def move_ios14(): # delete unrelated data for bid in mylib.appids_in_data(['*']): if bid not in study and bid[:-2] not in study: diir = mylib.path_data_app(bid) mylib.rm_dir(diir) print('del', diir) diir = os.path.dirname(diir) while not os.listdir(diir): print('del', diir) mylib.rm_dir(diir) diir = os.path.dirname(diir) # delete unrelated out for bid in mylib.appids_in_out(['*']): diir = mylib.path_out_app(bid) if bid not in study: if bid[:-2] not in study: print('del', diir) mylib.rm_dir(diir) for bid in study: diir = mylib.path_out_app(bid) try: shutil.copytree(diir, diir + '.2') except FileExistsError: pass # copy meta for bid in study: diir = mylib.path_data_app(bid) mylib.mkdir(mylib.path_add(diir, '2')) # continue for x in ['info_de', 'info_us', 'combined', 'evaluated']: try: shutil.copy(mylib.path_add(diir, x + '.json'), mylib.path_add(diir, '2', x + '.json')) except: pass # move ios 14 for fname, json in mylib.enum_jsons(bid): fiil = os.path.basename(fname) try: ios = json['ios'].split('.')[0] except KeyError: ios = '14' if ios == '14' and os.path.getmtime(fname) > 1600258000: mylib.mv(fname, mylib.path_add(diir, '2', fiil)) write_temporary_lists()
def process(bundle_ids): print('generating html: apps ...') i = 0 for bid in mylib.appids_in_out(bundle_ids): gen_page(bid, bundle_combine.get_evaluated(bid)) mylib.symlink(bundle_combine.fname_evaluated(bid), mylib.path_out_app(bid, 'data.json')) mylib.printf(' .' if i == 0 else '.') i = (i + 1) % 50 if i == 0: print('') # close printf print('') # close printf print('')
def del_id(bundle_ids): def delete_from_all_indices(bundle_ids): index_rank.process(bundle_ids, deleteOnly=True) index_domains.process(bundle_ids, deleteOnly=True) index_app_names.process(bundle_ids, deleteOnly=True) print('removing apps from website:') for bid in mylib.appids_in_out(bundle_ids): dest = mylib.path_out_app(bid) if mylib.dir_exists(dest): print(' ' + bid) mylib.rm_dir(dest) print('') delete_from_all_indices(bundle_ids) rebuild_html()
def gen_page(bundle_id, obj): name = index_app_names.get_name(bundle_id) gernes = index_categories.get_categories(bundle_id) obj['tracker'] = list(filter(lambda x: x[2], obj['subdom'])) HTML.write(mylib.path_out_app(bundle_id), f''' <h2 class="title">{name}</h2> <p class="subtitle snd"><i class="mg_lr">Bundle-id:</i>{ bundle_id }</p> <div id="meta"> <div class="icons"> { Graph.pie_chart_tracker(obj['tracker_percent']) } <img class="app-icon" src="icon.png" alt="app-icon" width="100" height="100"> </div> <table> <tr><td>App Categories:</td><td>{ ', '.join([HTML.a_category(i, name) for i, name in gernes]) }</td></tr> <tr><td>Last Update:</td><td>{HTML.date_utc(obj['last_date'])}</td></tr> </table> </div> <div id="stats"> { stat(1, 'Number of recordings:', 'sum_rec', obj['sum_rec']) } { stat(1, 'Average recording time:', 'avg_time', HTML.seconds_to_time(obj['avg_time'])) } { stat(2, 'Cumulative recording time:', 'sum_time', HTML.seconds_to_time(obj['sum_time'])) } { stat(1, 'Average number of requests:', 'avg_logs_pm', HTML.fmt_round_num(obj['avg_logs']), HTML.fmt_as_pm(obj['avg_logs_pm'])) } { stat(2, 'Total number of requests:', 'sum_logs_pm', str(obj['sum_logs']), HTML.fmt_as_pm(obj['sum_logs_pm'])) } { stat(1, 'Number of domains:', 'pardom', len(obj['pardom'])) } { stat(2, 'Number of subdomains:', 'subdom', len(obj['subdom'])) } { stat(3, 'Tracker percentage:', 'tracker_percent', HTML.fmt_as_percent(obj['tracker_percent'])) } </div> <h3>Connections</h3> <div> <h4>Potential Trackers ({ len(obj['tracker']) }):</h4> { gen_dom_tags(obj['tracker'], HTML.a_subdomain, onlyTrackers=True) } <h4>Domains ({ len(obj['pardom']) }):</h4> { gen_dotgraph(obj['pardom']) } { gen_dom_tags(obj['pardom'], HTML.a_domain) } <h4>Subdomains ({ len(obj['subdom']) }):</h4> { gen_dotgraph(obj['subdom']) } { gen_dom_tags(obj['subdom'], HTML.a_subdomain) } </div> { HTML.p_download_json('data.json', bundle_id + '.json') } <script type="text/javascript" src="/static/lookup-rank.js"></script> <script type="text/javascript"> lookup_rank_js('/results/rank.json', '{bundle_id}'); </script>''', title=name)
def needs_icon_path(bundle_id): icon_file = mylib.path_out_app(bundle_id, 'icon.png') return (mylib.file_exists(icon_file), icon_file)
def url_for_icon(bundle_id): if mylib.file_exists(mylib.path_out_app(bundle_id, 'icon.png')): return '/app/{0}/icon.png'.format(bundle_id) else: return '/static/app-template.svg'