def move_ios14():
# delete unrelated data
for bid in mylib.appids_in_data(['*']):
if bid not in study and bid[:-2] not in study:
diir = mylib.path_data_app(bid)
mylib.rm_dir(diir)
print('del', diir)
diir = os.path.dirname(diir)
while not os.listdir(diir):
print('del', diir)
mylib.rm_dir(diir)
diir = os.path.dirname(diir)
# delete unrelated out
for bid in mylib.appids_in_out(['*']):
diir = mylib.path_out_app(bid)
if bid not in study:
if bid[:-2] not in study:
print('del', diir)
mylib.rm_dir(diir)
for bid in study:
diir = mylib.path_out_app(bid)
try:
shutil.copytree(diir, diir + '.2')
except FileExistsError:
pass
# copy meta
for bid in study:
diir = mylib.path_data_app(bid)
mylib.mkdir(mylib.path_add(diir, '2'))
# continue
for x in ['info_de', 'info_us', 'combined', 'evaluated']:
try:
shutil.copy(mylib.path_add(diir, x + '.json'),
mylib.path_add(diir, '2', x + '.json'))
except:
pass
# move ios 14
for fname, json in mylib.enum_jsons(bid):
fiil = os.path.basename(fname)
try:
ios = json['ios'].split('.')[0]
except KeyError:
ios = '14'
if ios == '14' and os.path.getmtime(fname) > 1600258000:
mylib.mv(fname, mylib.path_add(diir, '2', fiil))
write_temporary_lists()
def process(bundle_ids):
print('generating html: apps ...')
i = 0
for bid in mylib.appids_in_out(bundle_ids):
gen_page(bid, bundle_combine.get_evaluated(bid))
mylib.symlink(bundle_combine.fname_evaluated(bid),
mylib.path_out_app(bid, 'data.json'))
mylib.printf(' .' if i == 0 else '.')
i = (i + 1) % 50
if i == 0:
print('') # close printf
print('') # close printf
print('')
def del_id(bundle_ids):
def delete_from_all_indices(bundle_ids):
index_rank.process(bundle_ids, deleteOnly=True)
index_domains.process(bundle_ids, deleteOnly=True)
index_app_names.process(bundle_ids, deleteOnly=True)
print('removing apps from website:')
for bid in mylib.appids_in_out(bundle_ids):
dest = mylib.path_out_app(bid)
if mylib.dir_exists(dest):
print(' ' + bid)
mylib.rm_dir(dest)
print('')
delete_from_all_indices(bundle_ids)
rebuild_html()
def gen_page(bundle_id, obj):
name = index_app_names.get_name(bundle_id)
gernes = index_categories.get_categories(bundle_id)
obj['tracker'] = list(filter(lambda x: x[2], obj['subdom']))
HTML.write(mylib.path_out_app(bundle_id), f'''
<h2 class="title">{name}</h2>
<p class="subtitle snd"><i class="mg_lr">Bundle-id:</i>{ bundle_id }</p>
<div id="meta">
<div class="icons">
{ Graph.pie_chart_tracker(obj['tracker_percent']) }
<img class="app-icon" src="icon.png" alt="app-icon" width="100" height="100">
</div>
<table>
<tr><td>App Categories:</td><td>{
', '.join([HTML.a_category(i, name) for i, name in gernes])
}</td></tr>
<tr><td>Last Update:</td><td>{HTML.date_utc(obj['last_date'])}</td></tr>
</table>
</div>
<div id="stats">
{ stat(1, 'Number of recordings:', 'sum_rec', obj['sum_rec']) }
{ stat(1, 'Average recording time:', 'avg_time', HTML.seconds_to_time(obj['avg_time'])) }
{ stat(2, 'Cumulative recording time:', 'sum_time', HTML.seconds_to_time(obj['sum_time'])) }
{ stat(1, 'Average number of requests:', 'avg_logs_pm', HTML.fmt_round_num(obj['avg_logs']), HTML.fmt_as_pm(obj['avg_logs_pm'])) }
{ stat(2, 'Total number of requests:', 'sum_logs_pm', str(obj['sum_logs']), HTML.fmt_as_pm(obj['sum_logs_pm'])) }
{ stat(1, 'Number of domains:', 'pardom', len(obj['pardom'])) }
{ stat(2, 'Number of subdomains:', 'subdom', len(obj['subdom'])) }
{ stat(3, 'Tracker percentage:', 'tracker_percent', HTML.fmt_as_percent(obj['tracker_percent'])) }
</div>
<h3>Connections</h3>
<div>
<h4>Potential Trackers ({ len(obj['tracker']) }):</h4>
{ gen_dom_tags(obj['tracker'], HTML.a_subdomain, onlyTrackers=True) }
<h4>Domains ({ len(obj['pardom']) }):</h4>
{ gen_dotgraph(obj['pardom']) }
{ gen_dom_tags(obj['pardom'], HTML.a_domain) }
<h4>Subdomains ({ len(obj['subdom']) }):</h4>
{ gen_dotgraph(obj['subdom']) }
{ gen_dom_tags(obj['subdom'], HTML.a_subdomain) }
</div>
{ HTML.p_download_json('data.json', bundle_id + '.json') }
<script type="text/javascript" src="/static/lookup-rank.js"></script>
<script type="text/javascript">
lookup_rank_js('/results/rank.json', '{bundle_id}');
</script>''', title=name)
def needs_icon_path(bundle_id):
icon_file = mylib.path_out_app(bundle_id, 'icon.png')
return (mylib.file_exists(icon_file), icon_file)
def url_for_icon(bundle_id):
if mylib.file_exists(mylib.path_out_app(bundle_id, 'icon.png')):
return '/app/{0}/icon.png'.format(bundle_id)
else:
return '/static/app-template.svg'