def json_combine(bundle_id):
def inc_dic(ddic, key, num):
try:
ddic[key][1].append(num)
except KeyError:
ddic[key] = (download_tracker.is_tracker(key), [num])
res = {'rec_len': []}
pardom = {}
subdom = {}
latest = 0
for fname, jdata in mylib.enum_jsons(bundle_id):
# TODO: load combined and append newest only, then update evaluated
latest = max(latest, os.path.getmtime(fname)) # or getctime
res['rec_len'].append(jdata['duration'])
try:
logs = jdata['logs']
uniq_par = {}
for subdomain in logs:
occurs = len(logs[subdomain])
subdomain = cleanup_domain_name(subdomain)
inc_dic(subdom, subdomain, occurs)
par_dom = mylib.parent_domain(subdomain)
try:
uniq_par[par_dom] += occurs
except KeyError:
uniq_par[par_dom] = occurs
for name, val in uniq_par.items():
inc_dic(pardom, name, val)
except KeyError:
mylib.err('bundle-combine', 'skip: ' + fname)
res['pardom'] = pardom
res['subdom'] = subdom
res['last_date'] = int(latest)
return res
def process(bundle_ids, deleteOnly=False):
global _app_names_dict
print('writing index: app names ...')
if bundle_ids == ['*']:
print(' full reset')
mylib.rm_file(fname_apps_all()) # rebuild from ground up
mylib.rm_file(fname_apps_compact())
index = mylib.json_safe_read(fname_apps_all(), {})
did_change = False
for bid in mylib.appids_in_data(bundle_ids):
if deleteOnly:
did_change |= mylib.try_del(index, [bid])
continue
names = download_itunes.get_app_names(bid)
if not names:
mylib.err('index-app-names', 'could not load: {}'.format(bid))
continue
try:
if index[bid] == names:
continue
except KeyError:
pass
index[bid] = names
did_change = True
if did_change:
print(' writing')
mylib.json_write(fname_apps_all(), index, pretty=False)
_app_names_dict = {bid: download_itunes.choose_lang(names)
for bid, names in index.items()}
mylib.json_write(fname_apps_compact(), _app_names_dict, pretty=False)
else:
print(' no change')
print('')
def enum_local(list_name):
path = fname_tracker(list_name)
if not mylib.file_exists(path):
mylib.err('download-tracker', 'List not found: ' + list_name)
return []
with open(path, 'r') as fp:
for line in fp.readlines():
yield line.strip()
def enum_remote(url, ignore=None):
try:
whole = mylib.download(url)
for line in whole.split(b'\n'):
if not line or ignore and line.startswith(ignore):
continue
yield line
except Exception as e:
mylib.err('tracker-download', str(e) + ' in ' + url)
def cleanup_domain_name(domain):
safe_domain = re_domain.sub('', domain)
if domain == safe_domain:
return domain
mylib.err('bundle-combine', 'invalid character in domain name: ' + domain)
without_null = domain.replace('(null)', '')
if domain != without_null:
return cleanup_domain_name(without_null)
return safe_domain
def lowe(fname):
res = set()
for x in enum_remote(
'https://pgl.yoyo.org/adservers/serverlist.php'
'?hostformat=hosts&mimetype=plaintext', b'#'):
p = x.split()
if len(p) != 2:
mylib.err('tracker-list', 'Lowe: parsing error')
continue
res.add(p[1])
return save_list(res, fname, binary=True)
def download(bundle_id, force=False):
if not mylib.valid_bundle_id(bundle_id):
mylib.err('apple-download', 'invalid id: ' + bundle_id)
return False
exists, _ = needs_icon_path(bundle_id)
if exists and not force:
return False
mylib.printf(' {} => '.format(bundle_id))
for lang in AVAILABLE_LANGS:
try:
mylib.printf(lang)
download_info(bundle_id, lang, force=force)
mylib.printf('[✔] ')
except Exception:
mylib.printf('[✘] ')
mylib.err('apple-download', 'json {}: {}'.format(
lang.upper(), bundle_id), logOnly=True)
try:
mylib.printf('icon')
index_needs_update = download_icon(bundle_id, force=force)
mylib.printf('[✔] ')
except Exception:
index_needs_update = False
mylib.printf('[✘] ')
mylib.err('apple-download', 'img for ' + bundle_id, logOnly=True)
print('') # end printf line
return index_needs_update
def import_update():
print('checking incoming data ...')
needs_update = set()
then_delete = set()
for fname, bid in mylib.enum_newly_added():
if bid == '_manually':
# TODO: notify admin that manual action is required
mylib.err('import', 'manual action required!')
elif bid == '_longterm':
mylib.err('import', 'manual action required! (background)')
else:
print(' ' + bid)
needs_update.add(bid)
then_delete.add(fname)
print('')
if len(needs_update) > 0:
combine_and_update(needs_update)
if len(then_delete) > 0:
print('cleanup _in folder ...')
for x in then_delete:
mylib.rm_file(fname)
print('')
print_usage_and_exit()
cmd = args[0]
params = args[1:]
if cmd == 'import':
import_update()
elif cmd == 'tracker':
tracker_update()
# download_tracker.combine_all()
elif cmd == 'icons':
bundle_ids = download_itunes.download_missing_icons(force=False)
if bundle_ids:
rebuild_html(bundle_ids)
elif cmd == 'index':
index_categories.process(['*'], force=True)
index_rank.process(['*'])
index_domains.process(['*'])
rebuild_html()
elif cmd == 'run':
if len(params) == 0:
print_usage_and_exit()
combine_and_update(params)
elif cmd == 'del':
if len(params) == 0:
print_usage_and_exit()
del_id(params) # ['_manually']
else:
print_usage_and_exit()
except Exception:
mylib.err('critical', traceback.format_exc(), logOnly=True)
raise