def parse_apk(filepath):
command = "%s dump badging %s" % (settings.AAPT_PATH, filepath)
logger.d('execute command: %s' % command)
package_name = ''
sdk_version = ''
version_name = ''
version_code = 0
lines = os.popen(command).readlines()
for line in lines:
values = line.split(":", 1)
if len(values) < 2:
continue
k = values[0].strip().lower()
v = values[1].strip()
if k == "package":
m = re.match(r"name='(.+)' versionCode='(\d*)' versionName='(.*)'",
v)
if m:
package_name = m.group(1)
version_code = m.group(2)
version_name = m.group(3)
else:
print "fail to extract info from '%s'" % v
elif k == "sdkversion":
m = re.match(r"'(\d+)'", v)
if m:
sdk_version = m.group(1)
return package_name, version_code, version_name, sdk_version
def parse_apk(filepath):
command = "%s dump badging %s" % (settings.AAPT_PATH, filepath)
logger.d('execute command: %s' % command)
package_name = ''
sdk_version = ''
version_name = ''
version_code = 0
lines = os.popen(command).readlines()
for line in lines:
values = line.split(":", 1)
if len(values) < 2:
continue
k = values[0].strip().lower()
v = values[1].strip()
if k == "package":
m = re.match(r"name='(.+)' versionCode='(\d*)' versionName='(.*)'", v)
if m:
package_name = m.group(1)
version_code = m.group(2)
version_name = m.group(3)
else:
print "fail to extract info from '%s'" % v
elif k == "sdkversion":
m = re.match(r"'(\d+)'", v)
if m:
sdk_version = m.group(1)
return package_name, version_code, version_name, sdk_version
def get_final_app(id_from, id_to, count, hash):
try:
c = _db.cursor()
if hash > 0:
rand_result = random.randrange(1, hash)
sql = '''
select id, package_name, version_code
from final_app
where id < %s and id >= %s and file_type='apk'
and (labels is null or labels = '')
and (id - %s) %% %s = %s
order by id desc
limit %s
''' % (id_from, id_to, id_to, hash, rand_result, count)
else:
sql = '''
select id, package_name, version_code
from final_app
where id < %s and id >= %s and file_type='apk'
and (labels is null or labels = '')
order by id desc
limit %s
''' % (id_from, id_to, count)
logger.d(sql)
c.execute(sql)
result = c.fetchall()
return result
finally:
c.close()
def get_wandoujia_app_labels(pn, vc):
url = 'http://apps.wandoujia.com/api/v1/apps/%s' % pn
data = Net.read_json(url)
# logger.d(simplejson.dumps(data))
apks = []
if 'latestApk' in data:
apks.append(data['latestApk'])
if 'apks' in data:
for apk in data['apks']:
apks.append(apk)
def extract_labels(apk):
securityDetail = apk['securityDetail']
labels = []
for d in securityDetail:
if d['status'] == 'SAFE' and d['provider'] in label_maps['wandoujia']:
labels.append(label_maps['wandoujia'][d['provider']])
if 'adsType' in apk and apk['adsType'] == 'NONE':
labels.append(label_maps['common']['noads'])
else:
labels.append(label_maps['common']['ads'])
labels = ','.join(labels)
return labels
for apk in apks:
if 'versionCode' in apk and str(apk['versionCode']) == str(vc):
if 'securityDetail' in apk:
return extract_labels(apk)
else:
logger.d('securityDetail not found for app(pn=%s, vc=%s)' % (pn, vc))
else:
logger.d('versionCode(%s) not match for app(pn=%s, vc=%s)' % (apk['versionCode'], pn, vc))
return ''
def update_last_id(last_id):
try:
c = _db.cursor()
sql = '''
update `config` set value=%s, updated_at=now() where `key`='update_meta_last_id';
''' % last_id
logger.d(sql)
c.execute(sql)
_db.conn.commit()
finally:
c.close()
def callback(ch, method, properties, body):
id = body
try:
if not process(id):
logger.d('process id(%s) return False, add to queue')
ch.basic_publish(exchange='', routing_key=_queue_name,
body=id, properties=pika.BasicProperties(delivery_mode=2))
except Exception as e:
logger.e('process id(%s) failed: %s' % (id, e))
logger.d('process id(%s) return False, add to queue')
ch.basic_publish(exchange='', routing_key=_queue_name,
body=id, properties=pika.BasicProperties(delivery_mode=2))
ch.basic_ack(delivery_tag=method.delivery_tag)
def update_appchina_real_downloadlink(avail_download_links):
logger.d(str(avail_download_links))
if avail_download_links:
avail_download_links = avail_download_links.split(' ')
newurls = []
for l in avail_download_links:
if l.find('http://www.appchina.com') == 0 or l.find('http://www.d.appchina.com') == 0:
try:
newl = get_real_url(l)
if newl != l and newl not in avail_download_links:
newurls.append(newl)
except Exception as e:
logger.e('%s' % e)
if newurls:
for l in newurls:
avail_download_links.append(l)
avail_download_links = ' '.join(avail_download_links)
logger.d(str(avail_download_links))
return avail_download_links
else:
logger.d('nothing need to update.')
return None
else:
logger.d('avail_download_links is empty, do not need to update.')
return None
def update_finalapp(id, urls):
sql = '''
update final_app set avail_download_links=%s, updated_at=current_timestamp(),
status=status&0xfffffffffffffff8 where id = %s
'''
logger.d(sql % (urls, id))
try:
c = _db.cursor()
c.execute(sql, (urls, id))
_db.conn.commit()
except MySQLError:
_db.reconnect()
finally:
c.close()
def get_final_app_cols(pn, vc, cols):
try:
c = _db.cursor()
sql = '''
select id, %s from final_app
where package_name='%s' and version_code=%s
''' % (','.join(cols), pn.replace("'", ''), vc)
if debug:
logger.d(sql)
c.execute(sql)
result = c.fetchone()
return result
finally:
c.close()
def update_finalapp(id, urls):
sql = '''
update final_app set avail_download_links=%s, updated_at=current_timestamp(),
status=status&0xfffffffffffffff8 where id = %s
'''
logger.d(sql % (urls, id))
try:
c = _db.cursor()
c.execute(sql, (urls, id))
_db.conn.commit()
except MySQLError:
_db.reconnect()
finally:
c.close()
def process(id):
logger.i('process id: %s' % id)
file_path, vol_id = get_url(id)
if file_path:
data = '{"authkey":"9ORmsDOAJ3zcD21w", "url":"http://estoredwnld7.189store.com/downloads/vol%s/%s"}' % \
(vol_id, file_path)
url = 'http://%s/vol%s/%s' % ('estoredwnld7.189store.com', vol_id,
file_path)
url1 = 'http://%s/vol%s/%s' % (download_host, vol_id, file_path)
if url404(url, url1):
logger.i('found app(id=%s) url(%s) 404' % (id, url))
update_finalapp_safe(id, 4, '')
return True
tencent_url = 'https://api.scan.qq.com/browser/scan'
ret = ''
try:
ret = post(tencent_url, data)
ret = ret.replace('\r', '')
ret = ret.replace('\n', '')
ret = simplejson.loads(ret)
except Exception as e:
logger.e(u'ret format error: %s' % ret)
raise e
logger.d('tencent returns: %s' % simplejson.dumps(ret))
if 'safetype' not in ret:
raise 'error format: %s' % simplejson.dumps(ret)
if ret['safetype'] == 'unknown':
update_finalapp_safe(id, None, '')
return False
if ret['safetype'] == 'virus':
logger.i('found app(id=%s) has virus.' % id)
update_finalapp_safe(id, 0, simplejson.dumps(ret))
return True
if ret['safetype'] == 'lowrisk':
logger.i('found app(id=%s) lowrisk.' % id)
update_finalapp_safe(id, 2, simplejson.dumps(ret))
return True
if ret['safetype'] == 'midrisk':
logger.i('found app(id=%s) midrisk.' % id)
update_finalapp_safe(id, 3, simplejson.dumps(ret))
return True
if ret['safetype'] == 'safe':
logger.i('found app(id=%s) safe.' % id)
update_finalapp_safe(id, 1, '')
return True
else:
logger.i('can not find id in db, ignore: %s' % id)
return True
def update_finalapp_safe(id, tencent_safe, info):
sql = '''
insert into final_app_safe(final_app_id, last_verify_time, tencent_safe, info)
values (%s, %s, %s, %s)
on duplicate key update last_verify_time=%s, tencent_safe=%s, info=%s
'''
try:
c = get_db().cursor()
now = datetime.datetime.now()
logger.d(sql % (id, now, tencent_safe, info, now, tencent_safe, info))
c.execute(sql, (id, now, tencent_safe, info, now, tencent_safe, info))
get_db().conn.commit()
except MySQLError:
get_db().reconnect()
finally:
c.close()
def update_finalapp_safe(id, tencent_safe, info):
sql = '''
insert into final_app_safe(final_app_id, last_verify_time, tencent_safe, info)
values (%s, %s, %s, %s)
on duplicate key update last_verify_time=%s, tencent_safe=%s, info=%s
'''
try:
c = get_db().cursor()
now = datetime.datetime.now()
logger.d(sql % (id, now, tencent_safe, info, now, tencent_safe, info))
c.execute(sql, (id, now, tencent_safe, info, now, tencent_safe, info))
get_db().conn.commit()
except MySQLError:
get_db().reconnect()
finally:
c.close()
def process(id):
logger.i('process id: %s' % id)
file_path, vol_id = get_url(id)
if file_path:
data = '{"authkey":"9ORmsDOAJ3zcD21w", "url":"http://estoredwnld7.189store.com/downloads/vol%s/%s"}' % \
(vol_id, file_path)
url = 'http://%s/vol%s/%s' % ('estoredwnld7.189store.com', vol_id, file_path)
url1 = 'http://%s/vol%s/%s' % (download_host, vol_id, file_path)
if url404(url, url1):
logger.i('found app(id=%s) url(%s) 404' % (id, url))
update_finalapp_safe(id, 4, '')
return True
tencent_url = 'https://api.scan.qq.com/browser/scan'
ret = ''
try:
ret = post(tencent_url, data)
ret = ret.replace('\r', '')
ret = ret.replace('\n', '')
ret = simplejson.loads(ret)
except Exception as e:
logger.e(u'ret format error: %s' % ret)
raise e
logger.d('tencent returns: %s' % simplejson.dumps(ret))
if 'safetype' not in ret:
raise 'error format: %s' % simplejson.dumps(ret)
if ret['safetype'] == 'unknown':
update_finalapp_safe(id, None, '')
return False
if ret['safetype'] == 'virus':
logger.i('found app(id=%s) has virus.' % id)
update_finalapp_safe(id, 0, simplejson.dumps(ret))
return True
if ret['safetype'] == 'lowrisk':
logger.i('found app(id=%s) lowrisk.' % id)
update_finalapp_safe(id, 2, simplejson.dumps(ret))
return True
if ret['safetype'] == 'midrisk':
logger.i('found app(id=%s) midrisk.' % id)
update_finalapp_safe(id, 3, simplejson.dumps(ret))
return True
if ret['safetype'] == 'safe':
logger.i('found app(id=%s) safe.' % id)
update_finalapp_safe(id, 1, '')
return True
else:
logger.i('can not find id in db, ignore: %s' % id)
return True
def callback(ch, method, properties, body):
id = body
try:
if not process(id):
logger.d('process id(%s) return False, add to queue')
ch.basic_publish(exchange='',
routing_key=_queue_name,
body=id,
properties=pika.BasicProperties(delivery_mode=2))
except Exception as e:
logger.e('process id(%s) failed: %s' % (id, e))
logger.d('process id(%s) return False, add to queue')
ch.basic_publish(exchange='',
routing_key=_queue_name,
body=id,
properties=pika.BasicProperties(delivery_mode=2))
ch.basic_ack(delivery_tag=method.delivery_tag)
def get_rand_apps(self, fields=None, top_count=20000, rand_count=1000):
try:
c = get_db().cursor()
sql = '''
select package_name from
(select distinct package_name from final_app
where file_type='apk'
order by cast(downloads as unsigned) desc limit %s) a
order by rand() limit %s
'''
logger.d(sql % (top_count, rand_count))
c.execute(sql, (top_count, rand_count))
ret = c.fetchall()
return ret
except MySQLError:
get_db().reconnect()
finally:
c.close()
def get_url(packagename, version_code):
try:
c = _db.cursor()
sql = '''
select id, avail_download_links from final_app
where package_name=%s and version_code=%s and file_type='apk'
'''
logger.d(sql % (packagename, version_code))
c.execute(sql, (packagename, version_code))
ret = c.fetchone()
if not ret:
return None, None
else:
return ret
except MySQLError:
_db.reconnect()
finally:
c.close()
def get_rand_apps(self, fields=None, top_count=20000, rand_count=1000):
try:
c = get_db().cursor()
sql = '''
select package_name from
(select distinct package_name from final_app
where file_type='apk'
order by cast(downloads as unsigned) desc limit %s) a
order by rand() limit %s
'''
logger.d(sql % (top_count, rand_count))
c.execute(sql, (top_count, rand_count))
ret = c.fetchall()
return ret
except MySQLError:
get_db().reconnect()
finally:
c.close()
def get_url(packagename, version_code):
try:
c = _db.cursor()
sql = '''
select id, avail_download_links from final_app
where package_name=%s and version_code=%s and file_type='apk'
'''
logger.d(sql % (packagename, version_code))
c.execute(sql, (packagename, version_code))
ret = c.fetchone()
if not ret:
return None, None
else:
return ret
except MySQLError:
_db.reconnect()
finally:
c.close()
def get_url(id):
try:
c = get_db().cursor()
sql = '''
select file_path, vol_id from final_app
where id=%s and file_type='apk'
'''
logger.d(sql % (id))
c.execute(sql, (id))
ret = c.fetchone()
if not ret:
return None, None
else:
return ret
except MySQLError:
get_db().reconnect()
finally:
c.close()
def get_url(id):
try:
c = get_db().cursor()
sql = '''
select file_path, vol_id from final_app
where id=%s and file_type='apk'
'''
logger.d(sql % (id))
c.execute(sql, (id))
ret = c.fetchone()
if not ret:
return None, None
else:
return ret
except MySQLError:
get_db().reconnect()
finally:
c.close()
def get_last_id():
try:
c = _db.cursor()
sql = '''
insert ignore into `config` (`key`, `value`, `created_at`, `updated_at`)
values ('update_meta_last_id', 2425920, now(), now());
'''
logger.d(sql)
c.execute(sql)
_db.conn.commit()
sql = '''
select value from `config` where `key`='update_meta_last_id';
'''
logger.d(sql)
c.execute(sql)
result = c.fetchone()
return result[0]
finally:
c.close()
def update_app(package_name, version_code, labels=None, avail_download_links=None):
try:
updatecols = {}
if labels:
updatecols['labels'] = labels
if avail_download_links:
updatecols['avail_download_links'] = avail_download_links
if not updatecols.keys():
logger.i('nothing need to update')
return
c = _db.cursor()
updatecols = ["%s='%s'" % (k, updatecols[k].replace("'", '')) for k in updatecols.keys()]
sql = '''
update final_app set %s, status=status&0xfffffffffffffff8, updated_at=now()
where version_code=%s and package_name='%s'
''' % (','.join(updatecols), version_code, package_name.replace("'", ''))
logger.d(sql)
ret = c.execute(sql)
logger.i('updated %s rows.' % ret)
_db.conn.commit()
finally:
c.close()
