def parse_apk(filepath): command = "%s dump badging %s" % (settings.AAPT_PATH, filepath) logger.d('execute command: %s' % command) package_name = '' sdk_version = '' version_name = '' version_code = 0 lines = os.popen(command).readlines() for line in lines: values = line.split(":", 1) if len(values) < 2: continue k = values[0].strip().lower() v = values[1].strip() if k == "package": m = re.match(r"name='(.+)' versionCode='(\d*)' versionName='(.*)'", v) if m: package_name = m.group(1) version_code = m.group(2) version_name = m.group(3) else: print "fail to extract info from '%s'" % v elif k == "sdkversion": m = re.match(r"'(\d+)'", v) if m: sdk_version = m.group(1) return package_name, version_code, version_name, sdk_version
def get_final_app(id_from, id_to, count, hash): try: c = _db.cursor() if hash > 0: rand_result = random.randrange(1, hash) sql = ''' select id, package_name, version_code from final_app where id < %s and id >= %s and file_type='apk' and (labels is null or labels = '') and (id - %s) %% %s = %s order by id desc limit %s ''' % (id_from, id_to, id_to, hash, rand_result, count) else: sql = ''' select id, package_name, version_code from final_app where id < %s and id >= %s and file_type='apk' and (labels is null or labels = '') order by id desc limit %s ''' % (id_from, id_to, count) logger.d(sql) c.execute(sql) result = c.fetchall() return result finally: c.close()
def get_wandoujia_app_labels(pn, vc): url = 'http://apps.wandoujia.com/api/v1/apps/%s' % pn data = Net.read_json(url) # logger.d(simplejson.dumps(data)) apks = [] if 'latestApk' in data: apks.append(data['latestApk']) if 'apks' in data: for apk in data['apks']: apks.append(apk) def extract_labels(apk): securityDetail = apk['securityDetail'] labels = [] for d in securityDetail: if d['status'] == 'SAFE' and d['provider'] in label_maps['wandoujia']: labels.append(label_maps['wandoujia'][d['provider']]) if 'adsType' in apk and apk['adsType'] == 'NONE': labels.append(label_maps['common']['noads']) else: labels.append(label_maps['common']['ads']) labels = ','.join(labels) return labels for apk in apks: if 'versionCode' in apk and str(apk['versionCode']) == str(vc): if 'securityDetail' in apk: return extract_labels(apk) else: logger.d('securityDetail not found for app(pn=%s, vc=%s)' % (pn, vc)) else: logger.d('versionCode(%s) not match for app(pn=%s, vc=%s)' % (apk['versionCode'], pn, vc)) return ''
def update_last_id(last_id): try: c = _db.cursor() sql = ''' update `config` set value=%s, updated_at=now() where `key`='update_meta_last_id'; ''' % last_id logger.d(sql) c.execute(sql) _db.conn.commit() finally: c.close()
def callback(ch, method, properties, body): id = body try: if not process(id): logger.d('process id(%s) return False, add to queue') ch.basic_publish(exchange='', routing_key=_queue_name, body=id, properties=pika.BasicProperties(delivery_mode=2)) except Exception as e: logger.e('process id(%s) failed: %s' % (id, e)) logger.d('process id(%s) return False, add to queue') ch.basic_publish(exchange='', routing_key=_queue_name, body=id, properties=pika.BasicProperties(delivery_mode=2)) ch.basic_ack(delivery_tag=method.delivery_tag)
def update_appchina_real_downloadlink(avail_download_links): logger.d(str(avail_download_links)) if avail_download_links: avail_download_links = avail_download_links.split(' ') newurls = [] for l in avail_download_links: if l.find('http://www.appchina.com') == 0 or l.find('http://www.d.appchina.com') == 0: try: newl = get_real_url(l) if newl != l and newl not in avail_download_links: newurls.append(newl) except Exception as e: logger.e('%s' % e) if newurls: for l in newurls: avail_download_links.append(l) avail_download_links = ' '.join(avail_download_links) logger.d(str(avail_download_links)) return avail_download_links else: logger.d('nothing need to update.') return None else: logger.d('avail_download_links is empty, do not need to update.') return None
def update_finalapp(id, urls): sql = ''' update final_app set avail_download_links=%s, updated_at=current_timestamp(), status=status&0xfffffffffffffff8 where id = %s ''' logger.d(sql % (urls, id)) try: c = _db.cursor() c.execute(sql, (urls, id)) _db.conn.commit() except MySQLError: _db.reconnect() finally: c.close()
def get_final_app_cols(pn, vc, cols): try: c = _db.cursor() sql = ''' select id, %s from final_app where package_name='%s' and version_code=%s ''' % (','.join(cols), pn.replace("'", ''), vc) if debug: logger.d(sql) c.execute(sql) result = c.fetchone() return result finally: c.close()
def process(id): logger.i('process id: %s' % id) file_path, vol_id = get_url(id) if file_path: data = '{"authkey":"9ORmsDOAJ3zcD21w", "url":"http://estoredwnld7.189store.com/downloads/vol%s/%s"}' % \ (vol_id, file_path) url = 'http://%s/vol%s/%s' % ('estoredwnld7.189store.com', vol_id, file_path) url1 = 'http://%s/vol%s/%s' % (download_host, vol_id, file_path) if url404(url, url1): logger.i('found app(id=%s) url(%s) 404' % (id, url)) update_finalapp_safe(id, 4, '') return True tencent_url = 'https://api.scan.qq.com/browser/scan' ret = '' try: ret = post(tencent_url, data) ret = ret.replace('\r', '') ret = ret.replace('\n', '') ret = simplejson.loads(ret) except Exception as e: logger.e(u'ret format error: %s' % ret) raise e logger.d('tencent returns: %s' % simplejson.dumps(ret)) if 'safetype' not in ret: raise 'error format: %s' % simplejson.dumps(ret) if ret['safetype'] == 'unknown': update_finalapp_safe(id, None, '') return False if ret['safetype'] == 'virus': logger.i('found app(id=%s) has virus.' % id) update_finalapp_safe(id, 0, simplejson.dumps(ret)) return True if ret['safetype'] == 'lowrisk': logger.i('found app(id=%s) lowrisk.' % id) update_finalapp_safe(id, 2, simplejson.dumps(ret)) return True if ret['safetype'] == 'midrisk': logger.i('found app(id=%s) midrisk.' % id) update_finalapp_safe(id, 3, simplejson.dumps(ret)) return True if ret['safetype'] == 'safe': logger.i('found app(id=%s) safe.' % id) update_finalapp_safe(id, 1, '') return True else: logger.i('can not find id in db, ignore: %s' % id) return True
def update_finalapp_safe(id, tencent_safe, info): sql = ''' insert into final_app_safe(final_app_id, last_verify_time, tencent_safe, info) values (%s, %s, %s, %s) on duplicate key update last_verify_time=%s, tencent_safe=%s, info=%s ''' try: c = get_db().cursor() now = datetime.datetime.now() logger.d(sql % (id, now, tencent_safe, info, now, tencent_safe, info)) c.execute(sql, (id, now, tencent_safe, info, now, tencent_safe, info)) get_db().conn.commit() except MySQLError: get_db().reconnect() finally: c.close()
def get_rand_apps(self, fields=None, top_count=20000, rand_count=1000): try: c = get_db().cursor() sql = ''' select package_name from (select distinct package_name from final_app where file_type='apk' order by cast(downloads as unsigned) desc limit %s) a order by rand() limit %s ''' logger.d(sql % (top_count, rand_count)) c.execute(sql, (top_count, rand_count)) ret = c.fetchall() return ret except MySQLError: get_db().reconnect() finally: c.close()
def get_url(packagename, version_code): try: c = _db.cursor() sql = ''' select id, avail_download_links from final_app where package_name=%s and version_code=%s and file_type='apk' ''' logger.d(sql % (packagename, version_code)) c.execute(sql, (packagename, version_code)) ret = c.fetchone() if not ret: return None, None else: return ret except MySQLError: _db.reconnect() finally: c.close()
def get_url(id): try: c = get_db().cursor() sql = ''' select file_path, vol_id from final_app where id=%s and file_type='apk' ''' logger.d(sql % (id)) c.execute(sql, (id)) ret = c.fetchone() if not ret: return None, None else: return ret except MySQLError: get_db().reconnect() finally: c.close()
def get_last_id(): try: c = _db.cursor() sql = ''' insert ignore into `config` (`key`, `value`, `created_at`, `updated_at`) values ('update_meta_last_id', 2425920, now(), now()); ''' logger.d(sql) c.execute(sql) _db.conn.commit() sql = ''' select value from `config` where `key`='update_meta_last_id'; ''' logger.d(sql) c.execute(sql) result = c.fetchone() return result[0] finally: c.close()
def update_app(package_name, version_code, labels=None, avail_download_links=None): try: updatecols = {} if labels: updatecols['labels'] = labels if avail_download_links: updatecols['avail_download_links'] = avail_download_links if not updatecols.keys(): logger.i('nothing need to update') return c = _db.cursor() updatecols = ["%s='%s'" % (k, updatecols[k].replace("'", '')) for k in updatecols.keys()] sql = ''' update final_app set %s, status=status&0xfffffffffffffff8, updated_at=now() where version_code=%s and package_name='%s' ''' % (','.join(updatecols), version_code, package_name.replace("'", '')) logger.d(sql) ret = c.execute(sql) logger.i('updated %s rows.' % ret) _db.conn.commit() finally: c.close()