def get_apps_by_packagename(self, packagename_list, fields=None):
if not isinstance(packagename_list, list):
packagename_list = [packagename_list]
url = 'http://estoresrvice.189store.com/api/app/updates2.json'
param = {
'clientid': 'kpitest',
'source': 'kpitest_1',
'appinfos': '|'.join(['%s:0' % pn for pn in packagename_list]),
'all_apps': 2,
}
header = {'Accept-Encoding': 'gzip,deflate,sdch'}
json_data = Net.read_json(url,
method='post',
data=param,
header=header)
ret = []
ret_packages = set()
if not fields:
return ret
for data in json_data:
res_data = []
for f in fields:
f = FIELDS[f]
if f not in data:
logger.e('key %s not in object: %s' % (f, data))
break
res_data.append(data[f])
ret_packages.add(data[FIELDS[datasource.FIELD_PACKAGE_NAME]])
if len(res_data) == len(fields):
ret.append(res_data)
logger.e('can not find new version for app: %s' %
(set(packagename_list) - ret_packages))
return ret
def run():
logger.i('rmq_channel.start_consuming')
_pika = None
rmq_channel = None
fail_times = 0
while True and not should_stop and fail_times < max_fail_times:
try:
_pika = pika.BlockingConnection(
pika.ConnectionParameters(
**settings.RABBITMQ['interception_download_urls']))
pikas.append(_pika)
rmq_channel = _pika.channel()
rmq_channel.queue_declare(queue='downloadurls', durable=True)
rmq_channel.basic_qos(prefetch_count=1)
rmq_channel.basic_consume(callback, queue='downloadurls')
rmq_channels.append(rmq_channel)
rmq_channel.start_consuming()
except Exception as e:
fail_times = fail_times + 1
logger.e('error consuming: %s' % e)
if rmq_channel in rmq_channels:
try:
rmq_channel.stop_consuming()
rmq_channel.close()
except:
pass
rmq_channels.remove(rmq_channel)
if _pika in pikas:
try:
_pika.close()
except:
pass
pikas.remove(_pika)
time.sleep(5)
def get_apps_by_packagename(self, packagename_list, fields=None):
if not isinstance(packagename_list, list):
packagename_list = [packagename_list]
url = 'http://estoresrvice.189store.com/api/app/updates2.json'
param = {
'clientid': 'kpitest',
'source': 'kpitest_1',
'appinfos': '|'.join(['%s:0' % pn for pn in packagename_list]),
'all_apps': 2,
}
header = {'Accept-Encoding': 'gzip,deflate,sdch'}
json_data = Net.read_json(url, method='post', data=param, header=header)
ret = []
ret_packages = set()
if not fields:
return ret
for data in json_data:
res_data = []
for f in fields:
f = FIELDS[f]
if f not in data:
logger.e('key %s not in object: %s' % (f, data))
break
res_data.append(data[f])
ret_packages.add(data[FIELDS[datasource.FIELD_PACKAGE_NAME]])
if len(res_data) == len(fields):
ret.append(res_data)
logger.e('can not find new version for app: %s' % (set(packagename_list) - ret_packages))
return ret
def run():
logger.i('rmq_channel.start_consuming')
_pika = None
rmq_channel = None
fail_times = 0
while True and not should_stop and fail_times < max_fail_times:
try:
_pika = pika.BlockingConnection(
pika.ConnectionParameters(**settings.RABBITMQ['interception_download_urls']))
pikas.append(_pika)
rmq_channel = _pika.channel()
rmq_channel.queue_declare(queue='downloadurls', durable=True)
rmq_channel.basic_qos(prefetch_count=1)
rmq_channel.basic_consume(callback, queue='downloadurls')
rmq_channels.append(rmq_channel)
rmq_channel.start_consuming()
except Exception as e:
fail_times = fail_times + 1
logger.e('error consuming: %s' % e)
if rmq_channel in rmq_channels:
try:
rmq_channel.stop_consuming()
rmq_channel.close()
except:
pass
rmq_channels.remove(rmq_channel)
if _pika in pikas:
try:
_pika.close()
except:
pass
pikas.remove(_pika)
time.sleep(5)
def update_appchina_real_downloadlink(avail_download_links):
logger.d(str(avail_download_links))
if avail_download_links:
avail_download_links = avail_download_links.split(' ')
newurls = []
for l in avail_download_links:
if l.find('http://www.appchina.com') == 0 or l.find('http://www.d.appchina.com') == 0:
try:
newl = get_real_url(l)
if newl != l and newl not in avail_download_links:
newurls.append(newl)
except Exception as e:
logger.e('%s' % e)
if newurls:
for l in newurls:
avail_download_links.append(l)
avail_download_links = ' '.join(avail_download_links)
logger.d(str(avail_download_links))
return avail_download_links
else:
logger.d('nothing need to update.')
return None
else:
logger.d('avail_download_links is empty, do not need to update.')
return None
def callback(ch, method, properties, body):
id = body
try:
if not process(id):
logger.d('process id(%s) return False, add to queue')
ch.basic_publish(exchange='', routing_key=_queue_name,
body=id, properties=pika.BasicProperties(delivery_mode=2))
except Exception as e:
logger.e('process id(%s) failed: %s' % (id, e))
logger.d('process id(%s) return False, add to queue')
ch.basic_publish(exchange='', routing_key=_queue_name,
body=id, properties=pika.BasicProperties(delivery_mode=2))
ch.basic_ack(delivery_tag=method.delivery_tag)
def update_apps_labels(apps):
suc_count = 0
for pn, vc in apps:
try:
labels = get_wandoujia_app_labels(pn, vc)
if labels:
update_app(pn, vc, labels)
suc_count += 1
logger.i('update app(pn=%s, vc=%s) success.' % (pn, vc))
else:
logger.i('ignore app(pn=%s, vc=%s).' % (pn, vc))
except Exception as e:
logger.e('update app(pn=%s, vc=%s) failed: %s' % (pn, vc, e))
return suc_count
def graceful_exit(*args, **kargs):
logger.i('request rabbit_consumer stop(%s, %s). stopping...' % (args, kargs))
global should_stop
should_stop = True
for c in rmq_channels:
try:
c.stop_consuming()
c.close()
except Exception as e:
logger.e('stop_consuming failed: %s' % e)
for p in pikas:
try:
p.close()
except Exception as e:
logger.e('close rabbit connection failed: %s' % e)
def process(id):
logger.i('process id: %s' % id)
file_path, vol_id = get_url(id)
if file_path:
data = '{"authkey":"9ORmsDOAJ3zcD21w", "url":"http://estoredwnld7.189store.com/downloads/vol%s/%s"}' % \
(vol_id, file_path)
url = 'http://%s/vol%s/%s' % ('estoredwnld7.189store.com', vol_id,
file_path)
url1 = 'http://%s/vol%s/%s' % (download_host, vol_id, file_path)
if url404(url, url1):
logger.i('found app(id=%s) url(%s) 404' % (id, url))
update_finalapp_safe(id, 4, '')
return True
tencent_url = 'https://api.scan.qq.com/browser/scan'
ret = ''
try:
ret = post(tencent_url, data)
ret = ret.replace('\r', '')
ret = ret.replace('\n', '')
ret = simplejson.loads(ret)
except Exception as e:
logger.e(u'ret format error: %s' % ret)
raise e
logger.d('tencent returns: %s' % simplejson.dumps(ret))
if 'safetype' not in ret:
raise 'error format: %s' % simplejson.dumps(ret)
if ret['safetype'] == 'unknown':
update_finalapp_safe(id, None, '')
return False
if ret['safetype'] == 'virus':
logger.i('found app(id=%s) has virus.' % id)
update_finalapp_safe(id, 0, simplejson.dumps(ret))
return True
if ret['safetype'] == 'lowrisk':
logger.i('found app(id=%s) lowrisk.' % id)
update_finalapp_safe(id, 2, simplejson.dumps(ret))
return True
if ret['safetype'] == 'midrisk':
logger.i('found app(id=%s) midrisk.' % id)
update_finalapp_safe(id, 3, simplejson.dumps(ret))
return True
if ret['safetype'] == 'safe':
logger.i('found app(id=%s) safe.' % id)
update_finalapp_safe(id, 1, '')
return True
else:
logger.i('can not find id in db, ignore: %s' % id)
return True
def graceful_exit(*args, **kargs):
logger.i('request rabbit_consumer stop(%s, %s). stopping...' %
(args, kargs))
global should_stop
should_stop = True
for c in rmq_channels:
try:
c.stop_consuming()
c.close()
except Exception as e:
logger.e('stop_consuming failed: %s' % e)
for p in pikas:
try:
p.close()
except Exception as e:
logger.e('close rabbit connection failed: %s' % e)
def process(id):
logger.i('process id: %s' % id)
file_path, vol_id = get_url(id)
if file_path:
data = '{"authkey":"9ORmsDOAJ3zcD21w", "url":"http://estoredwnld7.189store.com/downloads/vol%s/%s"}' % \
(vol_id, file_path)
url = 'http://%s/vol%s/%s' % ('estoredwnld7.189store.com', vol_id, file_path)
url1 = 'http://%s/vol%s/%s' % (download_host, vol_id, file_path)
if url404(url, url1):
logger.i('found app(id=%s) url(%s) 404' % (id, url))
update_finalapp_safe(id, 4, '')
return True
tencent_url = 'https://api.scan.qq.com/browser/scan'
ret = ''
try:
ret = post(tencent_url, data)
ret = ret.replace('\r', '')
ret = ret.replace('\n', '')
ret = simplejson.loads(ret)
except Exception as e:
logger.e(u'ret format error: %s' % ret)
raise e
logger.d('tencent returns: %s' % simplejson.dumps(ret))
if 'safetype' not in ret:
raise 'error format: %s' % simplejson.dumps(ret)
if ret['safetype'] == 'unknown':
update_finalapp_safe(id, None, '')
return False
if ret['safetype'] == 'virus':
logger.i('found app(id=%s) has virus.' % id)
update_finalapp_safe(id, 0, simplejson.dumps(ret))
return True
if ret['safetype'] == 'lowrisk':
logger.i('found app(id=%s) lowrisk.' % id)
update_finalapp_safe(id, 2, simplejson.dumps(ret))
return True
if ret['safetype'] == 'midrisk':
logger.i('found app(id=%s) midrisk.' % id)
update_finalapp_safe(id, 3, simplejson.dumps(ret))
return True
if ret['safetype'] == 'safe':
logger.i('found app(id=%s) safe.' % id)
update_finalapp_safe(id, 1, '')
return True
else:
logger.i('can not find id in db, ignore: %s' % id)
return True
def update_apps_avail_download_links(apps):
suc_count = 0
for pn, vc in apps:
try:
avail_download_links = get_final_app_cols(pn, vc, ['avail_download_links'])
if avail_download_links:
avail_download_links = avail_download_links[1]
avail_download_links = update_appchina_real_downloadlink(avail_download_links)
if avail_download_links:
update_app(pn, vc, avail_download_links=avail_download_links)
suc_count += 1
logger.i('update app(pn=%s, vc=%s) success.' % (pn, vc))
else:
logger.i('ignore app(pn=%s, vc=%s).' % (pn, vc))
except Exception as e:
logger.e('update app(pn=%s, vc=%s) failed: %s' % (pn, vc, e))
return suc_count
def get_app_info_by_packagename(self, packagename, fields=None):
url = ('http://apps.wandoujia.com/api/v1/apps/%s?opt_fields=apks.compatible,apks.incompatibleDetail,'
'ad,apks.adsType,apks.beta,apks.bytes,apks.downloadUrl.*,apks.size,apks.versionCode,apks.versionName,'
'apks.securityStatus,apks.md5,apks.permissionLevel,apks.superior,apks.dangerousPermissions,'
'apks.permissions,apks.securityDetail.*,apks.resolution,categories.alias,categories.name,description,'
'title,commentsCount,likeCount,dislikesCount,installedCountStr,icons.px78,'
'packageName,id,likesRate,screenshots.*,apks.verified,apks.paidType')
url = url % packagename
try:
json_data = Net.read_json(url)
except Exception as e:
logger.e('read_json from url(%s) failed: %s' % (url, e))
return None
if 'apks' not in json_data:
logger.w('one or more fields(%s) can not be found in object: %s' % (fields, json_data))
return None
return self._get_data(json_data, fields, packagename=None)
def callback(ch, method, properties, body):
id = body
try:
if not process(id):
logger.d('process id(%s) return False, add to queue')
ch.basic_publish(exchange='',
routing_key=_queue_name,
body=id,
properties=pika.BasicProperties(delivery_mode=2))
except Exception as e:
logger.e('process id(%s) failed: %s' % (id, e))
logger.d('process id(%s) return False, add to queue')
ch.basic_publish(exchange='',
routing_key=_queue_name,
body=id,
properties=pika.BasicProperties(delivery_mode=2))
ch.basic_ack(delivery_tag=method.delivery_tag)
def update_avail_download_links(packagename, version_code, url):
logger.i('update_avail_download_links packagename: %s, version_code: %s, url: %s' %
(packagename, version_code, url))
id, urls = get_url(packagename, version_code)
if urls and urls.find(url) != -1:
logger.i('final app already has url: %s' % url)
return
elif not urls:
logger.i('final app not found')
return
else:
new_urls = '%s %s' % (urls, url)
new_urls = reform_urls(new_urls)
while len(new_urls) > 2048:
new_urls1 = handle_download_links(new_urls)
if new_urls1 == new_urls:
break
new_urls = new_urls1
if len(new_urls) > 2048:
logger.e('new url too long: %s, will not process.' % new_urls)
return
update_finalapp(id, new_urls)
def callback(ch, method, properties, body):
url = body
queue_name_running = 'downloadurls_running'
queue_name_success = 'downloadurls_success'
queue_name_failed = 'downloadurls_failed'
if _redis.hexists(queue_name_success, url):
ch.basic_ack(delivery_tag=method.delivery_tag)
elif _redis.hexists(queue_name_failed, url):
ch.basic_ack(delivery_tag=method.delivery_tag)
elif _redis.hexists(queue_name_running, url):
_redis.hincrby(queue_name_running, url, 1)
ch.basic_ack(delivery_tag=method.delivery_tag)
else:
_redis.hset(queue_name_running, url, 0)
try:
process(url)
_redis.hdel(queue_name_running, url)
_redis.hset(queue_name_success, url, 0)
except Exception as e:
logger.e('process url(%s) failed: %s' % (url, e))
_redis.hset(queue_name_failed, url, 0)
ch.basic_ack(delivery_tag=method.delivery_tag)
def callback(ch, method, properties, body):
url = body
queue_name_running = 'downloadurls_running'
queue_name_success = 'downloadurls_success'
queue_name_failed = 'downloadurls_failed'
if _redis.hexists(queue_name_success, url):
ch.basic_ack(delivery_tag=method.delivery_tag)
elif _redis.hexists(queue_name_failed, url):
ch.basic_ack(delivery_tag=method.delivery_tag)
elif _redis.hexists(queue_name_running, url):
_redis.hincrby(queue_name_running, url, 1)
ch.basic_ack(delivery_tag=method.delivery_tag)
else:
_redis.hset(queue_name_running, url, 0)
try:
process(url)
_redis.hdel(queue_name_running, url)
_redis.hset(queue_name_success, url, 0)
except Exception as e:
logger.e('process url(%s) failed: %s' % (url, e))
_redis.hset(queue_name_failed, url, 0)
ch.basic_ack(delivery_tag=method.delivery_tag)
def update_avail_download_links(packagename, version_code, url):
logger.i(
'update_avail_download_links packagename: %s, version_code: %s, url: %s'
% (packagename, version_code, url))
id, urls = get_url(packagename, version_code)
if urls and urls.find(url) != -1:
logger.i('final app already has url: %s' % url)
return
elif not urls:
logger.i('final app not found')
return
else:
new_urls = '%s %s' % (urls, url)
new_urls = reform_urls(new_urls)
while len(new_urls) > 2048:
new_urls1 = handle_download_links(new_urls)
if new_urls1 == new_urls:
break
new_urls = new_urls1
if len(new_urls) > 2048:
logger.e('new url too long: %s, will not process.' % new_urls)
return
update_finalapp(id, new_urls)
def kpi2():
compare_count = 1000
stores_to_compare = [
BaiduDataSource(),
WandoujiaDataSource(),
Zhushou360DataSource()
]
estore = EstoreDataSource()
kpi_res = {}
for store in stores_to_compare:
try:
apps = store.get_apps_from_list(compare_count / 2,
datasource.LIST_TYPE_SOFT, fields)
except Exception as e:
logger.e(e)
apps = []
try:
apps1 = store.get_apps_from_list(compare_count / 2,
datasource.LIST_TYPE_GAME, fields)
except Exception as e:
logger.e(e)
apps1 = []
for a in apps1:
apps.append(a)
kpi_res.update({
store.NAME: {
'source': apps,
'equal': [],
'better': [],
'worse': [],
'none': [],
}
})
apps1_dict = {}
for i in range(0, len(apps), 50):
apps1 = estore.get_apps_by_packagename(
[a[0] for a in apps[i:i + 50]], fields)
for a in apps1:
apps1_dict[a[0]] = a
for app in apps:
app1 = apps1_dict.get(app[0], None)
if not app1:
kpi_res[store.NAME]['none'].append(app)
elif int(app[1]) == int(app1[1]):
kpi_res[store.NAME]['equal'].append(app)
elif int(app[1]) > int(app1[1]):
kpi_res[store.NAME]['better'].append((app, app1))
elif int(app[1]) < int(app1[1]):
kpi_res[store.NAME]['worse'].append((app, app1))
result = []
result.append('test apps from top list of other market, result: ')
result.append(
u'(equal: 都有此应用，并且版本一致; better: estore市场应用版本更低;worse: estore市场应用版本更高; none: estore市场无此应用;)\n'
)
result.append('%s\t%s' %
('catetory', '\t'.join([s.NAME for s in stores_to_compare])))
count_str = 'count\t%s' % ('\t'.join(
[str(len(kpi_res[s.NAME]['source'])) for s in stores_to_compare]))
result.append(count_str)
for k in ['equal', 'better', 'worse', 'none']:
result.append('%s\t%s' % (k, '\t'.join(
[str(len(kpi_res[s.NAME][k])) for s in stores_to_compare])))
result.append('\n%s\t%s' %
('catetory', '\t'.join([s.NAME for s in stores_to_compare])))
result.append(count_str)
for k in ['equal', 'better', 'worse', 'none']:
str_ = [
'%.2f%%' % (len(kpi_res[s.NAME][k]) * 100 /
(0.0 + len(kpi_res[s.NAME]['source'])))
for s in stores_to_compare
]
result.append('%s\t%s' % (k, '\t'.join(str_)))
result.append('\n\n我们市场版本更低的应用: ')
for s in stores_to_compare:
result.append('\n%s:' % s.NAME)
for app, app1 in kpi_res[s.NAME]['better']:
result.append(u'%s(%s): %s->%s %s->%s' %
(app[3], app[0], app[1], app1[1], app[2], app1[2]))
result.append(u'\n\n我们市场没有的应用: ')
for s in stores_to_compare:
result.append('\n%s:' % s.NAME)
for app in kpi_res[s.NAME]['none']:
result.append(u'%s(%s): %s %s' % (app[3], app[0], app[1], app[2]))
print '\n'.join(result)
def _get_data(self, data, fields):
try:
return BaseDataSource.json_get_data(self, data, fields, FIELDS)
except Exception as e:
logger.e('BaiduDataSource _get_data failed: %s' % e)
return None
def kpi2():
compare_count = 1000
stores_to_compare = [BaiduDataSource(), WandoujiaDataSource(), Zhushou360DataSource()]
estore = EstoreDataSource()
kpi_res = {}
for store in stores_to_compare:
try:
apps = store.get_apps_from_list(compare_count / 2, datasource.LIST_TYPE_SOFT, fields)
except Exception as e:
logger.e(e)
apps = []
try:
apps1 = store.get_apps_from_list(compare_count / 2, datasource.LIST_TYPE_GAME, fields)
except Exception as e:
logger.e(e)
apps1 = []
for a in apps1:
apps.append(a)
kpi_res.update({
store.NAME: {
'source': apps,
'equal': [],
'better': [],
'worse': [],
'none': [],
}
})
apps1_dict = {}
for i in range(0, len(apps), 50):
apps1 = estore.get_apps_by_packagename([a[0] for a in apps[i:i + 50]], fields)
for a in apps1:
apps1_dict[a[0]] = a
for app in apps:
app1 = apps1_dict.get(app[0], None)
if not app1:
kpi_res[store.NAME]['none'].append(app)
elif int(app[1]) == int(app1[1]):
kpi_res[store.NAME]['equal'].append(app)
elif int(app[1]) > int(app1[1]):
kpi_res[store.NAME]['better'].append((app, app1))
elif int(app[1]) < int(app1[1]):
kpi_res[store.NAME]['worse'].append((app, app1))
result = []
result.append('test apps from top list of other market, result: ')
result.append(u'(equal: 都有此应用，并且版本一致; better: estore市场应用版本更低;worse: estore市场应用版本更高; none: estore市场无此应用;)\n')
result.append('%s\t%s' % ('catetory', '\t'.join([s.NAME for s in stores_to_compare])))
count_str = 'count\t%s' % ('\t'.join([str(len(kpi_res[s.NAME]['source'])) for s in stores_to_compare]))
result.append(count_str)
for k in ['equal', 'better', 'worse', 'none']:
result.append('%s\t%s' % (k, '\t'.join([str(len(kpi_res[s.NAME][k])) for s in stores_to_compare])))
result.append('\n%s\t%s' % ('catetory', '\t'.join([s.NAME for s in stores_to_compare])))
result.append(count_str)
for k in ['equal', 'better', 'worse', 'none']:
str_ = ['%.2f%%' % (len(kpi_res[s.NAME][k]) * 100 / (0.0 + len(kpi_res[s.NAME]['source'])))
for s in stores_to_compare]
result.append('%s\t%s' % (k, '\t'.join(str_)))
result.append('\n\n我们市场版本更低的应用: ')
for s in stores_to_compare:
result.append('\n%s:' % s.NAME)
for app, app1 in kpi_res[s.NAME]['better']:
result.append(u'%s(%s): %s->%s %s->%s' % (app[3], app[0], app[1], app1[1], app[2], app1[2]))
result.append(u'\n\n我们市场没有的应用: ')
for s in stores_to_compare:
result.append('\n%s:' % s.NAME)
for app in kpi_res[s.NAME]['none']:
result.append(u'%s(%s): %s %s' % (app[3], app[0], app[1], app[2]))
print '\n'.join(result)
