def get_apps_by_packagename(self, packagename_list, fields=None): if not isinstance(packagename_list, list): packagename_list = [packagename_list] url = 'http://estoresrvice.189store.com/api/app/updates2.json' param = { 'clientid': 'kpitest', 'source': 'kpitest_1', 'appinfos': '|'.join(['%s:0' % pn for pn in packagename_list]), 'all_apps': 2, } header = {'Accept-Encoding': 'gzip,deflate,sdch'} json_data = Net.read_json(url, method='post', data=param, header=header) ret = [] ret_packages = set() if not fields: return ret for data in json_data: res_data = [] for f in fields: f = FIELDS[f] if f not in data: logger.e('key %s not in object: %s' % (f, data)) break res_data.append(data[f]) ret_packages.add(data[FIELDS[datasource.FIELD_PACKAGE_NAME]]) if len(res_data) == len(fields): ret.append(res_data) logger.e('can not find new version for app: %s' % (set(packagename_list) - ret_packages)) return ret
def run(): logger.i('rmq_channel.start_consuming') _pika = None rmq_channel = None fail_times = 0 while True and not should_stop and fail_times < max_fail_times: try: _pika = pika.BlockingConnection( pika.ConnectionParameters( **settings.RABBITMQ['interception_download_urls'])) pikas.append(_pika) rmq_channel = _pika.channel() rmq_channel.queue_declare(queue='downloadurls', durable=True) rmq_channel.basic_qos(prefetch_count=1) rmq_channel.basic_consume(callback, queue='downloadurls') rmq_channels.append(rmq_channel) rmq_channel.start_consuming() except Exception as e: fail_times = fail_times + 1 logger.e('error consuming: %s' % e) if rmq_channel in rmq_channels: try: rmq_channel.stop_consuming() rmq_channel.close() except: pass rmq_channels.remove(rmq_channel) if _pika in pikas: try: _pika.close() except: pass pikas.remove(_pika) time.sleep(5)
def run(): logger.i('rmq_channel.start_consuming') _pika = None rmq_channel = None fail_times = 0 while True and not should_stop and fail_times < max_fail_times: try: _pika = pika.BlockingConnection( pika.ConnectionParameters(**settings.RABBITMQ['interception_download_urls'])) pikas.append(_pika) rmq_channel = _pika.channel() rmq_channel.queue_declare(queue='downloadurls', durable=True) rmq_channel.basic_qos(prefetch_count=1) rmq_channel.basic_consume(callback, queue='downloadurls') rmq_channels.append(rmq_channel) rmq_channel.start_consuming() except Exception as e: fail_times = fail_times + 1 logger.e('error consuming: %s' % e) if rmq_channel in rmq_channels: try: rmq_channel.stop_consuming() rmq_channel.close() except: pass rmq_channels.remove(rmq_channel) if _pika in pikas: try: _pika.close() except: pass pikas.remove(_pika) time.sleep(5)
def update_appchina_real_downloadlink(avail_download_links): logger.d(str(avail_download_links)) if avail_download_links: avail_download_links = avail_download_links.split(' ') newurls = [] for l in avail_download_links: if l.find('http://www.appchina.com') == 0 or l.find('http://www.d.appchina.com') == 0: try: newl = get_real_url(l) if newl != l and newl not in avail_download_links: newurls.append(newl) except Exception as e: logger.e('%s' % e) if newurls: for l in newurls: avail_download_links.append(l) avail_download_links = ' '.join(avail_download_links) logger.d(str(avail_download_links)) return avail_download_links else: logger.d('nothing need to update.') return None else: logger.d('avail_download_links is empty, do not need to update.') return None
def callback(ch, method, properties, body): id = body try: if not process(id): logger.d('process id(%s) return False, add to queue') ch.basic_publish(exchange='', routing_key=_queue_name, body=id, properties=pika.BasicProperties(delivery_mode=2)) except Exception as e: logger.e('process id(%s) failed: %s' % (id, e)) logger.d('process id(%s) return False, add to queue') ch.basic_publish(exchange='', routing_key=_queue_name, body=id, properties=pika.BasicProperties(delivery_mode=2)) ch.basic_ack(delivery_tag=method.delivery_tag)
def update_apps_labels(apps): suc_count = 0 for pn, vc in apps: try: labels = get_wandoujia_app_labels(pn, vc) if labels: update_app(pn, vc, labels) suc_count += 1 logger.i('update app(pn=%s, vc=%s) success.' % (pn, vc)) else: logger.i('ignore app(pn=%s, vc=%s).' % (pn, vc)) except Exception as e: logger.e('update app(pn=%s, vc=%s) failed: %s' % (pn, vc, e)) return suc_count
def graceful_exit(*args, **kargs): logger.i('request rabbit_consumer stop(%s, %s). stopping...' % (args, kargs)) global should_stop should_stop = True for c in rmq_channels: try: c.stop_consuming() c.close() except Exception as e: logger.e('stop_consuming failed: %s' % e) for p in pikas: try: p.close() except Exception as e: logger.e('close rabbit connection failed: %s' % e)
def process(id): logger.i('process id: %s' % id) file_path, vol_id = get_url(id) if file_path: data = '{"authkey":"9ORmsDOAJ3zcD21w", "url":"http://estoredwnld7.189store.com/downloads/vol%s/%s"}' % \ (vol_id, file_path) url = 'http://%s/vol%s/%s' % ('estoredwnld7.189store.com', vol_id, file_path) url1 = 'http://%s/vol%s/%s' % (download_host, vol_id, file_path) if url404(url, url1): logger.i('found app(id=%s) url(%s) 404' % (id, url)) update_finalapp_safe(id, 4, '') return True tencent_url = 'https://api.scan.qq.com/browser/scan' ret = '' try: ret = post(tencent_url, data) ret = ret.replace('\r', '') ret = ret.replace('\n', '') ret = simplejson.loads(ret) except Exception as e: logger.e(u'ret format error: %s' % ret) raise e logger.d('tencent returns: %s' % simplejson.dumps(ret)) if 'safetype' not in ret: raise 'error format: %s' % simplejson.dumps(ret) if ret['safetype'] == 'unknown': update_finalapp_safe(id, None, '') return False if ret['safetype'] == 'virus': logger.i('found app(id=%s) has virus.' % id) update_finalapp_safe(id, 0, simplejson.dumps(ret)) return True if ret['safetype'] == 'lowrisk': logger.i('found app(id=%s) lowrisk.' % id) update_finalapp_safe(id, 2, simplejson.dumps(ret)) return True if ret['safetype'] == 'midrisk': logger.i('found app(id=%s) midrisk.' % id) update_finalapp_safe(id, 3, simplejson.dumps(ret)) return True if ret['safetype'] == 'safe': logger.i('found app(id=%s) safe.' % id) update_finalapp_safe(id, 1, '') return True else: logger.i('can not find id in db, ignore: %s' % id) return True
def update_apps_avail_download_links(apps): suc_count = 0 for pn, vc in apps: try: avail_download_links = get_final_app_cols(pn, vc, ['avail_download_links']) if avail_download_links: avail_download_links = avail_download_links[1] avail_download_links = update_appchina_real_downloadlink(avail_download_links) if avail_download_links: update_app(pn, vc, avail_download_links=avail_download_links) suc_count += 1 logger.i('update app(pn=%s, vc=%s) success.' % (pn, vc)) else: logger.i('ignore app(pn=%s, vc=%s).' % (pn, vc)) except Exception as e: logger.e('update app(pn=%s, vc=%s) failed: %s' % (pn, vc, e)) return suc_count
def get_app_info_by_packagename(self, packagename, fields=None): url = ('http://apps.wandoujia.com/api/v1/apps/%s?opt_fields=apks.compatible,apks.incompatibleDetail,' 'ad,apks.adsType,apks.beta,apks.bytes,apks.downloadUrl.*,apks.size,apks.versionCode,apks.versionName,' 'apks.securityStatus,apks.md5,apks.permissionLevel,apks.superior,apks.dangerousPermissions,' 'apks.permissions,apks.securityDetail.*,apks.resolution,categories.alias,categories.name,description,' 'title,commentsCount,likeCount,dislikesCount,installedCountStr,icons.px78,' 'packageName,id,likesRate,screenshots.*,apks.verified,apks.paidType') url = url % packagename try: json_data = Net.read_json(url) except Exception as e: logger.e('read_json from url(%s) failed: %s' % (url, e)) return None if 'apks' not in json_data: logger.w('one or more fields(%s) can not be found in object: %s' % (fields, json_data)) return None return self._get_data(json_data, fields, packagename=None)
def update_avail_download_links(packagename, version_code, url): logger.i('update_avail_download_links packagename: %s, version_code: %s, url: %s' % (packagename, version_code, url)) id, urls = get_url(packagename, version_code) if urls and urls.find(url) != -1: logger.i('final app already has url: %s' % url) return elif not urls: logger.i('final app not found') return else: new_urls = '%s %s' % (urls, url) new_urls = reform_urls(new_urls) while len(new_urls) > 2048: new_urls1 = handle_download_links(new_urls) if new_urls1 == new_urls: break new_urls = new_urls1 if len(new_urls) > 2048: logger.e('new url too long: %s, will not process.' % new_urls) return update_finalapp(id, new_urls)
def callback(ch, method, properties, body): url = body queue_name_running = 'downloadurls_running' queue_name_success = 'downloadurls_success' queue_name_failed = 'downloadurls_failed' if _redis.hexists(queue_name_success, url): ch.basic_ack(delivery_tag=method.delivery_tag) elif _redis.hexists(queue_name_failed, url): ch.basic_ack(delivery_tag=method.delivery_tag) elif _redis.hexists(queue_name_running, url): _redis.hincrby(queue_name_running, url, 1) ch.basic_ack(delivery_tag=method.delivery_tag) else: _redis.hset(queue_name_running, url, 0) try: process(url) _redis.hdel(queue_name_running, url) _redis.hset(queue_name_success, url, 0) except Exception as e: logger.e('process url(%s) failed: %s' % (url, e)) _redis.hset(queue_name_failed, url, 0) ch.basic_ack(delivery_tag=method.delivery_tag)
def update_avail_download_links(packagename, version_code, url): logger.i( 'update_avail_download_links packagename: %s, version_code: %s, url: %s' % (packagename, version_code, url)) id, urls = get_url(packagename, version_code) if urls and urls.find(url) != -1: logger.i('final app already has url: %s' % url) return elif not urls: logger.i('final app not found') return else: new_urls = '%s %s' % (urls, url) new_urls = reform_urls(new_urls) while len(new_urls) > 2048: new_urls1 = handle_download_links(new_urls) if new_urls1 == new_urls: break new_urls = new_urls1 if len(new_urls) > 2048: logger.e('new url too long: %s, will not process.' % new_urls) return update_finalapp(id, new_urls)
def kpi2(): compare_count = 1000 stores_to_compare = [ BaiduDataSource(), WandoujiaDataSource(), Zhushou360DataSource() ] estore = EstoreDataSource() kpi_res = {} for store in stores_to_compare: try: apps = store.get_apps_from_list(compare_count / 2, datasource.LIST_TYPE_SOFT, fields) except Exception as e: logger.e(e) apps = [] try: apps1 = store.get_apps_from_list(compare_count / 2, datasource.LIST_TYPE_GAME, fields) except Exception as e: logger.e(e) apps1 = [] for a in apps1: apps.append(a) kpi_res.update({ store.NAME: { 'source': apps, 'equal': [], 'better': [], 'worse': [], 'none': [], } }) apps1_dict = {} for i in range(0, len(apps), 50): apps1 = estore.get_apps_by_packagename( [a[0] for a in apps[i:i + 50]], fields) for a in apps1: apps1_dict[a[0]] = a for app in apps: app1 = apps1_dict.get(app[0], None) if not app1: kpi_res[store.NAME]['none'].append(app) elif int(app[1]) == int(app1[1]): kpi_res[store.NAME]['equal'].append(app) elif int(app[1]) > int(app1[1]): kpi_res[store.NAME]['better'].append((app, app1)) elif int(app[1]) < int(app1[1]): kpi_res[store.NAME]['worse'].append((app, app1)) result = [] result.append('test apps from top list of other market, result: ') result.append( u'(equal: 都有此应用,并且版本一致; better: estore市场应用版本更低;worse: estore市场应用版本更高; none: estore市场无此应用;)\n' ) result.append('%s\t%s' % ('catetory', '\t'.join([s.NAME for s in stores_to_compare]))) count_str = 'count\t%s' % ('\t'.join( [str(len(kpi_res[s.NAME]['source'])) for s in stores_to_compare])) result.append(count_str) for k in ['equal', 'better', 'worse', 'none']: result.append('%s\t%s' % (k, '\t'.join( [str(len(kpi_res[s.NAME][k])) for s in stores_to_compare]))) result.append('\n%s\t%s' % ('catetory', '\t'.join([s.NAME for s in stores_to_compare]))) result.append(count_str) for k in ['equal', 'better', 'worse', 'none']: str_ = [ '%.2f%%' % (len(kpi_res[s.NAME][k]) * 100 / (0.0 + len(kpi_res[s.NAME]['source']))) for s in stores_to_compare ] result.append('%s\t%s' % (k, '\t'.join(str_))) result.append('\n\n我们市场版本更低的应用: ') for s in stores_to_compare: result.append('\n%s:' % s.NAME) for app, app1 in kpi_res[s.NAME]['better']: result.append(u'%s(%s): %s->%s %s->%s' % (app[3], app[0], app[1], app1[1], app[2], app1[2])) result.append(u'\n\n我们市场没有的应用: ') for s in stores_to_compare: result.append('\n%s:' % s.NAME) for app in kpi_res[s.NAME]['none']: result.append(u'%s(%s): %s %s' % (app[3], app[0], app[1], app[2])) print '\n'.join(result)
def _get_data(self, data, fields): try: return BaseDataSource.json_get_data(self, data, fields, FIELDS) except Exception as e: logger.e('BaiduDataSource _get_data failed: %s' % e) return None
def kpi2(): compare_count = 1000 stores_to_compare = [BaiduDataSource(), WandoujiaDataSource(), Zhushou360DataSource()] estore = EstoreDataSource() kpi_res = {} for store in stores_to_compare: try: apps = store.get_apps_from_list(compare_count / 2, datasource.LIST_TYPE_SOFT, fields) except Exception as e: logger.e(e) apps = [] try: apps1 = store.get_apps_from_list(compare_count / 2, datasource.LIST_TYPE_GAME, fields) except Exception as e: logger.e(e) apps1 = [] for a in apps1: apps.append(a) kpi_res.update({ store.NAME: { 'source': apps, 'equal': [], 'better': [], 'worse': [], 'none': [], } }) apps1_dict = {} for i in range(0, len(apps), 50): apps1 = estore.get_apps_by_packagename([a[0] for a in apps[i:i + 50]], fields) for a in apps1: apps1_dict[a[0]] = a for app in apps: app1 = apps1_dict.get(app[0], None) if not app1: kpi_res[store.NAME]['none'].append(app) elif int(app[1]) == int(app1[1]): kpi_res[store.NAME]['equal'].append(app) elif int(app[1]) > int(app1[1]): kpi_res[store.NAME]['better'].append((app, app1)) elif int(app[1]) < int(app1[1]): kpi_res[store.NAME]['worse'].append((app, app1)) result = [] result.append('test apps from top list of other market, result: ') result.append(u'(equal: 都有此应用,并且版本一致; better: estore市场应用版本更低;worse: estore市场应用版本更高; none: estore市场无此应用;)\n') result.append('%s\t%s' % ('catetory', '\t'.join([s.NAME for s in stores_to_compare]))) count_str = 'count\t%s' % ('\t'.join([str(len(kpi_res[s.NAME]['source'])) for s in stores_to_compare])) result.append(count_str) for k in ['equal', 'better', 'worse', 'none']: result.append('%s\t%s' % (k, '\t'.join([str(len(kpi_res[s.NAME][k])) for s in stores_to_compare]))) result.append('\n%s\t%s' % ('catetory', '\t'.join([s.NAME for s in stores_to_compare]))) result.append(count_str) for k in ['equal', 'better', 'worse', 'none']: str_ = ['%.2f%%' % (len(kpi_res[s.NAME][k]) * 100 / (0.0 + len(kpi_res[s.NAME]['source']))) for s in stores_to_compare] result.append('%s\t%s' % (k, '\t'.join(str_))) result.append('\n\n我们市场版本更低的应用: ') for s in stores_to_compare: result.append('\n%s:' % s.NAME) for app, app1 in kpi_res[s.NAME]['better']: result.append(u'%s(%s): %s->%s %s->%s' % (app[3], app[0], app[1], app1[1], app[2], app1[2])) result.append(u'\n\n我们市场没有的应用: ') for s in stores_to_compare: result.append('\n%s:' % s.NAME) for app in kpi_res[s.NAME]['none']: result.append(u'%s(%s): %s %s' % (app[3], app[0], app[1], app[2])) print '\n'.join(result)