def convert_taobaoke_widget(items, fn_join_iids=lambda x:','.join(x), batch_size=40, calllimit=60, expire=600, outer_code='jcn', appkey=TAOBAOKE_APPKEY, appsec=TAOBAOKE_APPSECRET): ts = int(time.time()*1000) msg = appsec + 'app_key' + str(appkey) + "timestamp" + str(ts) + appsec sign = hmac.HMAC(appsec, msg).hexdigest().upper() headers = {'User-Agent' : DEFAULT_UA, 'Referer' : "http://www.j.cn/"} for chunk in waitlimit(calllimit, 60.0, chunks(items, batch_size)): # calllimit for minutes params = {'app_key' : appkey, '_t_sys' : 'args=4', 'method' : 'taobao.taobaoke.widget.items.convert', 'sign' : sign, 'timestamp' : ts, 'fields' : "num_iid,nick,price,click_url,commission,commission_rate,commission_num,commission_volume,shop_click_url,seller_credit_score", 'callback' : 'TOP.io.jsonpCbs.t%s' % md5( str(random.random()) ).hexdigest()[:13], 'partner_id' : 'top-sdk-js-20120801', } params['num_iids'] = fn_join_iids(chunk) if outer_code: params['outer_code'] = outer_code url = "http://gw.api.taobao.com/widget/rest?%s" % urllib.urlencode(params) results = download(url, headers=headers) if results: Statsd.increment('guang.taobaoapi.widget_succ') else: Statsd.increment('guang.taobaoapi.widget_err') #logger.debug('Calling %s(%s) -> %s', request.method_name, request.api_params, results) yield (chunk, results)
def get_taobao_items(top, items, fn_join_iids=lambda x:','.join(x), batch_size=20, calllimit=60, expire=600): request = TOPRequest('taobao.items.list.get') for chunk in waitlimit(calllimit, 60.0, batch(items, batch_size)): # calllimit for minutes chunk = list(chunk) request['fields'] = "detail_url,cid,num_iid,title,nick,pic_url,num,price,has_showcase,approve_status,list_time,delist_time,modified,stuff_status,is_timing,post_fee,express_fee,ems_fee, has_discount,freight_payer" request['num_iids'] = fn_join_iids(chunk) results = try_execute(top, request, expire) #logger.debug('Calling %s(%s) -> %s', request.method_name, request.api_params, results) yield TopRequests(chunk, results)
def get_deleted_items(top, items, fn_join_iids=lambda x:','.join(map(str, x)), batch_size=20, calllimit=60, expire=600): request = TOPRequest('taobao.items.list.get') for chunk in waitlimit(calllimit, 60.0, batch(items, batch_size)): chunk = list(chunk) request['fields'] = "num_iid,approve_status" request['num_iids'] = fn_join_iids(chunk) results = try_execute(top, request, expire) logger.debug('Calling %s(%s) -> %s', request.method_name, request.api_params, results) try: # TODO: request may not show in result for item in results['items']['item']: if item['approve_status'] != 'onsale': yield item except: logger.warn("get_deleted_items %s", traceback.format_exc())
def get_taobao_items(top, items, fn_join_iids=lambda x: ','.join(x), batch_size=20, calllimit=60, expire=600): request = TOPRequest('taobao.items.list.get') for chunk in waitlimit(calllimit, 60.0, batch(items, batch_size)): # calllimit for minutes chunk = list(chunk) request[ 'fields'] = "detail_url,cid,num_iid,title,nick,pic_url,num,price,has_showcase,approve_status,list_time,delist_time,modified,stuff_status,is_timing,post_fee,express_fee,ems_fee, has_discount,freight_payer" request['num_iids'] = fn_join_iids(chunk) results = try_execute(top, request, expire) #logger.debug('Calling %s(%s) -> %s', request.method_name, request.api_params, results) yield TopRequests(chunk, results)
def convert_taobaoke_widget(items, fn_join_iids=lambda x: ','.join(x), batch_size=40, calllimit=60, expire=600, outer_code='jcn', appkey=TAOBAOKE_APPKEY, appsec=TAOBAOKE_APPSECRET): ts = int(time.time() * 1000) msg = appsec + 'app_key' + str(appkey) + "timestamp" + str(ts) + appsec sign = hmac.HMAC(appsec, msg).hexdigest().upper() headers = {'User-Agent': DEFAULT_UA, 'Referer': "http://www.j.cn/"} for chunk in waitlimit(calllimit, 60.0, chunks(items, batch_size)): # calllimit for minutes params = { 'app_key': appkey, '_t_sys': 'args=4', 'method': 'taobao.taobaoke.widget.items.convert', 'sign': sign, 'timestamp': ts, 'fields': "num_iid,nick,price,click_url,commission,commission_rate,commission_num,commission_volume,shop_click_url,seller_credit_score", 'callback': 'TOP.io.jsonpCbs.t%s' % md5(str(random.random())).hexdigest()[:13], 'partner_id': 'top-sdk-js-20120801', } params['num_iids'] = fn_join_iids(chunk) if outer_code: params['outer_code'] = outer_code url = "http://gw.api.taobao.com/widget/rest?%s" % urllib.urlencode( params) results = download(url, headers=headers) if results: Statsd.increment('guang.taobaoapi.widget_succ') else: Statsd.increment('guang.taobaoapi.widget_err') #logger.debug('Calling %s(%s) -> %s', request.method_name, request.api_params, results) yield (chunk, results)
def get_deleted_items(top, items, fn_join_iids=lambda x: ','.join(map(str, x)), batch_size=20, calllimit=60, expire=600): request = TOPRequest('taobao.items.list.get') for chunk in waitlimit(calllimit, 60.0, batch(items, batch_size)): chunk = list(chunk) request['fields'] = "num_iid,approve_status" request['num_iids'] = fn_join_iids(chunk) results = try_execute(top, request, expire) logger.debug('Calling %s(%s) -> %s', request.method_name, request.api_params, results) try: # TODO: request may not show in result for item in results['items']['item']: if item['approve_status'] != 'onsale': yield item except: logger.warn("get_deleted_items %s", traceback.format_exc())
def main(): if FLAGS.sessionid == "": logger.error( "Get SESSION from http://container.api.taobao.com/container?appkey=12525923" ) db = None csv_w = None if not FLAGS.dryrun: db = get_db_engine() if FLAGS.csv: csv_w = csv.writer(open(FLAGS.csv_filename, "wb"), delimiter=FLAGS.csv_split, quotechar=FLAGS.csv_quote, quoting=csv.QUOTE_NONNUMERIC) csv_w.writerow([ "report_date", "outer_code", "commission_rate", "item_title", "seller_nick", "num_iid", "shop_title", "app_key", "commission", "trade_id", "pay_time", "item_num", "category_id", "pay_price", "real_pay_fee", "category_name" ]) for d in waitlimit(FLAGS.limit, 60.0, dates()): logger.info("Fetching %s %s", d, FLAGS.sessionid) try: pageno = 1 total = 100 result_len = 100 got = 0 while result_len >= total: report = get_report(get_top(), d, FLAGS.sessionid, pageno, total) if not report: logger.info("result %s %s null", d, pageno) break else: logger.info("result %s %s", d, pageno) pageno += 1 result_len = len( report['taobaoke_report']['taobaoke_report_members'] ['taobaoke_report_member']) got += result_len logger.info( "result %s %s -> %s %s", d, pageno, got, len(report['taobaoke_report']['taobaoke_report_members'] ['taobaoke_report_member'])) if result_len > 0: members = report['taobaoke_report'][ 'taobaoke_report_members']['taobaoke_report_member'] for m in members: try: #import pdb; pdb.set_trace() check_sql = """select outer_code, commission_rate, item_title, seller_nick, num_iid, shop_title, app_key, commission, trade_id, pay_time, item_num, category_id, pay_price, real_pay_fee, category_name, create_time, confirm_time, status from taobao_report where trade_id=%s""" % m['trade_id'] result = list(db.execute(check_sql)) if result: if result[0][0] == m.get( 'outer_code', '') and result[0][4] == m['num_iid']: logger.debug( "already exists in db, skip %s vs %s", result[0], m) else: logger.warn( "same trade id, something wrong! %s %s" % (m, result)) continue sql = """insert into taobao_report (outer_code, commission_rate, item_title, seller_nick, num_iid, shop_title, app_key, commission, trade_id, pay_time, item_num, category_id, pay_price, real_pay_fee, category_name) values ( "%s", "%s", "%s", "%s", %s, "%s", "%s", "%s", %s, "%s", %s, %s, "%s", "%s", "%s" )""" % ( m.get('outer_code', ''), m['commission_rate'].replace( '%', '%%'), m['item_title'].replace( '%', '%%'), m['seller_nick'].replace( '%', '%%'), m['num_iid'], m['shop_title'].replace('%', '%%'), m['app_key'], m['commission'], m['trade_id'], m['pay_time'], m['item_num'], m['category_id'], m['pay_price'], m['real_pay_fee'], m.get('category_name', '').replace('%', '%%')) logger.debug(sql) if db: try: db.execute(sql) except: logger.warn( "insert failed sql %s --> err %s", sql, traceback.format_exc()) if csv_w: writecsv(csv_w, [ d, m.get('outer_code', ''), m['commission_rate'], m['item_title'], m['seller_nick'], m['num_iid'], m['shop_title'], m['app_key'], m['commission'], m['trade_id'], m['pay_time'], m['item_num'], m['category_id'], m['pay_price'], m['real_pay_fee'], m.get('category_name', '') ]) except: logger.error("Got error %s %s", m, traceback.format_exc()) except: logger.error("Got fatal error %s %s", d, traceback.format_exc())
def main(): if FLAGS.sessionid == "": logger.error("Get SESSION from http://container.api.taobao.com/container?appkey=12525923") db = None csv_w = None if not FLAGS.dryrun: db = get_db_engine() if FLAGS.csv: csv_w = csv.writer(open(FLAGS.csv_filename, "wb"), delimiter=FLAGS.csv_split, quotechar=FLAGS.csv_quote, quoting=csv.QUOTE_NONNUMERIC) csv_w.writerow(["report_date", "outer_code", "commission_rate", "item_title", "seller_nick", "num_iid", "shop_title", "app_key", "commission", "trade_id", "pay_time", "item_num", "category_id", "pay_price", "real_pay_fee", "category_name"]) for d in waitlimit(FLAGS.limit, 60.0, dates()): logger.info("Fetching %s %s", d, FLAGS.sessionid) try: pageno = 1 total = 100 result_len = 100 got = 0 while result_len >= total: report = get_report(get_top(), d, FLAGS.sessionid, pageno, total) if not report: logger.info("result %s %s null", d, pageno) break else: logger.info("result %s %s", d, pageno) result_len = len(report['taobaoke_payments']['taobaoke_payment']) got += result_len logger.info("result %s %s -> %s %s", d, pageno, got, len(report['taobaoke_payments']['taobaoke_payment'])) if result_len > 0: members = report['taobaoke_payments']['taobaoke_payment'] for m in members: try: #import pdb; pdb.set_trace() check_sql = """select outer_code, commission_rate, item_title, seller_nick, num_iid, shop_title, app_key, commission, trade_id, pay_time, item_num, category_id, pay_price, real_pay_fee, category_name, create_time, confirm_time, status from taobao_report where trade_id=%s""" % m['trade_id'] result = list(db.execute(check_sql)) if result: if result[0][0] == m.get('outer_code', '') and result[0][4] == m['num_iid']: logger.debug("already exists in db, skip %s vs %s", result[0], m) else: logger.warn("same trade id, something wrong! %s %s" % (m, result)) continue sql = """insert into taobao_report (outer_code, commission_rate, item_title, seller_nick, num_iid, shop_title, app_key, commission, trade_id, pay_time, item_num, category_id, pay_price, real_pay_fee, category_name, create_time) values ( "%s", "%s", "%s", "%s", %s, "%s", "%s", "%s", %s, "%s", %s, %s, "%s", "%s", "%s", now() )""" % ( m.get('outer_code', ''), m['commission_rate'].replace('%', '%%'), m['item_title'].replace('%', '%%'), m['seller_nick'].replace('%', '%%'), m['num_iid'], m['shop_title'].replace('%', '%%'), m['app_key'], m['commission'], m['trade_id'], m['pay_time'], m['item_num'], m['category_id'], m['pay_price'], m['real_pay_fee'], m.get('category_name','').replace('%', '%%') ) logger.debug(sql) if db: try: db.execute(sql) except: logger.warn("insert failed sql %s --> err %s", sql, traceback.format_exc()) if csv_w: writecsv(csv_w, [d, m.get('outer_code', ''), m['commission_rate'], m['item_title'], m['seller_nick'], m['num_iid'], m['shop_title'], m['app_key'], m['commission'], m['trade_id'], m['pay_time'], m['item_num'], m['category_id'], m['pay_price'], m['real_pay_fee'], m.get('category_name', '')]) except: logger.error("Got error %s %s", m, traceback.format_exc()) pageno += 1 except: logger.error("Got fatal error %s %s", d, traceback.format_exc())