def convert_taobaoke_widget(items, fn_join_iids=lambda x:','.join(x), batch_size=40, calllimit=60, expire=600, outer_code='jcn', appkey=TAOBAOKE_APPKEY, appsec=TAOBAOKE_APPSECRET):
ts = int(time.time()*1000)
msg = appsec + 'app_key' + str(appkey) + "timestamp" + str(ts) + appsec
sign = hmac.HMAC(appsec, msg).hexdigest().upper()
headers = {'User-Agent' : DEFAULT_UA, 'Referer' : "http://www.j.cn/"}
for chunk in waitlimit(calllimit, 60.0, chunks(items, batch_size)): # calllimit for minutes
params = {'app_key' : appkey,
'_t_sys' : 'args=4',
'method' : 'taobao.taobaoke.widget.items.convert',
'sign' : sign,
'timestamp' : ts,
'fields' : "num_iid,nick,price,click_url,commission,commission_rate,commission_num,commission_volume,shop_click_url,seller_credit_score",
'callback' : 'TOP.io.jsonpCbs.t%s' % md5( str(random.random()) ).hexdigest()[:13],
'partner_id' : 'top-sdk-js-20120801',
}
params['num_iids'] = fn_join_iids(chunk)
if outer_code:
params['outer_code'] = outer_code
url = "http://gw.api.taobao.com/widget/rest?%s" % urllib.urlencode(params)
results = download(url, headers=headers)
if results:
Statsd.increment('guang.taobaoapi.widget_succ')
else:
Statsd.increment('guang.taobaoapi.widget_err')
#logger.debug('Calling %s(%s) -> %s', request.method_name, request.api_params, results)
yield (chunk, results)
def get_taobao_items(top, items, fn_join_iids=lambda x:','.join(x), batch_size=20, calllimit=60, expire=600):
request = TOPRequest('taobao.items.list.get')
for chunk in waitlimit(calllimit, 60.0, batch(items, batch_size)): # calllimit for minutes
chunk = list(chunk)
request['fields'] = "detail_url,cid,num_iid,title,nick,pic_url,num,price,has_showcase,approve_status,list_time,delist_time,modified,stuff_status,is_timing,post_fee,express_fee,ems_fee, has_discount,freight_payer"
request['num_iids'] = fn_join_iids(chunk)
results = try_execute(top, request, expire)
#logger.debug('Calling %s(%s) -> %s', request.method_name, request.api_params, results)
yield TopRequests(chunk, results)
def get_deleted_items(top, items, fn_join_iids=lambda x:','.join(map(str, x)), batch_size=20, calllimit=60, expire=600):
request = TOPRequest('taobao.items.list.get')
for chunk in waitlimit(calllimit, 60.0, batch(items, batch_size)):
chunk = list(chunk)
request['fields'] = "num_iid,approve_status"
request['num_iids'] = fn_join_iids(chunk)
results = try_execute(top, request, expire)
logger.debug('Calling %s(%s) -> %s', request.method_name, request.api_params, results)
try:
# TODO: request may not show in result
for item in results['items']['item']:
if item['approve_status'] != 'onsale':
yield item
except:
logger.warn("get_deleted_items %s", traceback.format_exc())
def get_taobao_items(top,
items,
fn_join_iids=lambda x: ','.join(x),
batch_size=20,
calllimit=60,
expire=600):
request = TOPRequest('taobao.items.list.get')
for chunk in waitlimit(calllimit, 60.0,
batch(items, batch_size)): # calllimit for minutes
chunk = list(chunk)
request[
'fields'] = "detail_url,cid,num_iid,title,nick,pic_url,num,price,has_showcase,approve_status,list_time,delist_time,modified,stuff_status,is_timing,post_fee,express_fee,ems_fee, has_discount,freight_payer"
request['num_iids'] = fn_join_iids(chunk)
results = try_execute(top, request, expire)
#logger.debug('Calling %s(%s) -> %s', request.method_name, request.api_params, results)
yield TopRequests(chunk, results)
def convert_taobaoke_widget(items,
fn_join_iids=lambda x: ','.join(x),
batch_size=40,
calllimit=60,
expire=600,
outer_code='jcn',
appkey=TAOBAOKE_APPKEY,
appsec=TAOBAOKE_APPSECRET):
ts = int(time.time() * 1000)
msg = appsec + 'app_key' + str(appkey) + "timestamp" + str(ts) + appsec
sign = hmac.HMAC(appsec, msg).hexdigest().upper()
headers = {'User-Agent': DEFAULT_UA, 'Referer': "http://www.j.cn/"}
for chunk in waitlimit(calllimit, 60.0,
chunks(items, batch_size)): # calllimit for minutes
params = {
'app_key':
appkey,
'_t_sys':
'args=4',
'method':
'taobao.taobaoke.widget.items.convert',
'sign':
sign,
'timestamp':
ts,
'fields':
"num_iid,nick,price,click_url,commission,commission_rate,commission_num,commission_volume,shop_click_url,seller_credit_score",
'callback':
'TOP.io.jsonpCbs.t%s' % md5(str(random.random())).hexdigest()[:13],
'partner_id':
'top-sdk-js-20120801',
}
params['num_iids'] = fn_join_iids(chunk)
if outer_code:
params['outer_code'] = outer_code
url = "http://gw.api.taobao.com/widget/rest?%s" % urllib.urlencode(
params)
results = download(url, headers=headers)
if results:
Statsd.increment('guang.taobaoapi.widget_succ')
else:
Statsd.increment('guang.taobaoapi.widget_err')
#logger.debug('Calling %s(%s) -> %s', request.method_name, request.api_params, results)
yield (chunk, results)
def get_deleted_items(top,
items,
fn_join_iids=lambda x: ','.join(map(str, x)),
batch_size=20,
calllimit=60,
expire=600):
request = TOPRequest('taobao.items.list.get')
for chunk in waitlimit(calllimit, 60.0, batch(items, batch_size)):
chunk = list(chunk)
request['fields'] = "num_iid,approve_status"
request['num_iids'] = fn_join_iids(chunk)
results = try_execute(top, request, expire)
logger.debug('Calling %s(%s) -> %s', request.method_name,
request.api_params, results)
try:
# TODO: request may not show in result
for item in results['items']['item']:
if item['approve_status'] != 'onsale':
yield item
except:
logger.warn("get_deleted_items %s", traceback.format_exc())
def main():
if FLAGS.sessionid == "":
logger.error(
"Get SESSION from http://container.api.taobao.com/container?appkey=12525923"
)
db = None
csv_w = None
if not FLAGS.dryrun:
db = get_db_engine()
if FLAGS.csv:
csv_w = csv.writer(open(FLAGS.csv_filename, "wb"),
delimiter=FLAGS.csv_split,
quotechar=FLAGS.csv_quote,
quoting=csv.QUOTE_NONNUMERIC)
csv_w.writerow([
"report_date", "outer_code", "commission_rate", "item_title",
"seller_nick", "num_iid", "shop_title", "app_key", "commission",
"trade_id", "pay_time", "item_num", "category_id", "pay_price",
"real_pay_fee", "category_name"
])
for d in waitlimit(FLAGS.limit, 60.0, dates()):
logger.info("Fetching %s %s", d, FLAGS.sessionid)
try:
pageno = 1
total = 100
result_len = 100
got = 0
while result_len >= total:
report = get_report(get_top(), d, FLAGS.sessionid, pageno,
total)
if not report:
logger.info("result %s %s null", d, pageno)
break
else:
logger.info("result %s %s", d, pageno)
pageno += 1
result_len = len(
report['taobaoke_report']['taobaoke_report_members']
['taobaoke_report_member'])
got += result_len
logger.info(
"result %s %s -> %s %s", d, pageno, got,
len(report['taobaoke_report']['taobaoke_report_members']
['taobaoke_report_member']))
if result_len > 0:
members = report['taobaoke_report'][
'taobaoke_report_members']['taobaoke_report_member']
for m in members:
try:
#import pdb; pdb.set_trace()
check_sql = """select outer_code, commission_rate, item_title, seller_nick, num_iid,
shop_title, app_key, commission, trade_id, pay_time, item_num,
category_id, pay_price, real_pay_fee, category_name, create_time,
confirm_time, status from taobao_report
where trade_id=%s""" % m['trade_id']
result = list(db.execute(check_sql))
if result:
if result[0][0] == m.get(
'outer_code',
'') and result[0][4] == m['num_iid']:
logger.debug(
"already exists in db, skip %s vs %s",
result[0], m)
else:
logger.warn(
"same trade id, something wrong! %s %s"
% (m, result))
continue
sql = """insert into taobao_report (outer_code, commission_rate, item_title, seller_nick,
num_iid, shop_title, app_key, commission, trade_id, pay_time, item_num,
category_id, pay_price, real_pay_fee, category_name) values (
"%s", "%s", "%s", "%s", %s, "%s", "%s", "%s", %s, "%s", %s, %s, "%s", "%s", "%s"
)""" % (
m.get('outer_code', ''),
m['commission_rate'].replace(
'%', '%%'), m['item_title'].replace(
'%', '%%'), m['seller_nick'].replace(
'%', '%%'), m['num_iid'],
m['shop_title'].replace('%', '%%'),
m['app_key'], m['commission'], m['trade_id'],
m['pay_time'], m['item_num'], m['category_id'],
m['pay_price'], m['real_pay_fee'],
m.get('category_name', '').replace('%', '%%'))
logger.debug(sql)
if db:
try:
db.execute(sql)
except:
logger.warn(
"insert failed sql %s --> err %s", sql,
traceback.format_exc())
if csv_w:
writecsv(csv_w, [
d,
m.get('outer_code',
''), m['commission_rate'],
m['item_title'], m['seller_nick'],
m['num_iid'], m['shop_title'],
m['app_key'], m['commission'],
m['trade_id'], m['pay_time'],
m['item_num'], m['category_id'],
m['pay_price'], m['real_pay_fee'],
m.get('category_name', '')
])
except:
logger.error("Got error %s %s", m,
traceback.format_exc())
except:
logger.error("Got fatal error %s %s", d, traceback.format_exc())
def main():
if FLAGS.sessionid == "":
logger.error("Get SESSION from http://container.api.taobao.com/container?appkey=12525923")
db = None
csv_w = None
if not FLAGS.dryrun:
db = get_db_engine()
if FLAGS.csv:
csv_w = csv.writer(open(FLAGS.csv_filename, "wb"), delimiter=FLAGS.csv_split,
quotechar=FLAGS.csv_quote, quoting=csv.QUOTE_NONNUMERIC)
csv_w.writerow(["report_date", "outer_code", "commission_rate", "item_title", "seller_nick", "num_iid",
"shop_title", "app_key", "commission", "trade_id", "pay_time", "item_num",
"category_id", "pay_price", "real_pay_fee", "category_name"])
for d in waitlimit(FLAGS.limit, 60.0, dates()):
logger.info("Fetching %s %s", d, FLAGS.sessionid)
try:
pageno = 1
total = 100
result_len = 100
got = 0
while result_len >= total:
report = get_report(get_top(), d, FLAGS.sessionid, pageno, total)
if not report:
logger.info("result %s %s null", d, pageno)
break
else:
logger.info("result %s %s", d, pageno)
result_len = len(report['taobaoke_payments']['taobaoke_payment'])
got += result_len
logger.info("result %s %s -> %s %s", d, pageno, got, len(report['taobaoke_payments']['taobaoke_payment']))
if result_len > 0:
members = report['taobaoke_payments']['taobaoke_payment']
for m in members:
try:
#import pdb; pdb.set_trace()
check_sql = """select outer_code, commission_rate, item_title, seller_nick, num_iid,
shop_title, app_key, commission, trade_id, pay_time, item_num,
category_id, pay_price, real_pay_fee, category_name, create_time,
confirm_time, status from taobao_report
where trade_id=%s""" % m['trade_id']
result = list(db.execute(check_sql))
if result:
if result[0][0] == m.get('outer_code', '') and result[0][4] == m['num_iid']:
logger.debug("already exists in db, skip %s vs %s", result[0], m)
else:
logger.warn("same trade id, something wrong! %s %s" % (m, result))
continue
sql = """insert into taobao_report (outer_code, commission_rate, item_title, seller_nick,
num_iid, shop_title, app_key, commission, trade_id, pay_time, item_num,
category_id, pay_price, real_pay_fee, category_name, create_time) values (
"%s", "%s", "%s", "%s", %s, "%s", "%s", "%s", %s, "%s", %s, %s, "%s", "%s", "%s", now()
)""" % (
m.get('outer_code', ''), m['commission_rate'].replace('%', '%%'), m['item_title'].replace('%', '%%'),
m['seller_nick'].replace('%', '%%'), m['num_iid'],
m['shop_title'].replace('%', '%%'), m['app_key'], m['commission'], m['trade_id'], m['pay_time'], m['item_num'],
m['category_id'], m['pay_price'], m['real_pay_fee'], m.get('category_name','').replace('%', '%%')
)
logger.debug(sql)
if db:
try:
db.execute(sql)
except:
logger.warn("insert failed sql %s --> err %s", sql, traceback.format_exc())
if csv_w:
writecsv(csv_w, [d, m.get('outer_code', ''), m['commission_rate'], m['item_title'], m['seller_nick'], m['num_iid'],
m['shop_title'], m['app_key'], m['commission'], m['trade_id'], m['pay_time'], m['item_num'],
m['category_id'], m['pay_price'], m['real_pay_fee'], m.get('category_name', '')])
except:
logger.error("Got error %s %s", m, traceback.format_exc())
pageno += 1
except:
logger.error("Got fatal error %s %s", d, traceback.format_exc())
