def sync_douban_collections(did, index, new_latest):
d = DataSource.objects.get(pk=did)
info = JSONDecoder().decode(d.auth_info)
consumer = oauth.Consumer(key=DOUBAN_KEY, secret=DOUBAN_SECRET)
token = oauth.Token(key=info['oauth_token'], secret=info['oauth_token_secret'])
client = oauth.Client(consumer, token)
url = "http://api.douban.com/people/%s/collection?start-index=%s&max-results=50&alt=json" % (info['douban_user_id'], index)
resp, content = client.request(url, "GET")
collections_info = JSONDecoder().decode(content)
if collections_info['entry']:
if (index == 1):
new_latest = collections_info['entry'][0]['id']['$t']
for collect in collections_info['entry']:
if (info.has_key('latest_collection') and collect['id']['$t'] == info['latest_collection']):
info['latest_collection'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_douban_collections.apply_async(args = [did, 1, None], countdown = 84600)
break
new_paw = Paw.objects.create(source=d, \
type="douban collection", \
content=collect['title']['$t'], \
create_time=datetime.strptime(collect['updated']['$t'][:-6], "%Y-%m-%dT%H:%M:%S"))
if not (info.has_key('latest_collection') and collect['id']['$t'] == info['latest_collection']):
sync_douban_collections.apply_async(args = [did, index+50, new_latest], countdown = 60)
else:
if new_latest is not None:
info['latest_collection'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_douban_collections.apply_async(args = [did, 1, None], countdown = 84600)
def sync_weibo(did, page, new_latest):
d = DataSource.objects.get(pk=did)
info = JSONDecoder().decode(d.auth_info)
url = "https://api.weibo.com/2/statuses/user_timeline.json?access_token=%s&uid=%s&page=%s&count=200" % (info['access_token'], info['uid'], page)
statuses_info = JSONDecoder().decode(urllib2.urlopen(url).read())
if statuses_info['statuses']:
if (page == 1):
new_latest = statuses_info['statuses'][0]['id']
for status in statuses_info['statuses']:
if (info.has_key('latest') and (status['id'] == info['latest'])):
info['latest'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_weibo.apply_async(args = [did, 1, None], countdown = 84600)
break
new_paw = Paw.objects.create(source=d, type="weibo", content=status['text'], create_time=datetime.strptime(status['created_at'][0:20]+status['created_at'][-4:], "%a %b %d %H:%M:%S %Y"))
if status.has_key('original_pic'):
new_img = Img(album = new_paw, upload_time=new_paw.create_time)
origin_tmp = NamedTemporaryFile(delete=True)
origin_tmp.write(urllib2.urlopen(status['original_pic']).read())
origin_tmp.flush()
new_img.original.save(origin_tmp.name.split('/')[-1]+'.jpg', File(origin_tmp), save=False)
thumb_tmp = NamedTemporaryFile(delete=True)
thumb_tmp.write(urllib2.urlopen(status['thumbnail_pic']).read())
thumb_tmp.flush()
new_img.thumbnail.save(thumb_tmp.name.split('/')[-1]+'.jpg', File(thumb_tmp), save=True)
if not (info.has_key('latest') and (status['id'] == info['latest'])):
sync_weibo.apply_async(args = [did, page+1, new_latest], countdown = 60)
else:
if new_latest is not None:
info['latest'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_weibo.apply_async(args = [did, 1, None], countdown = 84600)
def sync_renren_status(did, page, new_latest):
d = DataSource.objects.get(pk=did)
info = JSONDecoder().decode(d.auth_info)
sig_calc = u"access_token=%scount=1000format=jsonmethod=status.getspage=%suid=%sv=1.0%s" % (info['access_token'], page, info['user']['id'], RENREN_SECRET)
sig = calc_md5(sig_calc)
statues_url = "http://api.renren.com/restserver.do?method=status.gets&v=1.0&access_token=%s&count=1000&format=json&page=%s&uid=%s&sig=%s" % (info['access_token'], page, info['user']['id'], sig)
statues = JSONDecoder().decode(urllib2.urlopen(statues_url, "").read())
if statues:
if (page == 1):
new_latest = statues[0]['status_id']
for status in statues:
if info.has_key('latest_status') and status['status_id'] == info['latest_status']:
info['latest_status'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_renren_status.apply_async(args = [did, 1, None], countdown = 84600)
sync_renren_albums.apply_async(args = [d.id, 1], countdown = 1)
break
p = Paw.objects.create(source=d,\
type="renren status",\
content=status['message'],\
create_time=datetime.strptime(status['time'], "%Y-%m-%d %H:%M:%S"))
if not (info.has_key('latest_status') and status['status_id'] == info['latest_status']):
sync_renren_status.apply_async(args = [did, page+1, new_latest], countdown = 60)
else:
if new_latest is not None:
info['latest_status'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_renren_status.apply_async(args = [did, 1, None], countdown = 84600)
sync_renren_albums.apply_async(args = [d.id, 1], countdown = 1)
def sync_tqq(did, flag, time, lid, new_latest):
d = DataSource.objects.get(pk=did)
info = JSONDecoder().decode(d.auth_info)
consumer = oauth.Consumer(key=QQ_KEY, secret=QQ_SECRET)
token = oauth.Token(key=info['oauth_token'],
secret=info['oauth_token_secret'])
client = oauth.Client(consumer, token)
url = "http://open.t.qq.com/api/statuses/broadcast_timeline?format=json&pageflag=%s&reqnum=200&pagetime=%s&lastid=%s&type=1&contenttype=0&accesslevel=1" % (
flag, time, lid)
resp, content = client.request(url, "GET")
posts_info = JSONDecoder().decode(content)
if posts_info['data']:
if (flag == 0):
new_latest = posts_info['data']['info'][0]['id']
for post in posts_info['data']['info']:
if (info.has_key('latest') and post['id'] == info['latest']):
info['latest'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_tqq.apply_async(args=[did, 0, 0, 0, None],
countdown=84600)
break
new_paw = Paw.objects.create(source=d, \
type="tqq", \
content=post['text'], \
create_time=datetime.fromtimestamp(post['timestamp']))
if post['image']:
for img_url in post['image']:
new_img = Img(album=new_paw,
upload_time=datetime.fromtimestamp(
post['timestamp']))
origin_tmp = NamedTemporaryFile(delete=True)
origin_tmp.write(urllib2.urlopen(img_url + "/160").read())
origin_tmp.flush()
new_img.original.save(origin_tmp.name.split('/')[-1] +
'.jpg',
File(origin_tmp),
save=False)
thumb_tmp = NamedTemporaryFile(delete=True)
thumb_tmp.write(urllib2.urlopen(img_url + "/2000").read())
thumb_tmp.flush()
new_img.thumbnail.save(thumb_tmp.name.split('/')[-1] +
'.jpg',
File(thumb_tmp),
save=True)
new_img.save()
if not (info.has_key('latest') and post['id'] == info['latest']):
sync_tqq.apply_async(
args=[did, 1, post['timestamp'], post['id'], new_latest],
countdown=1)
else:
info['latest'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_tqq.apply_async(args=[did, 0, 0, 0, None], countdown=84600)
def sync_renren_blogs(did, page, new_latest):
d = DataSource.objects.get(pk=did)
info = JSONDecoder().decode(d.auth_info)
sig_calc = u"access_token=%sformat=jsonmethod=blog.getspage=%suid=%sv=1.0%s" % (
info['access_token'], page, info['user']['id'], RENREN_SECRET)
sig = calc_md5(sig_calc)
blogs_url = "http://api.renren.com/restserver.do?method=blog.gets&v=1.0&access_token=%s&format=json&page=%s&uid=%s&sig=%s" % (
info['access_token'], page, info['user']['id'], sig)
blogs_info = JSONDecoder().decode(urllib2.urlopen(blogs_url, "").read())
if blogs_info.has_key('blogs') and blogs_info['blogs']:
if (page == 1):
new_latest = blogs_info['blogs'][0]['id']
for blog in blogs_info['blogs']:
if (info.has_key('latest_blog')
and blog['id'] == info['latest_blog']):
info['latest_blog'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_renren_blogs.apply_async(args=[did, 1, None],
countdown=84600)
sync_renren_status.apply_async(args=[d.id, 1, None],
countdown=1)
break
sig_calc = u"access_token=%sformat=jsonid=%smethod=blog.getuid=%sv=1.0%s" % (
info['access_token'], blog['id'], info['user']['id'],
RENREN_SECRET)
sig = calc_md5(sig_calc)
blog_url = "http://api.renren.com/restserver.do?method=blog.get&v=1.0&access_token=%s&format=json&id=%s&uid=%s&sig=%s" % (
info['access_token'], blog['id'], info['user']['id'], sig)
blog_info = JSONDecoder().decode(
urllib2.urlopen(blog_url, "").read())
p = Paw.objects.create(source=d,\
type="renren blog",\
title=blog_info['title'],\
summary=blog['content'],\
content=blog_info['content'],\
create_time=datetime.strptime(blog_info['time'], "%Y-%m-%d %H:%M:%S"))
if not (info.has_key('latest_blog')
and blog['id'] == info['latest_blog']):
sync_renren_blogs.apply_async(args=[did, page + 1, new_latest],
countdown=60)
else:
if new_latest is not None:
info['latest_blog'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_renren_blogs.apply_async(args=[did, 1, None], countdown=84600)
sync_renren_status.apply_async(args=[d.id, 1, None], countdown=1)
def sync_douban_reviews(did, index, new_latest):
d = DataSource.objects.get(pk=did)
info = JSONDecoder().decode(d.auth_info)
consumer = oauth.Consumer(key=DOUBAN_KEY, secret=DOUBAN_SECRET)
token = oauth.Token(key=info['oauth_token'],
secret=info['oauth_token_secret'])
client = oauth.Client(consumer, token)
url = "http://api.douban.com/people/%s/reviews?start-index=%s&max-results=50&alt=json" % (
info['douban_user_id'], index)
resp, content = client.request(url, "GET")
reviews_info = JSONDecoder().decode(content)
if reviews_info['entry']:
if (index == 1):
new_latest = reviews_info['entry'][0]['id']['$t']
for review in reviews_info['entry']:
if (info.has_key('latest_review')
and review['id']['$t'] == info['latest_review']):
info['latest_review'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_douban_reviews.apply_async(args=[did, 1, None],
countdown=84600)
sync_douban_collections.apply_async(args=[did, 1, None],
countdown=1)
break
#get the specific review's content
review_url = review['id']['$t'] + "?alt=json"
resp, content = client.request(review_url, "GET")
review_content = JSONDecoder().decode(content)
new_paw = Paw.objects.create(source=d, \
type="douban review", \
title=review['title']['$t'], \
summary=review['summary']['$t'], \
content=review_content['content']['$t'], \
create_time=datetime.strptime(review['published']['$t'][:-6], "%Y-%m-%dT%H:%M:%S"))
if not (info.has_key('latest_review')
and review['id']['$t'] == info['latest_review']):
sync_douban_reviews.apply_async(args=[did, index + 50, new_latest],
countdown=60)
else:
if new_latest is not None:
info['latest_review'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_douban_reviews.apply_async(args=[did, 1, None], countdown=84600)
sync_douban_collections.apply_async(args=[did, 1, None], countdown=1)
def sync_weibo(did, page, new_latest):
d = DataSource.objects.get(pk=did)
info = JSONDecoder().decode(d.auth_info)
url = "https://api.weibo.com/2/statuses/user_timeline.json?access_token=%s&uid=%s&page=%s&count=200" % (
info['access_token'], info['uid'], page)
statuses_info = JSONDecoder().decode(urllib2.urlopen(url).read())
if statuses_info['statuses']:
if (page == 1):
new_latest = statuses_info['statuses'][0]['id']
for status in statuses_info['statuses']:
if (info.has_key('latest') and (status['id'] == info['latest'])):
info['latest'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_weibo.apply_async(args=[did, 1, None], countdown=84600)
break
new_paw = Paw.objects.create(
source=d,
type="weibo",
content=status['text'],
create_time=datetime.strptime(
status['created_at'][0:20] + status['created_at'][-4:],
"%a %b %d %H:%M:%S %Y"))
if status.has_key('original_pic'):
new_img = Img(album=new_paw, upload_time=new_paw.create_time)
origin_tmp = NamedTemporaryFile(delete=True)
origin_tmp.write(
urllib2.urlopen(status['original_pic']).read())
origin_tmp.flush()
new_img.original.save(origin_tmp.name.split('/')[-1] + '.jpg',
File(origin_tmp),
save=False)
thumb_tmp = NamedTemporaryFile(delete=True)
thumb_tmp.write(
urllib2.urlopen(status['thumbnail_pic']).read())
thumb_tmp.flush()
new_img.thumbnail.save(thumb_tmp.name.split('/')[-1] + '.jpg',
File(thumb_tmp),
save=True)
if not (info.has_key('latest') and (status['id'] == info['latest'])):
sync_weibo.apply_async(args=[did, page + 1, new_latest],
countdown=60)
else:
if new_latest is not None:
info['latest'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_weibo.apply_async(args=[did, 1, None], countdown=84600)
def sync_tqq(did, flag, time, lid, new_latest):
d = DataSource.objects.get(pk=did)
info = JSONDecoder().decode(d.auth_info)
consumer = oauth.Consumer(key=QQ_KEY, secret=QQ_SECRET)
token = oauth.Token(key=info['oauth_token'], secret=info['oauth_token_secret'])
client = oauth.Client(consumer, token)
url = "http://open.t.qq.com/api/statuses/broadcast_timeline?format=json&pageflag=%s&reqnum=200&pagetime=%s&lastid=%s&type=1&contenttype=0&accesslevel=1" % (flag, time, lid)
resp, content = client.request(url, "GET")
posts_info = JSONDecoder().decode(content)
if posts_info['data']:
if (flag == 0):
new_latest = posts_info['data']['info'][0]['id']
for post in posts_info['data']['info']:
if (info.has_key('latest') and post['id'] == info['latest']):
info['latest'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_tqq.apply_async(args=[did, 0, 0, 0, None], countdown = 84600)
break
new_paw = Paw.objects.create(source=d, \
type="tqq", \
content=post['text'], \
create_time=datetime.fromtimestamp(post['timestamp']))
if post['image']:
for img_url in post['image']:
new_img = Img(album = new_paw, upload_time=datetime.fromtimestamp(post['timestamp']))
origin_tmp = NamedTemporaryFile(delete=True)
origin_tmp.write(urllib2.urlopen(img_url+"/160").read())
origin_tmp.flush()
new_img.original.save(origin_tmp.name.split('/')[-1]+'.jpg', File(origin_tmp), save=False)
thumb_tmp = NamedTemporaryFile(delete=True)
thumb_tmp.write(urllib2.urlopen(img_url+"/2000").read())
thumb_tmp.flush()
new_img.thumbnail.save(thumb_tmp.name.split('/')[-1]+'.jpg', File(thumb_tmp), save=True)
new_img.save()
if not (info.has_key('latest') and post['id'] == info['latest']):
sync_tqq.apply_async(args=[did, 1, post['timestamp'], post['id'], new_latest], countdown = 1)
else:
info['latest'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_tqq.apply_async(args=[did, 0, 0, 0, None], countdown = 84600)
def sync_renren_blogs(did, page, new_latest):
d = DataSource.objects.get(pk=did)
info = JSONDecoder().decode(d.auth_info)
sig_calc = u"access_token=%sformat=jsonmethod=blog.getspage=%suid=%sv=1.0%s" % (info['access_token'], page, info['user']['id'], RENREN_SECRET)
sig = calc_md5(sig_calc)
blogs_url = "http://api.renren.com/restserver.do?method=blog.gets&v=1.0&access_token=%s&format=json&page=%s&uid=%s&sig=%s" % (info['access_token'], page, info['user']['id'], sig)
blogs_info = JSONDecoder().decode(urllib2.urlopen(blogs_url, "").read())
if blogs_info.has_key('blogs') and blogs_info['blogs']:
if (page == 1):
new_latest = blogs_info['blogs'][0]['id']
for blog in blogs_info['blogs']:
if (info.has_key('latest_blog') and blog['id'] == info['latest_blog']):
info['latest_blog'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_renren_blogs.apply_async(args = [did, 1, None], countdown = 84600)
sync_renren_status.apply_async(args = [d.id, 1, None], countdown = 1)
break
sig_calc = u"access_token=%sformat=jsonid=%smethod=blog.getuid=%sv=1.0%s" % (info['access_token'], blog['id'], info['user']['id'], RENREN_SECRET)
sig = calc_md5(sig_calc)
blog_url = "http://api.renren.com/restserver.do?method=blog.get&v=1.0&access_token=%s&format=json&id=%s&uid=%s&sig=%s" % (info['access_token'], blog['id'], info['user']['id'], sig)
blog_info = JSONDecoder().decode(urllib2.urlopen(blog_url, "").read())
p = Paw.objects.create(source=d,\
type="renren blog",\
title=blog_info['title'],\
summary=blog['content'],\
content=blog_info['content'],\
create_time=datetime.strptime(blog_info['time'], "%Y-%m-%d %H:%M:%S"))
if not (info.has_key('latest_blog') and blog['id'] == info['latest_blog']):
sync_renren_blogs.apply_async(args = [did, page+1, new_latest], countdown = 60)
else:
if new_latest is not None:
info['latest_blog'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_renren_blogs.apply_async(args = [did, 1, None], countdown = 84600)
sync_renren_status.apply_async(args = [d.id, 1, None], countdown = 1)
def sync_renren_status(did, page, new_latest):
d = DataSource.objects.get(pk=did)
info = JSONDecoder().decode(d.auth_info)
sig_calc = u"access_token=%scount=1000format=jsonmethod=status.getspage=%suid=%sv=1.0%s" % (
info['access_token'], page, info['user']['id'], RENREN_SECRET)
sig = calc_md5(sig_calc)
statues_url = "http://api.renren.com/restserver.do?method=status.gets&v=1.0&access_token=%s&count=1000&format=json&page=%s&uid=%s&sig=%s" % (
info['access_token'], page, info['user']['id'], sig)
statues = JSONDecoder().decode(urllib2.urlopen(statues_url, "").read())
if statues:
if (page == 1):
new_latest = statues[0]['status_id']
for status in statues:
if info.has_key('latest_status'
) and status['status_id'] == info['latest_status']:
info['latest_status'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_renren_status.apply_async(args=[did, 1, None],
countdown=84600)
sync_renren_albums.apply_async(args=[d.id, 1], countdown=1)
break
p = Paw.objects.create(source=d,\
type="renren status",\
content=status['message'],\
create_time=datetime.strptime(status['time'], "%Y-%m-%d %H:%M:%S"))
if not (info.has_key('latest_status')
and status['status_id'] == info['latest_status']):
sync_renren_status.apply_async(args=[did, page + 1, new_latest],
countdown=60)
else:
if new_latest is not None:
info['latest_status'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_renren_status.apply_async(args=[did, 1, None], countdown=84600)
sync_renren_albums.apply_async(args=[d.id, 1], countdown=1)
def sync_rss(did):
d = DataSource.objects.get(pk=did)
info = JSONDecoder().decode(d.auth_info)
f = feedparser.parse(info['url'])
for entry in f["entries"]:
# If the item didn't come with a GUID, then
# use link and then title as an identifier.
if not "id" in entry:
if "link" in entry:
entry["id"] = entry["link"]
else:
entry["id"] = entry["title"]
entry["create_time"]=datetime(entry.updated_parsed.tm_year, \
entry.updated_parsed.tm_mon, \
entry.updated_parsed.tm_mday, \
entry.updated_parsed.tm_hour, \
entry.updated_parsed.tm_min, \
entry.updated_parsed.tm_sec)
if not "content" in entry:
if "description" in entry:
entry["real_content"] = entry["description"]
else:
entry["real_content"] = entry["summary"]
else:
entry["real_content"] = entry["content"][0]["value"]
if (info.has_key('latest') and (entry["id"] == info['latest'])):
break
else:
p = Paw.objects.create(source=d, \
type="rss", \
title=entry["title"][:100], \
summary=entry["summary"], \
content=entry["real_content"], \
create_time=entry["create_time"])
if f["entries"]:
info['latest'] = f.entries[0]["id"]
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_rss.apply_async(args=[did], countdown=84600)
def sync_rss(did):
d = DataSource.objects.get(pk=did)
info = JSONDecoder().decode(d.auth_info)
f = feedparser.parse(info['url'])
for entry in f["entries"]:
# If the item didn't come with a GUID, then
# use link and then title as an identifier.
if not "id" in entry:
if "link" in entry:
entry["id"] = entry["link"]
else:
entry["id"] = entry["title"]
entry["create_time"]=datetime(entry.updated_parsed.tm_year, \
entry.updated_parsed.tm_mon, \
entry.updated_parsed.tm_mday, \
entry.updated_parsed.tm_hour, \
entry.updated_parsed.tm_min, \
entry.updated_parsed.tm_sec)
if not "content" in entry:
if "description" in entry:
entry["real_content"] = entry["description"]
else:
entry["real_content"] = entry["summary"]
else:
entry["real_content"] = entry["content"][0]["value"]
if (info.has_key('latest') and (entry["id"] == info['latest'])):
break
else:
p = Paw.objects.create(source=d, \
type="rss", \
title=entry["title"][:100], \
summary=entry["summary"], \
content=entry["real_content"], \
create_time=entry["create_time"])
if f["entries"]:
info['latest'] = f.entries[0]["id"]
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_rss.apply_async(args = [did], countdown = 84600)
def sync_taobao(did, page, new_latest):
d = DataSource.objects.get(pk=did)
# here refresh the sessionkey
info = JSONDecoder().decode(d.auth_info)
sign_str = "appkey" + TAOBAO_KEY + "refresh_token" + info[
'refresh_token'] + "sessionkey" + info['top_session'] + TAOBAO_SECRET
hash = md5.new()
hash.update(sign_str)
sign = hash.hexdigest().upper()
access_token_url = 'http://container.open.taobao.com/container/refresh?appkey=%s&refresh_token=%s&sessionkey=%s&sign=%s' \
% (TAOBAO_KEY, info['refresh_token'], info['top_session'], sign)
new_info = JSONDecoder().decode(
urllib2.urlopen(access_token_url, "").read())
info['top_session'] = new_info['top_session']
info['refresh_token'] = new_info['refresh_token']
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
# here begin the goods sync
info = JSONDecoder().decode(d.auth_info)
params = {
'method': 'taobao.trades.bought.get',
'session': info['top_session'],
'fields':
'tid,created,seller_nick,orders.oid,orders.pic_path,orders.total_fee,orders.title',
'partner_id': 'top-apitools',
'format': 'json',
'page_no': page,
'status': 'TRADE_FINISHED'
}
op = OpenTaobao(TAOBAO_KEY, TAOBAO_SECRET)
deal_info = JSONDecoder().decode(
op.get_result(params))['trades_bought_get_response']
if deal_info.has_key('trades') and deal_info['trades']:
if (page == 1):
print "now is page1"
new_latest = deal_info['trades']['trade'][0]['tid']
for trade in deal_info['trades']['trade']:
if (info.has_key('latest') and (trade['tid'] == info['latest'])):
info['latest'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_taobao.apply_async(
args=[did, 1, None],
countdown=84000) # wait for tommorow to do again this task
print "I am break and do it tomorrow"
break
for order in trade['orders']['order']:
print "I am in again"
new_paw = Paw.objects.create(
source=d,
type="taobao",
content=u"在" + trade['seller_nick'] + u"店</br>花费" +
order['total_fee'] + u"</br>购买了" + order["title"],
create_time=trade['created'])
if order.has_key('pic_path'):
new_img = Img(album=new_paw,
upload_time=new_paw.create_time)
origin_tmp = NamedTemporaryFile(delete=True)
origin_tmp.write(urllib2.urlopen(order['pic_path']).read())
origin_tmp.flush()
new_img.original.save(origin_tmp.name.split('/')[-1] +
'.jpg',
File(origin_tmp),
save=False)
thumb_tmp = NamedTemporaryFile(delete=True)
thumb_tmp.write(
urllib2.urlopen(order['pic_path'] + '_sum.jpg').read())
thumb_tmp.flush()
new_img.thumbnail.save(thumb_tmp.name.split('/')[-1] +
'.jpg',
File(thumb_tmp),
save=True)
if not (info.has_key('latest') and (trade['tid'] == info['latest'])):
print "do it 1min later"
sync_taobao.apply_async(args=[did, page + 1, new_latest],
countdown=60)
else:
if new_latest is not None:
info['latest'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
print "do it tommorrow"
sync_taobao.apply_async(args=[did, 1, None], countdown=84000)
def sync_taobao(did, page, new_latest):
d = DataSource.objects.get(pk=did)
# here refresh the sessionkey
info = JSONDecoder().decode(d.auth_info)
sign_str = "appkey"+TAOBAO_KEY+"refresh_token"+info['refresh_token']+"sessionkey"+info['top_session']+TAOBAO_SECRET
hash = md5.new()
hash.update(sign_str)
sign = hash.hexdigest().upper()
access_token_url = 'http://container.open.taobao.com/container/refresh?appkey=%s&refresh_token=%s&sessionkey=%s&sign=%s' \
% (TAOBAO_KEY, info['refresh_token'], info['top_session'], sign)
new_info = JSONDecoder().decode(urllib2.urlopen(access_token_url, "").read())
info['top_session'] = new_info['top_session']
info['refresh_token'] = new_info['refresh_token']
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
# here begin the goods sync
info = JSONDecoder().decode(d.auth_info)
params = {
'method':'taobao.trades.bought.get',
'session':info['top_session'],
'fields':'tid,created,seller_nick,orders.oid,orders.pic_path,orders.total_fee,orders.title',
'partner_id':'top-apitools',
'format':'json',
'page_no':page,
'status':'TRADE_FINISHED'
}
op = OpenTaobao(TAOBAO_KEY,TAOBAO_SECRET)
deal_info = JSONDecoder().decode(op.get_result(params))['trades_bought_get_response']
if deal_info.has_key('trades') and deal_info['trades']:
if (page == 1):
print "now is page1"
new_latest = deal_info['trades']['trade'][0]['tid']
for trade in deal_info['trades']['trade']:
if (info.has_key('latest') and (trade['tid'] == info['latest'])):
info['latest'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
sync_taobao.apply_async(args = [did, 1, None], countdown = 84000) # wait for tommorow to do again this task
print "I am break and do it tomorrow"
break
for order in trade['orders']['order']:
print "I am in again"
new_paw = Paw.objects.create(source=d, type="taobao", content=u"在"+trade['seller_nick']+u"店</br>花费"+order['total_fee']+u"</br>购买了"+order["title"], create_time=trade['created'])
if order.has_key('pic_path'):
new_img = Img(album = new_paw, upload_time=new_paw.create_time)
origin_tmp = NamedTemporaryFile(delete=True)
origin_tmp.write(urllib2.urlopen(order['pic_path']).read())
origin_tmp.flush()
new_img.original.save(origin_tmp.name.split('/')[-1]+'.jpg', File(origin_tmp), save=False)
thumb_tmp = NamedTemporaryFile(delete=True)
thumb_tmp.write(urllib2.urlopen(order['pic_path']+'_sum.jpg').read())
thumb_tmp.flush()
new_img.thumbnail.save(thumb_tmp.name.split('/')[-1]+'.jpg', File(thumb_tmp), save=True)
if not (info.has_key('latest') and (trade['tid'] == info['latest'])):
print "do it 1min later"
sync_taobao.apply_async(args = [did, page+1, new_latest], countdown = 60)
else:
if new_latest is not None:
info['latest'] = new_latest
d.auth_info = JSONEncoder().encode(info)
d.save(force_update=True)
print "do it tommorrow"
sync_taobao.apply_async(args = [did, 1, None], countdown = 84000)