def check_result(self, uri, resp, content): user_id = self.user_alias and self.user_alias.user_id or None excp = OAuthTokenExpiredError( user_id, config.OPENID_TYPE_DICT[config.OPENID_DOUBAN], content) jdata = json_decode(content) if content else None if str(resp.status) == "200": excp.clear_the_profile() return jdata log.warning("get %s fail, status code=%s, msg=%s. go to refresh token" \ % (uri, resp.status, content)) if jdata and isinstance(jdata, dict): error_code = jdata.get("code") if str(error_code) == "103" or str(error_code) == "123": excp.set_the_profile() raise excp elif str(error_code) == "106" and self.user_alias: try: new_tokens = super(Douban, self).refresh_tokens() if new_tokens and isinstance(new_tokens, dict): OAuth2Token.add(self.user_alias.id, new_tokens.get("access_token"), new_tokens.get("refresh_token")) excp.clear_the_profile() except OAuthError, e: log.warn("refresh token fail: %s" % e) excp.set_the_profile() raise e
def _save_user_and_token(token_dict, user_info, openid_type): ua = UserAlias.get(openid_type, user_info.get_user_id()) if not ua: if not g.user: ua = UserAlias.create_new_user(openid_type, user_info.get_user_id(), user_info.get_nickname()) else: ua = UserAlias.bind_to_exists_user(g.user, openid_type, user_info.get_user_id()) if not ua: return None ##设置个人资料(头像等等) u = User.get(ua.user_id) u.set_avatar_url(user_info.get_avatar()) u.set_icon_url(user_info.get_icon()) ##保存access token if openid_type == config.OPENID_TYPE_DICT[config.OPENID_TWITTER]: OAuth2Token.add(ua.id, token_dict.get("access_token"), token_dict.get("access_token_secret", "")) else: OAuth2Token.add(ua.id, token_dict.get("access_token"), token_dict.get("refresh_token", "")) ##set cookie,保持登录状态 if not g.user: g.user = User.get(ua.user_id) set_user_cookie(g.user, session) return g.user
def check_result(self, uri, resp, content): user_id = self.user_alias and self.user_alias.user_id or None excp = OAuthTokenExpiredError(user_id, config.OPENID_TYPE_DICT[config.OPENID_DOUBAN], content) jdata = json_decode(content) if content else None if str(resp.status) == "200": excp.clear_the_profile() return jdata log.warning("get %s fail, status code=%s, msg=%s. go to refresh token" \ % (uri, resp.status, content)) if jdata and isinstance(jdata, dict): error_code = jdata.get("code") if str(error_code) == "103" or str(error_code) == "123": excp.set_the_profile() raise excp elif str(error_code) == "106" and self.user_alias: try: new_tokens = super(Douban, self).refresh_tokens() if new_tokens and isinstance(new_tokens, dict): OAuth2Token.add(self.user_alias.id, new_tokens.get("access_token"), new_tokens.get("refresh_token")) excp.clear_the_profile() except OAuthError, e: log.warn("refresh token fail: %s" % e) excp.set_the_profile() raise e
def _save_user_and_token(token_dict, thirdparty_user, openid_type): first_connect = False ua = UserAlias.get(openid_type, thirdparty_user.get_user_id()) if not ua: if not g.user: ua = UserAlias.create_new_user(openid_type, thirdparty_user.get_user_id(), thirdparty_user.get_nickname()) else: ua = UserAlias.bind_to_exists_user(g.user, openid_type, thirdparty_user.get_user_id()) first_connect = True if not ua: return None ##设置个人资料(头像等等) u = User.get(ua.user_id) u.set_avatar_url(thirdparty_user.get_avatar()) u.set_icon_url(thirdparty_user.get_icon()) ##把各个第三方的uid保存到profile里面 k = openid_type v = { "uid": thirdparty_user.get_uid(), "name": thirdparty_user.get_nickname(), "intro": thirdparty_user.get_intro(), "signature": thirdparty_user.get_signature(), "avatar": thirdparty_user.get_avatar(), "icon": thirdparty_user.get_icon(), "email": thirdparty_user.get_email(), "first_connect": "Y" if first_connect else "N", } u.set_profile_item(k, json_encode(v)) ##保存access token if openid_type == config.OPENID_TYPE_DICT[config.OPENID_TWITTER]: OAuth2Token.add(ua.id, token_dict.get("access_token"), token_dict.get("access_token_secret", "")) else: OAuth2Token.add(ua.id, token_dict.get("access_token"), token_dict.get("refresh_token", "")) ##set cookie,保持登录状态 if not g.user: g.user = User.get(ua.user_id) set_user_cookie(g.user, session) return g.user
def _request(self, api, method="POST", extra_dict=None): if extra_dict is None: extra_dict = {} params = { "method": api, "v": "1.0", "access_token": self.access_token, "format": "json", } params.update(extra_dict) _, qs = Renren.sign(self.apikey_secret, **params) uri = "%s?%s" % (self.api_host, qs) log.info('getting %s...' % uri) resp, content = httplib2_request(uri, method) if resp.status == 200: user_id = self.user_alias and self.user_alias.user_id or None excp = OAuthTokenExpiredError( user_id=None, openid_type=config.OPENID_TYPE_DICT[config.OPENID_RENREN], msg=content) jdata = json_decode(content) if content else None if jdata and isinstance(jdata, dict): error_code = jdata.get("error_code") error_msg = jdata.get("error_msg") if error_code: if str(error_code) == "105": ## 无效的token excp.set_the_profile() raise excp elif str(error_code) == "106" and self.user_alias: ## FIXME: 过期的token, 是不是106? try: new_tokens = super(Renren, self).refresh_tokens() if new_tokens and isinstance(new_tokens, dict): OAuth2Token.add( self.user_alias.id, new_tokens.get("access_token"), new_tokens.get("refresh_token")) excp.clear_the_profile() except OAuthError, e: log.warn("refresh token fail: %s" % e) excp.set_the_profile() raise e return jdata
def _request(self, api, method="POST", extra_dict=None): if extra_dict is None: extra_dict = {} params = { "method": api, "v": "1.0", "access_token": self.access_token, "format": "json", } params.update(extra_dict) _, qs = Renren.sign(self.apikey_secret, **params) uri = "%s?%s" % (self.api_host, qs) log.info('getting %s...' % uri) resp, content = httplib2_request(uri, method) if resp.status == 200: user_id = self.user_alias and self.user_alias.user_id or None excp = OAuthTokenExpiredError(user_id=None, openid_type=config.OPENID_TYPE_DICT[config.OPENID_RENREN], msg=content) jdata = json_decode(content) if content else None if jdata and isinstance(jdata, dict): error_code = jdata.get("error_code") error_msg = jdata.get("error_msg") if error_code: if str(error_code) == "105": ## 无效的token excp.set_the_profile() raise excp elif str(error_code) == "106" and self.user_alias: ## FIXME: 过期的token, 是不是106? try: new_tokens = super(Renren, self).refresh_tokens() if new_tokens and isinstance(new_tokens, dict): OAuth2Token.add(self.user_alias.id, new_tokens.get("access_token"), new_tokens.get("refresh_token")) excp.clear_the_profile() except OAuthError, e: log.warn("refresh token fail: %s" % e) excp.set_the_profile() raise e return jdata
def get_client(cls, user_id): alias = UserAlias.get_by_user_and_type(user_id, config.OPENID_TYPE_DICT[config.OPENID_DOUBAN]) if not alias: return None token = OAuth2Token.get(alias.id) if not token: return None return cls(alias.alias, token.access_token, token.refresh_token)
def __init__(self, alias, apikey=None, apikey_secret=None, access_token=None, access_token_secret=None): ua = UserAlias.get(config.OPENID_TYPE_DICT[config.OPENID_TWITTER], alias) self.apikey = apikey if apikey is not None else config.APIKEY_DICT[config.OPENID_TWITTER].get("key") self.apikey_secret = apikey_secret if apikey_secret is not None else config.APIKEY_DICT[config.OPENID_TWITTER].get("secret") token = OAuth2Token.get(ua.id) self.access_token = access_token if access_token is not None else token.access_token self.access_token_secret = access_token_secret if access_token_secret is not None else token.refresh_token self.auth = tweepy.OAuthHandler(self.apikey, self.apikey_secret) self.auth.set_access_token(self.access_token, self.access_token_secret)
def get_client(cls, user_id): alias = UserAlias.get_by_user_and_type( user_id, config.OPENID_TYPE_DICT[config.OPENID_DOUBAN]) if not alias: return None token = OAuth2Token.get(alias.id) if not token: return None return cls(alias.alias, token.access_token, token.refresh_token)
def __init__(self, alias, apikey=None, apikey_secret=None, access_token=None, access_token_secret=None): ua = UserAlias.get(config.OPENID_TYPE_DICT[config.OPENID_QQ], alias) self.apikey = apikey if apikey is not None else config.APIKEY_DICT[config.OPENID_QQ].get("key") self.apikey_secret = apikey_secret if apikey_secret is not None else config.APIKEY_DICT[config.OPENID_QQ].get("secret") ##TODO:这里的OAuth2Token也变相的存储了OAuth1的token和secret,需要后续改一改 token = OAuth2Token.get(ua.id) self.access_token = access_token if access_token is not None else token.access_token self.access_token_secret = access_token_secret if access_token_secret is not None else token.refresh_token self.auth = QQOAuth1Login(self.apikey, self.apikey_secret, token=self.access_token, token_secret=self.access_token_secret)
def update_tokens(self, refresh_token): qs = {} qs["client_id"] = self.apikey qs["client_secret"] = self.apikey_secret qs["redirect_uri"] = self.redirect_uri qs["grant_type"] = "refresh_token" qs["refresh_token"] = refresh_token resp, content = httplib2_request(self.access_token_uri, "POST", body=urllib.urlencode(qs)) if resp.status != 200: raise OAuthLoginError('refres_tokens fail, status=%s:reason=%s:content=%s' \ %(resp.status, resp.reason, content)) r = json_decode(content) return OAuth2Token.add(r.get("douban_user_id"), r.get("access_token"), r.get("refresh_token"))
def __init__(self, alias, apikey=None, apikey_secret=None, access_token=None, access_token_secret=None): ua = UserAlias.get(config.OPENID_TYPE_DICT[config.OPENID_TWITTER], alias) self.apikey = apikey if apikey is not None else config.APIKEY_DICT[ config.OPENID_TWITTER].get("key") self.apikey_secret = apikey_secret if apikey_secret is not None else config.APIKEY_DICT[ config.OPENID_TWITTER].get("secret") token = OAuth2Token.get(ua.id) self.access_token = access_token if access_token is not None else token.access_token self.access_token_secret = access_token_secret if access_token_secret is not None else token.refresh_token self.auth = tweepy.OAuthHandler(self.apikey, self.apikey_secret) self.auth.set_access_token(self.access_token, self.access_token_secret)
def sync(cates): cates = cates.split("|") if not (cates and isinstance(cates, list)): return "no cates" cates = filter(lambda x: x in [str(y) for y in config.CATE_LIST], cates) if not cates: abort(400, "not support such cates") provider = category2provider(int(cates[0])) redir = "/connect/%s" % provider if not g.user: print '--- no g.user...' return redirect(redir) if request.form.get("remove"): for c in cates: r = SyncTask.gets_by_user_and_cate(g.user, str(c)) for x in r: x.remove() return json_encode({'ok': 'true'}) uas = UserAlias.gets_by_user_id(g.user.id) r = filter(lambda x: x.type == config.OPENID_TYPE_DICT[provider], uas) user_alias = r and r[0] if not user_alias: print '--- no user_alias...' return json_encode({'ok': 'false', 'redir': redir}) token = OAuth2Token.get(user_alias.id) if not token: print '--- no token...' return json_encode({'ok': 'false', 'redir': redir}) for c in cates: SyncTask.add(c, g.user.id) return json_encode({'ok': 'true'})
def sync(cates): cates = cates.split("|") if not (cates and isinstance(cates, list)): return "no cates" cates = filter(lambda x: x in [str(y) for y in config.CATE_LIST], cates) if not cates: abort(400, "not support such cates") provider = category2provider(int(cates[0])) redir = "/connect/%s" % provider if not g.user: print '--- no g.user...' return redirect(redir) if request.form.get("remove"): for c in cates: r = SyncTask.gets_by_user_and_cate(g.user, str(c)) for x in r: x.remove() return json_encode({'ok':'true'}) uas = UserAlias.gets_by_user_id(g.user.id) r = filter(lambda x: x.type == config.OPENID_TYPE_DICT[provider], uas) user_alias = r and r[0] if not user_alias: print '--- no user_alias...' return json_encode({'ok':'false', 'redir':redir}) token = OAuth2Token.get(user_alias.id) if not token: print '--- no token...' return json_encode({'ok':'false', 'redir':redir}) for c in cates: SyncTask.add(c, g.user.id) return json_encode({'ok':'true'})
def __init__(self, alias, apikey=None, apikey_secret=None, access_token=None, access_token_secret=None): ua = UserAlias.get(config.OPENID_TYPE_DICT[config.OPENID_QQ], alias) self.apikey = apikey if apikey is not None else config.APIKEY_DICT[ config.OPENID_QQ].get("key") self.apikey_secret = apikey_secret if apikey_secret is not None else config.APIKEY_DICT[ config.OPENID_QQ].get("secret") ##TODO:这里的OAuth2Token也变相的存储了OAuth1的token和secret,需要后续改一改 token = OAuth2Token.get(ua.id) self.access_token = access_token if access_token is not None else token.access_token self.access_token_secret = access_token_secret if access_token_secret is not None else token.refresh_token self.auth = QQOAuth1Login(self.apikey, self.apikey_secret, token=self.access_token, token_secret=self.access_token_secret)
def sync(t, old=False): if not t: print 'no such task' return 0 log.info("the sync task is :%s" % t) try: alias = None provider = category2provider(t.category) if provider == config.OPENID_DOUBAN: alias = UserAlias.get_by_user_and_type(t.user_id, config.OPENID_TYPE_DICT[config.OPENID_DOUBAN]) elif provider == config.OPENID_SINA: alias = UserAlias.get_by_user_and_type(t.user_id, config.OPENID_TYPE_DICT[config.OPENID_SINA]) elif provider == config.OPENID_TWITTER: alias = UserAlias.get_by_user_and_type(t.user_id, config.OPENID_TYPE_DICT[config.OPENID_TWITTER]) elif provider == config.OPENID_QQ: alias = UserAlias.get_by_user_and_type(t.user_id, config.OPENID_TYPE_DICT[config.OPENID_QQ]) if not alias: log.warn("no alias...") return 0 token = OAuth2Token.get(alias.id) if not token: log.warn("no access token, break...") return 0 client = None if provider == config.OPENID_DOUBAN: client = Douban(alias.alias, token.access_token, token.refresh_token) elif provider == config.OPENID_SINA: client = SinaWeibo(alias.alias, token.access_token) elif provider == config.OPENID_TWITTER: client = Twitter(alias.alias) elif provider == config.OPENID_QQ: client = QQWeibo(alias.alias) if not client: log.warn("get client fail, break...") return 0 if t.category == config.CATE_DOUBAN_NOTE: if old: start = Status.get_count_by_cate(t.category, t.user_id) else: start = 0 note_list = client.get_notes(start, 50) if note_list: for x in note_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(note_list) elif t.category == config.CATE_DOUBAN_MINIBLOG: if old: start = Status.get_count_by_cate(t.category, t.user_id) else: start = 0 miniblog_list = client.get_miniblogs(start, 50) if miniblog_list: for x in miniblog_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(miniblog_list) elif t.category == config.CATE_DOUBAN_STATUS: origin_min_id = Status.get_min_origin_id(t.category, t.user_id) if old: log.info("will get douban status order than %s..." % origin_min_id) status_list = client.get_timeline(until_id=origin_min_id) else: log.info("will get douban status newer than %s..." % origin_min_id) status_list = client.get_timeline(since_id=origin_min_id, count=20) if status_list: log.info("get douban status succ, len is %s" % len(status_list)) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) elif t.category == config.CATE_SINA_STATUS: origin_min_id = Status.get_min_origin_id(t.category, t.user_id) #means max_id if old: log.info("will get sinaweibo order than %s..." % origin_min_id) status_list = client.get_timeline(until_id=origin_min_id) ## 如果根据max_id拿不到数据,那么根据page再拿一次 if len(status_list) <= 1: downloaded_status_count = Status.get_count_by_user(t.user_id) page = downloaded_status_count / 100 + 2 status_list = client.get_timeline_by_page(page=page) else: log.info("will get sinaweibo newer than %s..." % origin_min_id) status_list = client.get_timeline(since_id=origin_min_id, count=20) if status_list: log.info("get sinaweibo succ, len is %s" % len(status_list)) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(status_list) elif t.category == config.CATE_TWITTER_STATUS: origin_min_id = Status.get_min_origin_id(t.category, t.user_id) if old: log.info("will get tweets order than %s..." % origin_min_id) status_list = client.get_timeline(max_id=origin_min_id) else: log.info("will get tweets newer than %s..." % origin_min_id) status_list = client.get_timeline(since_id=origin_min_id, count=20) if status_list: log.info("get tweets succ, len is %s" % len(status_list)) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(status_list) elif t.category == config.CATE_QQWEIBO_STATUS: if old: oldest_create_time = Status.get_oldest_create_time(t.category, t.user_id) log.info("will get qqweibo order than %s" % oldest_create_time) if oldest_create_time is not None: oldest_create_time = datetime2timestamp(oldest_create_time) status_list = client.get_old_timeline(oldest_create_time, reqnum=200) else: log.info("will get qqweibo new timeline") status_list = client.get_new_timeline(reqnum=20) if status_list: log.info("get qqweibo succ, result length is:%s" % len(status_list)) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(status_list) except: import traceback; print traceback.format_exc() return 0
def sync(t, old=False): if not t: print 'no such task' return 0 log.info("the sync task is :%s" % t) try: alias = None provider = category2provider(t.category) alias = UserAlias.get_by_user_and_type(t.user_id, config.OPENID_TYPE_DICT[provider]) if not alias: log.warn("no alias...") return 0 token = OAuth2Token.get(alias.id) if not token: log.warn("no access token, break...") return 0 client = None if provider == config.OPENID_DOUBAN: client = Douban.get_client(alias.user_id) elif provider == config.OPENID_SINA: client = SinaWeibo.get_client(alias.user_id) elif provider == config.OPENID_TWITTER: client = TwitterOAuth1.get_client(alias.user_id) elif provider == config.OPENID_QQ: client = QQWeibo.get_client(alias.user_id) elif provider == config.OPENID_RENREN: client = Renren.get_client(alias.user_id) elif provider == config.OPENID_INSTAGRAM: client = Instagram.get_client(alias.user_id) if not client: log.warn("get client fail, break...") return 0 if t.category == config.CATE_DOUBAN_NOTE: if old: start = Status.get_count_by_cate(t.category, t.user_id) else: start = 0 note_list = client.get_notes(start, 50) if note_list: for x in note_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(note_list) elif t.category == config.CATE_DOUBAN_MINIBLOG: if old: start = Status.get_count_by_cate(t.category, t.user_id) else: start = 0 miniblog_list = client.get_miniblogs(start, 50) if miniblog_list: for x in miniblog_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(miniblog_list) elif t.category == config.CATE_DOUBAN_STATUS: origin_min_id = Status.get_min_origin_id(t.category, t.user_id) if old: log.info("will get douban status order than %s..." % origin_min_id) status_list = client.get_timeline(until_id=origin_min_id) else: log.info("will get douban status newer than %s..." % origin_min_id) status_list = client.get_timeline(since_id=origin_min_id, count=20) if status_list: log.info("get douban status succ, len is %s" % len(status_list)) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) elif t.category == config.CATE_SINA_STATUS: origin_min_id = Status.get_min_origin_id(t.category, t.user_id) #means the earliest id origin_max_id = Status.get_max_origin_id(t.category, t.user_id) #meas the latest id if old: log.info("will get sinaweibo order than %s..." % origin_min_id) status_list = client.get_timeline(until_id=origin_min_id) ## 如果根据max_id拿不到数据,那么根据page再fetch一次或者until_id - 1 if status_list and len(status_list) < 20 and origin_min_id is not None: log.info("again will get sinaweibo order than %s..." % (int(origin_min_id)-1)) status_list = client.get_timeline(until_id=int(origin_min_id)-1) else: log.info("will get sinaweibo newer than %s..." % origin_max_id) status_list = client.get_timeline(since_id=origin_max_id, count=50) if status_list: log.info("get sinaweibo succ, len is %s" % len(status_list)) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(status_list) elif t.category == config.CATE_TWITTER_STATUS: origin_min_id = Status.get_min_origin_id(t.category, t.user_id) origin_max_id = Status.get_max_origin_id(t.category, t.user_id) if old: log.info("will get tweets order than %s..." % origin_min_id) status_list = client.get_timeline(max_id=origin_min_id) else: log.info("will get tweets newer than %s..." % origin_max_id) status_list = client.get_timeline(since_id=origin_max_id, count=50) if status_list: log.info("get tweets succ, len is %s" % len(status_list)) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(status_list) elif t.category == config.CATE_QQWEIBO_STATUS: if old: oldest_create_time = Status.get_oldest_create_time(t.category, t.user_id) log.info("will get qqweibo order than %s" % oldest_create_time) if oldest_create_time is not None: oldest_create_time = datetime2timestamp(oldest_create_time) status_list = client.get_old_timeline(oldest_create_time, reqnum=200) else: log.info("will get qqweibo new timeline") status_list = client.get_new_timeline(reqnum=20) if status_list: log.info("get qqweibo succ, result length is:%s" % len(status_list)) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(status_list) elif t.category == config.CATE_RENREN_STATUS: if old: count = 100 total_count = Status.get_count_by_cate(t.category, t.user_id) page = int(total_count / count) + 1 log.info("will get older renren status, page=%s, count=%s" %(page, count)) status_list = client.get_timeline(page, count) else: count = 20 page = 1 log.info("will get newest renren status, page=%s, count=%s" %(page, count)) status_list = client.get_timeline(page, count) if status_list: log.info("get renren status succ, result length is:%s" % len(status_list)) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(status_list) elif t.category == config.CATE_RENREN_BLOG: if old: count = 50 total_count = Status.get_count_by_cate(t.category, t.user_id) page = int(total_count / count) + 1 log.info("will get older renren blog, page=%s, count=%s" %(page, count)) blogs = client.get_blogs(page, count) else: count = 20 page = 1 log.info("will get newest renren blog, page=%s, count=%s" %(page, count)) blogs = client.get_blogs(page, count) if blogs: uid = blogs.get("uid") blog_ids = filter(None, [v.get("id") for v in blogs.get("blogs", [])]) log.info("get renren blog ids succ, result length is:%s" % len(blog_ids)) for blog_id in blog_ids: blog = client.get_blog(blog_id, uid) if blog: Status.add_from_obj(t.user_id, blog, json_encode(blog.get_data())) return len(blog_ids) elif t.category == config.CATE_RENREN_ALBUM: status_list = client.get_albums() if status_list: log.info("get renren album succ, result length is:%s" % len(status_list)) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(status_list) elif t.category == config.CATE_RENREN_PHOTO: albums_ids = Status.get_ids(user_id=t.user_id, limit=1000, cate=config.CATE_RENREN_ALBUM) albums = Status.gets(albums_ids) if not albums: return 0 for x in albums: d = x.get_data() if not d: continue aid = d.get_origin_id() size = int(d.get_size()) count = 50 for i in xrange(1, size/count + 2): status_list = client.get_photos(aid, i, count) if status_list: log.info("get renren photo of album %s succ, result length is:%s" \ % (aid, len(status_list))) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) elif t.category == config.CATE_INSTAGRAM_STATUS: origin_min_id = Status.get_min_origin_id(t.category, t.user_id) #means the earliest id origin_max_id = Status.get_max_origin_id(t.category, t.user_id) #means the latest id if old: log.info("will get instagram earlier than %s..." % origin_min_id) status_list = client.get_timeline(max_id=origin_min_id) else: log.info("will get instagram later than %s..." % origin_max_id) status_list = client.get_timeline(min_id=origin_max_id, count=50) if status_list: log.info("get instagram succ, len is %s" % len(status_list)) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(status_list) except Exception, e: print "---sync_exception_catched:", e
def sync(t, old=False): if not t: print 'no such task' return 0 log.info("the sync task is :%s" % t) try: alias = None provider = category2provider(t.category) if provider == config.OPENID_DOUBAN: alias = UserAlias.get_by_user_and_type( t.user_id, config.OPENID_TYPE_DICT[config.OPENID_DOUBAN]) elif provider == config.OPENID_SINA: alias = UserAlias.get_by_user_and_type( t.user_id, config.OPENID_TYPE_DICT[config.OPENID_SINA]) elif provider == config.OPENID_TWITTER: alias = UserAlias.get_by_user_and_type( t.user_id, config.OPENID_TYPE_DICT[config.OPENID_TWITTER]) elif provider == config.OPENID_QQ: alias = UserAlias.get_by_user_and_type( t.user_id, config.OPENID_TYPE_DICT[config.OPENID_QQ]) if not alias: log.warn("no alias...") return 0 token = OAuth2Token.get(alias.id) if not token: log.warn("no access token, break...") return 0 client = None if provider == config.OPENID_DOUBAN: client = Douban(alias.alias, token.access_token, token.refresh_token) elif provider == config.OPENID_SINA: client = SinaWeibo(alias.alias, token.access_token) elif provider == config.OPENID_TWITTER: client = Twitter(alias.alias) elif provider == config.OPENID_QQ: client = QQWeibo(alias.alias) if not client: log.warn("get client fail, break...") return 0 if t.category == config.CATE_DOUBAN_NOTE: if old: start = Status.get_count_by_cate(t.category, t.user_id) else: start = 0 note_list = client.get_notes(start, 50) if note_list: for x in note_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(note_list) elif t.category == config.CATE_DOUBAN_MINIBLOG: if old: start = Status.get_count_by_cate(t.category, t.user_id) else: start = 0 miniblog_list = client.get_miniblogs(start, 50) if miniblog_list: for x in miniblog_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(miniblog_list) elif t.category == config.CATE_DOUBAN_STATUS: origin_min_id = Status.get_min_origin_id(t.category, t.user_id) if old: log.info("will get douban status order than %s..." % origin_min_id) status_list = client.get_timeline(until_id=origin_min_id) else: log.info("will get douban status newer than %s..." % origin_min_id) status_list = client.get_timeline(since_id=origin_min_id, count=20) if status_list: log.info("get douban status succ, len is %s" % len(status_list)) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) elif t.category == config.CATE_SINA_STATUS: origin_min_id = Status.get_min_origin_id(t.category, t.user_id) #means max_id if old: log.info("will get sinaweibo order than %s..." % origin_min_id) status_list = client.get_timeline(until_id=origin_min_id) ## 如果根据max_id拿不到数据,那么根据page再fetch一次或者until_id - 1 if len(status_list) < 20: log.info("again will get sinaweibo order than %s..." % (int(origin_min_id) - 1)) status_list = client.get_timeline( until_id=int(origin_min_id) - 1) #if len(status_list) <= 1: # downloaded_status_count = Status.get_count_by_user(t.user_id) # page = downloaded_status_count / 100 + 2 # status_list = client.get_timeline_by_page(page=page) else: log.info("will get sinaweibo newer than %s..." % origin_min_id) status_list = client.get_timeline(since_id=origin_min_id, count=20) if status_list: log.info("get sinaweibo succ, len is %s" % len(status_list)) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(status_list) elif t.category == config.CATE_TWITTER_STATUS: origin_min_id = Status.get_min_origin_id(t.category, t.user_id) if old: log.info("will get tweets order than %s..." % origin_min_id) status_list = client.get_timeline(max_id=origin_min_id) else: log.info("will get tweets newer than %s..." % origin_min_id) status_list = client.get_timeline(since_id=origin_min_id, count=20) if status_list: log.info("get tweets succ, len is %s" % len(status_list)) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(status_list) elif t.category == config.CATE_QQWEIBO_STATUS: if old: oldest_create_time = Status.get_oldest_create_time( t.category, t.user_id) log.info("will get qqweibo order than %s" % oldest_create_time) if oldest_create_time is not None: oldest_create_time = datetime2timestamp(oldest_create_time) status_list = client.get_old_timeline(oldest_create_time, reqnum=200) else: log.info("will get qqweibo new timeline") status_list = client.get_new_timeline(reqnum=20) if status_list: log.info("get qqweibo succ, result length is:%s" % len(status_list)) for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(status_list) except: import traceback print traceback.format_exc() return 0
def sync(t, old=False): alias = None provider = category2provider(t.category) if provider == config.OPENID_DOUBAN: alias = UserAlias.get_by_user_and_type(t.user_id, config.OPENID_TYPE_DICT[config.OPENID_DOUBAN]) elif provider == config.OPENID_SINA: alias = UserAlias.get_by_user_and_type(t.user_id, config.OPENID_TYPE_DICT[config.OPENID_SINA]) elif provider == config.OPENID_TWITTER: alias = UserAlias.get_by_user_and_type(t.user_id, config.OPENID_TYPE_DICT[config.OPENID_TWITTER]) elif provider == config.OPENID_QQ: alias = UserAlias.get_by_user_and_type(t.user_id, config.OPENID_TYPE_DICT[config.OPENID_QQ]) if not alias: log.warn("no alias...") return 0 token = OAuth2Token.get(alias.id) if not token: log.warn("no access token, break...") return 0 client = None if provider == config.OPENID_DOUBAN: client = Douban(alias.alias, token.access_token, token.refresh_token) elif provider == config.OPENID_SINA: client = SinaWeibo(alias.alias, token.access_token) elif provider == config.OPENID_TWITTER: client = Twitter(alias.alias) elif provider == config.OPENID_QQ: client = QQWeibo(alias.alias) if not client: log.warn("get client fail, break...") return 0 if t.category == config.CATE_DOUBAN_NOTE: if old: start = Status.get_count_by_cate(t.category, t.user_id) else: start = 0 note_list = client.get_notes(start, 50) if note_list: for x in note_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(note_list) elif t.category == config.CATE_DOUBAN_MINIBLOG: if old: start = Status.get_count_by_cate(t.category, t.user_id) else: start = 0 miniblog_list = client.get_miniblogs(start, 50) if miniblog_list: for x in miniblog_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(miniblog_list) elif t.category == config.CATE_DOUBAN_STATUS or t.category == config.CATE_SINA_STATUS: if old: until_id = Status.get_min_origin_id(t.category, t.user_id) #means max_id status_list = client.get_timeline(until_id=until_id) else: since_id = Status.get_min_origin_id(t.category, t.user_id) status_list = client.get_timeline(since_id=since_id) if status_list: for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(status_list) elif t.category == config.CATE_TWITTER_STATUS: if old: until_id = Status.get_min_origin_id(t.category, t.user_id) #means max_id status_list = client.get_timeline(max_id=until_id) else: since_id = Status.get_min_origin_id(t.category, t.user_id) status_list = client.get_timeline(since_id=since_id) if status_list: for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(status_list) elif t.category == config.CATE_QQWEIBO_STATUS: if old: oldest_create_time = Status.get_oldest_create_time(t.category, t.user_id) oldest_create_time = datetime2timestamp(oldest_create_time) status_list = client.get_old_timeline(oldest_create_time, reqnum=200) else: status_list = client.get_new_timeline(reqnum=20) if status_list: for x in status_list: Status.add_from_obj(t.user_id, x, json_encode(x.get_data())) return len(status_list) return 0