def check_result(self, uri, resp, content):
user_id = self.user_alias and self.user_alias.user_id or None
excp = OAuthTokenExpiredError(
user_id, config.OPENID_TYPE_DICT[config.OPENID_DOUBAN], content)
jdata = json_decode(content) if content else None
if str(resp.status) == "200":
excp.clear_the_profile()
return jdata
log.warning("get %s fail, status code=%s, msg=%s. go to refresh token" \
% (uri, resp.status, content))
if jdata and isinstance(jdata, dict):
error_code = jdata.get("code")
if str(error_code) == "103" or str(error_code) == "123":
excp.set_the_profile()
raise excp
elif str(error_code) == "106" and self.user_alias:
try:
new_tokens = super(Douban, self).refresh_tokens()
if new_tokens and isinstance(new_tokens, dict):
OAuth2Token.add(self.user_alias.id,
new_tokens.get("access_token"),
new_tokens.get("refresh_token"))
excp.clear_the_profile()
except OAuthError, e:
log.warn("refresh token fail: %s" % e)
excp.set_the_profile()
raise e
def _save_user_and_token(token_dict, user_info, openid_type):
ua = UserAlias.get(openid_type, user_info.get_user_id())
if not ua:
if not g.user:
ua = UserAlias.create_new_user(openid_type,
user_info.get_user_id(), user_info.get_nickname())
else:
ua = UserAlias.bind_to_exists_user(g.user,
openid_type, user_info.get_user_id())
if not ua:
return None
##设置个人资料（头像等等）
u = User.get(ua.user_id)
u.set_avatar_url(user_info.get_avatar())
u.set_icon_url(user_info.get_icon())
##保存access token
if openid_type == config.OPENID_TYPE_DICT[config.OPENID_TWITTER]:
OAuth2Token.add(ua.id, token_dict.get("access_token"),
token_dict.get("access_token_secret", ""))
else:
OAuth2Token.add(ua.id, token_dict.get("access_token"),
token_dict.get("refresh_token", ""))
##set cookie，保持登录状态
if not g.user:
g.user = User.get(ua.user_id)
set_user_cookie(g.user, session)
return g.user
def check_result(self, uri, resp, content):
user_id = self.user_alias and self.user_alias.user_id or None
excp = OAuthTokenExpiredError(user_id,
config.OPENID_TYPE_DICT[config.OPENID_DOUBAN], content)
jdata = json_decode(content) if content else None
if str(resp.status) == "200":
excp.clear_the_profile()
return jdata
log.warning("get %s fail, status code=%s, msg=%s. go to refresh token" \
% (uri, resp.status, content))
if jdata and isinstance(jdata, dict):
error_code = jdata.get("code")
if str(error_code) == "103" or str(error_code) == "123":
excp.set_the_profile()
raise excp
elif str(error_code) == "106" and self.user_alias:
try:
new_tokens = super(Douban, self).refresh_tokens()
if new_tokens and isinstance(new_tokens, dict):
OAuth2Token.add(self.user_alias.id,
new_tokens.get("access_token"),
new_tokens.get("refresh_token"))
excp.clear_the_profile()
except OAuthError, e:
log.warn("refresh token fail: %s" % e)
excp.set_the_profile()
raise e
def _save_user_and_token(token_dict, thirdparty_user, openid_type):
first_connect = False
ua = UserAlias.get(openid_type, thirdparty_user.get_user_id())
if not ua:
if not g.user:
ua = UserAlias.create_new_user(openid_type,
thirdparty_user.get_user_id(),
thirdparty_user.get_nickname())
else:
ua = UserAlias.bind_to_exists_user(g.user, openid_type,
thirdparty_user.get_user_id())
first_connect = True
if not ua:
return None
##设置个人资料（头像等等）
u = User.get(ua.user_id)
u.set_avatar_url(thirdparty_user.get_avatar())
u.set_icon_url(thirdparty_user.get_icon())
##把各个第三方的uid保存到profile里面
k = openid_type
v = {
"uid": thirdparty_user.get_uid(),
"name": thirdparty_user.get_nickname(),
"intro": thirdparty_user.get_intro(),
"signature": thirdparty_user.get_signature(),
"avatar": thirdparty_user.get_avatar(),
"icon": thirdparty_user.get_icon(),
"email": thirdparty_user.get_email(),
"first_connect": "Y" if first_connect else "N",
}
u.set_profile_item(k, json_encode(v))
##保存access token
if openid_type == config.OPENID_TYPE_DICT[config.OPENID_TWITTER]:
OAuth2Token.add(ua.id, token_dict.get("access_token"),
token_dict.get("access_token_secret", ""))
else:
OAuth2Token.add(ua.id, token_dict.get("access_token"),
token_dict.get("refresh_token", ""))
##set cookie，保持登录状态
if not g.user:
g.user = User.get(ua.user_id)
set_user_cookie(g.user, session)
return g.user
def _request(self, api, method="POST", extra_dict=None):
if extra_dict is None:
extra_dict = {}
params = {
"method": api,
"v": "1.0",
"access_token": self.access_token,
"format": "json",
}
params.update(extra_dict)
_, qs = Renren.sign(self.apikey_secret, **params)
uri = "%s?%s" % (self.api_host, qs)
log.info('getting %s...' % uri)
resp, content = httplib2_request(uri, method)
if resp.status == 200:
user_id = self.user_alias and self.user_alias.user_id or None
excp = OAuthTokenExpiredError(
user_id=None,
openid_type=config.OPENID_TYPE_DICT[config.OPENID_RENREN],
msg=content)
jdata = json_decode(content) if content else None
if jdata and isinstance(jdata, dict):
error_code = jdata.get("error_code")
error_msg = jdata.get("error_msg")
if error_code:
if str(error_code) == "105":
## 无效的token
excp.set_the_profile()
raise excp
elif str(error_code) == "106" and self.user_alias:
## FIXME: 过期的token, 是不是106?
try:
new_tokens = super(Renren, self).refresh_tokens()
if new_tokens and isinstance(new_tokens, dict):
OAuth2Token.add(
self.user_alias.id,
new_tokens.get("access_token"),
new_tokens.get("refresh_token"))
excp.clear_the_profile()
except OAuthError, e:
log.warn("refresh token fail: %s" % e)
excp.set_the_profile()
raise e
return jdata
def _save_user_and_token(token_dict, thirdparty_user, openid_type):
first_connect = False
ua = UserAlias.get(openid_type, thirdparty_user.get_user_id())
if not ua:
if not g.user:
ua = UserAlias.create_new_user(openid_type,
thirdparty_user.get_user_id(), thirdparty_user.get_nickname())
else:
ua = UserAlias.bind_to_exists_user(g.user,
openid_type, thirdparty_user.get_user_id())
first_connect = True
if not ua:
return None
##设置个人资料（头像等等）
u = User.get(ua.user_id)
u.set_avatar_url(thirdparty_user.get_avatar())
u.set_icon_url(thirdparty_user.get_icon())
##把各个第三方的uid保存到profile里面
k = openid_type
v = {
"uid": thirdparty_user.get_uid(),
"name": thirdparty_user.get_nickname(),
"intro": thirdparty_user.get_intro(),
"signature": thirdparty_user.get_signature(),
"avatar": thirdparty_user.get_avatar(),
"icon": thirdparty_user.get_icon(),
"email": thirdparty_user.get_email(),
"first_connect": "Y" if first_connect else "N",
}
u.set_profile_item(k, json_encode(v))
##保存access token
if openid_type == config.OPENID_TYPE_DICT[config.OPENID_TWITTER]:
OAuth2Token.add(ua.id, token_dict.get("access_token"),
token_dict.get("access_token_secret", ""))
else:
OAuth2Token.add(ua.id, token_dict.get("access_token"),
token_dict.get("refresh_token", ""))
##set cookie，保持登录状态
if not g.user:
g.user = User.get(ua.user_id)
set_user_cookie(g.user, session)
return g.user
def _request(self, api, method="POST", extra_dict=None):
if extra_dict is None:
extra_dict = {}
params = {
"method": api,
"v": "1.0",
"access_token": self.access_token,
"format": "json",
}
params.update(extra_dict)
_, qs = Renren.sign(self.apikey_secret, **params)
uri = "%s?%s" % (self.api_host, qs)
log.info('getting %s...' % uri)
resp, content = httplib2_request(uri, method)
if resp.status == 200:
user_id = self.user_alias and self.user_alias.user_id or None
excp = OAuthTokenExpiredError(user_id=None,
openid_type=config.OPENID_TYPE_DICT[config.OPENID_RENREN],
msg=content)
jdata = json_decode(content) if content else None
if jdata and isinstance(jdata, dict):
error_code = jdata.get("error_code")
error_msg = jdata.get("error_msg")
if error_code:
if str(error_code) == "105":
## 无效的token
excp.set_the_profile()
raise excp
elif str(error_code) == "106" and self.user_alias:
## FIXME: 过期的token, 是不是106?
try:
new_tokens = super(Renren, self).refresh_tokens()
if new_tokens and isinstance(new_tokens, dict):
OAuth2Token.add(self.user_alias.id,
new_tokens.get("access_token"),
new_tokens.get("refresh_token"))
excp.clear_the_profile()
except OAuthError, e:
log.warn("refresh token fail: %s" % e)
excp.set_the_profile()
raise e
return jdata
def get_client(cls, user_id):
alias = UserAlias.get_by_user_and_type(user_id, config.OPENID_TYPE_DICT[config.OPENID_DOUBAN])
if not alias:
return None
token = OAuth2Token.get(alias.id)
if not token:
return None
return cls(alias.alias, token.access_token, token.refresh_token)
def __init__(self, alias, apikey=None, apikey_secret=None, access_token=None, access_token_secret=None):
ua = UserAlias.get(config.OPENID_TYPE_DICT[config.OPENID_TWITTER], alias)
self.apikey = apikey if apikey is not None else config.APIKEY_DICT[config.OPENID_TWITTER].get("key")
self.apikey_secret = apikey_secret if apikey_secret is not None else config.APIKEY_DICT[config.OPENID_TWITTER].get("secret")
token = OAuth2Token.get(ua.id)
self.access_token = access_token if access_token is not None else token.access_token
self.access_token_secret = access_token_secret if access_token_secret is not None else token.refresh_token
self.auth = tweepy.OAuthHandler(self.apikey, self.apikey_secret)
self.auth.set_access_token(self.access_token, self.access_token_secret)
def get_client(cls, user_id):
alias = UserAlias.get_by_user_and_type(
user_id, config.OPENID_TYPE_DICT[config.OPENID_DOUBAN])
if not alias:
return None
token = OAuth2Token.get(alias.id)
if not token:
return None
return cls(alias.alias, token.access_token, token.refresh_token)
def __init__(self, alias, apikey=None, apikey_secret=None, access_token=None, access_token_secret=None):
ua = UserAlias.get(config.OPENID_TYPE_DICT[config.OPENID_QQ], alias)
self.apikey = apikey if apikey is not None else config.APIKEY_DICT[config.OPENID_QQ].get("key")
self.apikey_secret = apikey_secret if apikey_secret is not None else config.APIKEY_DICT[config.OPENID_QQ].get("secret")
##TODO:这里的OAuth2Token也变相的存储了OAuth1的token和secret，需要后续改一改
token = OAuth2Token.get(ua.id)
self.access_token = access_token if access_token is not None else token.access_token
self.access_token_secret = access_token_secret if access_token_secret is not None else token.refresh_token
self.auth = QQOAuth1Login(self.apikey, self.apikey_secret,
token=self.access_token, token_secret=self.access_token_secret)
def update_tokens(self, refresh_token):
qs = {}
qs["client_id"] = self.apikey
qs["client_secret"] = self.apikey_secret
qs["redirect_uri"] = self.redirect_uri
qs["grant_type"] = "refresh_token"
qs["refresh_token"] = refresh_token
resp, content = httplib2_request(self.access_token_uri, "POST",
body=urllib.urlencode(qs))
if resp.status != 200:
raise OAuthLoginError('refres_tokens fail, status=%s:reason=%s:content=%s' \
%(resp.status, resp.reason, content))
r = json_decode(content)
return OAuth2Token.add(r.get("douban_user_id"), r.get("access_token"), r.get("refresh_token"))
def update_tokens(self, refresh_token):
qs = {}
qs["client_id"] = self.apikey
qs["client_secret"] = self.apikey_secret
qs["redirect_uri"] = self.redirect_uri
qs["grant_type"] = "refresh_token"
qs["refresh_token"] = refresh_token
resp, content = httplib2_request(self.access_token_uri,
"POST",
body=urllib.urlencode(qs))
if resp.status != 200:
raise OAuthLoginError('refres_tokens fail, status=%s:reason=%s:content=%s' \
%(resp.status, resp.reason, content))
r = json_decode(content)
return OAuth2Token.add(r.get("douban_user_id"), r.get("access_token"),
r.get("refresh_token"))
def __init__(self,
alias,
apikey=None,
apikey_secret=None,
access_token=None,
access_token_secret=None):
ua = UserAlias.get(config.OPENID_TYPE_DICT[config.OPENID_TWITTER],
alias)
self.apikey = apikey if apikey is not None else config.APIKEY_DICT[
config.OPENID_TWITTER].get("key")
self.apikey_secret = apikey_secret if apikey_secret is not None else config.APIKEY_DICT[
config.OPENID_TWITTER].get("secret")
token = OAuth2Token.get(ua.id)
self.access_token = access_token if access_token is not None else token.access_token
self.access_token_secret = access_token_secret if access_token_secret is not None else token.refresh_token
self.auth = tweepy.OAuthHandler(self.apikey, self.apikey_secret)
self.auth.set_access_token(self.access_token, self.access_token_secret)
def sync(cates):
cates = cates.split("|")
if not (cates and isinstance(cates, list)):
return "no cates"
cates = filter(lambda x: x in [str(y) for y in config.CATE_LIST], cates)
if not cates:
abort(400, "not support such cates")
provider = category2provider(int(cates[0]))
redir = "/connect/%s" % provider
if not g.user:
print '--- no g.user...'
return redirect(redir)
if request.form.get("remove"):
for c in cates:
r = SyncTask.gets_by_user_and_cate(g.user, str(c))
for x in r:
x.remove()
return json_encode({'ok': 'true'})
uas = UserAlias.gets_by_user_id(g.user.id)
r = filter(lambda x: x.type == config.OPENID_TYPE_DICT[provider], uas)
user_alias = r and r[0]
if not user_alias:
print '--- no user_alias...'
return json_encode({'ok': 'false', 'redir': redir})
token = OAuth2Token.get(user_alias.id)
if not token:
print '--- no token...'
return json_encode({'ok': 'false', 'redir': redir})
for c in cates:
SyncTask.add(c, g.user.id)
return json_encode({'ok': 'true'})
def sync(cates):
cates = cates.split("|")
if not (cates and isinstance(cates, list)):
return "no cates"
cates = filter(lambda x: x in [str(y) for y in config.CATE_LIST], cates)
if not cates:
abort(400, "not support such cates")
provider = category2provider(int(cates[0]))
redir = "/connect/%s" % provider
if not g.user:
print '--- no g.user...'
return redirect(redir)
if request.form.get("remove"):
for c in cates:
r = SyncTask.gets_by_user_and_cate(g.user, str(c))
for x in r:
x.remove()
return json_encode({'ok':'true'})
uas = UserAlias.gets_by_user_id(g.user.id)
r = filter(lambda x: x.type == config.OPENID_TYPE_DICT[provider], uas)
user_alias = r and r[0]
if not user_alias:
print '--- no user_alias...'
return json_encode({'ok':'false', 'redir':redir})
token = OAuth2Token.get(user_alias.id)
if not token:
print '--- no token...'
return json_encode({'ok':'false', 'redir':redir})
for c in cates:
SyncTask.add(c, g.user.id)
return json_encode({'ok':'true'})
def __init__(self,
alias,
apikey=None,
apikey_secret=None,
access_token=None,
access_token_secret=None):
ua = UserAlias.get(config.OPENID_TYPE_DICT[config.OPENID_QQ], alias)
self.apikey = apikey if apikey is not None else config.APIKEY_DICT[
config.OPENID_QQ].get("key")
self.apikey_secret = apikey_secret if apikey_secret is not None else config.APIKEY_DICT[
config.OPENID_QQ].get("secret")
##TODO:这里的OAuth2Token也变相的存储了OAuth1的token和secret，需要后续改一改
token = OAuth2Token.get(ua.id)
self.access_token = access_token if access_token is not None else token.access_token
self.access_token_secret = access_token_secret if access_token_secret is not None else token.refresh_token
self.auth = QQOAuth1Login(self.apikey,
self.apikey_secret,
token=self.access_token,
token_secret=self.access_token_secret)
def sync(t, old=False):
if not t:
print 'no such task'
return 0
log.info("the sync task is :%s" % t)
try:
alias = None
provider = category2provider(t.category)
if provider == config.OPENID_DOUBAN:
alias = UserAlias.get_by_user_and_type(t.user_id,
config.OPENID_TYPE_DICT[config.OPENID_DOUBAN])
elif provider == config.OPENID_SINA:
alias = UserAlias.get_by_user_and_type(t.user_id,
config.OPENID_TYPE_DICT[config.OPENID_SINA])
elif provider == config.OPENID_TWITTER:
alias = UserAlias.get_by_user_and_type(t.user_id,
config.OPENID_TYPE_DICT[config.OPENID_TWITTER])
elif provider == config.OPENID_QQ:
alias = UserAlias.get_by_user_and_type(t.user_id,
config.OPENID_TYPE_DICT[config.OPENID_QQ])
if not alias:
log.warn("no alias...")
return 0
token = OAuth2Token.get(alias.id)
if not token:
log.warn("no access token, break...")
return 0
client = None
if provider == config.OPENID_DOUBAN:
client = Douban(alias.alias, token.access_token, token.refresh_token)
elif provider == config.OPENID_SINA:
client = SinaWeibo(alias.alias, token.access_token)
elif provider == config.OPENID_TWITTER:
client = Twitter(alias.alias)
elif provider == config.OPENID_QQ:
client = QQWeibo(alias.alias)
if not client:
log.warn("get client fail, break...")
return 0
if t.category == config.CATE_DOUBAN_NOTE:
if old:
start = Status.get_count_by_cate(t.category, t.user_id)
else:
start = 0
note_list = client.get_notes(start, 50)
if note_list:
for x in note_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(note_list)
elif t.category == config.CATE_DOUBAN_MINIBLOG:
if old:
start = Status.get_count_by_cate(t.category, t.user_id)
else:
start = 0
miniblog_list = client.get_miniblogs(start, 50)
if miniblog_list:
for x in miniblog_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(miniblog_list)
elif t.category == config.CATE_DOUBAN_STATUS:
origin_min_id = Status.get_min_origin_id(t.category, t.user_id)
if old:
log.info("will get douban status order than %s..." % origin_min_id)
status_list = client.get_timeline(until_id=origin_min_id)
else:
log.info("will get douban status newer than %s..." % origin_min_id)
status_list = client.get_timeline(since_id=origin_min_id, count=20)
if status_list:
log.info("get douban status succ, len is %s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
elif t.category == config.CATE_SINA_STATUS:
origin_min_id = Status.get_min_origin_id(t.category, t.user_id) #means max_id
if old:
log.info("will get sinaweibo order than %s..." % origin_min_id)
status_list = client.get_timeline(until_id=origin_min_id)
## 如果根据max_id拿不到数据，那么根据page再拿一次
if len(status_list) <= 1:
downloaded_status_count = Status.get_count_by_user(t.user_id)
page = downloaded_status_count / 100 + 2
status_list = client.get_timeline_by_page(page=page)
else:
log.info("will get sinaweibo newer than %s..." % origin_min_id)
status_list = client.get_timeline(since_id=origin_min_id, count=20)
if status_list:
log.info("get sinaweibo succ, len is %s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_TWITTER_STATUS:
origin_min_id = Status.get_min_origin_id(t.category, t.user_id)
if old:
log.info("will get tweets order than %s..." % origin_min_id)
status_list = client.get_timeline(max_id=origin_min_id)
else:
log.info("will get tweets newer than %s..." % origin_min_id)
status_list = client.get_timeline(since_id=origin_min_id, count=20)
if status_list:
log.info("get tweets succ, len is %s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_QQWEIBO_STATUS:
if old:
oldest_create_time = Status.get_oldest_create_time(t.category, t.user_id)
log.info("will get qqweibo order than %s" % oldest_create_time)
if oldest_create_time is not None:
oldest_create_time = datetime2timestamp(oldest_create_time)
status_list = client.get_old_timeline(oldest_create_time, reqnum=200)
else:
log.info("will get qqweibo new timeline")
status_list = client.get_new_timeline(reqnum=20)
if status_list:
log.info("get qqweibo succ, result length is:%s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
except:
import traceback; print traceback.format_exc()
return 0
def sync(t, old=False):
if not t:
print 'no such task'
return 0
log.info("the sync task is :%s" % t)
try:
alias = None
provider = category2provider(t.category)
alias = UserAlias.get_by_user_and_type(t.user_id,
config.OPENID_TYPE_DICT[provider])
if not alias:
log.warn("no alias...")
return 0
token = OAuth2Token.get(alias.id)
if not token:
log.warn("no access token, break...")
return 0
client = None
if provider == config.OPENID_DOUBAN:
client = Douban.get_client(alias.user_id)
elif provider == config.OPENID_SINA:
client = SinaWeibo.get_client(alias.user_id)
elif provider == config.OPENID_TWITTER:
client = TwitterOAuth1.get_client(alias.user_id)
elif provider == config.OPENID_QQ:
client = QQWeibo.get_client(alias.user_id)
elif provider == config.OPENID_RENREN:
client = Renren.get_client(alias.user_id)
elif provider == config.OPENID_INSTAGRAM:
client = Instagram.get_client(alias.user_id)
if not client:
log.warn("get client fail, break...")
return 0
if t.category == config.CATE_DOUBAN_NOTE:
if old:
start = Status.get_count_by_cate(t.category, t.user_id)
else:
start = 0
note_list = client.get_notes(start, 50)
if note_list:
for x in note_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(note_list)
elif t.category == config.CATE_DOUBAN_MINIBLOG:
if old:
start = Status.get_count_by_cate(t.category, t.user_id)
else:
start = 0
miniblog_list = client.get_miniblogs(start, 50)
if miniblog_list:
for x in miniblog_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(miniblog_list)
elif t.category == config.CATE_DOUBAN_STATUS:
origin_min_id = Status.get_min_origin_id(t.category, t.user_id)
if old:
log.info("will get douban status order than %s..." % origin_min_id)
status_list = client.get_timeline(until_id=origin_min_id)
else:
log.info("will get douban status newer than %s..." % origin_min_id)
status_list = client.get_timeline(since_id=origin_min_id, count=20)
if status_list:
log.info("get douban status succ, len is %s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
elif t.category == config.CATE_SINA_STATUS:
origin_min_id = Status.get_min_origin_id(t.category, t.user_id) #means the earliest id
origin_max_id = Status.get_max_origin_id(t.category, t.user_id) #meas the latest id
if old:
log.info("will get sinaweibo order than %s..." % origin_min_id)
status_list = client.get_timeline(until_id=origin_min_id)
## 如果根据max_id拿不到数据，那么根据page再fetch一次或者until_id - 1
if status_list and len(status_list) < 20 and origin_min_id is not None:
log.info("again will get sinaweibo order than %s..." % (int(origin_min_id)-1))
status_list = client.get_timeline(until_id=int(origin_min_id)-1)
else:
log.info("will get sinaweibo newer than %s..." % origin_max_id)
status_list = client.get_timeline(since_id=origin_max_id, count=50)
if status_list:
log.info("get sinaweibo succ, len is %s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_TWITTER_STATUS:
origin_min_id = Status.get_min_origin_id(t.category, t.user_id)
origin_max_id = Status.get_max_origin_id(t.category, t.user_id)
if old:
log.info("will get tweets order than %s..." % origin_min_id)
status_list = client.get_timeline(max_id=origin_min_id)
else:
log.info("will get tweets newer than %s..." % origin_max_id)
status_list = client.get_timeline(since_id=origin_max_id, count=50)
if status_list:
log.info("get tweets succ, len is %s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_QQWEIBO_STATUS:
if old:
oldest_create_time = Status.get_oldest_create_time(t.category, t.user_id)
log.info("will get qqweibo order than %s" % oldest_create_time)
if oldest_create_time is not None:
oldest_create_time = datetime2timestamp(oldest_create_time)
status_list = client.get_old_timeline(oldest_create_time, reqnum=200)
else:
log.info("will get qqweibo new timeline")
status_list = client.get_new_timeline(reqnum=20)
if status_list:
log.info("get qqweibo succ, result length is:%s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_RENREN_STATUS:
if old:
count = 100
total_count = Status.get_count_by_cate(t.category, t.user_id)
page = int(total_count / count) + 1
log.info("will get older renren status, page=%s, count=%s" %(page, count))
status_list = client.get_timeline(page, count)
else:
count = 20
page = 1
log.info("will get newest renren status, page=%s, count=%s" %(page, count))
status_list = client.get_timeline(page, count)
if status_list:
log.info("get renren status succ, result length is:%s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_RENREN_BLOG:
if old:
count = 50
total_count = Status.get_count_by_cate(t.category, t.user_id)
page = int(total_count / count) + 1
log.info("will get older renren blog, page=%s, count=%s" %(page, count))
blogs = client.get_blogs(page, count)
else:
count = 20
page = 1
log.info("will get newest renren blog, page=%s, count=%s" %(page, count))
blogs = client.get_blogs(page, count)
if blogs:
uid = blogs.get("uid")
blog_ids = filter(None, [v.get("id") for v in blogs.get("blogs", [])])
log.info("get renren blog ids succ, result length is:%s" % len(blog_ids))
for blog_id in blog_ids:
blog = client.get_blog(blog_id, uid)
if blog:
Status.add_from_obj(t.user_id, blog, json_encode(blog.get_data()))
return len(blog_ids)
elif t.category == config.CATE_RENREN_ALBUM:
status_list = client.get_albums()
if status_list:
log.info("get renren album succ, result length is:%s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_RENREN_PHOTO:
albums_ids = Status.get_ids(user_id=t.user_id, limit=1000, cate=config.CATE_RENREN_ALBUM)
albums = Status.gets(albums_ids)
if not albums:
return 0
for x in albums:
d = x.get_data()
if not d:
continue
aid = d.get_origin_id()
size = int(d.get_size())
count = 50
for i in xrange(1, size/count + 2):
status_list = client.get_photos(aid, i, count)
if status_list:
log.info("get renren photo of album %s succ, result length is:%s" \
% (aid, len(status_list)))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
elif t.category == config.CATE_INSTAGRAM_STATUS:
origin_min_id = Status.get_min_origin_id(t.category, t.user_id) #means the earliest id
origin_max_id = Status.get_max_origin_id(t.category, t.user_id) #means the latest id
if old:
log.info("will get instagram earlier than %s..." % origin_min_id)
status_list = client.get_timeline(max_id=origin_min_id)
else:
log.info("will get instagram later than %s..." % origin_max_id)
status_list = client.get_timeline(min_id=origin_max_id, count=50)
if status_list:
log.info("get instagram succ, len is %s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
except Exception, e:
print "---sync_exception_catched:", e
def sync(t, old=False):
if not t:
print 'no such task'
return 0
log.info("the sync task is :%s" % t)
try:
alias = None
provider = category2provider(t.category)
if provider == config.OPENID_DOUBAN:
alias = UserAlias.get_by_user_and_type(
t.user_id, config.OPENID_TYPE_DICT[config.OPENID_DOUBAN])
elif provider == config.OPENID_SINA:
alias = UserAlias.get_by_user_and_type(
t.user_id, config.OPENID_TYPE_DICT[config.OPENID_SINA])
elif provider == config.OPENID_TWITTER:
alias = UserAlias.get_by_user_and_type(
t.user_id, config.OPENID_TYPE_DICT[config.OPENID_TWITTER])
elif provider == config.OPENID_QQ:
alias = UserAlias.get_by_user_and_type(
t.user_id, config.OPENID_TYPE_DICT[config.OPENID_QQ])
if not alias:
log.warn("no alias...")
return 0
token = OAuth2Token.get(alias.id)
if not token:
log.warn("no access token, break...")
return 0
client = None
if provider == config.OPENID_DOUBAN:
client = Douban(alias.alias, token.access_token,
token.refresh_token)
elif provider == config.OPENID_SINA:
client = SinaWeibo(alias.alias, token.access_token)
elif provider == config.OPENID_TWITTER:
client = Twitter(alias.alias)
elif provider == config.OPENID_QQ:
client = QQWeibo(alias.alias)
if not client:
log.warn("get client fail, break...")
return 0
if t.category == config.CATE_DOUBAN_NOTE:
if old:
start = Status.get_count_by_cate(t.category, t.user_id)
else:
start = 0
note_list = client.get_notes(start, 50)
if note_list:
for x in note_list:
Status.add_from_obj(t.user_id, x,
json_encode(x.get_data()))
return len(note_list)
elif t.category == config.CATE_DOUBAN_MINIBLOG:
if old:
start = Status.get_count_by_cate(t.category, t.user_id)
else:
start = 0
miniblog_list = client.get_miniblogs(start, 50)
if miniblog_list:
for x in miniblog_list:
Status.add_from_obj(t.user_id, x,
json_encode(x.get_data()))
return len(miniblog_list)
elif t.category == config.CATE_DOUBAN_STATUS:
origin_min_id = Status.get_min_origin_id(t.category, t.user_id)
if old:
log.info("will get douban status order than %s..." %
origin_min_id)
status_list = client.get_timeline(until_id=origin_min_id)
else:
log.info("will get douban status newer than %s..." %
origin_min_id)
status_list = client.get_timeline(since_id=origin_min_id,
count=20)
if status_list:
log.info("get douban status succ, len is %s" %
len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x,
json_encode(x.get_data()))
elif t.category == config.CATE_SINA_STATUS:
origin_min_id = Status.get_min_origin_id(t.category,
t.user_id) #means max_id
if old:
log.info("will get sinaweibo order than %s..." % origin_min_id)
status_list = client.get_timeline(until_id=origin_min_id)
## 如果根据max_id拿不到数据，那么根据page再fetch一次或者until_id - 1
if len(status_list) < 20:
log.info("again will get sinaweibo order than %s..." %
(int(origin_min_id) - 1))
status_list = client.get_timeline(
until_id=int(origin_min_id) - 1)
#if len(status_list) <= 1:
# downloaded_status_count = Status.get_count_by_user(t.user_id)
# page = downloaded_status_count / 100 + 2
# status_list = client.get_timeline_by_page(page=page)
else:
log.info("will get sinaweibo newer than %s..." % origin_min_id)
status_list = client.get_timeline(since_id=origin_min_id,
count=20)
if status_list:
log.info("get sinaweibo succ, len is %s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x,
json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_TWITTER_STATUS:
origin_min_id = Status.get_min_origin_id(t.category, t.user_id)
if old:
log.info("will get tweets order than %s..." % origin_min_id)
status_list = client.get_timeline(max_id=origin_min_id)
else:
log.info("will get tweets newer than %s..." % origin_min_id)
status_list = client.get_timeline(since_id=origin_min_id,
count=20)
if status_list:
log.info("get tweets succ, len is %s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x,
json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_QQWEIBO_STATUS:
if old:
oldest_create_time = Status.get_oldest_create_time(
t.category, t.user_id)
log.info("will get qqweibo order than %s" % oldest_create_time)
if oldest_create_time is not None:
oldest_create_time = datetime2timestamp(oldest_create_time)
status_list = client.get_old_timeline(oldest_create_time,
reqnum=200)
else:
log.info("will get qqweibo new timeline")
status_list = client.get_new_timeline(reqnum=20)
if status_list:
log.info("get qqweibo succ, result length is:%s" %
len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x,
json_encode(x.get_data()))
return len(status_list)
except:
import traceback
print traceback.format_exc()
return 0
def sync(t, old=False):
alias = None
provider = category2provider(t.category)
if provider == config.OPENID_DOUBAN:
alias = UserAlias.get_by_user_and_type(t.user_id,
config.OPENID_TYPE_DICT[config.OPENID_DOUBAN])
elif provider == config.OPENID_SINA:
alias = UserAlias.get_by_user_and_type(t.user_id,
config.OPENID_TYPE_DICT[config.OPENID_SINA])
elif provider == config.OPENID_TWITTER:
alias = UserAlias.get_by_user_and_type(t.user_id,
config.OPENID_TYPE_DICT[config.OPENID_TWITTER])
elif provider == config.OPENID_QQ:
alias = UserAlias.get_by_user_and_type(t.user_id,
config.OPENID_TYPE_DICT[config.OPENID_QQ])
if not alias:
log.warn("no alias...")
return 0
token = OAuth2Token.get(alias.id)
if not token:
log.warn("no access token, break...")
return 0
client = None
if provider == config.OPENID_DOUBAN:
client = Douban(alias.alias, token.access_token, token.refresh_token)
elif provider == config.OPENID_SINA:
client = SinaWeibo(alias.alias, token.access_token)
elif provider == config.OPENID_TWITTER:
client = Twitter(alias.alias)
elif provider == config.OPENID_QQ:
client = QQWeibo(alias.alias)
if not client:
log.warn("get client fail, break...")
return 0
if t.category == config.CATE_DOUBAN_NOTE:
if old:
start = Status.get_count_by_cate(t.category, t.user_id)
else:
start = 0
note_list = client.get_notes(start, 50)
if note_list:
for x in note_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(note_list)
elif t.category == config.CATE_DOUBAN_MINIBLOG:
if old:
start = Status.get_count_by_cate(t.category, t.user_id)
else:
start = 0
miniblog_list = client.get_miniblogs(start, 50)
if miniblog_list:
for x in miniblog_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(miniblog_list)
elif t.category == config.CATE_DOUBAN_STATUS or t.category == config.CATE_SINA_STATUS:
if old:
until_id = Status.get_min_origin_id(t.category, t.user_id) #means max_id
status_list = client.get_timeline(until_id=until_id)
else:
since_id = Status.get_min_origin_id(t.category, t.user_id)
status_list = client.get_timeline(since_id=since_id)
if status_list:
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_TWITTER_STATUS:
if old:
until_id = Status.get_min_origin_id(t.category, t.user_id) #means max_id
status_list = client.get_timeline(max_id=until_id)
else:
since_id = Status.get_min_origin_id(t.category, t.user_id)
status_list = client.get_timeline(since_id=since_id)
if status_list:
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_QQWEIBO_STATUS:
if old:
oldest_create_time = Status.get_oldest_create_time(t.category, t.user_id)
oldest_create_time = datetime2timestamp(oldest_create_time)
status_list = client.get_old_timeline(oldest_create_time, reqnum=200)
else:
status_list = client.get_new_timeline(reqnum=20)
if status_list:
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
return 0
