def sync_helper(cate,old=False):
log.info("%s syncing old %s... cate=%s" % (datetime.datetime.now(), old, cate))
ids = SyncTask.get_ids()
task_list = SyncTask.gets(ids)
if cate:
task_list = [x for x in task_list if x.category==cate]
if not task_list:
log.warn("no task list, so sleep 10s and continue...")
return
log.info("task_list length is %s" % len(task_list))
for t in task_list:
sync(t, old)
def _add_sync_task_and_push_queue(provider, user):
task_ids = [x.category for x in SyncTask.gets_by_user(user)]
if provider == config.OPENID_DOUBAN:
if str(config.CATE_DOUBAN_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_DOUBAN_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
elif provider == config.OPENID_SINA:
if str(config.CATE_SINA_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_SINA_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
elif provider == config.OPENID_TWITTER:
if str(config.CATE_TWITTER_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_TWITTER_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
elif provider == config.OPENID_QQ:
if str(config.CATE_QQWEIBO_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_QQWEIBO_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
elif provider == config.OPENID_RENREN:
for cate in (config.CATE_RENREN_STATUS, config.CATE_RENREN_BLOG,
config.CATE_RENREN_ALBUM, config.CATE_RENREN_PHOTO):
if str(cate) not in task_ids:
t = SyncTask.add(cate, user.id)
t and TaskQueue.add(t.id, t.kind)
elif provider == config.OPENID_INSTAGRAM:
if str(config.CATE_INSTAGRAM_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_INSTAGRAM_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
def _add_sync_task_and_push_queue(provider, user):
task_ids = [x.category for x in SyncTask.gets_by_user(user)]
if provider == config.OPENID_DOUBAN:
if str(config.CATE_DOUBAN_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_DOUBAN_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
elif provider == config.OPENID_SINA:
if str(config.CATE_SINA_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_SINA_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
elif provider == config.OPENID_TWITTER:
if str(config.CATE_TWITTER_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_TWITTER_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
elif provider == config.OPENID_QQ:
if str(config.CATE_QQWEIBO_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_QQWEIBO_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
elif provider == config.OPENID_RENREN:
for cate in (config.CATE_RENREN_STATUS, config.CATE_RENREN_BLOG,
config.CATE_RENREN_ALBUM, config.CATE_RENREN_PHOTO):
if str(cate) not in task_ids:
t = SyncTask.add(cate, user.id)
t and TaskQueue.add(t.id, t.kind)
elif provider == config.OPENID_INSTAGRAM:
if str(config.CATE_INSTAGRAM_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_INSTAGRAM_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
def _add_sync_task_and_push_queue(provider, user):
task_ids = [x.category for x in SyncTask.gets_by_user(user)]
if provider == config.OPENID_DOUBAN:
if str(config.CATE_DOUBAN_MINIBLOG) not in task_ids:
t = SyncTask.add(config.CATE_DOUBAN_MINIBLOG, user.id)
t and TaskQueue.add(t.id, t.kind)
if str(config.CATE_DOUBAN_NOTE) not in task_ids:
t = SyncTask.add(config.CATE_DOUBAN_NOTE, user.id)
t and TaskQueue.add(t.id, t.kind)
elif provider == config.OPENID_SINA:
if str(config.CATE_SINA_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_SINA_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
elif provider == config.OPENID_TWITTER:
if str(config.CATE_TWITTER_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_TWITTER_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
def sync_helper(cate,old=False):
log.info("%s syncing old %s... cate=%s" % (datetime.datetime.now(), old, cate))
ids = SyncTask.get_ids()
task_list = SyncTask.gets(ids)
if cate:
task_list = [x for x in task_list if x.category==cate]
if not task_list:
log.warn("no task list, so sleep 10s and continue...")
return
log.info("task_list length is %s" % len(task_list))
for t in task_list:
try:
if t.category == config.CATE_WORDPRESS_POST:
sync_wordpress(t)
else:
sync(t, old)
except Exception, e:
import traceback
print "%s %s" % (datetime.datetime.now(), traceback.format_exc())
def sync_helper(cate,old=False):
log.info("%s syncing old %s... cate=%s" % (datetime.datetime.now(), old, cate))
ids = SyncTask.get_ids()
task_list = SyncTask.gets(ids)
if cate:
task_list = [x for x in task_list if x.category==cate]
if not task_list:
log.warn("no task list, so sleep 10s and continue...")
return
log.info("task_list length is %s" % len(task_list))
for t in task_list:
try:
if t.category == config.CATE_WORDPRESS_POST:
sync_wordpress(t)
else:
sync(t, old)
except Exception, e:
import traceback
print "%s %s" % (datetime.datetime.now(), traceback.format_exc())
def sync(cates):
cates = cates.split("|")
if not (cates and isinstance(cates, list)):
return "no cates"
cates = filter(lambda x: x in [str(y) for y in config.CATE_LIST], cates)
if not cates:
abort(400, "not support such cates")
provider = category2provider(int(cates[0]))
redir = "/connect/%s" % provider
if not g.user:
print '--- no g.user...'
return redirect(redir)
if request.form.get("remove"):
for c in cates:
r = SyncTask.gets_by_user_and_cate(g.user, str(c))
for x in r:
x.remove()
return json_encode({'ok': 'true'})
uas = UserAlias.gets_by_user_id(g.user.id)
r = filter(lambda x: x.type == config.OPENID_TYPE_DICT[provider], uas)
user_alias = r and r[0]
if not user_alias:
print '--- no user_alias...'
return json_encode({'ok': 'false', 'redir': redir})
token = OAuth2Token.get(user_alias.id)
if not token:
print '--- no token...'
return json_encode({'ok': 'false', 'redir': redir})
for c in cates:
SyncTask.add(c, g.user.id)
return json_encode({'ok': 'true'})
def sync(cates):
cates = cates.split("|")
if not (cates and isinstance(cates, list)):
return "no cates"
cates = filter(lambda x: x in [str(y) for y in config.CATE_LIST], cates)
if not cates:
abort(400, "not support such cates")
provider = category2provider(int(cates[0]))
redir = "/connect/%s" % provider
if not g.user:
print '--- no g.user...'
return redirect(redir)
if request.form.get("remove"):
for c in cates:
r = SyncTask.gets_by_user_and_cate(g.user, str(c))
for x in r:
x.remove()
return json_encode({'ok':'true'})
uas = UserAlias.gets_by_user_id(g.user.id)
r = filter(lambda x: x.type == config.OPENID_TYPE_DICT[provider], uas)
user_alias = r and r[0]
if not user_alias:
print '--- no user_alias...'
return json_encode({'ok':'false', 'redir':redir})
token = OAuth2Token.get(user_alias.id)
if not token:
print '--- no token...'
return json_encode({'ok':'false', 'redir':redir})
for c in cates:
SyncTask.add(c, g.user.id)
return json_encode({'ok':'true'})
def _add_sync_task_and_push_queue(provider, user):
task_ids = [x.category for x in SyncTask.gets_by_user(user)]
if provider == config.OPENID_DOUBAN:
if str(config.CATE_DOUBAN_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_DOUBAN_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
elif provider == config.OPENID_SINA:
if str(config.CATE_SINA_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_SINA_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
elif provider == config.OPENID_TWITTER:
if str(config.CATE_TWITTER_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_TWITTER_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
elif provider == config.OPENID_QQ:
if str(config.CATE_QQWEIBO_STATUS) not in task_ids:
t = SyncTask.add(config.CATE_QQWEIBO_STATUS, user.id)
t and TaskQueue.add(t.id, t.kind)
def profile():
u = g.user
sync_tasks = SyncTask.gets_by_user(u)
my_sync_cates = [x.category for x in sync_tasks]
site_homepage_list = []
for ua in g.user_alias:
if ua.type == config.OPENID_TYPE_DICT[config.OPENID_DOUBAN]:
site_homepage_list.append({'site':u'豆瓣', 'homepage':'http://www.douban.com/people/%s' %ua.alias})
elif ua.type == config.OPENID_TYPE_DICT[config.OPENID_SINA]:
site_homepage_list.append({'site':u'新浪微博', 'homepage':'http://www.weibo.com/%s' %ua.alias})
elif ua.type == config.OPENID_TYPE_DICT[config.OPENID_TWITTER]:
site_homepage_list.append({'site':u'twitter', 'homepage':'http://www.twitter.com/%s' %ua.alias})
return render_template("profile.html", user=u,
my_sync_cates = my_sync_cates, site_homepage_list=site_homepage_list, config=config)
def bind(uid, feed_uri):
user = User.get(uid)
if not user:
print 'no user'
return
ua = UserAlias.bind_to_exists_user(user,
config.OPENID_TYPE_DICT[config.OPENID_WORDPRESS], feed_uri)
if not ua:
print "no user alias"
else:
##添加同步任务
t = SyncTask.add(config.CATE_WORDPRESS_POST, user.id)
t and TaskQueue.add(t.id, t.kind)
##删除confiration记录
mc.delete("wordpress_bind:%s" %user.id)
def bind(uid, feed_uri):
user = User.get(uid)
if not user:
print 'no user'
return
ua = UserAlias.bind_to_exists_user(
user, config.OPENID_TYPE_DICT[config.OPENID_WORDPRESS], feed_uri)
if not ua:
print "no user alias"
else:
##添加同步任务
t = SyncTask.add(config.CATE_WORDPRESS_POST, user.id)
t and TaskQueue.add(t.id, t.kind)
##删除confiration记录
mc.delete("wordpress_bind:%s" % user.id)
import past
import jobs
from past.model.status import TaskQueue, SyncTask, Status
from past import config
if __name__ == "__main__":
while True:
try:
queue_ids = TaskQueue.get_all_ids()
print 'queue length:', len(queue_ids)
for qid in queue_ids:
queue = TaskQueue.get(qid)
if queue and queue.task_kind == config.K_SYNCTASK:
print 'syncing task id:', queue.task_id
sync_task = SyncTask.get(queue.task_id)
max_sync_times = 0
min_id = Status.get_min_origin_id(sync_task.category, sync_task.user_id)
if sync_task:
while True:
if max_sync_times >= 3:
break
r = jobs.sync(sync_task, old=True)
new_min_id = Status.get_min_origin_id(sync_task.category, sync_task.user_id)
if r == 0 or new_min_id == min_id:
break
min_id = new_min_id
max_sync_times += 1
queue.remove()
time.sleep(5)
time.sleep(30)
import past
import jobs
from past.model.status import TaskQueue, SyncTask, Status
from past import config
if __name__ == "__main__":
try:
queue_ids = TaskQueue.get_all_ids()
print '%s queue length: %s' % (datetime.datetime.now(), len(queue_ids))
for qid in queue_ids:
queue = TaskQueue.get(qid)
if queue and queue.task_kind == config.K_SYNCTASK:
print 'syncing task id:', queue.task_id
sync_task = SyncTask.get(queue.task_id)
if not sync_task:
continue
## 现在不同步豆瓣日记
if str(sync_task.category) == str(config.CATE_DOUBAN_NOTE):
continue
## 同步wordpress rss
if str(sync_task.category) == str(config.CATE_WORDPRESS_POST):
jobs.sync_wordpress(sync_task)
queue.remove()
continue
max_sync_times = 0
min_id = Status.get_min_origin_id(sync_task.category,
#-*- coding:utf-8 -*-
import sys
sys.path.append('../')
import past
from past import config
from past.model.user import UserAlias
from past.model.status import SyncTask
all_alias_ids = UserAlias.get_ids()
for id_ in all_alias_ids:
print id_
ua = UserAlias.get_by_id(id_)
if not ua:
continue
print ua
if ua.type == 'D':
SyncTask.add(config.CATE_DOUBAN_STATUS, ua.user_id)
#SyncTask.add(config.CATE_DOUBAN_MINIBLOG, ua.user_id)
#if ua.type == 'S':
# SyncTask.add(config.CATE_SINA_STATUS, ua.user_id)
#if ua.type == 'T':
# SyncTask.add(config.CATE_TWITTER_STATUS, ua.user_id)
def bind_wordpress():
if not g.user:
flash(u"请先使用豆瓣、微博、QQ、Twitter任意一个帐号登录后，再来做绑定blog的操作^^", "tip")
return redirect("/home")
user = g.user
intros = [
g.user.get_thirdparty_profile(x).get("intro")
for x in config.OPENID_TYPE_DICT.values()
]
intros = filter(None, intros)
uas = g.user.get_alias()
wordpress_alias_list = [
x for x in uas
if x.type == config.OPENID_TYPE_DICT[config.OPENID_WORDPRESS]
]
step = "1"
random_id = mc.get("wordpress_bind:%s" % g.user.id)
c = random_id and Confirmation.get_by_random_id(random_id)
if c:
_, feed_uri = c.text.split(":", 1)
step = "2"
else:
feed_uri = ""
if request.method == "GET":
return render_template("v2/bind_wordpress.html",
consts=consts,
**locals())
elif request.method == "POST":
ret = {}
ret['ok'] = False
if step == '1':
feed_uri = request.form.get("feed_uri")
if not feed_uri:
ret['msg'] = 'feed地址不能为空'
elif not (feed_uri.startswith("http://")
or feed_uri.startswith("https://")):
ret['msg'] = 'feed地址貌似不对'
else:
ua = UserAlias.get(
config.OPENID_TYPE_DICT[config.OPENID_WORDPRESS], feed_uri)
if ua:
ret['msg'] = '该feed地址已被绑定'
else:
##设置一个激活码
code = randbytes(16)
val = "%s:%s" % (g.user.id, feed_uri)
r = Confirmation.add(code, val)
if r:
ret['ok'] = True
ret['msg'] = '为了验证blog的主人^^，请发一篇blog，内容为 %s，完成该步骤后，请点下一步完成绑定' \
% code
mc.set("wordpress_bind:%s" % g.user.id, code)
else:
ret['msg'] = '抱歉，出错了，请重试, 或者给管理员捎个话:[email protected]'
return json_encode(ret)
elif step == '2':
if not (random_id and c):
ret['msg'] = '出错了，激活码不对^^'
else:
text = c.text
user_id, feed_uri = text.split(":", 1)
## 同步一下，看看验证码的文章是否正确
client = Wordpress(feed_uri)
rs = client.get_feeds(refresh=True)
if not rs:
ret['msg'] = '没有发现含有验证码的文章，请检查后再提交验证'
else:
latest_post = rs[0]
if not latest_post:
ret['msg'] = "你的feed地址可能无法访问，请检查下"
else:
content = latest_post.get_content(
) or latest_post.get_summary()
if content and content.encode("utf8")[:100].find(
str(random_id)) != -1:
ua = UserAlias.bind_to_exists_user(
g.user, config.OPENID_TYPE_DICT[
config.OPENID_WORDPRESS], feed_uri)
if not ua:
ret['msg'] = '出错了，麻烦你重试一下吧^^'
else:
##添加同步任务
t = SyncTask.add(config.CATE_WORDPRESS_POST,
g.user.id)
t and TaskQueue.add(t.id, t.kind)
##删除confiration记录
c.delete()
mc.delete("wordpress_bind:%s" % g.user.id)
ret['ok'] = True
ret['msg'] = '恭喜，绑定成功啦'
else:
ret['msg'] = "没有发现含有验证码的文章，请检查后再提交验证"
return json_encode(ret)
else:
return "method not allowed"
#-*- coding:utf-8 -*-
import sys
sys.path.append('../')
import past
from past import config
from past.model.user import UserAlias
from past.model.status import SyncTask
all_alias_ids = UserAlias.get_ids()
for id_ in all_alias_ids:
print id_
ua = UserAlias.get_by_id(id_)
if not ua:
continue
print ua
if ua.type == 'D':
SyncTask.add(config.CATE_DOUBAN_NOTE, ua.user_id)
SyncTask.add(config.CATE_DOUBAN_MINIBLOG, ua.user_id)
if ua.type == 'S':
SyncTask.add(config.CATE_SINA_STATUS, ua.user_id)
if ua.type == 'T':
SyncTask.add(config.CATE_TWITTER_STATUS, ua.user_id)
def profile():
u = g.user
sync_tasks = SyncTask.gets_by_user(u)
my_sync_cates = [x.category for x in sync_tasks]
return render_template("profile.html", user=u, my_sync_cates=my_sync_cates, config=config)
def bind_wordpress():
if not g.user:
flash(u"请先使用豆瓣、微博、QQ、Twitter任意一个帐号登录后，再来做绑定blog的操作^^", "tip")
return redirect("/home")
intros = [g.user.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
uas = g.user.get_alias()
wordpress_alias_list = [x for x in uas if x.type == config.OPENID_TYPE_DICT[config.OPENID_WORDPRESS]]
step = "1"
random_id = mc.get("wordpress_bind:%s" % g.user.id)
c = random_id and Confirmation.get_by_random_id(random_id)
if c:
_, feed_uri = c.text.split(":", 1)
step = "2"
else:
feed_uri = ""
if request.method == "GET":
return render_template("bind_wordpress.html", consts=consts, **locals())
elif request.method == "POST":
ret = {}
ret['ok'] = False
if step == '1':
feed_uri = request.form.get("feed_uri")
if not feed_uri:
ret['msg'] = 'feed地址不能为空'
elif not (feed_uri.startswith("http://") or feed_uri.startswith("https://")):
ret['msg'] = 'feed地址貌似不对'
else:
ua = UserAlias.get(config.OPENID_TYPE_DICT[config.OPENID_WORDPRESS], feed_uri)
if ua:
ret['msg'] = '该feed地址已被绑定'
else:
##设置一个激活码
code = randbytes(16)
val = "%s:%s" % (g.user.id, feed_uri)
r = Confirmation.add(code, val)
if r:
ret['ok'] = True
ret['msg'] = '为了验证blog的主人^^，请发一篇blog，内容为 %s，完成该步骤后，请点下一步完成绑定' \
% code
mc.set("wordpress_bind:%s" %g.user.id, code)
else:
ret['msg'] = '抱歉，出错了，请重试, 或者给管理员捎个话:[email protected]'
return json_encode(ret)
elif step == '2':
if not (random_id and c):
ret['msg'] = '出错了，激活码不对^^'
else:
text = c.text
user_id, feed_uri = text.split(":", 1)
## 同步一下，看看验证码的文章是否正确
client = Wordpress(feed_uri)
rs = client.get_feeds(refresh=True)
if not rs:
ret['msg'] = '没有发现含有验证码的文章，请检查后再提交验证'
else:
latest_post = rs[0]
if not latest_post:
ret['msg'] = "你的feed地址可能无法访问，请检查下"
else:
content = latest_post.get_content() or latest_post.get_summary()
if content and content.encode("utf8")[:100].find(str(random_id)) != -1:
ua = UserAlias.bind_to_exists_user(g.user,
config.OPENID_TYPE_DICT[config.OPENID_WORDPRESS], feed_uri)
if not ua:
ret['msg'] = '出错了，麻烦你重试一下吧^^'
else:
##添加同步任务
t = SyncTask.add(config.CATE_WORDPRESS_POST, g.user.id)
t and TaskQueue.add(t.id, t.kind)
##删除confiration记录
c.delete()
mc.delete("wordpress_bind:%s" %g.user.id)
ret['ok'] = True
ret['msg'] = '恭喜，绑定成功啦'
else:
ret['msg'] = "没有发现含有验证码的文章，请检查后再提交验证"
return json_encode(ret)
else:
return "method not allowed"
import datetime
from past import config
from past.model.status import SyncTask
from past.model.user import User
import jobs
if __name__ == '__main__':
user = User.get(sys.argv[1])
old = sys.argv[2] == "old"
if not user:
print "no such user"
exit(1)
ts = SyncTask.gets_by_user(user)
if not ts:
print "no sync tasks"
for t in ts:
try:
if t.category == config.CATE_WORDPRESS_POST:
jobs.sync_wordpress(t)
else:
jobs.sync(t, old=old)
except Exception, e:
import traceback
print "%s %s" % (datetime.datetime.now(), traceback.format_exc())