def sync_wordpress(t, refresh=False):
if not t:
log.warning('no_wordpress_sync_task')
return
#一个人可以有多个wordpress的rss源地址
rs = UserAlias.gets_by_user_id(t.user_id)
uas = []
for x in rs:
if x.type == config.OPENID_TYPE_DICT[config.OPENID_WORDPRESS]:
uas.append(x)
if not uas:
log.warning('no_wordpress_alias')
return
for ua in uas:
try:
client = Wordpress(ua.alias)
rs = client.get_feeds(refresh)
if rs:
log.info("get wordpress succ, result length is:%s" % len(rs))
for x in rs:
Status.add_from_obj(t.user_id, x,
json_encode(x.get_data()))
return
except:
import traceback
print traceback.format_exc()
def user_by_domain(uid):
u = User.get(uid)
if not u:
abort(404, "no such user")
r = check_access_user(u)
if r:
flash(r[1].decode("utf8"), "tip")
return redirect(url_for("home"))
ids = Status.get_ids(user_id=u.id, start=g.start, limit=g.count, cate=g.cate)
status_list = Status.gets(ids)
if g.user and g.user.id == uid:
pass
elif g.user and g.user.id != uid:
status_list = [x for x in status_list if x.privacy() != consts.STATUS_PRIVACY_PRIVATE]
elif not g.user:
status_list = [x for x in status_list if x.privacy() == consts.STATUS_PRIVACY_PUBLIC]
status_list = statuses_timelize(status_list)
tags_list = []
intros = [u.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
if g.user:
sync_list = get_sync_list(g.user)
else:
sync_list = []
return render_template("timeline.html", user=u, unbinded=[],
tags_list=tags_list, intros=intros,
status_list=status_list, config=config, sync_list=sync_list)
def user(uid):
u = User.get(uid)
if not u:
abort(404, "no such user")
if g.user and g.user.id == u.id:
return redirect(url_for("timeline"))
if u.get_profile_item('user_privacy') == consts.USER_PRIVACY_PRIVATE:
flash(u"由于该用户设置了仅自己可见的权限，所以，我们就看不到了", "tip")
return redirect(url_for("timeline"))
#TODO:增加可否查看其他用户的权限检查
cate = request.args.get("cate", None)
ids = Status.get_ids(user_id=u.id, start=g.start, limit=g.count, cate=g.cate)
status_list = Status.gets(ids)
status_list = statuses_timelize(status_list)
if status_list:
##XXX:暂时去除了个人关键字的功能
#tags_list = [x[0] for x in get_keywords(u.id, 30)]
tags_list = []
else:
tags_list = []
intros = [u.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
return render_template("timeline.html", user=u, unbinded=[],
tags_list=tags_list, intros=intros, status_list=status_list, config=config)
def remove_user(uid, clear_status=True):
user = User.get(uid)
if not user:
print "---no user:%s" % uid
suicide_log.info("---- delete from user, uid=%s" % uid)
db_conn.execute("delete from user where id=%s", uid)
db_conn.commit()
User._clear_cache(uid)
if clear_status:
cursor = db_conn.execute("select id from status where user_id=%s", uid)
if cursor:
rows = cursor.fetchall()
for row in rows:
sid = row[0]
suicide_log.info("---- delete status text, sid=%s" % sid)
RawStatus.remove(sid)
suicide_log.info("---- delete from status, uid=" % uid)
db_conn.execute("delete from status where user_id=%s", uid)
db_conn.commit()
Status._clear_cache(uid, None)
suicide_log.info("---- delete from passwd, uid=%s" % uid)
db_conn.execute("delete from passwd where user_id=%s", uid)
suicide_log.info("---- delete from sync_task, uid=%s" % uid)
db_conn.execute("delete from sync_task where user_id=%s", uid)
suicide_log.info("---- delete from user_alias, uid=%s" % uid)
db_conn.execute("delete from user_alias where user_id=%s", uid)
db_conn.commit()
def remove_user(uid):
user = User.get(uid)
if not user:
print '---no user:%s' % uid
print "---- delete from user, uid=", uid
db_conn.execute("delete from user where id=%s", uid)
db_conn.commit()
User._clear_cache(uid)
cursor = db_conn.execute("select id from status where user_id=%s", uid)
if cursor:
rows = cursor.fetchall()
for row in rows:
sid = row[0]
print "---- delete mongo text, sid=", sid
RawStatus.remove(sid)
print "---- delete from status, uid=", uid
db_conn.execute("delete from status where user_id=%s", uid)
db_conn.commit()
Status._clear_cache(uid, None)
print "---- delete from passwd, uid=", uid
db_conn.execute("delete from passwd where user_id=%s", uid)
print "---- delete from sync_task, uid=", uid
db_conn.execute("delete from sync_task where user_id=%s", uid)
print "---- delete from user_alias, uid=", uid
db_conn.execute("delete from user_alias where user_id=%s", uid)
db_conn.commit()
def remove_user(uid, clear_status=True):
user = User.get(uid)
if not user:
print '---no user:%s' % uid
suicide_log.info("---- delete from user, uid=%s" % uid)
db_conn.execute("delete from user where id=%s", uid)
db_conn.commit()
User._clear_cache(uid)
if clear_status:
cursor = db_conn.execute("select id from status where user_id=%s", uid)
if cursor:
rows = cursor.fetchall()
for row in rows:
sid = row[0]
suicide_log.info("---- delete status text, sid=%s" % sid)
RawStatus.remove(sid)
suicide_log.info("---- delete from status, uid=" % uid)
db_conn.execute("delete from status where user_id=%s", uid)
db_conn.commit()
Status._clear_cache(uid, None)
suicide_log.info("---- delete from passwd, uid=%s" % uid)
db_conn.execute("delete from passwd where user_id=%s", uid)
suicide_log.info("---- delete from sync_task, uid=%s" % uid)
db_conn.execute("delete from sync_task where user_id=%s", uid)
suicide_log.info("---- delete from user_alias, uid=%s" % uid)
db_conn.execute("delete from user_alias where user_id=%s", uid)
db_conn.commit()
def sync_wordpress(t, refresh=False):
if not t:
log.warning('no_wordpress_sync_task')
return
#一个人可以有多个wordpress的rss源地址
rs = UserAlias.gets_by_user_id(t.user_id)
uas = []
for x in rs:
if x.type == config.OPENID_TYPE_DICT[config.OPENID_WORDPRESS]:
uas.append(x)
if not uas:
log.warning('no_wordpress_alias')
return
for ua in uas:
try:
client = Wordpress(ua.alias)
rs = client.get_feeds(refresh)
if rs:
log.info("get wordpress succ, result length is:%s" % len(rs))
for x in rs:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return
except:
import traceback; print traceback.format_exc()
def user_more_by_domain(uid):
u = User.get(uid)
if not u:
abort(404, "no such user")
r = check_access_user(u)
if r:
abort(400, "no priv to access")
ids = Status.get_ids(user_id=u.id, start=g.start, limit=g.count, cate=g.cate)
status_list = Status.gets(ids)
if g.user and g.user.id == uid:
pass
elif g.user and g.user.id != uid:
status_list = [x for x in status_list if x.privacy() != consts.STATUS_PRIVACY_PRIVATE]
elif not g.user:
status_list = [x for x in status_list if x.privacy() == consts.STATUS_PRIVACY_PUBLIC]
status_list = statuses_timelize(status_list)
intros = [u.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
if g.user:
sync_list = get_sync_list(g.user)
else:
sync_list = []
now = datetime.datetime.now().strftime("%Y年%m月%d日 %H:%M:%S")
return render_template("v2/user_more.html", user=u, intros=intros,
status_list=status_list, config=config, sync_list=sync_list,
now = now)
def generate_pdf(filename, uid, start, count, cate=None, with_head=True, capacity=50*1024):
#########Set FONT################
from xhtml2pdf.default import DEFAULT_FONT
from xhtml2pdf.document import pisaDocument
from reportlab.pdfbase import pdfmetrics
from reportlab.pdfbase.ttfonts import TTFont
pdfmetrics.registerFont(TTFont('zhfont', os.path.join(app.root_path, 'static/font/yahei-consolas.ttf')))
DEFAULT_FONT["helvetica"] = "zhfont"
css = open(os.path.join(app.root_path, "static/css/pdf.css")).read()
#result = StringIO.StringIO()
full_file_name = get_pdf_full_filename(filename)
if not full_file_name:
return None
result = open(full_file_name, 'wb', 1024*1000)
user = User.get(uid)
if not user:
return None
# get status
ids = Status.get_ids(user_id=uid, start=start, limit=count, cate=cate)
status_list = Status.gets(ids)
_html = render(user, status_list, with_head)
_pdf = pisaDocument(_html, result, default_css=css, link_callback=link_callback, capacity=capacity)
result.close()
if not _pdf.err:
return full_file_name
else:
return None
def home():
user_ids = User.get_ids(limit=10000)
user = None
i = 0
while i <= 3:
random_uid = random.choice(user_ids)
user = User.get(random_uid)
r = check_access_user(user)
if not r:
break
i+=1
if user:
history_ids = Status.get_ids(user.id, start=0, limit=20)
status_list = Status.gets(history_ids)
status_list = statuses_timelize(status_list)
intros = [user.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
else:
status_list = []
intros = []
if g.user:
sync_list = get_sync_list(g.user)
else:
sync_list = []
d = defaultdict(list)
for x in status_list:
t = x.create_time.strftime("%Y年%m月%d日")
d[t].append(x)
history_status = d
return render_template("v2/explore.html", **locals())
def send_today_in_history(user_id):
u = User.get(user_id)
if not u:
return
setting = u.get_profile_item("email_remind_today_in_history")
if setting == 'N':
print '---user %s does not like to receive remind mail' % u.id
return
email = u.get_email()
if not email:
print '---- user %s no email' % u.id
return
yesterday_ids = get_status_ids_yesterday(u.id,
day=(datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y-%m-%d"))
status_of_yesterday = Status.gets(yesterday_ids)
history_ids = get_status_ids_today_in_history(u.id,
day=datetime.datetime.now().strftime("%Y-%m-%d"))
status_of_today_in_history = Status.gets(history_ids)
intros = [u.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
history_ids = get_status_ids_today_in_history(u.id,
day=datetime.datetime.now().strftime("%Y-%m-%d"))
d = {}
for s in Status.gets(history_ids):
t = s.create_time.strftime("%Y-%m-%d")
if d.has_key(t):
d[t].append(s)
else:
d[t] = [s]
status_of_today_in_history = d
from past.consts import YESTERDAY
if not (status_of_yesterday or status_of_today_in_history):
print '--- user %s has no status in history' % u.id
return
from jinja2 import Environment, PackageLoader
env = Environment(loader=PackageLoader('past', 'templates'))
env.filters['wrap_long_line'] = wrap_long_line
env.filters['nl2br'] = filters.nl2br
env.filters['clear_html_element'] = clear_html_element
t = env.get_template('mail.html')
m = t.module
html = m.status_in_past(status_of_yesterday, status_of_today_in_history, YESTERDAY, config, intros)
html = html.encode("utf8")
subject = '''来自thepast.me的提醒 %s''' % datetime.datetime.now().strftime("%Y-%m-%d")
text = ''
print '--- send reminding to %s %s' %(user_id, email)
send_mail(["%s" % email], "Today of The Past<*****@*****.**>", subject, text, html, files=[], server="localhost")
def index():
if not g.user:
return redirect(url_for("user_explore"))
ids = Status.get_ids(user_id=g.user.id, start=g.start, limit=g.count, cate=g.cate)
status_list = Status.gets(ids)
return render_template("timeline.html", user=g.user, status_list=status_list, config=config)
def render(user, status_list, with_head=True):
if not status_list:
return
date = status_list[0].create_time.strftime("%Y年%m月")
date = date.decode("utf8")
if with_head:
_html = u"""<html> <body>
<div id="Top">
<img src="%s"/> The Past of Me | 个人杂志计划 %s CopyRight©%s
<br/>
</div>
<br/> <br/>
<div class="box">
""" % (os.path.join(app.root_path, "static/img/logo.png"),
date, user.name)
else:
_html = u"""<html> <body><div class="box">"""
from jinja2 import Environment, PackageLoader
env = Environment(loader=PackageLoader('past', 'templates'))
env.filters['wrap_long_line'] = wrap_long_line
env.filters['nl2br'] = filters.nl2br
env.filters['stream_time'] = filters.stream_time
env.filters['clear_html_element'] = clear_html_element
env.filters['isstr'] = lambda x: isinstance(x, basestring)
t = env.get_template('status.html')
m = t.module
for s in status_list:
if not s:
continue
if s.category == config.CATE_DOUBAN_STATUS:
r = m.douban_status(s, pdf=True)
elif s.category == config.CATE_SINA_STATUS:
r = m.sina_status(s, pdf=True)
elif s.category == config.CATE_TWITTER_STATUS:
r = m.twitter_status(s, pdf=True)
elif s.category == config.CATE_QQWEIBO_STATUS:
r = m.qq_weibo_status(s, pdf=True)
elif s.category == config.CATE_WORDPRESS_POST:
r = m.wordpress_status(s, pdf=True)
elif s.category == config.CATE_THEPAST_NOTE:
r = m.thepast_note_status(s, pdf=True)
elif s.category == config.CATE_RENREN_STATUS:
r = m.thepast_renren_status(s, pdf=True)
elif s.category == config.CATE_RENREN_BLOG:
r = m.thepast_renren_blog(s, pdf=True)
elif s.category == config.CATE_RENREN_PHOTO or s.category == config.CATE_RENREN_ALBUM:
r = m.thepast_renren_photo(s, pdf=True)
elif s.category == config.CATE_INSTAGRAM_STATUS:
r = m.thepast_default_status(s, pdf=True)
else:
r = ''
if not r:
continue
_html += '''<div class="cell">''' + r + '''</div>'''
Status._clear_cache(user_id = s.user_id, status_id = s.id)
_html += """</div></body></html>"""
return _html
def timeline():
ids = Status.get_ids(user_id=g.user.id, start=g.start, limit=g.count, cate=g.cate)
status_list = Status.gets(ids)
status_list = statuses_timelize(status_list)
if status_list:
tags_list = [x[0] for x in get_keywords(g.user.id, 30)]
else:
tags_list = []
intros = [g.user.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
return render_template("timeline.html", user=g.user, tags_list=tags_list,
intros=intros, status_list=status_list, config=config)
def render(user, status_list, with_head=True):
if not status_list:
return
date = status_list[0].create_time.strftime("%Y年%m月")
date = date.decode("utf8")
if with_head:
_html = u"""<html> <body>
<div id="Top">
<img src="%s"/> The Past of Me | 个人杂志计划 %s CopyRight©%s
<br/>
</div>
<br/> <br/>
<div class="box">
""" % (
os.path.join(app.root_path, "static/img/logo.png"),
date,
user.name,
)
else:
_html = u"""<html> <body><div class="box">"""
from jinja2 import Environment, PackageLoader
env = Environment(loader=PackageLoader("past", "templates"))
env.filters["wrap_long_line"] = wrap_long_line
env.filters["nl2br"] = filters.nl2br
env.filters["clear_html_element"] = clear_html_element
t = env.get_template("status.html")
m = t.module
for s in status_list:
if not s:
continue
if s.category == config.CATE_DOUBAN_STATUS:
r = m.douban_status(s, pdf=True)
elif s.category == config.CATE_SINA_STATUS:
r = m.sina_status(s, pdf=True)
elif s.category == config.CATE_TWITTER_STATUS:
r = m.twitter_status(s, pdf=True)
elif s.category == config.CATE_QQWEIBO_STATUS:
r = m.qq_weibo_status(s, pdf=True)
elif s.category == config.CATE_WORDPRESS_POST:
r = m.wordpress_status(s, pdf=True)
elif s.category == config.CATE_THEPAST_NOTE:
r = m.thepast_note_status(s, pdf=True)
else:
r = ""
if not r:
continue
_html += """<div class="cell">""" + r + """</div>"""
Status._clear_cache(user_id=s.user_id, status_id=s.id)
_html += """</div></body></html>"""
return _html
def update(self, title, content, fmt):
if title and title != self.title or fmt and fmt != self.fmt or content and content != self.content:
_fmt = fmt or self.fmt
_title = title or self.title
_content = content or self.content
db_conn.execute('''update note set title = %s, content = %s, fmt = %s where id = %s''',
(_title, _content, _fmt, self.id))
db_conn.commit()
self.flush_note()
if title != self.title:
from past.model.status import Status
Status._clear_cache(None, self.get_status_id(), None)
def remove_status(uid):
cursor = db_conn.execute("select id from status where user_id=%s", uid)
if cursor:
rows = cursor.fetchall()
for row in rows:
sid = row[0]
print "---- delete mongo text, sid=", sid
RawStatus.remove(sid)
print "---- delete from status, uid=", uid
db_conn.execute("delete from status where user_id=%s", uid)
db_conn.commit()
Status._clear_cache(uid, None)
def user(uid):
u = User.get(uid)
if not u:
abort(404, "no such user")
if g.user and g.user.id == u.id:
return redirect(url_for("index"))
#TODO:增加可否查看其他用户的权限检查
cate = request.args.get("cate", None)
ids = Status.get_ids(user_id=u.id, start=g.start, limit=g.count, cate=g.cate)
status_list = Status.gets(ids)
return render_template("timeline.html", user=u, status_list=status_list, config=config)
def update(self, title, content, fmt, privacy):
if title and title != self.title or fmt and fmt != self.fmt or content and content != self.content or privacy and privacy != self.privacy:
_fmt = fmt or self.fmt
_title = title or self.title
_content = content or self.content
_privacy = privacy or self.privacy
db_conn.execute('''update note set title = %s, content = %s, fmt = %s, privacy = %s where id = %s''',
(_title, _content, _fmt, _privacy, self.id))
db_conn.commit()
self.flush_note()
if title != self.title:
from past.model.status import Status
Status._clear_cache(None, self.get_status_id(), None)
def user_by_domain(uid):
u = User.get(uid)
if not u:
abort(404, "no such user")
r = check_access_user(u)
if r:
flash(r[1].decode("utf8"), "tip")
return redirect(url_for("home"))
ids = Status.get_ids(user_id=u.id,
start=g.start,
limit=g.count,
cate=g.cate)
status_list = Status.gets(ids)
if g.user and g.user.id == uid:
pass
elif g.user and g.user.id != uid:
status_list = [
x for x in status_list
if x.privacy() != consts.STATUS_PRIVACY_PRIVATE
]
elif not g.user:
status_list = [
x for x in status_list
if x.privacy() == consts.STATUS_PRIVACY_PUBLIC
]
status_list = statuses_timelize(status_list)
tags_list = []
intros = [
u.get_thirdparty_profile(x).get("intro")
for x in config.OPENID_TYPE_DICT.values()
]
intros = filter(None, intros)
if g.user:
sync_list = get_sync_list(g.user)
else:
sync_list = []
return render_template("timeline.html",
user=u,
unbinded=[],
tags_list=tags_list,
intros=intros,
status_list=status_list,
config=config,
sync_list=sync_list)
def pdf(uid):
user = User.get(uid)
if not user:
abort(404, "No such user")
if uid != g.user.id and user.get_profile_item('user_privacy') == consts.USER_PRIVACY_PRIVATE:
flash(u"由于该用户设置了仅自己可见的权限，所以，我们就看不到了", "tip")
return redirect("/")
intros = [g.user.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
pdf_files = []
start_date = Status.get_oldest_create_time(None, user.id)
now = datetime.now()
d = start_date
while d and d <= now:
pdf_filename = get_pdf_filename(user.id, d.strftime("%Y%m"))
if is_pdf_file_exists(pdf_filename):
full_file_name = get_pdf_full_filename(pdf_filename)
pdf_files.append([d, pdf_filename, sizeof_fmt(os.path.getsize(full_file_name))])
days = calendar.monthrange(d.year, d.month)[1]
d += timedelta(days=days)
d = datetime(d.year, d.month, 1)
files_dict = defaultdict(list)
for date, filename, filesize in pdf_files:
files_dict[date.year].append([date, filename, filesize])
pdf_applyed = PdfSettings.is_user_id_exists(g.user.id)
return render_template("v2/pdf.html", **locals())
def generate(user_id, date, order='asc'):
try:
uas = UserAlias.gets_by_user_id(user_id)
if not uas:
return
start_date = datetime.datetime(date.year, date.month, 1)
end_date = datetime.datetime(
date.year, date.month,
calendar.monthrange(date.year, date.month)[1], 23, 59, 59)
pdf_filename = get_pdf_filename(user_id, date.strftime("%Y%m"))
print '----generate pdf:', start_date, ' to ', end_date, ' file is', pdf_filename
if is_pdf_file_exists(pdf_filename):
print '---- %s exists, so ignore...' % pdf_filename
return
status_ids = Status.get_ids_by_date(user_id, start_date,
end_date)[:900]
if order == 'asc':
status_ids = status_ids[::-1]
if not status_ids:
print '----- status ids is none', status_ids
return
generate_pdf(pdf_filename, user_id, status_ids, capacity=-1)
if not is_pdf_file_exists(pdf_filename):
print '----%s generate pdf for user:%s fail' % (
datetime.datetime.now(), user_id)
else:
print '----%s generate pdf for user:%s succ' % (
datetime.datetime.now(), user_id)
except Exception, e:
import traceback
print '%s %s' % (datetime.datetime.now(), traceback.format_exc())
def generate_pdf(filename, uid, status_ids, with_head=True, capacity=50*1024):
#########Set FONT################
from xhtml2pdf.default import DEFAULT_FONT
from xhtml2pdf.document import pisaDocument
from reportlab.pdfbase import pdfmetrics
from reportlab.pdfbase.ttfonts import TTFont
pdfmetrics.registerFont(TTFont('zhfont', os.path.join(app.root_path, 'static/font/yahei-consolas.ttf')))
DEFAULT_FONT["helvetica"] = "zhfont"
css = open(os.path.join(app.root_path, "static/css/pdf.css")).read()
#result = StringIO.StringIO()
full_file_name = get_pdf_full_filename(filename)
if not full_file_name:
return None
result = open(full_file_name, 'wb', 1024*1000)
user = User.get(uid)
if not user:
return None
# get status
status_list = Status.gets(status_ids)
_html = render(user, status_list, with_head)
_pdf = pisaDocument(_html, result, default_css=css, link_callback=link_callback, capacity=capacity)
result.close()
if not _pdf.err:
return full_file_name
else:
return None
def generate(user_id, date, order='asc'):
try:
uas = UserAlias.gets_by_user_id(user_id)
if not uas:
return
start_date = datetime.datetime(date.year, date.month, 1)
end_date = datetime.datetime(date.year, date.month,
calendar.monthrange(date.year, date.month)[1], 23, 59, 59)
pdf_filename = get_pdf_filename(user_id, date.strftime("%Y%m"))
print '----generate pdf:', start_date, ' to ', end_date, ' file is', pdf_filename
if is_pdf_file_exists(pdf_filename):
print '---- %s exists, so ignore...' % pdf_filename
return
status_ids = Status.get_ids_by_date(user_id, start_date, end_date)[:900]
if order == 'asc':
status_ids = status_ids[::-1]
if not status_ids:
print '----- status ids is none', status_ids
return
generate_pdf(pdf_filename, user_id, status_ids, capacity=-1)
if not is_pdf_file_exists(pdf_filename):
print '----%s generate pdf for user:%s fail' % (datetime.datetime.now(), user_id)
else:
print '----%s generate pdf for user:%s succ' % (datetime.datetime.now(), user_id)
except Exception, e:
import traceback
print '%s %s' % (datetime.datetime.now(), traceback.format_exc())
def pdf(uid):
user = User.get(uid)
if not user:
abort(404, "No such user")
if uid != g.user.id and user.get_profile_item('user_privacy') == consts.USER_PRIVACY_PRIVATE:
flash(u"由于该用户设置了仅自己可见的权限，所以，我们就看不到了", "tip")
return redirect(url_for("timeline"))
intros = [g.user.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
pdf_files = []
start_date = Status.get_oldest_create_time(None, user.id)
now = datetime.now()
d = start_date
while d and d <= now:
pdf_filename = get_pdf_filename(user.id, d.strftime("%Y%m"))
if is_pdf_file_exists(pdf_filename):
full_file_name = get_pdf_full_filename(pdf_filename)
pdf_files.append([d, pdf_filename, sizeof_fmt(os.path.getsize(full_file_name))])
days = calendar.monthrange(d.year, d.month)[1]
d += timedelta(days=days)
d = datetime(d.year, d.month, 1)
files_dict = defaultdict(list)
for date, filename, filesize in pdf_files:
files_dict[date.year].append([date, filename, filesize])
pdf_applyed = PdfSettings.is_user_id_exists(g.user.id)
return render_template("pdf.html", **locals())
def user_more_by_domain(uid):
u = User.get(uid)
if not u:
abort(404, "no such user")
r = check_access_user(u)
if r:
abort(400, "no priv to access")
ids = Status.get_ids(user_id=u.id,
start=g.start,
limit=g.count,
cate=g.cate)
status_list = Status.gets(ids)
if g.user and g.user.id == uid:
pass
elif g.user and g.user.id != uid:
status_list = [
x for x in status_list
if x.privacy() != consts.STATUS_PRIVACY_PRIVATE
]
elif not g.user:
status_list = [
x for x in status_list
if x.privacy() == consts.STATUS_PRIVACY_PUBLIC
]
status_list = statuses_timelize(status_list)
intros = [
u.get_thirdparty_profile(x).get("intro")
for x in config.OPENID_TYPE_DICT.values()
]
intros = filter(None, intros)
if g.user:
sync_list = get_sync_list(g.user)
else:
sync_list = []
now = datetime.datetime.now().strftime("%Y年%m月%d日 %H:%M:%S")
return render_template("v2/user_more.html",
user=u,
intros=intros,
status_list=status_list,
config=config,
sync_list=sync_list,
now=now)
def home():
user_ids = Status.get_recent_updated_user_ids()
users = filter(None, [User.get(x) for x in user_ids])
users = [
x for x in users
if x.get_profile_item('user_privacy') != consts.USER_PRIVACY_PRIVATE
]
return render_template("home.html", users=users, config=config)
def post(id):
intros = [g.user.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
status = Status.get(id)
if not status:
flash(u"访问的文章不存在^^","error")
return render_template("post.html", **locals())
def past():
intros = [g.user.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
yesterday_ids = get_status_ids_yesterday(g.user.id)
status_of_yesterday = Status.gets(yesterday_ids)
history_ids = get_status_ids_today_in_history(g.user.id)
d = {}
for s in Status.gets(history_ids):
t = s.create_time.strftime("%Y-%m-%d")
if d.has_key(t):
d[t].append(s)
else:
d[t] = [s]
status_of_today_in_history = d
from past.consts import YESTERDAY
return render_template("past.html", **locals())
def user(uid):
u = User.get(uid)
if not u:
abort(404, "no such user")
if g.user and g.user.id == u.id:
return redirect(url_for("timeline"))
# TODO:增加可否查看其他用户的权限检查
cate = request.args.get("cate", None)
ids = Status.get_ids(user_id=u.id, start=g.start, limit=g.count, cate=g.cate)
status_list = Status.gets(ids)
status_list = statuses_timelize(status_list)
unbinded = list(set(config.OPENID_TYPE_DICT.values()) - set([ua.type for ua in g.user.get_alias()]))
tmp = {}
for k, v in config.OPENID_TYPE_DICT.items():
tmp[v] = k
unbinded = [[x, tmp[x], config.OPENID_TYPE_NAME_DICT[x]] for x in unbinded]
return render_template("timeline.html", user=u, unbinded=unbinded, status_list=status_list, config=config)
def add(cls, user_id, title, content, fmt=consts.NOTE_FMT_PLAIN, privacy=consts.STATUS_PRIVACY_PUBLIC):
cursor = None
try:
cursor = db_conn.execute('''insert into note (user_id, title, content, create_time, fmt, privacy)
values (%s, %s, %s, %s, %s, %s)''',
(user_id, title, content, datetime.datetime.now(), fmt, privacy))
db_conn.commit()
note_id = cursor.lastrowid
note = cls.get(note_id)
from past.model.status import Status
Status.add(user_id, note_id,
note.create_time, config.OPENID_TYPE_DICT[config.OPENID_THEPAST],
config.CATE_THEPAST_NOTE, "")
cls._clear_cache(user_id, None)
return note
except IntegrityError:
db_conn.rollback()
finally:
cursor and cursor.close()
def send_yesterday(user_id, now=None):
if not now:
now = datetime.datetime.now()
u = User.get(user_id)
if not u:
return
setting = u.get_profile_item("email_remind_today_in_history")
if setting == 'N':
print '---user %s does not like to receive remind mail' % u.id
return
email = u.get_email()
if not email:
print '---- user %s no email' % u.id
return
yesterday_ids = get_status_ids_yesterday(u.id, now)
status_of_yesterday = Status.gets(yesterday_ids)
intros = [
u.get_thirdparty_profile(x).get("intro")
for x in config.OPENID_TYPE_DICT.values()
]
intros = filter(None, intros)
from past.consts import YESTERDAY
if not status_of_yesterday:
print '--- user %s has no status in yesterday' % u.id
return
from jinja2 import Environment, PackageLoader
env = Environment(loader=PackageLoader('past', 'templates'))
env.filters['wrap_long_line'] = wrap_long_line
env.filters['nl2br'] = filters.nl2br
env.filters['stream_time'] = filters.stream_time
env.filters['clear_html_element'] = clear_html_element
t = env.get_template('mail.html')
m = t.module
if now:
y = (now - datetime.timedelta(days=1)).strftime("%Y-%m-%d")
else:
y = YESTERDAY
html = m.status_in_past(status_of_yesterday, None, y, config, intros)
html = html.encode("utf8")
subject = '''thepast.me|整理自己的故事 %s''' % now.strftime("%Y-%m-%d")
text = ''
print '--- send reminding to %s %s' % (user_id, email)
send_mail(["%s" % email], "thepast<*****@*****.**>", subject, text, html)
def user(uid):
u = User.get(uid)
if not u:
abort(404, "no such user")
if g.user and g.user.id == u.id:
return redirect(url_for("timeline"))
#TODO:增加可否查看其他用户的权限检查
cate = request.args.get("cate", None)
ids = Status.get_ids(user_id=u.id, start=g.start, limit=g.count, cate=g.cate)
status_list = Status.gets(ids)
status_list = statuses_timelize(status_list)
if status_list:
tags_list = [x[0] for x in get_keywords(u.id, 30)]
else:
tags_list = []
intros = [u.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
return render_template("timeline.html", user=u, unbinded=[],
tags_list=tags_list, intros=intros, status_list=status_list, config=config)
def send_yesterday(user_id, now=None):
if not now:
now = datetime.datetime.now()
u = User.get(user_id)
if not u:
return
setting = u.get_profile_item("email_remind_today_in_history")
if setting == "N":
print "---user %s does not like to receive remind mail" % u.id
return
email = u.get_email()
if not email:
print "---- user %s no email" % u.id
return
yesterday_ids = get_status_ids_yesterday(u.id, now)
status_of_yesterday = Status.gets(yesterday_ids)
intros = [u.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
from past.consts import YESTERDAY
if not status_of_yesterday:
print "--- user %s has no status in yesterday" % u.id
return
from jinja2 import Environment, PackageLoader
env = Environment(loader=PackageLoader("past", "templates"))
env.filters["wrap_long_line"] = wrap_long_line
env.filters["nl2br"] = filters.nl2br
env.filters["stream_time"] = filters.stream_time
env.filters["clear_html_element"] = clear_html_element
t = env.get_template("mail.html")
m = t.module
if now:
y = (now - datetime.timedelta(days=1)).strftime("%Y-%m-%d")
else:
y = YESTERDAY
html = m.status_in_past(status_of_yesterday, None, y, config, intros)
html = html.encode("utf8")
subject = """thepast.me|整理自己的故事 %s""" % now.strftime("%Y-%m-%d")
text = ""
print "--- send reminding to %s %s" % (user_id, email)
send_mail(["%s" % email], "thepast<*****@*****.**>", subject, text, html)
def home():
user_ids = User.get_ids(limit=10000)
user = None
i = 0
while i <= 3:
random_uid = random.choice(user_ids)
user = User.get(random_uid)
r = check_access_user(user)
if not r:
break
i += 1
if user:
history_ids = Status.get_ids(user.id, start=0, limit=20)
status_list = Status.gets(history_ids)
status_list = statuses_timelize(status_list)
intros = [
user.get_thirdparty_profile(x).get("intro")
for x in config.OPENID_TYPE_DICT.values()
]
intros = filter(None, intros)
else:
status_list = []
intros = []
if g.user:
sync_list = get_sync_list(g.user)
else:
sync_list = []
d = defaultdict(list)
for x in status_list:
t = x.create_time.strftime("%Y年%m月%d日")
d[t].append(x)
history_status = d
return render_template("v2/explore.html", **locals())
def post(id):
status = Status.get(id)
if not status:
abort(404, "访问的文章不存在^^")
else:
user = User.get(status.user_id)
if user and not check_access_user(user):
if status.category == config.CATE_THEPAST_NOTE:
return redirect("/note/%s" % status.origin_id)
intros = [user.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
return render_template("post.html", config=config, **locals())
else:
abort(403, "没有权限访问该文章")
def past():
intros = [
g.user.get_thirdparty_profile(x).get("intro")
for x in config.OPENID_TYPE_DICT.values()
]
intros = filter(None, intros)
now = datetime.datetime.now()
yesterday_ids = get_status_ids_yesterday(g.user.id, now)
status_of_yesterday = Status.gets(yesterday_ids)
history_ids = get_status_ids_today_in_history(g.user.id, now)
d = {}
for s in Status.gets(history_ids):
t = s.create_time.strftime("%Y-%m-%d")
if d.has_key(t):
d[t].append(s)
else:
d[t] = [s]
status_of_today_in_history = d
from past.consts import YESTERDAY
return render_template("past.html", **locals())
def post(id):
status = Status.get(id)
if not status:
abort(404, "访问的文章不存在^^")
else:
user = User.get(status.user_id)
if user and not check_access_user(user):
if status.category == config.CATE_THEPAST_NOTE:
return redirect("/note/%s" % status.origin_id)
intros = [user.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
return render_template("post.html", config=config, **locals())
else:
abort(403, "没有权限访问该文章")
def user_past(uid):
user = User.get(uid)
if not user:
abort(404, "no such user")
r = check_access_user(user)
if r:
flash(r[1].decode("utf8"), "tip")
return redirect("/")
try:
now = datetime.datetime.strptime(request.args.get("now"), "%Y-%m-%d")
except:
now = datetime.datetime.now()
history_ids = get_status_ids_today_in_history(user.id, now)
status_list = Status.gets(history_ids)
if g.user and g.user.id == uid:
pass
elif g.user and g.user.id != uid:
status_list = [
x for x in status_list
if x.privacy() != consts.STATUS_PRIVACY_PRIVATE
]
elif not g.user:
status_list = [
x for x in status_list
if x.privacy() == consts.STATUS_PRIVACY_PUBLIC
]
status_list = statuses_timelize(status_list)
if g.user:
sync_list = get_sync_list(g.user)
else:
sync_list = []
intros = [
user.get_thirdparty_profile(x).get("intro")
for x in config.OPENID_TYPE_DICT.values()
]
intros = filter(None, intros)
d = defaultdict(list)
for x in status_list:
t = x.create_time.strftime("%Y年%m月%d日")
d[t].append(x)
history_status = d
return render_template("v2/user_past.html", **locals())
def generate(user_id, date, order='asc'):
try:
uas = UserAlias.gets_by_user_id(user_id)
if not uas:
return
start_date = datetime.datetime(date.year, date.month, 1)
end_date = datetime.datetime(
date.year, date.month,
calendar.monthrange(date.year, date.month)[1], 23, 59, 59)
pdf_filename = get_pdf_filename(user_id, date.strftime("%Y%m"), "")
pdf_filename_compressed = get_pdf_filename(user_id,
date.strftime("%Y%m"))
print '----generate pdf:', start_date, ' to ', end_date, ' file is', pdf_filename
if is_pdf_file_exists(pdf_filename_compressed):
print '---- %s exists, so ignore...' % pdf_filename_compressed
return
status_ids = Status.get_ids_by_date(user_id, start_date,
end_date)[:900]
if order == 'asc':
status_ids = status_ids[::-1]
if not status_ids:
print '----- status ids is none', status_ids
return
generate_pdf(pdf_filename, user_id, status_ids)
if not is_pdf_file_exists(pdf_filename):
print '----%s generate pdf for user:%s fail' % (
datetime.datetime.now(), user_id)
else:
commands.getoutput(
"cd %s && tar -zcf %s %s && rm %s" %
(config.PDF_FILE_DOWNLOAD_DIR, pdf_filename_compressed,
pdf_filename, pdf_filename))
print '----%s generate pdf for user:%s succ' % (
datetime.datetime.now(), user_id)
except Exception, e:
import traceback
print '%s %s' % (datetime.datetime.now(), traceback.format_exc())
def generate_pdf_by_user(user_id):
user = User.get(user_id)
if not user:
return
#XXX:暂时只生成2012年的(uid从98开始的用户)
#XXX:暂时只生成2012年3月份的(uid从166开始的用户)
start_date = Status.get_oldest_create_time(None, user_id)
if not start_date:
return
now = datetime.datetime.now()
now = datetime.datetime(now.year, now.month, now.day) - datetime.timedelta(
days=calendar.monthrange(now.year, now.month)[1])
d = start_date
while d <= now:
generate(user_id, d)
days = calendar.monthrange(d.year, d.month)[1]
d += datetime.timedelta(days=days)
d = datetime.datetime(d.year, d.month, 1)
def generate_pdf_by_user(user_id):
user = User.get(user_id)
if not user:
return
#XXX:暂时只生成2012年的(uid从98开始的用户)
#XXX:暂时只生成2012年3月份的(uid从166开始的用户)
start_date = Status.get_oldest_create_time(None, user_id)
#start_date = datetime.datetime(2012, 3, 1, 0, 0, 0)
if not start_date:
return
now = datetime.datetime.now()
now = datetime.datetime(now.year, now.month, now.day) - datetime.timedelta(days = calendar.monthrange(now.year, now.month)[1])
d = start_date
while d <= now:
generate(user_id, d)
days = calendar.monthrange(d.year, d.month)[1]
d += datetime.timedelta(days=days)
d = datetime.datetime(d.year, d.month, 1)
def user_past(uid):
user = User.get(uid)
if not user:
abort(404, "no such user")
r = check_access_user(user)
if r:
flash(r[1].decode("utf8"), "tip")
return redirect("/")
try:
now = datetime.datetime.strptime(request.args.get("now"), "%Y-%m-%d")
except:
now = datetime.datetime.now()
history_ids = get_status_ids_today_in_history(user.id, now)
status_list = Status.gets(history_ids)
if g.user and g.user.id == uid:
pass
elif g.user and g.user.id != uid:
status_list = [x for x in status_list if x.privacy() != consts.STATUS_PRIVACY_PRIVATE]
elif not g.user:
status_list = [x for x in status_list if x.privacy() == consts.STATUS_PRIVACY_PUBLIC]
status_list = statuses_timelize(status_list)
if g.user:
sync_list = get_sync_list(g.user)
else:
sync_list = []
intros = [user.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
d = defaultdict(list)
for x in status_list:
t = x.create_time.strftime("%Y年%m月%d日")
d[t].append(x)
history_status = d
return render_template("v2/user_past.html", **locals())
def past():
intros = [g.user.get_thirdparty_profile(x).get("intro") for x in config.OPENID_TYPE_DICT.values()]
intros = filter(None, intros)
try:
now = datetime.datetime.strptime(request.args.get("now"), "%Y-%m-%d")
except:
now = datetime.datetime.now()
history_ids = get_status_ids_today_in_history(g.user.id, now)
status_list = Status.gets(history_ids)
status_list = statuses_timelize(status_list)
sync_list = get_sync_list(g.user)
d = defaultdict(list)
for x in status_list:
t = x.create_time.strftime("%Y年%m月%d日")
d[t].append(x)
history_status = d
return render_template("past.html", **locals())
def user_past(uid):
user = User.get(uid)
if not user:
abort(404, "no such user")
try:
now = datetime.datetime.strptime(request.args.get("now"), "%Y-%m-%d")
except:
now = datetime.datetime.now()
history_ids = get_status_ids_today_in_history(user.id, now)
status_list = Status.gets(history_ids)
status_list = statuses_timelize(status_list)
if g.user:
sync_list = get_sync_list(g.user)
else:
sync_list = []
d = defaultdict(list)
for x in status_list:
t = x.create_time.strftime("%Y年%m月%d日")
d[t].append(x)
history_status = d
return render_template("v2/user_past.html", **locals())
passwd=config.DB_PASSWD,
db="wp_linjuly",
use_unicode=True,
charset="utf8")
return conn
except Exception, e:
print "connect db fail:%s" % e
return None
db_conn = connect_db()
user_id = 34
limit = 250
status_ids = Status.get_ids(user_id, limit=limit, order="create_time desc")
for s in Status.gets(status_ids):
try:
_t = ''.join([x for x in s.text])
retweeted_data = s.get_retweeted_data()
if retweeted_data:
if isinstance(retweeted_data, basestring):
_t += retweeted_data
else:
_t += retweeted_data.get_content()
print '---sid:', s.id
post_author = 1
post_date = s.create_time
post_date_gmt = s.create_time - timedelta(hours=8)
def sync(t, old=False):
if not t:
print 'no such task'
return 0
log.info("the sync task is :%s" % t)
try:
alias = None
provider = category2provider(t.category)
alias = UserAlias.get_by_user_and_type(t.user_id,
config.OPENID_TYPE_DICT[provider])
if not alias:
log.warn("no alias...")
return 0
token = OAuth2Token.get(alias.id)
if not token:
log.warn("no access token, break...")
return 0
client = None
if provider == config.OPENID_DOUBAN:
client = Douban.get_client(alias.user_id)
elif provider == config.OPENID_SINA:
client = SinaWeibo.get_client(alias.user_id)
elif provider == config.OPENID_TWITTER:
client = TwitterOAuth1.get_client(alias.user_id)
elif provider == config.OPENID_QQ:
client = QQWeibo.get_client(alias.user_id)
elif provider == config.OPENID_RENREN:
client = Renren.get_client(alias.user_id)
elif provider == config.OPENID_INSTAGRAM:
client = Instagram.get_client(alias.user_id)
if not client:
log.warn("get client fail, break...")
return 0
if t.category == config.CATE_DOUBAN_NOTE:
if old:
start = Status.get_count_by_cate(t.category, t.user_id)
else:
start = 0
note_list = client.get_notes(start, 50)
if note_list:
for x in note_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(note_list)
elif t.category == config.CATE_DOUBAN_MINIBLOG:
if old:
start = Status.get_count_by_cate(t.category, t.user_id)
else:
start = 0
miniblog_list = client.get_miniblogs(start, 50)
if miniblog_list:
for x in miniblog_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(miniblog_list)
elif t.category == config.CATE_DOUBAN_STATUS:
origin_min_id = Status.get_min_origin_id(t.category, t.user_id)
if old:
log.info("will get douban status order than %s..." % origin_min_id)
status_list = client.get_timeline(until_id=origin_min_id)
else:
log.info("will get douban status newer than %s..." % origin_min_id)
status_list = client.get_timeline(since_id=origin_min_id, count=20)
if status_list:
log.info("get douban status succ, len is %s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
elif t.category == config.CATE_SINA_STATUS:
origin_min_id = Status.get_min_origin_id(t.category, t.user_id) #means the earliest id
origin_max_id = Status.get_max_origin_id(t.category, t.user_id) #meas the latest id
if old:
log.info("will get sinaweibo order than %s..." % origin_min_id)
status_list = client.get_timeline(until_id=origin_min_id)
## 如果根据max_id拿不到数据，那么根据page再fetch一次或者until_id - 1
if status_list and len(status_list) < 20 and origin_min_id is not None:
log.info("again will get sinaweibo order than %s..." % (int(origin_min_id)-1))
status_list = client.get_timeline(until_id=int(origin_min_id)-1)
else:
log.info("will get sinaweibo newer than %s..." % origin_max_id)
status_list = client.get_timeline(since_id=origin_max_id, count=50)
if status_list:
log.info("get sinaweibo succ, len is %s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_TWITTER_STATUS:
origin_min_id = Status.get_min_origin_id(t.category, t.user_id)
origin_max_id = Status.get_max_origin_id(t.category, t.user_id)
if old:
log.info("will get tweets order than %s..." % origin_min_id)
status_list = client.get_timeline(max_id=origin_min_id)
else:
log.info("will get tweets newer than %s..." % origin_max_id)
status_list = client.get_timeline(since_id=origin_max_id, count=50)
if status_list:
log.info("get tweets succ, len is %s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_QQWEIBO_STATUS:
if old:
oldest_create_time = Status.get_oldest_create_time(t.category, t.user_id)
log.info("will get qqweibo order than %s" % oldest_create_time)
if oldest_create_time is not None:
oldest_create_time = datetime2timestamp(oldest_create_time)
status_list = client.get_old_timeline(oldest_create_time, reqnum=200)
else:
log.info("will get qqweibo new timeline")
status_list = client.get_new_timeline(reqnum=20)
if status_list:
log.info("get qqweibo succ, result length is:%s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_RENREN_STATUS:
if old:
count = 100
total_count = Status.get_count_by_cate(t.category, t.user_id)
page = int(total_count / count) + 1
log.info("will get older renren status, page=%s, count=%s" %(page, count))
status_list = client.get_timeline(page, count)
else:
count = 20
page = 1
log.info("will get newest renren status, page=%s, count=%s" %(page, count))
status_list = client.get_timeline(page, count)
if status_list:
log.info("get renren status succ, result length is:%s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_RENREN_BLOG:
if old:
count = 50
total_count = Status.get_count_by_cate(t.category, t.user_id)
page = int(total_count / count) + 1
log.info("will get older renren blog, page=%s, count=%s" %(page, count))
blogs = client.get_blogs(page, count)
else:
count = 20
page = 1
log.info("will get newest renren blog, page=%s, count=%s" %(page, count))
blogs = client.get_blogs(page, count)
if blogs:
uid = blogs.get("uid")
blog_ids = filter(None, [v.get("id") for v in blogs.get("blogs", [])])
log.info("get renren blog ids succ, result length is:%s" % len(blog_ids))
for blog_id in blog_ids:
blog = client.get_blog(blog_id, uid)
if blog:
Status.add_from_obj(t.user_id, blog, json_encode(blog.get_data()))
return len(blog_ids)
elif t.category == config.CATE_RENREN_ALBUM:
status_list = client.get_albums()
if status_list:
log.info("get renren album succ, result length is:%s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_RENREN_PHOTO:
albums_ids = Status.get_ids(user_id=t.user_id, limit=1000, cate=config.CATE_RENREN_ALBUM)
albums = Status.gets(albums_ids)
if not albums:
return 0
for x in albums:
d = x.get_data()
if not d:
continue
aid = d.get_origin_id()
size = int(d.get_size())
count = 50
for i in xrange(1, size/count + 2):
status_list = client.get_photos(aid, i, count)
if status_list:
log.info("get renren photo of album %s succ, result length is:%s" \
% (aid, len(status_list)))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
elif t.category == config.CATE_INSTAGRAM_STATUS:
origin_min_id = Status.get_min_origin_id(t.category, t.user_id) #means the earliest id
origin_max_id = Status.get_max_origin_id(t.category, t.user_id) #means the latest id
if old:
log.info("will get instagram earlier than %s..." % origin_min_id)
status_list = client.get_timeline(max_id=origin_min_id)
else:
log.info("will get instagram later than %s..." % origin_max_id)
status_list = client.get_timeline(min_id=origin_max_id, count=50)
if status_list:
log.info("get instagram succ, len is %s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x, json_encode(x.get_data()))
return len(status_list)
except Exception, e:
print "---sync_exception_catched:", e
def timeline_json(uid, start):
limit = 50
start = int(start)
u = User.get(uid)
if not u:
abort(404, "no such user")
r = check_access_user(u)
if r:
abort(r[0], r[1])
ids = Status.get_ids(user_id=u.id, start=start, limit=limit, cate=g.cate)
status_list = Status.gets(ids)
if g.user and g.user.id == uid:
pass
elif g.user and g.user.id != uid:
status_list = [
x for x in status_list
if x.privacy() != consts.STATUS_PRIVACY_PRIVATE
]
elif not g.user:
status_list = [
x for x in status_list
if x.privacy() == consts.STATUS_PRIVACY_PUBLIC
]
if not status_list:
return json_encode({})
date = []
for s in status_list:
headline = s.title or s.summary or ""
text = s.text or ""
data = s.get_data()
images = data and data.get_images() or []
uri = s.get_origin_uri()
if uri:
headline = '<a href="%s" target="_blank">%s</a>' % (uri and uri[1],
headline)
if not (headline or text or images):
continue
t = s.create_time
if s.category in [config.CATE_DOUBAN_STATUS, config.CATE_SINA_STATUS]:
re_tweet = s.get_retweeted_data()
re_images = re_tweet and re_tweet.get_images() or []
images.extend(re_images)
text = re_tweet and re_tweet.get_content() or ''
if s.category in [config.CATE_DOUBAN_STATUS]:
atts = data and data.get_attachments()
if atts:
for att in atts:
text += att.get_title() + "\n" + att.get_description()
if s.category in [config.CATE_QQWEIBO_STATUS]:
text = s.get_retweeted_data() or ''
if s.category in [
config.CATE_WORDPRESS_POST,
config.CATE_THEPAST_NOTE,
config.CATE_RENREN_BLOG,
]:
uri = s.get_origin_uri()
headline = '<a href="%s" target="_blank">%s</a>' % (uri and uri[1],
s.title)
text = s.text or ''
if s.category in [
config.CATE_RENREN_STATUS, config.CATE_RENREN_PHOTO,
config.CATE_RENREN_ALBUM
]:
headline = s.title
text = s.text or ''
tmp = {
'startDate': t.strftime("%Y,%m,%d,%H,%M,%S"),
'headline': headline,
'text': text,
'asset': {
'media': images and images[0],
'credit': '',
'caption': ''
},
}
try:
json_encode(tmp)
date.append(tmp)
except:
pass
if date:
more = {
'startDate': (status_list[-1].create_time -
timedelta(0, 0, 1)).strftime("%Y,%m,%d,%H,%M,%S"),
'headline':
'<a href="/visual/%s?start=%s">查看更早的内容...</a>' %
(u.id, start + limit),
'text':
'',
'asset': {
'media': '',
'credit': '',
'caption': ''
},
}
date.append(more)
if start == 0:
cover = {
'startDate': datetime.now().strftime("%Y,%m,%d,%H,%M,%S"),
'headline': 'The past of you',
'text': 'Storytelling about yourself...',
'asset': {
'media': '',
'credit': '',
'caption': ''
},
}
date.insert(0, cover)
json_data = {
'timeline': {
'headline': 'The past of you',
'type': 'default',
'startDate': date[0]['startDate'],
'text': 'Storytelling about yourself...',
'asset': {
'media': '',
'credit': '',
'caption': ''
},
'date': date,
}
}
return json_encode(json_data)
sync_task = SyncTask.get(queue.task_id)
if not sync_task:
continue
## 现在不同步豆瓣日记
if str(sync_task.category) == str(config.CATE_DOUBAN_NOTE):
continue
## 同步wordpress rss
if str(sync_task.category) == str(config.CATE_WORDPRESS_POST):
jobs.sync_wordpress(sync_task)
queue.remove()
continue
max_sync_times = 0
min_id = Status.get_min_origin_id(sync_task.category,
sync_task.user_id)
if sync_task:
while True:
if max_sync_times >= 6:
break
r = jobs.sync(sync_task, old=True)
new_min_id = Status.get_min_origin_id(
sync_task.category, sync_task.user_id)
if r == 0 or new_min_id == min_id:
break
min_id = new_min_id
max_sync_times += 1
queue.remove()
time.sleep(5)
time.sleep(5)
except Exception, e:
def cmd_past(from_user, date_, msg_type="text"):
thepast_id = UserWeixin.get_by_weixin(from_user).user_id
now = datetime.datetime.now()
date_ = date_ or now.strftime("%m-%d")
date_ = "%s-%s" % (now.year + 1, date_)
try:
date_ = datetime.datetime.strptime(date_, "%Y-%m-%d")
except:
date_ = datetime.datetime(year=now.year + 1,
month=now.month,
day=now.day)
history_ids = get_status_ids_today_in_history(thepast_id, date_)
status_list = Status.gets(history_ids)
status_list = [
x for x in status_list if x.privacy() == consts.STATUS_PRIVACY_PUBLIC
]
r = ''
if msg_type == "text":
for x in status_list:
r += x.create_time.strftime(
"%Y-%m-%d %H:%M") + "\n" + x.text + "\n~~~~~~~~~~\n"
elif msg_type == "news":
article_count = min(len(status_list) + 1, 9)
r += "<ArticleCount>%s</ArticleCount>" % article_count
r += "<Articles>"
date_str = u"{m}月{d}日".format(m=date_.month, d=date_.day)
title0 = u"{d},找到{l}条往事,点击看更多".format(d=date_str, l=len(status_list))
r += u'''
<item>
<Title><![CDATA[{title0}]]></Title>
<Description><![CDATA[{desc0}]]></Description>
<PicUrl><![CDATA[{picurl0}]]></PicUrl>
<Url><![CDATA[{url0}]]></Url>
</item>
'''.format(title0=title0,
desc0="",
picurl0="",
url0="http://thepast.me/laiwei")
for i in range(1, article_count):
item_xml = '<item>'
s = status_list[i - 1]
s_data = s.get_data()
s_atts = s_data and s_data.get_attachments() or []
s_images = s_data and s_data.get_images() or []
s_re = s.get_retweeted_data()
s_re_atts = s_re and s_re.get_attachments() or []
s_re_images = s_re and s_re.get_images() or []
s_re_user = s_re and s_re.get_user() or ""
s_re_user_nickname = s_re_user if isinstance(
s_re_user, basestring) else s_re_user.get_nickname()
title = s.title
desc = s.create_time.strftime("%Y-%m-%d %H:%M")
desc += "\n" + s.summary
for att in s_atts:
desc += "\n" + att.get_href()
desc += "\n" + att.get_description()
if s_re_user_nickname and s_re.get_content():
desc += "\n//@" + s_re_user_nickname + ":" + s_re.get_content()
for att in s_re_atts:
desc += "\n" + att.get_href()
desc += "\n" + att.get_description()
s_images.extend(s_re_images)
pic_url = ""
if s_images:
pic_url = s_images[0]
s_from = s.get_origin_uri()
url = s_from and s_from[1] or pic_url
item_xml += "<Title><![CDATA[" + title + desc + "]]></Title>"
item_xml += "<Description><![CDATA[" + title + desc + "]]></Description>"
item_xml += "<PicUrl><![CDATA[" + pic_url + "]]></PicUrl>"
item_xml += "<Url><![CDATA[" + url + "]]></Url>"
item_xml += '</item>'
r += item_xml
r += "</Articles>"
return r
from past.model.status import TaskQueue, SyncTask, Status
from past import config
if __name__ == "__main__":
while True:
try:
queue_ids = TaskQueue.get_all_ids()
print 'queue length:', len(queue_ids)
for qid in queue_ids:
queue = TaskQueue.get(qid)
if queue and queue.task_kind == config.K_SYNCTASK:
print 'syncing task id:', queue.task_id
sync_task = SyncTask.get(queue.task_id)
max_sync_times = 0
min_id = Status.get_min_origin_id(sync_task.category, sync_task.user_id)
if sync_task:
while True:
if max_sync_times >= 3:
break
r = jobs.sync(sync_task, old=True)
new_min_id = Status.get_min_origin_id(sync_task.category, sync_task.user_id)
if r == 0 or new_min_id == min_id:
break
min_id = new_min_id
max_sync_times += 1
queue.remove()
time.sleep(5)
time.sleep(30)
except Exception, e:
print '----except: %s' %e
def sync(t, old=False):
if not t:
print 'no such task'
return 0
log.info("the sync task is :%s" % t)
try:
alias = None
provider = category2provider(t.category)
if provider == config.OPENID_DOUBAN:
alias = UserAlias.get_by_user_and_type(
t.user_id, config.OPENID_TYPE_DICT[config.OPENID_DOUBAN])
elif provider == config.OPENID_SINA:
alias = UserAlias.get_by_user_and_type(
t.user_id, config.OPENID_TYPE_DICT[config.OPENID_SINA])
elif provider == config.OPENID_TWITTER:
alias = UserAlias.get_by_user_and_type(
t.user_id, config.OPENID_TYPE_DICT[config.OPENID_TWITTER])
elif provider == config.OPENID_QQ:
alias = UserAlias.get_by_user_and_type(
t.user_id, config.OPENID_TYPE_DICT[config.OPENID_QQ])
if not alias:
log.warn("no alias...")
return 0
token = OAuth2Token.get(alias.id)
if not token:
log.warn("no access token, break...")
return 0
client = None
if provider == config.OPENID_DOUBAN:
client = Douban(alias.alias, token.access_token,
token.refresh_token)
elif provider == config.OPENID_SINA:
client = SinaWeibo(alias.alias, token.access_token)
elif provider == config.OPENID_TWITTER:
client = Twitter(alias.alias)
elif provider == config.OPENID_QQ:
client = QQWeibo(alias.alias)
if not client:
log.warn("get client fail, break...")
return 0
if t.category == config.CATE_DOUBAN_NOTE:
if old:
start = Status.get_count_by_cate(t.category, t.user_id)
else:
start = 0
note_list = client.get_notes(start, 50)
if note_list:
for x in note_list:
Status.add_from_obj(t.user_id, x,
json_encode(x.get_data()))
return len(note_list)
elif t.category == config.CATE_DOUBAN_MINIBLOG:
if old:
start = Status.get_count_by_cate(t.category, t.user_id)
else:
start = 0
miniblog_list = client.get_miniblogs(start, 50)
if miniblog_list:
for x in miniblog_list:
Status.add_from_obj(t.user_id, x,
json_encode(x.get_data()))
return len(miniblog_list)
elif t.category == config.CATE_DOUBAN_STATUS:
origin_min_id = Status.get_min_origin_id(t.category, t.user_id)
if old:
log.info("will get douban status order than %s..." %
origin_min_id)
status_list = client.get_timeline(until_id=origin_min_id)
else:
log.info("will get douban status newer than %s..." %
origin_min_id)
status_list = client.get_timeline(since_id=origin_min_id,
count=20)
if status_list:
log.info("get douban status succ, len is %s" %
len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x,
json_encode(x.get_data()))
elif t.category == config.CATE_SINA_STATUS:
origin_min_id = Status.get_min_origin_id(t.category,
t.user_id) #means max_id
if old:
log.info("will get sinaweibo order than %s..." % origin_min_id)
status_list = client.get_timeline(until_id=origin_min_id)
## 如果根据max_id拿不到数据，那么根据page再fetch一次或者until_id - 1
if len(status_list) < 20:
log.info("again will get sinaweibo order than %s..." %
(int(origin_min_id) - 1))
status_list = client.get_timeline(
until_id=int(origin_min_id) - 1)
#if len(status_list) <= 1:
# downloaded_status_count = Status.get_count_by_user(t.user_id)
# page = downloaded_status_count / 100 + 2
# status_list = client.get_timeline_by_page(page=page)
else:
log.info("will get sinaweibo newer than %s..." % origin_min_id)
status_list = client.get_timeline(since_id=origin_min_id,
count=20)
if status_list:
log.info("get sinaweibo succ, len is %s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x,
json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_TWITTER_STATUS:
origin_min_id = Status.get_min_origin_id(t.category, t.user_id)
if old:
log.info("will get tweets order than %s..." % origin_min_id)
status_list = client.get_timeline(max_id=origin_min_id)
else:
log.info("will get tweets newer than %s..." % origin_min_id)
status_list = client.get_timeline(since_id=origin_min_id,
count=20)
if status_list:
log.info("get tweets succ, len is %s" % len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x,
json_encode(x.get_data()))
return len(status_list)
elif t.category == config.CATE_QQWEIBO_STATUS:
if old:
oldest_create_time = Status.get_oldest_create_time(
t.category, t.user_id)
log.info("will get qqweibo order than %s" % oldest_create_time)
if oldest_create_time is not None:
oldest_create_time = datetime2timestamp(oldest_create_time)
status_list = client.get_old_timeline(oldest_create_time,
reqnum=200)
else:
log.info("will get qqweibo new timeline")
status_list = client.get_new_timeline(reqnum=20)
if status_list:
log.info("get qqweibo succ, result length is:%s" %
len(status_list))
for x in status_list:
Status.add_from_obj(t.user_id, x,
json_encode(x.get_data()))
return len(status_list)
except:
import traceback
print traceback.format_exc()
return 0