def GET(self):
mc_user = mc.get('users')
if mc_user:
users = mc_user
else:
users = get_users(active_users)
has_error = False
allrss = []
for user in users:
try:
rss = []
rss_url = user.rss_url
data = urllib.urlopen(rss_url)
data = data.read()
xml = minidom.parseString(data)
items = xml.getElementsByTagName('item')[:4]
for item in items:
rss_item = Rss()
title = get_xml_text(item.getElementsByTagName('title')[0].childNodes)
date = get_xml_text(item.getElementsByTagName('pubDate')[0].childNodes)
link = get_xml_text(item.getElementsByTagName('link')[0].childNodes)
rss_item.title = title.encode('utf-8')
rss_item.date = date.split('+')[0].encode('utf-8')
rss_item.link = link.encode('utf-8')
rss.append(rss_item)
allrss.append(rss_item)
user.rss = rss
except:
has_error = True
user.rss = None
if not has_error:
mc.set('users', users, 3600 * 12)
return st('blog.mako', **locals())
def rss_feed_update(res, id, user_id, limit=None):
rss = Rss.mc_get(id)
zsite = Zsite.mc_get(user_id)
for count, i in enumerate(res):
if limit:
if count > limit:
break
r = link_title_uid_txt(i)
if not r:
continue
link, title, rss_uid, txt = r
if '相片: ' in title:
continue
title_txt = '%s\n%s' % (title, txt)
if duplicator_rss.txt_is_duplicate(title_txt):
continue
#print title, link, duplicator_rss.txt_is_duplicate(title_txt)
if rss.auto:
state = RSS_PRE_PO
else:
state = RSS_UNCHECK
c = RssPo.raw_sql(
'insert into rss_po (user_id,rss_id,rss_uid,title,txt,link,state) values (%s,%s,%s,%s,%s,%s,%s) on duplicate key update title=%s , txt=%s ',
user_id, id, rss_uid, title, txt, link, state, title, txt)
duplicator_rss.set_record(title_txt, c.lastrowid)
def get(self, id=0):
if id:
id = int(id)
rss = Rss.mc_get(id)
self.render(rss=rss)
else:
self.render()
def rss_feed_update(res, id, user_id, limit=None):
rss = Rss.mc_get(id)
zsite = Zsite.mc_get(user_id)
for count , i in enumerate(res):
if limit:
if count > limit:
break
r = link_title_uid_txt(i)
if not r:
continue
link, title, rss_uid, txt = r
if '相片: ' in title:
continue
title_txt = '%s\n%s'%(title, txt)
if duplicator_rss.txt_is_duplicate(title_txt):
continue
#print title, link, duplicator_rss.txt_is_duplicate(title_txt)
if rss.auto:
state = RSS_PRE_PO
else:
state = RSS_UNCHECK
c = RssPo.raw_sql(
'insert into rss_po (user_id,rss_id,rss_uid,title,txt,link,state) values (%s,%s,%s,%s,%s,%s,%s) on duplicate key update title=%s , txt=%s ',
user_id, id, rss_uid, title, txt, link, state,
title, txt
)
duplicator_rss.set_record(title_txt, c.lastrowid)
def unread_feed_update(greader, feed):
rs = Rss.raw_sql('select id,user_id from rss where url = %s', feed[5:]).fetchone()
if rs:
id, user_id = rs
res = greader.unread(feed)
rss_feed_update(res, id , user_id)
def get_douban_site():
with open('data/intro') as i:
intro = json.loads(i.read())
with open('data/info') as i:
info = json.loads(i.read())
with open('data/meta') as i:
meta = json.loads(i.read())
#zs = ZsiteLink.where(link='http://site.douban.com/110633/（1号厅的光影传奇）')
#if zs:
# zs = zs[0]
# zs.link = 'http://site.douban.com/110633/'
# zs.save()
for zl in ZsiteLink.where(name='豆瓣小站').order_by('id desc').col_list(col='zsite_id'):
if not Rss.where(user_id=zl):
zs = ZsiteLink.raw_sql('select link from zsite_link where link like %s and zsite_id=%s and cid=2', 'http://site.douban.com%', zl).fetchone()
if zs:
id = zs[0].split('/')[-1] or zs[0].split('/')[-2]
if info.get(id):
like, link, img, name = info.get(id)
if meta.get(id):
motto = meta.get(id)[0][0]
motto = motto.split('<br />')[0]
motto = motto.split('<a')[0]
img_src = meta.get(id)[0][1]
rss_new(zl, 'http://rss-tidy.42qu.com/douban/site/%s'%id, name, link, auto=1)
print zl, 'http://site.douban.com/%s'%id
else:
print id, 'no motto data'
else:
print id, '数据未录入', zl
def post(self, id):
rss = Rss.mc_get(id)
next = self.get_argument('next', None) or '/rss/index'
#url = self.get_argument('url', None)
#link = self.get_argument('link', None)
#user_id = self.get_argument('user_id', None)
#name = self.get_argument('name', None)
url , link , user_id , name , auto = _rss_post_argument(self)
if url:
rss.url = url
if link:
rss.link = link
if name:
rss.name = name
if user_id:
rss.user_id = user_id
rss.auto = int(bool(auto))
rss.save()
self.redirect(next)
def unread_feed_update(greader, feed):
rs = Rss.raw_sql('select id,user_id from rss where url = %s',
feed[5:]).fetchone()
if rs:
id, user_id = rs
res = greader.unread(feed)
rss_feed_update(res, id, user_id)
def get(self, id):
id = int(id)
rss = Rss.mc_get(id)
if not rss.gid:
rss.delete()
if rss.gid > 0:
rss.gid = -rss.gid
rss.save()
self.redirect('/rss/gid/1')
def check():
s = set()
for r in Rss.where():
s.add(r.url.split('/')[-1] or r.url.split('/')[-2])
with open('makeexcel.txt') as me:
for i in me:
i = i.strip()
if i not in s:
print r.user_id
get_in(i)
time.sleep(1)
else:
count = po_cid_count_by_zsite_id(r.user_id, CID_NOTE)
if count == 0:
zsite_show_rm(Zsite.mc_get(r.user_id))
#zsite_fav_rm_all_by_zsite_id(r.user_id)
zsite_admin_empty(r.user_id)
print r.user_id, '!!'
print i
def post(self):
url , link , user_id , name , auto = _rss_post_argument(self)
rss = Rss.mc_get(id)
if rss:
if url:
rss.url = url
if link:
rss.link = link
if user_id:
rss.user_id = user_id
if name:
rss.name = name
rss.auto = int(bool(auto))
rss.save()
def check():
s = set()
for r in Rss.where():
s.add(r.url.split('/')[-1] or r.url.split('/')[-2])
with open('makeexcel.txt') as me:
for i in me:
i = i.strip()
if i not in s:
print r.user_id
get_in(i)
time.sleep(1)
else:
count = po_cid_count_by_zsite_id(r.user_id, CID_NOTE)
if count == 0:
zsite_show_rm(Zsite.mc_get(r.user_id))
#zsite_fav_rm_all_by_zsite_id(r.user_id)
zsite_admin_empty(r.user_id)
print r.user_id, '!!'
print i
def recover():
rec = []
for z in Zsite.where(cid=3, state=0):
if Rss.get(user_id=z.id):
rec.append(z)
# print rec
rec_name = [i.name for i in rec]
# print rec_name
rm = []
for z in Zsite.where(cid=3, state=40):
if z.name in rec_name:
rm.append(z)
for z in rec:
z.state = 40
zsite_admin_new(z.id, 10017321)
z.save()
zsite_show_new(z.id, 3)
for z in rm:
zsite_site_rm(z.id)
def recover():
rec = []
for z in Zsite.where(cid=3, state=0):
if Rss.get(user_id=z.id):
rec.append(z)
# print rec
rec_name = [i.name for i in rec]
# print rec_name
rm = []
for z in Zsite.where(cid=3, state=40):
if z.name in rec_name:
rm.append(z)
for z in rec:
z.state = 40
zsite_admin_new(z.id, 10017321)
z.save()
zsite_show_new(z.id, 3)
for z in rm:
zsite_site_rm(z.id)
def get_douban_site():
with open('data/intro') as i:
intro = json.loads(i.read())
with open('data/info') as i:
info = json.loads(i.read())
with open('data/meta') as i:
meta = json.loads(i.read())
#zs = ZsiteLink.where(link='http://site.douban.com/110633/（1号厅的光影传奇）')
#if zs:
# zs = zs[0]
# zs.link = 'http://site.douban.com/110633/'
# zs.save()
for zl in ZsiteLink.where(name='豆瓣小站').order_by('id desc').col_list(
col='zsite_id'):
if not Rss.where(user_id=zl):
zs = ZsiteLink.raw_sql(
'select link from zsite_link where link like %s and zsite_id=%s and cid=2',
'http://site.douban.com%', zl).fetchone()
if zs:
id = zs[0].split('/')[-1] or zs[0].split('/')[-2]
if info.get(id):
like, link, img, name = info.get(id)
if meta.get(id):
motto = meta.get(id)[0][0]
motto = motto.split('<br />')[0]
motto = motto.split('<a')[0]
img_src = meta.get(id)[0][1]
rss_new(zl,
'http://rss-tidy.42qu.com/douban/site/%s' % id,
name,
link,
auto=1)
print zl, 'http://site.douban.com/%s' % id
else:
print id, 'no motto data'
else:
print id, '数据未录入', zl
txt = get_rss_link_title_by_rss(j)[-1]
except:
continue
else:
output.write("""<outline text="%s"
title="%s"
type="rss" xmlUrl="%s"
htmlUrl="%s"/>
""" % (txt, txt, j, j))
output.write("""</body></opml>""")
def print_uri():
links = get_uri()
with open('rss2user_id.txt', 'w') as f:
for i in links:
f.write('%s %s\n' % (i[0], i[1]))
if __name__ == '__main__':
#print_uri()
#print_rss()
#print_xml()
from model.rss import Rss
from model.zsite import Zsite
for i in Rss.where():
zsite = Zsite.mc_get( i.user_id )
if zsite.cid == CID_SITE:
print 'http:%s'%zsite.link, '\t\t\t', zsite.name
def rss_subscribe(greader=None):
from zkit.google.findrss import get_rss_link_title_by_url
rss_list = []
for i in Rss.where(gid=0):
url = i.url.strip()
#print url
if not all((i.link, i.url, i.name)):
rss, link, name = get_rss_link_title_by_url(url)
#print link, name
if rss:
i.url = rss
if link:
i.link = link
if not name:
name = link.split('://', 1)[-1]
if name:
i.name = name
i.save()
rss_list.append(i)
if rss_list:
if greader is None:
greader = Reader(GREADER_USERNAME, GREADER_PASSWORD)
for i in rss_list:
#print i.url
url = quote(i.url)
try:
greader.subscribe(url)
i.gid = 1
i.save()
except:
traceback.print_exc()
print i.url, i.user_id
i.delete()
try:
#print i.url
feed = 'feed/%s'%url
user_id = i.user_id
duplicator_set_by_user_id(user_id)
rss_feed_update(greader.feed(feed), i.id, user_id, 1024)
# greader.mark_as_read(feed)
except:
traceback.print_exc()
for i in Rss.where('gid<0'):
if greader is None:
greader = Reader(GREADER_USERNAME, GREADER_PASSWORD)
try:
greader.unsubscribe('feed/'+quote(i.url))
except:
traceback.print_exc()
print i.url, i.user_id
i.delete()
def get(self, id):
rss = Rss.mc_get(id)
next = self.request.headers.get('Referer', '')
self.render(rss=rss,
next=next)
def rss_subscribe(greader=None):
from zkit.google.findrss import get_rss_link_title_by_url
rss_list = []
for i in Rss.where(gid=0):
url = i.url.strip()
#print url
if not all((i.link, i.url, i.name)):
rss, link, name = get_rss_link_title_by_url(url)
#print link, name
if rss:
i.url = rss
if link:
i.link = link
if not name:
name = link.split('://', 1)[-1]
if name:
i.name = name
i.save()
rss_list.append(i)
if rss_list:
if greader is None:
greader = Reader(GREADER_USERNAME, GREADER_PASSWORD)
for i in rss_list:
#print i.url
url = quote(i.url)
try:
greader.subscribe(url)
i.gid = 1
i.save()
except:
traceback.print_exc()
print i.url, i.user_id
i.delete()
try:
#print i.url
feed = 'feed/%s' % url
user_id = i.user_id
duplicator_set_by_user_id(user_id)
rss_feed_update(greader.feed(feed), i.id, user_id, 1024)
# greader.mark_as_read(feed)
except:
traceback.print_exc()
for i in Rss.where('gid<0'):
if greader is None:
greader = Reader(GREADER_USERNAME, GREADER_PASSWORD)
try:
greader.unsubscribe('feed/' + quote(i.url))
except:
traceback.print_exc()
print i.url, i.user_id
i.delete()
