def GET(self): mc_user = mc.get('users') if mc_user: users = mc_user else: users = get_users(active_users) has_error = False allrss = [] for user in users: try: rss = [] rss_url = user.rss_url data = urllib.urlopen(rss_url) data = data.read() xml = minidom.parseString(data) items = xml.getElementsByTagName('item')[:4] for item in items: rss_item = Rss() title = get_xml_text(item.getElementsByTagName('title')[0].childNodes) date = get_xml_text(item.getElementsByTagName('pubDate')[0].childNodes) link = get_xml_text(item.getElementsByTagName('link')[0].childNodes) rss_item.title = title.encode('utf-8') rss_item.date = date.split('+')[0].encode('utf-8') rss_item.link = link.encode('utf-8') rss.append(rss_item) allrss.append(rss_item) user.rss = rss except: has_error = True user.rss = None if not has_error: mc.set('users', users, 3600 * 12) return st('blog.mako', **locals())
def rss_feed_update(res, id, user_id, limit=None): rss = Rss.mc_get(id) zsite = Zsite.mc_get(user_id) for count, i in enumerate(res): if limit: if count > limit: break r = link_title_uid_txt(i) if not r: continue link, title, rss_uid, txt = r if '相片: ' in title: continue title_txt = '%s\n%s' % (title, txt) if duplicator_rss.txt_is_duplicate(title_txt): continue #print title, link, duplicator_rss.txt_is_duplicate(title_txt) if rss.auto: state = RSS_PRE_PO else: state = RSS_UNCHECK c = RssPo.raw_sql( 'insert into rss_po (user_id,rss_id,rss_uid,title,txt,link,state) values (%s,%s,%s,%s,%s,%s,%s) on duplicate key update title=%s , txt=%s ', user_id, id, rss_uid, title, txt, link, state, title, txt) duplicator_rss.set_record(title_txt, c.lastrowid)
def get(self, id=0): if id: id = int(id) rss = Rss.mc_get(id) self.render(rss=rss) else: self.render()
def rss_feed_update(res, id, user_id, limit=None): rss = Rss.mc_get(id) zsite = Zsite.mc_get(user_id) for count , i in enumerate(res): if limit: if count > limit: break r = link_title_uid_txt(i) if not r: continue link, title, rss_uid, txt = r if '相片: ' in title: continue title_txt = '%s\n%s'%(title, txt) if duplicator_rss.txt_is_duplicate(title_txt): continue #print title, link, duplicator_rss.txt_is_duplicate(title_txt) if rss.auto: state = RSS_PRE_PO else: state = RSS_UNCHECK c = RssPo.raw_sql( 'insert into rss_po (user_id,rss_id,rss_uid,title,txt,link,state) values (%s,%s,%s,%s,%s,%s,%s) on duplicate key update title=%s , txt=%s ', user_id, id, rss_uid, title, txt, link, state, title, txt ) duplicator_rss.set_record(title_txt, c.lastrowid)
def unread_feed_update(greader, feed): rs = Rss.raw_sql('select id,user_id from rss where url = %s', feed[5:]).fetchone() if rs: id, user_id = rs res = greader.unread(feed) rss_feed_update(res, id , user_id)
def get_douban_site(): with open('data/intro') as i: intro = json.loads(i.read()) with open('data/info') as i: info = json.loads(i.read()) with open('data/meta') as i: meta = json.loads(i.read()) #zs = ZsiteLink.where(link='http://site.douban.com/110633/(1号厅的光影传奇)') #if zs: # zs = zs[0] # zs.link = 'http://site.douban.com/110633/' # zs.save() for zl in ZsiteLink.where(name='豆瓣小站').order_by('id desc').col_list(col='zsite_id'): if not Rss.where(user_id=zl): zs = ZsiteLink.raw_sql('select link from zsite_link where link like %s and zsite_id=%s and cid=2', 'http://site.douban.com%', zl).fetchone() if zs: id = zs[0].split('/')[-1] or zs[0].split('/')[-2] if info.get(id): like, link, img, name = info.get(id) if meta.get(id): motto = meta.get(id)[0][0] motto = motto.split('<br />')[0] motto = motto.split('<a')[0] img_src = meta.get(id)[0][1] rss_new(zl, 'http://rss-tidy.42qu.com/douban/site/%s'%id, name, link, auto=1) print zl, 'http://site.douban.com/%s'%id else: print id, 'no motto data' else: print id, '数据未录入', zl
def post(self, id): rss = Rss.mc_get(id) next = self.get_argument('next', None) or '/rss/index' #url = self.get_argument('url', None) #link = self.get_argument('link', None) #user_id = self.get_argument('user_id', None) #name = self.get_argument('name', None) url , link , user_id , name , auto = _rss_post_argument(self) if url: rss.url = url if link: rss.link = link if name: rss.name = name if user_id: rss.user_id = user_id rss.auto = int(bool(auto)) rss.save() self.redirect(next)
def unread_feed_update(greader, feed): rs = Rss.raw_sql('select id,user_id from rss where url = %s', feed[5:]).fetchone() if rs: id, user_id = rs res = greader.unread(feed) rss_feed_update(res, id, user_id)
def get(self, id): id = int(id) rss = Rss.mc_get(id) if not rss.gid: rss.delete() if rss.gid > 0: rss.gid = -rss.gid rss.save() self.redirect('/rss/gid/1')
def check(): s = set() for r in Rss.where(): s.add(r.url.split('/')[-1] or r.url.split('/')[-2]) with open('makeexcel.txt') as me: for i in me: i = i.strip() if i not in s: print r.user_id get_in(i) time.sleep(1) else: count = po_cid_count_by_zsite_id(r.user_id, CID_NOTE) if count == 0: zsite_show_rm(Zsite.mc_get(r.user_id)) #zsite_fav_rm_all_by_zsite_id(r.user_id) zsite_admin_empty(r.user_id) print r.user_id, '!!' print i
def post(self): url , link , user_id , name , auto = _rss_post_argument(self) rss = Rss.mc_get(id) if rss: if url: rss.url = url if link: rss.link = link if user_id: rss.user_id = user_id if name: rss.name = name rss.auto = int(bool(auto)) rss.save()
def recover(): rec = [] for z in Zsite.where(cid=3, state=0): if Rss.get(user_id=z.id): rec.append(z) # print rec rec_name = [i.name for i in rec] # print rec_name rm = [] for z in Zsite.where(cid=3, state=40): if z.name in rec_name: rm.append(z) for z in rec: z.state = 40 zsite_admin_new(z.id, 10017321) z.save() zsite_show_new(z.id, 3) for z in rm: zsite_site_rm(z.id)
def get_douban_site(): with open('data/intro') as i: intro = json.loads(i.read()) with open('data/info') as i: info = json.loads(i.read()) with open('data/meta') as i: meta = json.loads(i.read()) #zs = ZsiteLink.where(link='http://site.douban.com/110633/(1号厅的光影传奇)') #if zs: # zs = zs[0] # zs.link = 'http://site.douban.com/110633/' # zs.save() for zl in ZsiteLink.where(name='豆瓣小站').order_by('id desc').col_list( col='zsite_id'): if not Rss.where(user_id=zl): zs = ZsiteLink.raw_sql( 'select link from zsite_link where link like %s and zsite_id=%s and cid=2', 'http://site.douban.com%', zl).fetchone() if zs: id = zs[0].split('/')[-1] or zs[0].split('/')[-2] if info.get(id): like, link, img, name = info.get(id) if meta.get(id): motto = meta.get(id)[0][0] motto = motto.split('<br />')[0] motto = motto.split('<a')[0] img_src = meta.get(id)[0][1] rss_new(zl, 'http://rss-tidy.42qu.com/douban/site/%s' % id, name, link, auto=1) print zl, 'http://site.douban.com/%s' % id else: print id, 'no motto data' else: print id, '数据未录入', zl
txt = get_rss_link_title_by_rss(j)[-1] except: continue else: output.write("""<outline text="%s" title="%s" type="rss" xmlUrl="%s" htmlUrl="%s"/> """ % (txt, txt, j, j)) output.write("""</body></opml>""") def print_uri(): links = get_uri() with open('rss2user_id.txt', 'w') as f: for i in links: f.write('%s %s\n' % (i[0], i[1])) if __name__ == '__main__': #print_uri() #print_rss() #print_xml() from model.rss import Rss from model.zsite import Zsite for i in Rss.where(): zsite = Zsite.mc_get( i.user_id ) if zsite.cid == CID_SITE: print 'http:%s'%zsite.link, '\t\t\t', zsite.name
def rss_subscribe(greader=None): from zkit.google.findrss import get_rss_link_title_by_url rss_list = [] for i in Rss.where(gid=0): url = i.url.strip() #print url if not all((i.link, i.url, i.name)): rss, link, name = get_rss_link_title_by_url(url) #print link, name if rss: i.url = rss if link: i.link = link if not name: name = link.split('://', 1)[-1] if name: i.name = name i.save() rss_list.append(i) if rss_list: if greader is None: greader = Reader(GREADER_USERNAME, GREADER_PASSWORD) for i in rss_list: #print i.url url = quote(i.url) try: greader.subscribe(url) i.gid = 1 i.save() except: traceback.print_exc() print i.url, i.user_id i.delete() try: #print i.url feed = 'feed/%s'%url user_id = i.user_id duplicator_set_by_user_id(user_id) rss_feed_update(greader.feed(feed), i.id, user_id, 1024) # greader.mark_as_read(feed) except: traceback.print_exc() for i in Rss.where('gid<0'): if greader is None: greader = Reader(GREADER_USERNAME, GREADER_PASSWORD) try: greader.unsubscribe('feed/'+quote(i.url)) except: traceback.print_exc() print i.url, i.user_id i.delete()
def get(self, id): rss = Rss.mc_get(id) next = self.request.headers.get('Referer', '') self.render(rss=rss, next=next)
def rss_subscribe(greader=None): from zkit.google.findrss import get_rss_link_title_by_url rss_list = [] for i in Rss.where(gid=0): url = i.url.strip() #print url if not all((i.link, i.url, i.name)): rss, link, name = get_rss_link_title_by_url(url) #print link, name if rss: i.url = rss if link: i.link = link if not name: name = link.split('://', 1)[-1] if name: i.name = name i.save() rss_list.append(i) if rss_list: if greader is None: greader = Reader(GREADER_USERNAME, GREADER_PASSWORD) for i in rss_list: #print i.url url = quote(i.url) try: greader.subscribe(url) i.gid = 1 i.save() except: traceback.print_exc() print i.url, i.user_id i.delete() try: #print i.url feed = 'feed/%s' % url user_id = i.user_id duplicator_set_by_user_id(user_id) rss_feed_update(greader.feed(feed), i.id, user_id, 1024) # greader.mark_as_read(feed) except: traceback.print_exc() for i in Rss.where('gid<0'): if greader is None: greader = Reader(GREADER_USERNAME, GREADER_PASSWORD) try: greader.unsubscribe('feed/' + quote(i.url)) except: traceback.print_exc() print i.url, i.user_id i.delete()