def get(self): website = Setting().all().fetch(1) if 0 == len(website): w = Setting() w.webSite = "http://im.zxxsbook.com" w.put() website.append(w) web = website[0] self.searchRSS(web.webSite + "/RssSource")
def get(self): web = None syncerrornote = memcache.get("syncerrnote") if syncerrornote: try: website = Setting().all().fetch(1) if len(website) == 0: web = Setting() web.webSite = "http://im.zxxsbook.com" web.put() else: web = website[0] web.put() except: return content = Contents().all().filter("status =", "3").order("updateTime").fetch(3) if not web: website = Setting().all().fetch(1) if len(website) == 0: web = Setting() web.webSite = "http://im.zxxsbook.com" web.put() else: web = website[0] for c in content: piclist = memcache.get("piclist" + str(c.key().id())) if not piclist: piclist = Picture().all().filter("content =", c).fetch(20) else: piclist = piclist["list"] pam = {} for i in range(len(piclist)): img = piclist[i] # n=img.src.split('/') # pam['imgName'+str(i)]=n[-1] pam["img" + str(i)] = img.src pam["code"] = c.rss.code pam["title"] = c.title.encode("utf-8") pam["content"] = c.realContent.encode("utf-8") pam["username"] = setting.adminname # if c.rcode: # pam['rcode']=c.rcode.encode('utf-8') login_url = web.webSite + "/RssMsg" login_data = urllib.urlencode(pam) result = urlfetch.fetch( url=login_url, payload=login_data, method=urlfetch.POST, headers={ "Content-Type": "application/x-www-form-urlencoded", "User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", }, follow_redirects=False, deadline=50, ) if result.status_code == 200: c.code = result.content c.status = "4" try: c.put() memcache.set("syncerrnote", False, 3600) except: memcache.set("syncerrnote", True, 3600) else: logging.error("send news failure !" + c.link) logging.info("syncurl:" + login_url)
def handle_result(self, rpc, rssSources, url): try: result = rpc.get_result() if result.status_code == 200: html = result.content rsslist = feedparser.parse(html) alist = [] for rss in rsslist.entries: try: alist.append((rss.title, rss.link, rss.description)) except Exception, e: logging.error("rss jiexi" + str(e)) newContent = [] contentobj = ContentsObj().all().filter("rss =", rssSources).fetch(1) if not contentobj: contentobj = ContentsObj() contentobj.rss = rssSources else: contentobj = contentobj[0] nowNewsList = json.loads(contentobj.contentdata) for title, link, desc in alist: link = link.strip() if link not in nowNewsList: newContent.append((link, title, desc, [])) rpcs = [] notediclist = [] for linkyuanzu in newContent: pam = {} rpc = urlfetch.create_rpc(deadline=50) rpc.callback = self.rpc2_callback(rpc, rssSources, linkyuanzu, nowNewsList[-10:], pam) urlfetch.make_fetch_call( rpc, linkyuanzu[0], headers={"User-Agent": "Mozilla/5.0"}, follow_redirects=True ) rpcs.append(rpc) notediclist.append(pam) for rpc in rpcs: rpc.wait() website = Setting().all().fetch(1) if len(website) == 0: web = Setting() web.webSite = "http://localhoat:8888" web.put() else: web = website[0] allpam = {} tempNewList = [] allpam["notenum"] = str(len(notediclist)) for i, pam in enumerate(notediclist): if pam: tempNewList.append(newContent[i][0]) if not rssSources.isfilterImg: for num in range(pam["imgnum"]): allpam[str(i) + "img" + str(num)] = pam["img" + str(num)] allpam[str(i) + "code"] = pam["code"] allpam[str(i) + "title"] = pam["title"] allpam[str(i) + "content"] = pam["content"] allpam[str(i) + "username"] = pam["username"] if not notediclist: return login_url = web.webSite + "/RssMsg" login_data = urllib.urlencode(allpam) # logging.info('pam:'+str(allpam)) result = urlfetch.fetch( url=login_url, payload=login_data, method=urlfetch.POST, headers={ "Content-Type": "application/x-www-form-urlencoded", "User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", }, follow_redirects=False, deadline=50, ) if result.status_code == 200: nowNewsList.extend(tempNewList) contentobj.contentdata = json.dumps(nowNewsList[-60:]) contentobj.put() else: # linkyuanzu[3].append(False) logging.error("send news failure !" + linkyuanzu[0]) # for linkyuanzu in newContent: # if linkyuanzu[3]: # nowNewsList.append(linkyuanzu[0]) except Exception, e: logging.error("0000" + str(e) + url)