def get(self):
website = Setting().all().fetch(1)
if 0 == len(website):
w = Setting()
w.webSite = "http://im.zxxsbook.com"
w.put()
website.append(w)
web = website[0]
self.searchRSS(web.webSite + "/RssSource")
def get(self):
web = None
syncerrornote = memcache.get("syncerrnote")
if syncerrornote:
try:
website = Setting().all().fetch(1)
if len(website) == 0:
web = Setting()
web.webSite = "http://im.zxxsbook.com"
web.put()
else:
web = website[0]
web.put()
except:
return
content = Contents().all().filter("status =", "3").order("updateTime").fetch(3)
if not web:
website = Setting().all().fetch(1)
if len(website) == 0:
web = Setting()
web.webSite = "http://im.zxxsbook.com"
web.put()
else:
web = website[0]
for c in content:
piclist = memcache.get("piclist" + str(c.key().id()))
if not piclist:
piclist = Picture().all().filter("content =", c).fetch(20)
else:
piclist = piclist["list"]
pam = {}
for i in range(len(piclist)):
img = piclist[i]
# n=img.src.split('/')
# pam['imgName'+str(i)]=n[-1]
pam["img" + str(i)] = img.src
pam["code"] = c.rss.code
pam["title"] = c.title.encode("utf-8")
pam["content"] = c.realContent.encode("utf-8")
pam["username"] = setting.adminname
# if c.rcode:
# pam['rcode']=c.rcode.encode('utf-8')
login_url = web.webSite + "/RssMsg"
login_data = urllib.urlencode(pam)
result = urlfetch.fetch(
url=login_url,
payload=login_data,
method=urlfetch.POST,
headers={
"Content-Type": "application/x-www-form-urlencoded",
"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
},
follow_redirects=False,
deadline=50,
)
if result.status_code == 200:
c.code = result.content
c.status = "4"
try:
c.put()
memcache.set("syncerrnote", False, 3600)
except:
memcache.set("syncerrnote", True, 3600)
else:
logging.error("send news failure !" + c.link)
logging.info("syncurl:" + login_url)
def handle_result(self, rpc, rssSources, url):
try:
result = rpc.get_result()
if result.status_code == 200:
html = result.content
rsslist = feedparser.parse(html)
alist = []
for rss in rsslist.entries:
try:
alist.append((rss.title, rss.link, rss.description))
except Exception, e:
logging.error("rss jiexi" + str(e))
newContent = []
contentobj = ContentsObj().all().filter("rss =", rssSources).fetch(1)
if not contentobj:
contentobj = ContentsObj()
contentobj.rss = rssSources
else:
contentobj = contentobj[0]
nowNewsList = json.loads(contentobj.contentdata)
for title, link, desc in alist:
link = link.strip()
if link not in nowNewsList:
newContent.append((link, title, desc, []))
rpcs = []
notediclist = []
for linkyuanzu in newContent:
pam = {}
rpc = urlfetch.create_rpc(deadline=50)
rpc.callback = self.rpc2_callback(rpc, rssSources, linkyuanzu, nowNewsList[-10:], pam)
urlfetch.make_fetch_call(
rpc, linkyuanzu[0], headers={"User-Agent": "Mozilla/5.0"}, follow_redirects=True
)
rpcs.append(rpc)
notediclist.append(pam)
for rpc in rpcs:
rpc.wait()
website = Setting().all().fetch(1)
if len(website) == 0:
web = Setting()
web.webSite = "http://localhoat:8888"
web.put()
else:
web = website[0]
allpam = {}
tempNewList = []
allpam["notenum"] = str(len(notediclist))
for i, pam in enumerate(notediclist):
if pam:
tempNewList.append(newContent[i][0])
if not rssSources.isfilterImg:
for num in range(pam["imgnum"]):
allpam[str(i) + "img" + str(num)] = pam["img" + str(num)]
allpam[str(i) + "code"] = pam["code"]
allpam[str(i) + "title"] = pam["title"]
allpam[str(i) + "content"] = pam["content"]
allpam[str(i) + "username"] = pam["username"]
if not notediclist:
return
login_url = web.webSite + "/RssMsg"
login_data = urllib.urlencode(allpam)
# logging.info('pam:'+str(allpam))
result = urlfetch.fetch(
url=login_url,
payload=login_data,
method=urlfetch.POST,
headers={
"Content-Type": "application/x-www-form-urlencoded",
"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
},
follow_redirects=False,
deadline=50,
)
if result.status_code == 200:
nowNewsList.extend(tempNewList)
contentobj.contentdata = json.dumps(nowNewsList[-60:])
contentobj.put()
else:
# linkyuanzu[3].append(False)
logging.error("send news failure !" + linkyuanzu[0])
# for linkyuanzu in newContent:
# if linkyuanzu[3]:
# nowNewsList.append(linkyuanzu[0])
except Exception, e:
logging.error("0000" + str(e) + url)
