def main():
fname = config.log_path + 'article_parse.' + time.strftime("%Y%m%d")
log.set_logger(level='DEBUG', when="D", limit=1, filename=fname)
alist = Mongo().scan()
if not alist:
log.warn("no articles in mongodb")
return False
MyObj = Mysql()
mobj = Mongo()
for doc in alist:
if Parse(MyObj).do(doc):
mobj.update(doc.get("_id"), done=1)
log.info("insert mysql success, url:%s" % doc.get('url'))
else:
mobj.update(doc.get("_id"), done=-1)
log.warning("insert mysql failure, task_id:%s, url:%s" %
(doc.get('taskid'), doc.get('url')))
