def store_count(self, list): status = False count = len(list) element = 0 update = 0 if count > 0: for program in list: try: if self.check_for_exists(program) == 1: update += 1 else: id = genUUID() self.pc.save(program, id) element += 1 except Exception as e: import traceback msg = traceback.format_exc() LogGo.warning(msg) LogGo.info('Total :' + str(count) + ' (' + str(element) + ' Saved, ' + str(update) + ' Updated)') if element == 0 and update == 0: status = False else: status = True else: LogGo.info("0 Element!") # return False return status
def sogou_transfor(self): LogGo.info('搜狗转移') result = 0 # sogou = SougouTransforRuler() # # existsUrls = self.news.get_all_title() # order = self.news.get_max_order_code('wechat') # # news, article = sogou.ExtraList(existsUrls, order) # # content = '搜狗转移任务(' + ')' + '\r\n' # content += '此次采集数量: ' + str(len(news)) + '\r\n' # # if self.store(news, article): # result = 1 # else: # result = -1 # # if result == 1: # content += '存储成功!' # # SMTPServer.build_mission_report(content) # print(content) return result
def gs_detail(self, target, detail_page_bundle, content_ruler, encode): LogGo.info("On gs detail: " + str(target.data_key)) try: order = self.wechat_order gs = GsdataRuler() result = gs.scan_detail(target, detail_page_bundle, order, content_ruler, encode) code, detail_page_result = result if code == 1: if detail_page_result is not None: return (1, detail_page_result) elif code == 3: self.temp_list.append( (target, detail_page_bundle, content_ruler, encode)) except Exception as e: import traceback LogGo.warning(repr(e)) return (-1, e.args[0]) return (0, None)
def gensto(self, dao, list): status = False count = len(list) element = 0 if count > 0: for item in list: try: id = genUUID() dao.save_or_update(item, id) element += 1 except Exception as e: import traceback msg = traceback.format_exc() # print(msg) LogGo.warning(repr(e)) LogGo.info('Total :' + str(count) + ' / ' + str(element) + ' elements Saved!') if element == 0: status = False else: status = True else: LogGo.info("0 Element!") stauts = False return status
def start_mormal_mission(self): global all_target_transported target_list = get_target_list() self.target_producer = target_producer(target_list, self.config.target_pool_size, self.config.target_queue_size) self.target_consumer = target_consumer() self.upload_consumer = upload_consumer(self.config.uploader_queue_size) self.upload_consumer.start() self.target_consumer.start() self.target_producer.start() # self.target_producer.pool.close() # self.target_producer.pool.join() while True: # LogGo.debug(">>> target queue unfinishd count: " + str(self.target_producer.target_queue.queue.unfinished_tasks)) time.sleep(5) LogGo.debug("target_transported_over: " + str(target_producer.is_all_target_transported())) # self.target_consumer.queue.queue.join() # time.sleep(6000) LogGo.info('Loop Done! task count: ' + str(len(target_list))) SMTPServer.launch_mission_report()
def newrank_detail(self, target, detail_page_bundle, content_ruler, encode): LogGo.info("On newrank detail: " + str(target.data_key)) try: order = self.wechat_order newrank = NewrankRuler() result = newrank.scan_detail(target, detail_page_bundle, order, content_ruler, encode) code, detail_page_result = result if code == 1: if detail_page_result is not None: return (1, detail_page_result) elif code == -3: self.temp_list.append( (target, detail_page_bundle, content_ruler, encode)) except Exception: LogGo.warning('error in newrank detail') return (-1, None) return (0, None)
def check_data_base(self): """数据库结构检查""" list = [ TBNews(), TBArticle(), TBProgram(), TBWenzhangInfo(), TBDictionaryType(), TBDictionary(), TBNewsGroup(), TBNlpFilter(), TBScrapingTarget(), TBSoap(), TBSoapTarget(), TBGlobalTarget(), TBMR(), TBSpecialTarget(), TBProgramType(), TBSoapBlackList(), TBHeavyText() ] # list = [TBSoapBlackList()] guard = StructureGuard(Configs()) log = guard.check(list) if len(log) > 0: LogGo.info(str(log))
def genup(self, dao, list): status = False count = len(list) element = 0 if count > 0: for (update, where) in list: try: dao.update(update, where) element += 1 except Exception as e: import traceback msg = traceback.format_exc() # print(msg) LogGo.warning(repr(e)) LogGo.info('Total :' + str(count) + ' / ' + str(element) + ' elements Updated!') if element == 0: status = False else: status = True else: LogGo.info("0 Element!") stauts = False return status
def store_program_type(self, pair): status = False count = len(pair) element = 0 if count > 0: for item in pair: try: program, type = item self.program_type.save_by_program_type(program, type) element += 1 except Exception as e: import traceback msg = traceback.format_exc() LogGo.warning(msg) LogGo.info('Total :' + str(count) + ' / ' + str(element) + ' elements Saved!') if element == 0: status = False else: status = True else: LogGo.info("0 Element!") return status
def store_soap_target(self, targets: [], banned: bool = False): status = False element = 0 for target in targets: try: if banned: self.banned_program.save(target) else: self.soap_target.save(target) element += 1 except Exception as e: import traceback msg = traceback.format_exc() LogGo.warning(msg) LogGo.info('Total :' + str(element) + ' / ' + str(len(targets)) + ' elements Saved!') if element == 0: status = False else: status = True return status
def store_program(self, ids, programs): status = False if len(ids) != len(programs): LogGo.error("ids count unmatch programs count") return False count = len(ids) element = 0 if count > 0: for id, program in zip(ids, programs): try: self.program.save(program, id) element += 1 except Exception as e: import traceback msg = traceback.format_exc() LogGo.warning(msg) LogGo.info('Total :' + str(count) + ' / ' + str(element) + ' elements Saved!') if element == 0: status = False else: status = True else: LogGo.info("0 Element!") return status
def ExtraList(self, target, existsUrls, order): # order = ScrappdeDataDao.get_max_order_code() # 数据库中排序代码 result = [] url = str(target.extra0) next_index = "" """抓取地址""" raw = WechatRuler.req._get(url) try: trup = ExtraJSON.extraWechatList(raw, 'msgList', WechatRuler.keys) list = trup[0] next_index = str(trup[1]) except Exception as e: print(e) print("ERROR") return result while True: try: print('>>> scaning id: ' + next_index) LogGo.info('>>> scaning id: ' + next_index) tup = self.loopToFail(url, next_index) re_list = tup[0] next_index = str(tup[1]) is_continue = tup[2] if len(re_list) > 0: for item in re_list: list.append(item) # break else: break if is_continue != 1: break except Exception as e: print(e) break print('>>> list scaning completed') print('>>>') list.reverse() print('>>> Start Build SQL') result = self.build_base_dic(target,list,existsUrls,order) print('>>> Build SQL Success') print('>>>') return result
def start(self): """ 开启线程池,把所有 Target 放入池中,单独获取抓取列表 :return: """ while True: LogGo.info("Start target pool") [ self.pool.apply_async(target_producer.worker, (target, )) for target in self.targets ] time.sleep(Configs().work_interval)
def weibo_list(self, target): LogGo.info("On weibo list: " + str(target.data_key)) try: exists = self.exists_identifier weibo = WeiboRuler() code, value = weibo.scan_list(target, exists) return (code, value) except Exception as e: import traceback LogGo.warning(repr(e)) return (0, None)
def newrank_list(self, target): LogGo.info("On newrank list: " + str(target.data_key)) try: exists = self.exists_title newrank = NewrankRuler() code, value = newrank.scan_list(target, exists) return (code, value) except Exception as e: import traceback LogGo.warning(repr(e)) return (0, None)
def gs_list(self, target): LogGo.info("On gs list: " + str(target.data_key)) try: exists = self.exists_signature gs = GsdataRuler() code, value = gs.scan_list(target, exists) return (code, value) except Exception as e: import traceback LogGo.warning(repr(e)) return (0, None)
def store(self, news_list: [], article_list: [], heavy_list: [] = None): status = False if len(article_list) != len(news_list): LogGo.error("news count unmatch article count") return False count = len(news_list) element = 0 if count > 0: if heavy_list is not None: for news, article, heavy in zip(news_list, article_list, heavy_list): try: id = genUUID() self.news.save(news, id) self.article.save(article, id) self.heavy.save_with_news_id(heavy, id) element += 1 except Exception as e: LogGo.warning(repr(e)) else: for news, article in zip(news_list, article_list): try: id = genUUID() self.news.save(news, id) self.article.save(article, id) element += 1 except Exception as e: LogGo.warning(repr(e)) LogGo.info('Total :' + str(count) + ' / ' + str(element) + ' elements Saved!') if element == 0: status = False else: status = True # return True else: LogGo.info("0 Element!") # return False return status
def scrape_list(self, target): LogGo.info("On scrape list: " + str(target.data_key)) try: exists = self.exists_url ulweb = UlWebRuler() code, value = ulweb.scan_list(target, exists) return (code, value) except WebTargetOutOfDateException as e: LogGo.warning(e.args[0]) except Exception as e: import traceback LogGo.warning(repr(e)) return (0, None)
def start(self): result = 0 try: LogGo.info('搜狗爬虫') # Updatemp.loot() return result except Exception as e: import traceback msg = traceback.format_exc() print(msg) LogGo.warning(e) result = -1 return result
def weibo_detail(self, target, detail_page_bundle, content_ruler, encode): LogGo.info("On weibo detail: " + str(target.data_key)) try: order = self.weibo_order weibo = WeiboRuler() detail_page_result_dic = weibo.scan_detail(target, detail_page_bundle, order, content_ruler, encode) if detail_page_result_dic is not None: return (1, detail_page_result_dic) except Exception as e: import traceback LogGo.warning(repr(e)) return (-1, e.args[0]) return (0, None)
def store_soap(self, soap): status = False if len(soap) < 1: LogGo.info("no data to save!") return False try: id = genUUID() self.soap.save(soap, id) status = True except Exception as e: import traceback msg = traceback.format_exc() LogGo.warning(msg) LogGo.info('Soap Saved') return status
def extraAnyList(rawData, startStr, keys): index = rawData.index(startStr) elementlist = [] while (1): try: list = ExtraJSON.anyList(rawData, keys, index) if (list[1] == 0): # print("---- Over ----") break elementlist.insert(0, list[2]) index = list[1] except Exception as e: LogGo.info(e) break return elementlist
def send_request(self, result_dic): json_dic = dict() json_dic['date'] = DateGo.get_current_date() json_dic['targetId'] = 'No Target ID!' json_dic['rowList'] = [result_dic] try: LogGo.info("Ready to Post!") raw = RequestHelper.post(Configs.fish_data_post_url, json = json_dic) preview_dic = result_dic.copy() preview_dic['text_not_format_clob'] = 'DUMMY CONTENT' preview_dic['text_blob'] = 'DUMMY CONTENT' json_dic['rowList'] = preview_dic json_str = json.dumps(json_dic) LogGo.info("POST CONTENT: " + json_str) LogGo.info("POST RESPONSE: " + str(raw)) except Exception: E.out_err()
def worker(target): """ 普通扫描函数 首先获取列表 然后抓取详细页内容 """ # if not Configs.debuging: # time.sleep(20) # time.sleep(1) time.sleep(random.randint(1, 4)) last_access = target.last_access_date frequency = target.frequency if frequency is None: frequency = 0 if last_access is None: last_access = DateGo.date_befor_days(365, True) distance = DateGo.distance(last_access) if distance < frequency: ScrabingTarget.set_elog(target.id, "frequency") LogGo.info("type: " + str(target.type) + str(target.soap_type) + " name: " + str(target.data_key) + " skipped, last access data: " + str(last_access) + "\r\n") if target.type == 'ulweb' or target.type == 'jsweb': msg = '>>> Ulweb(' + target.data_key + ')' + ' last acc: ' + str( last_access) LogGo.info(msg) web = WebStrategy() result = web.scrape_list(target) target_producer.send_to_queue(result) # elif target.type == 'wechat':#微信官方 规律 历史消息 # msg = '\r\n' + '>>> Wechat(' + target.extra0 + ')' # LogGo.info(msg) # wechat = WechatStrategy() # result = wechat.ScanWechatTarget(target) # # return (target, 1, result) elif target.type == 'newrank': msg = '>>> Newrank(' + target.extra0 + ')' + ' last acc: ' + str( last_access) LogGo.info(msg) wechat = WechatStrategy() result = wechat.newrank_list(target) target_producer.send_to_queue(result) elif target.type == 'gsdata': msg = '>>> Gsdata(' + target.extra0 + ')' + ' last acc: ' + str( last_access) LogGo.info(msg) wechat = WechatStrategy() result = wechat.gs_list(target) target_producer.send_to_queue(result) elif target.type == 'weibo': msg = '>>> Weibo(' + target.extra0 + ')' + ' last acc: ' + str( last_access) LogGo.info(msg) weibo = WeiboStrategy() result = weibo.weibo_list(target) target_producer.send_to_queue(result)
def build_base_dic(self,target,list,existsUrls,order): news = TBNews() article = TBArticle() picture_dao = PictureDao() result = [] article_result = [] """抓取正文""" for i in list: try: i['content_url'] = UrlHelper.unify(i['content_url']) #StringHelper.unescape(i['content_url']) if existsUrls.count(i['content_url']) < 1: # getattr(i, 'url') LogGo.info(">>> file id: " + str(i['fileid'])) LogGo.info(">>> url: " + str(i['content_url'])) try: tup = ExtraJSON.wechat_extra_content(i['content_url']) # getattr(i, 'url') except Exception as e: print(e) print(">>> ") print(">>> extra content error.") print(">>> ") LogGo.info("extra content error.") LogGo.info("possible a deleted msg") # LogGo.info("url: " + i['content_url']) continue raw_content = tup[1] content = tup[2] """字典的 键 对应数据库中的字段名 值 对应要存储的值""" dic = dict() article_dic = dict() order = order + 5 dic[news.order_code.key] = order # """排序代码""" dic[news.create_date.key] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # """此条记录创建时间""" dic[news.valid.key] = 1 dic[news.text_not_format.key] = content #"""去除标签的正文内容""" # dic[news.text_blob.key] = raw_content #"""原始带标签字段""" dic[news.subscribe_time.key] = i['datetime'] # getattr(i, 'publicTime') """文章发表日期""" dic[news.author.key] = i['author'] # getattr(i, 'author')"""文章所属机构""" dic[news.title.key] = i['title'] # getattr(i, 'title')"""文章标题""" dic[news.subject.key] = i['digest'] # """摘要""" dic[news.status.key] = 2 picture_id = picture_dao.save_data(i['cover']) dic[news.main_pic_id.key] = picture_id #"""列表图片 id""" article_dic[article.fingerprint.key] = md5(i['content_url'])#"""由地址生成的指纹""" article_dic[article.target_id.key] = target.id article_dic[article.company.key] = target.data_key # getattr(i, 'author') """文章所属机构""" article_dic[article.content_url.key] = i['content_url'] # getattr(i, 'url')"""正文链接""" article_dic[article.scrabble_type.key] = 'wechat' # """文章类型 微信固定值为 wechat """ article_dic[article.is_scrabbled.key] = 1 # """在数据库中作为 这是一条抓取到的数据 的标记""" result.append(dic) article_result.append(article_dic) except Exception as e: import traceback msg = traceback.format_exc() print(msg) LogGo.warning(repr(e)) continue return result, article_result
def build_count_dic(self, pro_list): # pc = TBProgramPlayCount result = [] try: LogGo.info(">>> count: " + str(len(pro_list))) for programs in pro_list: """字典的 键 对应数据库中的字段名 值 对应要存储的值""" dic = dict() total = 0 for program in programs: try: dic[TBProgramPlayCount.program.key] = program[ TBSoap.program.key] plantform = program[TBSoap.plantform.key] count = program[TBSoap.play_count.key] total += count if self.td(plantform) == 'i': dic[TBProgramPlayCount.count1.key] = count elif self.td(plantform) == 'l': dic[TBProgramPlayCount.count2.key] = count elif self.td(plantform) == 't': dic[TBProgramPlayCount.count3.key] = count elif self.td(plantform) == 'm': dic[TBProgramPlayCount.count4.key] = count elif self.td(plantform) == 'y': dic[TBProgramPlayCount.count5.key] = count elif self.td(plantform) == 's': dic[TBProgramPlayCount.count6.key] = count except Exception as e: import traceback msg = traceback.format_exc() LogGo.info(msg) dic[TBProgramPlayCount.total_count.key] = total dic[TBProgramPlayCount. create_time.key] = datetime.datetime.now().strftime( '%Y-%m-%d') # """此条记录创建时间""" result.append(dic) except BaseDateLackException as e: msg = "Lake improtant data(" + str(e) + ')' LogGo.warning(msg) except DataFormatException as e: pass # msg = "Date format error: " + i['link'] + '\r\n' + str(e) # LogGo.warning(msg) except KeyError as e: import traceback msg = traceback.format_exc() LogGo.warning(msg) except Exception as e: import traceback msg = traceback.format_exc() LogGo.warning(msg) return result
def build_base_dic(self, target, result, order): soap = TBSoap() program_dao = ProgramDao() soap_result = [] try: name = '' if Configs.show_utf: try: name = target.data_key except: name = '<<error>>' LogGo.info(">>> name: " + str(name) + "(" + str(result['playCount']) + ")") """字典的 键 对应数据库中的字段名 值 对应要存储的值""" dic = dict() try: dic[soap.play_count.key] = result['playCount'] #瞬时播放量 except KeyError as e: raise BaseDateLackException(str(e)) try: dic[soap.keywords.key] = result['keywords'] # 关键字 except: pass try: dic[soap.bullet_count.key] = result['bullets'] # 弹幕量 except: pass try: dic[soap.hate_count.key] = result['hate'] # 怒踩量 except: pass try: dic[soap.like_count.key] = result['like'] # 点赞量 except: pass try: dic[soap.latest_order.key] = result['latestOrder'] # 最新剧集 except: pass try: dic[soap.name.key] = result['name'] # 剧名 except: pass try: dic[soap.name.key] = program_dao.get_title_by_id( target.program_id) except: pass try: dic[soap.score.key] = result['score'] # 分数 except: pass try: dic[soap.video_count.key] = result['videoCount'] # 视频数量 except: pass try: # pass dic[soap.program.key] = target.program_id # program dic[soap.target.key] = target.id # program except: pass dic[soap.plantform.key] = target.soap_type order += 1 dic[soap.order_code.key] = order # """排序代码""" dic[soap.create_date.key] = datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S') # """此条记录创建时间""" dic[soap.valid.key] = 1 soap_result.append(dic) except BaseDateLackException as e: msg = "Lake improtant data(" + str(e) + ')' LogGo.warning(msg) except DataFormatException as e: pass # msg = "Date format error: " + i['link'] + '\r\n' + str(e) # LogGo.warning(msg) except KeyError as e: import traceback msg = traceback.format_exc() LogGo.warning(msg) except Exception as e: import traceback msg = traceback.format_exc() LogGo.warning(msg) return soap_result
def build_single_page_dic(self, target, detail_page_bundle, order, content_ruler, encode): news = TBNews() article = TBArticle() result_dic = dict() try: LogGo.info(WeiboRuler.url_status + detail_page_bundle['id']) # blob = i['text'].encode("UTF-8") """字典的 键 对应数据库中的字段名 值 对应要存储的值""" news_dic = dict() article_dic = dict() """排序代码""" order += 2 news_dic[news.order_code.key] = order # dic[news.text_not_format.key] = i['text']#"""去除标签的正文内容""" # dic[news.text_blob.key] = blob #"""原始带标签字段""" sub_tim = detail_page_bundle['created_at'] if sub_tim is not None: news_dic[news.subscribe_time.key] = sub_tim # getattr(i, 'publicTime') """文章发表日期""" else: LogGo.warning("no subscribe time!") news_dic[news.create_date.key] = DateGo.get_current_date() # """此条记录创建时间""" news_dic[news.status.key] = 1 # """状态""" news_dic[news.valid.key] = 1 news_dic[news.title.key] = detail_page_bundle['text'] news_dic[news.text_not_format.key] = detail_page_bundle['text'] news_dic[news.text_blob.key] = detail_page_bundle['text'] # title = None # try: # title = i['page_info'] # title = title['content1'] # except Exception as e: # pass # if title is None: # dic[news.title.key] = i['text'] # getattr(i, 'title') """文章标题""" # else: # dic[news.title.key] = title # """文章标题""" """文章所属机构""" try: user = detail_page_bundle['user'] screen_name = user['screen_name'] article_dic[article.company.key] = screen_name # getattr except Exception as e: pass article_dic[article.vote_up_count.key] = detail_page_bundle['attitudes_count'] # getattr(i, 'likeCount') """点赞数""" article_dic[article.scrabble_type.key] = 'weibo' #"""文章类型""" article_dic[article.is_scrabbled.key] = 1 #"""在数据库中作为 这是一条抓取到的数据 的标记""" article_dic[article.identifier.key] = detail_page_bundle['id'] #"""数据在母体中的 id""" article_dic[article.target_id.key] = target.id article_dic[article.content_url.key] = WeiboRuler.url_status + detail_page_bundle['id'] # getattr(i, 'url') """正文链接""" article_dic[article.publishStatus.key] = 1 # article_dic[article.messageType.key] = random.randint(0, 1) """如果是回复 或者 引用 会有被引用的微博,记录那个微博的 id""" try: retweeted_status = detail_page_bundle['retweeted_status'] ret_id = retweeted_status['id'] article_dic[article.identifier_re.key] = ret_id except Exception as e: pass """阅读量""" # dic['click_count'] = i['clicksCount'] #getattr(i, 'clicksCount') """转发数""" """评论量""" # """图片组""" # try: # pics = i['pics'] # if len(pics) > 0: # group_id = PictureDao.save_group_data(pics) # if group_id is not None: # dic['group_picture_id'] = group_id # except Exception as e: # print(e) # LogGo.warning(dic['content_url']) # LogGo.warning(e) result_dic.update(article_dic) result_dic.update(news_dic) except Exception as e: import traceback msg = traceback.format_exc() LogGo.warning(msg) return None return result_dic
def scan_detail(self, target, detail_page_bundle, order, content_ruler, encode): news = TBNews() article = TBArticle() # picture_dao = PictureDao() result_dic = dict() try: info = self.ready_info(detail_page_bundle['title'], detail_page_bundle['url']) LogGo.info(info) try: # tup = ExtraJSON.wechat_extra_content(detail_page_bundle['url']) tup = self.jsons.wechat_extra_content( detail_page_bundle['url']) except HttpConnectionFailedException as e: LogGo.warning(repr(e)) return (-3, None) except AttributeError: LogGo.warning( "Maybe a deleted msg, complete the code to detect this error" ) return (-2, None) except Exception: LogGo.warning("Error when get detail message!") return (-2, None) raw_content = tup[1] content = tup[2] picture = tup[3] """字典的 键 对应数据库中的字段名 值 对应要存储的值""" news_dic = dict() article_dic = dict() ############################## NEWS ############################### """列表图片 id""" # if picture is not None: # picture_id = picture_dao.save_data(picture) # news_dic[news.main_pic_id.key] = picture_id news_dic[news.text_not_format.key] = content # """去除标签的正文内容""" # dic[news.text_blob.key] = raw_content#"""原始带标签字段""" news_dic[news.subscribe_time.key] = detail_page_bundle[ 'publicTime'] # """文章发表日期""" news_dic[news.create_date.key] = datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S') # """此条记录创建时间""" news_dic[news.subject.key] = detail_page_bundle[ 'summary'] # """摘要""" news_dic[news.valid.key] = 1 news_dic[news.author.key] = detail_page_bundle['author'] news_dic[news.title.key] = detail_page_bundle[ 'title'] # """文章标题""" news_dic[news.status.key] = 2 order += 5 news_dic[news.order_code.key] = order # """排序代码""" ############################## ARTICLE ############################### article_dic[article.content_url.key] = detail_page_bundle[ 'url'] # getattr(i, 'url')"""正文链接""" article_dic[article.fingerprint.key] = md5( detail_page_bundle['url']) # """由地址生成的指纹""" article_dic[article.company.key] = target.data_key # """文章所属机构""" article_dic[article.target_id.key] = target.id article_dic[article.raw_click_count.key] = detail_page_bundle[ 'clicksCount'] # getattr(i, 'clicksCount')#"""阅读量""" article_dic[article.vote_up_count.key] = detail_page_bundle[ 'likeCount'] # getattr(i, 'likeCount')"""点赞数""" article_dic[article.scrabble_type. key] = 'wechat' # """文章类型 微信固定值为 wechat """ article_dic[ article.is_scrabbled.key] = 1 # """在数据库中作为 这是一条抓取到的数据 的标记""" article_dic[article.publishStatus.key] = 1 # article_dic[article.messageType.key] = random.randint(0, 1) ############################## DIC ############################### result_dic.update(news_dic) result_dic.update(article_dic) except Exception: import traceback msg = traceback.format_exc() LogGo.warning(msg) return (-1, None) return (1, result_dic)
LogGo.init(Configs()) RequestHelper.init(Configs()) SMTPServer.init(Configs()) Download(Configs()) RequestHelperClassVer.init(Configs()) ProxyHelper.init(Configs()) MysqlHelper.init(Configs()) BaseStrategy.init() """ 程序入口 """ if __name__ == "__main__": base_init() LogGo.info('PID: ' + str(os.getpid())) FileHelper.record_pid(str(os.getpid())) LogGo.info("-- Xenoblade Online --") print(" ") print("---------Xenoblade v0.2(Super Beta Version)-----------") print(" ") print("dependencies:") print("APScheduler==3.3.1\r\n" "beautifulsoup4==4.5.3\r\n" "bosonnlp==0.8.0\r\n" "click==6.7\r\n" "crypto==1.4.1\r\n" "demjson==2.2.4\r\n" "Flask==0.12\r\n" "HTMLParser==0.0.2\r\n" "httplib2==0.10.3\r\n"