def reset_shutdown_status(): try: file_name = Configs().system_shutdown_flag_filename FileHelper.create(file_name, "0") except: LogGo.error("error while reseting shutdown flag file!!!")
def send_to_queue(result): """ 传送 code: 0 到达最大访问频率 1 正常结果 :param request: :param result: :return: """ global target_mutex, target_count, target_transported_count, all_target_transported code, value = result target, detail_page_bundle_list, content_ruler, encode = value if code == 1: for detail_page_bundle in detail_page_bundle_list: target_producer.target_queue.queue.put( (target, detail_page_bundle, content_ruler, encode)) ScrabingTarget.set_last_access_date(target.id) else: LogGo.error("List Page Error:" + str(target.data_key) + " Code: " + str(code)) ScrabingTarget.set_elog(target.id, "error code: " + str(code)) if target_mutex.acquire(): if target_count == target_transported_count: all_target_transported = True else: target_transported_count += 1 LogGo.debug('target_transported_count: ' + str(target_transported_count)) target_mutex.release()
def set_shutdown_status(): try: file_name = Configs().system_shutdown_flag_filename FileHelper.create(file_name, "1") except: LogGo.error("something wrong at setting shutdown flag file !!!")
def store_program(self, ids, programs): status = False if len(ids) != len(programs): LogGo.error("ids count unmatch programs count") return False count = len(ids) element = 0 if count > 0: for id, program in zip(ids, programs): try: self.program.save(program, id) element += 1 except Exception as e: import traceback msg = traceback.format_exc() LogGo.warning(msg) LogGo.info('Total :' + str(count) + ' / ' + str(element) + ' elements Saved!') if element == 0: status = False else: status = True else: LogGo.info("0 Element!") return status
def check_shutdown_status(): try: file_name = Configs().system_shutdown_flag_filename status = int(StringHelper.trim(FileHelper.read(file_name))) if status == 1: somebody_help.reset_shutdown_status() return True else: return False except: LogGo.error("system_shutdown_flag_file unavailable!") return False
def store(self, news_list: [], article_list: [], heavy_list: [] = None): status = False if len(article_list) != len(news_list): LogGo.error("news count unmatch article count") return False count = len(news_list) element = 0 if count > 0: if heavy_list is not None: for news, article, heavy in zip(news_list, article_list, heavy_list): try: id = genUUID() self.news.save(news, id) self.article.save(article, id) self.heavy.save_with_news_id(heavy, id) element += 1 except Exception as e: LogGo.warning(repr(e)) else: for news, article in zip(news_list, article_list): try: id = genUUID() self.news.save(news, id) self.article.save(article, id) element += 1 except Exception as e: LogGo.warning(repr(e)) LogGo.info('Total :' + str(count) + ' / ' + str(element) + ' elements Saved!') if element == 0: status = False else: status = True # return True else: LogGo.info("0 Element!") # return False return status
def fetch_detail(self, value): """value: (target, original dic, ruler, encode ) """ LogGo.debug("in detail :" + str(value)) target, detail_page_dic, content_ruler, encode = value delegate = None if target.type == 'ulweb' or target.type == 'jsweb': delegate = self.web.scrape_detail elif target.type == 'newrank': delegate = self.wechat.newrank_detail elif target.type == 'gsdata': delegate = self.wechat.gs_detail elif target.type == 'weibo': delegate = self.weibo.weibo_detail code, result_dic = delegate(target, detail_page_dic, content_ruler, encode) if code is 1: self.upload_queue.queue.put(result_dic) else: LogGo.error("Detail Error: " + str(detail_page_dic))
"requests==2.12.4\r\n" "shellescape==3.4.1\r\n" "six==1.10.0\r\n" "stem==1.6.0\r\n" "tzlocal==1.3\r\n" "Werkzeug==0.11.15\r\n") print(" ") print("------ ---- --- ---getIndexNumber--- ------ ---- --") print(" ") scan = Scanner() while True: LogGo.info(">>> New Loop -->") try: scan.start() except Exception as e: import traceback msg = traceback.format_exc() LogGo.error(msg) if Configs.infinity: while not somebody_help.isthataworkday(Configs().work_sequence): time.sleep(60 * 60) else: break LogGo.info("-- GH Offline --") print("---------------------------------------------")