def main(): config = 'config/offline_gsxt_parse.conf' province = None signal.signal(signal.SIGINT, process_quit) signal.signal(signal.SIGTERM, process_quit) signal.signal(signal.SIGQUIT, process_quit) signal.signal(signal.SIGUSR1, process_quit) length = len(sys.argv) if length > 1: config = sys.argv[1] if length > 2: province = sys.argv[2] else: province = None # 如果省份信息是all 则默认为全部站点进行抓取 if province == 'all': province = None global_log.info("当前需要处理的省份为: {}".format(province)) try: crawler = StartTaskCrawler(config_file=config, province=province) crawler.start_worker() except Exception as e: global_log.error("初始化错误:") global_log.exception(e)
def main(): config = 'config/online_gsxt_parse.conf' signal.signal(signal.SIGINT, process_quit) signal.signal(signal.SIGTERM, process_quit) signal.signal(signal.SIGQUIT, process_quit) signal.signal(signal.SIGUSR1, process_quit) length = len(sys.argv) global_log.info('当前参数个数为: length = {length}'.format(length=length)) if length > 1: config = sys.argv[1] crawler = StartTaskCrawler(config_file=config) crawler.start_worker()
def main(): config = 'config/offline_gsxt_searchlist.conf' source_table = "offline_all_list" province = None length = len(sys.argv) global_log.info('当前参数个数为: length = {length}'.format(length=length)) if length > 1: config = sys.argv[1] if length > 2: source_table = sys.argv[2] if length > 3: province = sys.argv[3] # 如果省份信息是all 则默认为全部站点进行抓取 if province == 'all': province = None global_log.info("当前需要处理的省份为: {}".format(province)) crawler = StartTaskCrawler(config_file=config, source_table=source_table, province=province) crawler.start_worker()
def main(): config = 'config/online_gsxt_crawl.conf' province = 'anhui' signal.signal(signal.SIGINT, process_quit) signal.signal(signal.SIGTERM, process_quit) signal.signal(signal.SIGQUIT, process_quit) signal.signal(signal.SIGUSR1, process_quit) length = len(sys.argv) global_log.info('当前参数个数为: length = {length}'.format(length=length)) if length > 1: config = sys.argv[1] if length > 2: province = sys.argv[2] try: crawler = StartTaskCrawler(config_file=config, province=province) crawler.start_worker() except Exception as e: global_log.error('初始化异常!!!!') global_log.exception(e)
def process_quit(signo, frame): global is_running global_log.info('收到信号退出进程...') is_running = False