Python info示例

示例#1

def main():
    config = 'config/offline_gsxt_parse.conf'
    province = None

    signal.signal(signal.SIGINT, process_quit)
    signal.signal(signal.SIGTERM, process_quit)
    signal.signal(signal.SIGQUIT, process_quit)
    signal.signal(signal.SIGUSR1, process_quit)

    length = len(sys.argv)
    if length > 1:
        config = sys.argv[1]
        if length > 2:
            province = sys.argv[2]
        else:
            province = None

    # 如果省份信息是all 则默认为全部站点进行抓取
    if province == 'all':
        province = None

    global_log.info("当前需要处理的省份为: {}".format(province))

    try:
        crawler = StartTaskCrawler(config_file=config, province=province)
        crawler.start_worker()
    except Exception as e:
        global_log.error("初始化错误:")
        global_log.exception(e)

示例#2

文件： start_online_parse_task.py 项目： xiaohui2856/gsxt_spider

def main():
    config = 'config/online_gsxt_parse.conf'
    signal.signal(signal.SIGINT, process_quit)
    signal.signal(signal.SIGTERM, process_quit)
    signal.signal(signal.SIGQUIT, process_quit)
    signal.signal(signal.SIGUSR1, process_quit)

    length = len(sys.argv)
    global_log.info('当前参数个数为: length = {length}'.format(length=length))
    if length > 1:
        config = sys.argv[1]

    crawler = StartTaskCrawler(config_file=config)
    crawler.start_worker()

示例#3

文件： start_offline_crawler.py 项目： xiaohui2856/gsxt_spider

def main():
    config = 'config/offline_gsxt_searchlist.conf'
    source_table = "offline_all_list"
    province = None

    length = len(sys.argv)
    global_log.info('当前参数个数为: length = {length}'.format(length=length))
    if length > 1:
        config = sys.argv[1]
        if length > 2:
            source_table = sys.argv[2]
            if length > 3:
                province = sys.argv[3]

    # 如果省份信息是all 则默认为全部站点进行抓取
    if province == 'all':
        province = None

    global_log.info("当前需要处理的省份为: {}".format(province))
    crawler = StartTaskCrawler(config_file=config,
                               source_table=source_table,
                               province=province)
    crawler.start_worker()

示例#4

文件： start_online_crawl.py 项目： xtuyaowu/gsxt_crawler

def main():
    config = 'config/online_gsxt_crawl.conf'
    province = 'anhui'

    signal.signal(signal.SIGINT, process_quit)
    signal.signal(signal.SIGTERM, process_quit)
    signal.signal(signal.SIGQUIT, process_quit)
    signal.signal(signal.SIGUSR1, process_quit)

    length = len(sys.argv)
    global_log.info('当前参数个数为: length = {length}'.format(length=length))
    if length > 1:
        config = sys.argv[1]
        if length > 2:
            province = sys.argv[2]

    try:
        crawler = StartTaskCrawler(config_file=config,
                                   province=province)
        crawler.start_worker()
    except Exception as e:
        global_log.error('初始化异常!!!!')
        global_log.exception(e)

示例#5

文件： start_online_parse_task.py 项目： xiaohui2856/gsxt_spider

def process_quit(signo, frame):
    global is_running
    global_log.info('收到信号退出进程...')
    is_running = False