Python setupConf Examples

Example #1

File: cli.py Project: importer/flask_learn

def scheduler(args):
    config.setupConf(args.cfg_file)
    logfile = "{}/{}".format(config.G_Conf.Scheduler.LogDir, config.G_Conf.Scheduler.LogFile)
    if args.ha:
        logfile = "{}.ha".format(logfile)

    initServer(logfile, config.G_Conf.Common.MysqlConn)
    main_scheduler = module_scheduler.MainScheduler(
        broker_url=config.G_Conf.Common.Broker,
        fetch_interval=config.G_Conf.Scheduler.FetchInterval,
        woker_timeout=config.G_Conf.Scheduler.WorkerTimeOut,
        retry_push_times=config.G_Conf.Scheduler.RetryQueueimes)

    # signal.signal(signal.SIGINT, sigint_handler)
    # signal.signal(signal.SIGTERM, sigint_handler)
    # signal.signal(signal.SIGQUIT, sigquit_handler)
    if args.ha:
        #haserver = ha.HaServer(host="etcd.in.codoon.com", port=2379)
        """
        start a main process for health checking, sub process for scheduler
        """
        haserver = ha.HaServer('127.0.0.1')
        haserver.run()
    else:
        main_scheduler.run()

Example #2

def scheduler(args):
    config.setupConf(args.cfg_file)
    logfile = "{}/{}".format(config.G_Conf.Scheduler.LogDir, config.G_Conf.Scheduler.LogFile)
    if args.ha:
        logfile = "{}.ha".format(logfile)

    initServer(logfile, config.G_Conf.Common.MysqlConn)
    main_scheduler = module_scheduler.MainScheduler(
        broker_url=config.G_Conf.Common.Broker,
        fetch_interval=config.G_Conf.Scheduler.FetchInterval,
        woker_timeout=config.G_Conf.Scheduler.WorkerTimeOut,
        retry_push_times=config.G_Conf.Scheduler.RetryQueueimes)

    signal.signal(signal.SIGINT, sigint_handler)
    signal.signal(signal.SIGTERM, sigint_handler)
    signal.signal(signal.SIGQUIT, sigquit_handler)
    if args.ha:
        #haserver = ha.HaServer(host="etcd.in.codoon.com", port=2379)
        """
        start a main process for health checking, sub process for scheduler
        """
        haserver = ha.HaServer()
        haserver.run()
    else:
        main_scheduler.run()

Example #3

File: cli.py Project: wuyongwen/goflow

def init_db(args):
    config.setupConf(args.cfg_file)
    initServer(
        "{}/{}".format(config.G_Conf.Worker.LogDir,
                       config.G_Conf.Worker.LogFile),
        config.G_Conf.Common.MysqlConn)
    Base.metadata.drop_all(db.engine)
    Base.metadata.create_all(db.engine)

Example #4

def kill(args):
    err, date_list = is_valid_run_args(args)
    if err:
        print err
        return
    print "kill task {} with date: {}".format(args.job_id, " ".join(date_list))
    config.setupConf(args.cfg_file)
    initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "kill.log"), config.G_Conf.Common.MysqlConn)
    cnt = module_scheduler.Admin().kill_task(args.job_id, date_list)
    print "kill {} instance".format(cnt)

Example #5

File: cli.py Project: importer/flask_learn

def kill(args):
    err, date_list = is_valid_run_args(args)
    if err:
        print (err)
        return
    print ("kill task {} with date: {}".format(args.job_id, " ".join(date_list)))
    config.setupConf(args.cfg_file)
    initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "kill.log"), config.G_Conf.Common.MysqlConn)
    cnt = module_scheduler.Admin().kill_task(args.job_id, date_list)
    print ("kill {} instance".format(cnt))

Example #6

def executer(args):
    config.setupConf(args.cfg_file)
    initServer("{}/{}".format(config.G_Conf.Worker.LogDir, config.G_Conf.Worker.LogFile), config.G_Conf.Common.MysqlConn)
    signal.signal(signal.SIGINT, sigint_handler)
    signal.signal(signal.SIGTERM, sigint_handler)
    signal.signal(signal.SIGQUIT, sigquit_handler)
    task_worker = worker.Worker(
        parallelism=config.G_Conf.Worker.Parallelism,
        heartbeat_interval=config.G_Conf.Worker.HearteatInterval,
        woker_timeout=config.G_Conf.Worker.TaskTimeOut,
        retry_times=config.G_Conf.Worker.Retry)
    task_worker.run()

Example #7

File: cli.py Project: importer/flask_learn

def executer(args):
    config.setupConf(args.cfg_file)
    initServer("{}/{}".format(config.G_Conf.Worker.LogDir, config.G_Conf.Worker.LogFile), config.G_Conf.Common.MysqlConn)
    # signal.signal(signal.SIGINT, sigint_handler)
    # signal.signal(signal.SIGTERM, sigint_handler)
    # signal.signal(signal.SIGQUIT, sigquit_handler)
    task_worker = worker.Worker(
        parallelism=config.G_Conf.Worker.Parallelism,
        heartbeat_interval=config.G_Conf.Worker.HearteatInterval,
        woker_timeout=config.G_Conf.Worker.TaskTimeOut,
        retry_times=config.G_Conf.Worker.Retry)
    task_worker.run()

Example #8

File: cli.py Project: importer/flask_learn

def do_all_job(args):
    """
    do_all_job run all scheduler job
    :param args:
    :return:
    """
    if not isValiddate(args.date):
        err = "time date format is YYYY-MM-DD, give {}".format(args.date)
        print (err)
        return

    print ("run all schedule_job with date: {}".format(args.date))
    config.setupConf(args.cfg_file)
    initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "do_all_job.log"), config.G_Conf.Common.MysqlConn)
    msg = module_scheduler.Admin().run_all_job([args.date])
    print ("run success, {}".format(msg))

Example #9

def do_all_job(args):
    """
    do_all_job run all scheduler job
    :param args:
    :return:
    """
    if not isValiddate(args.date):
        err = "time date format is YYYY-MM-DD, give {}".format(args.date)
        print err
        return

    print "run all schedule_job with date: {}".format(args.date)
    config.setupConf(args.cfg_file)
    initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "do_all_job.log"), config.G_Conf.Common.MysqlConn)
    msg = module_scheduler.Admin().run_all_job([args.date])
    print "run success, {}".format(msg)

Example #10

File: cli.py Project: importer/flask_learn

def migrate(args):
    """
    migrate dependency , extract last_id
    :param args:
    :return:
    """
    config.setupConf(args.cfg_file)
    initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "migrate.log"), config.G_Conf.Common.MysqlConn)
    if args.dependency:
        module_scheduler.Admin().migrate_dependency(None, args.delete)

    if args.last_id:
        if not isValiddate(args.date):
            print ("date format is YYYY-MM-DD, give {}".format(args.date))
            return
        d = args.date.replace("-", "")
        module_scheduler.Admin().migrate_lastid(d)

Example #11

def migrate(args):
    """
    migrate dependency , extract last_id
    :param args:
    :return:
    """
    config.setupConf(args.cfg_file)
    initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "migrate.log"), config.G_Conf.Common.MysqlConn)
    if args.dependency:
        module_scheduler.Admin().migrate_dependency(None, args.delete)

    if args.last_id:
        if not isValiddate(args.date):
            print "date format is YYYY-MM-DD, give {}".format(args.date)
            return
        d = args.date.replace("-", "")
        module_scheduler.Admin().migrate_lastid(d)

Example #12

def run(args):
    """
    rerun task
    :param args:
    :return:
    """
    err, date_list = is_valid_run_args(args)
    if err:
        print err
        return
    print "run task  {} with date: {}".format(args.job_id, " ".join(date_list))
    config.setupConf(args.cfg_file)
    initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "rerun.log"), config.G_Conf.Common.MysqlConn)
    msg = module_scheduler.Admin().rerun_task(args.job_id, date_list,
                                                 up_and_down=args.with_up_down,
                                                 run_up=args.with_upstream,
                                                 run_down=args.with_downstream,
                                              force=args.force)
    print "run success, {}".format(msg)

Example #13

def do_all_extract(args):
    """
    do_all_extract
    :param args:
    :return:
    """
    if not isValiddate(args.date):
        err = "time date format is YYYY-MM-DD, give {}".format(args.date)
        print err
        return
    msg = "run all extract with date: {}".format(args.date)
    if args.dest_table is not None:
        msg = "{}, dest_table: {}".format(msg, args.dest_table)
    print msg

    config.setupConf(args.cfg_file)
    initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "do_all_extract.log"), config.G_Conf.Common.MysqlConn)
    msg = module_scheduler.Admin().run_all_extract([args.date], dest_table=args.dest_table)
    print "run success, {}".format(msg)

Example #14

File: cli.py Project: importer/flask_learn

def do_all_extract(args):
    """
    do_all_extract
    :param args:
    :return:
    """
    if not isValiddate(args.date):
        err = "time date format is YYYY-MM-DD, give {}".format(args.date)
        print (err)
        return
    msg = "run all extract with date: {}".format(args.date)
    if args.dest_table is not None:
        msg = "{}, dest_table: {}".format(msg, args.dest_table)
    print (msg)

    config.setupConf(args.cfg_file)
    initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "do_all_extract.log"), config.G_Conf.Common.MysqlConn)
    msg = module_scheduler.Admin().run_all_extract([args.date], dest_table=args.dest_table)
    print ("run success, {}".format(msg))

Example #15

File: cli.py Project: importer/flask_learn

def run(args):
    """
    rerun task
    :param args:
    :return:
    """
    err, date_list = is_valid_run_args(args)
    if err:
        print (err)
        return
    print ("run task  {} with date: {}".format(args.job_id, " ".join(date_list)))
    config.setupConf(args.cfg_file)
    initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "rerun.log"), config.G_Conf.Common.MysqlConn)
    msg = module_scheduler.Admin().rerun_task(args.job_id, date_list,
                                                 up_and_down=args.with_up_down,
                                                 run_up=args.with_upstream,
                                                 run_down=args.with_downstream,
                                              force=args.force)
    print ("run success, {}".format(msg))

Example #16

def dep(args):
    """
    auto generate script dependency
    """
    config.setupConf(args.cfg_file)
    initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "dep.log"), config.G_Conf.Common.MysqlConn)

    if args.query:
        module_scheduler.Admin().query_dep(args.job_id, args.date)
    else:
        if os.path.isfile(args.script):
            gen_dep(args.script)
        elif os.path.isdir(args.script):
            for root, dirs, files in os.walk(args.script, followlinks=True):
                for f in files:
                    filepath = os.path.join(root, f)
                    if not os.path.isfile(filepath):
                        continue
                    mod_name, file_ext = os.path.splitext(
                        os.path.split(filepath)[-1])
                    if file_ext != '.py':
                        continue
                    gen_dep(filepath)

Example #17

File: cli.py Project: importer/flask_learn

def dep(args):
    """
    auto generate script dependency
    """
    config.setupConf(args.cfg_file)
    initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "dep.log"), config.G_Conf.Common.MysqlConn)

    if args.query:
        module_scheduler.Admin().query_dep(args.job_id, args.date)
    else:
        if os.path.isfile(args.script):
            gen_dep(args.script)
        elif os.path.isdir(args.script):
            for root, dirs, files in os.walk(args.script, followlinks=True):
                for f in files:
                    filepath = os.path.join(root, f)
                    if not os.path.isfile(filepath):
                        continue
                    mod_name, file_ext = os.path.splitext(
                        os.path.split(filepath)[-1])
                    if file_ext != '.py':
                        continue
                    gen_dep(filepath)

Example #18

File: cli.py Project: importer/flask_learn

def init_db(args):
    config.setupConf(args.cfg_file)
    initServer("{}/{}".format(config.G_Conf.Worker.LogDir, config.G_Conf.Worker.LogFile), config.G_Conf.Common.MysqlConn)
    Base.metadata.drop_all(db.engine) 
    Base.metadata.create_all(db.engine)