def scheduler(args):
config.setupConf(args.cfg_file)
logfile = "{}/{}".format(config.G_Conf.Scheduler.LogDir, config.G_Conf.Scheduler.LogFile)
if args.ha:
logfile = "{}.ha".format(logfile)
initServer(logfile, config.G_Conf.Common.MysqlConn)
main_scheduler = module_scheduler.MainScheduler(
broker_url=config.G_Conf.Common.Broker,
fetch_interval=config.G_Conf.Scheduler.FetchInterval,
woker_timeout=config.G_Conf.Scheduler.WorkerTimeOut,
retry_push_times=config.G_Conf.Scheduler.RetryQueueimes)
# signal.signal(signal.SIGINT, sigint_handler)
# signal.signal(signal.SIGTERM, sigint_handler)
# signal.signal(signal.SIGQUIT, sigquit_handler)
if args.ha:
#haserver = ha.HaServer(host="etcd.in.codoon.com", port=2379)
"""
start a main process for health checking, sub process for scheduler
"""
haserver = ha.HaServer('127.0.0.1')
haserver.run()
else:
main_scheduler.run()
def scheduler(args):
config.setupConf(args.cfg_file)
logfile = "{}/{}".format(config.G_Conf.Scheduler.LogDir, config.G_Conf.Scheduler.LogFile)
if args.ha:
logfile = "{}.ha".format(logfile)
initServer(logfile, config.G_Conf.Common.MysqlConn)
main_scheduler = module_scheduler.MainScheduler(
broker_url=config.G_Conf.Common.Broker,
fetch_interval=config.G_Conf.Scheduler.FetchInterval,
woker_timeout=config.G_Conf.Scheduler.WorkerTimeOut,
retry_push_times=config.G_Conf.Scheduler.RetryQueueimes)
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
signal.signal(signal.SIGQUIT, sigquit_handler)
if args.ha:
#haserver = ha.HaServer(host="etcd.in.codoon.com", port=2379)
"""
start a main process for health checking, sub process for scheduler
"""
haserver = ha.HaServer()
haserver.run()
else:
main_scheduler.run()
def init_db(args):
config.setupConf(args.cfg_file)
initServer(
"{}/{}".format(config.G_Conf.Worker.LogDir,
config.G_Conf.Worker.LogFile),
config.G_Conf.Common.MysqlConn)
Base.metadata.drop_all(db.engine)
Base.metadata.create_all(db.engine)
def kill(args):
err, date_list = is_valid_run_args(args)
if err:
print err
return
print "kill task {} with date: {}".format(args.job_id, " ".join(date_list))
config.setupConf(args.cfg_file)
initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "kill.log"), config.G_Conf.Common.MysqlConn)
cnt = module_scheduler.Admin().kill_task(args.job_id, date_list)
print "kill {} instance".format(cnt)
def kill(args):
err, date_list = is_valid_run_args(args)
if err:
print (err)
return
print ("kill task {} with date: {}".format(args.job_id, " ".join(date_list)))
config.setupConf(args.cfg_file)
initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "kill.log"), config.G_Conf.Common.MysqlConn)
cnt = module_scheduler.Admin().kill_task(args.job_id, date_list)
print ("kill {} instance".format(cnt))
def executer(args):
config.setupConf(args.cfg_file)
initServer("{}/{}".format(config.G_Conf.Worker.LogDir, config.G_Conf.Worker.LogFile), config.G_Conf.Common.MysqlConn)
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
signal.signal(signal.SIGQUIT, sigquit_handler)
task_worker = worker.Worker(
parallelism=config.G_Conf.Worker.Parallelism,
heartbeat_interval=config.G_Conf.Worker.HearteatInterval,
woker_timeout=config.G_Conf.Worker.TaskTimeOut,
retry_times=config.G_Conf.Worker.Retry)
task_worker.run()
def executer(args):
config.setupConf(args.cfg_file)
initServer("{}/{}".format(config.G_Conf.Worker.LogDir, config.G_Conf.Worker.LogFile), config.G_Conf.Common.MysqlConn)
# signal.signal(signal.SIGINT, sigint_handler)
# signal.signal(signal.SIGTERM, sigint_handler)
# signal.signal(signal.SIGQUIT, sigquit_handler)
task_worker = worker.Worker(
parallelism=config.G_Conf.Worker.Parallelism,
heartbeat_interval=config.G_Conf.Worker.HearteatInterval,
woker_timeout=config.G_Conf.Worker.TaskTimeOut,
retry_times=config.G_Conf.Worker.Retry)
task_worker.run()
def do_all_job(args):
"""
do_all_job run all scheduler job
:param args:
:return:
"""
if not isValiddate(args.date):
err = "time date format is YYYY-MM-DD, give {}".format(args.date)
print (err)
return
print ("run all schedule_job with date: {}".format(args.date))
config.setupConf(args.cfg_file)
initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "do_all_job.log"), config.G_Conf.Common.MysqlConn)
msg = module_scheduler.Admin().run_all_job([args.date])
print ("run success, {}".format(msg))
def do_all_job(args):
"""
do_all_job run all scheduler job
:param args:
:return:
"""
if not isValiddate(args.date):
err = "time date format is YYYY-MM-DD, give {}".format(args.date)
print err
return
print "run all schedule_job with date: {}".format(args.date)
config.setupConf(args.cfg_file)
initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "do_all_job.log"), config.G_Conf.Common.MysqlConn)
msg = module_scheduler.Admin().run_all_job([args.date])
print "run success, {}".format(msg)
def migrate(args):
"""
migrate dependency , extract last_id
:param args:
:return:
"""
config.setupConf(args.cfg_file)
initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "migrate.log"), config.G_Conf.Common.MysqlConn)
if args.dependency:
module_scheduler.Admin().migrate_dependency(None, args.delete)
if args.last_id:
if not isValiddate(args.date):
print ("date format is YYYY-MM-DD, give {}".format(args.date))
return
d = args.date.replace("-", "")
module_scheduler.Admin().migrate_lastid(d)
def migrate(args):
"""
migrate dependency , extract last_id
:param args:
:return:
"""
config.setupConf(args.cfg_file)
initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "migrate.log"), config.G_Conf.Common.MysqlConn)
if args.dependency:
module_scheduler.Admin().migrate_dependency(None, args.delete)
if args.last_id:
if not isValiddate(args.date):
print "date format is YYYY-MM-DD, give {}".format(args.date)
return
d = args.date.replace("-", "")
module_scheduler.Admin().migrate_lastid(d)
def run(args):
"""
rerun task
:param args:
:return:
"""
err, date_list = is_valid_run_args(args)
if err:
print err
return
print "run task {} with date: {}".format(args.job_id, " ".join(date_list))
config.setupConf(args.cfg_file)
initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "rerun.log"), config.G_Conf.Common.MysqlConn)
msg = module_scheduler.Admin().rerun_task(args.job_id, date_list,
up_and_down=args.with_up_down,
run_up=args.with_upstream,
run_down=args.with_downstream,
force=args.force)
print "run success, {}".format(msg)
def do_all_extract(args):
"""
do_all_extract
:param args:
:return:
"""
if not isValiddate(args.date):
err = "time date format is YYYY-MM-DD, give {}".format(args.date)
print err
return
msg = "run all extract with date: {}".format(args.date)
if args.dest_table is not None:
msg = "{}, dest_table: {}".format(msg, args.dest_table)
print msg
config.setupConf(args.cfg_file)
initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "do_all_extract.log"), config.G_Conf.Common.MysqlConn)
msg = module_scheduler.Admin().run_all_extract([args.date], dest_table=args.dest_table)
print "run success, {}".format(msg)
def do_all_extract(args):
"""
do_all_extract
:param args:
:return:
"""
if not isValiddate(args.date):
err = "time date format is YYYY-MM-DD, give {}".format(args.date)
print (err)
return
msg = "run all extract with date: {}".format(args.date)
if args.dest_table is not None:
msg = "{}, dest_table: {}".format(msg, args.dest_table)
print (msg)
config.setupConf(args.cfg_file)
initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "do_all_extract.log"), config.G_Conf.Common.MysqlConn)
msg = module_scheduler.Admin().run_all_extract([args.date], dest_table=args.dest_table)
print ("run success, {}".format(msg))
def run(args):
"""
rerun task
:param args:
:return:
"""
err, date_list = is_valid_run_args(args)
if err:
print (err)
return
print ("run task {} with date: {}".format(args.job_id, " ".join(date_list)))
config.setupConf(args.cfg_file)
initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "rerun.log"), config.G_Conf.Common.MysqlConn)
msg = module_scheduler.Admin().rerun_task(args.job_id, date_list,
up_and_down=args.with_up_down,
run_up=args.with_upstream,
run_down=args.with_downstream,
force=args.force)
print ("run success, {}".format(msg))
def dep(args):
"""
auto generate script dependency
"""
config.setupConf(args.cfg_file)
initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "dep.log"), config.G_Conf.Common.MysqlConn)
if args.query:
module_scheduler.Admin().query_dep(args.job_id, args.date)
else:
if os.path.isfile(args.script):
gen_dep(args.script)
elif os.path.isdir(args.script):
for root, dirs, files in os.walk(args.script, followlinks=True):
for f in files:
filepath = os.path.join(root, f)
if not os.path.isfile(filepath):
continue
mod_name, file_ext = os.path.splitext(
os.path.split(filepath)[-1])
if file_ext != '.py':
continue
gen_dep(filepath)
def dep(args):
"""
auto generate script dependency
"""
config.setupConf(args.cfg_file)
initServer("{}/{}".format(config.G_Conf.Scheduler.LogDir, "dep.log"), config.G_Conf.Common.MysqlConn)
if args.query:
module_scheduler.Admin().query_dep(args.job_id, args.date)
else:
if os.path.isfile(args.script):
gen_dep(args.script)
elif os.path.isdir(args.script):
for root, dirs, files in os.walk(args.script, followlinks=True):
for f in files:
filepath = os.path.join(root, f)
if not os.path.isfile(filepath):
continue
mod_name, file_ext = os.path.splitext(
os.path.split(filepath)[-1])
if file_ext != '.py':
continue
gen_dep(filepath)
def init_db(args):
config.setupConf(args.cfg_file)
initServer("{}/{}".format(config.G_Conf.Worker.LogDir, config.G_Conf.Worker.LogFile), config.G_Conf.Common.MysqlConn)
Base.metadata.drop_all(db.engine)
Base.metadata.create_all(db.engine)