def main():
""" Driver utility for the script."""
global logger
args = load_arguments()
logger = util_log.logger_setup(__name__,
log_file=args.log_file,
formatter=util_log.FORMATTER_4,
isrotate=True)
log("Daemon", "start ", os.getpid())
while True:
log("Daemon new loop", args.task_folder)
folders = get_list_valid_task_folder(args.task_folder)
if folders:
log("task folder:", folders)
pid_list = util_batch.batch_run_infolder(task_folders=folders,
log_file=args.log_file)
log("task folder started:", pid_list)
if args.mode != "daemon":
log("Daemon", "terminated", os.getpid())
break
sleep(args.waitsec)
def main3():
""" Driver utility for the script.
global_task_file contains the list of task ALREADY PROCESSED.
if a task is pick up --> this file is updated Globally.
"""
global logger
args = load_arguments()
logger = util_log.logger_setup(__name__,
log_file=args.log_file,
formatter=util_log.FORMATTER_4,
isrotate=True)
log("Daemon", "start ", os.getpid())
folder_main = args.task_folder
global_task_file = args.global_task_file
if not os.path.isdir(folder_main):
return 0
folder_check = json.load(open(global_task_file, mode="r"))
while True:
# log(folder_check)
log("Daemon new loop", folder_main)
for folder in os.listdir(folder_main):
# log(folder)
if not isvalid_folder(folder_main, folder, folder_check,
global_task_file):
continue
t0 = time.time()
folder_check = json.load(open(global_task_file,
mode="r")) # Refresh Global file
if folder not in folder_check:
global_task_file_save(
folder, folder_check,
global_task_file) # Update to prevent 2nd pick up
log("time to save", time.time() - t0)
folder = os.path.join(folder_main, folder)
files = [
file for file in os.listdir(folder)
if file == "main.sh" or file == "main.py"
]
log(files)
if files:
pid = subprocess_launch(folder, files[0])
log("task folder started:", folder, files[0], pid)
sleep(20)
os_wait_policy(waitsleep=10)
if args.mode != "daemon":
log("Daemon", "terminated", os.getpid())
break
sleep(args.waitsec)
os_wait_policy(waitsleep=5)
def main2():
""" Driver utility for the script."""
global logger
args = load_arguments()
logger = util_log.logger_setup(__name__,
log_file=args.log_file,
formatter=util_log.FORMATTER_4,
isrotate=True)
log("Daemon", "start ", os.getpid())
folder_main = args.task_folder
while True:
log("Daemon new loop", folder_main)
if not os.path.isdir(folder_main):
break
for root, dirs, files in os.walk(folder_main):
root_splits = root.split("/")
f = root_splits[-1]
for filename in files:
if (filename == "main.sh"
or filename == "main.py" and "_qstart" not in f
and "_qdone" not in f and "_ignore" not in f):
try:
#### Issue of collision if 2 instances rename the folder
folder_new = root + "_qstart"
os.rename(root, folder_new)
pid = subprocess_launch(folder_new, filename)
log("task folder started:", folder_new, pid)
except:
pass
os_wait_policy(waitsleep=5)
if args.mode != "daemon":
log("Daemon", "terminated", os.getpid())
break
sleep(args.waitsec)
os_wait_policy(waitsleep=5)
def autoscale_main():
### Variable initialization #####################################################
arg = load_arguments()
# ISTEST = not arg.prod
if not exists_dir(os.path.dirname(arg.log_file)):
# try and catch block for errors and bail out
os.makedirs(os.path.dirname(arg.log_file))
logger = logger_setup(__name__,
log_file=arg.log_file,
formatter=util_log.FORMATTER_4,
isrotate=True)
# print("arg input", arg)
log(MAIN_INSTANCE_TO_PROTECT, MAIN_INSTANCE_IP)
key_file = ec2_keypair_get(arg.keypair)
global_task_file = arg.global_task_file
if arg.reset_global_task_file:
task_globalfile_reset(global_task_file)
log("Daemon", "start: ", os.getpid(), global_task_file)
ii = 0
while True:
log("Daemon", "tasks folder: ", arg.task_folder)
# Retrieve tasks from github ##############################################
if ii % 5 == 0:
task_new, task_added = task_get_from_github(
repourl=arg.task_repourl,
reponame=arg.task_reponame,
branch=arg.task_repobranch,
to_task_folder=arg.task_s3_folder,
tmp_folder=arg.task_local_folder,
)
log("task", "new from github", task_added)
# Keep Global state of running instances
INSTANCE_DICT = ec2_instance_getallstate(arg.instance, key_file)
### Start instance by rules ###############################################
start_instance = instance_start_rule(arg.task_folder, global_task_file)
log("Instances to start", start_instance)
if start_instance:
# When instance start, batchdaemon will start and picks up task in COMMON DRIVE /zs3drive/
instance_list = ec2_spot_start(arg.ami, start_instance["type"],
start_instance["spotprice"],
arg.region, arg.spot_cfg_file,
arg.keypair)
log("Instances started", instance_list)
INSTANCE_DICT = ec2_instance_getallstate(arg.instance, key_file)
log("Instances running", INSTANCE_DICT)
##### Launch Batch system by No Blocking SSH #########################
ec2_instance_initialize_ssh(arg, key_file)
sleep(10)
### Stop instance by rules ################################################
stop_instances = instance_stop_rule(arg.task_folder,
arg.global_task_file, arg.instance,
key_file)
log("Instances to be stopped", stop_instances)
if stop_instances:
stop_instances_list = [v["id"] for v in stop_instances]
ec2_instance_backup(
stop_instances_list,
folder_list=arg.
folder_to_backup, # ["/home/ubuntu/zlog/", "/home/ubuntu/tasks_out/" ],
folder_backup=arg.backup_s3_folder,
) # "/home/ubuntu/zs3drive/backup/"
ec2_instance_stop(stop_instances_list)
log("Stopped instances", stop_instances_list)
### Upload results to github ##############################################
ii = ii + 1
if ii % 10 == 0: # 10 mins Freq
task_new, task_added = task_put_to_github(
repourl=arg.
taskout_repourl, # "https://github.com/arita37/tasks_out.git"
branch=arg.taskout_repobranch, # "tasks_out", branch="dev",
from_taskout_folder=arg.
taskout_s3_folder, # "/home/ubuntu/zs3drive/tasks_out/"
repo_folder=arg.taskout_local_folder,
) # "/home/ubuntu/data/github_tasks_out/"
log("task", "Add results to github", task_added)
### No Daemon mode ######################################################
if arg.mode != "daemon":
log("Daemon", "No Daemon mode", "terminated daemon")
break
sleep(arg.waitsec)