def os_folder_copy(from_folder_root, to_folder, isoverwrite=False, exclude_flag="ignore"): """ Copy with criteria """ log("Folder copy - src: %s" % from_folder_root, " to dst: %s" % to_folder, " overwrite: %s" % str(isoverwrite), " exclude: %s" % str(exclude_flag)) task_list_added, task_list = [], [] for f in os.listdir(from_folder_root): from_f = from_folder_root + f to_f = to_folder + f + "/" # Conditions of copy if os.path.isdir(from_f) and f not in {".git" } and exclude_flag not in f: if not os.path.exists(to_f) or isoverwrite: os_system("cp -r {f1} {f2}".format(f1=from_f, f2=to_f)) log("Copy - from: %s" % from_f, " to: %s" % to_f) if os.path.exists(to_f): task_list_added.append(to_f) task_list.append(from_f) else: print("Error copy", from_f, to_f) log("Copy Error - to: %s does not exist" % to_f) log(' task list: %s' % ','.join(task_list), ' task_list_added: %s' % ','.join(task_list_added)) return task_list, task_list_added
def ec2_instance_stop(instance_list): """ Stop the spot instances ainstances u stop any other instance, this should work """ instance_list = [ t for t in instance_list if t not in MAIN_INSTANCE_TO_PROTECT ] instances = instance_list if instances: if isinstance(instance_list, list): instances = ",".join(instance_list) cmdargs = [ "aws", "ec2", "terminate-instances", "--instance-ids", instances ] cmd = " ".join(cmdargs) log('Spot stop AWS CLI: %s' % cmd) os_system(cmd) return instances.split(",")
def task_getcount_cpurequired(folder_main, global_task_file): """ ncpu_required defined in task_config.py """ task_list = task_get_list_valid_folder_new(folder_main, global_task_file) ncpu_all = 0 for f in task_list: cmds = "python {a}/task_config.py --do ncpu_required ".format( a=folder_main + "/" + f) msg = os_system(cmds) ncpu = tofloat(msg, default=1.0) ncpu_all += ncpu return ncpu_all
def ec2_spot_start(amiId, instance_type, spot_price, region='us-west-2', spot_cfg_file='/home/ubuntu/test/ec_spot_config', keypair=None, waitsec=100): """ Request a spot instance based on the price for the instance type # Need a check if this request has been successful. 100 sec to be provisionned and started. """ if not instance_type: instance_type = 't3.small' ec2_config_build_template(amiId, instance_type, spot_cfg_file, keypair) cmdargs = [ "aws", "ec2", "request-spot-instances", "--region", region, "--spot-price", str(spot_price), "--instance-count", "1", " --type", "one-time", "--launch-specification", "file://%s" % spot_cfg_file, ] cmd = " ".join(cmdargs) log('Spot request AWS CLI: %s' % cmd) os_system(cmd) sleep(waitsec) # It may not be fulfilled in 50 secs. ll = ec2_spot_instance_list() return ll["SpotInstanceRequests"] if "SpotInstanceRequests" in ll else []
def task_get_from_github(repourl, reponame="tasks", branch="dev", to_task_folder="/home/ubuntu/zs3drive/tasks/", tmp_folder="/home/ubuntu/data/ztmp/"): """ Get tasks from github repo Retrieve tasks folder from Github repo and write on S3 drive for automatic processing. rm folder git clone https://github.com/arita37/tasks.git --branch <name> git checkout branch for folder1 in subfolder: cp folder1 folder_s3 """ # Git pull if not os.path.exists(tmp_folder): log('Creating tmp folder: %s' % tmp_folder) os.mkdir(tmp_folder) repo_folder = tmp_folder + os.sep + reponame + os.sep if to_task_folder and not to_task_folder.endswith(os.sep): to_task_folder = os.path.join(to_task_folder, os.sep) log("check", tmp_folder) os_system("rm -rf " + repo_folder) # cmds = "cd {a} && git clone {b} {c}".format(a=tmp_folder, b=repourl, c=reponame) cmds = "git clone {b} {c}".format(b=repourl, c=reponame) if branch: # Checkout the branch directly. cmds = "{a} --branch {b}".format(a=cmds, b=branch) log(cmds) os.chdir(tmp_folder) os_system(cmds) # Copy task_list, task_list_added = os_folder_copy(repo_folder, to_task_folder) # Rename folder to ignore and commit for f in task_list: os.rename(f, f + "_ignore" if f[-1] != "/" else f[:-1] + "_ignore") # cmds = " cd {a} && git add --all && git commit -m 'S3 copied '".format(a=repo_folder) cmds = "git add --all && git commit -m 'S3 copied '" cmds += " && git push --all --force " log(cmds) os.chdir(repo_folder) os_system(cmds) return task_list, task_list_added
def ec2_instance_backup(instances_list, folder_list=["zlog/"], folder_backup="/home/ubuntu/zs3drive/backup/"): """ Zip some local folders Transfer data from local to /zs3drive/backup/AMIname_YYYYMMDDss/ tar -czvf directorios.tar.gz folder """ now = datetime.today().strftime("%Y%m%d") for idx in instances_list: target_folder = folder_backup + "/a" + now + "_" + idx if not os.path.exists(target_folder): os.mkdir(target_folder) for f in folder_list: # fname = f.split("/")[-1] cmds = "cp -r {a} {b}".format(a=f, b=target_folder) msg = os_system(cmds) log('Backup commands: %s' % cmds, 'Command Response: %s' % msg) """
def task_put_to_github(repourl, branch="dev", from_taskout_folder="/home/ubuntu/zs3drive/tasks_out/", repo_folder="/home/ubuntu/data/github_tasks_out/"): """ Put results back to github git clone https://github.com/arita37/tasks_out.git github_tasks_out git pull --all copy S3 to github_tasks_out git add --all, push all """ if os.path.exists(repo_folder): cmds = " cd {a} && git pull --all ".format(a=repo_folder) cmds += " && git checkout {b}".format(b=branch) log("Git Pull results: %s" % cmds) os_system(cmds) else: cmds = "git clone {a} {b}".format(a=repourl, b=repo_folder) if branch: # Checkout the branch directly. cmds = "{a} --branch {b}".format(a=cmds, b=branch) log("Git clone: %s" % cmds) os_system(cmds) ### Copy with OVERWRITE task_list, task_list_added = os_folder_copy(from_taskout_folder, repo_folder, isoverwrite=True) ### Git push cmds = "cd {a} && git add --all".format(a=repo_folder) cmds += " && git commit -m 'oo{b}'".format(b=",".join(task_list_added)) cmds += " && git push --all --force " log("Git push task results: %s" % cmds) os_system(cmds) return task_list, task_list_added