def os_folder_copy(from_folder_root,
to_folder,
isoverwrite=False,
exclude_flag="ignore"):
"""
Copy with criteria
"""
log("Folder copy - src: %s" % from_folder_root, " to dst: %s" % to_folder,
" overwrite: %s" % str(isoverwrite),
" exclude: %s" % str(exclude_flag))
task_list_added, task_list = [], []
for f in os.listdir(from_folder_root):
from_f = from_folder_root + f
to_f = to_folder + f + "/"
# Conditions of copy
if os.path.isdir(from_f) and f not in {".git"
} and exclude_flag not in f:
if not os.path.exists(to_f) or isoverwrite:
os_system("cp -r {f1} {f2}".format(f1=from_f, f2=to_f))
log("Copy - from: %s" % from_f, " to: %s" % to_f)
if os.path.exists(to_f):
task_list_added.append(to_f)
task_list.append(from_f)
else:
print("Error copy", from_f, to_f)
log("Copy Error - to: %s does not exist" % to_f)
log(' task list: %s' % ','.join(task_list),
' task_list_added: %s' % ','.join(task_list_added))
return task_list, task_list_added
def ec2_instance_stop(instance_list):
""" Stop the spot instances ainstances u stop any other instance, this should work
"""
instance_list = [
t for t in instance_list if t not in MAIN_INSTANCE_TO_PROTECT
]
instances = instance_list
if instances:
if isinstance(instance_list, list):
instances = ",".join(instance_list)
cmdargs = [
"aws", "ec2", "terminate-instances", "--instance-ids", instances
]
cmd = " ".join(cmdargs)
log('Spot stop AWS CLI: %s' % cmd)
os_system(cmd)
return instances.split(",")
def task_getcount_cpurequired(folder_main, global_task_file):
"""
ncpu_required defined in task_config.py
"""
task_list = task_get_list_valid_folder_new(folder_main, global_task_file)
ncpu_all = 0
for f in task_list:
cmds = "python {a}/task_config.py --do ncpu_required ".format(
a=folder_main + "/" + f)
msg = os_system(cmds)
ncpu = tofloat(msg, default=1.0)
ncpu_all += ncpu
return ncpu_all
def ec2_spot_start(amiId,
instance_type,
spot_price,
region='us-west-2',
spot_cfg_file='/home/ubuntu/test/ec_spot_config',
keypair=None,
waitsec=100):
"""
Request a spot instance based on the price for the instance type
# Need a check if this request has been successful.
100 sec to be provisionned and started.
"""
if not instance_type:
instance_type = 't3.small'
ec2_config_build_template(amiId, instance_type, spot_cfg_file, keypair)
cmdargs = [
"aws",
"ec2",
"request-spot-instances",
"--region",
region,
"--spot-price",
str(spot_price),
"--instance-count",
"1",
" --type",
"one-time",
"--launch-specification",
"file://%s" % spot_cfg_file,
]
cmd = " ".join(cmdargs)
log('Spot request AWS CLI: %s' % cmd)
os_system(cmd)
sleep(waitsec) # It may not be fulfilled in 50 secs.
ll = ec2_spot_instance_list()
return ll["SpotInstanceRequests"] if "SpotInstanceRequests" in ll else []
def task_get_from_github(repourl,
reponame="tasks",
branch="dev",
to_task_folder="/home/ubuntu/zs3drive/tasks/",
tmp_folder="/home/ubuntu/data/ztmp/"):
"""
Get tasks from github repo
Retrieve tasks folder from Github repo and write on S3 drive for automatic processing.
rm folder
git clone https://github.com/arita37/tasks.git --branch <name>
git checkout branch
for folder1 in subfolder:
cp folder1 folder_s3
"""
# Git pull
if not os.path.exists(tmp_folder):
log('Creating tmp folder: %s' % tmp_folder)
os.mkdir(tmp_folder)
repo_folder = tmp_folder + os.sep + reponame + os.sep
if to_task_folder and not to_task_folder.endswith(os.sep):
to_task_folder = os.path.join(to_task_folder, os.sep)
log("check", tmp_folder)
os_system("rm -rf " + repo_folder)
# cmds = "cd {a} && git clone {b} {c}".format(a=tmp_folder, b=repourl, c=reponame)
cmds = "git clone {b} {c}".format(b=repourl, c=reponame)
if branch:
# Checkout the branch directly.
cmds = "{a} --branch {b}".format(a=cmds, b=branch)
log(cmds)
os.chdir(tmp_folder)
os_system(cmds)
# Copy
task_list, task_list_added = os_folder_copy(repo_folder, to_task_folder)
# Rename folder to ignore and commit
for f in task_list:
os.rename(f, f + "_ignore" if f[-1] != "/" else f[:-1] + "_ignore")
# cmds = " cd {a} && git add --all && git commit -m 'S3 copied '".format(a=repo_folder)
cmds = "git add --all && git commit -m 'S3 copied '"
cmds += " && git push --all --force "
log(cmds)
os.chdir(repo_folder)
os_system(cmds)
return task_list, task_list_added
def ec2_instance_backup(instances_list,
folder_list=["zlog/"],
folder_backup="/home/ubuntu/zs3drive/backup/"):
"""
Zip some local folders
Transfer data from local to /zs3drive/backup/AMIname_YYYYMMDDss/
tar -czvf directorios.tar.gz folder
"""
now = datetime.today().strftime("%Y%m%d")
for idx in instances_list:
target_folder = folder_backup + "/a" + now + "_" + idx
if not os.path.exists(target_folder):
os.mkdir(target_folder)
for f in folder_list:
# fname = f.split("/")[-1]
cmds = "cp -r {a} {b}".format(a=f, b=target_folder)
msg = os_system(cmds)
log('Backup commands: %s' % cmds, 'Command Response: %s' % msg)
"""
def task_put_to_github(repourl,
branch="dev",
from_taskout_folder="/home/ubuntu/zs3drive/tasks_out/",
repo_folder="/home/ubuntu/data/github_tasks_out/"):
"""
Put results back to github
git clone https://github.com/arita37/tasks_out.git github_tasks_out
git pull --all
copy S3 to github_tasks_out
git add --all, push all
"""
if os.path.exists(repo_folder):
cmds = " cd {a} && git pull --all ".format(a=repo_folder)
cmds += " && git checkout {b}".format(b=branch)
log("Git Pull results: %s" % cmds)
os_system(cmds)
else:
cmds = "git clone {a} {b}".format(a=repourl, b=repo_folder)
if branch:
# Checkout the branch directly.
cmds = "{a} --branch {b}".format(a=cmds, b=branch)
log("Git clone: %s" % cmds)
os_system(cmds)
### Copy with OVERWRITE
task_list, task_list_added = os_folder_copy(from_taskout_folder,
repo_folder,
isoverwrite=True)
### Git push
cmds = "cd {a} && git add --all".format(a=repo_folder)
cmds += " && git commit -m 'oo{b}'".format(b=",".join(task_list_added))
cmds += " && git push --all --force "
log("Git push task results: %s" % cmds)
os_system(cmds)
return task_list, task_list_added