def get_platform(provider_name: str = None,
platform_name: str = None,
stdout: bool = False,
cloud_profile: str = None,
vars: Dict = {},
**kwargs) -> ModuleType:
global PLATFORM_MODULE
if PLATFORM_MODULE:
return PLATFORM_MODULE
state = _get_state()
if not provider_name:
provider_name = state.get(CLOUD_PROVIDER)
if not platform_name:
platform_name = state.get(CLOUD_PLATFORM)
if not provider_name or not platform_name:
raise Exception("You need to set provider_name and platform_name")
PLATFORM_MODULE = _import_module("handoff.services.cloud." + provider_name)
cred_keys = PLATFORM_MODULE.find_cred_keys(vars)
response = PLATFORM_MODULE.login(cloud_profile, cred_keys=cred_keys)
if not response:
raise Exception(
f"Login to {provider_name} failed. Credentials may not be set correctly."
)
return PLATFORM_MODULE
def role_update(project_dir: str,
workspace_dir: str,
vars: Dict = {},
**kwargs) -> None:
"""`handoff cloud role update -p <project_directory> -v external_id=<id> grantee_account_id=<grantee_id>`
Update the role privilege information.
"""
state = _get_state()
platform = get_platform()
account_id = platform.get_account_id()
if not account_id:
raise Exception("Login failed")
# state.validate_env()
if not vars.get("grantee_account_id"):
LOGGER.warn("grantee_account_id was not set." +
"The grantee will be set for the same account. To set: ")
LOGGER.warn("-v grantee_account_id=xxxx")
if not vars.get("external_id"):
raise ValueError("external_id must be set. Do as:\n " +
"handoff cloud create_role -p <project-vir> " +
"-v external_id=yyyy")
return platform.update_role(grantee_account_id=str(
vars.get("grantee_account_id", account_id)),
external_id=vars.get("external_id"))
def run(project_dir: str,
workspace_dir: str,
envs: Dict = {},
vars: Dict = {},
extras: str = None,
**kwargs) -> None:
"""`handoff cloud run -v resource_group=<resource_group_name> task=<task_name> target_id=<target_id> -e vars='key1=val1 key2=val2...'`
Run a task once in the platform
If the environment variable vars is specified via -e option, it will be
used as:
`handoff run -v $(eval echo $vars)`
"""
state = _get_state()
platform = get_platform()
config = admin._config_get_local(project_dir, workspace_dir)
state.validate_env()
extras_obj = None
if extras:
with open(extras, "r") as f:
extras_obj = yaml.load(f)
target_id = vars.get("target_id")
target_envs = dict()
if target_id:
for s in config.get("schedules", []):
if str(s["target_id"]) == target_id:
for e in s.get("envs"):
target_envs[e["key"]] = e["value"]
break
target_envs.update(envs)
target_envs[STAGE] = state[STAGE]
return platform.run_task(env=target_envs, extras=extras_obj)
def _get_github(access_token=None):
state = _get_state()
if not access_token:
access_token = state.get(GITHUB_ACCESS_TOKEN)
global GITHUB
GITHUB = Github(access_token)
return GITHUB
def logs(project_dir: str,
workspace_dir: str,
vars: Dict = {},
**kwargs) -> None:
"""`handoff cloud logs -v start_time=<start_time> end_time=<end_time> follow=<True/False>`
Show logs
Use --vars (-v) option to:
- start_time: ISO 8086 formatted date time to indicate the start time
- end_time
- follow: If set, it waits for more logs until interrupted by ctrl-c
- filter: Filter log term
"""
state = _get_state()
platform = get_platform()
state.validate_env()
close = False
if vars.get("file"):
file_descriptor = open(vars.pop("file"), "a")
close = True
else:
file_descriptor = sys.stdout
if vars.get("role_arn"):
vars.pop("role_arn")
platform.write_logs(file_descriptor, **vars)
if close:
file_descriptor.close()
def clone(
project_dir: str,
workspace_dir: str,
vars: Dict = {},
**kwargs) -> None:
"""`handoff github clone -v organization=<github_org> repository=<github_repo> local_dir=<local_dir> force=False`
Clone remote repository. If local_dir is omitted, ./<repository> is used.
If force is set to True, an existing directory will be deleted before cloning.
"""
state = _get_state()
access_token = vars.get("access_token", state.get(GITHUB_ACCESS_TOKEN))
github = _get_github(access_token)
repo_name = vars["repository"]
org_name = vars.get("organization")
local_dir = vars.get("local_dir", "./" + repo_name)
if os.path.exists(local_dir):
if not vars.get("force", False):
raise Exception("The directory already exists.")
shutil.rmtree(local_dir)
git_url = vars.get("url", "https://github.com/" + str(org_name) + "/" + repo_name + ".git")
if vars.get("use_cli", False):
LOGGER.debug("Running git CLI")
git_url = git_url.replace("https://", f"https://{access_token}:x-oauth-basic@")
git_path = os.environ.get("GIT_PATH", "git")
os.system(f"{git_path} clone {git_url}")
else:
repo_clone = pygit2.clone_repository(
git_url, local_dir, callbacks=_get_callbacks(access_token))
return {"status": "success", "repository": repo_name}
def task_delete(project_dir: str, workspace_dir: str, **kwargs) -> None:
"""`handoff cloud task delete -p <project_directory>`
Delete the task
"""
state = _get_state()
platform = get_platform()
state.validate_env()
return platform.delete_task()
def bucket_delete(project_dir: str, workspace_dir: str, **kwargs) -> None:
"""`handoff cloud bucket delete -p <project_directory>`
Delete remote storage bucket.
"""
state = _get_state()
platform = get_platform()
state.validate_env()
return platform.delete_bucket()
def bucket_create(project_dir: str, workspace_dir: str, **kwargs) -> None:
"""`handoff cloud bucket create -p <project_directory>`
Create remote storage bucket. Bucket is attached to the resource group.
"""
state = _get_state()
platform = get_platform()
state.validate_env()
return platform.create_bucket()
def get(project_dir: str,
workspace_dir: str,
vars: Dict = {},
**kwargs) -> None:
"""`handoff envs get -p <project_dir> -d key=<env_var_key>`
Get the value of an evirohment varaible specified by <env_var_key>
"""
state = _get_state()
print(state[vars["key"]])
def resources_delete(project_dir: str, workspace_dir: str, **kwargs) -> None:
"""`handoff cloud resources delete -p <project_directory>`
Delete the resources
The resources are shared among the tasks under the same resource group.
"""
state = _get_state()
platform = get_platform()
state.validate_env()
return platform.delete_resources()
def task_update(project_dir: str,
workspace_dir: str,
vars: Dict = {},
**kwargs) -> None:
"""`handoff cloud task update -p <project_directory>`
Update the task
Optionally,
-v cpu=256, memory=512
"""
state = _get_state()
platform = get_platform()
state.validate_env([IMAGE_VERSION])
return platform.update_task(**vars)
def task_stop(project_dir: str,
workspace_dir: str,
vars: Dict = {},
**kwargs) -> None:
"""`handoff cloud task stop -p <project_directory> -v id=<task_id> reason=<reason>`
stop a running task
Options:
- reason
"""
state = _get_state()
platform = get_platform()
state.validate_env()
return platform.stop_task(**vars)
def task_status(project_dir: str,
workspace_dir: str,
vars: Dict = {},
**kwargs) -> None:
"""`handoff cloud task status -p <project_directory> -v full=False running=True stopped=True resource_group_level=False`
list task status
AWS options:
- full: When true, print the full description (default: false)
- running: When true, include desired status = RUNNING (default: true)
- stopped: When true, include desired status = STOPPED (default: true)
- resource_group_level: When true, list all the tasks under the same resource groups (default: false)
"""
state = _get_state()
platform = get_platform()
state.validate_env()
return platform.list_tasks(**vars)
def role_delete(project_dir: str,
workspace_dir: str,
vars: Dict = {},
**kwargs) -> None:
"""`handoff cloud role delete -p <project_directory> -v grantee_account_id=<grantee_id>`
Delete the role.
"""
state = _get_state()
platform = get_platform()
account_id = platform.get_account_id()
# state.validate_env()
if not vars.get("grantee_account_id"):
LOGGER.warn("grantee_account_id was not set." +
"The grantee will be set for the same account. To set: ")
LOGGER.warn("-v grantee_account_id=xxxx")
return platform.delete_role(
grantee_account_id=str(vars.get("grantee_account_id", account_id)))
def commit (
project_dir: str,
workspace_dir: str,
vars: Dict = {},
**kwargs) -> None:
"""`handoff github commit -v local_dir=<local_dir> commit_msg=<msg> branch=<branch>`
Commit all the outstanding changes to the remote branch. If the branch does not exist,
it will be created.
"""
state = _get_state()
access_token = vars.get("access_token", state.get(GITHUB_ACCESS_TOKEN))
github = _get_github(access_token)
local_dir = vars["local_dir"] # typically the "./{repository_name}"
repo = pygit2.Repository(local_dir + "/.git")
user = github.get_user()
branch = vars.get("branch", "master")
commit_msg = vars["commit_msg"]
# repo.remotes.set_url("origin", repo.clone_url)
index = repo.index
index.add_all()
index.remove_all("*.secrets*")
index.write()
author = pygit2.Signature(user.name, user.email)
commiter = pygit2.Signature(user.name, user.email)
tree = index.write_tree()
oid = repo.create_commit(
"refs/heads/" + branch,
author,
commiter,
commit_msg,
tree,
[repo.head.target],
# [repo.head.get_object().hex],
)
credentials = pygit2.UserPass(access_token, "x-oauth-basic")
# callbacks = pygit2.RemoteCallbacks(credentials=credentials)
remote = repo.remotes["origin"]
remote.credentials = credentials
remote.push(["refs/heads/" + branch], callbacks=_get_callbacks(access_token))
def schedule_list(project_dir: str,
workspace_dir: str,
vars: Dict = {},
**kwargs) -> None:
"""`handoff cloud schedule list`
List the scheduled tasks
"""
state = _get_state()
platform = get_platform()
state.validate_env()
schedules = platform.list_schedules(**vars)
target_ids = []
for s in schedules["schedules"]:
target_ids.append(str(s["target_id"]))
s["status"] = "scheduled"
if vars.get("include_unpublished"):
config = admin._config_get_local(project_dir, workspace_dir)
local_schedules = config.get("schedules", [])
for s in local_schedules:
status = ""
try:
index = target_ids.index(str(s["target_id"]))
except ValueError:
index = len(target_ids)
target_ids.append(s["target_id"])
schedules["schedules"].append({})
status = "draft"
remote = {}
remote.update(schedules["schedules"][index])
if remote.get("status"):
remote.pop("status")
if remote.get("name"):
remote.pop("name")
if s.get("description"):
s.pop("description")
if s != remote:
schedules["schedules"][index] = s
status = status or "edited"
schedules["schedules"][index]["status"] = status
return schedules
def resources_create(project_dir: str,
workspace_dir: str,
vars: Dict = {},
**kwargs) -> None:
"""`handoff cloud resources create -p <project_directory>`
Create resources necessary for task execution.
The resources are shared among the tasks under the same resource group.
Optionally:
-v static_ip=True: [AWS] Create Elastic IP and NAT Gateway to obtain a static IP. This [costs more](https://aws.amazon.com/vpc/pricing/).
AWS:
- Please refer to the
[CloudFormation template](https://github.com/anelenvars/handoff/blob/master/handoff/services/cloud/aws/cloudformation_templates/resources.yml) for the resources created with this command.
"""
state = _get_state()
platform = get_platform()
state.validate_env()
return platform.create_resources(**vars)
def schedule_delete(project_dir: str,
workspace_dir: str,
vars: Dict = {},
**kwargs) -> None:
"""`handoff cloud schedule delete -v target_id=<target_id>`
Unschedule a task named <target_id>
"""
state = _get_state()
platform = get_platform()
state.validate_env()
if not vars.get("target_id"):
return {
"status": "error",
"message": "Forgot to set '-v target_id=<ID>' ?",
}
target_id = str(vars["target_id"])
try:
response = platform.unschedule_task(target_id)
except Exception as e:
response = str(e)
return response
def pull(
project_dir: str,
workspace_dir: str,
vars: Dict = {},
**kwargs) -> None:
"""`handoff github commit -v local_dir=<local_dir> commit_msg=<msg> branch=<branch>`
Commit all the outstanding changes to the remote branch. If the branch does not exist,
it will be created.
"""
state = _get_state()
access_token = vars.get("access_token", state.get(GITHUB_ACCESS_TOKEN))
github = _get_github(access_token)
local_dir = vars["local_dir"] # typically the "./{repository_name}"
repo = pygit2.Repository(local_dir + "/.git")
remote_name = "origin"
# Adopted from https://github.com/MichaelBoselowitz/pygit2-examples/blob/68e889e50a592d30ab4105a2e7b9f28fac7324c8/examples.py#L48
for remote in repo.remotes:
if remote.name == remote_name:
remote.fetch()
remote_master_id = repo.lookup_reference('refs/remotes/origin/master').target
merge_result, _ = repo.merge_analysis(remote_master_id)
# Up to date, do nothing
if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE:
return {
"status": "success",
"message": "Repository is up-to-date",
}
# We can just fastforward
elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD:
repo.checkout_tree(repo.get(remote_master_id))
master_ref = repo.lookup_reference('refs/heads/master')
master_ref.set_target(remote_master_id)
repo.head.set_target(remote_master_id)
elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL:
repo.merge(remote_master_id)
if repo.index_conflicts:
return {
"status": "error",
"mesage": repo.index.conflicts,
}
user = repo.default_signature
tree = repo.index.write_tree()
commit = repo.create_commit('HEAD',
user,
user,
'Merge!',
tree,
[repo.head.target, remote_master_id])
repo.state_cleanup()
return {
"status": "success",
"message": "successfully fast forwarded the repository",
}
else:
return {
"status": "error",
"mesage": "Unknown merge analysis result"
}
def schedule_create(project_dir: str,
workspace_dir: str,
envs: Dict = {},
vars: Dict = {},
extras: str = None,
**kwargs) -> None:
"""`handoff cloud schedule create -v target_id=<target_id> cron="<cron_format>" -e vars='key1=val1 key2=val2...'`
Schedule a task named <target_id> at the recurring scheduled specified
as <cron_format>. An example of cron-format string is `10 01 * * ? *`
for every day at 01:10 (1:10AM)
If the environment variable vars is specified via -e option, it will be
used as:
`handoff run -v $(eval echo $vars)`
"""
state = _get_state()
platform = get_platform()
config = admin._config_get_local(project_dir, workspace_dir)
state.validate_env()
schedules = config.get("schedules")
target_id = vars.get("target_id")
cronexp = vars.get("cron")
extras_obj = None
if extras:
with open(extras, "r") as f:
extras_obj = yaml.load(f)
if not schedules:
schedules = [{
"target_id": target_id,
"cron": cronexp,
"extras_obj": extras_obj
}]
if not schedules and (not target_id or not cronexp):
return {
"status":
"error",
"message":
"Schedules not found in project.yml. You can also set "
"'-v target_id=<ID> cron=<CRON>'",
}
envs[STAGE] = state[STAGE]
responses = []
for s in schedules:
if target_id is not None and str(s["target_id"]) != str(target_id):
continue
if target_id is not None and cronexp:
s["cron"] = cronexp
e = dict()
for e1 in s.get("envs", list()):
e[e1["key"]] = e1["value"]
# priority of the values schedule section < envs section < command line
e.update(envs)
r = platform.schedule_task(str(s["target_id"]),
"cron(" + s["cron"] + ")",
env=e,
extras=s.get("extras_obj"))
responses.append(r)
return responses