def handle(request):
"""
Noop executor function definition. This handler function controls
all functionality
:param request: The request object from Flask
This object is required to have the following JSON parameters:
* run_id: The run_id of the task
* thread_id: The thread_id, from gman, that this execution is forked from.
* project: The project name of the run
* configs: A list containing the configuration dictionaries for the run.
* stage: The stage that is being run.
* artifacts: A list of dictionaries containing information on the artifacts
required for this run
:param request:
:return:
"""
run_id = request.get_json().get("run_id")
stage = request.get_json()["stage"]
configs = request.get_json()["configs"]
artifacts = request.get_json()["artifacts"]
task = g.task
access_key = read_secrets().get("access_key")
secret_key = read_secrets().get("secret_key")
minio_client = storage_client("minio",
hostname=storage_url,
access_key=access_key,
secret_key=secret_key)
with tempfile.TemporaryDirectory() as temp_directory:
for art_name, art_data in artifacts.items():
minio_client.download_file(art_data["artifact_uri"],
os.path.join(temp_directory, art_name))
unzip_files(f"{temp_directory}/{art_name}", temp_directory)
os.chdir(temp_directory)
log_file = f"{temp_directory}/noop.log"
with open(log_file, "w") as f:
f.write(
f"Noop performed for stage {stage} with the following configs {configs}"
)
minio_client.upload_file(run_id, f"artifacts/logs/{stage}/noop.log",
log_file)
project_artifact_hash = generate_sri(log_file)
artifact_uri = f"minio://{storage_url}/{run_id}/logs/{stage}/noop.log"
artman_client.post_artifact(
task_id=task["task"]["task_id"],
artman_url=gman_url,
uri=artifact_uri,
caller=function_executor,
sri=str(project_artifact_hash),
)
gman_client.update_task_id(
gman_url=gman_url,
task_id=task["task"]["task_id"],
status="info",
message="Uploaded log artifact",
)
def wrapper_func(*args, **kwargs):
gman_url = Config["gman"]["url"]
function_name = f"{Config['name']}"
run_id = request.get_json().get("run_id")
project = request.get_json().get("project")
task = gman_client.request_new_task_id(
run_id=run_id,
gman_url=gman_url,
status=status,
project=project,
caller=function_name,
)
g.task = task
try:
func(*args, **kwargs)
gman_client.update_task_id(
gman_url=gman_url,
task_id=task["task"]["task_id"],
status="completed",
message=f"{function_name} completed successfully.",
)
return task
except Exception:
message = traceback.format_exc()
gman_client.update_task_id(
gman_url=gman_url,
status="failed",
task_id=task["task"]["task_id"],
message=
f"Failed to execute {function_name}. Exception: {message}",
)
return task
def gman_delegate(r, *args, **kwargs):
gman_url = Config["gman"]["url"]
if r.status_code == 202:
gman_client.update_task_id(
gman_url=gman_url,
task_id=g.task["task"]["task_id"],
status="delegated",
message=f"Delegated execution to {r.url}",
)
else:
gman_client.update_task_id(
gman_url=gman_url,
task_id=g.task["task"]["task_id"],
status="failed",
message=f"Failed to delegate execution to {r.url}",
)
return r