def wrapper_func(*args, **kwargs):
gman_url = Config["gman"]["url"]
function_name = f"{Config['name']}"
run_id = request.get_json().get("run_id")
project = request.get_json().get("project")
thread_id = request.get_json().get("thread_id")
task = gman_client.request_new_task_id(
run_id=run_id,
gman_url=gman_url,
status=status,
project=project,
caller=function_name,
thread_id=thread_id,
)
g.task = task
try:
func(*args, **kwargs)
gman_client.update_task_id(
gman_url=gman_url,
task_id=task["task"]["task_id"],
status="completed",
message=f"{function_name} completed successfully.",
)
return task
except Exception:
message = traceback.format_exc()
gman_client.update_task_id(
gman_url=gman_url,
status="failed",
task_id=task["task"]["task_id"],
message=
f"Failed to execute {function_name}. Exception: {message}",
)
return task
def handle(request):
"""
Noop executor function definition. This handler function controls
all functionality
:param request: The request object from Flask
This object is required to have the following JSON parameters:
* run_id: The run_id of the task
* thread_id: The thread_id, from gman, that this execution is forked from.
* project: The project name of the run
* configs: A list containing the configuration dictionaries for the run.
* stage: The stage that is being run.
* artifacts: A list of dictionaries containing information on the artifacts
required for this run
:param request:
:return:
"""
run_id = request.get_json().get("run_id")
stage = request.get_json()["stage"]
configs = request.get_json()["configs"]
artifacts = request.get_json()["artifacts"]
task = g.task
access_key = read_secrets().get("access_key")
secret_key = read_secrets().get("secret_key")
minio_client = storage_client("minio",
hostname=storage_url,
access_key=access_key,
secret_key=secret_key)
with tempfile.TemporaryDirectory() as temp_directory:
for art_name, art_data in artifacts.items():
minio_client.download_file(art_data["artifact_uri"],
os.path.join(temp_directory, art_name))
unzip_files(f"{temp_directory}/{art_name}", temp_directory)
os.chdir(temp_directory)
log_file = f"{temp_directory}/noop.log"
with open(log_file, "w") as f:
f.write(
f"Noop performed for stage {stage} with the following configs {configs}"
)
minio_client.upload_file(run_id, f"artifacts/logs/{stage}/noop.log",
log_file)
project_artifact_hash = generate_sri(log_file)
artifact_uri = f"minio://{storage_url}/{run_id}/logs/{stage}/noop.log"
artman_client.post_artifact(
task_id=task["task"]["task_id"],
artman_url=gman_url,
uri=artifact_uri,
caller=function_executor,
sri=str(project_artifact_hash),
)
gman_client.update_task_id(
gman_url=gman_url,
task_id=task["task"]["task_id"],
status="info",
message="Uploaded log artifact",
)
def test_update_task_id_request_exception(mock_put_request_exception):
with pytest.raises(requests.exceptions.RequestException):
client.update_task_id(
task_id="1234",
status="running",
message="blank message",
gman_url="http://gman_url",
)
def test_update_task_id_fails_request(response_code):
responses.add(responses.PUT,
"http://gman_url/task/1234",
status=response_code)
with pytest.raises(requests.exceptions.HTTPError):
client.update_task_id(
task_id="1234",
status="running",
message="blank message",
gman_url="http://gman_url",
)
def gman_delegate(r, *args, **kwargs):
gman_url = Config["gman"]["url"]
if r.status_code == 202:
gman_client.update_task_id(
gman_url=gman_url,
task_id=g.task["task"]["task_id"],
status="delegated",
message=f"Delegated execution to {r.url}",
)
else:
gman_client.update_task_id(
gman_url=gman_url,
task_id=g.task["task"]["task_id"],
status="failed",
message=f"Failed to delegate execution to {r.url}",
)
return r
def test_update_task_id():
task_event = {
"message": "blank message",
"status": "running",
"thread_id": "",
"timestamp": "2019-05-16T19:56:33.231452+00:00",
"task": {
"project": "python_project",
"run_id": "574b1db2-ae55-41bb-8680-43703f3031f2",
"caller": "gateway",
"task_id": "1234",
},
}
responses.add(responses.PUT, "http://gman_url/task/1234", json=task_event)
client.update_task_id(
task_id="1234",
status="running",
message="blank message",
gman_url="http://gman_url",
)
assert (responses.calls[0].request.body ==
'{"message": "blank message", "status": "running"}')
def handle(request):
"""
Executor function definition. This handler function controls
all functionality
:param request: The request object from Flask
This object is required to have the following JSON parameters:
* run_id: The run_id of the task
* thread_id: The thread_id, from gman, that this execution is forked from.
* project: The project name of the run
* configs: A list containing the configuration dictionaries for the run.
* stage: The stage that is being run.
* artifacts: A list of dictionaries containing information on the artifacts
required for this run
fixme
Below is a basic definition of a function that does practically nothing.
Define your own functionality in this function using the following as guidance.
It is generally a good idea to do everything in the context of a TemporaryDirectory
due to the nature of OpenFaaS functions. Not every function is guaranteed to have
a clean execution environment. OpenFaaS function environments can be reused across
requests so any writes to the local filesystem should occur within a context which
will be removed upon completion.
:param request:
:return:
"""
run_id = request.get_json().get("run_id")
stage = request.get_json()["stage"]
configs = request.get_json()["configs"]
artifacts = request.get_json()["artifacts"]
task = g.task
access_key = read_secrets().get("access_key")
secret_key = read_secrets().get("secret_key")
minio_client = storage_client("minio",
hostname=storage_url,
access_key=access_key,
secret_key=secret_key)
with tempfile.TemporaryDirectory() as temp_directory:
for art_name, art_data in artifacts.items():
minio_client.download_file(art_data["artifact_uri"],
os.path.join(temp_directory, art_name))
unzip_files(f"{temp_directory}/{art_name}", temp_directory)
os.chdir(temp_directory)
log_file = f"{temp_directory}/noop.log"
with open(log_file, "w") as f:
f.write(
f"Noop performed for stage {stage} with the following configs {configs}"
)
minio_client.upload_file(run_id, f"artifacts/logs/{stage}/noop.log",
log_file)
project_artifact_hash = generate_sri(log_file)
artifact_uri = f"minio://{storage_url}/{run_id}/logs/{stage}/noop.log"
artman_client.post_artifact(
task_id=task["task"]["task_id"],
artman_url=gman_url,
uri=artifact_uri,
caller=function_executor,
sri=str(project_artifact_hash),
)
gman_client.update_task_id(
gman_url=gman_url,
task_id=task["task"]["task_id"],
status="info",
message="Uploaded log artifact",
)