def f(*args, **kwargs):
task = args[0]
task_id = task.request.id
start = time.time()
traceback_str = None
res = {}
try:
outputs = func(*args, **kwargs)
if task.name.endswith("sim"):
version = outputs.pop("version", OUTPUTS_VERSION)
if version == "v0":
res["model_version"] = "NA"
res.update(dict(outputs, **{"version": version}))
else:
outputs = cs_storage.write(task_id, outputs)
res.update({
"model_version": functions.get_version(),
"outputs": outputs,
"version": version,
})
else:
res.update(outputs)
except Exception:
traceback_str = traceback.format_exc()
finish = time.time()
if "meta" not in res:
res["meta"] = {}
res["meta"]["task_times"] = [finish - start]
if traceback_str is None:
res["status"] = "SUCCESS"
else:
res["status"] = "FAIL"
res["traceback"] = traceback_str
return res
def test_run_model(self):
self.test_all_data_specified()
inputs = self.get_inputs({})
check_get_inputs(inputs)
class MetaParams(Parameters):
defaults = inputs["meta_parameters"]
mp_spec = MetaParams().specification(serializable=True)
result = self.run_model(mp_spec, self.ok_adjustment)
assert cs_storage.LocalResult().load(result)
assert cs_storage.write(uuid.uuid4(), result, do_upload=False)
required_bokeh_keys = ("target_id", "root_id", "doc")
for output_type, outputs in result.items():
for output in outputs:
if output["media_type"] == "bokeh":
data = output["data"]
if not (isinstance(data, dict) and all(
[key in data.keys() for key in required_bokeh_keys])):
link = "https://docs.compute.studio/publish/outputs/#bokeh"
raise CSKitError(
f"Bokeh outputs must be created with json_item. Reference: {link}"
)
def test_cs_storage(exp_loc_res):
dummy_uuid = "c7a65ad2-0c2c-45d7-b0f7-d9fd524c49b3"
task_id = "1868c4a7-b03c-4fe4-ab45-0aa95c0bfa53"
rem_res = cs_storage.write(task_id, exp_loc_res)
loc_res = cs_storage.read(rem_res, json_serializable=False)
for output_type in ["renderable", "downloadable"]:
loc_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in loc_res[output_type]]
exp_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in exp_loc_res[output_type]]
assert exp_res_without_id == loc_res_without_id
assert json.dumps(cs_storage.read(rem_res, json_serializable=True))
loc_res1 = cs_storage.read({"renderable": rem_res["renderable"]},
json_serializable=False)
loc_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in loc_res1["renderable"]]
exp_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in exp_loc_res["renderable"]]
assert exp_res_without_id == loc_res_without_id
assert json.dumps(
cs_storage.read({"renderable": rem_res["renderable"]},
json_serializable=True))
def done_callback(future, job_id, comp_url, comp_api_token, start_time):
"""
This should be called like:
callback = functools.partial(
done_callback,
job_id=job_id,
comp_url=os.environ.get("COMP_URL"),
comp_api_token=os.environ.get("comp_api_token"),
start_time=time.time()
)
This is because this function needs the job id, comp url,
api token, and start time arguments, but dask only passes the
future object.
"""
finish = time.time()
print(f"job_id: {job_id}")
print(f"from dask")
print(f"state: {future.status}")
res = {}
traceback_str = None
try:
outputs = future.result()
outputs = cs_storage.write(job_id, outputs)
res.update({
"model_version": functions.get_version(),
"outputs": outputs,
"version": "v1",
})
except Exception:
traceback_str = traceback.format_exc()
print(f"exception in callback with job_id: {job_id}")
print(traceback_str)
if "meta" not in res:
res["meta"] = {}
res["meta"]["task_times"] = [finish - start_time]
if traceback_str is None:
res["status"] = "SUCCESS"
else:
res["status"] = "FAIL"
res["traceback"] = traceback_str
res["job_id"] = job_id
print("got result", res)
print(f"posting data to {comp_url}/outputs/api/")
resp = requests.put(
f"{comp_url}/outputs/api/",
json=res,
headers={"Authorization": f"Token {comp_api_token}"},
)
print("resp", resp.status_code)
if resp.status_code == 400:
print("errors", resp.json())
def test_errors():
with pytest.raises(exceptions.ValidationError):
cs_storage.write("123", {"bad": "data"})
with pytest.raises(exceptions.ValidationError):
cs_storage.read({"bad": "data"})
def test_cs_storage(png, jpg, bokeh_plot):
dummy_uuid = "c7a65ad2-0c2c-45d7-b0f7-d9fd524c49b3"
exp_loc_res = {
"renderable": [
{
"media_type": "bokeh",
"title": "bokeh plot",
"data": bokeh_plot,
},
{
"media_type": "table",
"title": "table stuff",
"data": "<table/>"
},
{
"media_type": "PNG",
"title": "PNG data",
"data": png
},
{
"media_type": "JPEG",
"title": "JPEG data",
"data": jpg
},
{
"media_type": "MP3",
"title": "MP3 data",
"data": b"MP3 bytes"
},
{
"media_type": "MP4",
"title": "MP4 data",
"data": b"MP4 bytes"
},
],
"downloadable": [
{
"media_type": "CSV",
"title": "CSV file",
"data": "comma,sep,values\n"
},
{
"media_type": "HDF5",
"title": "HDF5 file",
"data": b"serialized numpy arrays and such\n",
},
{
"media_type": "PDF",
"title": "PDF file",
"data": b"some pdf like data."
},
{
"media_type": "Markdown",
"title": "Markdown file",
"data": "**hello world**",
},
{
"media_type": "Text",
"title": "Text file",
"data": "text data"
},
],
}
task_id = "1868c4a7-b03c-4fe4-ab45-0aa95c0bfa53"
rem_res = cs_storage.write(task_id, exp_loc_res)
loc_res = cs_storage.read(rem_res, json_serializable=False)
for output_type in ["renderable", "downloadable"]:
loc_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in loc_res[output_type]]
exp_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in exp_loc_res[output_type]]
assert exp_res_without_id == loc_res_without_id
assert json.dumps(cs_storage.read(rem_res, json_serializable=True))
loc_res1 = cs_storage.read({"renderable": rem_res["renderable"]},
json_serializable=False)
loc_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in loc_res1["renderable"]]
exp_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in exp_loc_res["renderable"]]
assert exp_res_without_id == loc_res_without_id
assert json.dumps(
cs_storage.read({"renderable": rem_res["renderable"]},
json_serializable=True))
def write(task_id, outputs):
outputs = cs_storage.deserialize_from_json(outputs)
res = cs_storage.write(task_id, outputs)
return res