def test_cs_storage(exp_loc_res):
dummy_uuid = "c7a65ad2-0c2c-45d7-b0f7-d9fd524c49b3"
task_id = "1868c4a7-b03c-4fe4-ab45-0aa95c0bfa53"
rem_res = cs_storage.write(task_id, exp_loc_res)
loc_res = cs_storage.read(rem_res, json_serializable=False)
for output_type in ["renderable", "downloadable"]:
loc_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in loc_res[output_type]]
exp_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in exp_loc_res[output_type]]
assert exp_res_without_id == loc_res_without_id
assert json.dumps(cs_storage.read(rem_res, json_serializable=True))
loc_res1 = cs_storage.read({"renderable": rem_res["renderable"]},
json_serializable=False)
loc_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in loc_res1["renderable"]]
exp_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in exp_loc_res["renderable"]]
assert exp_res_without_id == loc_res_without_id
assert json.dumps(
cs_storage.read({"renderable": rem_res["renderable"]},
json_serializable=True))
def get(self, request, *args, **kwargs):
self.object = self.get_object(kwargs["model_pk"], kwargs["username"],
kwargs["title"])
sim = SimulationSerializer(self.object)
if self.object.outputs:
data = sim.data
outputs = data["outputs"]["outputs"]
data["outputs"] = cs_storage.read(outputs)
return Response(data, status=status.HTTP_200_OK)
elif self.object.traceback is not None:
return Response(sim.data, status=status.HTTP_200_OK)
job_id = str(self.object.job_id)
compute = Compute()
try:
job_ready = compute.results_ready(job_id)
except JobFailError:
self.object.traceback = ""
self.object.save()
return Response({"error": "model error"},
status=status.HTTP_400_BAD_REQUEST)
# something happened and the exception was not caught
if job_ready == "FAIL":
result = compute.get_results(job_id)
if result["traceback"]:
traceback_ = result["traceback"]
else:
traceback_ = "Error: The traceback for this error is unavailable."
self.object.traceback = traceback_
self.object.status = "WORKER_FAILURE"
self.object.save()
return Response({"error": "model error"},
status=status.HTTP_400_BAD_REQUEST)
return Response(sim.data, status=status.HTTP_202_ACCEPTED)
def get_sim_data(self, user, as_remote, username, title, model_pk):
self.object = self.get_object(model_pk, username, title)
sim = SimulationSerializer(self.object,
context={"request": self.request})
data = sim.data
if self.object.outputs_version() == "v0":
return Response(data, status=status.HTTP_200_OK)
elif self.object.outputs:
outputs = data["outputs"]["outputs"]
if not as_remote:
data["outputs"] = cs_storage.read(outputs)
else:
if self.request.is_secure():
protocol = "https"
else:
protocol = "http" # local dev.
base_url = f"{protocol}://{self.request.get_host()}"
for rem_output in data["outputs"]["outputs"]["renderable"][
"outputs"]:
rem_output[
"screenshot"] = f"{base_url}/storage/screenshots/{rem_output['id']}.png"
return Response(data, status=status.HTTP_200_OK)
elif self.object.traceback is not None:
return Response(data, status=status.HTTP_200_OK)
elif self.object.status == "STARTED":
return Response(data, status=status.HTTP_200_OK)
else:
return Response(data, status=status.HTTP_202_ACCEPTED)
def test_taxcruncher_outputs():
with open(f"{CURRENT_DIR}/test-tc-remote.json") as f:
remote_outputs = json.loads(f.read())
outputs = cs_storage.read(remote_outputs["outputs"])
for output in outputs["renderable"]:
basename = f"{output['title'] or 'template'}.html"
print(f"screenshotting: {basename}")
cs_storage.screenshot(output)
def test_use_with_dask():
try:
import dask
import dask.distributed
from distributed import Client
except ImportError:
import warnings
warnings.warn("Dask and/or Distributed are not installed")
return
with open(f"{CURRENT_DIR}/test-ogusa-remote.json") as f:
remote_outputs = json.loads(f.read())
outputs = cs_storage.read(remote_outputs["outputs"])
c = Client()
futures = c.map(cs_storage.screenshot, outputs["renderable"])
results = c.gather(futures)
for result in results:
assert isinstance(result, bytes)
def get_sim_data(self, user, as_remote, username, title, model_pk):
self.object = self.get_object(model_pk, username, title)
sim = SimulationSerializer(self.object,
context={"request": self.request})
data = sim.data
if self.object.outputs_version() == "v0":
return Response(data, status=status.HTTP_200_OK)
elif self.object.outputs:
outputs = data["outputs"]["outputs"]
if not as_remote:
data["outputs"] = cs_storage.read(outputs)
else:
data["outputs"]["outputs"] = cs_storage.add_screenshot_links(
data["outputs"]["outputs"])
return Response(data, status=status.HTTP_200_OK)
elif self.object.traceback is not None:
return Response(data, status=status.HTTP_200_OK)
elif self.object.status == "STARTED":
return Response(data, status=status.HTTP_200_OK)
compute = Compute()
try:
job_ready = compute.results_ready(self.object)
except JobFailError:
self.object.traceback = ""
self.object.save()
return Response({"error": "model error"},
status=status.HTTP_400_BAD_REQUEST)
# something happened and the exception was not caught
if job_ready == "FAIL":
result = compute.get_results(self.object)
if result["traceback"]:
traceback_ = result["traceback"]
else:
traceback_ = "Error: The traceback for this error is unavailable."
self.object.traceback = traceback_
self.object.status = "WORKER_FAILURE"
self.object.save()
return Response({"error": "model error"},
status=status.HTTP_400_BAD_REQUEST)
data.update(sim.data)
return Response(data, status=status.HTTP_202_ACCEPTED)
def render_v1(self, request):
renderable = {
"renderable": self.object.outputs["outputs"]["renderable"]
}
outputs = cs_storage.read(renderable)
return render(
request,
"comp/outputs/v1/sim_detail.html",
{
"outputs": outputs,
"object": self.object,
"result_header": "Results",
"bokeh_scripts": {
"cdn_js": CDN.js_files[0],
"cdn_css": CDN.css_files[0],
"widget_js": CDN.js_files[1],
"widget_css": CDN.css_files[1],
},
},
)
def test_errors():
with pytest.raises(exceptions.ValidationError):
cs_storage.write("123", {"bad": "data"})
with pytest.raises(exceptions.ValidationError):
cs_storage.read({"bad": "data"})
def test_cs_storage(png, jpg, bokeh_plot):
dummy_uuid = "c7a65ad2-0c2c-45d7-b0f7-d9fd524c49b3"
exp_loc_res = {
"renderable": [
{
"media_type": "bokeh",
"title": "bokeh plot",
"data": bokeh_plot,
},
{
"media_type": "table",
"title": "table stuff",
"data": "<table/>"
},
{
"media_type": "PNG",
"title": "PNG data",
"data": png
},
{
"media_type": "JPEG",
"title": "JPEG data",
"data": jpg
},
{
"media_type": "MP3",
"title": "MP3 data",
"data": b"MP3 bytes"
},
{
"media_type": "MP4",
"title": "MP4 data",
"data": b"MP4 bytes"
},
],
"downloadable": [
{
"media_type": "CSV",
"title": "CSV file",
"data": "comma,sep,values\n"
},
{
"media_type": "HDF5",
"title": "HDF5 file",
"data": b"serialized numpy arrays and such\n",
},
{
"media_type": "PDF",
"title": "PDF file",
"data": b"some pdf like data."
},
{
"media_type": "Markdown",
"title": "Markdown file",
"data": "**hello world**",
},
{
"media_type": "Text",
"title": "Text file",
"data": "text data"
},
],
}
task_id = "1868c4a7-b03c-4fe4-ab45-0aa95c0bfa53"
rem_res = cs_storage.write(task_id, exp_loc_res)
loc_res = cs_storage.read(rem_res, json_serializable=False)
for output_type in ["renderable", "downloadable"]:
loc_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in loc_res[output_type]]
exp_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in exp_loc_res[output_type]]
assert exp_res_without_id == loc_res_without_id
assert json.dumps(cs_storage.read(rem_res, json_serializable=True))
loc_res1 = cs_storage.read({"renderable": rem_res["renderable"]},
json_serializable=False)
loc_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in loc_res1["renderable"]]
exp_res_without_id = [{k: v
for k, v in output.items() if k != "id"}
for output in exp_loc_res["renderable"]]
assert exp_res_without_id == loc_res_without_id
assert json.dumps(
cs_storage.read({"renderable": rem_res["renderable"]},
json_serializable=True))