def test_cs_storage(exp_loc_res): dummy_uuid = "c7a65ad2-0c2c-45d7-b0f7-d9fd524c49b3" task_id = "1868c4a7-b03c-4fe4-ab45-0aa95c0bfa53" rem_res = cs_storage.write(task_id, exp_loc_res) loc_res = cs_storage.read(rem_res, json_serializable=False) for output_type in ["renderable", "downloadable"]: loc_res_without_id = [{k: v for k, v in output.items() if k != "id"} for output in loc_res[output_type]] exp_res_without_id = [{k: v for k, v in output.items() if k != "id"} for output in exp_loc_res[output_type]] assert exp_res_without_id == loc_res_without_id assert json.dumps(cs_storage.read(rem_res, json_serializable=True)) loc_res1 = cs_storage.read({"renderable": rem_res["renderable"]}, json_serializable=False) loc_res_without_id = [{k: v for k, v in output.items() if k != "id"} for output in loc_res1["renderable"]] exp_res_without_id = [{k: v for k, v in output.items() if k != "id"} for output in exp_loc_res["renderable"]] assert exp_res_without_id == loc_res_without_id assert json.dumps( cs_storage.read({"renderable": rem_res["renderable"]}, json_serializable=True))
def get(self, request, *args, **kwargs): self.object = self.get_object(kwargs["model_pk"], kwargs["username"], kwargs["title"]) sim = SimulationSerializer(self.object) if self.object.outputs: data = sim.data outputs = data["outputs"]["outputs"] data["outputs"] = cs_storage.read(outputs) return Response(data, status=status.HTTP_200_OK) elif self.object.traceback is not None: return Response(sim.data, status=status.HTTP_200_OK) job_id = str(self.object.job_id) compute = Compute() try: job_ready = compute.results_ready(job_id) except JobFailError: self.object.traceback = "" self.object.save() return Response({"error": "model error"}, status=status.HTTP_400_BAD_REQUEST) # something happened and the exception was not caught if job_ready == "FAIL": result = compute.get_results(job_id) if result["traceback"]: traceback_ = result["traceback"] else: traceback_ = "Error: The traceback for this error is unavailable." self.object.traceback = traceback_ self.object.status = "WORKER_FAILURE" self.object.save() return Response({"error": "model error"}, status=status.HTTP_400_BAD_REQUEST) return Response(sim.data, status=status.HTTP_202_ACCEPTED)
def get_sim_data(self, user, as_remote, username, title, model_pk): self.object = self.get_object(model_pk, username, title) sim = SimulationSerializer(self.object, context={"request": self.request}) data = sim.data if self.object.outputs_version() == "v0": return Response(data, status=status.HTTP_200_OK) elif self.object.outputs: outputs = data["outputs"]["outputs"] if not as_remote: data["outputs"] = cs_storage.read(outputs) else: if self.request.is_secure(): protocol = "https" else: protocol = "http" # local dev. base_url = f"{protocol}://{self.request.get_host()}" for rem_output in data["outputs"]["outputs"]["renderable"][ "outputs"]: rem_output[ "screenshot"] = f"{base_url}/storage/screenshots/{rem_output['id']}.png" return Response(data, status=status.HTTP_200_OK) elif self.object.traceback is not None: return Response(data, status=status.HTTP_200_OK) elif self.object.status == "STARTED": return Response(data, status=status.HTTP_200_OK) else: return Response(data, status=status.HTTP_202_ACCEPTED)
def test_taxcruncher_outputs(): with open(f"{CURRENT_DIR}/test-tc-remote.json") as f: remote_outputs = json.loads(f.read()) outputs = cs_storage.read(remote_outputs["outputs"]) for output in outputs["renderable"]: basename = f"{output['title'] or 'template'}.html" print(f"screenshotting: {basename}") cs_storage.screenshot(output)
def test_use_with_dask(): try: import dask import dask.distributed from distributed import Client except ImportError: import warnings warnings.warn("Dask and/or Distributed are not installed") return with open(f"{CURRENT_DIR}/test-ogusa-remote.json") as f: remote_outputs = json.loads(f.read()) outputs = cs_storage.read(remote_outputs["outputs"]) c = Client() futures = c.map(cs_storage.screenshot, outputs["renderable"]) results = c.gather(futures) for result in results: assert isinstance(result, bytes)
def get_sim_data(self, user, as_remote, username, title, model_pk): self.object = self.get_object(model_pk, username, title) sim = SimulationSerializer(self.object, context={"request": self.request}) data = sim.data if self.object.outputs_version() == "v0": return Response(data, status=status.HTTP_200_OK) elif self.object.outputs: outputs = data["outputs"]["outputs"] if not as_remote: data["outputs"] = cs_storage.read(outputs) else: data["outputs"]["outputs"] = cs_storage.add_screenshot_links( data["outputs"]["outputs"]) return Response(data, status=status.HTTP_200_OK) elif self.object.traceback is not None: return Response(data, status=status.HTTP_200_OK) elif self.object.status == "STARTED": return Response(data, status=status.HTTP_200_OK) compute = Compute() try: job_ready = compute.results_ready(self.object) except JobFailError: self.object.traceback = "" self.object.save() return Response({"error": "model error"}, status=status.HTTP_400_BAD_REQUEST) # something happened and the exception was not caught if job_ready == "FAIL": result = compute.get_results(self.object) if result["traceback"]: traceback_ = result["traceback"] else: traceback_ = "Error: The traceback for this error is unavailable." self.object.traceback = traceback_ self.object.status = "WORKER_FAILURE" self.object.save() return Response({"error": "model error"}, status=status.HTTP_400_BAD_REQUEST) data.update(sim.data) return Response(data, status=status.HTTP_202_ACCEPTED)
def render_v1(self, request): renderable = { "renderable": self.object.outputs["outputs"]["renderable"] } outputs = cs_storage.read(renderable) return render( request, "comp/outputs/v1/sim_detail.html", { "outputs": outputs, "object": self.object, "result_header": "Results", "bokeh_scripts": { "cdn_js": CDN.js_files[0], "cdn_css": CDN.css_files[0], "widget_js": CDN.js_files[1], "widget_css": CDN.css_files[1], }, }, )
def test_errors(): with pytest.raises(exceptions.ValidationError): cs_storage.write("123", {"bad": "data"}) with pytest.raises(exceptions.ValidationError): cs_storage.read({"bad": "data"})
def test_cs_storage(png, jpg, bokeh_plot): dummy_uuid = "c7a65ad2-0c2c-45d7-b0f7-d9fd524c49b3" exp_loc_res = { "renderable": [ { "media_type": "bokeh", "title": "bokeh plot", "data": bokeh_plot, }, { "media_type": "table", "title": "table stuff", "data": "<table/>" }, { "media_type": "PNG", "title": "PNG data", "data": png }, { "media_type": "JPEG", "title": "JPEG data", "data": jpg }, { "media_type": "MP3", "title": "MP3 data", "data": b"MP3 bytes" }, { "media_type": "MP4", "title": "MP4 data", "data": b"MP4 bytes" }, ], "downloadable": [ { "media_type": "CSV", "title": "CSV file", "data": "comma,sep,values\n" }, { "media_type": "HDF5", "title": "HDF5 file", "data": b"serialized numpy arrays and such\n", }, { "media_type": "PDF", "title": "PDF file", "data": b"some pdf like data." }, { "media_type": "Markdown", "title": "Markdown file", "data": "**hello world**", }, { "media_type": "Text", "title": "Text file", "data": "text data" }, ], } task_id = "1868c4a7-b03c-4fe4-ab45-0aa95c0bfa53" rem_res = cs_storage.write(task_id, exp_loc_res) loc_res = cs_storage.read(rem_res, json_serializable=False) for output_type in ["renderable", "downloadable"]: loc_res_without_id = [{k: v for k, v in output.items() if k != "id"} for output in loc_res[output_type]] exp_res_without_id = [{k: v for k, v in output.items() if k != "id"} for output in exp_loc_res[output_type]] assert exp_res_without_id == loc_res_without_id assert json.dumps(cs_storage.read(rem_res, json_serializable=True)) loc_res1 = cs_storage.read({"renderable": rem_res["renderable"]}, json_serializable=False) loc_res_without_id = [{k: v for k, v in output.items() if k != "id"} for output in loc_res1["renderable"]] exp_res_without_id = [{k: v for k, v in output.items() if k != "id"} for output in exp_loc_res["renderable"]] assert exp_res_without_id == loc_res_without_id assert json.dumps( cs_storage.read({"renderable": rem_res["renderable"]}, json_serializable=True))