def get(self, request, *args, **kwargs): self.object = self.get_object(kwargs["model_pk"], kwargs["username"], kwargs["title"]) sim = SimulationSerializer(self.object) if self.object.outputs: data = sim.data outputs = data["outputs"]["outputs"] data["outputs"] = cs_storage.read(outputs) return Response(data, status=status.HTTP_200_OK) elif self.object.traceback is not None: return Response(sim.data, status=status.HTTP_200_OK) job_id = str(self.object.job_id) compute = Compute() try: job_ready = compute.results_ready(job_id) except JobFailError: self.object.traceback = "" self.object.save() return Response({"error": "model error"}, status=status.HTTP_400_BAD_REQUEST) # something happened and the exception was not caught if job_ready == "FAIL": result = compute.get_results(job_id) if result["traceback"]: traceback_ = result["traceback"] else: traceback_ = "Error: The traceback for this error is unavailable." self.object.traceback = traceback_ self.object.status = "WORKER_FAILURE" self.object.save() return Response({"error": "model error"}, status=status.HTTP_400_BAD_REQUEST) return Response(sim.data, status=status.HTTP_202_ACCEPTED)
def remote_submit_job(self, url, data, timeout, headers=None): with requests_mock.Mocker() as mock: resp = {"job_id": dummy_uuid, "qlength": 2} resp = json.dumps(resp) mock.register_uri("POST", url, text=resp) self.last_posted = data return Compute.remote_submit_job(self, url, data, timeout)
def post(self, request, *args, **kwargs): compute = Compute() project = get_object_or_404( self.queryset, owner__user__username=kwargs["username"], title=kwargs["title"], ) ioutils = get_ioutils(project, Parser=APIParser) try: submit_inputs = SubmitInputs(request, project, ioutils, compute) result = submit_inputs.submit() except BadPostException as bpe: return Response(bpe.errors, status=status.HTTP_400_BAD_REQUEST) except AppError as ae: try: send_mail( f"Compute Studio AppError", (f"An error has occurred:\n {ae.parameters}\n causing: " f"{ae.traceback}\n user:{request.user.username}\n " f"project: {project.app_url}."), "*****@*****.**", ["*****@*****.**"], fail_silently=True, ) # Http 401 exception if mail credentials are not set up. except Exception: pass return Response(ae.traceback, status=status.HTTP_500_INTERNAL_SERVER_ERROR) inputs = InputsSerializer(result) return Response(inputs.data, status=status.HTTP_201_CREATED)
def submit(request, success_status, project, sim): compute = Compute() ioutils = get_ioutils(project, Parser=APIParser) try: submit_inputs = SubmitInputs(request, project, ioutils, compute, sim) result = submit_inputs.submit() except BadPostException as bpe: return Response(bpe.errors, status=status.HTTP_400_BAD_REQUEST) except AppError as ae: try: send_mail( f"Compute Studio AppError", (f"An error has occurred:\n {ae.parameters}\n causing: " f"{ae.traceback}\n user:{request.user.username}\n " f"project: {project.app_url}."), "*****@*****.**", ["*****@*****.**"], fail_silently=True, ) # Http 401 exception if mail credentials are not set up. except Exception: pass return Response(ae.traceback, status=success_status) inputs = InputsSerializer(result) return Response(inputs.data, status=status.HTTP_201_CREATED)
def put(self, request, *args, **kwargs): print("myinputs api method=PUT", kwargs) if request.user.username == "comp-api-user": data = request.data ser = InputsSerializer(data=request.data) if ser.is_valid(): data = ser.validated_data inputs = get_object_or_404(Inputs, job_id=data["job_id"]) print("data") print(data) if inputs.status == "PENDING": # successful run if data["status"] == "SUCCESS": inputs.errors_warnings = data["errors_warnings"] inputs.custom_adjustment = data.get( "custom_adjustment", None) inputs.status = "SUCCESS" if is_valid( inputs) else "INVALID" inputs.save() if inputs.status == "SUCCESS": submit_sim = SubmitSim(inputs, compute=Compute()) submit_sim.submit() # failed run, exception was caught else: inputs.status = "FAIL" inputs.traceback = data["traceback"] inputs.save() return Response(status=status.HTTP_200_OK) else: print("inputs put error", ser.errors) return Response(ser.errors, status=status.HTTP_400_BAD_REQUEST) else: return Response(status=status.HTTP_401_UNAUTHORIZED)
def dispatch(self, request, *args, **kwargs): compute = Compute() model_pk, username, title = ( kwargs["model_pk"], kwargs["username"], kwargs["title"], ) self.object = self.get_object(model_pk, username, title) if self.object.outputs or self.object.aggr_outputs: return self.render_outputs(request) elif self.object.traceback is not None: return self.fail(model_pk, username, title) else: job_id = str(self.object.job_id) try: job_ready = compute.results_ready(job_id) except JobFailError as jfe: self.object.traceback = "" self.object.save() return self.fail(model_pk, username, title) # something happened and the exception was not caught if job_ready == "FAIL": result = compute.get_results(job_id) if result["traceback"]: traceback_ = result["traceback"] else: traceback_ = "Error: The traceback for this error is unavailable." self.object.traceback = traceback_ self.object.status = "WORKER_FAILURE" self.object.save() return self.fail(model_pk, username, title) else: if request.method == "POST": # if not ready yet, insert number of minutes remaining exp_num_minutes = self.object.compute_eta() orig_eta = self.object.compute_eta( self.object.creation_date) return JsonResponse( { "eta": exp_num_minutes, "origEta": orig_eta }, status=202) else: context = {"eta": "100", "origEta": "0"} return render(request, "comp/not_ready.html", context)
def remote_query_job(self, url, params): self.client = None self.sim = None with requests_mock.Mocker() as mock: print("mocking: ", url) text = "FAIL" mock.register_uri("GET", url, text=text) return Compute.remote_query_job(self, url, params)
def remote_submit_job(self, url, data, timeout, headers=None): print("mocking:", url) with requests_mock.Mocker() as mock: resp = {"task_id": str(uuid.uuid4())} resp = json.dumps(resp) print("mocking", url) mock.register_uri("POST", url, text=resp) self.last_posted = data return Compute.remote_submit_job(self, url, data, timeout)
def remote_query_job(self, url, params): with requests_mock.Mocker() as mock: if self.num_times_to_wait > 0: text = "NO" else: text = "YES" mock.register_uri("GET", url, text=text) self.num_times_to_wait -= 1 return Compute.remote_query_job(self, url, params)
def get_sim_data(self, user, as_remote, username, title, model_pk): self.object = self.get_object(model_pk, username, title) sim = SimulationSerializer(self.object, context={"request": self.request}) data = sim.data if self.object.outputs_version() == "v0": return Response(data, status=status.HTTP_200_OK) elif self.object.outputs: outputs = data["outputs"]["outputs"] if not as_remote: data["outputs"] = cs_storage.read(outputs) else: data["outputs"]["outputs"] = cs_storage.add_screenshot_links( data["outputs"]["outputs"]) return Response(data, status=status.HTTP_200_OK) elif self.object.traceback is not None: return Response(data, status=status.HTTP_200_OK) elif self.object.status == "STARTED": return Response(data, status=status.HTTP_200_OK) compute = Compute() try: job_ready = compute.results_ready(self.object) except JobFailError: self.object.traceback = "" self.object.save() return Response({"error": "model error"}, status=status.HTTP_400_BAD_REQUEST) # something happened and the exception was not caught if job_ready == "FAIL": result = compute.get_results(self.object) if result["traceback"]: traceback_ = result["traceback"] else: traceback_ = "Error: The traceback for this error is unavailable." self.object.traceback = traceback_ self.object.status = "WORKER_FAILURE" self.object.save() return Response({"error": "model error"}, status=status.HTTP_400_BAD_REQUEST) data.update(sim.data) return Response(data, status=status.HTTP_202_ACCEPTED)
def __init__(self, project: "Project", compute: Union[SyncCompute, Compute] = None): self.project = project print(self.project) if compute is not None: self.comptue = compute elif self.project.cluster.version == "v0": self.compute = compute or SyncCompute() else: self.compute = compute or Compute() self.config = None
def put(self, request, *args, **kwargs): print("myinputs api method=PUT", kwargs) print("authenticator", request.user, request.successful_authenticator) ser = InputsSerializer(data=request.data) if ser.is_valid(): data = ser.validated_data inputs = get_object_or_404( Inputs.objects.prefetch_related("project"), job_id=data["job_id"]) if not inputs.project.has_write_access(request.user): return Response(status=status.HTTP_401_UNAUTHORIZED) if inputs.status in ("PENDING", "INVALID", "FAIL"): # successful run if data["status"] == "SUCCESS": inputs.errors_warnings = data["errors_warnings"] inputs.custom_adjustment = data.get( "custom_adjustment", None) inputs.status = "SUCCESS" if is_valid( inputs) else "INVALID" inputs.save() if inputs.status == "SUCCESS": submit_sim = SubmitSim(inputs.sim, compute=Compute()) submit_sim.submit() # failed run, exception was caught else: inputs.status = "FAIL" inputs.traceback = data["traceback"] inputs.save() if self.request.is_secure(): protocol = "https" else: protocol = "http" # local dev. base_url = f"{protocol}://{self.request.get_host()}" url = base_url + inputs.get_absolute_url() fail( project=inputs.project, model_pk=inputs.sim.model_pk, traceback=inputs.traceback, url=url, ) return Response(status=status.HTTP_200_OK) else: print("inputs put error", ser.errors) return Response(ser.errors, status=status.HTTP_400_BAD_REQUEST)
def remote_query_job(self, url, params): # Need to login as the comp-api-user self.client.login(username=self.user, password=self.password) if isinstance(self.client, APIClient): format_kwarg = {"format": "json"} else: format_kwarg = {"content_type": "application/json"} resp = self.client.put("/outputs/api/", data=dict(json.loads(self.outputs), **{"job_id": self.sim.job_id}), **format_kwarg) assert resp.status_code == 200 self.client = None self.sim = None with requests_mock.Mocker() as mock: text = "NO" mock.register_uri("GET", url, text=text) return Compute.remote_query_job(self, url, params)
def __init__(self, project, displayer, clean_inputs, compute=None, **valid_meta_params): self.project = project self.clean_inputs = clean_inputs self.compute = compute or Compute() self.valid_meta_params = valid_meta_params for param, value in valid_meta_params.items(): setattr(self, param, value) defaults = displayer.package_defaults() self.grouped_defaults = defaults["model_parameters"] self.flat_defaults = { k: v for _, sect in self.grouped_defaults.items() for k, v in sect.items() }
def remote_query_job(self, url): # Need to login as the comp-api-user self.client.login(username=self.user, password=self.password) if isinstance(self.client, APIClient): format_kwarg = {"format": "json"} else: format_kwarg = {"content_type": "application/json"} resp = self.client.put( "/outputs/api/", data=dict(json.loads(self.outputs), **{"job_id": self.sim.job_id}), **format_kwarg, ) if resp.status_code != 200: raise CallbackException( f"Status code: {resp.status_code}\n {json.dumps(resp.data, indent=4)}" ) self.client = None self.sim = None with requests_mock.Mocker() as mock: text = "NO" mock.register_uri("GET", url, text=text) return Compute.remote_query_job(self, url)
def __init__(self, project, model_parameters, clean_inputs, compute=None, **valid_meta_params): self.project = project self.clean_inputs = clean_inputs self.compute = compute or Compute() self.valid_meta_params = valid_meta_params for param, value in valid_meta_params.items(): setattr(self, param, value) try: defaults = model_parameters.defaults(self.valid_meta_params) except NotReady: self.grouped_defaults = {} self.flat_defaults = {} else: self.grouped_defaults = defaults["model_parameters"] self.flat_defaults = { k: v for _, sect in self.grouped_defaults.items() for k, v in sect.items() }
def remote_get_job(self, url, params): self.count += 1 with requests_mock.Mocker() as mock: mock.register_uri("GET", url, text=self.outputs) return Compute.remote_get_job(self, url, params)