def create_projection(self, pca, input_data=None, args=None, wait_time=3, retries=10): """Creates a new projection. The pca parameter can be a pca resource or ID """ pca_id = None resource_type = get_resource_type(pca) if resource_type != PCA_PATH: raise Exception("A PCA resource id is needed" " to create a projection. %s found." % resource_type) pca_id = get_resource_id(pca) if pca_id is not None: check_resource(pca_id, query_string=TINY_RESOURCE, wait_time=wait_time, retries=retries, raise_on_error=True, api=self) if input_data is None: input_data = {} create_args = {} if args is not None: create_args.update(args) create_args.update({ "input_data": input_data}) if pca_id is not None: create_args.update({ "pca": pca_id}) body = json.dumps(create_args) return self._create(self.projection_url, body, verify=self.verify)
def status(self, resource): """Maps status code to string. """ resource_id = get_resource_id(resource) if resource_id: resource = self._get("%s%s" % (self.url, resource_id)) status = get_status(resource) code = status['code'] return STATUSES.get(code, "UNKNOWN") else: status = get_status(resource) if status['code'] != UPLOADING: LOGGER.error("Wrong resource id") return return STATUSES[UPLOADING]
def reify_resource(self, resource_id): """Redirects to the reify method according to the resource type """ # first check if this is a valid id resource_id = get_resource_id(resource_id) if resource_id is not None: resource_type = get_resource_type(resource_id) reify_handler = getattr(self, 'reify_%s' % resource_type) message = "Analyzing %s.\n" % resource_id self.logger(message) reify_handler(resource_id) if self.delete: self.delete_stored_resource(resource_id)
def create_prediction(self, model, input_data=None, args=None, wait_time=3, retries=10): """Creates a new prediction. The model parameter can be: - a simple tree model - a simple logistic regression model - an ensemble - a deepnet . a linear regression - a fusion Note that the old `by_name` argument has been deprecated. """ model_id = None resource_type = get_resource_type(model) if resource_type not in SUPERVISED_PATHS: raise Exception("A supervised model resource id is needed" " to create a prediction. %s found." % resource_type) model_id = get_resource_id(model) if model_id is not None: check_resource(model_id, query_string=TINY_RESOURCE, wait_time=wait_time, retries=retries, raise_on_error=True, api=self) if input_data is None: input_data = {} create_args = {} if args is not None: create_args.update(args) create_args.update({"input_data": input_data}) if model_id is not None: create_args.update({"model": model_id}) body = json.dumps(create_args) return self._create(self.prediction_url, body, verify=self.verify_prediction)
def reify_resources(args, api, logger): """ Extracts the properties of the created resources and generates code to rebuild them """ resource_id = get_resource_id(args.resource_id) if resource_id is None: sys.exit("Failed to match a valid resource ID. Please, check: %s" % args.resource_id) # check whether the resource exists try: check_resource(resource_id, raise_on_error=True, api=api) except Exception, exc: sys.exit("Failed to find the resource %s. Please, check its ID and" " the connection info (domain and credentials)." % resource_id)
def create_prediction(self, model, input_data=None, args=None, wait_time=3, retries=10): """Creates a new prediction. The model parameter can be: - a simple tree model - a simple logistic regression model - an ensemble - a deepnet . a linear regression - a fusion Note that the old `by_name` argument has been deprecated. """ model_id = None resource_type = get_resource_type(model) if resource_type not in SUPERVISED_PATHS: raise Exception("A supervised model resource id is needed" " to create a prediction. %s found." % resource_type) model_id = get_resource_id(model) if model_id is not None: check_resource(model_id, query_string=TINY_RESOURCE, wait_time=wait_time, retries=retries, raise_on_error=True, api=self) if input_data is None: input_data = {} create_args = {} if args is not None: create_args.update(args) create_args.update({ "input_data": input_data}) if model_id is not None: create_args.update({ "model": model_id}) body = json.dumps(create_args) return self._create(self.prediction_url, body, verify=self.verify_prediction)
def reify_resources(args, api, logger): """ Extracts the properties of the created resources and generates code to rebuild them """ resource_id = get_resource_id(args.resource_id) if resource_id is None: sys.exit("Failed to match a valid resource ID. Please, check: %s" % args.resource_id) api_calls = RESTChain(api, resource_id, args.add_fields, logger, args.output_dir) output = api_calls.reify("python") if PYTHON3: with open(args.output, "w", encoding="utf-8") as reify_file: reify_file.write(output) else: with open(args.output, "w") as reify_file: reify_file.write(output.encode("utf-8"))
def create_projection(self, pca, input_data=None, args=None, wait_time=3, retries=10): """Creates a new projection. The pca parameter can be a pca resource or ID """ pca_id = None resource_type = get_resource_type(pca) if resource_type != PCA_PATH: raise Exception("A PCA resource id is needed" " to create a projection. %s found." % resource_type) pca_id = get_resource_id(pca) if pca_id is not None: check_resource(pca_id, query_string=TINY_RESOURCE, wait_time=wait_time, retries=retries, raise_on_error=True, api=self) if input_data is None: input_data = {} create_args = {} if args is not None: create_args.update(args) create_args.update({"input_data": input_data}) if pca_id is not None: create_args.update({"pca": pca_id}) body = json.dumps(create_args) return self._create(self.projection_url, body, verify=self.verify)
def reify_resources(args, api): """ Extracts the properties of the created resources and generates code to rebuild them """ resource_id = get_resource_id(args.resource_id) if resource_id is None: sys.exit("Failed to match a valid resource ID. Please, check: %s" % args.resource_id) # check whether the resource exists try: check_resource(resource_id, raise_on_error=True, api=api) except Exception: sys.exit("Failed to find the resource %s. Please, check its ID and" " the connection info (domain and credentials)." % resource_id) reify_script = whizzml_script(args, api) # apply the reify script to the resource execute_command = [ 'execute', '--script', reify_script, '--output-dir', args.output_dir ] command_args, _, _, _, _ = get_context( \ execute_command, EXE_SETTINGS) command_args.arguments_ = [["res-id", resource_id]] command_args.inputs = json.dumps(command_args.arguments_) # process the command session_file = None execute_whizzml(command_args, api, session_file) with open("%s.json" % command_args.output) as file_handler: exe_output = json.load(file_handler)['result'] if args.language == "nb": write_nb_output(resource_id, \ exe_output, args.output.replace(".py", ".ipynb"), api) return elif args.language == "whizzml": output = exe_output["source_code"] args.output = args.output.replace(".py", ".whizzml") exe_output["source_code"] = args.output exe_output["kind"] = "script" with open(os.path.join(os.path.dirname(args.output), "metadata.json"), "w") as meta_handler: meta_handler.write(json.dumps(exe_output)) else: output = python_output(exe_output, api) prefix = u"""\ #!/usr/bin/env python # -*- coding: utf-8 -*- \"\"\"Python code to reify %s Generated by BigMLer \"\"\" def main(): """ % resource_id suffix = u"""\ if __name__ == "__main__": main() """ output = "%s%s\n%s" % (prefix, output, suffix) write_to_utf8(args.output, output) sts = os.stat(args.output) os.chmod(args.output, sts.st_mode | stat.S_IEXEC)