def upload(self, path, local_path) -> str: """ Parameters ---------- path local_path Returns ------- dataset_id:int """ response = send_requests(self.db_params, "signature/upload", params={"object": path}, method="get") data = response.json()["data"] url = data.get("url") if url is None: # path already exists return str(response.json()["message"][0]) s3_header = {'Content-Type': 'multipart/form-data'} with open(local_path, 'rb') as f: requests.put(url, data=f, headers=s3_header) response = send_requests(self.db_params, "dataset", params={ "file_name": path, "file_size": str(os.path.getsize(local_path)) }, method="put") data = response.json()["data"] dataset_id = data["id"] return str(dataset_id)
def _get_trial_records_by_ids(self, trial_ids, task_id, user_id): result = [] for sub_trial_ids in list_.chunk(trial_ids, 10): target = f"trial?" + "&".join( [f"trial_id={trial_id}" for trial_id in sub_trial_ids]) response = send_requests(self.db_params, target, method="get") result.extend(extend_to_list(response.json()["data"])) return result
def _insert_hdl_record(self, task_id: str, hdl_id: str, user_id: int, hdl: dict, hdl_metadata: Dict[str, Any]): local = get_valid_params_in_kwargs(self._insert_hdl_record, locals()) local.pop("user_id") local.pop("task_id") # fixme: 没加 task_id target = "hdl" response = send_requests(self.db_params, target, local) return hdl_id
def _finish_experiment_update_info(self, experiment_id: int, final_model_path: str, log_path: str, end_time: datetime.datetime): local = get_valid_params_in_kwargs(self._finish_experiment_update_info, locals()) local.pop("experiment_id") target = f"experiment/{experiment_id}" response = send_requests(self.db_params, target, local, method="patch")
def get_url_of_dataset_id(self, dataset_id): response = send_requests(self.db_params, "signature/download", params={"object": str(dataset_id)}, method="get") data = response.json()["data"] if len(data) > 0 and "url" in data[0]: return data[0]["url"] return None
def _insert_experiment_record(self, user_id: int, hdl_id: str, task_id: str, experiment_type: str, experiment_config: Dict[str, Any], additional_info: Dict[str, Any]): local = get_valid_params_in_kwargs(self._insert_experiment_record, locals()) local.pop("user_id") target = "experiment" response = send_requests(self.db_params, target, local) data = response.json()["data"] if "experiment_id" not in data: raise ValueError("insert experiment failed.") return data["experiment_id"]
def _insert_task_record(self, task_id: str, user_id: int, metric: str, splitter: Dict[str, str], ml_task: Dict[str, str], train_set_id: str, test_set_id: str, train_label_id: str, test_label_id: str, specific_task_token: str, task_metadata: Dict[str, Any], sub_sample_indexes: List[int], sub_feature_indexes: List[str]): local = get_valid_params_in_kwargs(self._insert_task_record, locals()) local["sub_sample_indexes"] = json.dumps(local["sub_sample_indexes"]) local["sub_feature_indexes"] = json.dumps(local["sub_feature_indexes"]) local.pop("user_id") target = "task" data = send_requests(self.db_params, target, local).json()["data"] return task_id
def login(self) -> Tuple[int, str]: user = self.db_params["user"] password = self.db_params["password"] response = send_requests(self.db_params, "login", { "user": user, "password": password }) response_json = response.json() data = response_json["data"] self.user_token = data["user_token"] self.user_id = data["user_id"] self.db_params["headers"].update({ "user_id": str(self.user_id), "user_token": self.user_token, }) return self.user_id, self.user_token
def _insert_dataset_record(self, user_id: int, dataset_id: str, dataset_metadata: Dict[str, Any], dataset_type: str, dataset_path: str, upload_type: str, dataset_source: str, column_descriptions: Dict[str, Any], columns_mapper: Dict[str, str], columns: List[str]): local = get_valid_params_in_kwargs(self._insert_dataset_record, locals()) local.pop("user_id") local["columns"] = json.dumps(local["columns"]) target = "dataset" response = send_requests(self.db_params, target, local) data = response.json()["data"] if "dataset_id" not in data: data["dataset_id"] = dataset_id data["dataset_path"] = dataset_path data["length"] = 0 else: data["length"] = 1 return data
def _insert_trial_record(self, user_id: int, task_id: str, hdl_id: str, experiment_id: int, info: Dict[str, Any]): local = get_valid_params_in_kwargs(self._insert_trial_record, locals()) local.pop("user_id") target = "trial" info = local.pop("info") local.update(**info) local.pop("hdl_id") local.pop("task_id") local.pop("finally_fit_model", None) local["final_model_path"] = local.pop("finally_fit_model_path", None) # local["intermediate_results"]=json.dumps(local["intermediate_results"]) local["losses"] = json.dumps(local["losses"]) local["all_scores"] = json.dumps(local["all_scores"]) # fixme # local["test_loss"]={} # local["test_all_score"]={} response = send_requests(self.db_params, target, local) data = response.json()["data"] if "trial_id" not in data: self.logger.warning("insert trial failed.") return data.get("trial_id", -1)
def _get_trial_records_by_id(self, trial_id, task_id, user_id): target = f"trial/{trial_id}" response = send_requests(self.db_params, target, method="get") # 返回值是一个字典 return response.json()["data"]
def _get_sorted_trial_records(self, task_id, user_id, limit): target = f"task/{task_id}/trial?page_num={1}&page_size={limit}" response = send_requests(self.db_params, target, method="get") return extend_to_list(response.json()["data"])
def _get_task_records(self, task_id: str, user_id: int): target = f"task/{task_id}" response = send_requests(self.db_params, target, method="get") return extend_to_list(response.json()["data"])
def _get_experiment_record(self, experiment_id): target = f"experiment/{experiment_id}" response = send_requests(self.db_params, target, method="get") data = response.json()["data"] return extend_to_list(data)
def _get_dataset_records(self, dataset_id, user_id) -> List[Dict[str, Any]]: target = f"dataset/{dataset_id}" response = send_requests(self.db_params, target, method="get") data = response.json()["data"] return extend_to_list(data)