def upload(self, path, local_path) -> str:
"""
Parameters
----------
path
local_path
Returns
-------
dataset_id:int
"""
response = send_requests(self.db_params,
"signature/upload",
params={"object": path},
method="get")
data = response.json()["data"]
url = data.get("url")
if url is None: # path already exists
return str(response.json()["message"][0])
s3_header = {'Content-Type': 'multipart/form-data'}
with open(local_path, 'rb') as f:
requests.put(url, data=f, headers=s3_header)
response = send_requests(self.db_params,
"dataset",
params={
"file_name": path,
"file_size":
str(os.path.getsize(local_path))
},
method="put")
data = response.json()["data"]
dataset_id = data["id"]
return str(dataset_id)
def _get_trial_records_by_ids(self, trial_ids, task_id, user_id):
result = []
for sub_trial_ids in list_.chunk(trial_ids, 10):
target = f"trial?" + "&".join(
[f"trial_id={trial_id}" for trial_id in sub_trial_ids])
response = send_requests(self.db_params, target, method="get")
result.extend(extend_to_list(response.json()["data"]))
return result
def _insert_hdl_record(self, task_id: str, hdl_id: str, user_id: int,
hdl: dict, hdl_metadata: Dict[str, Any]):
local = get_valid_params_in_kwargs(self._insert_hdl_record, locals())
local.pop("user_id")
local.pop("task_id") # fixme: 没加 task_id
target = "hdl"
response = send_requests(self.db_params, target, local)
return hdl_id
def _finish_experiment_update_info(self, experiment_id: int,
final_model_path: str, log_path: str,
end_time: datetime.datetime):
local = get_valid_params_in_kwargs(self._finish_experiment_update_info,
locals())
local.pop("experiment_id")
target = f"experiment/{experiment_id}"
response = send_requests(self.db_params, target, local, method="patch")
def get_url_of_dataset_id(self, dataset_id):
response = send_requests(self.db_params,
"signature/download",
params={"object": str(dataset_id)},
method="get")
data = response.json()["data"]
if len(data) > 0 and "url" in data[0]:
return data[0]["url"]
return None
def _insert_experiment_record(self, user_id: int, hdl_id: str,
task_id: str, experiment_type: str,
experiment_config: Dict[str, Any],
additional_info: Dict[str, Any]):
local = get_valid_params_in_kwargs(self._insert_experiment_record,
locals())
local.pop("user_id")
target = "experiment"
response = send_requests(self.db_params, target, local)
data = response.json()["data"]
if "experiment_id" not in data:
raise ValueError("insert experiment failed.")
return data["experiment_id"]
def _insert_task_record(self, task_id: str, user_id: int, metric: str,
splitter: Dict[str, str], ml_task: Dict[str, str],
train_set_id: str, test_set_id: str,
train_label_id: str, test_label_id: str,
specific_task_token: str, task_metadata: Dict[str,
Any],
sub_sample_indexes: List[int],
sub_feature_indexes: List[str]):
local = get_valid_params_in_kwargs(self._insert_task_record, locals())
local["sub_sample_indexes"] = json.dumps(local["sub_sample_indexes"])
local["sub_feature_indexes"] = json.dumps(local["sub_feature_indexes"])
local.pop("user_id")
target = "task"
data = send_requests(self.db_params, target, local).json()["data"]
return task_id
def login(self) -> Tuple[int, str]:
user = self.db_params["user"]
password = self.db_params["password"]
response = send_requests(self.db_params, "login", {
"user": user,
"password": password
})
response_json = response.json()
data = response_json["data"]
self.user_token = data["user_token"]
self.user_id = data["user_id"]
self.db_params["headers"].update({
"user_id": str(self.user_id),
"user_token": self.user_token,
})
return self.user_id, self.user_token
def _insert_dataset_record(self, user_id: int, dataset_id: str,
dataset_metadata: Dict[str, Any],
dataset_type: str, dataset_path: str,
upload_type: str, dataset_source: str,
column_descriptions: Dict[str, Any],
columns_mapper: Dict[str,
str], columns: List[str]):
local = get_valid_params_in_kwargs(self._insert_dataset_record,
locals())
local.pop("user_id")
local["columns"] = json.dumps(local["columns"])
target = "dataset"
response = send_requests(self.db_params, target, local)
data = response.json()["data"]
if "dataset_id" not in data:
data["dataset_id"] = dataset_id
data["dataset_path"] = dataset_path
data["length"] = 0
else:
data["length"] = 1
return data
def _insert_trial_record(self, user_id: int, task_id: str, hdl_id: str,
experiment_id: int, info: Dict[str, Any]):
local = get_valid_params_in_kwargs(self._insert_trial_record, locals())
local.pop("user_id")
target = "trial"
info = local.pop("info")
local.update(**info)
local.pop("hdl_id")
local.pop("task_id")
local.pop("finally_fit_model", None)
local["final_model_path"] = local.pop("finally_fit_model_path", None)
# local["intermediate_results"]=json.dumps(local["intermediate_results"])
local["losses"] = json.dumps(local["losses"])
local["all_scores"] = json.dumps(local["all_scores"])
# fixme
# local["test_loss"]={}
# local["test_all_score"]={}
response = send_requests(self.db_params, target, local)
data = response.json()["data"]
if "trial_id" not in data:
self.logger.warning("insert trial failed.")
return data.get("trial_id", -1)
def _get_trial_records_by_id(self, trial_id, task_id, user_id):
target = f"trial/{trial_id}"
response = send_requests(self.db_params, target, method="get")
# 返回值是一个字典
return response.json()["data"]
def _get_sorted_trial_records(self, task_id, user_id, limit):
target = f"task/{task_id}/trial?page_num={1}&page_size={limit}"
response = send_requests(self.db_params, target, method="get")
return extend_to_list(response.json()["data"])
def _get_task_records(self, task_id: str, user_id: int):
target = f"task/{task_id}"
response = send_requests(self.db_params, target, method="get")
return extend_to_list(response.json()["data"])
def _get_experiment_record(self, experiment_id):
target = f"experiment/{experiment_id}"
response = send_requests(self.db_params, target, method="get")
data = response.json()["data"]
return extend_to_list(data)
def _get_dataset_records(self, dataset_id,
user_id) -> List[Dict[str, Any]]:
target = f"dataset/{dataset_id}"
response = send_requests(self.db_params, target, method="get")
data = response.json()["data"]
return extend_to_list(data)
