def add_dataset(self, name, datasource_id, subid=None, creator=None, owner=None, current_process=None,
last_modifier=None, data_file_format=None, default_filter_string=None, description=None,
filter=None, patterns=None, target_entity_class=None):
"""
创建数据集元信息
:param name: 数据集名称
:param datasource_id: 数据源ID
:param subid:
:param creator:
:param owner:
:param current_process:
:param last_modifier:
:param data_file_format:
:param default_filter_string:
:param description:
:param filter:
:param patterns:
:param target_entity_class:
:return: 数据集ID
"""
check_name = self.db_session.query(Dataset).filter(Dataset.name == name).all()
if check_name :
raise DUPLICATE_NAME
id = generate_primary_key('DSET')
create_time = datetime.now()
if creator is None:
creator = 'admin'
if owner is None:
owner = 'admin'
if last_modifier is None:
last_modifier = 'admin'
try:
dataset = Dataset(id=id, subid=subid, creator=creator, owner=owner, current_process=current_process,
last_modifier=last_modifier, create_time=create_time, name=name,
data_file_format=data_file_format, datasource_id=datasource_id,
default_filter_string=default_filter_string, description=description, filter=filter,
patterns=patterns, target_entity_class=target_entity_class)
self.db_session.add(dataset)
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise ADD_FAILED
return id
def get_all_dataset(self):
"""
获取全部的数据集元信息
:return: 数据集元信息列表
"""
try:
datasets = self.db_session.query(Dataset).all()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
raise QUERY_FAILED
return datasets
def get_all_model(self):
"""
查询全部模型元信息
:return: 模型元信息列表
"""
try:
model_list = self.db_session.query(Model).all()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
raise QUERY_FAILED
return model_list
def get_all_algorithms(self):
# NO.A313
# Get all algorithms' information.
# Args：
# None
# Returns：
# algorithm_info list
try:
algorithms = self.db_session.query(Algorithm).all()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
raise QUERY_FAILED
return algorithms
def get_dataset(self, dataset_id):
"""
通过ID获取数据集元信息
:param dataset_id: 数据集ID
:return: 数据集元信息
"""
try:
dataset = self.db_session.query(Dataset).get(dataset_id)
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
raise QUERY_FAILED
return dataset
def get_model(self, model_id):
"""
根据ID查询模型元信息
:param model_id: 模型ID
:return: 模型元信息
"""
try:
pending = self.db_session.query(Model).get(model_id)
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
raise QUERY_FAILED
return pending
def delete_model(self, model_id):
"""
删除模型元信息
:param model_id: 模型ID
:return: 无
"""
try:
pending = self.db_session.query(Model).get(model_id)
self.db_session.delete(pending)
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
raise DELETE_FAILED
def delete_dataset(self, dataset_id):
"""
删除数据集元信息
:param dataset_id: 数据集ID
:return: 无
"""
try:
pending = self.db_session.query(Dataset).get(dataset_id)
self.db_session.delete(pending)
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise DELETE_FAILED
def get_algorithm(self, algorithm_id):
# NO.A312
# Get algorithm information.
# Args：
# algorithm_id - The ID of algorithm.
# Returns：
# algorithm_object
# Exceptions:
# NON_EXISTING_ALGORITHM - The given algorithm_id does not exist.
try:
algorithm = self.db_session.query(Algorithm).get(algorithm_id)
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
raise QUERY_FAILED
if algorithm is None:
logger.error('Algorithm is not found')
raise NON_EXISTING_ALGORITHM
return algorithm
def update_algorithm(self, algorithm_id, name=None, display_name=None, description=None, entry_name=None,
hyperparameter_config=None, train_input_pattern=None, train_output_pattern=None,
model_input_pattern=None, model_output_pattern=None, runtime=None, learning=None,
package_id=None):
# NO.A302
# Update a deep-learning algorithm into the metadata DB.
# Attention: It is DANGEROUS that the last hyperparameter_config will be covered.
# Args:
# algorithm_id - id of algorithm
# name - name of algorithm
# display_name - display name of algorithm
# discription - discription of algorithm
# entry_name - name of entry in the package
# algorithm_type - type of algorithm
# hyperparameter_config - hyperparameter config
# available - whether it is available
# train_input_pattern - [train_input_pattern_id1, train_input_pattern_id2 ...]
# train_output_pattern - [train_output_pattern_id1, train_output_pattern_id2 ...]
# model_input_pattern - [model_input_pattern_id1, model_input_pattern_id2 ...]
# model_output_pattern - [model_output_pattern_id1, model_output_pattern_id2 ...]
# runtime - running environment eg. xlearn.pytorch/ xlearn.tensorflow
# learning - whether it is a learning algorithm, 0/1
# package_id - id of package
# Returns:
# Algorithm id.
# Exceptions:
# NON_EXISTING_ALGORITHM - The given algorithm_id does not exist.
if algorithm_id is None:
logger.error('缺少算法ID')
raise PARAM_LACK
check_name = self.db_session.query(Algorithm).filter(Algorithm.name == name).all()
if check_name:
raise DUPLICATE_NAME
pending = self.db_session.query(Algorithm).get(algorithm_id)
if pending is None:
logger.error('Algorithm is not found')
raise NON_EXISTING_ALGORITHM
try:
if name is not None:
pending.name = name
if display_name is not None:
pending.display_name = display_name
if description is not None:
pending.description = description
if entry_name is not None:
pending.entry_name = entry_name
if hyperparameter_config is not None:
pending.parameters = hyperparameter_config
if train_input_pattern is not None:
pending.alg_input_patterns = train_input_pattern
if train_output_pattern is not None:
pending.alg_output_patterns = train_output_pattern
if model_input_pattern is not None:
pending.model_input_patterns = model_input_pattern
if model_output_pattern is not None:
pending.model_output_patterns = model_output_pattern
if runtime is not None:
pending.runtime = runtime
if learning is not None:
pending.islearning = learning
if package_id is not None:
pending.package_id = package_id
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise UPDATE_FAILED
return algorithm_id
def add_algorithm(self, name, creator=None, owner=None, last_modifier=None, display_name=None, description=None,
entry_name=None, algorithm_type=None,
hyperparameter_config=None, available=None, train_input_pattern=None, train_output_pattern=None,
model_input_pattern=None, model_output_pattern=None, runtime=None, learning=None,
package_id=None):
# NO.A301
# Add an algorithm into the metadata DB.
# Args:
# name - name of algorithm
# display_name - display name of algorithm
# discription - discription of algorithm
# entry_name - name of entry in the package
# algorithm_type - type of algorithm
# hyperparameter_config - hyperparameter config
# available - whether it is available
# train_input_pattern - [train_input_pattern_id1, train_input_pattern_id2 ...]
# train_output_pattern - [train_output_pattern_id1, train_output_pattern_id2 ...]
# model_input_pattern - [model_input_pattern_id1, model_input_pattern_id2 ...]
# model_output_pattern - [model_output_pattern_id1, model_output_pattern_id2 ...]
# runtime - running environment eg. xlearn.pytorch/ xlearn.tensorflow
# learning - whether it is a learning algorithm, 0/1
# package_id - id of package
# Returns:
# Algorithm id
# Exceptions:
#
check_name = self.db_session.query(Algorithm).filter(Algorithm.name == name).all()
if check_name:
raise DUPLICATE_NAME
id = generate_primary_key('ALGO')
create_time = datetime.now()
if creator is None:
creator = 'admin'
if owner is None:
owner = 'admin'
if last_modifier is None:
last_modifier = 'admin'
if available is None:
available = 1
try:
algorithm = Algorithm(id=id, subid=id, creator=creator, owner=owner, last_modifier=last_modifier,
create_time=create_time, name=name, display_name=display_name,
algorithm_type=algorithm_type, description=description,
alg_input_patterns=train_input_pattern, alg_output_patterns=train_output_pattern,
parameters=hyperparameter_config, entry_name=entry_name, available=available,
isbuiltin=0, isdeleted=0, islearning=learning,
model_input_patterns=model_input_pattern,
model_output_patterns=model_output_pattern, package_id=package_id,
prog_language='python',
reference_count=0, runtime=runtime)
self.db_session.add(algorithm)
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise ADD_FAILED
return id
def update_dataset(self, dataset_id, subid=None, creator=None, owner=None, current_process=None, last_modifier=None,
name=None, datasource_id=None, data_file_format=None, default_filter_string=None,
description=None, filter=None, patterns=None, target_entity_class=None):
"""
更新数据集元信息
:param dataset_id: 数据集ID
:param subid:
:param creator:
:param owner:
:param current_process:
:param last_modifier:
:param name: 数据集名称
:param datasource_id: 数据源ID
:param data_file_format:
:param default_filter_string:
:param description:
:param filter:
:param patterns:
:param target_entity_class:
:return: 无
"""
if dataset_id is None:
logger.error('缺少数据集ID')
raise PARAM_LACK
check_name = self.db_session.query(Dataset).filter(Dataset.name == name).all()
if check_name :
raise DUPLICATE_NAME
pending = self.db_session.query(Dataset).get(dataset_id)
try:
if subid is not None:
pending.subid = subid
if creator is not None:
pending.creator = creator
if owner is not None:
pending.owner = owner
if current_process is not None:
pending.current_process = current_process
if last_modifier is not None:
pending.last_modifier = last_modifier
if name is not None:
pending.name = name
if datasource_id is not None:
pending.datasource_id = datasource_id
if data_file_format is not None:
pending.data_file_format = data_file_format
if default_filter_string is not None:
pending.default_filter_string = default_filter_string
if description is not None:
pending.description = description
if filter is not None:
pending.filter = filter
if patterns is not None:
pending.patterns = patterns
if target_entity_class is not None:
pending.target_entity_class = target_entity_class
pending.update_time = datetime.now()
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise UPDATE_FAILED
def add_model(self,
name,
algorithm_id,
input_data_patterns,
output_data_patterns,
subid=None,
creator=None,
owner=None,
current_process=None,
last_modifier=None,
description=None,
model_path=None,
model_resource=None,
usage=None):
"""
创建模型元信息
:param name: 模型名称
:param algorithm_id: 算法ID
:param input_data_patterns:
:param output_data_patterns:
:param subid:
:param creator:
:param owner:
:param current_process:
:param last_modifier:
:param description:
:param model_path:
:param model_resource:
:param usage:
:return: 模型ID
"""
check_name = self.db_session.query(Model).filter(
Model.name == name).all()
if check_name:
raise DUPLICATE_NAME
id = generate_primary_key('MODE')
create_time = datetime.now()
if creator is None:
creator = 'admin'
if owner is None:
owner = 'admin'
if last_modifier is None:
last_modifier = 'admin'
try:
model = Model(id=id,
subid=subid,
creator=creator,
owner=owner,
current_process=current_process,
last_modifier=last_modifier,
create_time=create_time,
name=name,
algorithm_id=algorithm_id,
description=description,
input_data_patterns=input_data_patterns,
output_data_patterns=output_data_patterns,
model_path=model_path,
model_resource=model_resource,
usage=usage)
self.db_session.add(model)
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise ADD_FAILED
return id
def update_model(self,
model_id,
subid=None,
creator=None,
owner=None,
current_process=None,
last_modifier=None,
name=None,
algorithm_id=None,
description=None,
input_data_patterns=None,
output_data_patterns=None,
model_path=None,
model_resource=None,
usage=None):
"""
更新模型元信息
:param model_id: 模型ID
:param subid:
:param creator:
:param owner:
:param current_process:
:param last_modifier:
:param name:
:param algorithm_id:
:param description:
:param input_data_patterns:
:param output_data_patterns:
:param model_path:
:param model_resource:
:param usage:
:return: 无
"""
if model_id is None:
logger.error('缺少模型ID')
raise PARAM_LACK
check_name = self.db_session.query(Model).filter(
Model.name == name).all()
if check_name:
raise DUPLICATE_NAME
pending = self.db_session.query(Model).get(model_id)
try:
if subid is not None:
pending.subid = subid
if creator is not None:
pending.creator = creator
if owner is not None:
pending.owner = owner
if current_process is not None:
pending.current_process = current_process
if last_modifier is not None:
pending.last_modifier = last_modifier
if name is not None:
pending.name = name
if algorithm_id is not None:
pending.algorithm_id = algorithm_id
if description is not None:
pending.description = description
if input_data_patterns is not None:
pending.input_data_patterns = input_data_patterns
if output_data_patterns is not None:
pending.output_data_patterns = output_data_patterns
if model_path is not None:
pending.model_path = model_path
if model_resource is not None:
pending.model_resource = model_resource
if usage is not None:
pending.usage = usage
pending.update_time = datetime.now()
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise UPDATE_FAILED