def add_dataset(self, name, datasource_id, subid=None, creator=None, owner=None, current_process=None,
last_modifier=None, data_file_format=None, default_filter_string=None, description=None,
filter=None, patterns=None, target_entity_class=None):
"""
创建数据集元信息
:param name: 数据集名称
:param datasource_id: 数据源ID
:param subid:
:param creator:
:param owner:
:param current_process:
:param last_modifier:
:param data_file_format:
:param default_filter_string:
:param description:
:param filter:
:param patterns:
:param target_entity_class:
:return: 数据集ID
"""
check_name = self.db_session.query(Dataset).filter(Dataset.name == name).all()
if check_name :
raise DUPLICATE_NAME
id = generate_primary_key('DSET')
create_time = datetime.now()
if creator is None:
creator = 'admin'
if owner is None:
owner = 'admin'
if last_modifier is None:
last_modifier = 'admin'
try:
dataset = Dataset(id=id, subid=subid, creator=creator, owner=owner, current_process=current_process,
last_modifier=last_modifier, create_time=create_time, name=name,
data_file_format=data_file_format, datasource_id=datasource_id,
default_filter_string=default_filter_string, description=description, filter=filter,
patterns=patterns, target_entity_class=target_entity_class)
self.db_session.add(dataset)
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise ADD_FAILED
return id
def get_package(self, package_id):
# NO.A322
# Get package information.
# Args：
# package_id - The ID of package.
# Returns：
# package_object
# Exceptions:
# NON_EXISTING_PACKAGE - The given package_id does not exist.
package = self.db_session.query(Package).get(package_id)
if package is None:
logger.error('Package is not found')
raise NON_EXISTING_PACKAGE
return package
def get_all_model(self):
"""
查询全部模型元信息
:return: 模型元信息列表
"""
try:
model_list = self.db_session.query(Model).all()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
raise QUERY_FAILED
return model_list
def get_all_dataset(self):
"""
获取全部的数据集元信息
:return: 数据集元信息列表
"""
try:
datasets = self.db_session.query(Dataset).all()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
raise QUERY_FAILED
return datasets
def recover_algorithm(self, algorithm_id):
# NO.A315
# Recover deleted algorithm.
# Args：
# algorithm_id - The ID of algorithm.
# Returns：
# None.
# Exceptions:
# NON_EXISTING_ALGORITHM - The given algorithm_id does not exist.
algorithm = self.db_session.query(Algorithm).get(algorithm_id)
if algorithm is None:
logger.error('Algorithm is not found')
raise NON_EXISTING_ALGORITHM
algorithm.isdeleted = 0
self.db_session.commit()
def make_algorithm_unavailable(self, algorithm_id):
# NO.A316
# Make one algorithm unavailable --- cannot be used in tasks.
# Args：
# algorithm_id - The ID of algorithm.
# Returns：
# None.
# Exceptions:
# NON_EXISTING_ALGORITHM - The given algorithm_id does not exist.
algorithm = self.db_session.query(Algorithm).get(algorithm_id)
if algorithm is None:
logger.error('Algorithm is not found')
raise NON_EXISTING_ALGORITHM
algorithm.available = 0
self.db_session.commit()
def delete_model(self, model_id):
"""
删除模型元信息
:param model_id: 模型ID
:return: 无
"""
try:
pending = self.db_session.query(Model).get(model_id)
self.db_session.delete(pending)
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
raise DELETE_FAILED
def get_model(self, model_id):
"""
根据ID查询模型元信息
:param model_id: 模型ID
:return: 模型元信息
"""
try:
pending = self.db_session.query(Model).get(model_id)
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
raise QUERY_FAILED
return pending
def get_dataset(self, dataset_id):
"""
通过ID获取数据集元信息
:param dataset_id: 数据集ID
:return: 数据集元信息
"""
try:
dataset = self.db_session.query(Dataset).get(dataset_id)
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
raise QUERY_FAILED
return dataset
def get_all_algorithms(self):
# NO.A313
# Get all algorithms' information.
# Args：
# None
# Returns：
# algorithm_info list
try:
algorithms = self.db_session.query(Algorithm).all()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
raise QUERY_FAILED
return algorithms
def clean_algorithm(self, algorithm_id):
# NO.A317
# Clean one algorithm.
# It will be no longer saved in DB. It cannot be recovered.
# If an algorithm is using, it cannot be deleted.
# Args:
# algorithm_id - The ID of algorithm.
# Returns:
# None.
# Exceptions:
# NON_EXISTING_ALGORITHM - The given algorithm_id does not exist.
algorithm = self.db_session.query(Algorithm).get(algorithm_id)
if algorithm is None:
logger.error('Algorithm is not found')
raise NON_EXISTING_ALGORITHM
self.db_session.delete(algorithm)
self.db_session.commit()
def delete_dataset(self, dataset_id):
"""
删除数据集元信息
:param dataset_id: 数据集ID
:return: 无
"""
try:
pending = self.db_session.query(Dataset).get(dataset_id)
self.db_session.delete(pending)
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise DELETE_FAILED
def delete_algorithm(self, algorithm_id):
# NO.A314
# Set "isdeleted" of algorithm to 1 in the metadata DB. The algorithm cannot be used after deleted,
# It's still saved in DB. Thus when needed, deleted algorithms can be recovered.
# If an algorithm is running, it cannot be deleted.
# Args:
# algorithm_id - The ID of algorithm.
# Returns:
# None.
# Exceptions:
# NON_EXISTING_ALGORITHM - The given algorithm_id does not exist.
algorithm = self.db_session.query(Algorithm).get(algorithm_id)
if algorithm is None:
logger.error('Algorithm is not found')
raise NON_EXISTING_ALGORITHM
algorithm.isdeleted = 1
self.db_session.commit()
def get_algorithm(self, algorithm_id):
# NO.A312
# Get algorithm information.
# Args：
# algorithm_id - The ID of algorithm.
# Returns：
# algorithm_object
# Exceptions:
# NON_EXISTING_ALGORITHM - The given algorithm_id does not exist.
try:
algorithm = self.db_session.query(Algorithm).get(algorithm_id)
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
raise QUERY_FAILED
if algorithm is None:
logger.error('Algorithm is not found')
raise NON_EXISTING_ALGORITHM
return algorithm
def update_package(self,
package_id,
name=None,
package_source=None,
package_path=None,
description=None):
# NO.A321
# Update one package.
# Args：
# package_name - The name of package.
# deployment_path - The path to deploy algorithm.
# upload_source - The upload source of package.
# requirement_list - Dependencies of algorithms in this package.
# Returns：
# package_id - The id of package.
# conda_id - The id of conda.
# Exceptions:
# UPDATE_FAILURE - Fail to add this package.
pending = self.db_session.query(Package).get(package_id)
if package_id is None:
logger.error('package_id is needed')
raise PARAM_LACK
if name is not None:
pending.name = name
if package_source is not None:
pending.package_source = package_source
if package_path is not None:
pending.package_path = package_path
if description is not None:
pending.description = description
pending.last_modifier = 'admin'
pending.update_time = datetime.now()
self.db_session.commit()
return pending
def update_user(self,
user_id,
name=None,
date=None,
creator=None,
subid=None,
last_modifier=None,
comment=None,
password=None,
display_name=None,
email=None,
expiredtime=None,
isfrozen=False):
"""
:param user_id:
:param name:
:param date:
:param creator:
:param subid:
:param last_modifier:
:param comment:
:param password:
:param display_name:
:param email:
:param expiredtime:
:param isfrozen:
:return:
"""
if user_id is None:
logger.error('缺少用户ID')
raise PARAM_LACK
user = self.db_session.query(User).get(user_id)
if name is not None:
user.name = name
if date is not None:
user.date = date
if creator is not None:
user.creator = creator
if subid is not None:
user.subid = subid
if last_modifier is not None:
user.last_modifier = last_modifier
if comment is not None:
user.comment = comment
if password is not None:
user.password = password
if display_name is not None:
user.display_name = display_name
if email is not None:
user.email = email
if expiredtime is not None:
user.expiredtime = expiredtime
if isfrozen is not None:
user.isfrozen = isfrozen
user.last_modify_time = datetime.now()
self.db_session.commit()
def add_algorithm(self, name, creator=None, owner=None, last_modifier=None, display_name=None, description=None,
entry_name=None, algorithm_type=None,
hyperparameter_config=None, available=None, train_input_pattern=None, train_output_pattern=None,
model_input_pattern=None, model_output_pattern=None, runtime=None, learning=None,
package_id=None):
# NO.A301
# Add an algorithm into the metadata DB.
# Args:
# name - name of algorithm
# display_name - display name of algorithm
# discription - discription of algorithm
# entry_name - name of entry in the package
# algorithm_type - type of algorithm
# hyperparameter_config - hyperparameter config
# available - whether it is available
# train_input_pattern - [train_input_pattern_id1, train_input_pattern_id2 ...]
# train_output_pattern - [train_output_pattern_id1, train_output_pattern_id2 ...]
# model_input_pattern - [model_input_pattern_id1, model_input_pattern_id2 ...]
# model_output_pattern - [model_output_pattern_id1, model_output_pattern_id2 ...]
# runtime - running environment eg. xlearn.pytorch/ xlearn.tensorflow
# learning - whether it is a learning algorithm, 0/1
# package_id - id of package
# Returns:
# Algorithm id
# Exceptions:
#
check_name = self.db_session.query(Algorithm).filter(Algorithm.name == name).all()
if check_name:
raise DUPLICATE_NAME
id = generate_primary_key('ALGO')
create_time = datetime.now()
if creator is None:
creator = 'admin'
if owner is None:
owner = 'admin'
if last_modifier is None:
last_modifier = 'admin'
if available is None:
available = 1
try:
algorithm = Algorithm(id=id, subid=id, creator=creator, owner=owner, last_modifier=last_modifier,
create_time=create_time, name=name, display_name=display_name,
algorithm_type=algorithm_type, description=description,
alg_input_patterns=train_input_pattern, alg_output_patterns=train_output_pattern,
parameters=hyperparameter_config, entry_name=entry_name, available=available,
isbuiltin=0, isdeleted=0, islearning=learning,
model_input_patterns=model_input_pattern,
model_output_patterns=model_output_pattern, package_id=package_id,
prog_language='python',
reference_count=0, runtime=runtime)
self.db_session.add(algorithm)
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise ADD_FAILED
return id
def update_model(self,
model_id,
subid=None,
creator=None,
owner=None,
current_process=None,
last_modifier=None,
name=None,
algorithm_id=None,
description=None,
input_data_patterns=None,
output_data_patterns=None,
model_path=None,
model_resource=None,
usage=None):
"""
更新模型元信息
:param model_id: 模型ID
:param subid:
:param creator:
:param owner:
:param current_process:
:param last_modifier:
:param name:
:param algorithm_id:
:param description:
:param input_data_patterns:
:param output_data_patterns:
:param model_path:
:param model_resource:
:param usage:
:return: 无
"""
if model_id is None:
logger.error('缺少模型ID')
raise PARAM_LACK
check_name = self.db_session.query(Model).filter(
Model.name == name).all()
if check_name:
raise DUPLICATE_NAME
pending = self.db_session.query(Model).get(model_id)
try:
if subid is not None:
pending.subid = subid
if creator is not None:
pending.creator = creator
if owner is not None:
pending.owner = owner
if current_process is not None:
pending.current_process = current_process
if last_modifier is not None:
pending.last_modifier = last_modifier
if name is not None:
pending.name = name
if algorithm_id is not None:
pending.algorithm_id = algorithm_id
if description is not None:
pending.description = description
if input_data_patterns is not None:
pending.input_data_patterns = input_data_patterns
if output_data_patterns is not None:
pending.output_data_patterns = output_data_patterns
if model_path is not None:
pending.model_path = model_path
if model_resource is not None:
pending.model_resource = model_resource
if usage is not None:
pending.usage = usage
pending.update_time = datetime.now()
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise UPDATE_FAILED
def update_dataset(self, dataset_id, subid=None, creator=None, owner=None, current_process=None, last_modifier=None,
name=None, datasource_id=None, data_file_format=None, default_filter_string=None,
description=None, filter=None, patterns=None, target_entity_class=None):
"""
更新数据集元信息
:param dataset_id: 数据集ID
:param subid:
:param creator:
:param owner:
:param current_process:
:param last_modifier:
:param name: 数据集名称
:param datasource_id: 数据源ID
:param data_file_format:
:param default_filter_string:
:param description:
:param filter:
:param patterns:
:param target_entity_class:
:return: 无
"""
if dataset_id is None:
logger.error('缺少数据集ID')
raise PARAM_LACK
check_name = self.db_session.query(Dataset).filter(Dataset.name == name).all()
if check_name :
raise DUPLICATE_NAME
pending = self.db_session.query(Dataset).get(dataset_id)
try:
if subid is not None:
pending.subid = subid
if creator is not None:
pending.creator = creator
if owner is not None:
pending.owner = owner
if current_process is not None:
pending.current_process = current_process
if last_modifier is not None:
pending.last_modifier = last_modifier
if name is not None:
pending.name = name
if datasource_id is not None:
pending.datasource_id = datasource_id
if data_file_format is not None:
pending.data_file_format = data_file_format
if default_filter_string is not None:
pending.default_filter_string = default_filter_string
if description is not None:
pending.description = description
if filter is not None:
pending.filter = filter
if patterns is not None:
pending.patterns = patterns
if target_entity_class is not None:
pending.target_entity_class = target_entity_class
pending.update_time = datetime.now()
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise UPDATE_FAILED
def update_datasource(self,
datasource_id,
name=None,
subid=None,
creator=None,
owner=None,
current_process=None,
last_modifier=None,
data_file_format=None,
database_name=None,
datasource_type=None,
description=None,
folder_depth=None,
paramone=None,
password=None,
server_ip=None,
server_port=None,
username=None,
workbench_url=None):
"""
更新数据源元信息
:param datasource_id: 数据源ID
:param name: 数据源名称
:param subid:
:param creator:
:param owner:
:param current_process:
:param last_modifier:
:param data_file_format:
:param database_name:
:param datasource_type:
:param description:
:param folder_depth:
:param paramone:
:param password:
:param server_ip:
:param server_port:
:param username:
:param workbench_url:
:return: 无
"""
if datasource_id is None:
logger.error('缺少数据源ID')
raise PARAM_LACK
pending = self.db_session.query(Datasource).get(datasource_id)
if subid is not None:
pending.subid = subid
if creator is not None:
pending.creator = creator
if owner is not None:
pending.owner = owner
if current_process is not None:
pending.current_process = current_process
if last_modifier is not None:
pending.last_modifier = last_modifier
if name is not None:
pending.name = name
if database_name is not None:
pending.database_name = database_name
if server_ip is not None:
pending.server_ip = server_ip
if server_port is not None:
pending.server_port = server_port
if workbench_url is not None:
pending.workbench_url = workbench_url
if data_file_format is not None:
pending.data_file_format = data_file_format
if datasource_type is not None:
pending.datasource_type = datasource_type
if folder_depth is not None:
pending.folder_depth = folder_depth
if paramone is not None:
pending.paramone = paramone
if password is not None:
pending.password = password
if username is not None:
pending.username = username
if description is not None:
pending.description = description
pending.update_time = datetime.now()
self.db_session.commit()
def update_algorithm(self, algorithm_id, name=None, display_name=None, description=None, entry_name=None,
hyperparameter_config=None, train_input_pattern=None, train_output_pattern=None,
model_input_pattern=None, model_output_pattern=None, runtime=None, learning=None,
package_id=None):
# NO.A302
# Update a deep-learning algorithm into the metadata DB.
# Attention: It is DANGEROUS that the last hyperparameter_config will be covered.
# Args:
# algorithm_id - id of algorithm
# name - name of algorithm
# display_name - display name of algorithm
# discription - discription of algorithm
# entry_name - name of entry in the package
# algorithm_type - type of algorithm
# hyperparameter_config - hyperparameter config
# available - whether it is available
# train_input_pattern - [train_input_pattern_id1, train_input_pattern_id2 ...]
# train_output_pattern - [train_output_pattern_id1, train_output_pattern_id2 ...]
# model_input_pattern - [model_input_pattern_id1, model_input_pattern_id2 ...]
# model_output_pattern - [model_output_pattern_id1, model_output_pattern_id2 ...]
# runtime - running environment eg. xlearn.pytorch/ xlearn.tensorflow
# learning - whether it is a learning algorithm, 0/1
# package_id - id of package
# Returns:
# Algorithm id.
# Exceptions:
# NON_EXISTING_ALGORITHM - The given algorithm_id does not exist.
if algorithm_id is None:
logger.error('缺少算法ID')
raise PARAM_LACK
check_name = self.db_session.query(Algorithm).filter(Algorithm.name == name).all()
if check_name:
raise DUPLICATE_NAME
pending = self.db_session.query(Algorithm).get(algorithm_id)
if pending is None:
logger.error('Algorithm is not found')
raise NON_EXISTING_ALGORITHM
try:
if name is not None:
pending.name = name
if display_name is not None:
pending.display_name = display_name
if description is not None:
pending.description = description
if entry_name is not None:
pending.entry_name = entry_name
if hyperparameter_config is not None:
pending.parameters = hyperparameter_config
if train_input_pattern is not None:
pending.alg_input_patterns = train_input_pattern
if train_output_pattern is not None:
pending.alg_output_patterns = train_output_pattern
if model_input_pattern is not None:
pending.model_input_patterns = model_input_pattern
if model_output_pattern is not None:
pending.model_output_patterns = model_output_pattern
if runtime is not None:
pending.runtime = runtime
if learning is not None:
pending.islearning = learning
if package_id is not None:
pending.package_id = package_id
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise UPDATE_FAILED
return algorithm_id
def add_model(self,
name,
algorithm_id,
input_data_patterns,
output_data_patterns,
subid=None,
creator=None,
owner=None,
current_process=None,
last_modifier=None,
description=None,
model_path=None,
model_resource=None,
usage=None):
"""
创建模型元信息
:param name: 模型名称
:param algorithm_id: 算法ID
:param input_data_patterns:
:param output_data_patterns:
:param subid:
:param creator:
:param owner:
:param current_process:
:param last_modifier:
:param description:
:param model_path:
:param model_resource:
:param usage:
:return: 模型ID
"""
check_name = self.db_session.query(Model).filter(
Model.name == name).all()
if check_name:
raise DUPLICATE_NAME
id = generate_primary_key('MODE')
create_time = datetime.now()
if creator is None:
creator = 'admin'
if owner is None:
owner = 'admin'
if last_modifier is None:
last_modifier = 'admin'
try:
model = Model(id=id,
subid=subid,
creator=creator,
owner=owner,
current_process=current_process,
last_modifier=last_modifier,
create_time=create_time,
name=name,
algorithm_id=algorithm_id,
description=description,
input_data_patterns=input_data_patterns,
output_data_patterns=output_data_patterns,
model_path=model_path,
model_resource=model_resource,
usage=usage)
self.db_session.add(model)
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise ADD_FAILED
return id
