def add_package(self,
name,
package_source,
package_path,
description=None):
# NO.A317
# Add one new package, create a new conda or find an existed conda according to requirement_list.
# Args：
# package_name - The name of package.
# deployment_path - The path to deploy algorithm.
# upload_source - The upload source of package.
# requirement_list - Dependencies of algorithms in this package.
# Returns：
# package_id
# hyperparameter_config
# Exceptions:
# ADD_FAILURE
id = generate_primary_key('PACK')
create_time = datetime.now()
package = Package(id=id,
subid=id,
creator='admin',
owner='admin',
last_modifier='admin',
create_time=create_time,
name=name,
package_source=package_source,
package_path=package_path,
description=description)
self.db_session.add(package)
self.db_session.commit()
return id
def add_user(self,
name,
date=None,
creator=None,
subid=None,
last_modifier=None,
last_modify_time=None,
comment=None,
password=None,
display_name=None,
email=None,
expiredtime=None,
isfrozen=False):
"""
:param name:
:param date:
:param creator:
:param subid:
:param last_modifier:
:param last_modify_time:
:param comment:
:param password:
:param display_name:
:param email:
:param expiredtime:
:param isfrozen:
:return:
"""
id = generate_primary_key('USER')
create_time = datetime.now()
if creator is None:
creator = 'admin'
if last_modifier is None:
last_modifier = 'admin'
user = User(id=id,
date=date,
creator=creator,
subid=subid,
last_modifier=last_modifier,
last_modify_time=last_modify_time,
comment=comment,
name=name,
password=password,
create_time=create_time,
display_name=display_name,
email=email,
expiredtime=expiredtime,
isfrozen=isfrozen)
self.db_session.add(user)
self.db_session.commit()
return id
def add_dataset(self, name, datasource_id, subid=None, creator=None, owner=None, current_process=None,
last_modifier=None, data_file_format=None, default_filter_string=None, description=None,
filter=None, patterns=None, target_entity_class=None):
"""
创建数据集元信息
:param name: 数据集名称
:param datasource_id: 数据源ID
:param subid:
:param creator:
:param owner:
:param current_process:
:param last_modifier:
:param data_file_format:
:param default_filter_string:
:param description:
:param filter:
:param patterns:
:param target_entity_class:
:return: 数据集ID
"""
check_name = self.db_session.query(Dataset).filter(Dataset.name == name).all()
if check_name :
raise DUPLICATE_NAME
id = generate_primary_key('DSET')
create_time = datetime.now()
if creator is None:
creator = 'admin'
if owner is None:
owner = 'admin'
if last_modifier is None:
last_modifier = 'admin'
try:
dataset = Dataset(id=id, subid=subid, creator=creator, owner=owner, current_process=current_process,
last_modifier=last_modifier, create_time=create_time, name=name,
data_file_format=data_file_format, datasource_id=datasource_id,
default_filter_string=default_filter_string, description=description, filter=filter,
patterns=patterns, target_entity_class=target_entity_class)
self.db_session.add(dataset)
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise ADD_FAILED
return id
def add_algorithm(self, name, creator=None, owner=None, last_modifier=None, display_name=None, description=None,
entry_name=None, algorithm_type=None,
hyperparameter_config=None, available=None, train_input_pattern=None, train_output_pattern=None,
model_input_pattern=None, model_output_pattern=None, runtime=None, learning=None,
package_id=None):
# NO.A301
# Add an algorithm into the metadata DB.
# Args:
# name - name of algorithm
# display_name - display name of algorithm
# discription - discription of algorithm
# entry_name - name of entry in the package
# algorithm_type - type of algorithm
# hyperparameter_config - hyperparameter config
# available - whether it is available
# train_input_pattern - [train_input_pattern_id1, train_input_pattern_id2 ...]
# train_output_pattern - [train_output_pattern_id1, train_output_pattern_id2 ...]
# model_input_pattern - [model_input_pattern_id1, model_input_pattern_id2 ...]
# model_output_pattern - [model_output_pattern_id1, model_output_pattern_id2 ...]
# runtime - running environment eg. xlearn.pytorch/ xlearn.tensorflow
# learning - whether it is a learning algorithm, 0/1
# package_id - id of package
# Returns:
# Algorithm id
# Exceptions:
#
check_name = self.db_session.query(Algorithm).filter(Algorithm.name == name).all()
if check_name:
raise DUPLICATE_NAME
id = generate_primary_key('ALGO')
create_time = datetime.now()
if creator is None:
creator = 'admin'
if owner is None:
owner = 'admin'
if last_modifier is None:
last_modifier = 'admin'
if available is None:
available = 1
try:
algorithm = Algorithm(id=id, subid=id, creator=creator, owner=owner, last_modifier=last_modifier,
create_time=create_time, name=name, display_name=display_name,
algorithm_type=algorithm_type, description=description,
alg_input_patterns=train_input_pattern, alg_output_patterns=train_output_pattern,
parameters=hyperparameter_config, entry_name=entry_name, available=available,
isbuiltin=0, isdeleted=0, islearning=learning,
model_input_patterns=model_input_pattern,
model_output_patterns=model_output_pattern, package_id=package_id,
prog_language='python',
reference_count=0, runtime=runtime)
self.db_session.add(algorithm)
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise ADD_FAILED
return id
def add_datasource(self,
name,
subid=None,
creator=None,
owner=None,
current_process=None,
last_modifier=None,
data_file_format=None,
database_name=None,
datasource_type='LOCAL_FS',
description=None,
folder_depth=None,
paramone=None,
password=None,
server_ip=None,
server_port=None,
username=None,
workbench_url=None):
"""
创建数据源元信息
:param name: 数据源名称
:param subid:
:param creator:
:param owner:
:param current_process:
:param last_modifier:
:param data_file_format:
:param database_name:
:param datasource_type:
:param description:
:param folder_depth:
:param paramone:
:param password:
:param server_ip:
:param server_port:
:param username:
:param workbench_url:
:return: 数据源ID
"""
id = generate_primary_key('DSOU')
create_time = datetime.now()
if creator is None:
creator = 'admin'
if owner is None:
owner = 'admin'
if last_modifier is None:
last_modifier = 'admin'
if database_name is None:
database_name = '/'
if folder_depth is None:
folder_depth = -1
new_datasource = Datasource(id=id,
subid=subid,
creator=creator,
owner=owner,
current_process=current_process,
last_modifier=last_modifier,
create_time=create_time,
name=name,
database_name=database_name,
data_file_format=data_file_format,
datasource_type=datasource_type,
description=description,
folder_depth=folder_depth,
paramone=paramone,
password=password,
server_ip=server_ip,
server_port=server_port,
username=username,
workbench_url=workbench_url)
self.db_session.add(new_datasource)
self.db_session.commit()
return id
def add_model(self,
name,
algorithm_id,
input_data_patterns,
output_data_patterns,
subid=None,
creator=None,
owner=None,
current_process=None,
last_modifier=None,
description=None,
model_path=None,
model_resource=None,
usage=None):
"""
创建模型元信息
:param name: 模型名称
:param algorithm_id: 算法ID
:param input_data_patterns:
:param output_data_patterns:
:param subid:
:param creator:
:param owner:
:param current_process:
:param last_modifier:
:param description:
:param model_path:
:param model_resource:
:param usage:
:return: 模型ID
"""
check_name = self.db_session.query(Model).filter(
Model.name == name).all()
if check_name:
raise DUPLICATE_NAME
id = generate_primary_key('MODE')
create_time = datetime.now()
if creator is None:
creator = 'admin'
if owner is None:
owner = 'admin'
if last_modifier is None:
last_modifier = 'admin'
try:
model = Model(id=id,
subid=subid,
creator=creator,
owner=owner,
current_process=current_process,
last_modifier=last_modifier,
create_time=create_time,
name=name,
algorithm_id=algorithm_id,
description=description,
input_data_patterns=input_data_patterns,
output_data_patterns=output_data_patterns,
model_path=model_path,
model_resource=model_resource,
usage=usage)
self.db_session.add(model)
self.db_session.commit()
except Exception as e:
logger.error(e)
logger.debug(traceback.format_exc())
self.db_session.rollback()
raise ADD_FAILED
return id