def __init__(self, model, param_file=None, output_format=None):
LOG.debug('Starting build...')
self.model = model
self.param = Parameters()
self.conveyor = Conveyor()
# load parameters
if param_file is not None:
# use the param_file to update existing parameters at the model
# directory and save changes to make them persistent
success, message = self.param.delta(model, 0, param_file)
else:
# load parameter file at the model directory
success, message = self.param.loadYaml(model, 0)
# being unable to load parameters is a critical error
if not success:
LOG.critical(
f'Unable to load model parameters. "{message}" Aborting...')
sys.exit(1)
# add additional output formats included in the constructor
# this is requiered to add JSON format as output when the object is
# instantiated from a web service call, requiring this output
if output_format is not None:
if output_format not in self.param.getVal('output_format'):
self.param.appVal('output_format', output_format)
def __init__(self,
space,
param_file=None,
param_string=None,
output_format=None):
LOG.debug('Starting sbuild...')
self.space = space
self.param = Parameters()
self.conveyor = Conveyor()
# identify the workflow type
self.conveyor.setOrigin('slearn')
# generate a unique modelID
self.conveyor.addMeta('modelID', utils.id_generator())
LOG.debug(
f'Generated new space with modelID: {self.conveyor.getMeta("modelID")}'
)
# load parameters
if param_file is not None:
# use the param_file to update existing parameters at the space
# directory and save changes to make them persistent
success, message = self.param.delta(space,
0,
param_file,
iformat='YAML',
isSpace=True)
elif param_string is not None:
success, message = self.param.delta(space,
0,
param_string,
iformat='JSONS',
isSpace=True)
else:
# load parameter file at the space directory
success, message = self.param.loadYaml(space, 0, isSpace=True)
# being unable to load parameters is a critical error
if not success:
LOG.critical(
f'Unable to load space parameters. {message}. Aborting...')
sys.exit(1)
# add additional output formats included in the constructor
# this is requiered to add JSON format as output when the object is
# instantiated from a web service call, requiring this output
if output_format is not None:
if output_format not in self.param.getVal('output_format'):
self.param.appVal('output_format', output_format)
def __init__(self, space, version, output_format=None, label=None):
LOG.debug('Starting search...')
self.space = space
self.version = version
self.label = label
self.param = Parameters()
self.conveyor = Conveyor()
# identify the workflow type
self.conveyor.setOrigin('sapply')
# load modelID
path = utils.space_path(space, version)
meta = os.path.join(path, 'space-meta.pkl')
try:
with open(meta, 'rb') as handle:
modelID = pickle.load(handle)
except:
LOG.critical(f'Unable to load modelID from {meta}. Aborting...')
sys.exit()
self.conveyor.addMeta('modelID', modelID)
LOG.debug(f'Loaded space with modelID: {modelID}')
# assign prediction (search) label
self.conveyor.addVal(label, 'prediction_label', 'prediction label',
'method', 'single',
'Label used to identify the prediction')
success, results = self.param.loadYaml(space, version, isSpace=True)
if not success:
LOG.critical(
f'Unable to load space parameters. {results}. Aborting...')
sys.exit()
# add additional output formats included in the constructor
# this is requiered to add JSON format as output when the object is
# instantiated from a web service call, requiring this output
if output_format != None:
if output_format not in self.param.getVal('output_format'):
self.param.appVal('output_format', output_format)
return
def __init__(self, model, version, output_format=None):
LOG.debug('Starting predict...')
self.model = model
self.version = version
self.param = Parameters()
self.conveyor = Conveyor()
if not self.param.loadYaml(model, version):
LOG.critical('Unable to load model parameters. Aborting...')
sys.exit()
# add additional output formats included in the constructor
# this is requiered to add JSON format as output when the object is
# instantiated from a web service call, requiring this output
if output_format != None:
if output_format not in self.param.getVal('output_format'):
self.param.appVal('output_format',output_format)
return
def action_parameters(space, version=None, oformat='text'):
''' Returns a JSON with whole results info for a given space and version '''
if space is None:
return False, 'Empty space label'
from flame.parameters import Parameters
param = Parameters()
param.loadYaml(space, version, isSpace=True)
if oformat != 'text':
return True, param
else:
yaml = param.dumpYAML()
for line in yaml:
print (line)
return True, 'parameters listed'
def __init__(self, model, version=0, context='model'):
''' Load the fields from the documentation file'''
self.model = model
self.version = version
self.fields = None
self.parameters = Parameters()
self.conveyor = None
# obtain the path and the default name of the model documents
documentation_file_path = utils.model_path(self.model, self.version)
documentation_file_name = os.path.join(documentation_file_path,
'documentation.yaml')
# load the main class dictionary (p) from this yaml file
if not os.path.isfile(documentation_file_name):
raise Exception('Documentation file not found')
try:
with open(documentation_file_name, 'r') as documentation_file:
self.fields = yaml.safe_load(documentation_file)
except Exception as e:
# LOG.error(f'Error loading documentation file with exception: {e}')
raise e
success, message = self.parameters.loadYaml(model, version)
if not success:
print(
f'Parameters could not be loaded. {message}. Please make sure endpoint and version are correct'
)
return
# Remove this after acc
#self.load_parameters()
if context == 'model':
self.load_results()
self.assign_parameters()
self.assign_results()
self.autocomplete_documentation()
self.setVal('md5', self.idataHash())
def __init__(self, model, version=0, output_format=None, label=None):
LOG.debug('Starting predict...')
self.model = model
self.version = version
self.param = Parameters()
self.conveyor = Conveyor()
# identify the workflow type
self.conveyor.setOrigin('apply')
# load modelID
success, result = utils.getModelID(model, version, 'model')
if not success:
LOG.critical(f'{result}. Aborting...')
sys.exit()
self.conveyor.addMeta('modelID', result)
LOG.debug(f'Loaded model with modelID: {result}')
# assign prediction label
self.conveyor.addVal(label, 'prediction_label', 'prediction label',
'method', 'single',
'Label used to identify the prediction')
success, results = self.param.loadYaml(model, version)
if not success:
LOG.critical(
f'Unable to load model parameters. {results}. Aborting...')
sys.exit()
# add additional output formats included in the constructor
# this is requiered to add JSON format as output when the object is
# instantiated from a web service call, requiring this output
if output_format != None:
if output_format not in self.param.getVal('output_format'):
self.param.appVal('output_format', output_format)
if 'ghost' in output_format:
self.param.setVal('output_similar', False)
return
def action_parameters(model, version=None, oformat='text'):
''' Returns an object with whole results info for a given model and version '''
if model is None:
return False, 'Empty model label'
from flame.parameters import Parameters
param = Parameters()
success, results = param.loadYaml(model, version)
if not success:
print(f'error obtaining parametes for model {model} : {results}')
return False, results
if oformat != 'text':
return True, param
else:
yaml = param.dumpYAML()
for line in yaml:
print(line)
return True, 'parameters listed'
def __init__(self, space, version, output_format=None, label=None):
LOG.debug('Starting predict...')
self.space = space
self.version = version
self.label = label
self.param = Parameters()
self.conveyor = Conveyor()
self.conveyor.addVal(label, 'prediction_label', 'prediction label',
'method', 'single',
'Label used to identify the prediction')
if not self.param.loadYaml(space, version, isSpace=True):
LOG.critical('Unable to load space parameters. Aborting...')
sys.exit()
# add additional output formats included in the constructor
# this is requiered to add JSON format as output when the object is
# instantiated from a web service call, requiring this output
if output_format != None:
if output_format not in self.param.getVal('output_format'):
self.param.appVal('output_format', output_format)
return
def action_parameters(space, version=None, oformat='text'):
''' Returns a JSON with whole results info for a given space and version '''
if space is None:
return False, 'Empty space label'
from flame.parameters import Parameters
param = Parameters()
param.loadYaml(space, version, isSpace=True)
if oformat == 'JSON':
return True, param.dumpJSON()
else:
order = [
'input_type',
'quantitative',
'SDFile_activity',
'SDFile_name',
'SDFile_id',
'SDFile_experimental',
'normalize_method',
'ionize_method',
'convert3D_method',
'computeMD_method',
'model',
'modelAutoscaling',
'tune',
'conformal',
'conformalSignificance',
'ModelValidationCV',
'ModelValidationLC',
'ModelValidationN',
'ModelValidationP',
'output_format',
'output_md',
'TSV_activity',
'TSV_objnames',
'TSV_varnames',
'imbalance',
'feature_selection',
'feature_number',
'mol_batch',
'ensemble_models',
'ensemble_versions',
'numCPUs',
'verbose_error',
'modelingToolkit',
'endpoint',
'model_path',
#'md5',
'version'
]
order += [
'MD_settings', 'RF_parameters', 'RF_optimize', 'SVM_parameters',
'SVM_optimize', 'PLSDA_parameters', 'PLSDA_optimize',
'PLSR_parameters', 'PLSR_optimize', 'GNB_parameters'
]
for ik in order:
if ik in param.p:
k = ik
v = param.p[k]
ivalue = ''
idescr = ''
ioptio = ''
## newest parameter formats are extended and contain
## rich metainformation for each entry
if param.extended:
if 'value' in v:
if not isinstance(v['value'], dict):
ivalue = v['value']
else:
# print header of dictionaty
print(f'{k} :')
# iterate keys assuming existence of value and description
for intk in v['value']:
intv = v['value'][intk]
iivalue = ''
if "value" in intv:
iivalue = intv["value"]
iidescr = ''
if "description" in intv and intv[
"description"] is not None:
iidescr = intv["description"]
iioptio = ''
if 'options' in intv:
toptio = intv['options']
if isinstance(toptio, list):
if toptio != [None]:
iioptio = f' {toptio}'
if isinstance(iivalue, float):
iivalue = f'{iivalue:f}'
elif iivalue is None:
iivalue = ''
print(
f' {intk:27} : {str(iivalue):30} #{iioptio} {iidescr}'
)
continue
if 'description' in v:
idescr = v['description']
if 'options' in v:
toptio = v['options']
if isinstance(toptio, list):
ioptio = f' {toptio}'
### compatibility: old stile parameters
else:
if not isinstance(v, dict):
ivalue = v
else:
ivalue = '*dictionary*'
### end compatibility
if isinstance(ivalue, float):
ivalue = f'{ivalue:f}'
elif ivalue is None:
ivalue = ''
print(f'{k:30} : {str(ivalue):30} #{ioptio} {idescr}')
return True, 'parameters listed'
def action_refresh(model=None, version=None, GUI=False):
'''
Rebuild one or many models making use of existing parameter files and
locally stored training series.
'''
import flame.context as context
from flame.parameters import Parameters
# from flame.documentation import Documentation
import logging
if GUI:
token_file = os.path.join(tempfile.gettempdir(), 'refreshing_' + model)
# update token file with content 'working'
with open(token_file, 'w') as f:
f.write('Analyzing and sorting models...')
# list endpoints relevant for the arguments
if model is not None:
model_list = [model]
else:
model_root = pathlib.Path(utils.model_repository_path())
model_list = [x.stem for x in model_root.iterdir() if x.is_dir()]
# list versions relevant for the arguments
task_list = []
for imodel in model_list:
if version is not None:
task_list.append((imodel, version))
else:
model_root = pathlib.Path(utils.model_tree_path(imodel))
itask_list = [(imodel, utils.modeldir2ver(x.stem))
for x in model_root.iterdir() if x.is_dir()]
task_list += itask_list # use "+=" and not "append" to merge the new list with the old one
# analize task_list and add at the end ensemble models
# this is needed to have low models refreshed BEFORE refreshing the high models
# eliminating the need to refresh them recursively
LOG.info("Analyzing and sorting models...")
# make sure the lower models are in task_list and, if not, force the inclussion
for itask in task_list:
param = Parameters()
success, results = param.loadYaml(itask[0], itask[1])
if not success:
continue
if param.getVal('input_type') == 'model_ensemble':
ens_nams = param.getVal('ensemble_names')
ens_vers = param.getVal('ensemble_versions')
for i in range(len(ens_nams)):
iver = 0
inam = ens_nams[i]
if (i < len(ens_vers)):
iver = ens_vers[i]
if ((inam, iver)) not in task_list:
task_list.append((inam, iver))
# create separate lists for regular and ensemble models
# and add ensemble models at the end
# this needs to be carried out after the previos step because
# some of the lower level models could be an ensemble model
# itself
mol_list = []
ens_list = []
for itask in task_list:
param = Parameters()
success, results = param.loadYaml(itask[0], itask[1])
if not success:
mol_list.append(itask)
continue
if param.getVal('input_type') == 'model_ensemble':
ens_list.append(itask)
else:
mol_list.append(itask)
task_list = mol_list + ens_list
# show all models before stating
LOG.info(
"Starting model refreshing task for the following models and versions")
for itask in task_list:
LOG.info(f' model: {itask[0]} version: {itask[1]}')
LOG.info("This can take some time, please be patient...")
source_dir = os.path.dirname(os.path.abspath(__file__))
children_dir = os.path.join(source_dir, 'children')
master_parameters = os.path.join(children_dir, 'parameters.yaml')
master_documentation = os.path.join(children_dir, 'documentation.yaml')
# now send the build command for each task
for itask in task_list:
destinat_path = utils.model_path(itask[0], 0) # dev
if itask[1] != 0:
# move version to /dev for building
original_path = utils.model_path(itask[0], itask[1]) # veri
security_path = destinat_path + '_security' # dev_sec
shutil.move(destinat_path, security_path) # dev --> dev_sec
shutil.move(original_path, destinat_path) # veri --> dev
LOG.info(
f' refreshing model: {itask[0]} version: {itask[1]} ({task_list.index(itask)+1} of {len(task_list)})...'
)
if GUI:
with open(token_file, 'w') as f:
f.write(
f'model: {itask[0]} version: {itask[1]} ({task_list.index(itask)+1} of {len(task_list)})'
)
# dissable LOG output
logging.disable(logging.ERROR)
# update parameters
dump_parameters = os.path.join(destinat_path, 'parameters_dump.yaml')
success, param = action_parameters(itask[0], 0, oformat='bin')
if success:
param_yaml = param.dumpYAML()
with open(dump_parameters, 'w') as f:
for line in param_yaml:
f.write(line + '\n')
else:
LOG.info(
' ERROR: unable to merge parameters for model: {itask[0]} version: {itask[1]}'
)
dump_parameters = None
original_parameters = os.path.join(destinat_path, 'parameters.yaml')
shutil.copy(master_parameters, original_parameters)
#update documentation
dump_documentation = os.path.join(destinat_path,
'documentation_dump.yaml')
success, documentation = action_documentation(itask[0],
0,
doc_file=None,
oformat='bin')
original_documentation = os.path.join(destinat_path,
'documentation.yaml')
shutil.copy(master_documentation, original_documentation)
if success:
documentation_yaml = documentation.dumpYAML()
with open(dump_documentation, 'w') as f:
for line in documentation_yaml:
line = line.encode("ascii", "ignore")
line = line.decode("ascii", "ignore")
f.write(line + '\n')
s2, documentation = action_documentation(itask[0],
0,
doc_file=None,
oformat='bin')
s3, r3 = documentation.delta(itask[0], 0, dump_documentation)
else:
LOG.info(
' ERROR: unable to merge documentation for model: {itask[0]} version: {itask[1]}'
)
# rebuild the model
command_build = {
'endpoint': itask[0],
'infile': None,
'param_file': dump_parameters,
'incremental': False
}
success, results = context.build_cmd(command_build)
# enable LOG output
logging.disable(logging.NOTSET)
if itask[1] != 0:
shutil.move(destinat_path, original_path) # dev --> veri
shutil.move(security_path, destinat_path) # dev_sec --> dev
if not success:
LOG.error(results)
LOG.info("Model refreshing task finished")
if GUI:
# update token file with status 'ready'
with open(token_file, 'w') as f:
f.write('ready')
return True, 'OK'
def action_refresh(model=None, version=None):
'''
Rebuild one or many models making use of existing parameter files and
locally stored training series.
'''
import flame.context as context
from flame.parameters import Parameters
import logging
# list endpoints relevant for the arguments
if model is not None:
model_list = [model]
else:
model_root = pathlib.Path(utils.model_repository_path())
model_list = [x.stem for x in model_root.iterdir() if x.is_dir()]
# list versions relevant for the arguments
task_list = []
for imodel in model_list:
if version is not None:
task_list.append((imodel, version))
else:
model_root = pathlib.Path(utils.model_tree_path(imodel))
itask_list = [(imodel, utils.modeldir2ver(x.stem))
for x in model_root.iterdir() if x.is_dir()]
task_list += itask_list # use "+=" and not "append" to merge the new list with the old one
# analize task_list and add at the end ensemble models
# this is needed to have low models refreshed BEFORE refreshing the high models
# eliminating the need to refresh them recursively
LOG.info("Analyzing and sorting models...")
# make sure the lower models are in task_list and, if not, force the inclussion
for itask in task_list:
param = Parameters()
success, results = param.loadYaml(itask[0], itask[1])
if not success:
continue
if param.getVal('input_type') == 'model_ensemble':
ens_nams = param.getVal('ensemble_names')
ens_vers = param.getVal('ensemble_versions')
for i in range(len(ens_nams)):
iver = 0
inam = ens_nams[i]
if (i < len(ens_vers)):
iver = ens_vers[i]
if ((inam, iver)) not in task_list:
task_list.append((inam, iver))
# create separate lists for regular and ensemble models
# and add ensemble models at the end
# this needs to be carried out after the previos step because
# some of the lower level models could be an ensemble model
# itself
mol_list = []
ens_list = []
for itask in task_list:
param = Parameters()
success, results = param.loadYaml(itask[0], itask[1])
if not success:
mol_list.append(itask)
continue
if param.getVal('input_type') == 'model_ensemble':
ens_list.append(itask)
else:
mol_list.append(itask)
task_list = mol_list + ens_list
# show all models before stating
LOG.info(
"Starting model refreshing task for the following models and versions")
for itask in task_list:
LOG.info(f' model: {itask[0]} version: {itask[1]}')
LOG.info("This can take some time, please be patient...")
# now send the build command for each task
for itask in task_list:
if itask[1] != 0:
# move version to /dev for building
original_path = utils.model_path(itask[0], itask[1]) # veri
destinat_path = utils.model_path(itask[0], 0) # dev
security_path = destinat_path + '_security' # dev_sec
shutil.move(destinat_path, security_path) # dev --> dev_sec
shutil.move(original_path, destinat_path) # veri --> dev
LOG.info(
f' refreshing model: {itask[0]} version: {itask[1]} ({task_list.index(itask)+1} of {len(task_list)})...'
)
# dissable LOG output
logging.disable(logging.ERROR)
command_build = {
'endpoint': itask[0],
'infile': None,
'param_file': None,
'incremental': False
}
success, results = context.build_cmd(command_build)
# enable LOG output
logging.disable(logging.NOTSET)
if itask[1] != 0:
shutil.move(destinat_path, original_path) # dev --> veri
shutil.move(security_path, destinat_path) # dev_sec --> dev
if not success:
LOG.error(results)
LOG.info("Model refreshing task finished")
return True, 'OK'
def action_parameters(model, version=None, oformat='text'):
''' Returns an object with whole results info for a given model and version '''
if model is None:
return False, 'Empty model label'
from flame.parameters import Parameters
param = Parameters()
success, results = param.loadYaml(model, version)
if not success:
print(f'error obtaining parametes for model {model} : {results}')
return False, results
if oformat != 'text':
return True, param
else:
yaml = param.dumpYAML()
for line in yaml:
print(line)
# order = ['input_type', 'quantitative', 'SDFile_activity', 'SDFile_name', 'SDFile_id',
# 'SDFile_experimental', 'SDFile_complementary', 'normalize_method', 'ionize_method', 'convert3D_method',
# 'computeMD_method', 'model', 'modelAutoscaling', 'tune', 'conformal',
# 'conformalConfidence', 'ModelValidationCV', 'ModelValidationLC',
# 'ModelValidationN', 'ModelValidationP', 'output_format', 'output_md', 'output_similar',
# 'TSV_activity', 'TSV_objnames', 'TSV_varnames', 'imbalance',
# 'feature_selection', 'feature_number', 'mol_batch',
# 'ensemble_names','ensemble_versions',
# 'similarity_metric', 'similarity_cutoff_num', 'similarity_cutoff_distance',
# 'numCPUs', 'verbose_error', 'modelingToolkit',
# 'endpoint', 'model_path',
# #'md5',
# 'version']
# order += ['MD_settings', 'RF_parameters','RF_optimize',
# 'SVM_parameters','SVM_optimize',
# 'PLSDA_parameters','PLSDA_optimize',
# 'PLSR_parameters','PLSR_optimize',
# 'GNB_parameters']
# for ik in order:
# if ik in param.p:
# k = ik
# v = param.p[k]
# ivalue = ''
# idescr = ''
# ioptio = ''
# ## newest parameter formats are extended and contain
# ## rich metainformation for each entry
# if param.extended:
# if 'value' in v:
# if not isinstance(v['value'] ,dict):
# ivalue = v['value']
# else:
# # print header of dictionaty
# print (f'{k} :')
# # iterate keys assuming existence of value and description
# for intk in v['value']:
# intv = v['value'][intk]
# iivalue = ''
# if "value" in intv:
# iivalue = intv["value"]
# iidescr = ''
# if "description" in intv and intv["description"] is not None:
# iidescr = intv["description"]
# iioptio = ''
# if 'options' in intv:
# toptio = intv['options']
# if isinstance(toptio, list):
# if toptio != [None]:
# iioptio = f' {toptio}'
# if isinstance (iivalue, float):
# iivalue = f'{iivalue:f}'
# elif iivalue is None:
# iivalue = ''
# print (f' {intk:27} : {str(iivalue):30} #{iioptio} {iidescr}')
# continue
# if 'description' in v:
# idescr = v['description']
# if 'options' in v:
# toptio = v['options']
# if isinstance(toptio, list):
# ioptio = f' {toptio}'
# ### compatibility: old stile parameters
# else:
# if not isinstance(v ,dict):
# ivalue = v
# else:
# ivalue = '*dictionary*'
# ### end compatibility
# if isinstance (ivalue, float):
# ivalue = f'{ivalue:f}'
# elif ivalue is None:
# ivalue = ''
# print (f'{k:30} : {str(ivalue):30} #{ioptio} {idescr}')
return True, 'parameters listed'