def action_publish(model):
'''
clone the development "dev" version as a new model version,
assigning a sequential version number
'''
if not model:
return False, 'empty model label'
bdir = utils.model_tree_path(model)
if not os.path.isdir(bdir):
LOG.error(f'Model {model} not found')
return False, 'model not found'
# gets version number
v = [int(x[-6:]) for x in os.listdir(bdir) if x.startswith("ver")]
if not v:
max_version = 0
else:
max_version = max(v)
new_dir = bdir+'/ver%0.6d' % (max_version+1)
if os.path.isdir(new_dir):
LOG.error(f'Versin {v} of model {model} not found')
return False, 'version already exists'
src_path = bdir+'/dev'
shutil.copytree(src_path, new_dir)
LOG.info(f'New model version created from {src_path} to {new_dir}')
return True, 'development version published as version '+str(max_version+1)
def action_list(model):
'''
Lists available models (if no argument is provided)
and model versions (if "model" is provided as argument)
'''
# TODO: if no argument is provided, also list all models
if not model:
rdir = utils.model_repository_path()
num_models = 0
models = []
print(' Models found in repository:')
for x in os.listdir(rdir):
num_models += 1
models.append(x)
print('\t- ', x)
LOG.debug(f'Retrieved list of models from {rdir}')
return True, ''
bdir = utils.model_tree_path(model)
num_versions = 0
for x in os.listdir(bdir):
if x.startswith("ver"):
num_versions += 1
print(model, ':', x)
LOG.info(f'model {model} has {num_versions} published versions')
return True, 'model '+model+' has '+str(num_versions)+' published versions'
def action_export(model):
'''
Exports the whole model tree indicated in the argument as a single
tarball file with the same name.
'''
if not model:
return False, 'Empty model label'
current_path = os.getcwd()
exportfile = os.path.join(current_path, model + '.tgz')
base_path = utils.model_tree_path(model)
if not os.path.isdir(base_path):
return False, 'Unable to export, endpoint directory not found'
# change to model repository to tar the file from there
os.chdir(base_path)
itemend = os.listdir()
itemend.sort()
with tarfile.open(exportfile, 'w:gz') as tar:
for iversion in itemend:
if not os.path.isdir(iversion):
continue
tar.add(iversion)
# return to current directory
os.chdir(current_path)
LOG.info(f'Model {model} exported as {model}.tgz')
return True, f'Model {model} exported as {model}.tgz'
def action_export(model):
'''
Exports the whole model tree indicated in the argument as a single
tarball file with the same name.
'''
if not model:
return False, 'empty model label'
current_path = os.getcwd()
exportfile = current_path+'/'+model+'.tgz'
bdir = utils.model_tree_path(model)
if not os.path.isdir(bdir):
LOG.error('Unable to export, model directory not found')
return False, 'endpoint directory not found'
os.chdir(bdir)
itemend = os.listdir()
itemend.sort()
with tarfile.open(exportfile, 'w:gz') as tar:
for iversion in itemend:
if not os.path.isdir(iversion):
continue
tar.add(iversion)
os.chdir(current_path)
LOG.info('Model exported as {}.tgz'.format(model))
return True, 'endpoint '+model+' exported as '+model+'.tgz'
def action_dir():
'''
Returns a JSON with the list of models and versions
'''
# get de model repo path
models_path = pathlib.Path(utils.model_repository_path())
# get directories in model repo path
dirs = [x for x in models_path.iterdir() if x.is_dir()]
# if dir contains dev/ -> is model (NAIVE APPROACH)
# get last dir name [-1]: model name
model_dirs = [d.parts[-1] for d in dirs if list(d.glob('dev'))]
results = []
for imodel in model_dirs:
# versions = ['dev']
versions = [{'text': 'dev'}]
for iversion in os.listdir(utils.model_tree_path(imodel)):
if iversion.startswith('ver'):
# versions.append (iversion)
versions.append({'text': iversion})
# results.append ((imodel,versions))
results.append({'text': imodel, 'nodes': versions})
return True, json.dumps(results)
def action_list(model):
'''
In no argument is provided lists all models present at the repository
otherwyse lists all versions for the model provided as argument
'''
# if no model name is provided, just list the model names
if not model:
rdir = utils.model_repository_path()
num_models = 0
LOG.info('Models found in repository:')
for x in os.listdir(rdir):
xpath = os.path.join(rdir, x)
# discard if the item is not a directory
if not os.path.isdir(xpath):
continue
# discard if the directory does not contain a 'dev' directory inside
if not os.path.isdir(os.path.join(xpath, 'dev')):
continue
num_models += 1
LOG.info('\t' + x)
LOG.debug(f'Retrieved list of models from {rdir}')
return True, f'{num_models} models found'
# if a model name is provided, list versions
base_path = utils.model_tree_path(model)
num_versions = 0
for x in os.listdir(base_path):
if x.startswith("ver"):
num_versions += 1
LOG.info(f'\t{model} : {x}')
return True, f'Model {model} has {num_versions} published versions'
def action_dir():
'''
Returns a JSON with the list of models and versions
'''
# get de model repo path
models_path = pathlib.Path(utils.model_repository_path())
# get directories in model repo path
dirs = [x for x in models_path.iterdir() if x.is_dir()]
# if dir contains dev/ -> is model (NAIVE APPROACH)
# get last dir name [-1]: model name
model_dirs = [d.parts[-1] for d in dirs if list(d.glob('dev'))]
results = []
for imodel in model_dirs:
idict = {}
idict ["modelname"] = imodel
versions = [0]
for iversion in os.listdir(utils.model_tree_path(imodel)):
if iversion.startswith('ver'):
versions.append(utils.modeldir2ver(iversion))
idict ["versions"] = versions
results.append(idict)
#print (json.dumps(results))
return True, json.dumps(results)
def action_new(model):
'''
Create a new model tree, using the given name.
This creates the development version "dev",
copying inside default child classes
'''
if not model:
return False, 'empty model label'
# importlib does not allow using 'test' and issues a misterious error when we
# try to use this name. This is a simple workaround to prevent creating models
# with this name
if model == 'test':
#LOG.warning(f'the name "test" is disallowed, please use any other name')
return False, 'the name "test" is disallowed, please use any other name'
# Model directory with /dev (default) level
ndir = pathlib.Path(utils.model_tree_path(model)) / 'dev'
# check if there is already a tree for this endpoint
if ndir.exists():
#LOG.warning(f'Endpoint {model} already exists')
return False, f'Endpoint {model} already exists'
try:
ndir.mkdir(parents=True)
LOG.debug(f'{ndir} created')
except:
return False, f'Unable to create path for {model} endpoint'
# Copy classes skeletons to ndir
wkd = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
children_names = ['apply', 'idata', 'odata', 'learn']
for cname in children_names:
filename = cname + '_child.py'
src_path = wkd / 'children' / filename
dst_path = ndir / filename
try:
shutil.copy(src_path, dst_path)
except:
return False, f'Unable to copy {cname} file'
LOG.debug(f'copied class skeletons from {src_path} to {dst_path}')
# copy parameter yml file
params_path = wkd / 'children/parameters.yaml'
shutil.copy(params_path, ndir)
# copy documentation yml file
documentation_path = wkd / 'children/documentation.yaml'
shutil.copy(documentation_path, ndir)
LOG.info(f'New endpoint {model} created')
#print(f'New endpoint {model} created')
return True, 'new endpoint '+model+' created'
def action_dir():
'''
Returns a list of models and versions
'''
# get de model repo path
models_path = pathlib.Path(utils.model_repository_path())
if models_path.is_dir() is False:
return False, 'the model repository path does not exist. Please run "flame -c config".'
# get directories in model repo path
dirs = [x for x in models_path.iterdir() if x.is_dir()]
# if dir contains dev/ -> is model (NAIVE APPROACH)
# get last dir name [-1]: model name
model_dirs = [d.parts[-1] for d in dirs if list(d.glob('dev'))]
label_defaults = {
'maturity': 'dev',
'type': 'unk',
'subtype': 'unk',
'endpoint': 'unk',
'species': 'unk'
}
results = []
for imodel in model_dirs:
idict = {}
idict["modelname"] = imodel
idict["version"] = 0
idict["info"] = action_info(imodel, 0, output=None)[1]
success, label = action_label(imodel, 0, oformat='object')
if success:
idict["label"] = label
else:
idict["label"] = label_defaults
results.append(idict)
for iversion in os.listdir(utils.model_tree_path(imodel)):
if iversion.startswith('ver'):
idict = {}
idict["modelname"] = imodel
idict["version"] = utils.modeldir2ver(iversion)
idict["info"] = action_info(imodel,
idict["version"],
output=None)[1]
success, label = action_label(imodel,
idict["version"],
oformat='object')
if success:
idict["label"] = label
else:
idict["label"] = label_defaults
results.append(idict)
# print (results)
return True, results
def action_kill(model):
'''
removes the model tree described by the argument
'''
if not model:
return False, 'empty model label'
ndir = utils.model_tree_path(model)
if not os.path.isdir(ndir):
LOG.error(f'Model {model} not found')
return False, 'model not found'
shutil.rmtree(ndir, ignore_errors=True)
LOG.info(f'Model {model} removed')
return True, 'model '+model+' removed'
def action_import(model):
'''
Creates a new model tree from a tarbal file with the name "model.tgz"
'''
if not model:
return False, 'empty model label'
# convert model to endpoint string
base_model = os.path.basename(model)
endpoint = os.path.splitext(base_model)[0]
ext = os.path.splitext(base_model)[1]
bdir = utils.model_tree_path(endpoint)
if os.path.isdir(bdir):
LOG.error(f'Endpoint already exists: {model}')
return False, 'endpoint already exists'
if ext != '.tgz':
importfile = os.path.abspath(model+'.tgz')
else:
importfile = model
LOG.info('importing {}'.format(importfile))
if not os.path.isfile(importfile):
LOG.info('importing package {} not found'.format(importfile))
return False, 'importing package '+importfile+' not found'
try:
os.mkdir(bdir)
except Exception as e:
LOG.error(f'error creating directory {bdir}: {e}')
raise e
# return False, 'error creating directory '+bdir
with tarfile.open(importfile, 'r:gz') as tar:
tar.extractall(bdir)
LOG.info('Endpoint {} imported OK'.format(endpoint))
return True, 'endpoint '+endpoint+' imported OK'
def action_import(model):
'''
Creates a new model tree from a tarbal file with the name "model.tgz"
'''
if not model:
return False, 'Empty model label'
# convert model to endpoint string
base_model = os.path.basename(model)
endpoint = os.path.splitext(base_model)[0]
ext = os.path.splitext(base_model)[1]
base_path = utils.model_tree_path(endpoint)
if os.path.isdir(base_path):
return False, f'Endpoint {endpoint} already exists'
if ext != '.tgz':
importfile = os.path.abspath(model + '.tgz')
else:
importfile = model
LOG.info(f'Importing {importfile} ...')
if not os.path.isfile(importfile):
LOG.info(f'Importing package {importfile} not found')
return False, f'Importing package {importfile} not found'
try:
os.mkdir(base_path)
except Exception as e:
return False, f'error creating directory {base_path}: {e}'
with tarfile.open(importfile, 'r:gz') as tar:
tar.extractall(base_path)
LOG.info(f'Endpoint {endpoint} imported OK')
return True, 'Endpoint ' + endpoint + ' imported OK'
def action_kill(model):
'''
removes the model tree described by the argument
'''
if not model:
return False, 'Empty model name'
ndir = utils.model_tree_path(model)
if not os.path.isdir(ndir):
#LOG.error(f'Model {model} not found')
return False, f'Model {model} not found'
try:
shutil.rmtree(ndir, ignore_errors=True)
except:
return False, f'Failed to remove model {model}'
LOG.info(f'Model {model} removed')
#print(f'Model {model} removed')
return True, f'Model {model} removed'
def action_publish(model):
'''
clone the development "dev" version as a new model version,
assigning a sequential version number
'''
if not model:
return False, 'Empty model label'
base_path = utils.model_tree_path(model)
if not os.path.isdir(base_path):
#LOG.error(f'Model {model} not found')
return False, f'Model {model} not found'
# gets version number
v = [int(x[-6:]) for x in os.listdir(base_path) if x.startswith("ver")]
if not v:
max_version = 0
else:
max_version = max(v)
new_path = os.path.join(base_path, f'ver{max_version+1:06}')
if os.path.isdir(new_path):
#LOG.error(f'Versin {v} of model {model} not found')
return False, f'Version {max_version+1} of model {model} already exists'
src_path = os.path.join(base_path, 'dev')
try:
shutil.copytree(src_path, new_path)
except:
return False, f'Unable to copy contents of dev version for model {model}'
LOG.info(f'New model version created from {src_path} to {new_path}')
return True, f'New model version created from {src_path} to {new_path}'
def action_refresh(model=None, version=None):
'''
Rebuild one or many models making use of existing parameter files and
locally stored training series.
'''
import flame.context as context
from flame.parameters import Parameters
import logging
# list endpoints relevant for the arguments
if model is not None:
model_list = [model]
else:
model_root = pathlib.Path(utils.model_repository_path())
model_list = [x.stem for x in model_root.iterdir() if x.is_dir()]
# list versions relevant for the arguments
task_list = []
for imodel in model_list:
if version is not None:
task_list.append((imodel, version))
else:
model_root = pathlib.Path(utils.model_tree_path(imodel))
itask_list = [(imodel, utils.modeldir2ver(x.stem))
for x in model_root.iterdir() if x.is_dir()]
task_list += itask_list # use "+=" and not "append" to merge the new list with the old one
# analize task_list and add at the end ensemble models
# this is needed to have low models refreshed BEFORE refreshing the high models
# eliminating the need to refresh them recursively
LOG.info("Analyzing and sorting models...")
# make sure the lower models are in task_list and, if not, force the inclussion
for itask in task_list:
param = Parameters()
success, results = param.loadYaml(itask[0], itask[1])
if not success:
continue
if param.getVal('input_type') == 'model_ensemble':
ens_nams = param.getVal('ensemble_names')
ens_vers = param.getVal('ensemble_versions')
for i in range(len(ens_nams)):
iver = 0
inam = ens_nams[i]
if (i < len(ens_vers)):
iver = ens_vers[i]
if ((inam, iver)) not in task_list:
task_list.append((inam, iver))
# create separate lists for regular and ensemble models
# and add ensemble models at the end
# this needs to be carried out after the previos step because
# some of the lower level models could be an ensemble model
# itself
mol_list = []
ens_list = []
for itask in task_list:
param = Parameters()
success, results = param.loadYaml(itask[0], itask[1])
if not success:
mol_list.append(itask)
continue
if param.getVal('input_type') == 'model_ensemble':
ens_list.append(itask)
else:
mol_list.append(itask)
task_list = mol_list + ens_list
# show all models before stating
LOG.info(
"Starting model refreshing task for the following models and versions")
for itask in task_list:
LOG.info(f' model: {itask[0]} version: {itask[1]}')
LOG.info("This can take some time, please be patient...")
# now send the build command for each task
for itask in task_list:
if itask[1] != 0:
# move version to /dev for building
original_path = utils.model_path(itask[0], itask[1]) # veri
destinat_path = utils.model_path(itask[0], 0) # dev
security_path = destinat_path + '_security' # dev_sec
shutil.move(destinat_path, security_path) # dev --> dev_sec
shutil.move(original_path, destinat_path) # veri --> dev
LOG.info(
f' refreshing model: {itask[0]} version: {itask[1]} ({task_list.index(itask)+1} of {len(task_list)})...'
)
# dissable LOG output
logging.disable(logging.ERROR)
command_build = {
'endpoint': itask[0],
'infile': None,
'param_file': None,
'incremental': False
}
success, results = context.build_cmd(command_build)
# enable LOG output
logging.disable(logging.NOTSET)
if itask[1] != 0:
shutil.move(destinat_path, original_path) # dev --> veri
shutil.move(security_path, destinat_path) # dev_sec --> dev
if not success:
LOG.error(results)
LOG.info("Model refreshing task finished")
return True, 'OK'
def action_import(model):
'''
Creates a new model tree from a tarbal file with the name "model.tgz"
'''
if not model:
return False, 'Empty model label'
# convert model to endpoint string
base_model = os.path.basename(model)
endpoint = os.path.splitext(base_model)[0]
ext = os.path.splitext(base_model)[1]
base_path = utils.model_tree_path(endpoint)
if os.path.isdir(base_path):
return False, f'Endpoint {endpoint} already exists'
if ext != '.tgz':
importfile = os.path.abspath(model + '.tgz')
else:
importfile = model
LOG.info(f'Importing {importfile} ...')
if not os.path.isfile(importfile):
LOG.info(f'Importing package {importfile} not found')
return False, f'Importing package {importfile} not found'
try:
os.mkdir(base_path)
except Exception as e:
return False, f'Error creating directory {base_path}: {e}'
with tarfile.open(importfile, 'r:gz') as tar:
tar.extractall(base_path)
# get libraries
message = f'Endpoint {endpoint} imported OK'
compatible = True
for x in os.listdir(base_path):
model_path = os.path.join(base_path, x)
model_pkl = os.path.join(model_path, 'estimator.pkl')
try:
with open(model_pkl, "rb") as input_file:
dict_estimator = pickle.load(input_file)
except FileNotFoundError:
continue
# check if the libraries used to build this model are similar to current libraries
if 'libraries' in dict_estimator:
# print (dict_estimator['libraries'])
success, results = utils.compatible_modules(
dict_estimator['libraries'])
if not success:
message = f"WARNING: Incompatible libraries detected, {results}. Use at your own risk"
return False, message
LOG.info(
'Libraries used to generate the imported model are compatible with local libraries'
)
LOG.info(message)
return True, message
def action_refresh(model=None, version=None, GUI=False):
'''
Rebuild one or many models making use of existing parameter files and
locally stored training series.
'''
import flame.context as context
from flame.parameters import Parameters
# from flame.documentation import Documentation
import logging
if GUI:
token_file = os.path.join(tempfile.gettempdir(), 'refreshing_' + model)
# update token file with content 'working'
with open(token_file, 'w') as f:
f.write('Analyzing and sorting models...')
# list endpoints relevant for the arguments
if model is not None:
model_list = [model]
else:
model_root = pathlib.Path(utils.model_repository_path())
model_list = [x.stem for x in model_root.iterdir() if x.is_dir()]
# list versions relevant for the arguments
task_list = []
for imodel in model_list:
if version is not None:
task_list.append((imodel, version))
else:
model_root = pathlib.Path(utils.model_tree_path(imodel))
itask_list = [(imodel, utils.modeldir2ver(x.stem))
for x in model_root.iterdir() if x.is_dir()]
task_list += itask_list # use "+=" and not "append" to merge the new list with the old one
# analize task_list and add at the end ensemble models
# this is needed to have low models refreshed BEFORE refreshing the high models
# eliminating the need to refresh them recursively
LOG.info("Analyzing and sorting models...")
# make sure the lower models are in task_list and, if not, force the inclussion
for itask in task_list:
param = Parameters()
success, results = param.loadYaml(itask[0], itask[1])
if not success:
continue
if param.getVal('input_type') == 'model_ensemble':
ens_nams = param.getVal('ensemble_names')
ens_vers = param.getVal('ensemble_versions')
for i in range(len(ens_nams)):
iver = 0
inam = ens_nams[i]
if (i < len(ens_vers)):
iver = ens_vers[i]
if ((inam, iver)) not in task_list:
task_list.append((inam, iver))
# create separate lists for regular and ensemble models
# and add ensemble models at the end
# this needs to be carried out after the previos step because
# some of the lower level models could be an ensemble model
# itself
mol_list = []
ens_list = []
for itask in task_list:
param = Parameters()
success, results = param.loadYaml(itask[0], itask[1])
if not success:
mol_list.append(itask)
continue
if param.getVal('input_type') == 'model_ensemble':
ens_list.append(itask)
else:
mol_list.append(itask)
task_list = mol_list + ens_list
# show all models before stating
LOG.info(
"Starting model refreshing task for the following models and versions")
for itask in task_list:
LOG.info(f' model: {itask[0]} version: {itask[1]}')
LOG.info("This can take some time, please be patient...")
source_dir = os.path.dirname(os.path.abspath(__file__))
children_dir = os.path.join(source_dir, 'children')
master_parameters = os.path.join(children_dir, 'parameters.yaml')
master_documentation = os.path.join(children_dir, 'documentation.yaml')
# now send the build command for each task
for itask in task_list:
destinat_path = utils.model_path(itask[0], 0) # dev
if itask[1] != 0:
# move version to /dev for building
original_path = utils.model_path(itask[0], itask[1]) # veri
security_path = destinat_path + '_security' # dev_sec
shutil.move(destinat_path, security_path) # dev --> dev_sec
shutil.move(original_path, destinat_path) # veri --> dev
LOG.info(
f' refreshing model: {itask[0]} version: {itask[1]} ({task_list.index(itask)+1} of {len(task_list)})...'
)
if GUI:
with open(token_file, 'w') as f:
f.write(
f'model: {itask[0]} version: {itask[1]} ({task_list.index(itask)+1} of {len(task_list)})'
)
# dissable LOG output
logging.disable(logging.ERROR)
# update parameters
dump_parameters = os.path.join(destinat_path, 'parameters_dump.yaml')
success, param = action_parameters(itask[0], 0, oformat='bin')
if success:
param_yaml = param.dumpYAML()
with open(dump_parameters, 'w') as f:
for line in param_yaml:
f.write(line + '\n')
else:
LOG.info(
' ERROR: unable to merge parameters for model: {itask[0]} version: {itask[1]}'
)
dump_parameters = None
original_parameters = os.path.join(destinat_path, 'parameters.yaml')
shutil.copy(master_parameters, original_parameters)
#update documentation
dump_documentation = os.path.join(destinat_path,
'documentation_dump.yaml')
success, documentation = action_documentation(itask[0],
0,
doc_file=None,
oformat='bin')
original_documentation = os.path.join(destinat_path,
'documentation.yaml')
shutil.copy(master_documentation, original_documentation)
if success:
documentation_yaml = documentation.dumpYAML()
with open(dump_documentation, 'w') as f:
for line in documentation_yaml:
line = line.encode("ascii", "ignore")
line = line.decode("ascii", "ignore")
f.write(line + '\n')
s2, documentation = action_documentation(itask[0],
0,
doc_file=None,
oformat='bin')
s3, r3 = documentation.delta(itask[0], 0, dump_documentation)
else:
LOG.info(
' ERROR: unable to merge documentation for model: {itask[0]} version: {itask[1]}'
)
# rebuild the model
command_build = {
'endpoint': itask[0],
'infile': None,
'param_file': dump_parameters,
'incremental': False
}
success, results = context.build_cmd(command_build)
# enable LOG output
logging.disable(logging.NOTSET)
if itask[1] != 0:
shutil.move(destinat_path, original_path) # dev --> veri
shutil.move(security_path, destinat_path) # dev_sec --> dev
if not success:
LOG.error(results)
LOG.info("Model refreshing task finished")
if GUI:
# update token file with status 'ready'
with open(token_file, 'w') as f:
f.write('ready')
return True, 'OK'
def action_export(model, version=0):
'''
Exports the whole model tree indicated in the argument as a single
tarball file with the same name.
'''
confidential_files = [
'confidential_model.yaml', 'confidential_preprocess.yaml',
'parameters.yaml', 'documentation.yaml'
]
if not model:
return False, 'Empty model label'
current_path = os.getcwd()
base_path = utils.model_tree_path(model)
if not os.path.isdir(base_path):
return False, 'Unable to export, endpoint directory not found'
# change to model repository to tar the file from there
os.chdir(base_path)
confidential = False
# single version export
if version != 0:
compressedname = f'{model}_v{version:06d}.tgz'
compressedfile = os.path.join(current_path, compressedname)
iversion = f'ver{version:06d}'
if (os.path.isfile(os.path.join(iversion, 'confidential_model.yaml'))):
confidential = True
with tarfile.open(compressedfile, 'w:gz') as tar:
for ifile in confidential_files:
ipath = os.path.join(iversion, ifile)
if os.path.isfile(ipath):
tar.add(ipath)
else:
with tarfile.open(compressedfile, 'w:gz') as tar:
tar.add(iversion)
else:
compressedname = model + '.tgz'
compressedfile = os.path.join(current_path, compressedname)
itemend = os.listdir()
itemend.sort()
# t1 = time.time()
with tarfile.open(compressedfile, 'w:gz') as tar:
for iversion in itemend:
if not os.path.isdir(iversion):
continue
tar.add(iversion)
# print (time.time()-t1)
# touch a file o indicate that the compressed file is ready
file = open(os.path.join(current_path, model + '.completed'), 'w')
file.close()
# return to current directory
os.chdir(current_path)
message = ''
if confidential:
message = f'Model {model} version {version} was exported in CONFIDENTIAL mode as {compressedname}'
else:
if version != 0:
message = f'Model {model} version {version} was exported as {compressedname}'
else:
message = f'All the versions for model {model} were exported as {compressedname}'
LOG.info(message)
return True, message
def action_import(model):
'''
Creates a new model tree from a tarbal file with the name "model.tgz"
'''
import re
if not model:
return False, 'Empty model label'
# convert model to endpoint string
base_model = os.path.basename(model)
endpoint = os.path.splitext(base_model)[0]
# find version in case of single version exports
version = None
if re.match("_v[0-9]{6}", endpoint[-8:]):
version = int(endpoint[-6:])
endpoint = endpoint[:-8]
ext = os.path.splitext(base_model)[1]
base_path = utils.model_tree_path(endpoint)
# safety checks
if os.path.isdir(base_path):
return False, f'Endpoint {endpoint} already exists'
if ext != '.tgz':
importfile = os.path.abspath(model + '.tgz')
else:
importfile = model
LOG.info(f'Importing {importfile} ...')
if not os.path.isfile(importfile):
LOG.info(f'Importing package {importfile} not found')
return False, f'Importing package {importfile} not found'
confidential = False
# create directory
try:
os.mkdir(base_path)
except Exception as e:
return False, f'Error creating directory {base_path}: {e}'
# unpack tar.gz. This is done for any kind of export file
with tarfile.open(importfile, 'r:gz') as tar:
tar.extractall(base_path)
# when importing a single version we need to clone the last folder to 'dev'
inner_dirs = os.listdir(base_path)
if not 'dev' in inner_dirs and version is not None:
# assign single version using file name
version_dir = f'ver{version:06d}'
# as a fallback assign the last internal folder
if not os.path.isdir(version_dir):
version_dir = inner_dirs[-1]
version_path = os.path.join(base_path, version_dir)
confidential_model = os.path.join(version_path,
'confidential_model.yaml')
# check if it is a confidential model
if (os.path.isfile(confidential_model)):
confidential = True
flame_source = os.path.dirname(os.path.abspath(__file__))
children_source = os.path.join(flame_source, 'children')
children_names = [
'apply', 'idata', 'odata', 'learn', 'slearn', 'sapply'
]
for cname in children_names:
cpath = os.path.join(children_source, cname + '_child.py')
shutil.copy(cpath, version_path)
LOG.info(f'Adding local children: {cpath} ...')
# open confidential_model.yaml
with open(confidential_model, 'r') as fc:
cmodel = yaml.safe_load(fc)
# create model-results.pkl
model_building_info = [('nobj', '', cmodel['nobj'])]
model_building_info += [('nvarx', '', cmodel['nvarx'])]
model_building_info += [('model', '', cmodel['model'])]
model_type_info = [('quantitative', '', cmodel['quantitative'])]
model_type_info += [('conformal', '', cmodel['conformal'])]
model_type_info += [('conformal_confidence', '',
cmodel['conformal_confidence'])]
model_type_info += [('ensemble', '', False)]
model_type_info += [('ensemble_names', '', [])]
model_type_info += [('ensemble_versions', '', [])]
model_type_info += [('confidential', '', True)]
model_type_info += [('secret', '', True)]
if cmodel['quantitative']:
model_validation_info = [('R2', '', cmodel['R2'])]
model_validation_info += [('Q2', '', cmodel['Q2'])]
model_validation_info += [('SDEC', '', cmodel['SDEC'])]
model_validation_info += [('SDEP', '', cmodel['SDEP'])]
model_validation_info += [('scoringP', '', cmodel['scoringP'])]
model_validation_info += [('scoringR', '', cmodel['scoringR'])]
else:
model_validation_info = [('MCC_f', '', cmodel['MCC_f'])]
model_validation_info += [('MCC', '', cmodel['MCC'])]
model_validation_info += [('Sensitivity_f', '',
cmodel['Sensitivity_f'])]
model_validation_info += [('Sensitivity', '',
cmodel['Sensitivity'])]
model_validation_info += [('Specificity_f', '',
cmodel['Specificity_f'])]
model_validation_info += [('Specificity', '',
cmodel['Specificity'])]
model_validation_info += [('FP_f', '', cmodel['FP_f'])]
model_validation_info += [('FP', '', cmodel['FP'])]
model_validation_info += [('FN_f', '', cmodel['FN_f'])]
model_validation_info += [('FN', '', cmodel['FN'])]
model_validation_info += [('TP_f', '', cmodel['TP_f'])]
model_validation_info += [('TP', '', cmodel['TP'])]
model_validation_info += [('TN_f', '', cmodel['TN_f'])]
model_validation_info += [('TN', '', cmodel['TN'])]
conveyor = Conveyor()
conveyor.addMeta('modelID', cmodel['modelID'])
conveyor.addMeta('endpoint', endpoint)
conveyor.addMeta('version', version)
conveyor.addMeta('quantitative', True)
conveyor.addMeta('secret', True)
conveyor.addVal(model_building_info, 'model_build_info',
'model building information', 'method', 'single',
'Information about the model building')
conveyor.addVal(model_validation_info, 'model_valid_info',
'model validation information', 'method', 'single',
'Information about the model validation')
conveyor.addVal(model_type_info, 'model_type_info',
'model type information', 'method', 'single',
'Information about the model type')
results_file_name = os.path.join(version_path, 'model-results.pkl')
with open(results_file_name, 'wb') as handle:
conveyor.save(handle)
meta_file_name = os.path.join(version_path, 'model-meta.pkl')
with open(meta_file_name, 'wb') as handle:
pickle.dump(cmodel['modelID'], handle)
pickle.dump(None, handle)
pickle.dump(None, handle)
pickle.dump(model_building_info, handle)
pickle.dump(model_validation_info, handle)
pickle.dump(model_type_info, handle)
# clone the version in dev
shutil.copytree(version_path, os.path.join(base_path, 'dev'))
LOG.info(f'Cloning version {version} to version 0 ...')
if confidential:
LOG.info(
f'Import of CONFIDENTIAL model {model} version {version} was successfull'
)
return True, 'OK'
# get libraries
message = f'Endpoint {endpoint} imported OK'
for x in os.listdir(base_path):
model_path = os.path.join(base_path, x)
model_pkl = os.path.join(model_path, 'estimator.pkl')
dict_estimator = {}
if os.path.isfile(model_pkl):
with open(model_pkl, "rb") as input_file:
try:
dict_estimator = pickle.load(input_file)
except Exception as e:
return False, f'Incompatible libraries found!. Import aborted with message "{str(e)}"'
# check if the libraries used to build this model are similar to current libraries
if 'libraries' in dict_estimator:
# print (dict_estimator['libraries'])
success, results = utils.compatible_modules(
dict_estimator['libraries'])
if not success:
message = f"WARNING: Incompatible libraries detected, {results}. Use at your own risk"
return False, message
LOG.info(
'Libraries used to generate the imported model are compatible with local libraries'
)
LOG.info(message)
return True, message
