def action_publish(model): ''' clone the development "dev" version as a new model version, assigning a sequential version number ''' if not model: return False, 'empty model label' bdir = utils.model_tree_path(model) if not os.path.isdir(bdir): LOG.error(f'Model {model} not found') return False, 'model not found' # gets version number v = [int(x[-6:]) for x in os.listdir(bdir) if x.startswith("ver")] if not v: max_version = 0 else: max_version = max(v) new_dir = bdir+'/ver%0.6d' % (max_version+1) if os.path.isdir(new_dir): LOG.error(f'Versin {v} of model {model} not found') return False, 'version already exists' src_path = bdir+'/dev' shutil.copytree(src_path, new_dir) LOG.info(f'New model version created from {src_path} to {new_dir}') return True, 'development version published as version '+str(max_version+1)
def action_list(model): ''' Lists available models (if no argument is provided) and model versions (if "model" is provided as argument) ''' # TODO: if no argument is provided, also list all models if not model: rdir = utils.model_repository_path() num_models = 0 models = [] print(' Models found in repository:') for x in os.listdir(rdir): num_models += 1 models.append(x) print('\t- ', x) LOG.debug(f'Retrieved list of models from {rdir}') return True, '' bdir = utils.model_tree_path(model) num_versions = 0 for x in os.listdir(bdir): if x.startswith("ver"): num_versions += 1 print(model, ':', x) LOG.info(f'model {model} has {num_versions} published versions') return True, 'model '+model+' has '+str(num_versions)+' published versions'
def action_export(model): ''' Exports the whole model tree indicated in the argument as a single tarball file with the same name. ''' if not model: return False, 'Empty model label' current_path = os.getcwd() exportfile = os.path.join(current_path, model + '.tgz') base_path = utils.model_tree_path(model) if not os.path.isdir(base_path): return False, 'Unable to export, endpoint directory not found' # change to model repository to tar the file from there os.chdir(base_path) itemend = os.listdir() itemend.sort() with tarfile.open(exportfile, 'w:gz') as tar: for iversion in itemend: if not os.path.isdir(iversion): continue tar.add(iversion) # return to current directory os.chdir(current_path) LOG.info(f'Model {model} exported as {model}.tgz') return True, f'Model {model} exported as {model}.tgz'
def action_export(model): ''' Exports the whole model tree indicated in the argument as a single tarball file with the same name. ''' if not model: return False, 'empty model label' current_path = os.getcwd() exportfile = current_path+'/'+model+'.tgz' bdir = utils.model_tree_path(model) if not os.path.isdir(bdir): LOG.error('Unable to export, model directory not found') return False, 'endpoint directory not found' os.chdir(bdir) itemend = os.listdir() itemend.sort() with tarfile.open(exportfile, 'w:gz') as tar: for iversion in itemend: if not os.path.isdir(iversion): continue tar.add(iversion) os.chdir(current_path) LOG.info('Model exported as {}.tgz'.format(model)) return True, 'endpoint '+model+' exported as '+model+'.tgz'
def action_dir(): ''' Returns a JSON with the list of models and versions ''' # get de model repo path models_path = pathlib.Path(utils.model_repository_path()) # get directories in model repo path dirs = [x for x in models_path.iterdir() if x.is_dir()] # if dir contains dev/ -> is model (NAIVE APPROACH) # get last dir name [-1]: model name model_dirs = [d.parts[-1] for d in dirs if list(d.glob('dev'))] results = [] for imodel in model_dirs: # versions = ['dev'] versions = [{'text': 'dev'}] for iversion in os.listdir(utils.model_tree_path(imodel)): if iversion.startswith('ver'): # versions.append (iversion) versions.append({'text': iversion}) # results.append ((imodel,versions)) results.append({'text': imodel, 'nodes': versions}) return True, json.dumps(results)
def action_list(model): ''' In no argument is provided lists all models present at the repository otherwyse lists all versions for the model provided as argument ''' # if no model name is provided, just list the model names if not model: rdir = utils.model_repository_path() num_models = 0 LOG.info('Models found in repository:') for x in os.listdir(rdir): xpath = os.path.join(rdir, x) # discard if the item is not a directory if not os.path.isdir(xpath): continue # discard if the directory does not contain a 'dev' directory inside if not os.path.isdir(os.path.join(xpath, 'dev')): continue num_models += 1 LOG.info('\t' + x) LOG.debug(f'Retrieved list of models from {rdir}') return True, f'{num_models} models found' # if a model name is provided, list versions base_path = utils.model_tree_path(model) num_versions = 0 for x in os.listdir(base_path): if x.startswith("ver"): num_versions += 1 LOG.info(f'\t{model} : {x}') return True, f'Model {model} has {num_versions} published versions'
def action_dir(): ''' Returns a JSON with the list of models and versions ''' # get de model repo path models_path = pathlib.Path(utils.model_repository_path()) # get directories in model repo path dirs = [x for x in models_path.iterdir() if x.is_dir()] # if dir contains dev/ -> is model (NAIVE APPROACH) # get last dir name [-1]: model name model_dirs = [d.parts[-1] for d in dirs if list(d.glob('dev'))] results = [] for imodel in model_dirs: idict = {} idict ["modelname"] = imodel versions = [0] for iversion in os.listdir(utils.model_tree_path(imodel)): if iversion.startswith('ver'): versions.append(utils.modeldir2ver(iversion)) idict ["versions"] = versions results.append(idict) #print (json.dumps(results)) return True, json.dumps(results)
def action_new(model): ''' Create a new model tree, using the given name. This creates the development version "dev", copying inside default child classes ''' if not model: return False, 'empty model label' # importlib does not allow using 'test' and issues a misterious error when we # try to use this name. This is a simple workaround to prevent creating models # with this name if model == 'test': #LOG.warning(f'the name "test" is disallowed, please use any other name') return False, 'the name "test" is disallowed, please use any other name' # Model directory with /dev (default) level ndir = pathlib.Path(utils.model_tree_path(model)) / 'dev' # check if there is already a tree for this endpoint if ndir.exists(): #LOG.warning(f'Endpoint {model} already exists') return False, f'Endpoint {model} already exists' try: ndir.mkdir(parents=True) LOG.debug(f'{ndir} created') except: return False, f'Unable to create path for {model} endpoint' # Copy classes skeletons to ndir wkd = pathlib.Path(os.path.dirname(os.path.abspath(__file__))) children_names = ['apply', 'idata', 'odata', 'learn'] for cname in children_names: filename = cname + '_child.py' src_path = wkd / 'children' / filename dst_path = ndir / filename try: shutil.copy(src_path, dst_path) except: return False, f'Unable to copy {cname} file' LOG.debug(f'copied class skeletons from {src_path} to {dst_path}') # copy parameter yml file params_path = wkd / 'children/parameters.yaml' shutil.copy(params_path, ndir) # copy documentation yml file documentation_path = wkd / 'children/documentation.yaml' shutil.copy(documentation_path, ndir) LOG.info(f'New endpoint {model} created') #print(f'New endpoint {model} created') return True, 'new endpoint '+model+' created'
def action_dir(): ''' Returns a list of models and versions ''' # get de model repo path models_path = pathlib.Path(utils.model_repository_path()) if models_path.is_dir() is False: return False, 'the model repository path does not exist. Please run "flame -c config".' # get directories in model repo path dirs = [x for x in models_path.iterdir() if x.is_dir()] # if dir contains dev/ -> is model (NAIVE APPROACH) # get last dir name [-1]: model name model_dirs = [d.parts[-1] for d in dirs if list(d.glob('dev'))] label_defaults = { 'maturity': 'dev', 'type': 'unk', 'subtype': 'unk', 'endpoint': 'unk', 'species': 'unk' } results = [] for imodel in model_dirs: idict = {} idict["modelname"] = imodel idict["version"] = 0 idict["info"] = action_info(imodel, 0, output=None)[1] success, label = action_label(imodel, 0, oformat='object') if success: idict["label"] = label else: idict["label"] = label_defaults results.append(idict) for iversion in os.listdir(utils.model_tree_path(imodel)): if iversion.startswith('ver'): idict = {} idict["modelname"] = imodel idict["version"] = utils.modeldir2ver(iversion) idict["info"] = action_info(imodel, idict["version"], output=None)[1] success, label = action_label(imodel, idict["version"], oformat='object') if success: idict["label"] = label else: idict["label"] = label_defaults results.append(idict) # print (results) return True, results
def action_kill(model): ''' removes the model tree described by the argument ''' if not model: return False, 'empty model label' ndir = utils.model_tree_path(model) if not os.path.isdir(ndir): LOG.error(f'Model {model} not found') return False, 'model not found' shutil.rmtree(ndir, ignore_errors=True) LOG.info(f'Model {model} removed') return True, 'model '+model+' removed'
def action_import(model): ''' Creates a new model tree from a tarbal file with the name "model.tgz" ''' if not model: return False, 'empty model label' # convert model to endpoint string base_model = os.path.basename(model) endpoint = os.path.splitext(base_model)[0] ext = os.path.splitext(base_model)[1] bdir = utils.model_tree_path(endpoint) if os.path.isdir(bdir): LOG.error(f'Endpoint already exists: {model}') return False, 'endpoint already exists' if ext != '.tgz': importfile = os.path.abspath(model+'.tgz') else: importfile = model LOG.info('importing {}'.format(importfile)) if not os.path.isfile(importfile): LOG.info('importing package {} not found'.format(importfile)) return False, 'importing package '+importfile+' not found' try: os.mkdir(bdir) except Exception as e: LOG.error(f'error creating directory {bdir}: {e}') raise e # return False, 'error creating directory '+bdir with tarfile.open(importfile, 'r:gz') as tar: tar.extractall(bdir) LOG.info('Endpoint {} imported OK'.format(endpoint)) return True, 'endpoint '+endpoint+' imported OK'
def action_import(model): ''' Creates a new model tree from a tarbal file with the name "model.tgz" ''' if not model: return False, 'Empty model label' # convert model to endpoint string base_model = os.path.basename(model) endpoint = os.path.splitext(base_model)[0] ext = os.path.splitext(base_model)[1] base_path = utils.model_tree_path(endpoint) if os.path.isdir(base_path): return False, f'Endpoint {endpoint} already exists' if ext != '.tgz': importfile = os.path.abspath(model + '.tgz') else: importfile = model LOG.info(f'Importing {importfile} ...') if not os.path.isfile(importfile): LOG.info(f'Importing package {importfile} not found') return False, f'Importing package {importfile} not found' try: os.mkdir(base_path) except Exception as e: return False, f'error creating directory {base_path}: {e}' with tarfile.open(importfile, 'r:gz') as tar: tar.extractall(base_path) LOG.info(f'Endpoint {endpoint} imported OK') return True, 'Endpoint ' + endpoint + ' imported OK'
def action_kill(model): ''' removes the model tree described by the argument ''' if not model: return False, 'Empty model name' ndir = utils.model_tree_path(model) if not os.path.isdir(ndir): #LOG.error(f'Model {model} not found') return False, f'Model {model} not found' try: shutil.rmtree(ndir, ignore_errors=True) except: return False, f'Failed to remove model {model}' LOG.info(f'Model {model} removed') #print(f'Model {model} removed') return True, f'Model {model} removed'
def action_publish(model): ''' clone the development "dev" version as a new model version, assigning a sequential version number ''' if not model: return False, 'Empty model label' base_path = utils.model_tree_path(model) if not os.path.isdir(base_path): #LOG.error(f'Model {model} not found') return False, f'Model {model} not found' # gets version number v = [int(x[-6:]) for x in os.listdir(base_path) if x.startswith("ver")] if not v: max_version = 0 else: max_version = max(v) new_path = os.path.join(base_path, f'ver{max_version+1:06}') if os.path.isdir(new_path): #LOG.error(f'Versin {v} of model {model} not found') return False, f'Version {max_version+1} of model {model} already exists' src_path = os.path.join(base_path, 'dev') try: shutil.copytree(src_path, new_path) except: return False, f'Unable to copy contents of dev version for model {model}' LOG.info(f'New model version created from {src_path} to {new_path}') return True, f'New model version created from {src_path} to {new_path}'
def action_refresh(model=None, version=None): ''' Rebuild one or many models making use of existing parameter files and locally stored training series. ''' import flame.context as context from flame.parameters import Parameters import logging # list endpoints relevant for the arguments if model is not None: model_list = [model] else: model_root = pathlib.Path(utils.model_repository_path()) model_list = [x.stem for x in model_root.iterdir() if x.is_dir()] # list versions relevant for the arguments task_list = [] for imodel in model_list: if version is not None: task_list.append((imodel, version)) else: model_root = pathlib.Path(utils.model_tree_path(imodel)) itask_list = [(imodel, utils.modeldir2ver(x.stem)) for x in model_root.iterdir() if x.is_dir()] task_list += itask_list # use "+=" and not "append" to merge the new list with the old one # analize task_list and add at the end ensemble models # this is needed to have low models refreshed BEFORE refreshing the high models # eliminating the need to refresh them recursively LOG.info("Analyzing and sorting models...") # make sure the lower models are in task_list and, if not, force the inclussion for itask in task_list: param = Parameters() success, results = param.loadYaml(itask[0], itask[1]) if not success: continue if param.getVal('input_type') == 'model_ensemble': ens_nams = param.getVal('ensemble_names') ens_vers = param.getVal('ensemble_versions') for i in range(len(ens_nams)): iver = 0 inam = ens_nams[i] if (i < len(ens_vers)): iver = ens_vers[i] if ((inam, iver)) not in task_list: task_list.append((inam, iver)) # create separate lists for regular and ensemble models # and add ensemble models at the end # this needs to be carried out after the previos step because # some of the lower level models could be an ensemble model # itself mol_list = [] ens_list = [] for itask in task_list: param = Parameters() success, results = param.loadYaml(itask[0], itask[1]) if not success: mol_list.append(itask) continue if param.getVal('input_type') == 'model_ensemble': ens_list.append(itask) else: mol_list.append(itask) task_list = mol_list + ens_list # show all models before stating LOG.info( "Starting model refreshing task for the following models and versions") for itask in task_list: LOG.info(f' model: {itask[0]} version: {itask[1]}') LOG.info("This can take some time, please be patient...") # now send the build command for each task for itask in task_list: if itask[1] != 0: # move version to /dev for building original_path = utils.model_path(itask[0], itask[1]) # veri destinat_path = utils.model_path(itask[0], 0) # dev security_path = destinat_path + '_security' # dev_sec shutil.move(destinat_path, security_path) # dev --> dev_sec shutil.move(original_path, destinat_path) # veri --> dev LOG.info( f' refreshing model: {itask[0]} version: {itask[1]} ({task_list.index(itask)+1} of {len(task_list)})...' ) # dissable LOG output logging.disable(logging.ERROR) command_build = { 'endpoint': itask[0], 'infile': None, 'param_file': None, 'incremental': False } success, results = context.build_cmd(command_build) # enable LOG output logging.disable(logging.NOTSET) if itask[1] != 0: shutil.move(destinat_path, original_path) # dev --> veri shutil.move(security_path, destinat_path) # dev_sec --> dev if not success: LOG.error(results) LOG.info("Model refreshing task finished") return True, 'OK'
def action_import(model): ''' Creates a new model tree from a tarbal file with the name "model.tgz" ''' if not model: return False, 'Empty model label' # convert model to endpoint string base_model = os.path.basename(model) endpoint = os.path.splitext(base_model)[0] ext = os.path.splitext(base_model)[1] base_path = utils.model_tree_path(endpoint) if os.path.isdir(base_path): return False, f'Endpoint {endpoint} already exists' if ext != '.tgz': importfile = os.path.abspath(model + '.tgz') else: importfile = model LOG.info(f'Importing {importfile} ...') if not os.path.isfile(importfile): LOG.info(f'Importing package {importfile} not found') return False, f'Importing package {importfile} not found' try: os.mkdir(base_path) except Exception as e: return False, f'Error creating directory {base_path}: {e}' with tarfile.open(importfile, 'r:gz') as tar: tar.extractall(base_path) # get libraries message = f'Endpoint {endpoint} imported OK' compatible = True for x in os.listdir(base_path): model_path = os.path.join(base_path, x) model_pkl = os.path.join(model_path, 'estimator.pkl') try: with open(model_pkl, "rb") as input_file: dict_estimator = pickle.load(input_file) except FileNotFoundError: continue # check if the libraries used to build this model are similar to current libraries if 'libraries' in dict_estimator: # print (dict_estimator['libraries']) success, results = utils.compatible_modules( dict_estimator['libraries']) if not success: message = f"WARNING: Incompatible libraries detected, {results}. Use at your own risk" return False, message LOG.info( 'Libraries used to generate the imported model are compatible with local libraries' ) LOG.info(message) return True, message
def action_refresh(model=None, version=None, GUI=False): ''' Rebuild one or many models making use of existing parameter files and locally stored training series. ''' import flame.context as context from flame.parameters import Parameters # from flame.documentation import Documentation import logging if GUI: token_file = os.path.join(tempfile.gettempdir(), 'refreshing_' + model) # update token file with content 'working' with open(token_file, 'w') as f: f.write('Analyzing and sorting models...') # list endpoints relevant for the arguments if model is not None: model_list = [model] else: model_root = pathlib.Path(utils.model_repository_path()) model_list = [x.stem for x in model_root.iterdir() if x.is_dir()] # list versions relevant for the arguments task_list = [] for imodel in model_list: if version is not None: task_list.append((imodel, version)) else: model_root = pathlib.Path(utils.model_tree_path(imodel)) itask_list = [(imodel, utils.modeldir2ver(x.stem)) for x in model_root.iterdir() if x.is_dir()] task_list += itask_list # use "+=" and not "append" to merge the new list with the old one # analize task_list and add at the end ensemble models # this is needed to have low models refreshed BEFORE refreshing the high models # eliminating the need to refresh them recursively LOG.info("Analyzing and sorting models...") # make sure the lower models are in task_list and, if not, force the inclussion for itask in task_list: param = Parameters() success, results = param.loadYaml(itask[0], itask[1]) if not success: continue if param.getVal('input_type') == 'model_ensemble': ens_nams = param.getVal('ensemble_names') ens_vers = param.getVal('ensemble_versions') for i in range(len(ens_nams)): iver = 0 inam = ens_nams[i] if (i < len(ens_vers)): iver = ens_vers[i] if ((inam, iver)) not in task_list: task_list.append((inam, iver)) # create separate lists for regular and ensemble models # and add ensemble models at the end # this needs to be carried out after the previos step because # some of the lower level models could be an ensemble model # itself mol_list = [] ens_list = [] for itask in task_list: param = Parameters() success, results = param.loadYaml(itask[0], itask[1]) if not success: mol_list.append(itask) continue if param.getVal('input_type') == 'model_ensemble': ens_list.append(itask) else: mol_list.append(itask) task_list = mol_list + ens_list # show all models before stating LOG.info( "Starting model refreshing task for the following models and versions") for itask in task_list: LOG.info(f' model: {itask[0]} version: {itask[1]}') LOG.info("This can take some time, please be patient...") source_dir = os.path.dirname(os.path.abspath(__file__)) children_dir = os.path.join(source_dir, 'children') master_parameters = os.path.join(children_dir, 'parameters.yaml') master_documentation = os.path.join(children_dir, 'documentation.yaml') # now send the build command for each task for itask in task_list: destinat_path = utils.model_path(itask[0], 0) # dev if itask[1] != 0: # move version to /dev for building original_path = utils.model_path(itask[0], itask[1]) # veri security_path = destinat_path + '_security' # dev_sec shutil.move(destinat_path, security_path) # dev --> dev_sec shutil.move(original_path, destinat_path) # veri --> dev LOG.info( f' refreshing model: {itask[0]} version: {itask[1]} ({task_list.index(itask)+1} of {len(task_list)})...' ) if GUI: with open(token_file, 'w') as f: f.write( f'model: {itask[0]} version: {itask[1]} ({task_list.index(itask)+1} of {len(task_list)})' ) # dissable LOG output logging.disable(logging.ERROR) # update parameters dump_parameters = os.path.join(destinat_path, 'parameters_dump.yaml') success, param = action_parameters(itask[0], 0, oformat='bin') if success: param_yaml = param.dumpYAML() with open(dump_parameters, 'w') as f: for line in param_yaml: f.write(line + '\n') else: LOG.info( ' ERROR: unable to merge parameters for model: {itask[0]} version: {itask[1]}' ) dump_parameters = None original_parameters = os.path.join(destinat_path, 'parameters.yaml') shutil.copy(master_parameters, original_parameters) #update documentation dump_documentation = os.path.join(destinat_path, 'documentation_dump.yaml') success, documentation = action_documentation(itask[0], 0, doc_file=None, oformat='bin') original_documentation = os.path.join(destinat_path, 'documentation.yaml') shutil.copy(master_documentation, original_documentation) if success: documentation_yaml = documentation.dumpYAML() with open(dump_documentation, 'w') as f: for line in documentation_yaml: line = line.encode("ascii", "ignore") line = line.decode("ascii", "ignore") f.write(line + '\n') s2, documentation = action_documentation(itask[0], 0, doc_file=None, oformat='bin') s3, r3 = documentation.delta(itask[0], 0, dump_documentation) else: LOG.info( ' ERROR: unable to merge documentation for model: {itask[0]} version: {itask[1]}' ) # rebuild the model command_build = { 'endpoint': itask[0], 'infile': None, 'param_file': dump_parameters, 'incremental': False } success, results = context.build_cmd(command_build) # enable LOG output logging.disable(logging.NOTSET) if itask[1] != 0: shutil.move(destinat_path, original_path) # dev --> veri shutil.move(security_path, destinat_path) # dev_sec --> dev if not success: LOG.error(results) LOG.info("Model refreshing task finished") if GUI: # update token file with status 'ready' with open(token_file, 'w') as f: f.write('ready') return True, 'OK'
def action_export(model, version=0): ''' Exports the whole model tree indicated in the argument as a single tarball file with the same name. ''' confidential_files = [ 'confidential_model.yaml', 'confidential_preprocess.yaml', 'parameters.yaml', 'documentation.yaml' ] if not model: return False, 'Empty model label' current_path = os.getcwd() base_path = utils.model_tree_path(model) if not os.path.isdir(base_path): return False, 'Unable to export, endpoint directory not found' # change to model repository to tar the file from there os.chdir(base_path) confidential = False # single version export if version != 0: compressedname = f'{model}_v{version:06d}.tgz' compressedfile = os.path.join(current_path, compressedname) iversion = f'ver{version:06d}' if (os.path.isfile(os.path.join(iversion, 'confidential_model.yaml'))): confidential = True with tarfile.open(compressedfile, 'w:gz') as tar: for ifile in confidential_files: ipath = os.path.join(iversion, ifile) if os.path.isfile(ipath): tar.add(ipath) else: with tarfile.open(compressedfile, 'w:gz') as tar: tar.add(iversion) else: compressedname = model + '.tgz' compressedfile = os.path.join(current_path, compressedname) itemend = os.listdir() itemend.sort() # t1 = time.time() with tarfile.open(compressedfile, 'w:gz') as tar: for iversion in itemend: if not os.path.isdir(iversion): continue tar.add(iversion) # print (time.time()-t1) # touch a file o indicate that the compressed file is ready file = open(os.path.join(current_path, model + '.completed'), 'w') file.close() # return to current directory os.chdir(current_path) message = '' if confidential: message = f'Model {model} version {version} was exported in CONFIDENTIAL mode as {compressedname}' else: if version != 0: message = f'Model {model} version {version} was exported as {compressedname}' else: message = f'All the versions for model {model} were exported as {compressedname}' LOG.info(message) return True, message
def action_import(model): ''' Creates a new model tree from a tarbal file with the name "model.tgz" ''' import re if not model: return False, 'Empty model label' # convert model to endpoint string base_model = os.path.basename(model) endpoint = os.path.splitext(base_model)[0] # find version in case of single version exports version = None if re.match("_v[0-9]{6}", endpoint[-8:]): version = int(endpoint[-6:]) endpoint = endpoint[:-8] ext = os.path.splitext(base_model)[1] base_path = utils.model_tree_path(endpoint) # safety checks if os.path.isdir(base_path): return False, f'Endpoint {endpoint} already exists' if ext != '.tgz': importfile = os.path.abspath(model + '.tgz') else: importfile = model LOG.info(f'Importing {importfile} ...') if not os.path.isfile(importfile): LOG.info(f'Importing package {importfile} not found') return False, f'Importing package {importfile} not found' confidential = False # create directory try: os.mkdir(base_path) except Exception as e: return False, f'Error creating directory {base_path}: {e}' # unpack tar.gz. This is done for any kind of export file with tarfile.open(importfile, 'r:gz') as tar: tar.extractall(base_path) # when importing a single version we need to clone the last folder to 'dev' inner_dirs = os.listdir(base_path) if not 'dev' in inner_dirs and version is not None: # assign single version using file name version_dir = f'ver{version:06d}' # as a fallback assign the last internal folder if not os.path.isdir(version_dir): version_dir = inner_dirs[-1] version_path = os.path.join(base_path, version_dir) confidential_model = os.path.join(version_path, 'confidential_model.yaml') # check if it is a confidential model if (os.path.isfile(confidential_model)): confidential = True flame_source = os.path.dirname(os.path.abspath(__file__)) children_source = os.path.join(flame_source, 'children') children_names = [ 'apply', 'idata', 'odata', 'learn', 'slearn', 'sapply' ] for cname in children_names: cpath = os.path.join(children_source, cname + '_child.py') shutil.copy(cpath, version_path) LOG.info(f'Adding local children: {cpath} ...') # open confidential_model.yaml with open(confidential_model, 'r') as fc: cmodel = yaml.safe_load(fc) # create model-results.pkl model_building_info = [('nobj', '', cmodel['nobj'])] model_building_info += [('nvarx', '', cmodel['nvarx'])] model_building_info += [('model', '', cmodel['model'])] model_type_info = [('quantitative', '', cmodel['quantitative'])] model_type_info += [('conformal', '', cmodel['conformal'])] model_type_info += [('conformal_confidence', '', cmodel['conformal_confidence'])] model_type_info += [('ensemble', '', False)] model_type_info += [('ensemble_names', '', [])] model_type_info += [('ensemble_versions', '', [])] model_type_info += [('confidential', '', True)] model_type_info += [('secret', '', True)] if cmodel['quantitative']: model_validation_info = [('R2', '', cmodel['R2'])] model_validation_info += [('Q2', '', cmodel['Q2'])] model_validation_info += [('SDEC', '', cmodel['SDEC'])] model_validation_info += [('SDEP', '', cmodel['SDEP'])] model_validation_info += [('scoringP', '', cmodel['scoringP'])] model_validation_info += [('scoringR', '', cmodel['scoringR'])] else: model_validation_info = [('MCC_f', '', cmodel['MCC_f'])] model_validation_info += [('MCC', '', cmodel['MCC'])] model_validation_info += [('Sensitivity_f', '', cmodel['Sensitivity_f'])] model_validation_info += [('Sensitivity', '', cmodel['Sensitivity'])] model_validation_info += [('Specificity_f', '', cmodel['Specificity_f'])] model_validation_info += [('Specificity', '', cmodel['Specificity'])] model_validation_info += [('FP_f', '', cmodel['FP_f'])] model_validation_info += [('FP', '', cmodel['FP'])] model_validation_info += [('FN_f', '', cmodel['FN_f'])] model_validation_info += [('FN', '', cmodel['FN'])] model_validation_info += [('TP_f', '', cmodel['TP_f'])] model_validation_info += [('TP', '', cmodel['TP'])] model_validation_info += [('TN_f', '', cmodel['TN_f'])] model_validation_info += [('TN', '', cmodel['TN'])] conveyor = Conveyor() conveyor.addMeta('modelID', cmodel['modelID']) conveyor.addMeta('endpoint', endpoint) conveyor.addMeta('version', version) conveyor.addMeta('quantitative', True) conveyor.addMeta('secret', True) conveyor.addVal(model_building_info, 'model_build_info', 'model building information', 'method', 'single', 'Information about the model building') conveyor.addVal(model_validation_info, 'model_valid_info', 'model validation information', 'method', 'single', 'Information about the model validation') conveyor.addVal(model_type_info, 'model_type_info', 'model type information', 'method', 'single', 'Information about the model type') results_file_name = os.path.join(version_path, 'model-results.pkl') with open(results_file_name, 'wb') as handle: conveyor.save(handle) meta_file_name = os.path.join(version_path, 'model-meta.pkl') with open(meta_file_name, 'wb') as handle: pickle.dump(cmodel['modelID'], handle) pickle.dump(None, handle) pickle.dump(None, handle) pickle.dump(model_building_info, handle) pickle.dump(model_validation_info, handle) pickle.dump(model_type_info, handle) # clone the version in dev shutil.copytree(version_path, os.path.join(base_path, 'dev')) LOG.info(f'Cloning version {version} to version 0 ...') if confidential: LOG.info( f'Import of CONFIDENTIAL model {model} version {version} was successfull' ) return True, 'OK' # get libraries message = f'Endpoint {endpoint} imported OK' for x in os.listdir(base_path): model_path = os.path.join(base_path, x) model_pkl = os.path.join(model_path, 'estimator.pkl') dict_estimator = {} if os.path.isfile(model_pkl): with open(model_pkl, "rb") as input_file: try: dict_estimator = pickle.load(input_file) except Exception as e: return False, f'Incompatible libraries found!. Import aborted with message "{str(e)}"' # check if the libraries used to build this model are similar to current libraries if 'libraries' in dict_estimator: # print (dict_estimator['libraries']) success, results = utils.compatible_modules( dict_estimator['libraries']) if not success: message = f"WARNING: Incompatible libraries detected, {results}. Use at your own risk" return False, message LOG.info( 'Libraries used to generate the imported model are compatible with local libraries' ) LOG.info(message) return True, message