def buildThread(command, output): print("Thread Start NEW") success, results = context.build_cmd(command, output_format=output) if not success: error_file = os.path.join(tempfile.gettempdir(), 'building_' + command['endpoint']) with open(error_file, 'w') as f: f.write(results) print("Thread End")
def main(): LOG.debug('-------------NEW RUN-------------\n') parser = argparse.ArgumentParser( description= 'Use Flame to either build a model from or apply a model to the input file.' ) parser.add_argument('-f', '--infile', help='Input file.', required=False) parser.add_argument('-e', '--endpoint', help='Endpoint model name.', required=False) parser.add_argument('-v', '--version', help='Endpoint model version.', required=False) parser.add_argument('-a', '--action', help='Manage action.', required=False) parser.add_argument( '-c', '--command', action='store', choices=['predict', 'build', 'manage', 'config'], help='Action type: \'predict\' or \'build\' or \'manage\'', required=True) # parser.add_argument('-log', '--loglevel', # help='Logger level of verbosity',) parser.add_argument('-p', '--path', help='Defines de new path for models repository.', required=False) args = parser.parse_args() # init logger Level and set general config # another way around would be create a handler with the level # and append it to the global instance of logger # if args.loglevel: # numeric_level = getattr(logging, args.loglevel.upper(), None) # if not isinstance(numeric_level, int): # raise ValueError('Invalid log level: {}'.format(args.loglevel)) # logging.basicConfig(level=numeric_level) # make sure flame has been configured before running any command, unless this command if used to # configure flame if args.command != 'config': configuration_warning() if args.command == 'predict': if (args.endpoint is None) or (args.infile is None): print( 'flame predict : endpoint and input file arguments are compulsory' ) return version = utils.intver(args.version) command_predict = { 'endpoint': args.endpoint, 'version': version, 'infile': args.infile } LOG.info(f'Starting prediction with model {args.endpoint}' f' version {version} for file {args.infile}') success, results = context.predict_cmd(command_predict) # print('flame predict : ', success, results) elif args.command == 'build': if (args.endpoint is None) or (args.infile is None): print( 'flame build : endpoint and input file arguments are compulsory' ) return command_build = {'endpoint': args.endpoint, 'infile': args.infile} LOG.info(f'Starting building model {args.endpoint}' f' with file {args.infile}') success, results = context.build_cmd(command_build) # print('flame build : ', success, results) elif args.command == 'manage': manage_cmd(args) elif args.command == 'config': config(args.path) change_config_status()
def buildThread(command, output): print("Thread Start") success, results = context.build_cmd(command, output_format=output) print("Thread End")
def action_refresh(model=None, version=None, GUI=False): ''' Rebuild one or many models making use of existing parameter files and locally stored training series. ''' import flame.context as context from flame.parameters import Parameters # from flame.documentation import Documentation import logging if GUI: token_file = os.path.join(tempfile.gettempdir(), 'refreshing_' + model) # update token file with content 'working' with open(token_file, 'w') as f: f.write('Analyzing and sorting models...') # list endpoints relevant for the arguments if model is not None: model_list = [model] else: model_root = pathlib.Path(utils.model_repository_path()) model_list = [x.stem for x in model_root.iterdir() if x.is_dir()] # list versions relevant for the arguments task_list = [] for imodel in model_list: if version is not None: task_list.append((imodel, version)) else: model_root = pathlib.Path(utils.model_tree_path(imodel)) itask_list = [(imodel, utils.modeldir2ver(x.stem)) for x in model_root.iterdir() if x.is_dir()] task_list += itask_list # use "+=" and not "append" to merge the new list with the old one # analize task_list and add at the end ensemble models # this is needed to have low models refreshed BEFORE refreshing the high models # eliminating the need to refresh them recursively LOG.info("Analyzing and sorting models...") # make sure the lower models are in task_list and, if not, force the inclussion for itask in task_list: param = Parameters() success, results = param.loadYaml(itask[0], itask[1]) if not success: continue if param.getVal('input_type') == 'model_ensemble': ens_nams = param.getVal('ensemble_names') ens_vers = param.getVal('ensemble_versions') for i in range(len(ens_nams)): iver = 0 inam = ens_nams[i] if (i < len(ens_vers)): iver = ens_vers[i] if ((inam, iver)) not in task_list: task_list.append((inam, iver)) # create separate lists for regular and ensemble models # and add ensemble models at the end # this needs to be carried out after the previos step because # some of the lower level models could be an ensemble model # itself mol_list = [] ens_list = [] for itask in task_list: param = Parameters() success, results = param.loadYaml(itask[0], itask[1]) if not success: mol_list.append(itask) continue if param.getVal('input_type') == 'model_ensemble': ens_list.append(itask) else: mol_list.append(itask) task_list = mol_list + ens_list # show all models before stating LOG.info( "Starting model refreshing task for the following models and versions") for itask in task_list: LOG.info(f' model: {itask[0]} version: {itask[1]}') LOG.info("This can take some time, please be patient...") source_dir = os.path.dirname(os.path.abspath(__file__)) children_dir = os.path.join(source_dir, 'children') master_parameters = os.path.join(children_dir, 'parameters.yaml') master_documentation = os.path.join(children_dir, 'documentation.yaml') # now send the build command for each task for itask in task_list: destinat_path = utils.model_path(itask[0], 0) # dev if itask[1] != 0: # move version to /dev for building original_path = utils.model_path(itask[0], itask[1]) # veri security_path = destinat_path + '_security' # dev_sec shutil.move(destinat_path, security_path) # dev --> dev_sec shutil.move(original_path, destinat_path) # veri --> dev LOG.info( f' refreshing model: {itask[0]} version: {itask[1]} ({task_list.index(itask)+1} of {len(task_list)})...' ) if GUI: with open(token_file, 'w') as f: f.write( f'model: {itask[0]} version: {itask[1]} ({task_list.index(itask)+1} of {len(task_list)})' ) # dissable LOG output logging.disable(logging.ERROR) # update parameters dump_parameters = os.path.join(destinat_path, 'parameters_dump.yaml') success, param = action_parameters(itask[0], 0, oformat='bin') if success: param_yaml = param.dumpYAML() with open(dump_parameters, 'w') as f: for line in param_yaml: f.write(line + '\n') else: LOG.info( ' ERROR: unable to merge parameters for model: {itask[0]} version: {itask[1]}' ) dump_parameters = None original_parameters = os.path.join(destinat_path, 'parameters.yaml') shutil.copy(master_parameters, original_parameters) #update documentation dump_documentation = os.path.join(destinat_path, 'documentation_dump.yaml') success, documentation = action_documentation(itask[0], 0, doc_file=None, oformat='bin') original_documentation = os.path.join(destinat_path, 'documentation.yaml') shutil.copy(master_documentation, original_documentation) if success: documentation_yaml = documentation.dumpYAML() with open(dump_documentation, 'w') as f: for line in documentation_yaml: line = line.encode("ascii", "ignore") line = line.decode("ascii", "ignore") f.write(line + '\n') s2, documentation = action_documentation(itask[0], 0, doc_file=None, oformat='bin') s3, r3 = documentation.delta(itask[0], 0, dump_documentation) else: LOG.info( ' ERROR: unable to merge documentation for model: {itask[0]} version: {itask[1]}' ) # rebuild the model command_build = { 'endpoint': itask[0], 'infile': None, 'param_file': dump_parameters, 'incremental': False } success, results = context.build_cmd(command_build) # enable LOG output logging.disable(logging.NOTSET) if itask[1] != 0: shutil.move(destinat_path, original_path) # dev --> veri shutil.move(security_path, destinat_path) # dev_sec --> dev if not success: LOG.error(results) LOG.info("Model refreshing task finished") if GUI: # update token file with status 'ready' with open(token_file, 'w') as f: f.write('ready') return True, 'OK'
def action_refresh(model=None, version=None): ''' Rebuild one or many models making use of existing parameter files and locally stored training series. ''' import flame.context as context from flame.parameters import Parameters import logging # list endpoints relevant for the arguments if model is not None: model_list = [model] else: model_root = pathlib.Path(utils.model_repository_path()) model_list = [x.stem for x in model_root.iterdir() if x.is_dir()] # list versions relevant for the arguments task_list = [] for imodel in model_list: if version is not None: task_list.append((imodel, version)) else: model_root = pathlib.Path(utils.model_tree_path(imodel)) itask_list = [(imodel, utils.modeldir2ver(x.stem)) for x in model_root.iterdir() if x.is_dir()] task_list += itask_list # use "+=" and not "append" to merge the new list with the old one # analize task_list and add at the end ensemble models # this is needed to have low models refreshed BEFORE refreshing the high models # eliminating the need to refresh them recursively LOG.info("Analyzing and sorting models...") # make sure the lower models are in task_list and, if not, force the inclussion for itask in task_list: param = Parameters() success, results = param.loadYaml(itask[0], itask[1]) if not success: continue if param.getVal('input_type') == 'model_ensemble': ens_nams = param.getVal('ensemble_names') ens_vers = param.getVal('ensemble_versions') for i in range(len(ens_nams)): iver = 0 inam = ens_nams[i] if (i < len(ens_vers)): iver = ens_vers[i] if ((inam, iver)) not in task_list: task_list.append((inam, iver)) # create separate lists for regular and ensemble models # and add ensemble models at the end # this needs to be carried out after the previos step because # some of the lower level models could be an ensemble model # itself mol_list = [] ens_list = [] for itask in task_list: param = Parameters() success, results = param.loadYaml(itask[0], itask[1]) if not success: mol_list.append(itask) continue if param.getVal('input_type') == 'model_ensemble': ens_list.append(itask) else: mol_list.append(itask) task_list = mol_list + ens_list # show all models before stating LOG.info( "Starting model refreshing task for the following models and versions") for itask in task_list: LOG.info(f' model: {itask[0]} version: {itask[1]}') LOG.info("This can take some time, please be patient...") # now send the build command for each task for itask in task_list: if itask[1] != 0: # move version to /dev for building original_path = utils.model_path(itask[0], itask[1]) # veri destinat_path = utils.model_path(itask[0], 0) # dev security_path = destinat_path + '_security' # dev_sec shutil.move(destinat_path, security_path) # dev --> dev_sec shutil.move(original_path, destinat_path) # veri --> dev LOG.info( f' refreshing model: {itask[0]} version: {itask[1]} ({task_list.index(itask)+1} of {len(task_list)})...' ) # dissable LOG output logging.disable(logging.ERROR) command_build = { 'endpoint': itask[0], 'infile': None, 'param_file': None, 'incremental': False } success, results = context.build_cmd(command_build) # enable LOG output logging.disable(logging.NOTSET) if itask[1] != 0: shutil.move(destinat_path, original_path) # dev --> veri shutil.move(security_path, destinat_path) # dev_sec --> dev if not success: LOG.error(results) LOG.info("Model refreshing task finished") return True, 'OK'
def main(): LOG.debug('-------------NEW RUN-------------\n') parser = argparse.ArgumentParser( description= f'Flame version {__version__}. Use Flame to build and manage predictive models or to predict using them.' ) parser.add_argument('-f', '--infile', help='Input file.', required=False) parser.add_argument('-e', '--endpoint', help='Endpoint model name.', required=False) parser.add_argument('-s', '--space', help='Chemical space name.', required=False) parser.add_argument('-v', '--version', help='Endpoint model version.', required=False) parser.add_argument('-a', '--action', help='Manage action.', required=False) parser.add_argument('-p', '--parameters', help='File with parameters for the current action.', required=False) parser.add_argument( '-c', '--command', action='store', choices=['predict', 'search', 'build', 'sbuild', 'manage', 'config'], help= 'Action type: \'predict\' or \'search\' or \'build\' \'sbuild\' or \'manage\' or \'config\'', required=True) # parser.add_argument('-log', '--loglevel', # help='Logger level of verbosity',) parser.add_argument( '-d', '--directory', help= 'Defines the root directory for the models and spaces repositories.', required=False) parser.add_argument('-t', '--documentation_file', help='File with manually filled documentation fields.', required=False) parser.add_argument( '-l', '--label', help='Label for facilitating the identification of the prediction.', required=False) parser.add_argument( '-inc', '--incremental', help= 'The input file must be added to the existing training series. Only for "build" command.', action='store_true', required=False) parser.add_argument('--smarts', help='SMARTS string used as input for similarity', required=False) args = parser.parse_args() # init logger Level and set general config # another way around would be create a handler with the level # and append it to the global instance of logger # if args.loglevel: # numeric_level = getattr(logging, args.loglevel.upper(), None) # if not isinstance(numeric_level, int): # raise ValueError('Invalid log level: {}'.format(args.loglevel)) # logging.basicConfig(level=numeric_level) if args.infile is not None: if not os.path.isfile(args.infile): LOG.error(f'Input file {args.infile} not found') return # make sure flame has been configured before running any command, unless this command if used to # configure flame if args.command != 'config': utils.config_test() if args.command == 'predict': if (args.endpoint is None) or (args.infile is None): LOG.error( 'flame predict : endpoint and input file arguments are compulsory' ) return version = utils.intver(args.version) if args.label is None: label = 'temp' else: label = args.label command_predict = { 'endpoint': args.endpoint, 'version': version, 'label': label, 'infile': args.infile } LOG.info( f'Starting prediction with model {args.endpoint}' f' version {version} for file {args.infile}, labelled as {label}') success, results = context.predict_cmd(command_predict) if not success: LOG.error(results) elif args.command == 'search': if (args.space is None) or (args.infile is None and args.smarts is None): LOG.error( 'flame search : space and input file arguments are compulsory') return version = utils.intver(args.version) if args.label is None: label = 'temp' else: label = args.label command_search = { 'space': args.space, 'version': version, 'infile': args.infile, 'smarts': args.smarts, 'runtime_param': args.parameters, 'label': label } LOG.info( f'Starting search on space {args.space}' f' version {version} for file {args.infile}, labelled as {label}') success, results = context.search_cmd(command_search) if not success: LOG.error(results) elif args.command == 'build': if (args.endpoint is None): LOG.error('flame build : endpoint argument is compulsory') return command_build = { 'endpoint': args.endpoint, 'infile': args.infile, 'param_file': args.parameters, 'incremental': args.incremental } LOG.info(f'Starting building model {args.endpoint}' f' with file {args.infile} and parameters {args.parameters}') success, results = context.build_cmd(command_build) if not success: LOG.error(results) elif args.command == 'sbuild': if (args.space is None): LOG.error('flame sbuild : space argument is compulsory') return command_build = { 'space': args.space, 'infile': args.infile, 'param_file': args.parameters } LOG.info(f'Starting building model {args.space}' f' with file {args.infile} and parameters {args.parameters}') success, results = context.sbuild_cmd(command_build) if not success: LOG.error(results) elif args.command == 'manage': success, results = context.manage_cmd(args) if not success: LOG.error(results) elif args.command == 'config': success, results = config.configure(args.directory, (args.action == 'silent')) if not success: LOG.error(f'{results}, configuration unchanged')