def main(parameters=[]): parser = CoreParser() if not parameters: parameters = get_parameters(parser) # each case id (aka, variable) has a set of parameters specified. # print out the parameters for each variable for trouble shootting #for p in parameters: # attrs = vars(p) # print (', '.join("%s: %s" % item for item in attrs.items())) if not os.path.exists(parameters[0].results_dir): os.makedirs(parameters[0].results_dir, 0o755) if not parameters[0].no_viewer: # Only save provenance for full runs. save_provenance(parameters[0].results_dir, parser) if container.is_container(): print('Running e3sm_diags in a container.') # Modify the parmeters so that it runs in # the container as if it usually runs. for p in parameters: container.containerize_parameter(p) if parameters[0].multiprocessing: parameters = cdp.cdp_run.multiprocess(run_diag, parameters, context='fork') elif parameters[0].distributed: parameters = cdp.cdp_run.distribute(run_diag, parameters) else: parameters = cdp.cdp_run.serial(run_diag, parameters) parameters = _collapse_results(parameters) if container.is_container(): for p in parameters: container.decontainerize_parameter(p) if not parameters: print( 'There was not a single valid diagnostics run, no viewer created.') else: # If you get `AttributeError: 'NoneType' object has no attribute 'no_viewer'` on this line # then `run_diag` likely returns `None`. if parameters[0].no_viewer: print( 'Viewer not created because the no_viewer parameter is True.') else: path = os.path.join(parameters[0].results_dir, 'viewer') if not os.path.exists(path): os.makedirs(path) index_path = create_viewer(path, parameters) print('Viewer HTML generated at {}'.format(index_path))
def create_metadata(parameter): """ From a set of parameters, extract the metadata. """ metadata = collections.OrderedDict() msg = 'Use this command to recreate this image:' metadata[msg] = '' cmd = 'e3sm_diags --no_viewer ' from acme_diags.acme_parser import ACMEParser parser = ACMEParser() args = parser.view_args() supported_cmd_args = list(args.__dict__.keys()) if 'other_parameters' in supported_cmd_args: supported_cmd_args.remove('other_parameters') if 'parameters' in supported_cmd_args: supported_cmd_args.remove('parameters') if container.is_container(): container.decontainerize_parameter(parameter) for param_name in parameter.__dict__: param = parameter.__dict__[param_name] # we don't want to include blank values if not param: continue if param_name in supported_cmd_args: if isinstance(param, list) or isinstance(param, tuple): # ex: --diff_levels -7, -6, -5, -4 cmd += "--{} ".format(param_name) for p in param: if isinstance(p, str) and p.isdigit(): cmd += " {} ".format(str(p)) else: cmd += " '{}' ".format(str(p)) elif isinstance(param, bool): # ex: --multiprocessing # note there's no value after the parameter, it's just a flag if param: # command is True, so add --command to set it to True cmd += "--{} ".format(param_name) elif isinstance(param, str) and param.isdigit(): cmd += "--{} {} ".format(param_name, param) else: cmd += "--{} '{}' ".format(param_name, param) metadata[msg] = cmd return metadata
def get_output_dir(set_num, parameter, ignore_container=False): """ Get the directory of where to save the outputs for a run. If ignore_container is True and the software is being ran in a container, get the path that the user passed in. """ results_dir = parameter.results_dir if ignore_container and container.is_container(): results_dir = parameter.orig_results_dir pth = os.path.join(results_dir, '{}'.format(set_num), parameter.case_id) if not os.path.exists(pth): # When running diags in parallel, sometimes another process will create the dir. try: os.makedirs(pth, 0o775) except OSError as e: if e.errno != os.errno.EEXIST: raise return pth
def _save_parameter_files(results_dir, parser): """ Save the command line arguments used, and any py or cfg files. """ cmd_used = " ".join(sys.argv) fnm = os.path.join(results_dir, "cmd_used.txt") with open(fnm, "w") as f: if container.is_container(): f.write("# e3sm_diags was ran in a container.\n") f.write(cmd_used) print("Saved command used to: {}".format(fnm)) args = parser.view_args() if hasattr(args, "parameters") and args.parameters: fnm = args.parameters if not os.path.isfile(fnm): print("File does not exist: {}".format(fnm)) else: with open(fnm, "r") as f: contents = "".join(f.readlines()) # Remove any path, just keep the filename. new_fnm = fnm.split("/")[-1] new_fnm = os.path.join(results_dir, new_fnm) with open(new_fnm, "w") as f: f.write(contents) print("Saved py file to: {}".format(new_fnm)) if hasattr(args, "other_parameters") and args.other_parameters: fnm = args.other_parameters[0] if not os.path.isfile(fnm): print("File does not exist: {}".format(fnm)) else: with open(fnm, "r") as f: contents = "".join(f.readlines()) # Remove any path, just keep the filename. new_fnm = fnm.split("/")[-1] new_fnm = os.path.join(results_dir, new_fnm) with open(new_fnm, "w") as f: f.write(contents) print("Saved cfg file to: {}".format(new_fnm))
def main(): parser = ACMEParser() parameters = get_parameters(parser) if not os.path.exists(parameters[0].results_dir): os.makedirs(parameters[0].results_dir, 0o775) if not parameters[0].no_viewer: # Only save provenance for full runs. save_provenance(parameters[0].results_dir, parser) if container.is_container(): print('Running e3sm_diags in a container.') # Parameters will decontainerized by the viewer later. # That's to make sure the command shown in the viewer works with or without the viewer. for p in parameters: container.containerize_parameter(p) if parameters[0].multiprocessing: parameters = cdp.cdp_run.multiprocess(run_diag, parameters) elif parameters[0].distributed: parameters = cdp.cdp_run.distribute(run_diag, parameters) else: parameters = cdp.cdp_run.serial(run_diag, parameters) parameters = _collapse_results(parameters) if not parameters: print('There was not a single valid diagnostics run, no viewer created.') else: if parameters[0].no_viewer: print('Viewer not created because the no_viewer parameter is True.') else: pth = os.path.join(parameters[0].results_dir, 'viewer') if not os.path.exists(pth): os.makedirs(pth) create_viewer(pth, parameters, parameters[0].output_format[0]) path = os.path.join(parameters[0].results_dir, 'viewer') print('Viewer HTML generated at {}/index.html'.format(path))
def main(parameters=[]): parser = CoreParser() if not parameters: parameters = get_parameters(parser) expected_parameters = create_parameter_dict(parameters) # each case id (aka, variable) has a set of parameters specified. # print out the parameters for each variable for trouble shootting # for p in parameters: # attrs = vars(p) # print (', '.join("%s: %s" % item for item in attrs.items())) if not os.path.exists(parameters[0].results_dir): os.makedirs(parameters[0].results_dir, 0o755) if not parameters[0].no_viewer: # Only save provenance for full runs. save_provenance(parameters[0].results_dir, parser) if container.is_container(): print("Running e3sm_diags in a container.") # Modify the parmeters so that it runs in # the container as if it usually runs. for p in parameters: container.containerize_parameter(p) if parameters[0].multiprocessing: parameters = cdp.cdp_run.multiprocess(run_diag, parameters, context="fork") elif parameters[0].distributed: parameters = cdp.cdp_run.distribute(run_diag, parameters) else: parameters = cdp.cdp_run.serial(run_diag, parameters) parameters = _collapse_results(parameters) if container.is_container(): for p in parameters: container.decontainerize_parameter(p) if not parameters: print( "There was not a single valid diagnostics run, no viewer created.") else: # If you get `AttributeError: 'NoneType' object has no attribute 'no_viewer'` on this line # then `run_diag` likely returns `None`. if parameters[0].no_viewer: print( "Viewer not created because the no_viewer parameter is True.") else: path = os.path.join(parameters[0].results_dir, "viewer") if not os.path.exists(path): os.makedirs(path) index_path = create_viewer(path, parameters) print("Viewer HTML generated at {}".format(index_path)) actual_parameters = create_parameter_dict(parameters) if parameters[0].fail_on_incomplete and (actual_parameters != expected_parameters): d: Dict[type, Tuple[int, int]] = dict() # Loop through all expected parameter types. for t in expected_parameters.keys(): d[t] = (actual_parameters[t], expected_parameters[t]) message = "Not all parameters completed successfully. Check output above for errors/exceptions. The following dictionary maps parameter types to their actual and expected numbers: {}".format( d) raise Exception(message)