def establish_machine_spec(candidate_spec, user_spec, env_vars): log.debug('Establish machine spec.') # TODO JLM: indicate in the candidate_spec how the machine_spec_file was set. machine_spec = establish_spec(candidate_spec['machine_spec_file']) candidate_spec['machine_spec_setby'] = '__file__' # TODO JLM: User spec is supposed to allow overrides to machine spec. # Apply overrides from user_spec return(machine_spec)
def establish_test(test_spec, candidate_spec, user_spec): log.debug("Establish test spec.") test_spec_file = Path(test_spec) candidate_spec['wrf_hydro_tests']['test_spec_setby'] = 'command line path/file' if not test_spec_file.exists(): candidate_spec['wrf_hydro_tests']['test_spec_setby'] = 'command line key' test_spec_list = \ list(Path(user_spec['wrf_hydro_tests_dir']+'/tests').glob(test_spec+'.py')) if len(test_spec_list) != 1: log.error('The test specification argument does not identify a unique test.') else: test_spec_file = str(test_spec_list[0]) candidate_spec['wrf_hydro_tests']['test_spec'] = test_spec_file return(True)
def establish_machine_spec(candidate_spec, user_spec, env_vars): log.debug('Establish machine spec.') candidate_spec['wrf_hydro_tests']['machine_spec_setby'] = 'wrf_hydro_tests_dir' machine_spec_file = user_spec['wrf_hydro_tests_dir']+'/machine_spec.yaml' candidate_spec['wrf_hydro_tests']['machine_spec'] = machine_spec_file # TODO JLM: indicate in the candidate_spec how the machine_spec_file was set. machine_spec = establish_spec(machine_spec_file) # TODO JLM: User spec is supposed to allow overrides to machine spec. # Apply overrides from user_spec return(machine_spec)
def clone_repo(repo_tag, candidate_spec, user_spec, dir_for_clone): # If ssh_priv_key is NOT supplied, use https, else ssh. if user_spec['github']['ssh_priv_key'] is None: protocol = 'https' url = form_authtoken_url(repo_tag, candidate_spec, user_spec) else: protocol = 'ssh' url = '[email protected]:'+candidate_spec[repo_tag]['fork'] log.debug('Cloning ' + candidate_spec[repo_tag]['fork'] + ' into ' + str(dir_for_clone) + ' using ' + protocol + ' ...') process = subprocess.run(['git', 'clone', url, dir_for_clone]) if process.returncode != 0: return(False) return(True)
def clone_repo(repo_tag, candidate_spec, user_spec, dir_for_clone): # If ssh_priv_key is NOT supplied, use https, else ssh. if user_spec['github']['ssh_priv_key'] is None: protocol = 'https' url = form_authtoken_url(repo_tag, candidate_spec, user_spec) else: protocol = 'ssh' url = '[email protected]:' + candidate_spec[repo_tag]['fork'] log.debug('Cloning ' + candidate_spec[repo_tag]['fork'] + ' into ' + str(dir_for_clone) + ' using ' + protocol + ' ...') process = subprocess.run(['git', 'clone', url, dir_for_clone]) if process.returncode != 0: return (False) return (True)
def establish_user_spec(candidate_spec, env_vars): log.debug('Establish user spec.') user_spec_file = None if ('wrf_hydro_tests' in candidate_spec) and \ ('user_spec' in candidate_spec['wrf_hydro_tests']): user_spec_file = candidate_spec['wrf_hydro_tests']['user_spec'] candidate_spec['wrf_hydro_tests']['user_spec_setby'] = 'candidate spec' if user_spec_file == '' or user_spec_file is None: default_files = establish_default_files() user_spec_file = default_files[0] candidate_spec['wrf_hydro_tests']['user_spec_setby'] = 'env var' candidate_spec['wrf_hydro_tests']['user_spec'] = user_spec_file # TODO JLM: indicate in the candidate_spec how the user_spec_file was set. # TODO JLM: WARN if DNE user_spec = establish_spec(user_spec_file) return(user_spec)
def establish_user_spec(candidate_spec, env_vars): log.debug('Establish user spec.') user_spec_file = None if ('wrf_hydro_tests' in candidate_spec) and \ ('user_spec_file' in candidate_spec): user_spec_file = candidate_spec['user_spec_file'] candidate_spec['user_spec_setby'] = 'candidate spec' if user_spec_file == '' or user_spec_file is None: user_spec_file = os.environ['WRF_HYDRO_TESTS_USER_SPEC'] user_spec = establish_spec(user_spec_file) candidate_spec['user_spec_setby'] = 'env var' candidate_spec['user_spec_file'] = user_spec_file # TODO JLM: indicate in the candidate_spec how the user_spec_file was set. # TODO JLM: WARN if DNE #print('user_spec_file: ', user_spec_file) user_spec = establish_spec(user_spec_file) return(user_spec)
def establish_repo(repo_tag, candidate_spec, user_spec): repo_tag_base = repo_tag.split('_')[0] log.debug('') log.info(repo_tag_base.title() + ' repo') if candidate_spec[repo_tag]['local_path'] is None: # The case when local_path is not set. candidate_spec[repo_tag]['local_path_setby'] = 'fork & commitish' dir_for_clone = Path(candidate_spec['repos_dir'] + '/' + repo_tag_base) #print('dir_for_clone: ',dir_for_clone) candidate_spec[repo_tag]['local_path'] = dir_for_clone if dir_for_clone.exists(): delete_dir_and_contents(dir_for_clone) Path.mkdir(dir_for_clone, parents=True) clone_repo(repo_tag, candidate_spec, user_spec, dir_for_clone) # check out the commitish commitish = candidate_spec[repo_tag]['commitish'] if commitish is None: commitish = 'master' log.debug('Checking out commitish: '+commitish) subprocess.run(['git', 'checkout', commitish], cwd=dir_for_clone) git_log = subprocess.run(['git', 'log', '-n1'], stdout=subprocess.PIPE, cwd=dir_for_clone) log.debug(git_log.stdout.decode('utf-8')) else: candidate_spec[repo_tag]['local_path_setby'] = 'candidate spec'
def establish_repo(repo_tag, candidate_spec, user_spec): repo_tag_base = repo_tag.split('_')[0] log.debug('') log.info(repo_tag_base.title() + ' repo') if candidate_spec[repo_tag]['local_path'] is None: # The case when local_path is not set. candidate_spec[repo_tag]['local_path_setby'] = 'fork & commitish' dir_for_clone = Path(candidate_spec['repos_dir'] + '/' + repo_tag_base) print('dir_for_clone: ', dir_for_clone) candidate_spec[repo_tag]['local_path'] = dir_for_clone if dir_for_clone.exists(): delete_dir_and_contents(dir_for_clone) Path.mkdir(dir_for_clone, parents=True) clone_repo(repo_tag, candidate_spec, user_spec, dir_for_clone) # check out the commitish commitish = candidate_spec[repo_tag]['commitish'] if commitish is None: commitish = 'master' log.debug('Checking out commitish: ' + commitish) subprocess.run(['git', 'checkout', commitish], cwd=dir_for_clone) git_log = subprocess.run(['git', 'log', '-n1'], stdout=subprocess.PIPE, cwd=dir_for_clone) log.debug(git_log.stdout.decode('utf-8')) else: candidate_spec[repo_tag]['local_path_setby'] = 'candidate spec'
def log_spec(spec, name): log.info(horiz_bar) log.info(name+' spec: ') log.debug(pformat(spec)) log.debug('')
# this kind of log. log.setLevel(logging.DEBUG) stdout = logging.StreamHandler() stdout.setLevel(logging.DEBUG) log.addHandler(stdout) # log_file = "take_test.log" # log_file_handler = logging.FileHandler(log_file, mode='w') # log_file_handler.setLevel(logging.DEBUG) # log.addHandler(log_file_handler) horiz_bar = '=================================================================' log.info(horiz_bar) log.info("*** take_test.py: A wrf_hydro candidate takes a test. ***") log.debug('') # ###################################################### # Specification files to dictionaries. log.info(horiz_bar ) log.info( "Setup the specifictions (specs):") env_vars = os.environ.copy() candidate_spec = establish_candidate(candidate_spec_file) # The default candidate path is solved here based on the this_script_path. if not pathlib.PosixPath(candidate_spec['candidate_repo']['local_path']).exists(): candidate_spec['candidate_repo']['local_path'] = \ candidate_spec['candidate_repo']['local_path'].format( **{'this_repo_path': this_repo_path} )
def establish_candidate(candidate_spec_file): log.debug('Establish candidate spec.') candidate_spec = establish_spec(candidate_spec_file) candidate_spec['candidate_spec_file'] = candidate_spec_file return(candidate_spec)
# this kind of log. log.setLevel(logging.DEBUG) stdout = logging.StreamHandler() stdout.setLevel(logging.DEBUG) log.addHandler(stdout) # log_file = "take_test.log" # log_file_handler = logging.FileHandler(log_file, mode='w') # log_file_handler.setLevel(logging.DEBUG) # log.addHandler(log_file_handler) horiz_bar = '=================================================================' log.info(horiz_bar) log.info("*** take_test.py: A wrf_hydro candidate takes a test. ***") log.debug('') # ###################################################### # Specification files to dictionaries. log.info(horiz_bar) log.info("Setup the specifictions (specs):") env_vars = os.environ.copy() candidate_spec = establish_candidate(candidate_spec_file) # The default candidate path is solved here based on the this_script_path. if not pathlib.PosixPath( candidate_spec['candidate_repo']['local_path']).exists(): candidate_spec['candidate_repo']['local_path'] = \ candidate_spec['candidate_repo']['local_path'].format( **{'this_repo_path': this_repo_path}
def establish_candidate(candidate_spec_file): log.debug('Establish candidate spec.') candidate_spec = establish_spec(candidate_spec_file) candidate_spec['wrf_hydro_tests']['candidate_spec'] = candidate_spec_file return(candidate_spec)
# Enforce some namelist options up front # Make sure the lsm and hydro restart output timesteps are the same hydro_rst_dt = self.candidate_sim.hydro_namelist['hydro_nlist'][ 'rst_dt'] self.candidate_sim.namelist_hrldas['noahlsm_offline'][ 'restart_frequency_hours'] = int(hydro_rst_dt / 60) def compile_candidate(self, compiler: str, overwrite: bool = False, compile_options: dict = None): test_compile_candidate(self, compiler, overwrite, compile_options) horiz_bar = '=================================================================' log.info(horiz_bar) log.info('Take the test.') log.debug('') test = FundamentalTest(candidate_sim, reference_sim, '/home/docker/test', overwrite=True) # TODO JLM: wrf_hydro_py, print compile log on compile fail. log.info('C') log.debug('') test.compile_candidate(candidate_spec['compiler']) print(test.candidate_sim.model.compile_log.stdout.decode('utf-8'))
def log_boilerplate(candidate_spec, user_spec, env_vars, horiz_bar, script_path): log.debug( "Date : " + datetime.now().strftime('%Y %h %d %H:%M:%S %Z') ) if not 'USER' in env_vars: user = subprocess.Popen(["whoami"], stdout=subprocess.PIPE).communicate()[0] env_vars['USER'] = user.decode('utf-8').replace("\n",'') log.debug( "User : "******"hostname"], stdout=subprocess.PIPE).communicate()[0] env_vars['HOSTNAME'] = hostname.decode('utf-8').replace("\n",'') log.debug( "Machine : " + env_vars['HOSTNAME'] ) proc = subprocess.run( ['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, cwd=script_path ) the_commit = proc.stdout.decode('utf-8').split()[0] log.debug( "take_test.py location : " + script_path ) # TODO(JLM): should we check for uncommiteed changes in the script_path? Just # ones in the test dir? #is_uncommitted = \ # subprocess.run(['git', 'diff-index', '--quiet', 'HEAD', '--']).returncode #if is_uncommitted != 0: # log.warning( "There are uncommitted changes to the testing repo (" + script_path + ")") log.debug("Domain argument : " + str(candidate_spec['domain'])) log.debug("Config argument : " + str(candidate_spec['config'])) log.debug("Tests run in : " + candidate_spec['test_dir'] ) log.debug("Cloned repos in : " + candidate_spec['repos_dir'] ) log.debug("Candidate spec file : " + candidate_spec['candidate_spec_file'] ) log.debug("Machine spec file : " + candidate_spec['machine_spec_file'] ) log.debug("Machine spec set by : " + candidate_spec['machine_spec_setby'] ) log.debug("User spec file : " + candidate_spec['user_spec_file'] ) log.debug("User spec set by : " + candidate_spec['user_spec_setby'] ) #log.debug( "Test spec file : " + candidate_spec['test_spec_file'] ) #log.debug( "Test spec set by : " + # candidate_spec['test_spec_setby'] ) log.debug("Log file : " + script_path + '/take_test.log') log.debug("Will echo specs to log at end.") return(True)
# This coloring approach may only allow one log. log.setLevel(logging.DEBUG) stdout = logging.StreamHandler() stdout.setLevel(logging.DEBUG) log.addHandler(stdout) log_file = "example.log" log_file_handler = logging.FileHandler(log_file, mode='w') log_file_handler.setLevel(logging.DEBUG) log.addHandler(log_file_handler) horiz_bar = '=================================================================' log.info(horiz_bar) log.info("*** take_test.py: A wrf_hydro candidate takes a test. ***") log.debug('') # ###################################################### # Specification files to dictionaries. log.info(horiz_bar) log.info("Setup the specifictions (specs):") env_vars = os.environ.copy() candidate_spec = establish_candidate(candidate_spec_file) user_spec = establish_user_spec(candidate_spec, env_vars) machine_spec = establish_machine_spec(candidate_spec, user_spec, env_vars) #user_spec, machine_spec = establish_default_files() # Test spec is a bit different. # Writes to candidate_spec['wrf_hydro_tests']['test_spec']
def log_spec(spec, name): log.info(horiz_bar) log.info(name + ' spec: ') log.debug(pformat(spec)) log.debug('')
def log_boilerplate(candidate_spec, user_spec, env_vars, horiz_bar): log.debug("Date : " + datetime.now().strftime('%Y %h %d %H:%M:%S %Z')) if not 'USER' in env_vars: user = subprocess.Popen(["whoami"], stdout=subprocess.PIPE).communicate()[0] env_vars['USER'] = user.decode('utf-8').replace("\n", '') log.debug("User : "******"hostname"], stdout=subprocess.PIPE).communicate()[0] env_vars['HOSTNAME'] = hostname.decode('utf-8').replace("\n", '') log.debug("Machine : " + env_vars['HOSTNAME']) proc = subprocess.run(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE) the_commit = proc.stdout.decode('utf-8').split()[0] log.debug("wrf_hydro_tests commit: " + the_commit) is_uncommitted = \ subprocess.run(['git', 'diff-index', '--quiet', 'HEAD', '--']).returncode if is_uncommitted != 0: log.warning("There are uncommitted changes to wrf_hydro_tests.") log.debug("Candidate spec file : " + candidate_spec['wrf_hydro_tests']['candidate_spec']) log.debug("Machine spec file : " + candidate_spec['wrf_hydro_tests']['machine_spec']) log.debug("Machine spec set by : " + candidate_spec['wrf_hydro_tests']['machine_spec_setby']) log.debug("User spec file : " + candidate_spec['wrf_hydro_tests']['user_spec']) log.debug("User spec set by : " + candidate_spec['wrf_hydro_tests']['user_spec_setby']) log.debug("Test spec file : " + candidate_spec['wrf_hydro_tests']['test_spec']) log.debug("Test spec set by : " + candidate_spec['wrf_hydro_tests']['test_spec_setby']) log.debug("Log file : ") log.debug("Will echo specs to log at end.") return (True)