def establish_machine_spec(candidate_spec, user_spec, env_vars):
log.debug('Establish machine spec.')
# TODO JLM: indicate in the candidate_spec how the machine_spec_file was set.
machine_spec = establish_spec(candidate_spec['machine_spec_file'])
candidate_spec['machine_spec_setby'] = '__file__'
# TODO JLM: User spec is supposed to allow overrides to machine spec.
# Apply overrides from user_spec
return(machine_spec)
def establish_test(test_spec, candidate_spec, user_spec):
log.debug("Establish test spec.")
test_spec_file = Path(test_spec)
candidate_spec['wrf_hydro_tests']['test_spec_setby'] = 'command line path/file'
if not test_spec_file.exists():
candidate_spec['wrf_hydro_tests']['test_spec_setby'] = 'command line key'
test_spec_list = \
list(Path(user_spec['wrf_hydro_tests_dir']+'/tests').glob(test_spec+'.py'))
if len(test_spec_list) != 1:
log.error('The test specification argument does not identify a unique test.')
else:
test_spec_file = str(test_spec_list[0])
candidate_spec['wrf_hydro_tests']['test_spec'] = test_spec_file
return(True)
def establish_machine_spec(candidate_spec, user_spec, env_vars):
log.debug('Establish machine spec.')
candidate_spec['wrf_hydro_tests']['machine_spec_setby'] = 'wrf_hydro_tests_dir'
machine_spec_file = user_spec['wrf_hydro_tests_dir']+'/machine_spec.yaml'
candidate_spec['wrf_hydro_tests']['machine_spec'] = machine_spec_file
# TODO JLM: indicate in the candidate_spec how the machine_spec_file was set.
machine_spec = establish_spec(machine_spec_file)
# TODO JLM: User spec is supposed to allow overrides to machine spec.
# Apply overrides from user_spec
return(machine_spec)
def clone_repo(repo_tag, candidate_spec, user_spec, dir_for_clone):
# If ssh_priv_key is NOT supplied, use https, else ssh.
if user_spec['github']['ssh_priv_key'] is None:
protocol = 'https'
url = form_authtoken_url(repo_tag, candidate_spec, user_spec)
else:
protocol = 'ssh'
url = '[email protected]:'+candidate_spec[repo_tag]['fork']
log.debug('Cloning ' + candidate_spec[repo_tag]['fork'] +
' into ' + str(dir_for_clone) + ' using ' + protocol + ' ...')
process = subprocess.run(['git', 'clone', url, dir_for_clone])
if process.returncode != 0:
return(False)
return(True)
def clone_repo(repo_tag, candidate_spec, user_spec, dir_for_clone):
# If ssh_priv_key is NOT supplied, use https, else ssh.
if user_spec['github']['ssh_priv_key'] is None:
protocol = 'https'
url = form_authtoken_url(repo_tag, candidate_spec, user_spec)
else:
protocol = 'ssh'
url = '[email protected]:' + candidate_spec[repo_tag]['fork']
log.debug('Cloning ' + candidate_spec[repo_tag]['fork'] + ' into ' +
str(dir_for_clone) + ' using ' + protocol + ' ...')
process = subprocess.run(['git', 'clone', url, dir_for_clone])
if process.returncode != 0:
return (False)
return (True)
def establish_user_spec(candidate_spec, env_vars):
log.debug('Establish user spec.')
user_spec_file = None
if ('wrf_hydro_tests' in candidate_spec) and \
('user_spec' in candidate_spec['wrf_hydro_tests']):
user_spec_file = candidate_spec['wrf_hydro_tests']['user_spec']
candidate_spec['wrf_hydro_tests']['user_spec_setby'] = 'candidate spec'
if user_spec_file == '' or user_spec_file is None:
default_files = establish_default_files()
user_spec_file = default_files[0]
candidate_spec['wrf_hydro_tests']['user_spec_setby'] = 'env var'
candidate_spec['wrf_hydro_tests']['user_spec'] = user_spec_file
# TODO JLM: indicate in the candidate_spec how the user_spec_file was set.
# TODO JLM: WARN if DNE
user_spec = establish_spec(user_spec_file)
return(user_spec)
def establish_user_spec(candidate_spec, env_vars):
log.debug('Establish user spec.')
user_spec_file = None
if ('wrf_hydro_tests' in candidate_spec) and \
('user_spec_file' in candidate_spec):
user_spec_file = candidate_spec['user_spec_file']
candidate_spec['user_spec_setby'] = 'candidate spec'
if user_spec_file == '' or user_spec_file is None:
user_spec_file = os.environ['WRF_HYDRO_TESTS_USER_SPEC']
user_spec = establish_spec(user_spec_file)
candidate_spec['user_spec_setby'] = 'env var'
candidate_spec['user_spec_file'] = user_spec_file
# TODO JLM: indicate in the candidate_spec how the user_spec_file was set.
# TODO JLM: WARN if DNE
#print('user_spec_file: ', user_spec_file)
user_spec = establish_spec(user_spec_file)
return(user_spec)
def establish_repo(repo_tag, candidate_spec, user_spec):
repo_tag_base = repo_tag.split('_')[0]
log.debug('')
log.info(repo_tag_base.title() + ' repo')
if candidate_spec[repo_tag]['local_path'] is None:
# The case when local_path is not set.
candidate_spec[repo_tag]['local_path_setby'] = 'fork & commitish'
dir_for_clone = Path(candidate_spec['repos_dir'] + '/' + repo_tag_base)
#print('dir_for_clone: ',dir_for_clone)
candidate_spec[repo_tag]['local_path'] = dir_for_clone
if dir_for_clone.exists():
delete_dir_and_contents(dir_for_clone)
Path.mkdir(dir_for_clone, parents=True)
clone_repo(repo_tag, candidate_spec, user_spec, dir_for_clone)
# check out the commitish
commitish = candidate_spec[repo_tag]['commitish']
if commitish is None:
commitish = 'master'
log.debug('Checking out commitish: '+commitish)
subprocess.run(['git', 'checkout', commitish], cwd=dir_for_clone)
git_log = subprocess.run(['git', 'log', '-n1'], stdout=subprocess.PIPE, cwd=dir_for_clone)
log.debug(git_log.stdout.decode('utf-8'))
else:
candidate_spec[repo_tag]['local_path_setby'] = 'candidate spec'
def establish_repo(repo_tag, candidate_spec, user_spec):
repo_tag_base = repo_tag.split('_')[0]
log.debug('')
log.info(repo_tag_base.title() + ' repo')
if candidate_spec[repo_tag]['local_path'] is None:
# The case when local_path is not set.
candidate_spec[repo_tag]['local_path_setby'] = 'fork & commitish'
dir_for_clone = Path(candidate_spec['repos_dir'] + '/' + repo_tag_base)
print('dir_for_clone: ', dir_for_clone)
candidate_spec[repo_tag]['local_path'] = dir_for_clone
if dir_for_clone.exists():
delete_dir_and_contents(dir_for_clone)
Path.mkdir(dir_for_clone, parents=True)
clone_repo(repo_tag, candidate_spec, user_spec, dir_for_clone)
# check out the commitish
commitish = candidate_spec[repo_tag]['commitish']
if commitish is None:
commitish = 'master'
log.debug('Checking out commitish: ' + commitish)
subprocess.run(['git', 'checkout', commitish], cwd=dir_for_clone)
git_log = subprocess.run(['git', 'log', '-n1'],
stdout=subprocess.PIPE,
cwd=dir_for_clone)
log.debug(git_log.stdout.decode('utf-8'))
else:
candidate_spec[repo_tag]['local_path_setby'] = 'candidate spec'
def log_spec(spec, name):
log.info(horiz_bar)
log.info(name+' spec: ')
log.debug(pformat(spec))
log.debug('')
# this kind of log.
log.setLevel(logging.DEBUG)
stdout = logging.StreamHandler()
stdout.setLevel(logging.DEBUG)
log.addHandler(stdout)
# log_file = "take_test.log"
# log_file_handler = logging.FileHandler(log_file, mode='w')
# log_file_handler.setLevel(logging.DEBUG)
# log.addHandler(log_file_handler)
horiz_bar = '================================================================='
log.info(horiz_bar)
log.info("*** take_test.py: A wrf_hydro candidate takes a test. ***")
log.debug('')
# ######################################################
# Specification files to dictionaries.
log.info(horiz_bar )
log.info( "Setup the specifictions (specs):")
env_vars = os.environ.copy()
candidate_spec = establish_candidate(candidate_spec_file)
# The default candidate path is solved here based on the this_script_path.
if not pathlib.PosixPath(candidate_spec['candidate_repo']['local_path']).exists():
candidate_spec['candidate_repo']['local_path'] = \
candidate_spec['candidate_repo']['local_path'].format(
**{'this_repo_path': this_repo_path}
)
def establish_candidate(candidate_spec_file):
log.debug('Establish candidate spec.')
candidate_spec = establish_spec(candidate_spec_file)
candidate_spec['candidate_spec_file'] = candidate_spec_file
return(candidate_spec)
# this kind of log.
log.setLevel(logging.DEBUG)
stdout = logging.StreamHandler()
stdout.setLevel(logging.DEBUG)
log.addHandler(stdout)
# log_file = "take_test.log"
# log_file_handler = logging.FileHandler(log_file, mode='w')
# log_file_handler.setLevel(logging.DEBUG)
# log.addHandler(log_file_handler)
horiz_bar = '================================================================='
log.info(horiz_bar)
log.info("*** take_test.py: A wrf_hydro candidate takes a test. ***")
log.debug('')
# ######################################################
# Specification files to dictionaries.
log.info(horiz_bar)
log.info("Setup the specifictions (specs):")
env_vars = os.environ.copy()
candidate_spec = establish_candidate(candidate_spec_file)
# The default candidate path is solved here based on the this_script_path.
if not pathlib.PosixPath(
candidate_spec['candidate_repo']['local_path']).exists():
candidate_spec['candidate_repo']['local_path'] = \
candidate_spec['candidate_repo']['local_path'].format(
**{'this_repo_path': this_repo_path}
def establish_candidate(candidate_spec_file):
log.debug('Establish candidate spec.')
candidate_spec = establish_spec(candidate_spec_file)
candidate_spec['wrf_hydro_tests']['candidate_spec'] = candidate_spec_file
return(candidate_spec)
# Enforce some namelist options up front
# Make sure the lsm and hydro restart output timesteps are the same
hydro_rst_dt = self.candidate_sim.hydro_namelist['hydro_nlist'][
'rst_dt']
self.candidate_sim.namelist_hrldas['noahlsm_offline'][
'restart_frequency_hours'] = int(hydro_rst_dt / 60)
def compile_candidate(self,
compiler: str,
overwrite: bool = False,
compile_options: dict = None):
test_compile_candidate(self, compiler, overwrite, compile_options)
horiz_bar = '================================================================='
log.info(horiz_bar)
log.info('Take the test.')
log.debug('')
test = FundamentalTest(candidate_sim,
reference_sim,
'/home/docker/test',
overwrite=True)
# TODO JLM: wrf_hydro_py, print compile log on compile fail.
log.info('C')
log.debug('')
test.compile_candidate(candidate_spec['compiler'])
print(test.candidate_sim.model.compile_log.stdout.decode('utf-8'))
def log_boilerplate(candidate_spec, user_spec, env_vars, horiz_bar, script_path):
log.debug( "Date : " + datetime.now().strftime('%Y %h %d %H:%M:%S %Z') )
if not 'USER' in env_vars:
user = subprocess.Popen(["whoami"], stdout=subprocess.PIPE).communicate()[0]
env_vars['USER'] = user.decode('utf-8').replace("\n",'')
log.debug( "User : "******"hostname"], stdout=subprocess.PIPE).communicate()[0]
env_vars['HOSTNAME'] = hostname.decode('utf-8').replace("\n",'')
log.debug( "Machine : " + env_vars['HOSTNAME'] )
proc = subprocess.run(
['git', 'rev-parse', 'HEAD'],
stdout=subprocess.PIPE,
cwd=script_path
)
the_commit = proc.stdout.decode('utf-8').split()[0]
log.debug( "take_test.py location : " + script_path )
# TODO(JLM): should we check for uncommiteed changes in the script_path? Just
# ones in the test dir?
#is_uncommitted = \
# subprocess.run(['git', 'diff-index', '--quiet', 'HEAD', '--']).returncode
#if is_uncommitted != 0:
# log.warning( "There are uncommitted changes to the testing repo (" + script_path + ")")
log.debug("Domain argument : " + str(candidate_spec['domain']))
log.debug("Config argument : " + str(candidate_spec['config']))
log.debug("Tests run in : " + candidate_spec['test_dir'] )
log.debug("Cloned repos in : " + candidate_spec['repos_dir'] )
log.debug("Candidate spec file : " + candidate_spec['candidate_spec_file'] )
log.debug("Machine spec file : " + candidate_spec['machine_spec_file'] )
log.debug("Machine spec set by : " +
candidate_spec['machine_spec_setby'] )
log.debug("User spec file : " + candidate_spec['user_spec_file'] )
log.debug("User spec set by : " +
candidate_spec['user_spec_setby'] )
#log.debug( "Test spec file : " + candidate_spec['test_spec_file'] )
#log.debug( "Test spec set by : " +
# candidate_spec['test_spec_setby'] )
log.debug("Log file : " + script_path + '/take_test.log')
log.debug("Will echo specs to log at end.")
return(True)
# This coloring approach may only allow one log.
log.setLevel(logging.DEBUG)
stdout = logging.StreamHandler()
stdout.setLevel(logging.DEBUG)
log.addHandler(stdout)
log_file = "example.log"
log_file_handler = logging.FileHandler(log_file, mode='w')
log_file_handler.setLevel(logging.DEBUG)
log.addHandler(log_file_handler)
horiz_bar = '================================================================='
log.info(horiz_bar)
log.info("*** take_test.py: A wrf_hydro candidate takes a test. ***")
log.debug('')
# ######################################################
# Specification files to dictionaries.
log.info(horiz_bar)
log.info("Setup the specifictions (specs):")
env_vars = os.environ.copy()
candidate_spec = establish_candidate(candidate_spec_file)
user_spec = establish_user_spec(candidate_spec, env_vars)
machine_spec = establish_machine_spec(candidate_spec, user_spec, env_vars)
#user_spec, machine_spec = establish_default_files()
# Test spec is a bit different.
# Writes to candidate_spec['wrf_hydro_tests']['test_spec']
def log_spec(spec, name):
log.info(horiz_bar)
log.info(name + ' spec: ')
log.debug(pformat(spec))
log.debug('')
def log_boilerplate(candidate_spec, user_spec, env_vars, horiz_bar):
log.debug("Date : " +
datetime.now().strftime('%Y %h %d %H:%M:%S %Z'))
if not 'USER' in env_vars:
user = subprocess.Popen(["whoami"],
stdout=subprocess.PIPE).communicate()[0]
env_vars['USER'] = user.decode('utf-8').replace("\n", '')
log.debug("User : "******"hostname"],
stdout=subprocess.PIPE).communicate()[0]
env_vars['HOSTNAME'] = hostname.decode('utf-8').replace("\n", '')
log.debug("Machine : " + env_vars['HOSTNAME'])
proc = subprocess.run(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE)
the_commit = proc.stdout.decode('utf-8').split()[0]
log.debug("wrf_hydro_tests commit: " + the_commit)
is_uncommitted = \
subprocess.run(['git', 'diff-index', '--quiet', 'HEAD', '--']).returncode
if is_uncommitted != 0:
log.warning("There are uncommitted changes to wrf_hydro_tests.")
log.debug("Candidate spec file : " +
candidate_spec['wrf_hydro_tests']['candidate_spec'])
log.debug("Machine spec file : " +
candidate_spec['wrf_hydro_tests']['machine_spec'])
log.debug("Machine spec set by : " +
candidate_spec['wrf_hydro_tests']['machine_spec_setby'])
log.debug("User spec file : " +
candidate_spec['wrf_hydro_tests']['user_spec'])
log.debug("User spec set by : " +
candidate_spec['wrf_hydro_tests']['user_spec_setby'])
log.debug("Test spec file : " +
candidate_spec['wrf_hydro_tests']['test_spec'])
log.debug("Test spec set by : " +
candidate_spec['wrf_hydro_tests']['test_spec_setby'])
log.debug("Log file : ")
log.debug("Will echo specs to log at end.")
return (True)