def make_yaml_files(srcdir, tgtdir):
if not os.path.exists(tgtdir):
logger.info(f'{tgtdir}: make directory')
os.makedirs(tgtdir)
logger.info(f'{tgtdir}: send yaml files to here')
logger.info(f'{srcdir}: get yaml files from here')
for srcfile in glob.glob(f'{srcdir}/*.yaml'):
srcbase = os.path.basename(srcfile)
if srcbase.startswith('resources'): continue
if srcbase.startswith('settings'): continue
tgtfile = os.path.join(tgtdir, srcbase)
logger.info(f'{srcbase}: copy yaml file')
shutil.copyfile(srcfile, tgtfile)
# Deal with the settings:
doc = from_file(f"{srcdir}/settings.yaml")
settings_yaml = os.path.join(tgtdir, 'settings.yaml')
logger.info(f'{settings_yaml}: generate file')
with open(f'{tgtdir}/settings.yaml', 'wt') as fd:
fd.write('# This file is automatically generated from:\n')
fd.write(f'# {srcdir}/settings.yaml')
fd.write('# Changes to this file may be overwritten.\n\n')
fd.write(to_yaml(doc))
# Now the resources:
resource_basename = doc.settings.resource_file
resource_srcfile = os.path.join(srcdir, resource_basename)
resource_tgtfile = os.path.join(tgtdir, 'resources.yaml')
logger.info(f'{resource_srcfile}: use this resource yaml file')
shutil.copyfile(resource_srcfile, resource_tgtfile)
logger.info(f'{tgtdir}: yaml files created here')
def sandbox_platforms(userfile, platdir):
available = {}
plat = from_file(userfile, f'{platdir}/_common.yaml',
f'{platdir}/_sandbox.yaml')
# plat=from_file(f'{platdir}/_common.yaml',f'{platdir}/_sandbox.yaml')
available[plat.platform.name] = plat
return available
def find_available_platforms(platdir):
matches = {}
filenames = {}
can_skip = set()
for matching_file in glob.glob(f'{platdir}/[a-zA-Z]*.yaml'):
logger.info(f'{matching_file}: check this platform...')
plat = from_file('../user.yaml', f'{platdir}/_common.yaml',
matching_file)
if not 'platform' in plat or \
not 'detect' in plat.platform or \
not 'name' in plat.platform:
logger.warning(f'{matching_file}: does not contain a '
'"platform" map with "detect" and "name"')
continue
name = plat.platform.name
if plat.platform.detect:
logger.info(f'{matching_file}: platform {name} matches')
if name in filenames:
logger.error(
f'{filenames[name]}: same platform name "{name}" as {matching_file}'
)
exit(1)
matches[name] = plat
if plat.platform.get('skip_if_others_present', False):
can_skip.add(name)
available = copy(matches)
for k in can_skip:
if k in available: del available[k]
if available:
return available
else:
# All platforms "can be skipped" so skip none:
return matches
def make_config_files_in_expdir(doc, expdir):
for key in doc.keys():
if not key.startswith('config_'): continue
value = doc[key]
if not isinstance(value, collections.Mapping): continue
if not 'filename' in value or not 'content' in value:
logger.warning(
f'{key}: config files require "filename" and "content" entries.'
)
if value.get('disable', False): continue #
filename = os.path.join(expdir, str(value.filename))
logger.debug(f'{filename}: expand')
content = str(value.content)
logger.info(f'{filename}: write')
with open(filename, 'wt') as fd:
fd.write(content)
# value = doc['partition_common']['resources']
# test1 = from_file('tests/test_data/yaml-io/original.yaml')
doc.partition_common.Evaluate = True
crow.config.evaluate_immediates(doc.partition_common)
value = doc['partition_common']['resources']
# value = { 'resources_sum': doc['partition_common']['resources']}
filename = os.path.join(expdir, 'resources_sum.yaml')
logger.debug(f'{filename}: expand')
content = to_yaml(value)
with open(filename, 'wt') as fd:
fd.write(content)
resources_sum = from_file(filename)
value = {'resources_sum': resources_sum}
logger.debug(f'{filename}: expand')
content = to_yaml(value)
with open(filename, 'wt') as fd:
fd.write(content)
def make_yaml_files(srcdir, tgtdir):
if not os.path.exists(tgtdir):
logger.info(f'{tgtdir}: make directory')
os.makedirs(tgtdir)
logger.info(f'{tgtdir}: send yaml files to here')
logger.info(f'{srcdir}: get yaml files from here')
for srcfile in glob.glob(f'{srcdir}/*.yaml'):
srcbase = os.path.basename(srcfile)
if srcbase.startswith('resources'): continue
if srcbase.startswith('config'): continue
tgtfile = os.path.join(tgtdir, srcbase)
logger.info(f'{srcbase}: copy yaml file')
shutil.copyfile(srcfile, tgtfile)
del srcbase, tgtfile
readme = [os.path.join(srcdir, 'settings_validator.yaml')]
# Deal with the config files:
for srcfile in glob.glob(f'{srcdir}/config*.yaml'):
logger.info(f'{srcfile}: read file')
doc = from_file(srcfile)
tgtfile = os.path.join(tgtdir, os.path.basename(srcfile))
yaml = to_yaml(doc)
anchor = os.path.basename(srcfile)[:-5]
anchored = re.sub(r'\A([a-zA-Z][a-zA-Z0-9_]*):', r'\1: &' + anchor,
yaml)
logger.info(f'{tgtfile}: generate file')
with open(tgtfile, 'wt') as fd:
fd.write('# This file is automatically generated from:\n')
fd.write(f'# {srcfile}')
fd.write('# Changes to this file may be overwritten.\n\n')
fd.write(yaml)
readme.insert(0, tgtfile)
del doc, tgtfile
# Read the settings file
readme.append('settings.yaml')
logger.info(f'Read files: {", ".join(readme)}')
doc = from_file(*readme)
# Now the resources:
resource_basename = doc.settings.resource_file
resource_srcfile = os.path.join(srcdir, resource_basename)
resource_tgtfile = os.path.join(tgtdir, 'resources.yaml')
logger.info(f'{resource_srcfile}: use this resource yaml file')
shutil.copyfile(resource_srcfile, resource_tgtfile)
logger.info(f'{tgtdir}: yaml files created here')
def find_available_platforms(platdir):
available = {}
for matching_file in glob.glob(f'{platdir}/*.yaml'):
logger.info(f'{matching_file}: check this platform...')
plat = from_file('user.yaml', matching_file)
if not 'platform' in plat or \
not 'detect' in plat.platform or \
not 'name' in plat.platform:
logger.warning(f'{matching_file}: does not contain a '
'"platform" map with "detect" and "name"')
if plat.platform.detect:
logger.info(
f'{matching_file}: platform {plat.platform.name} matches')
available[plat.platform.name] = plat
return available
#! /usr/bin/env python3
f'This script requires Python 3.6 or newer.'
import os
from crow.metascheduler import to_ecflow
from crow.config import from_file, Suite
conf = from_file('ecftest.yaml')
suite = Suite(conf.suite)
suite_defs, ecf_files = to_ecflow(suite)
for defname in suite_defs:
#print(f'=== contents of suite def {defname}\n{suite_defs[defname]}')
filename = defname
print(filename)
dirname = os.path.dirname(filename)
if dirname and not os.path.exists(dirname):
os.makedirs(os.path.dirname(filename))
with open(filename, 'wt') as fd:
fd.write(suite_defs[defname]['def'])
for setname in ecf_files:
print(f'ecf file set {setname}:\n')
for filename in ecf_files[setname]:
print(f' file {filename}')
dirname = os.path.dirname(filename)
if dirname and not os.path.exists(dirname):
os.makedirs(os.path.dirname(filename))
with open(filename + ".ecf", 'wt') as fd:
fd.write(ecf_files[setname][filename])
def make_yaml_files_in_expdir(srcdir, case_name, experiment_name, platdoc,
force):
logger.info(f'{srcdir}: get yaml files from here')
logger.info(f'{case_name}: use this case')
case_file = find_case_yaml_file_for(case_name)
platform_yaml = to_yaml(platdoc)
names = {'names': {'experiment': experiment_name, 'case': case_name}}
names_yaml = to_yaml(names)
# Get the configuration from the source directory:
with io.StringIO() as fd:
fd.write(platform_yaml)
fd.write('\n\n')
fd.write(names_yaml)
fd.write('\n\n')
crow.config.follow_main(fd, srcdir)
fd.write('\n\n')
with open(case_file, 'rt') as cfd:
fd.write(cfd.read())
config_contents = fd.getvalue()
config = crow.config.from_string(config_contents)
workflow_file = os.path.join(srcdir, config.places.workflow_file)
tgtdir = config.places.EXPDIR
rotdir = config.places.ROTDIR
redo = False
logger.info(f'{rotdir}: COM files will be here')
logger.info(f'{tgtdir}: send yaml files to here')
gud = True
if os.path.exists(tgtdir):
gud = False
logger.warning(f'{tgtdir}: already exists!')
if os.path.exists(rotdir):
gud = False
logger.warning(f'{rotdir}: already exists!')
if not gud and not force:
logger.error('Target directories already exist.')
logger.error('I will not start a workflow unless you do -f.')
logger.critical(
'Use -f to force this workflow to start, but we aware that config, initial COM, and yaml files will be overwritten. Other files will remain unmodified.'
)
exit(1)
elif not gud:
logger.warning('Target directories already exist.')
logger.warning('Received -f, so I will start anyway.')
logger.warning('Will overwrite config, initial COM, and yaml files.')
logger.warning('All other files will remain unmodified.')
redo = True
del config
if not os.path.exists(tgtdir):
logger.info(f'{tgtdir}: make directory')
os.makedirs(tgtdir)
logger.info(f'{tgtdir}/names.yaml: write experiment name and case name')
with open(f'{tgtdir}/names.yaml', 'wt') as fd:
fd.write(names_yaml)
if redo and os.path.exists(f'{tgtdir}/platform.yaml'):
logger.warning(
'I am NOT replacing platform.yaml. You must edit this manually.')
logger.warning(
'This is a safeguard to prevent automatic scrub space detection from switching scrub spaces mid-workflow.'
)
logger.warning(f'{tgtdir}/platform.yaml: NOT replacing this file.')
else:
logger.info(f'{tgtdir}/platform.yaml: write platform logic')
with open(f'{tgtdir}/platform.yaml', 'wt') as fd:
fd.write(platform_yaml)
logger.info(f'{case_file}: use this case file')
shutil.copy2(case_file, os.path.join(tgtdir, 'case.yaml'))
logger.info(f'{workflow_file}: use this workflow file')
shutil.copy2(workflow_file, os.path.join(tgtdir, 'workflow.yaml'))
for srcfile, tgtbase in itertools.chain(iter(YAML_DIRS_TO_COPY.items()),
iter(YAML_FILES_TO_COPY.items())):
tgtfile = os.path.join(tgtdir, tgtbase)
if os.path.isdir(srcfile):
logger.info(f'{srcfile}: copy yaml directory tree to {tgtfile}')
if os.path.exists(tgtfile):
logger.info(f'{tgtfile}: delete directory')
shutil.rmtree(tgtfile)
shutil.copytree(srcfile, tgtfile)
else:
logger.info(f'{srcfile}: copy yaml file to {tgtfile}')
shutil.copyfile(srcfile, tgtfile)
del tgtfile
# Deal with the static files:
for srcfile in glob.glob(f'{srcdir}/static/*.yaml'):
logger.info(f'{srcfile}: read file')
doc = from_file(srcfile)
tgtfile = os.path.join(tgtdir, "static_" + os.path.basename(srcfile))
yaml = to_yaml(doc)
logger.info(f'{tgtfile}: generate file')
with open(tgtfile, 'wt') as fd:
fd.write('# This file is automatically generated from:\n')
fd.write(f'# {srcfile}')
fd.write('# Changes to this file may be overwritten.\n\n')
fd.write(yaml)
del doc, tgtfile
logger.info(f'{tgtdir}: yaml files created here')
return tgtdir
## Unit test program for crow.config module
import sys, os, shutil, collections, copy
sys.path.append(os.getcwd() + '/../../')
import logging
from datetime import timedelta
from crow.config import from_dir, Suite, from_file, to_yaml, evaluate_immediates, from_string
import crow.config.represent
from crow.config.eval_tools import list_eval, dict_eval
from crow.sysenv import JobResourceSpec
platdoc = from_file('_common.yaml', '_sandbox.yaml')
platdoc.platform.Evaluate = True
evaluate_immediates(platdoc.platform)
shutil.copyfile('resources_sum_sample.yaml', 'resources_sum.yaml')
doc = from_file('_common.yaml', '_sandbox.yaml', 'case.yaml',
'default_resources.yaml', 'resources_sum.yaml')
filename = 'resources_sum.yaml'
doc.writeme = {'resources_sum': doc.partition_common.resources}
content = to_yaml({'resources_sum': doc.partition_common.resources})
with open(filename, 'wt') as fd:
fd.write(content)
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
#! /usr/bin/env python3
f'This script requires Python 3.6 or newer.'
import os
from crow.metascheduler import to_ecflow
from crow.config import from_file, Suite
conf = from_file('taskarray.yaml')
suite = Suite(conf.suite)
suite_defs, ecf_files = to_ecflow(suite)
for defname in suite_defs:
#print(f'=== contents of suite def {defname}\n{suite_defs[defname]}')
filename = defname
print(filename)
dirname = os.path.dirname(filename)
if dirname and not os.path.exists(dirname):
os.makedirs(os.path.dirname(filename))
with open(filename, 'wt') as fd:
fd.write(suite_defs[defname]['def'])
for setname in ecf_files:
print(f'ecf file set {setname}:\n')
for filename in ecf_files[setname]:
print(f' file {filename}')
dirname = os.path.dirname(filename)
if dirname and not os.path.exists(dirname):
os.makedirs(os.path.dirname(filename))
with open(filename + ".ecf", 'wt') as fd:
fd.write(ecf_files[setname][filename])
