def make_yaml_files(srcdir, tgtdir): if not os.path.exists(tgtdir): logger.info(f'{tgtdir}: make directory') os.makedirs(tgtdir) logger.info(f'{tgtdir}: send yaml files to here') logger.info(f'{srcdir}: get yaml files from here') for srcfile in glob.glob(f'{srcdir}/*.yaml'): srcbase = os.path.basename(srcfile) if srcbase.startswith('resources'): continue if srcbase.startswith('settings'): continue tgtfile = os.path.join(tgtdir, srcbase) logger.info(f'{srcbase}: copy yaml file') shutil.copyfile(srcfile, tgtfile) # Deal with the settings: doc = from_file(f"{srcdir}/settings.yaml") settings_yaml = os.path.join(tgtdir, 'settings.yaml') logger.info(f'{settings_yaml}: generate file') with open(f'{tgtdir}/settings.yaml', 'wt') as fd: fd.write('# This file is automatically generated from:\n') fd.write(f'# {srcdir}/settings.yaml') fd.write('# Changes to this file may be overwritten.\n\n') fd.write(to_yaml(doc)) # Now the resources: resource_basename = doc.settings.resource_file resource_srcfile = os.path.join(srcdir, resource_basename) resource_tgtfile = os.path.join(tgtdir, 'resources.yaml') logger.info(f'{resource_srcfile}: use this resource yaml file') shutil.copyfile(resource_srcfile, resource_tgtfile) logger.info(f'{tgtdir}: yaml files created here')
def sandbox_platforms(userfile, platdir): available = {} plat = from_file(userfile, f'{platdir}/_common.yaml', f'{platdir}/_sandbox.yaml') # plat=from_file(f'{platdir}/_common.yaml',f'{platdir}/_sandbox.yaml') available[plat.platform.name] = plat return available
def find_available_platforms(platdir): matches = {} filenames = {} can_skip = set() for matching_file in glob.glob(f'{platdir}/[a-zA-Z]*.yaml'): logger.info(f'{matching_file}: check this platform...') plat = from_file('../user.yaml', f'{platdir}/_common.yaml', matching_file) if not 'platform' in plat or \ not 'detect' in plat.platform or \ not 'name' in plat.platform: logger.warning(f'{matching_file}: does not contain a ' '"platform" map with "detect" and "name"') continue name = plat.platform.name if plat.platform.detect: logger.info(f'{matching_file}: platform {name} matches') if name in filenames: logger.error( f'{filenames[name]}: same platform name "{name}" as {matching_file}' ) exit(1) matches[name] = plat if plat.platform.get('skip_if_others_present', False): can_skip.add(name) available = copy(matches) for k in can_skip: if k in available: del available[k] if available: return available else: # All platforms "can be skipped" so skip none: return matches
def make_config_files_in_expdir(doc, expdir): for key in doc.keys(): if not key.startswith('config_'): continue value = doc[key] if not isinstance(value, collections.Mapping): continue if not 'filename' in value or not 'content' in value: logger.warning( f'{key}: config files require "filename" and "content" entries.' ) if value.get('disable', False): continue # filename = os.path.join(expdir, str(value.filename)) logger.debug(f'{filename}: expand') content = str(value.content) logger.info(f'{filename}: write') with open(filename, 'wt') as fd: fd.write(content) # value = doc['partition_common']['resources'] # test1 = from_file('tests/test_data/yaml-io/original.yaml') doc.partition_common.Evaluate = True crow.config.evaluate_immediates(doc.partition_common) value = doc['partition_common']['resources'] # value = { 'resources_sum': doc['partition_common']['resources']} filename = os.path.join(expdir, 'resources_sum.yaml') logger.debug(f'{filename}: expand') content = to_yaml(value) with open(filename, 'wt') as fd: fd.write(content) resources_sum = from_file(filename) value = {'resources_sum': resources_sum} logger.debug(f'{filename}: expand') content = to_yaml(value) with open(filename, 'wt') as fd: fd.write(content)
def make_yaml_files(srcdir, tgtdir): if not os.path.exists(tgtdir): logger.info(f'{tgtdir}: make directory') os.makedirs(tgtdir) logger.info(f'{tgtdir}: send yaml files to here') logger.info(f'{srcdir}: get yaml files from here') for srcfile in glob.glob(f'{srcdir}/*.yaml'): srcbase = os.path.basename(srcfile) if srcbase.startswith('resources'): continue if srcbase.startswith('config'): continue tgtfile = os.path.join(tgtdir, srcbase) logger.info(f'{srcbase}: copy yaml file') shutil.copyfile(srcfile, tgtfile) del srcbase, tgtfile readme = [os.path.join(srcdir, 'settings_validator.yaml')] # Deal with the config files: for srcfile in glob.glob(f'{srcdir}/config*.yaml'): logger.info(f'{srcfile}: read file') doc = from_file(srcfile) tgtfile = os.path.join(tgtdir, os.path.basename(srcfile)) yaml = to_yaml(doc) anchor = os.path.basename(srcfile)[:-5] anchored = re.sub(r'\A([a-zA-Z][a-zA-Z0-9_]*):', r'\1: &' + anchor, yaml) logger.info(f'{tgtfile}: generate file') with open(tgtfile, 'wt') as fd: fd.write('# This file is automatically generated from:\n') fd.write(f'# {srcfile}') fd.write('# Changes to this file may be overwritten.\n\n') fd.write(yaml) readme.insert(0, tgtfile) del doc, tgtfile # Read the settings file readme.append('settings.yaml') logger.info(f'Read files: {", ".join(readme)}') doc = from_file(*readme) # Now the resources: resource_basename = doc.settings.resource_file resource_srcfile = os.path.join(srcdir, resource_basename) resource_tgtfile = os.path.join(tgtdir, 'resources.yaml') logger.info(f'{resource_srcfile}: use this resource yaml file') shutil.copyfile(resource_srcfile, resource_tgtfile) logger.info(f'{tgtdir}: yaml files created here')
def find_available_platforms(platdir): available = {} for matching_file in glob.glob(f'{platdir}/*.yaml'): logger.info(f'{matching_file}: check this platform...') plat = from_file('user.yaml', matching_file) if not 'platform' in plat or \ not 'detect' in plat.platform or \ not 'name' in plat.platform: logger.warning(f'{matching_file}: does not contain a ' '"platform" map with "detect" and "name"') if plat.platform.detect: logger.info( f'{matching_file}: platform {plat.platform.name} matches') available[plat.platform.name] = plat return available
#! /usr/bin/env python3 f'This script requires Python 3.6 or newer.' import os from crow.metascheduler import to_ecflow from crow.config import from_file, Suite conf = from_file('ecftest.yaml') suite = Suite(conf.suite) suite_defs, ecf_files = to_ecflow(suite) for defname in suite_defs: #print(f'=== contents of suite def {defname}\n{suite_defs[defname]}') filename = defname print(filename) dirname = os.path.dirname(filename) if dirname and not os.path.exists(dirname): os.makedirs(os.path.dirname(filename)) with open(filename, 'wt') as fd: fd.write(suite_defs[defname]['def']) for setname in ecf_files: print(f'ecf file set {setname}:\n') for filename in ecf_files[setname]: print(f' file {filename}') dirname = os.path.dirname(filename) if dirname and not os.path.exists(dirname): os.makedirs(os.path.dirname(filename)) with open(filename + ".ecf", 'wt') as fd: fd.write(ecf_files[setname][filename])
def make_yaml_files_in_expdir(srcdir, case_name, experiment_name, platdoc, force): logger.info(f'{srcdir}: get yaml files from here') logger.info(f'{case_name}: use this case') case_file = find_case_yaml_file_for(case_name) platform_yaml = to_yaml(platdoc) names = {'names': {'experiment': experiment_name, 'case': case_name}} names_yaml = to_yaml(names) # Get the configuration from the source directory: with io.StringIO() as fd: fd.write(platform_yaml) fd.write('\n\n') fd.write(names_yaml) fd.write('\n\n') crow.config.follow_main(fd, srcdir) fd.write('\n\n') with open(case_file, 'rt') as cfd: fd.write(cfd.read()) config_contents = fd.getvalue() config = crow.config.from_string(config_contents) workflow_file = os.path.join(srcdir, config.places.workflow_file) tgtdir = config.places.EXPDIR rotdir = config.places.ROTDIR redo = False logger.info(f'{rotdir}: COM files will be here') logger.info(f'{tgtdir}: send yaml files to here') gud = True if os.path.exists(tgtdir): gud = False logger.warning(f'{tgtdir}: already exists!') if os.path.exists(rotdir): gud = False logger.warning(f'{rotdir}: already exists!') if not gud and not force: logger.error('Target directories already exist.') logger.error('I will not start a workflow unless you do -f.') logger.critical( 'Use -f to force this workflow to start, but we aware that config, initial COM, and yaml files will be overwritten. Other files will remain unmodified.' ) exit(1) elif not gud: logger.warning('Target directories already exist.') logger.warning('Received -f, so I will start anyway.') logger.warning('Will overwrite config, initial COM, and yaml files.') logger.warning('All other files will remain unmodified.') redo = True del config if not os.path.exists(tgtdir): logger.info(f'{tgtdir}: make directory') os.makedirs(tgtdir) logger.info(f'{tgtdir}/names.yaml: write experiment name and case name') with open(f'{tgtdir}/names.yaml', 'wt') as fd: fd.write(names_yaml) if redo and os.path.exists(f'{tgtdir}/platform.yaml'): logger.warning( 'I am NOT replacing platform.yaml. You must edit this manually.') logger.warning( 'This is a safeguard to prevent automatic scrub space detection from switching scrub spaces mid-workflow.' ) logger.warning(f'{tgtdir}/platform.yaml: NOT replacing this file.') else: logger.info(f'{tgtdir}/platform.yaml: write platform logic') with open(f'{tgtdir}/platform.yaml', 'wt') as fd: fd.write(platform_yaml) logger.info(f'{case_file}: use this case file') shutil.copy2(case_file, os.path.join(tgtdir, 'case.yaml')) logger.info(f'{workflow_file}: use this workflow file') shutil.copy2(workflow_file, os.path.join(tgtdir, 'workflow.yaml')) for srcfile, tgtbase in itertools.chain(iter(YAML_DIRS_TO_COPY.items()), iter(YAML_FILES_TO_COPY.items())): tgtfile = os.path.join(tgtdir, tgtbase) if os.path.isdir(srcfile): logger.info(f'{srcfile}: copy yaml directory tree to {tgtfile}') if os.path.exists(tgtfile): logger.info(f'{tgtfile}: delete directory') shutil.rmtree(tgtfile) shutil.copytree(srcfile, tgtfile) else: logger.info(f'{srcfile}: copy yaml file to {tgtfile}') shutil.copyfile(srcfile, tgtfile) del tgtfile # Deal with the static files: for srcfile in glob.glob(f'{srcdir}/static/*.yaml'): logger.info(f'{srcfile}: read file') doc = from_file(srcfile) tgtfile = os.path.join(tgtdir, "static_" + os.path.basename(srcfile)) yaml = to_yaml(doc) logger.info(f'{tgtfile}: generate file') with open(tgtfile, 'wt') as fd: fd.write('# This file is automatically generated from:\n') fd.write(f'# {srcfile}') fd.write('# Changes to this file may be overwritten.\n\n') fd.write(yaml) del doc, tgtfile logger.info(f'{tgtdir}: yaml files created here') return tgtdir
## Unit test program for crow.config module import sys, os, shutil, collections, copy sys.path.append(os.getcwd() + '/../../') import logging from datetime import timedelta from crow.config import from_dir, Suite, from_file, to_yaml, evaluate_immediates, from_string import crow.config.represent from crow.config.eval_tools import list_eval, dict_eval from crow.sysenv import JobResourceSpec platdoc = from_file('_common.yaml', '_sandbox.yaml') platdoc.platform.Evaluate = True evaluate_immediates(platdoc.platform) shutil.copyfile('resources_sum_sample.yaml', 'resources_sum.yaml') doc = from_file('_common.yaml', '_sandbox.yaml', 'case.yaml', 'default_resources.yaml', 'resources_sum.yaml') filename = 'resources_sum.yaml' doc.writeme = {'resources_sum': doc.partition_common.resources} content = to_yaml({'resources_sum': doc.partition_common.resources}) with open(filename, 'wt') as fd: fd.write(content) logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
#! /usr/bin/env python3 f'This script requires Python 3.6 or newer.' import os from crow.metascheduler import to_ecflow from crow.config import from_file, Suite conf = from_file('taskarray.yaml') suite = Suite(conf.suite) suite_defs, ecf_files = to_ecflow(suite) for defname in suite_defs: #print(f'=== contents of suite def {defname}\n{suite_defs[defname]}') filename = defname print(filename) dirname = os.path.dirname(filename) if dirname and not os.path.exists(dirname): os.makedirs(os.path.dirname(filename)) with open(filename, 'wt') as fd: fd.write(suite_defs[defname]['def']) for setname in ecf_files: print(f'ecf file set {setname}:\n') for filename in ecf_files[setname]: print(f' file {filename}') dirname = os.path.dirname(filename) if dirname and not os.path.exists(dirname): os.makedirs(os.path.dirname(filename)) with open(filename + ".ecf", 'wt') as fd: fd.write(ecf_files[setname][filename])