def __init__(self, yaml_config_file):
self.yaml_config_file = yaml_config_file
self.root = Paths.dirname(self.yaml_config_file)
self.yamls = self._get_all_yamls()
self.cases = list()
self.common_config = None
# create dummy case for every yaml file in folder
if not Paths.exists(self.yaml_config_file):
self.common_config = deepcopy(DEFAULTS)
for y in self.yamls:
dummy_case = deepcopy(DEFAULTS)
dummy_case['file'] = [y]
self.cases.append(dummy_case)
else:
# setup common config values
self.yaml_config = self._read_yaml()
self.common_config = self.merge(DEFAULTS, self.yaml_config.get('common_config', {}))
# first process files which are specified in test_cases
missing = [Paths.basename(y) for y in self.yamls]
for case in self.yaml_config.get('test_cases', []):
case_config = self.merge(self.common_config, case)
self.cases.append(case_config)
for f in case_config['file']:
if f in missing:
missing.remove(f)
# process rest (dummy case)
for y in missing:
dummy_case = deepcopy(self.common_config)
dummy_case['file'] = [y]
self.cases.append(dummy_case)
def _run(self):
if Paths.exists(self.dir):
try:
shutil.rmtree(self.dir)
self.returncode = 0
except OSError as e:
self.returncode = 4
self.error = str(e)
def _run(self):
if Paths.exists(self.dir):
try:
shutil.rmtree(self.dir)
self.returncode = 0
except OSError as e:
self.returncode = 4
self.error = str(e)
def get_pbs_module(hostname_hint=None):
"""
file host_table.yaml serves as lookup table when using python script in queue mode
each key is hostname and each value names a module which should be loaded
modules are located in /src/python/scripts/pbs/modules
If no matching key for current machine exists try to use pbs_<hostname>
where all dots(.) are replaced with underscores(_)
if hostname_hint is not set node name will be used
:rtype : scripts.pbs.modules.pbs_tarkil_cesnet_cz
"""
pbs_module_path = None
host_file = Paths.join(Paths.flow123d_root(), 'config', 'host_table.yaml')
host_file_exists = Paths.exists(host_file)
hostname = hostname_hint or platform.node()
from_host = False
# try to get name from json file
if host_file_exists:
with open(host_file, 'r') as fp:
hosts = yaml.load(fp)
pbs_module_path = hosts.get(hostname, None)
from_host = pbs_module_path is not None
if not pbs_module_path:
hostname = hostname.replace('.', '_')
pbs_module_path = 'pbs_{}'.format(hostname)
# construct full path for import
full_module_path = 'scripts.pbs.modules.{module_name}'.format(
module_name=pbs_module_path)
# try to get pbs_module
try:
return importlib.import_module(full_module_path)
except ImportError:
Printer.all.err('Could not load module "{}" ({}) for hostname "{}"',
pbs_module_path, full_module_path, hostname)
with Printer.all.with_level(2):
if host_file_exists:
if from_host:
Printer.all.err(
'Value specified in host_table.yaml "{}" points to non-existing module',
pbs_module_path)
else:
Printer.all.err(
'Config file host_table.yaml does not have entry for hostname "{}"',
hostname)
else:
Printer.all.err(
'Config file host_table.yaml does not exists ({}) and auto module detection failed',
host_file)
raise
def run(self):
# wipe out dirs on demand
if self.wipeout_dir:
for d in self.wipeout_dir:
if Paths.exists(d):
Printer.all.out('Deleting directory {}', d)
shutil.rmtree(d)
total = 0
for p in self:
total += 1
p.copy()
Printer.all.out("Copied out {} files", total)
def get_pbs_module(hostname_hint=None):
"""
file host_table.yaml serves as lookup table when using python script in queue mode
each key is hostname and each value names a module which should be loaded
modules are located in /src/python/scripts/pbs/modules
If no matching key for current machine exists try to use pbs_<hostname>
where all dots(.) are replaced with underscores(_)
if hostname_hint is not set node name will be used
:rtype : scripts.pbs.modules.pbs_tarkil_cesnet_cz
"""
pbs_module_path = None
host_file = Paths.join(Paths.flow123d_root(), 'config', 'host_table.yaml')
host_file_exists = Paths.exists(host_file)
hostname = hostname_hint or platform.node()
from_host = False
# try to get name from json file
if host_file_exists:
with open(host_file, 'r') as fp:
hosts = yaml.load(fp)
pbs_module_path = hosts.get(hostname, None)
from_host = pbs_module_path is not None
if not pbs_module_path:
hostname = hostname.replace('.', '_')
pbs_module_path = 'pbs_{}'.format(hostname)
# construct full path for import
full_module_path = 'scripts.pbs.modules.{module_name}'.format(module_name=pbs_module_path)
# try to get pbs_module
try:
return importlib.import_module(full_module_path)
except ImportError:
Printer.all.err('Could not load module "{}" ({}) for hostname "{}"',
pbs_module_path, full_module_path, hostname)
with Printer.all.with_level(2):
if host_file_exists:
if from_host:
Printer.all.err('Value specified in host_table.yaml "{}" points to non-existing module', pbs_module_path)
else:
Printer.all.err('Config file host_table.yaml does not have entry for hostname "{}"', hostname)
else:
Printer.all.err('Config file host_table.yaml does not exists ({}) and auto module detection failed', host_file)
raise
def _run(self):
"""
Run method for this module
"""
if self.arg_options.random_output_dir:
import scripts.yamlc as yamlc
from core.base import System
yamlc.TEST_RESULTS = 'test_results-{}'.format(System.rnd8)
self.all_yamls = list()
for path in self.others:
if not Paths.exists(path):
Printer.all.err('given path does not exists, ignoring path "{}"', path)
sys.exit(3)
# append files to all_yamls
if Paths.is_dir(path):
self.all_yamls.extend(Paths.walk(path, ConfigPool.yaml_filters))
else:
self.all_yamls.append(path)
Printer.all.out("Found {} yaml file/s", len(self.all_yamls))
if not self.all_yamls:
Printer.all.wrn('No yaml files found in locations: \n {}', '\n '.join(self.others))
sys.exit(0)
self.configs = self.read_configs(self.all_yamls)
self.configs.update(
proc=self.arg_options.cpu,
time_limit=self.arg_options.time_limit,
memory_limit=self.arg_options.memory_limit,
)
# filter tags for includes and excludes
self.configs.filter_tags(
include=self.include,
exclude=self.exclude
)
if self.arg_options.queue:
Printer.all.out('Running in PBS mode')
return self.run_pbs_mode()
else:
Printer.all.out('Running in LOCAL mode')
return self.run_local_mode()
def _run(self):
"""
Run method for this module
"""
if self.arg_options.random_output_dir:
import scripts.yamlc as yamlc
yamlc.TEST_RESULTS = 'test_results-{}'.format(self.arg_options.random_output_dir)
self.all_yamls = list()
for path in self.arg_options.args:
if not Paths.exists(path):
Printer.all.err('given path does not exists, path "{}"', path)
sys.exit(3)
# append files to all_yamls
if Paths.is_dir(path):
self.all_yamls.extend(Paths.walk(path, ConfigPool.yaml_filters))
else:
self.all_yamls.append(path)
Printer.all.out("Found {} yaml file/s", len(self.all_yamls))
if not self.all_yamls:
Printer.all.wrn('No yaml files found in locations: \n {}', '\n '.join(self.arg_options.args))
sys.exit(0)
self.configs = self.read_configs(self.all_yamls)
self.configs.update(
proc=self.arg_options.cpu,
time_limit=self.arg_options.time_limit,
memory_limit=self.arg_options.memory_limit,
)
# filter tags for includes and excludes
self.configs.filter_tags(
include=self.include,
exclude=self.exclude
)
if self.arg_options.queue:
Printer.all.out('Running in PBS mode')
return self.run_pbs_mode()
else:
Printer.all.out('Running in LOCAL mode')
return self.run_local_mode()
def __init__(self,
yaml_config_file,
missing_policy=MISSING_POLICY_CREATE_DEFAULT):
self.yaml_config_file = yaml_config_file
self.root = Paths.dirname(self.yaml_config_file)
self.yamls = self._get_all_yamls()
self.cases = list()
self.common_config = None
self.missing_policy = missing_policy
# create dummy case for every yaml file in folder
if not Paths.exists(self.yaml_config_file):
self.common_config = deepcopy(yamlc.DEFAULTS)
for y in self.yamls:
dummy_case = deepcopy(yamlc.DEFAULTS)
dummy_case['files'] = [y]
self.cases.append(dummy_case)
else:
# setup common config values
self.yaml_config = self._read_yaml()
self.common_config = self.merge(
yamlc.DEFAULTS, self.yaml_config.get('common_config', {}))
# first process files which are specified in test_cases
missing = [Paths.basename(y) for y in self.yamls]
for case in self.yaml_config.get(yamlc.TAG_TEST_CASES, []):
case_config = self.merge(self.common_config, case)
# ensure that value are array
case_config[yamlc.TAG_FILES] = ensure_iterable(
case_config.get(yamlc.TAG_FILES, []))
# keep correct order
self.cases.append(case_config)
for f in case_config[yamlc.TAG_FILES]:
if f in missing:
missing.remove(f)
# process rest (dummy case)
if missing_policy == self.MISSING_POLICY_CREATE_DEFAULT:
for y in missing:
dummy_case = deepcopy(self.common_config)
dummy_case[yamlc.TAG_FILES] = [y]
self.cases.append(dummy_case)
def _walk_files(self):
# switching processing logic
self.dir_mode = False
# in this loop we are processing all given files/folders
all_yamls = list()
for path in self.arg_options.args:
if not Paths.exists(path):
printf.error('given path does not exists, path "{}"', path)
sys.exit(3)
# append files to all_yamls
if Paths.is_dir(path):
self.dir_mode = True
all_yamls.extend(Paths.walk(path, ConfigPool.yaml_filters))
else:
all_yamls.append(path)
return all_yamls
def get_pbs_module(hostname=None):
"""
:rtype : scripts.pbs.modules.pbs_tarkil_cesnet_cz
"""
pbs_module_path = None
if not hostname:
hostname = platform.node()
# try to get name from json file
host_file = Paths.join(Paths.source_dir(), 'host_table.json')
if Paths.exists(host_file):
with open(host_file, 'r') as fp:
hosts = json.load(fp)
pbs_module_path = hosts.get(hostname, None)
if not pbs_module_path:
hostname = hostname.replace('.', '_')
pbs_module_path = 'pbs_{}'.format(hostname)
Printer.wrn('Warning! no host specified assuming module {}', pbs_module_path)
# try to get pbs_module
return importlib.import_module('scripts.pbs.modules.{}'.format(pbs_module_path))
def __init__(self, yaml_config_file):
self.yaml_config_file = yaml_config_file
self.root = Paths.dirname(self.yaml_config_file)
self.yamls = self._get_all_yamls()
self.cases = list()
self.common_config = None
# create dummy case for every yaml file in folder
if not Paths.exists(self.yaml_config_file):
self.common_config = deepcopy(yamlc.DEFAULTS)
for y in self.yamls:
dummy_case = deepcopy(yamlc.DEFAULTS)
dummy_case['files'] = [y]
self.cases.append(dummy_case)
else:
# setup common config values
self.yaml_config = self._read_yaml()
self.common_config = self.merge(
yamlc.DEFAULTS, self.yaml_config.get('common_config', {}))
# first process files which are specified in test_cases
missing = [Paths.basename(y) for y in self.yamls]
for case in self.yaml_config.get(yamlc.TAG_TEST_CASES, []):
case_config = self.merge(self.common_config, case)
# ensure that value is array
case_config[yamlc.TAG_FILES] = ensure_iterable(
case_config.get(yamlc.TAG_FILES, []))
# keep correct order
self.cases.append(case_config)
for f in case_config[yamlc.TAG_FILES]:
if f in missing:
missing.remove(f)
# process rest (dummy case)
for y in missing:
dummy_case = deepcopy(self.common_config)
dummy_case[yamlc.TAG_FILES] = [y]
self.cases.append(dummy_case)
def do_work(parser, args=None, debug=False):
"""
:type parser: utils.argparser.ArgParser
"""
# parse arguments
global arg_options, arg_others, arg_rest
arg_options, arg_others, arg_rest = parser.parse(args)
Paths.format = PathFormat.ABSOLUTE
Paths.base_dir('' if not arg_options.root else arg_options.root)
# configure printer
Printer.batch_output = arg_options.batch
Printer.dynamic_output = not arg_options.batch
# we need flow123d, mpiexec and ndiff to exists in LOCAL mode
if not arg_options.queue and not Paths.test_paths('flow123d', 'mpiexec', 'ndiff'):
Printer.err('Missing obligatory files! Exiting')
GlobalResult.error = "missing obligatory files"
sys.exit(1)
# test yaml args
if not arg_others:
parser.exit_usage('Error: No yaml files or folder given')
GlobalResult.error = "no yaml files or folder given"
sys.exit(2)
all_yamls = list()
for path in arg_others:
if not Paths.exists(path):
Printer.err('Error! given path does not exists, ignoring path "{}"', path)
GlobalResult.error = "path does not exist"
sys.exit(3)
if Paths.is_dir(path):
all_yamls.extend(Paths.walk(path, filters=[
PathFilters.filter_type_is_file(),
PathFilters.filter_ext('.yaml'),
PathFilters.filter_not(PathFilters.filter_name('config.yaml'))
]))
else:
all_yamls.append(path)
Printer.out("Found {} .yaml file/s", len(all_yamls))
if not all_yamls:
Printer.wrn('Warning! No yaml files found in locations: \n {}', '\n '.join(arg_others))
GlobalResult.error = "no yaml files or folders given"
sys.exit(3)
configs = read_configs(all_yamls)
configs.update(
proc=arg_options.cpu,
time_limit=arg_options.time_limit,
memory_limit=arg_options.memory_limit,
)
if arg_options.queue:
Printer.out('Running in PBS mode')
return run_pbs_mode(configs, debug)
else:
Printer.out('Running in LOCAL mode')
return run_local_mode(configs, debug)
