def get_pbs_command(self, pbs_script_filename):
# total parallel process
np = self.case.proc
# processes per node, default value 1
ppn = self.ppn
# number of physical machines to be reserved
nodes = float(np) / ppn
if int(nodes) != nodes:
Printer.wrn('Warning: NP is not divisible by PPN')
nodes = int(math.ceil(nodes))
# memory limit
mem = int(self.case.memory_limit * ppn)
# time_limit
walltime = datetime.timedelta(seconds=int(self.case.time_limit))
# get queue, if only -q is set, 'default' queue will be set
# otherwise given string value will be used
queue = self.queue
queue = 'default' if type(queue) is not str else queue
# command
command = [
'qsub',
'-l', 'nodes={nodes}:ppn={ppn}'.format(**locals()), # :nfs4 option may be set
'-l', 'mem={mem}mb'.format(**locals()),
'-l', 'walltime={walltime}'.format(**locals()),
'-l', 'place=infiniband',
'-q', '{queue}'.format(**locals()),
'-o', self.case.fs.pbs_output,
pbs_script_filename
]
return command
def print_log_file(cls, f, n_lines):
log_file = IO.read(f)
if log_file:
if n_lines == 0:
Printer.out('Full log from file {}:', f)
else:
Printer.out('Last {} lines from file {}:', abs(n_lines), f)
Printer.wrn(format_n_lines(log_file.rstrip(), -n_lines, indent=Printer.indent * ' '))
def print_log_file(cls, f, n_lines):
log_file = IO.read(f)
if log_file:
if n_lines == 0:
Printer.out('Full log from file {}:', f)
else:
Printer.out('Last {} lines from file {}:', abs(n_lines), f)
Printer.wrn(
format_n_lines(log_file.rstrip(),
-n_lines,
indent=Printer.indent * ' '))
def get_pbs_command(self, pbs_script_filename):
# total parallel process
np = self.case.proc
# processes per node, default value 1
ppn = self.ppn
# number of physical machines to be reserved
nodes = float(np) / ppn
if int(nodes) != nodes:
Printer.wrn('Warning: NP is not divisible by PPN')
nodes = int(math.ceil(nodes))
# memory limit
mem = int(self.case.memory_limit * ppn)
# time_limit
walltime = datetime.timedelta(seconds=int(self.case.time_limit))
# get queue, if only -q is set, 'default' queue will be set
# otherwise given string value will be used
queue = self.queue
queue = 'default' if type(queue) is not str else queue
# command
command = [
'qsub',
'-l',
'nodes={nodes}:ppn={ppn}'.format(
**locals()), # :nfs4 option may be set
'-l',
'mem={mem}mb'.format(**locals()),
'-l',
'walltime={walltime}'.format(**locals()),
'-l',
'place=infiniband',
'-q',
'{queue}'.format(**locals()),
'-o',
self.case.fs.pbs_output,
pbs_script_filename
]
return command
def get_pbs_module(hostname=None):
"""
:rtype : scripts.pbs.modules.pbs_tarkil_cesnet_cz
"""
pbs_module_path = None
if not hostname:
hostname = platform.node()
# try to get name from json file
host_file = Paths.join(Paths.source_dir(), 'host_table.json')
if Paths.exists(host_file):
with open(host_file, 'r') as fp:
hosts = json.load(fp)
pbs_module_path = hosts.get(hostname, None)
if not pbs_module_path:
hostname = hostname.replace('.', '_')
pbs_module_path = 'pbs_{}'.format(hostname)
Printer.wrn('Warning! no host specified assuming module {}', pbs_module_path)
# try to get pbs_module
return importlib.import_module('scripts.pbs.modules.{}'.format(pbs_module_path))
def do_work(parser, args=None, debug=False):
"""
:type parser: utils.argparser.ArgParser
"""
# parse arguments
global arg_options, arg_others, arg_rest
arg_options, arg_others, arg_rest = parser.parse(args)
Paths.format = PathFormat.ABSOLUTE
Paths.base_dir('' if not arg_options.root else arg_options.root)
# configure printer
Printer.batch_output = arg_options.batch
Printer.dynamic_output = not arg_options.batch
# we need flow123d, mpiexec and ndiff to exists in LOCAL mode
if not arg_options.queue and not Paths.test_paths('flow123d', 'mpiexec', 'ndiff'):
Printer.err('Missing obligatory files! Exiting')
GlobalResult.error = "missing obligatory files"
sys.exit(1)
# test yaml args
if not arg_others:
parser.exit_usage('Error: No yaml files or folder given')
GlobalResult.error = "no yaml files or folder given"
sys.exit(2)
all_yamls = list()
for path in arg_others:
if not Paths.exists(path):
Printer.err('Error! given path does not exists, ignoring path "{}"', path)
GlobalResult.error = "path does not exist"
sys.exit(3)
if Paths.is_dir(path):
all_yamls.extend(Paths.walk(path, filters=[
PathFilters.filter_type_is_file(),
PathFilters.filter_ext('.yaml'),
PathFilters.filter_not(PathFilters.filter_name('config.yaml'))
]))
else:
all_yamls.append(path)
Printer.out("Found {} .yaml file/s", len(all_yamls))
if not all_yamls:
Printer.wrn('Warning! No yaml files found in locations: \n {}', '\n '.join(arg_others))
GlobalResult.error = "no yaml files or folders given"
sys.exit(3)
configs = read_configs(all_yamls)
configs.update(
proc=arg_options.cpu,
time_limit=arg_options.time_limit,
memory_limit=arg_options.memory_limit,
)
if arg_options.queue:
Printer.out('Running in PBS mode')
return run_pbs_mode(configs, debug)
else:
Printer.out('Running in LOCAL mode')
return run_local_mode(configs, debug)
