def get_pbs_command(self, pbs_script_filename): # total parallel process np = self.case.proc # processes per node, default value 1 ppn = self.ppn # number of physical machines to be reserved nodes = float(np) / ppn if int(nodes) != nodes: Printer.wrn('Warning: NP is not divisible by PPN') nodes = int(math.ceil(nodes)) # memory limit mem = int(self.case.memory_limit * ppn) # time_limit walltime = datetime.timedelta(seconds=int(self.case.time_limit)) # get queue, if only -q is set, 'default' queue will be set # otherwise given string value will be used queue = self.queue queue = 'default' if type(queue) is not str else queue # command command = [ 'qsub', '-l', 'nodes={nodes}:ppn={ppn}'.format(**locals()), # :nfs4 option may be set '-l', 'mem={mem}mb'.format(**locals()), '-l', 'walltime={walltime}'.format(**locals()), '-l', 'place=infiniband', '-q', '{queue}'.format(**locals()), '-o', self.case.fs.pbs_output, pbs_script_filename ] return command
def print_log_file(cls, f, n_lines): log_file = IO.read(f) if log_file: if n_lines == 0: Printer.out('Full log from file {}:', f) else: Printer.out('Last {} lines from file {}:', abs(n_lines), f) Printer.wrn(format_n_lines(log_file.rstrip(), -n_lines, indent=Printer.indent * ' '))
def print_log_file(cls, f, n_lines): log_file = IO.read(f) if log_file: if n_lines == 0: Printer.out('Full log from file {}:', f) else: Printer.out('Last {} lines from file {}:', abs(n_lines), f) Printer.wrn( format_n_lines(log_file.rstrip(), -n_lines, indent=Printer.indent * ' '))
def get_pbs_command(self, pbs_script_filename): # total parallel process np = self.case.proc # processes per node, default value 1 ppn = self.ppn # number of physical machines to be reserved nodes = float(np) / ppn if int(nodes) != nodes: Printer.wrn('Warning: NP is not divisible by PPN') nodes = int(math.ceil(nodes)) # memory limit mem = int(self.case.memory_limit * ppn) # time_limit walltime = datetime.timedelta(seconds=int(self.case.time_limit)) # get queue, if only -q is set, 'default' queue will be set # otherwise given string value will be used queue = self.queue queue = 'default' if type(queue) is not str else queue # command command = [ 'qsub', '-l', 'nodes={nodes}:ppn={ppn}'.format( **locals()), # :nfs4 option may be set '-l', 'mem={mem}mb'.format(**locals()), '-l', 'walltime={walltime}'.format(**locals()), '-l', 'place=infiniband', '-q', '{queue}'.format(**locals()), '-o', self.case.fs.pbs_output, pbs_script_filename ] return command
def get_pbs_module(hostname=None): """ :rtype : scripts.pbs.modules.pbs_tarkil_cesnet_cz """ pbs_module_path = None if not hostname: hostname = platform.node() # try to get name from json file host_file = Paths.join(Paths.source_dir(), 'host_table.json') if Paths.exists(host_file): with open(host_file, 'r') as fp: hosts = json.load(fp) pbs_module_path = hosts.get(hostname, None) if not pbs_module_path: hostname = hostname.replace('.', '_') pbs_module_path = 'pbs_{}'.format(hostname) Printer.wrn('Warning! no host specified assuming module {}', pbs_module_path) # try to get pbs_module return importlib.import_module('scripts.pbs.modules.{}'.format(pbs_module_path))
def do_work(parser, args=None, debug=False): """ :type parser: utils.argparser.ArgParser """ # parse arguments global arg_options, arg_others, arg_rest arg_options, arg_others, arg_rest = parser.parse(args) Paths.format = PathFormat.ABSOLUTE Paths.base_dir('' if not arg_options.root else arg_options.root) # configure printer Printer.batch_output = arg_options.batch Printer.dynamic_output = not arg_options.batch # we need flow123d, mpiexec and ndiff to exists in LOCAL mode if not arg_options.queue and not Paths.test_paths('flow123d', 'mpiexec', 'ndiff'): Printer.err('Missing obligatory files! Exiting') GlobalResult.error = "missing obligatory files" sys.exit(1) # test yaml args if not arg_others: parser.exit_usage('Error: No yaml files or folder given') GlobalResult.error = "no yaml files or folder given" sys.exit(2) all_yamls = list() for path in arg_others: if not Paths.exists(path): Printer.err('Error! given path does not exists, ignoring path "{}"', path) GlobalResult.error = "path does not exist" sys.exit(3) if Paths.is_dir(path): all_yamls.extend(Paths.walk(path, filters=[ PathFilters.filter_type_is_file(), PathFilters.filter_ext('.yaml'), PathFilters.filter_not(PathFilters.filter_name('config.yaml')) ])) else: all_yamls.append(path) Printer.out("Found {} .yaml file/s", len(all_yamls)) if not all_yamls: Printer.wrn('Warning! No yaml files found in locations: \n {}', '\n '.join(arg_others)) GlobalResult.error = "no yaml files or folders given" sys.exit(3) configs = read_configs(all_yamls) configs.update( proc=arg_options.cpu, time_limit=arg_options.time_limit, memory_limit=arg_options.memory_limit, ) if arg_options.queue: Printer.out('Running in PBS mode') return run_pbs_mode(configs, debug) else: Printer.out('Running in LOCAL mode') return run_local_mode(configs, debug)