def get_batch_directives(self, case, job, overrides=None): """ """ result = [] directive_prefix = None roots = self.get_children("batch_system") queue = self.get_value("JOB_QUEUE", subgroup=job) if self._batchtype != "none" and not queue in self._get_all_queue_names(): qnode = self.get_default_queue() queue = self.text(qnode) for root in roots: if root is not None: if directive_prefix is None: directive_prefix = self.get_element_text("batch_directive", root=root) dnodes = self.get_children("directives", root=root) for dnode in dnodes: nodes = self.get_children("directive", root=dnode) if self._match_attribs(self.attrib(dnode), case, queue): for node in nodes: directive = self.get_resolved_value("" if self.text(node) is None else self.text(node)) default = self.get(node, "default") if default is None: directive = transform_vars(directive, case=case, subgroup=job, default=default, overrides=overrides) else: directive = transform_vars(directive, default=default) custom_prefix = self.get(node, "prefix") prefix = directive_prefix if custom_prefix is None else custom_prefix result.append("{}{}".format("" if not prefix else (prefix + " "), directive)) return "\n".join(result)
def make_batch_script(self, input_template, job, case, outfile=None): expect(os.path.exists(input_template), "input file '{}' does not exist".format(input_template)) task_count = self.get_value("task_count", subgroup=job) overrides = {} if task_count is not None: overrides["total_tasks"] = int(task_count) overrides["num_nodes"] = int(math.ceil(float(task_count)/float(case.tasks_per_node))) else: task_count = case.get_value("TOTALPES")*int(case.thread_count) if int(task_count) < case.get_value("MAX_TASKS_PER_NODE"): overrides["max_tasks_per_node"] = int(task_count) overrides["job_id"] = case.get_value("CASE") + os.path.splitext(job)[1] if "pleiades" in case.get_value("MACH"): # pleiades jobname needs to be limited to 15 chars overrides["job_id"] = overrides["job_id"][:15] overrides["batchdirectives"] = self.get_batch_directives(case, job, overrides=overrides) overrides["mpirun"] = case.get_mpirun_cmd(job=job) output_text = transform_vars(open(input_template,"r").read(), case=case, subgroup=job, overrides=overrides) output_name = get_batch_script_for_job(job) if outfile is None else outfile logger.info("Creating file {}".format(output_name)) with open(output_name, "w") as fd: fd.write(output_text) # make sure batch script is exectuble os.chmod(output_name, os.stat(output_name).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
def make_batch_script(self, input_template, job, case, total_tasks, tasks_per_node, num_nodes, thread_count): expect(os.path.exists(input_template), "input file '%s' does not exist" % input_template) self.tasks_per_node = tasks_per_node self.num_tasks = total_tasks self.tasks_per_numa = tasks_per_node / 2 self.thread_count = thread_count task_count = self.get_value("task_count", subgroup=job) if task_count == "default": self.total_tasks = total_tasks self.num_nodes = num_nodes else: self.total_tasks = task_count self.num_nodes = int(math.ceil(float(task_count)/float(tasks_per_node))) self.pedocumentation = "" self.job_id = case.get_value("CASE") + os.path.splitext(job)[1] if "pleiades" in case.get_value("MACH"): # pleiades jobname needs to be limited to 15 chars self.job_id = self.job_id[:15] self.output_error_path = self.job_id self.batchdirectives = self.get_batch_directives(case, job) output_text = transform_vars(open(input_template,"r").read(), case=case, subgroup=job, check_members=self) with open(job, "w") as fd: fd.write(output_text) os.chmod(job, os.stat(job).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
def get_batch_directives(self, case, job, raw=False): """ """ result = [] directive_prefix = self.get_node("batch_directive").text directive_prefix = "" if directive_prefix is None else directive_prefix roots = self.get_nodes("batch_system") for root in roots: if root is not None: nodes = self.get_nodes("directive", root=root) for node in nodes: directive = self.get_resolved_value("" if node.text is None else node.text) default = node.get("default") if not raw: directive = transform_vars(directive, case=case, subgroup=job, default=default, check_members=self) elif default is not None: directive = transform_vars(directive, default=default) result.append("%s %s" % (directive_prefix, directive)) return "\n".join(result)
def get_batch_directives(self, case, job, overrides=None): """ """ result = [] directive_prefix = None roots = self.get_children("batch_system") for root in roots: if root is not None: if directive_prefix is None: directive_prefix = self.get_element_text("batch_directive", root=root) nodes = self.get_children("directive", root=root) for node in nodes: directive = self.get_resolved_value("" if self.text(node) is None else self.text(node)) default = self.get(node, "default") if default is None: directive = transform_vars(directive, case=case, subgroup=job, default=default, overrides=overrides) else: directive = transform_vars(directive, default=default) result.append("{} {}".format("" if directive_prefix is None else directive_prefix, directive)) return "\n".join(result)
def make_batch_script(self, input_template, job, case): expect(os.path.exists(input_template), "input file '%s' does not exist" % input_template) task_maker = TaskMaker(case) self.maxthreads = task_maker.maxthreads self.taskgeometry = task_maker.taskgeometry self.threadgeometry = task_maker.threadgeometry self.taskcount = task_maker.taskcount self.thread_count = task_maker.thread_count self.pedocumentation = task_maker.document() self.ptile = task_maker.ptile self.tasks_per_node = task_maker.tasks_per_node self.max_tasks_per_node = task_maker.MAX_TASKS_PER_NODE self.tasks_per_numa = task_maker.tasks_per_numa self.num_tasks = task_maker.totaltasks task_count = self.get_value("task_count", subgroup=job) if task_count == "default": self.sumpes = task_maker.fullsum self.totaltasks = task_maker.totaltasks self.fullsum = task_maker.fullsum self.sumtasks = task_maker.totaltasks self.task_count = task_maker.fullsum self.num_nodes = task_maker.num_nodes else: self.sumpes = task_count self.totaltasks = task_count self.fullsum = task_count self.sumtasks = task_count self.task_count = task_count self.num_nodes = task_count self.pedocumentation = "" self.job_id = case.get_value("CASE") + os.path.splitext(job)[1] if "pleiades" in case.get_value("MACH"): # pleiades jobname needs to be limited to 15 chars self.job_id = self.job_id[:15] self.output_error_path = self.job_id self.batchdirectives = self.get_batch_directives(case, job) output_text = transform_vars(open(input_template,"r").read(), case=case, subgroup=job, check_members=self) with open(job, "w") as fd: fd.write(output_text) os.chmod(job, os.stat(job).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
def make_batch_script(self, input_template, job, case): expect(os.path.exists(input_template), "input file '{}' does not exist".format(input_template)) task_count = self.get_value("task_count", subgroup=job) overrides = {} if task_count is not None: overrides["total_tasks"] = int(task_count) overrides["num_nodes"] = int(math.ceil(float(task_count)/float(case.tasks_per_node))) overrides["pedocumentation"] = "" # TODO? overrides["job_id"] = case.get_value("CASE") + os.path.splitext(job)[1] if "pleiades" in case.get_value("MACH"): # pleiades jobname needs to be limited to 15 chars overrides["job_id"] = overrides["job_id"][:15] overrides["batchdirectives"] = self.get_batch_directives(case, job, overrides=overrides) output_text = transform_vars(open(input_template,"r").read(), case=case, subgroup=job, overrides=overrides) with open(job, "w") as fd: fd.write(output_text) os.chmod(job, os.stat(job).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
directive = self.get_resolved_value("" if self.text(node) is None else self.text(node)) if output_format == 'cylc': if self._batchtype == 'pbs': # cylc includes the -N itself, no need to add if directive.startswith("-N"): directive='' continue m = re.match(r'\s*(-[\w])', directive) if m: directive = re.sub(r'(-[\w]) ','{} = '.format(m.group(1)), directive) default = self.get(node, "default") if default is None: directive = transform_vars(directive, case=case, subgroup=job, default=default, overrides=overrides) else: directive = transform_vars(directive, default=default) custom_prefix = self.get(node, "prefix") prefix = directive_prefix if custom_prefix is None else custom_prefix result.append("{}{}".format("" if not prefix else (prefix + " "), directive)) return "\n".join(result) def get_submit_args(self, case, job): ''' return a list of touples (flag, name) ''' submitargs = " " bs_nodes = self.get_children("batch_system") submit_arg_nodes = []
def get_mpirun(self, case, attribs, check_members=None, job="case.run"): """ Find best match, return (executable, {arg_name : text}) """ mpirun_nodes = self.get_nodes("mpirun") best_match = None best_num_matched = -1 default_match = None best_num_matched_default = -1 args = {} for mpirun_node in mpirun_nodes: xml_attribs = mpirun_node.attrib all_match = True matches = 0 is_default = False for key, value in attribs.iteritems(): if key in xml_attribs: if xml_attribs[key].lower() == "false": xml_attrib = False elif xml_attribs[key].lower() == "true": xml_attrib = True else: xml_attrib = xml_attribs[key] if xml_attrib == value: matches += 1 elif key == "mpilib" and xml_attrib == "default": is_default = True else: all_match = False break for key in xml_attribs: expect(key in attribs, "Unhandled MPI property '%s'" % key) if all_match: if is_default: if matches > best_num_matched_default: default_match = mpirun_node best_num_matched_default = matches else: if matches > best_num_matched: best_match = mpirun_node best_num_matched = matches expect(best_match is not None or default_match is not None, "Could not find a matching MPI for attributes: %s" % attribs) the_match = best_match if best_match is not None else default_match # Now that we know the best match, compute the arguments arg_node = self.get_optional_node("arguments", root=the_match) if arg_node is not None: arg_nodes = self.get_nodes("arg", root=arg_node) for arg_node in arg_nodes: arg_value = transform_vars(arg_node.text, case=case, subgroup=job, check_members=check_members, default=arg_node.get("default")) args[arg_node.get("name")] = arg_value exec_node = self.get_node("executable", root=the_match) expect(exec_node is not None,"No executable found") executable = exec_node.text return executable, args
def get_mpirun(self, case, attribs, job="case.run", exe_only=False): """ Find best match, return (executable, {arg_name : text}) """ mpirun_nodes = self.get_nodes("mpirun") best_match = None best_num_matched = -1 default_match = None best_num_matched_default = -1 args = [] for mpirun_node in mpirun_nodes: xml_attribs = mpirun_node.attrib all_match = True matches = 0 is_default = False for key, value in attribs.items(): expect(key in self._allowed_mpi_attributes, "Unexpected key {} in mpirun attributes".format(key)) if key in xml_attribs: if xml_attribs[key].lower() == "false": xml_attrib = False elif xml_attribs[key].lower() == "true": xml_attrib = True else: xml_attrib = xml_attribs[key] if xml_attrib == value: matches += 1 elif key == "mpilib" and value != "mpi-serial" and xml_attrib == "default": is_default = True else: all_match = False break if all_match: if is_default: if matches > best_num_matched_default: default_match = mpirun_node best_num_matched_default = matches else: if matches > best_num_matched: best_match = mpirun_node best_num_matched = matches # if there are no special arguments required for mpi-serial it need not have an entry in config_machines.xml if "mpilib" in attribs and attribs["mpilib"] == "mpi-serial" and best_match is None: return "",[] expect(best_match is not None or default_match is not None, "Could not find a matching MPI for attributes: {}".format(attribs)) the_match = best_match if best_match is not None else default_match # Now that we know the best match, compute the arguments if not exe_only: arg_node = self.get_optional_node("arguments", root=the_match) if arg_node is not None: arg_nodes = self.get_nodes("arg", root=arg_node) for arg_node in arg_nodes: arg_value = transform_vars(arg_node.text, case=case, subgroup=job, default=arg_node.get("default")) args.append(arg_value) exec_node = self.get_node("executable", root=the_match) expect(exec_node is not None,"No executable found") executable = exec_node.text return executable, args