def batch_resources(self, spec, **kwargs):
if kwargs:
spec = dict(spec, **kwargs)
space = self.indent_text
sio = StringIO()
if not isinstance(spec, JobResourceSpec):
spec = JobResourceSpec(spec)
result = ''
if spec[0].get('walltime', ''):
dt = tools.to_timedelta(spec[0]['walltime'])
dt = dt.total_seconds()
hours = int(dt // 3600)
minutes = int((dt % 3600) // 60)
seconds = int(math.floor(dt % 60))
sio.write(f'#BSUB -W {hours}:{minutes:02d}\n')
# Handle memory.
if spec[0].is_exclusive() and spec[0].get('batch_memory', ''):
bytes = tools.memory_in_bytes(spec[0]['batch_memory'])
elif not spec[0].is_exclusive() and spec[0].get('compute_memory', ''):
bytes = tools.memory_in_bytes(spec[0]['compute_memory'])
elif spec[0].get('memory', ''):
bytes = tools.memory_in_bytes(spec[0]['memory'])
else:
bytes = 2000 * 1048576.
megabytes = int(math.ceil(bytes / 1048576.))
sio.write(f'#BSUB -R rusage[mem={megabytes:d}]\n')
if spec[0].get('outerr', ''):
sio.write(f'#BSUB -o {spec[0]["outerr"]}\n')
else:
if spec[0].get('stdout', ''):
sio.write('#BSUB -o {spec[0]["stdout"]}\n')
if spec[0].get('stderr', ''):
sio.write('#BSUB -e {spec[0]["stderr"]}\n')
# --------------------------------------------------------------
# With LSF+ALPS on WCOSS Cray, to my knowledge, you can only
# request one node size for all ranks. This code calculates
# the largest node size required (hyperthreading vs. non)
requested_nodes = 1
nodesize = max([self.nodes.node_size(r) for r in spec])
if spec[0].is_exclusive() is False:
# Shared program. This requires a different batch card syntax
nranks = max(1, spec.total_ranks())
sio.write(f'#BSUB -n {nranks}\n')
else:
if not spec.is_pure_serial() and not spec.is_pure_openmp():
# This is an MPI program.
nodes_ranks = self.nodes.to_nodes_ppn(spec)
requested_nodes = sum([n for n, p in nodes_ranks])
sio.write('#BSUB -extsched CRAYLINUX[]\n')
if self.settings.get('use_export_nodes', True):
sio.write(f'export NODES={requested_nodes}')
else:
sio.write("#BSUB -R '1*{select[craylinux && !vnode]} + ")
sio.write('%d' % requested_nodes)
sio.write("*{select[craylinux && vnode]span[")
sio.write(f"ptile={nodesize}] cu[type=cabinet]}}'")
ret = sio.getvalue()
sio.close()
return ret
def batch_resources(self, spec, **kwargs):
if kwargs:
spec = dict(spec, **kwargs)
space = self.indent_text
sio = StringIO()
if not isinstance(spec, JobResourceSpec):
spec = JobResourceSpec(spec)
result = ''
if spec[0].get('walltime', ''):
dt = tools.to_timedelta(spec[0]['walltime'])
dt = dt.total_seconds()
hours = int(dt // 3600)
minutes = int((dt % 3600) // 60)
seconds = int(math.floor(dt % 60))
sio.write(f'#BSUB -W {hours}:{minutes:02d}\n')
# ------------------------------------------------------------
# Handle memory
rusage = ''
if self.specify_memory:
bytes = self._decide_memory_for(spec)
megabytes = int(math.ceil(bytes / 1048576.))
rusage = f'rusage[mem={megabytes:d}]'
# ------------------------------------------------------------
# stdout/stderr locations
self._batch_stdout_stderr(spec[0], sio)
# ------------------------------------------------------------
# ranks, affinity, and span
if spec[0].exclusive:
sio.write('#BSUB -x\n')
if len(spec) == 1:
# Special case: only one block. We'll put the affinity
# and span on their own line and use "-n" to specify the
# number of ranks.
# There are some specialer cases in here, including pure
# OpenMP or pure serial.
ras = self._ranks_affinity_and_span_for(spec)
if rusage:
sio.write(f"""#BSUB -R '{rusage}'\n""")
# Affinity is mandatory:
sio.write(f"""#BSUB -R 'affinity[{ras[0]["affinity"]}]'\n""")
# Span is only used when OpenMP or MPI are in use:
if not spec.is_pure_serial():
sio.write(f"""#BSUB -R 'span[{ras[0]["span"]}]'\n""")
# -n is used except in shared, non-mpi jobs
if spec[0].exclusive or spec.total_ranks() > 2:
sio.write(f"""#BSUB -n {ras[0]["ranks"]}\n""")
elif not self.use_task_geometry:
# General case: more than one block. Task geometry is
# disabled.
hyperthreads = max([self.nodes.hyperthreads_for(r) for r in spec])
node_size = self.nodes.cores_per_node
if hyperthreads > 1:
node_size *= self.nodes.cpus_per_core
max_ppn = min([self.nodes.max_ranks_per_node(r) for r in spec])
affinity_count = node_size // max_ppn
affinity_type = 'cpu' if hyperthreads > 1 else 'core'
affinity = f'{affinity_type}({affinity_count})'
if rusage:
sio.write(f"""#BSUB -R '{rusage}'\n""")
sio.write(f"""#BSUB -R 'affinity[{affinity}]'\n""")
sio.write(
f"""#BSUB -R 'span[{min(spec.total_ranks(),max_ppn)}]'\n""")
sio.write(f"""#BSUB -n {spec.total_ranks()}\n""")
else:
# General case: more than one block. Task geometry is
# enabled.
ras = self._ranks_affinity_and_span_for(spec)
sio.write("#BSUB -R '")
first = True
for ras1 in ras:
if first:
first = False
else:
sio.write(' + ')
sio.write(f'{ras1["ranks"]}*'
f'{{span[{ras1["span"]}]'
f'affinity[{ras1["affinity"]}]{rusage}}}')
sio.write("'\n")
ret = sio.getvalue()
sio.close()
return ret
def rocoto_resources(self, spec, indent=0):
sio = StringIO()
space = self.indent_text
if not isinstance(spec, JobResourceSpec):
spec = JobResourceSpec(spec)
if spec[0].get('walltime', ''):
dt = tools.to_timedelta(spec[0]['walltime'])
dt = dt.total_seconds()
hours = int(dt // 3600)
minutes = int((dt % 3600) // 60)
seconds = int(math.floor(dt % 60))
sio.write(
f'{indent*space}<walltime>{hours}:{minutes:02d}:{seconds:02d}</walltime>\n'
)
# Handle memory.
if spec[0].is_exclusive() and spec[0].get('batch_memory', ''):
bytes = tools.memory_in_bytes(spec[0]['batch_memory'])
elif not spec[0].is_exclusive() and spec[0].get('compute_memory', ''):
bytes = tools.memory_in_bytes(spec[0]['compute_memory'])
elif spec[0].get('memory', ''):
bytes = tools.memory_in_bytes(spec[0]['memory'])
else:
bytes = 2000 * 1048576.
megabytes = int(math.ceil(bytes / 1048576.))
sio.write(f'{indent*space}<memory>{megabytes:d}M</memory>\n')
if 'outerr' in spec:
sio.write(f'{indent*space}<join>{spec["outerr"]}</join>\n')
else:
if 'stdout' in spec:
sio.write('{indent*space}<stdout>{spec["stdout"]}</stdout>\n')
if 'stderr' in spec:
sio.write('{indent*space}<stderr>{spec["stderr"]}</stderr>\n')
nodesize = max([self.nodes.node_size(r) for r in spec])
requested_nodes = 1
if spec[0].is_exclusive() is False:
# Shared program. This requires a different batch card syntax
nranks = max(1, spec.total_ranks())
sio.write(
f'{indent*space}<cores>{max(1,spec.total_ranks())}</cores>\n'
f'{indent*space}<shared></shared>\n')
else:
if not spec.is_pure_serial() and not spec.is_pure_openmp():
# This is an MPI program.
nodes_ranks = self.nodes.to_nodes_ppn(spec)
requested_nodes = sum([n for n, p in nodes_ranks])
nodes_ranks = self.nodes.to_nodes_ppn(
spec, can_merge_ranks=lambda x, y: False)
sio.write(indent*space+'<nodes>' \
+ '+'.join([f'{max(n,1)}:ppn={max(p,1)}' for n,p in nodes_ranks ]) \
+ '</nodes>\n')
#sio.write(f'{indent*space}<nodes>{requested_nodes}:ppn={nodesize}</nodes>')
ret = sio.getvalue()
sio.close()
return ret