def DataFrameFromBundle(bundle):
"""Generates a DataFrame from a datreant bundle."""
df = pd.DataFrame(
columns=["module", "nodes", "ns/day", "run time [min]", "gpu", "host", "ncores"]
)
for i, sim in enumerate(bundle):
# older versions wrote a version category. This ensures backwards compatibility
if "module" in sim.categories:
module = sim.categories["module"]
else:
module = sim.categories["version"]
# call the engine specific analysis functions
engine = detect_md_engine(module)
df.loc[i] = utils.analyze_run(engine=engine, sim=sim)
if df.empty:
console.error("There is no data for the given path.")
# Sort values by `nodes`
df = df.sort_values(
["host", "module", "run time [min]", "gpu", "nodes"]
).reset_index(drop=True)
return df
def ensure_correct_environment():
old_packages = []
import datreant as dtr
try:
version = dtr.__version__ # noqa: F401,F841
except AttributeError:
old_packages.append("datreant.core")
try:
import datreant.data # noqa: F401
old_packages.append("datreant.data")
except ImportError:
pass
try:
import mdsynthesis # noqa: F401
old_packages.append("mdsynthesis")
except ImportError:
pass
if old_packages:
console.error(MIGRATION_WARNING, ", ".join(old_packages),
DOCUMENTATION_URL)
def prepare_benchmark(name, relative_path, *args, **kwargs):
benchmark = kwargs["benchmark"]
if not kwargs["multidir"] == 1:
console.error("The NAMD-engine currently only supports '--multidir 1'")
if name.endswith(".namd"):
name = name[:-5]
namd = "{}.namd".format(name)
psf = "{}.psf".format(name)
pdb = "{}.pdb".format(name)
namd_relpath = os.path.join(relative_path, namd)
psf_relpath = os.path.join(relative_path, psf)
pdb_relpath = os.path.join(relative_path, pdb)
with open(namd_relpath) as fh:
analyze_namd_file(fh)
fh.seek(0)
copyfile(namd_relpath, benchmark[namd].relpath)
copyfile(psf_relpath, benchmark[psf].relpath)
copyfile(pdb_relpath, benchmark[pdb].relpath)
return name
def parse_bundle(bundle, columns, sort_values_by, discard_performance=False):
"""Generates a DataFrame from a datreant.Bundle."""
data = []
with click.progressbar(
bundle, length=len(bundle), label="Analyzing benchmarks", show_pos=True
) as bar:
for treant in bar:
module = treant.categories["module"]
engine = detect_md_engine(module)
row = utils.analyze_benchmark(engine=engine, benchmark=treant)
version = 2
if "version" in treant.categories:
version = 3
if version == 2:
row.pop() # multidir is not a category for version 2 data
row += [version]
if discard_performance:
row = row[:2] + row[3:]
data.append(row)
df = pd.DataFrame(data, columns=columns)
# Exit if no data is available
if df.empty:
console.error("There is no data for the given path.")
# Sort values by `nodes`
df = df.sort_values(sort_values_by).reset_index(drop=True)
return df
def get_batch_command():
for p in PATHS:
for b in BATCH_SYSTEMS.values():
if glob(os.path.join(p, b)):
return b
console.error(
"Was not able to find a batch system. Are you trying to use this "
"package on a host with a queuing system?")
def filter_dataframe_for_plotting(df, host_name, module_name, gpu, cpu):
if gpu and cpu:
console.info("Plotting GPU and CPU data.")
elif gpu and not cpu:
df = df[df.gpu]
console.info("Plotting GPU data only.")
elif cpu and not gpu:
df = df[~df.gpu]
console.info("Plotting CPU data only.")
elif not cpu and not gpu:
console.error("CPU and GPU not set. Nothing to plot. Exiting.")
if df.empty:
console.error("Your filtering led to an empty dataset. Exiting.")
df_filtered_hosts = df[df["host"].isin(host_name)]
df_unique_hosts = np.unique(df_filtered_hosts["host"])
if df_unique_hosts.size != len(host_name):
console.error(
"Could not find all provided hosts. Available hosts are: {}".format(
", ".join(np.unique(df["host"]))
)
)
if not host_name:
console.info("Plotting all hosts in input file.")
else:
df = df_filtered_hosts
console.info(
"Data for the following hosts will be plotted: {}".format(
", ".join(df_unique_hosts)
)
)
for module in module_name:
if module in SUPPORTED_ENGINES.keys():
console.info("Plotting all modules for engine '{}'.", module)
elif module in df["module"].tolist():
console.info("Plotting module '{}'.", module)
elif module not in df["module"].tolist():
console.error(
"The module '{}' does not exist in your data. Exiting.", module
)
if not module_name:
console.info("Plotting all modules in your input data.")
if module_name:
df = df[df["module"].str.contains("|".join(module_name))]
if df.empty:
console.error(
"Your selections contained no benchmarking information. "
"Are you sure all your selections are correct?"
)
return df
def test_console_error():
"""Test the output of console.error()."""
fh = StringIO()
with pytest.raises(SystemExit) as error:
console.error("Does not compute.", filehandler=fh)
assert fh.getvalue() == "ERROR Does not compute.\n"
assert error.type == SystemExit
assert error.value.code == 1
def filter_dataframe_for_plotting(df, host_name, module_name, gpu, cpu):
# gpu/cpu can be plotted together or separately
if gpu and cpu:
# if no flags are given by the user or both are set everything is plotted
console.info("Plotting GPU and CPU data.")
elif gpu and not cpu:
df = df[df.gpu]
console.info("Plotting GPU data only.")
elif cpu and not gpu:
df = df[~df.gpu]
console.info("Plotting CPU data only.")
elif not cpu and not gpu:
console.error("CPU and GPU not set. Nothing to plot. Exiting.")
if df.empty:
console.error("Your filtering led to an empty dataset. Exiting.")
df_filtered_hosts = df[df["host"].isin(host_name)]
df_unique_hosts = np.unique(df_filtered_hosts["host"])
if df_unique_hosts.size != len(host_name):
console.error(
"Could not find all provided hosts. Available hosts are: {}".
format(", ".join(np.unique(df["host"]))))
if not host_name:
console.info("Plotting all hosts in input file.")
else:
df = df_filtered_hosts
console.info("Data for the following hosts will be plotted: {}".format(
", ".join(df_unique_hosts)))
for module in module_name:
if module in ["gromacs", "namd"]:
console.info("Plotting all modules for engine '{}'.", module)
elif module in df["module"].tolist():
console.info("Plotting module '{}'.", module)
elif module not in df["module"].tolist():
console.error(
"The module '{}' does not exist in your data. Exiting.",
module)
if not module_name:
console.info("Plotting all modules in your input data.")
# this should work but we need to check before whether any of the entered
# names are faulty/don't exist
if module_name:
df = df[df["module"].str.contains("|".join(module_name))]
if df.empty:
console.error("Your selections contained no benchmarking information. "
"Are you sure all your selections are correct?")
return df
def check_input_file_exists(name):
"""Check if the TPR file exists."""
fn = name
if fn.endswith(".tpr"):
fn = name[:-4]
tpr = fn + ".tpr"
if not os.path.exists(tpr):
console.error(
"File {} does not exist, but is needed for GROMACS benchmarks.",
tpr)
return True
def check_input_file_exists(name):
"""Check and append the correct file extensions for the NAMD module."""
# Check whether the needed files are there.
for extension in ["namd", "psf", "pdb"]:
if name.endswith(".{}".format(extension)):
name = name[:-1 - len(extension)]
fn = "{}.{}".format(name, extension)
if not os.path.exists(fn):
console.error(
"File {} does not exist, but is needed for NAMD benchmarks.",
fn)
return True
def do_analyze(directory, save_csv):
"""Analyze benchmarks."""
bundle = dtr.discover(directory)
version = VersionFactory(categories=bundle.categories).version_class
df = parse_bundle(
bundle,
columns=version.analyze_categories,
sort_values_by=version.analyze_sort,
)
# Remove the versions column from the DataFrame
columns_to_drop = ["version"]
df = df.drop(columns=columns_to_drop)
if save_csv is not None:
if not save_csv.endswith(".csv"):
save_csv = "{}.csv".format(save_csv)
df.to_csv(save_csv, index=False)
console.success("Successfully benchmark data to {}.", save_csv)
# Reformat NaN values nicely into question marks.
# move this to the bundle function!
df = df.replace(np.nan, "?")
if df.isnull().values.any():
console.warn(
"We were not able to gather informations for all systems. "
"Systems marked with question marks have either crashed or "
"were not started yet.")
# Warn user that we are going to print more than 50 benchmark results to the console
if df.shape[0] > 50:
if click.confirm(
"We are about to print the results of {} benchmarks to the console. Continue?"
.format(click.style(str(df.shape[0]), bold=True))):
pass
else:
console.error("Exiting.")
# Print the data to the console
print_dataframe(
df,
columns=map_columns(version.category_mapping,
version.analyze_printing),
)
def prepare_module_name(module, skip_validation=False):
"""Split the provided module name into its base MD engine and version.
Currently we only try to split via the delimiter `/`, but this could be
changed upon request or made configurable on a per-host basis.
"""
try:
basename, version = module.split("/")
except (ValueError, AttributeError):
if skip_validation:
console.error(
"Although you are using the {} option, you have to provide a valid "
"MD engine name, e.g., {} or {}.",
"--skip-validation",
"gromacs/dummy",
"namd/dummy",
)
console.error("We were not able to determine the module name.")
return basename, version
def do_submit(directory, force_restart, yes):
"""Submit the benchmarks."""
# Migrate from MDBenchmark<2 to MDBenchmark=>2
mds_to_dtr.migrate_to_datreant(directory)
bundle = dtr.discover(directory)
# Exit if no bundles were found in the current directory.
if not bundle:
console.error("No benchmarks found.")
grouped_bundles = bundle.categories.groupby("started")
try:
bundles_not_yet_started = grouped_bundles[False]
except KeyError:
bundles_not_yet_started = None
if not bundles_not_yet_started and not force_restart:
console.error(
"All generated benchmarks were already started once. "
"You can force a restart with {}.",
"--force",
)
# Start all benchmark simulations if a restart was requested. Otherwise
# only start the ones that were not run yet.
bundles_to_start = bundle
if not force_restart:
bundles_to_start = bundles_not_yet_started
df = DataFrameFromBundle(bundles_to_start)
# Reformat NaN values nicely into question marks.
df_to_print = df.replace(np.nan, "?")
df_to_print = df.drop(columns=["ns/day", "ncores"])
console.info("{}", "Benchmark Summary:")
df_short = ConsolidateDataFrame(df_to_print)
PrintDataFrame(df_short)
# Ask the user to confirm whether they want to submit the benchmarks
if yes:
console.info("The above benchmarks will be submitted.")
elif not click.confirm("The above benchmarks will be submitted. Continue?"):
console.error("Exiting. No benchmarks submitted.")
batch_cmd = get_batch_command()
console.info("Submitting a total of {} benchmarks.", len(bundles_to_start))
for sim in bundles_to_start:
# Remove files generated by previous mdbenchmark run
if force_restart:
engine = detect_md_engine(sim.categories["module"])
cleanup_before_restart(engine=engine, sim=sim)
sim.categories["started"] = True
os.chdir(sim.abspath)
subprocess.call([batch_cmd, "bench.job"])
console.info(
"Submitted all benchmarks. Run {} once they are finished to get the results.",
"mdbenchmark analyze",
)
def analyze_namd_file(fh):
""" Check whether the NAMD config file has any relative imports or variables
"""
lines = fh.readlines()
for line in lines:
# Continue if we do not need to do anything with the current line
if (("parameters" not in line) and ("coordinates" not in line)
and ("structure" not in line)):
continue
path = line.split()[1]
if "$" in path:
console.error(
"Variable Substitutions are not allowed in NAMD files!")
if ".." in path:
console.error("Relative file paths are not allowed in NAMD files!")
if "/" not in path or ("/" in path and not path.startswith("/")):
console.error("No absolute path detected in NAMD file!")
def do_plot(
csv,
output_name,
output_format,
template,
module,
gpu,
cpu,
plot_cores,
fit,
font_size,
dpi,
xtick_step,
watermark,
):
"""Creates plots of benchmarks."""
if not csv:
raise click.BadParameter(
"You must specify at least one CSV file.", param_hint='"--csv"'
)
df = pd.concat([pd.read_csv(c) for c in csv])
performance_column = "performance" if "performance" in df.columns else "ns/day"
df = filter_dataframe_for_plotting(df, template, module, gpu, cpu)
# Exit if there is no performance data
if df[performance_column].isnull().all():
console.error("There is no performance data to plot.")
rcParams["font.size"] = font_size
fig = Figure()
FigureCanvas(fig)
ax = fig.add_subplot(111)
ax = plot_over_group(
df=df,
plot_cores=plot_cores,
fit=fit,
performance_column=performance_column,
ax=ax,
)
# Update xticks
selection = "ncores" if plot_cores else "nodes"
min_x = df[selection].min() if plot_cores else 1
max_x = df[selection].max()
xticks_steps = min_x
xticks = np.arange(min_x, max_x + min_x, xticks_steps)
step = get_xsteps(xticks.size, min_x, plot_cores, xtick_step)
ax.set_xticks(xticks[::step])
xdiff = min_x * 0.5 * step
ax.set_xlim(min_x - xdiff, max_x + xdiff)
# Update yticks
max_y = df[performance_column].max() or 50
yticks_steps = int(((max_y + 1) // 10))
if yticks_steps == 0:
yticks_steps = 1
yticks = np.arange(0, max_y + (max_y * 0.25), yticks_steps)
ax.set_yticks(yticks)
ax.set_ylim(0, max_y + (max_y * 0.25))
# Add watermark
if watermark:
ax.text(0.025, 0.925, "MDBenchmark", transform=ax.transAxes, alpha=0.3)
legend = ax.legend(loc="upper center", bbox_to_anchor=(0.5, -0.175))
plt.tight_layout()
if output_name is None and len(csv) == 1:
csv_string = csv[0].split(".")[0]
output_name = "{}.{}".format(csv_string, output_format)
elif output_name is None and len(csv) != 1:
output_name = generate_output_name(output_format)
elif not output_name.endswith(".{}".format(output_format)):
output_name = "{}.{}".format(output_name, output_format)
fig.savefig(
output_name,
type=output_format,
bbox_extra_artists=(legend,),
bbox_inches="tight",
dpi=dpi,
)
console.info("The plot was saved as '{}'.", output_name)
def do_generate(
name,
cpu,
gpu,
module,
host,
min_nodes,
max_nodes,
time,
skip_validation,
job_name,
yes,
physical_cores,
logical_cores,
number_of_ranks,
enable_hyperthreading,
multidir,
):
"""Generate a bunch of benchmarks."""
# Instantiate the version we are going to use
benchmark_version = Version3Categories()
# Validate the CPU and GPU flags
validate_cpu_gpu_flags(cpu, gpu)
# Validate the number of nodes
validate_number_of_nodes(min_nodes=min_nodes, max_nodes=max_nodes)
if logical_cores < physical_cores:
console.error(
"The number of logical cores cannot be smaller than the number of physical cores."
)
if physical_cores and not logical_cores:
console.warn("Assuming logical_cores = 2 * physical_cores")
logical_cores = 2 * physical_cores
if physical_cores and logical_cores:
processor = Processor(physical_cores=physical_cores,
logical_cores=logical_cores)
else:
processor = Processor()
# Hyperthreading check
if enable_hyperthreading and not processor.supports_hyperthreading:
console.error(
"The processor of this machine does not support hyperthreading.")
if not number_of_ranks:
number_of_ranks = (processor.physical_cores, )
# Validate number of simulations
validate_number_of_simulations(multidir, min_nodes, max_nodes,
number_of_ranks)
# Grab the template name for the host. This should always work because
# click does the validation for us
template = utils.retrieve_host_template(host)
# Warn the user that NAMD support is still experimental.
if any(["namd" in m for m in module]):
console.warn(NAMD_WARNING, "--gpu")
# Stop if we cannot find any modules. If the user specified multiple
# modules, we will continue with only the valid ones.
modules = mdengines.normalize_modules(module, skip_validation)
if not modules:
console.error("No requested modules available!")
# Check if all needed files exist. Throw an error if they do not.
validate_required_files(name=name, modules=modules)
# Validate that we can use the number of ranks and threads.
# We can continue, if no ValueError is thrown
for ranks in number_of_ranks:
try:
processor.get_ranks_and_threads(
ranks, with_hyperthreading=enable_hyperthreading)
except ValueError as e:
console.error(e)
# Create all benchmark combinations and put them into a DataFrame
data = construct_generate_data(
name,
job_name,
modules,
host,
template,
cpu,
gpu,
time,
min_nodes,
max_nodes,
processor,
number_of_ranks,
enable_hyperthreading,
multidir,
)
df = pd.DataFrame(data, columns=benchmark_version.generate_categories)
# Consolidate the data by grouping on the number of nodes and print to the
# user as an overview.
consolidated_df = consolidate_dataframe(
df, columns=benchmark_version.consolidate_categories)
print_dataframe(
consolidated_df[benchmark_version.generate_printing],
columns=map_columns(
map_dict=benchmark_version.category_mapping,
columns=benchmark_version.generate_printing,
),
)
# Save the number of benchmarks for later printing
number_of_benchmarks = df.shape[0]
# Ask the user for confirmation to generate files.
# If the user defined `--yes`, we will skip the confirmation immediately.
if yes:
console.info(
"We will generate {} " +
"{benchmark}.".format(benchmark="benchmark" if number_of_benchmarks
== 1 else "benchmarks"),
number_of_benchmarks,
)
elif not click.confirm("We will generate {} benchmarks. Continue?".format(
number_of_benchmarks)):
console.error("Exiting. No benchmarks were generated.")
# Generate the benchmarks
with click.progressbar(
df.iterrows(),
length=number_of_benchmarks,
show_pos=True,
label="Generating benchmarks",
) as bar:
for _, row in bar:
relative_path, file_basename = os.path.split(row["name"])
mappings = benchmark_version.generate_mapping
kwargs = {"name": file_basename, "relative_path": relative_path}
for key, value in mappings.items():
kwargs[value] = row[key]
write_benchmark(**kwargs)
# Finish up by telling the user how to submit the benchmarks
console.info(
"Finished! You can submit the jobs with {}.",
"mdbenchmark submit",
)
def normalize_modules(modules, skip_validation):
"""Validate that the provided module names are available.
We first check whether the requested MD engine is supported by the package.
Next we try to discover all available modules on the host. If this is not
possible, or if the user has used the `--skip-validation` option, we skip
the check and notify the user.
If the user requested modules that were not found on the system, we inform
the user and show all modules for that corresponding MD engine that were
found.
"""
# Check if modules are from supported md engines
d = defaultdict(list)
for m in modules:
engine_name, version = prepare_module_name(m, skip_validation)
d[engine_name] = version
for engine_name in d.keys():
if detect_md_engine(engine_name) is None:
console.error(
"There is currently no support for '{}'. " +
"Supported MD engines are: {}.".format(", ".join(
sorted(SUPPORTED_ENGINES.keys()))),
engine_name,
)
if skip_validation:
console.warn("Not performing module name validation.")
return modules
available_modules = get_available_modules()
if available_modules is None:
console.warn(
"Cannot locate modules available on this host. Not performing module name validation."
)
return modules
good_modules = [
m for m in modules if validate_module_name(m, available_modules)
]
# Prepare to warn the user about missing modules
missing_modules = set(modules).difference(good_modules)
if missing_modules:
d = defaultdict(list)
for mm in sorted(missing_modules):
engine_name, version = mm.split("/")
d[engine_name].append(version)
err = "We have problems finding all of your requested modules on this host.\n"
args = []
for engine_name in sorted(d.keys()):
err += (
"We were not able to find the following modules for MD engine {}: {}.\n"
)
args.append(engine_name)
args.extend(d[engine_name])
# Show all available modules that we found for the requested MD engine
err += "Available modules are:\n{}\n"
args.extend([
"\n".join([
"{}/{}".format(engine_name, mde)
for mde in sorted(available_modules[engine_name])
])
])
console.warn(err, bold=True, *args)
return good_modules
def do_submit(directory, force_restart, yes):
"""Submit the benchmarks."""
bundle = dtr.discover(directory)
# Exit if no bundles were found in the current directory.
if not bundle:
console.error("No benchmarks found.")
grouped_bundles = bundle.categories.groupby("started")
try:
bundles_not_yet_started = grouped_bundles[False]
except KeyError:
bundles_not_yet_started = None
if not bundles_not_yet_started and not force_restart:
console.error(
"All generated benchmarks were already started once. "
"You can force a restart with {}.",
"--force",
)
# Start all benchmark simulations if a restart was requested. Otherwise
# only start the ones that were not run yet.
bundles_to_start = bundle
if not force_restart:
bundles_to_start = bundles_not_yet_started
benchmark_version = VersionFactory(
categories=bundles_to_start.categories).version_class
df = parse_bundle(
bundles_to_start,
columns=benchmark_version.submit_categories,
sort_values_by=benchmark_version.analyze_sort,
discard_performance=True,
)
# Reformat NaN values nicely into question marks.
df_to_print = df.replace(np.nan, "?")
columns_to_drop = ["ncores", "version"]
df_to_print = df.drop(columns=columns_to_drop)
# Consolidate the data by grouping on the number of nodes and print to the
# user as an overview.
consolidated_df = consolidate_dataframe(
df_to_print, columns=benchmark_version.consolidate_categories)
print_dataframe(
consolidated_df,
columns=map_columns(
map_dict=benchmark_version.category_mapping,
columns=benchmark_version.generate_printing[1:],
),
)
# Ask the user to confirm whether they want to submit the benchmarks
if yes:
console.info("The above benchmarks will be submitted.")
elif not click.confirm(
"The above benchmarks will be submitted. Continue?"):
console.error("Exiting. No benchmarks submitted.")
batch_cmd = get_batch_command()
console.info("Submitting a total of {} benchmarks.", len(bundles_to_start))
for sim in bundles_to_start:
# Remove files generated by previous mdbenchmark run
if force_restart:
engine = detect_md_engine(sim.categories["module"])
cleanup_before_restart(engine=engine, sim=sim)
sim.categories["started"] = True
os.chdir(sim.abspath)
subprocess.call([batch_cmd, "bench.job"])
console.info(
"Submitted all benchmarks. Run {} once they are finished to get the results.",
"mdbenchmark analyze",
)
def do_generate(
name,
cpu,
gpu,
module,
host,
min_nodes,
max_nodes,
time,
skip_validation,
job_name,
yes,
):
"""Generate a bunch of benchmarks."""
# Validate the CPU and GPU flags
validate_cpu_gpu_flags(cpu, gpu)
# Validate the number of nodes
validate_number_of_nodes(min_nodes=min_nodes, max_nodes=max_nodes)
# Grab the template name for the host. This should always work because
# click does the validation for us
template = utils.retrieve_host_template(host)
# Warn the user that NAMD support is still experimental.
if any(["namd" in m for m in module]):
console.warn(NAMD_WARNING, "--gpu")
module = mdengines.normalize_modules(module, skip_validation)
# If several modules were given and we only cannot find one of them, we
# continue.
if not module:
console.error("No requested modules available!")
df_overview = pd.DataFrame(columns=[
"name",
"job_name",
"base_directory",
"template",
"engine",
"module",
"nodes",
"run time [min]",
"gpu",
"host",
])
i = 1
for m in module:
# Here we detect the MD engine (supported: GROMACS and NAMD).
engine = mdengines.detect_md_engine(m)
# Check if all needed files exist. Throw an error if they do not.
engine.check_input_file_exists(name)
gpu_cpu = {"cpu": cpu, "gpu": gpu}
for pu, state in sorted(gpu_cpu.items()):
if not state:
continue
directory = "{}_{}".format(host, m)
gpu = False
gpu_string = ""
if pu == "gpu":
gpu = True
directory += "_gpu"
gpu_string = " with GPUs"
console.info("Creating benchmark system for {}.", m + gpu_string)
base_directory = dtr.Tree(directory)
for nodes in range(min_nodes, max_nodes + 1):
df_overview.loc[i] = [
name,
job_name,
base_directory,
template,
engine,
m,
nodes,
time,
gpu,
host,
]
i += 1
console.info("{}", "Benchmark Summary:")
df_short = ConsolidateDataFrame(df_overview)
PrintDataFrame(df_short)
if yes:
console.info("Generating the above benchmarks.")
elif not click.confirm(
"The above benchmarks will be generated. Continue?"):
console.error("Exiting. No benchmarks generated.")
for _, row in df_overview.iterrows():
relative_path, file_basename = os.path.split(row["name"])
write_benchmark(
engine=row["engine"],
base_directory=row["base_directory"],
template=row["template"],
nodes=row["nodes"],
gpu=row["gpu"],
module=row["module"],
name=file_basename,
relative_path=relative_path,
job_name=row["job_name"],
host=row["host"],
time=row["run time [min]"],
)
# Provide some output for the user
console.info(
"Finished generating all benchmarks.\n"
"You can now submit the jobs with {}.",
"mdbenchmark submit",
)
