def asynchronous_zfp():
sweep_parameters = [
p.ParamRunner('simulation', 'nprocs', [64]),
p.ParamCmdLineArg('simulation', 'settings', 1,
["settings-files.json"]),
p.ParamADIOS2XML('simulation', 'sim output engine', 'SimulationOutput',
'engine', [{
'BP4': {}
}]),
p.ParamRunner('pdf_calc', 'nprocs', [8]),
p.ParamCmdLineArg('pdf_calc', 'infile', 1, ['gs.bp']),
p.ParamCmdLineArg('pdf_calc', 'outfile', 2, ['pdf.bp']),
p.ParamCmdLineArg('pdf_calc', 'bins', 3, [100]),
p.ParamCmdLineArg('pdf_calc', 'write_orig_data', 4, ['YES']),
p.ParamADIOS2XML('pdf_calc', 'zfp compression', 'PDFAnalysisOutput',
'var_operation', [{
"U": {
"zfp": {
'accuracy': 0.001
}
}
}, {
"U": {
"zfp": {
'accuracy': 0.0001
}
}
}]),
]
sweep = p.Sweep(parameters=sweep_parameters,
rc_dependency={'pdf_calc': 'simulation'},
node_layout={'summit': node_layouts.separate_nodes()})
return sweep
def get_sweep_params(adios_engine):
# Setup the sweep parameters for a Sweep
return [
# ParamRunner 'nprocs' specifies the no. of ranks to be spawned
p.ParamRunner('simulation', 'nprocs', [1]),
# Create a ParamCmdLineArg parameter to specify a command line argument to run the application
p.ParamCmdLineArg('simulation', 'settings', 1, ["settings.json"]),
# Edit key-value pairs in the json file
# Sweep over two values for the F key in the json file. Alongwith 4 values for the nprocs property for
# the pdf_calc code, this Sweep will create 2*4 = 8 experiments.
p.ParamConfig('simulation', 'feed_rate_U', 'settings.json', 'F',
[0.01]),
p.ParamConfig('simulation', 'kill_rate_V', 'settings.json', 'k',
[0.048]),
# Setup an environment variable
# p.ParamEnvVar ('simulation', 'openmp', 'OMP_NUM_THREADS', [4]),
# Change the engine for the 'SimulationOutput' IO object in the adios xml file to SST for coupling.
# As both the applications use the same xml file, you need to do this just once.
p.ParamADIOS2XML('simulation', 'sim output engine', 'SimulationOutput',
'engine', [{
adios_engine: {}
}]),
# Now setup options for the pdf_calc application.
# Sweep over four values for the nprocs
p.ParamRunner('pdf_calc', 'nprocs', [1]),
p.ParamCmdLineArg('pdf_calc', 'infile', 1, ['gs.bp']),
p.ParamCmdLineArg('pdf_calc', 'outfile', 2, ['pdf']),
]
def insitu_analysis(sim_nprocs, analysis_nprocs, node_layout, adios_engine):
sweep_parameters = [
p.ParamRunner('simulation', 'nprocs', [sim_nprocs]),
p.ParamCmdLineOption('simulation', 'sim input', '-in',
["in.lj.nordf"]),
p.ParamADIOS2XML('simulation', 'sim output engine', 'custom', 'engine',
[{
adios_engine: {}
}]),
p.ParamRunner('rdf_calc', 'nprocs', [analysis_nprocs]),
p.ParamCmdLineOption('rdf_calc', 'input', '-in', ["in.lj.rdf.rerun"]),
p.ParamADIOS2XML('rdf_calc', 'analysis input engine', 'read_dump',
'engine', [{
adios_engine: {}
}]),
]
sweep = p.Sweep(parameters=sweep_parameters,
rc_dependency=None,
node_layout={'summit': node_layout})
return sweep
def create_experiment(writer_nprocs, reader_nprocs, config_file, adios_xml_file, engine, writer_decomposition, reader_decomposition, machine_name, node_layout):
"""
Creates a sweep object that tells Cheetah how to run the adios io test.
Assumes 1D decomposition.
"""
# print(adios_xml_file)
# print(engine)
params = [
# ParamRunner 'nprocs' specifies the no. of ranks to be spawned
p.ParamRunner ('writer', 'nprocs', [writer_nprocs]),
# Create a ParamCmdLineArg parameter to specify a command line argument to run the application
p.ParamCmdLineOption ('writer', 'app', '-a', [1]),
p.ParamCmdLineOption ('writer', 'app-config', '-c', [config_file]),
p.ParamCmdLineOption ('writer', 'adios-config', '-x', [adios_xml_file]),
p.ParamCmdLineOption ('writer', 'strongscaling', '-w', [None]),
p.ParamCmdLineOption ('writer', 'timing', '-t', [None]),
p.ParamCmdLineOption ('writer', 'decomposition', '-d', [writer_decomposition]),
# Change the engine for the 'SimulationOutput' IO object in the adios xml file
p.ParamADIOS2XML ('writer', 'dump_trajectory', 'trj_dump_out', 'engine', [engine]),
# Sweep over four values for the nprocs
p.ParamRunner ('reader', 'nprocs', [reader_nprocs]),
p.ParamCmdLineOption ('reader', 'app', '-a', [2]),
p.ParamCmdLineOption ('reader', 'app-config', '-c', [config_file]),
p.ParamCmdLineOption ('reader', 'adios-config', '-x', [adios_xml_file]),
p.ParamCmdLineOption ('reader', 'weakscaling', '-s', [None]),
p.ParamCmdLineOption ('reader', 'timing', '-t', [None]),
p.ParamCmdLineOption ('reader', 'decomposition', '-d', [reader_decomposition]),
# Change the engine for the 'SimulationOutput' IO object in the adios xml file
p.ParamADIOS2XML ('reader', 'load_trajectory', 'trj_dump_in', 'engine', [engine]),
]
sweep = p.Sweep(parameters=params)
if node_layout:
sweep.node_layout = {machine_name: node_layout}
return sweep
def create_experiment(writer_nprocs, reader_nprocs, trj_engine, sorted_trj_engine, machine_name, node_layout):
"""
Creates a sweep object that tells Cheetah how to run the adios io test.
Assumes 1D decomposition.
"""
# print(adios_xml_file)
# print(engine)
params = [
# ParamRunner 'nprocs' specifies the no. of ranks to be spawned
p.ParamRunner ('writer', 'nprocs', [writer_nprocs]),
# Create a ParamCmdLineArg parameter to specify a command line argument to run the application
p.ParamCmdLineArg ('writer', 'config', 1, ['copro.nw']),
# Change the engine for the 'SimulationOutput' IO object in the adios xml file
p.ParamADIOS2XML ('writer', 'trajectory', 'trj', 'engine', [trj_engine]),
# Sweep over four values for the nprocs
p.ParamRunner ('reader', 'nprocs', [reader_nprocs]),
p.ParamCmdLineArg ('reader', 'input_md', 1, ['copro_md']),
p.ParamCmdLineArg ('reader', 'verbose', 2, [1]),
# Change the engine for the 'SimulationOutput' IO object in the adios xml file
p.ParamADIOS2XML ('reader', 'sorted_trj', 'SortingOutput', 'engine', [sorted_trj_engine]),
p.ParamRunner ('analyzer', 'nprocs', [1]),
p.ParamCmdLineArg ('analyzer', 'script', 1, ['pca3d.R']),
p.ParamCmdLineArg ('analyzer', 'window', 2, [100]),
p.ParamCmdLineArg ('analyzer', 'stride', 3, [10]),
p.ParamCmdLineArg ('analyzer', 'k', 4, [5]),
p.ParamCmdLineArg ('analyzer', 'sorted_trj', 5, ['copro_md_trj.bp']),
p.ParamCmdLineArg ('analyzer', 'xml', 6, ['adios2.xml']),
p.ParamCmdLineArg ('analyzer', 'mcCore', 7, [1]),
p.ParamCmdLineArg ('analyzer', 'output', 8, ['pairs.pdf']),
]
sweep = p.Sweep(parameters=params)
if node_layout:
sweep.node_layout = {machine_name: node_layout}
return sweep
def posthoc_analysis(sim_nprocs, analysis_nprocs):
sweep_parameters = [
p.ParamRunner('simulation', 'nprocs', [sim_nprocs]),
p.ParamCmdLineOption('simulation', 'sim input', '-in',
["in.lj.nordf"]),
p.ParamADIOS2XML('simulation', 'sim output engine', 'custom', 'engine',
[{
'BP4': {}
}]),
p.ParamRunner('rdf_calc', 'nprocs', [analysis_nprocs]),
p.ParamCmdLineOption('rdf_calc', 'input', '-in', ["in.lj.rdf.rerun"]),
]
sweep = p.Sweep(parameters=sweep_parameters,
rc_dependency={'rdf_calc': 'simulation'},
node_layout={'summit': node_layouts.separate_nodes()})
return sweep
def posthoc_analysis():
sweep_parameters = [
p.ParamRunner('simulation', 'nprocs', [4]),
p.ParamCmdLineArg('simulation', 'settings', 1,
["settings-files.json"]),
p.ParamADIOS2XML('simulation', 'sim output engine', 'SimulationOutput',
'engine', [{
'BP4': {}
}]),
p.ParamRunner('pdf_calc', 'nprocs', [4]),
p.ParamCmdLineArg('pdf_calc', 'infile', 1, ['gs.bp']),
p.ParamCmdLineArg('pdf_calc', 'outfile', 2, ['pdf.bp']),
]
sweep = p.Sweep(parameters=sweep_parameters,
rc_dependency={'pdf_calc': 'simulation'},
node_layout={'summit': node_layouts.separate_nodes()})
return sweep
def insitu_analysis(node_layout):
sweep_parameters = [
p.ParamRunner('simulation', 'nprocs', [64]),
p.ParamCmdLineArg('simulation', 'settings', 1,
["settings-staging.json"]),
p.ParamADIOS2XML('simulation', 'sim output engine', 'SimulationOutput',
'engine', [{
'SST': {}
}]),
p.ParamRunner('pdf_calc', 'nprocs', [4, 8, 16, 32]),
p.ParamCmdLineArg('pdf_calc', 'infile', 1, ['gs.bp']),
p.ParamCmdLineArg('pdf_calc', 'outfile', 2, ['pdf.bp']),
]
sweep = p.Sweep(parameters=sweep_parameters,
rc_dependency=None,
node_layout={'summit': node_layout})
return sweep
class GrayScott(Campaign):
# A name for the campaign
name = "gray_scott"
# Define your workflow. Setup the applications that form the workflow.
# exe may be an absolute path.
# The adios xml file is automatically copied to the campaign directory.
# 'runner_override' may be used to launch the code on a login/service node as a serial code
# without a runner such as aprun/srun/jsrun etc.
codes = [
("simulation", dict(exe="gray-scott", adios_xml_file='adios2.xml')),
("pdf_calc",
dict(exe="pdf_calc",
adios_xml_file='adios2.xml',
runner_override=False)),
]
# List of machines on which this code can be run
supported_machines = ['local', 'titan', 'theta']
# Kill an experiment right away if any workflow components fail (just the experiment, not the whole group)
kill_on_partial_failure = True
# Any setup that you may need to do in an experiment directory before the experiment is run
run_dir_setup_script = None
# A post-process script that is run for every experiment after the experiment completes
run_post_process_script = None
# Directory permissions for the campaign sub-directories
umask = '027'
# Options for the underlying scheduler on the target system. Specify the project ID and job queue here.
scheduler_options = {
'theta': {
'project': 'CSC249ADCD01',
'queue': 'default'
}
}
# A way to setup your environment before the experiment runs. Export environment variables such as LD_LIBRARY_PATH here.
app_config_scripts = {'local': 'setup.sh', 'theta': 'env_setup.sh'}
# Setup the sweep parameters for a Sweep
sweep1_parameters = [
# ParamRunner 'nprocs' specifies the no. of ranks to be spawned
p.ParamRunner('simulation', 'nprocs', [512]),
# Create a ParamCmdLineArg parameter to specify a command line argument to run the application
p.ParamCmdLineArg('simulation', 'settings', 1, ["settings.json"]),
# Edit key-value pairs in the json file
# Sweep over two values for the F key in the json file. Alongwith 4 values for the nprocs property for
# the pdf_calc code, this Sweep will create 2*4 = 8 experiments.
p.ParamConfig('simulation', 'feed_rate_U', 'settings.json', 'F',
[0.01, 0.02]),
p.ParamConfig('simulation', 'kill_rate_V', 'settings.json', 'k',
[0.048]),
# Setup an environment variable
# p.ParamEnvVar ('simulation', 'openmp', 'OMP_NUM_THREADS', [4]),
# Change the engine for the 'SimulationOutput' IO object in the adios xml file to SST for coupling.
# As both the applications use the same xml file, you need to do this just once.
p.ParamADIOS2XML('simulation', 'SimulationOutput', 'engine', [{
'SST': {}
}]),
# Now setup options for the pdf_calc application.
# Sweep over four values for the nprocs
p.ParamRunner('pdf_calc', 'nprocs', [32, 64, 128, 256]),
p.ParamCmdLineArg('pdf_calc', 'infile', 1, ['gs.bp']),
p.ParamCmdLineArg('pdf_calc', 'outfile', 2, ['pdf']),
]
# Create a Sweep object. This one does not define a node-layout, and thus, all cores of a compute node will be
# utilized and mapped to application ranks.
sweep1 = p.Sweep(parameters=sweep1_parameters)
# Create another Sweep object and set its node-layout to spawn 16 simulation processes per node, and
# 4 processes of pdf_calc per node. On Theta, different executables reside on separate nodes as node-sharing
# is not permitted on Theta.
sweep2_parameters = copy.deepcopy(sweep1_parameters)
sweep2 = p.Sweep(
node_layout={'theta': [{
'simulation': 16
}, {
'pdf_calc': 4
}]},
parameters=sweep2_parameters)
# Create a SweepGroup and add the above Sweeps. Set batch job properties such as the no. of nodes,
sweepGroup1 = p.SweepGroup(
"sg-1", # A unique name for the SweepGroup
walltime=3600, # Total runtime for the SweepGroup
per_run_timeout=
600, # Timeout for each experiment
parameter_groups=[sweep1, sweep2], # Sweeps to include in this group
launch_mode='default', # Launch mode: default, or MPMD if supported
nodes=128, # No. of nodes for the batch job.
# rc_dependency={'pdf_calc':'simulation',}, # Specify dependencies between workflow components
run_repetitions=
2, # No. of times each experiment in the group must be repeated (Total no. of runs here will be 3)
)
# Activate the SweepGroup
sweeps = [sweepGroup1]
class ProducerConsumer(Campaign):
# A name for the campaign
name = "coupling-example"
# WORKFLOW SETUP
#---------------
# A list of the codes that will be part of the workflow
# If there is an adios xml file associated with the codes, list it here
# 'sleep_after' represents the time gap after which the next code is spawned
# Use runner_override to run the code without the default launcher (mpirun/aprun/jsrun etc.). This runs the
# code as a serial application
codes = [("producer",
dict(exe="producer.py",
adios_xml_file='adios2.xml',
sleep_after=5)),
("mean_calc",
dict(exe="mean_calculator.py",
adios_xml_file='adios2.xml',
runner_override=False))]
# CAMPAIGN SETTINGS
#------------------
# A list of machines that this campaign is supported on
supported_machines = ['local', 'titan', 'theta', 'summit']
# Option to kill an experiment (just one experiment, not the full sweep or campaign) if one of the codes fails
kill_on_partial_failure = True
# Some pre-processing in the experiment directory
# This is performed when the campaign directory is created (before the campaign is launched)
run_dir_setup_script = None
# A post-processing script to be run in the experiment directory after the experiment completes
# For example, removing some large files after the experiment is done
run_post_process_script = None
# umask applied to your directory in the campaign so that colleagues can view files
umask = '027'
# Scheduler information: job queue, account-id etc. Leave it to None if running on a local machine
scheduler_options = {
'theta': {
'project': '',
'queue': 'batch'
},
'summit': {
'project': 'CSC299'
}
}
# Setup your environment. Loading modules, setting the LD_LIBRARY_PATH etc.
# Ensure this script is executable
app_config_scripts = {'local': 'setup.sh', 'summit': 'env_setup.sh'}
# PARAMETER SWEEPS
#-----------------
# Setup how the workflow is run, and what values to 'sweep' over
# Use ParamCmdLineArg to setup a command line arg, ParamCmdLineOption to setup a command line option, and so on.
sweep1_parameters = [
p.ParamRunner('producer', 'nprocs', [128]),
p.ParamRunner('mean_calc', 'nprocs', [36]),
p.ParamCmdLineArg('producer', 'array_size_per_pe', 1,
[1024 * 1024]), # 1M, 2M, 10M
p.ParamCmdLineArg('producer', 'num_steps', 2, [10]),
p.ParamADIOS2XML('producer', 'staging', 'producer', 'engine', [{
"SST": {}
}]),
]
shared_node = SummitNode()
for i in range(18):
shared_node.cpu[i] = "producer:{}".format(math.floor(i / 6))
shared_node.cpu[i + 21] = "producer:{}".format(math.floor(
(i + 18) / 6))
for i in range(3):
shared_node.cpu[i + 18] = "mean_calc:0"
shared_node.cpu[i + 18 + 21] = "mean_calc:0"
for i in range(6):
shared_node.gpu[i] = ["producer:{}".format(i)]
shared_node_layout = [shared_node]
shared_node_1_per_rank = SummitNode()
for i in range(18):
shared_node_1_per_rank.cpu[i] = "producer:{}".format(i)
shared_node_1_per_rank.cpu[i + 21] = "producer:{}".format(i + 18)
for i in range(3):
shared_node_1_per_rank.cpu[i + 18] = "mean_calc:{}".format(i)
shared_node_1_per_rank.cpu[i + 18 + 21] = "mean_calc:{}".format(i + 3)
for i in range(6):
shared_node_1_per_rank.gpu[i] = ["producer:{}".format(i)]
shared_node_layout_2 = [shared_node_1_per_rank]
shared_node_shared_gpu = SummitNode()
for i in range(18):
shared_node_shared_gpu.cpu[i] = "producer:{}".format(math.floor(i / 6))
shared_node_shared_gpu.cpu[i + 21] = "producer:{}".format(
math.floor((i + 18) / 6))
for i in range(3):
shared_node_shared_gpu.cpu[i + 18] = "mean_calc:0"
shared_node_shared_gpu.cpu[i + 18 + 21] = "mean_calc:0"
shared_node_shared_gpu.gpu[0] = ["producer:0"]
shared_node_shared_gpu.gpu[1] = [
"producer:0",
"producer:1",
]
shared_node_shared_gpu.gpu[2] = ["producer:0", "producer:1", 'mean_calc:0']
shared_node_layout_3 = [shared_node_shared_gpu]
sep_node_producer = SummitNode()
sep_node_mean_calc = SummitNode()
for i in range(18):
sep_node_producer.cpu[i] = "producer:{}".format(math.floor(i / 6))
for i in range(3):
sep_node_mean_calc.cpu[i + 18] = "mean_calc:0"
sep_node_mean_calc.cpu[i + 18 + 21] = "mean_calc:0"
for i in range(3):
sep_node_producer.gpu[i] = ["producer:{}".format(i)]
sep_node_layout = [sep_node_producer, sep_node_mean_calc]
# Create a sweep
# node_layout represents no. of processes per node
sweep1 = p.Sweep(node_layout={'summit': shared_node_layout},
parameters=sweep1_parameters,
rc_dependency=None)
sweep2 = p.Sweep(node_layout={'summit': shared_node_layout_2},
parameters=sweep1_parameters,
rc_dependency=None)
sweep3 = p.Sweep(node_layout={'summit': shared_node_layout_3},
parameters=sweep1_parameters,
rc_dependency=None)
sweep4 = p.Sweep(node_layout={'summit': sep_node_layout},
parameters=sweep1_parameters,
rc_dependency=None)
# Create a sweep group from the above sweep. You can place multiple sweeps in the group.
# Each group is submitted as a separate job.
sweepGroup1 = p.SweepGroup(
"sg-1",
walltime=300,
per_run_timeout=60,
parameter_groups=[sweep1, sweep2, sweep3, sweep4],
launch_mode='default', # or MPMD
# optional:
# tau_profiling=True,
# tau_tracing=False,
# nodes=10,
# component_subdirs = True, <-- codes have their own separate workspace in the experiment directory
# component_inputs = {'producer': ['some_input_file'], 'norm_calc': [SymLink('some_large_file')] } <-- inputs required by codes
# max_procs = 64 <-- max no. of procs to run concurrently. depends on 'nodes'
)
# Sweep groups to be activated
sweeps = {'summit': [sweepGroup1]}
class NWChem(Campaign):
# A name for the campaign
name = "nwchem"
# Define your workflow. Setup the applications that form the workflow.
# exe may be an absolute path.
# The adios xml file is automatically copied to the campaign directory.
# 'runner_override' may be used to launch the code on a login/service node as a serial code
# without a runner such as aprun/srun/jsrun etc.
codes = [
("nwchem_main",
dict(
exe=
"/ccs/proj/e2e/pnorbert/ADIOS/ADIOS2/build.rhea.gcc/install/bin/adios2_iotest",
adios_xml_file='copro.xml')),
("sorting",
dict(
exe=
"/ccs/proj/e2e/pnorbert/ADIOS/ADIOS2/build.rhea.gcc/install/bin/adios2_iotest",
adios_xml_file='copro.xml',
runner_override=False)),
]
# List of machines on which this code can be run
supported_machines = ['local', 'titan', 'theta', 'rhea']
# Kill an experiment right away if any workflow components fail (just the experiment, not the whole group)
kill_on_partial_failure = True
# Any setup that you may need to do in an experiment directory before the experiment is run
run_dir_setup_script = None
# A post-process script that is run for every experiment after the experiment completes
run_post_process_script = None
# Directory permissions for the campaign sub-directories
umask = '027'
# Options for the underlying scheduler on the target system. Specify the project ID and job queue here.
# scheduler_options = {'theta': {'project':'CSC249ADCD01', 'queue': 'default'}}
scheduler_options = {'rhea': {'project': 'csc143'}}
# A way to setup your environment before the experiment runs. Export environment variables such as LD_LIBRARY_PATH here.
app_config_scripts = {
'local': 'setup.sh',
'theta': 'env_setup.sh',
'rhea': 'setup_nwchem_rhea.sh'
}
# Setup the sweep parameters for a Sweep
sweep1_parameters = [
# ParamRunner 'nprocs' specifies the no. of ranks to be spawned
p.ParamRunner('nwchem_main', 'nprocs', [80]),
# Create a ParamCmdLineArg parameter to specify a command line argument to run the application
p.ParamCmdLineOption('nwchem_main', 'app', '-a', [1]),
p.ParamCmdLineOption('nwchem_main', 'app-config', '-c',
['copro-80.txt']),
p.ParamCmdLineOption('nwchem_main', 'adios-config', '-x',
['copro.xml']),
p.ParamCmdLineOption('nwchem_main', 'strongscaling', '-w', [None]),
p.ParamCmdLineOption('nwchem_main', 'timing', '-t', [None]),
p.ParamCmdLineOption('nwchem_main', 'decomposition', '-d', [80]),
# Change the engine for the 'SimulationOutput' IO object in the adios xml file to SST for coupling.
p.ParamADIOS2XML('nwchem_main', 'dump_trajectory', 'trj_dump_out',
'engine', [{
'BP4': {
'OpenTimeoutSecs': '30.0'
}
}]),
# Now setup options for the pdf_calc application.
# Sweep over four values for the nprocs
p.ParamRunner('sorting', 'nprocs', [8]),
p.ParamCmdLineOption('sorting', 'app', '-a', [1]),
p.ParamCmdLineOption('sorting', 'app-config', '-c', ['copro-80.txt']),
p.ParamCmdLineOption('sorting', 'adios-config', '-x', ['copro.xml']),
p.ParamCmdLineOption('sorting', 'weakscaling', '-s', [None]),
p.ParamCmdLineOption('sorting', 'timing', '-t', [None]),
p.ParamCmdLineOption('sorting', 'decomposition', '-d', [8]),
# Change the engine for the 'SimulationOutput' IO object in the adios xml file to SST for coupling.
p.ParamADIOS2XML('sorting', 'load_trajectory', 'trj_dump_in', 'engine',
[{
'BP4': {
'OpenTimeoutSecs': '30.0'
}
}]),
]
#print(sweep1_parameters)
sweep2_parameters = sweep1_parameters.copy()
sweep2_parameters[7] = p.ParamADIOS2XML('nwchem_main', 'dump_trajectory',
'trj_dump_out', 'engine', [{
'SST': {}
}])
sweep2_parameters[15] = p.ParamADIOS2XML('sorting', 'load_trajectory',
'trj_dump_in', 'engine', [{
'SST': {}
}])
sweep3_parameters = sweep1_parameters.copy()
sweep3_parameters[7] = p.ParamADIOS2XML('nwchem_main', 'dump_trajectory',
'trj_dump_out', 'engine', [{
"Null": {}
}])
sweep3_parameters[15] = p.ParamADIOS2XML('sorting', 'load_trajectory',
'trj_dump_in', 'engine', [{
"Null": {}
}])
sweep4_parameters = sweep1_parameters.copy()
sweep4_parameters[7] = p.ParamADIOS2XML('nwchem_main', 'dump_trajectory',
'trj_dump_out', 'engine', [{
'BP4': {
'OpenTimeoutSecs': '30.0',
'BurstBufferPath': '/tmp'
}
}])
sweep4_parameters[15] = p.ParamADIOS2XML('sorting', 'load_trajectory',
'trj_dump_in', 'engine', [{
'BP4': {
'OpenTimeoutSecs': '30.0',
'BurstBufferPath': '/tmp'
}
}])
#sweep4_parameters = sweep1_parameters.copy()
#sweep4_parameters[7] = p.ParamADIOS2XML('nwchem_main', 'dump_trajectory', 'trj_dump_out', 'engine', [ {'SSC':{'DataTransport':'WAN'}} ])
#sweep4_parameters[15] = p.ParamADIOS2XML('sorting', 'load_trajectory', 'trj_dump_in', 'engine', [ {'SSC':{'DataTransport':'WAN'}} ])
# Create Sweep objects. This one does not define a node-layout, and thus, all cores of a compute node will be
# utilized and mapped to application ranks.
sweep1 = p.Sweep(parameters=sweep1_parameters)
sweep2 = p.Sweep(parameters=sweep2_parameters)
sweep3 = p.Sweep(parameters=sweep3_parameters)
sweep4 = p.Sweep(parameters=sweep4_parameters)
# Create a SweepGroup and add the above Sweeps. Set batch job properties such as the no. of nodes,
sweepGroup1 = p.SweepGroup(
"nwchem-adios", # A unique name for the SweepGroup
walltime=18060, # Total runtime for the SweepGroup
per_run_timeout=
500, # Timeout for each experiment
parameter_groups=[sweep1, sweep2, sweep3,
sweep4], # Sweeps to include in this group
launch_mode='default', # Launch mode: default, or MPMD if supported
nodes=6, # No. of nodes for the batch job.
run_repetitions=
2, # No. of times each experiment in the group must be repeated (Total no. of runs here will be 3)
component_inputs={'nwchem_main': ['copro-80.txt']},
)
# Activate the SweepGroup
sweeps = [sweepGroup1]
class ProducerConsumer(Campaign):
# A name for the campaign
name = "coupling-example"
# WORKFLOW SETUP
#---------------
# A list of the codes that will be part of the workflow
# If there is an adios xml file associated with the codes, list it here
# 'sleep_after' represents the time gap after which the next code is spawned
# Use runner_override to run the code without the default launcher (mpirun/aprun/jsrun etc.). This runs the
# code as a serial application
codes = [ ("producer", dict(exe="producer.py", adios_xml_file='adios2.xml', sleep_after=5)),
("mean_calc", dict(exe="mean_calculator.py", adios_xml_file='adios2.xml', runner_override=False))
]
# CAMPAIGN SETTINGS
#------------------
# A list of machines that this campaign is supported on
supported_machines = ['local', 'titan', 'theta', 'summit', 'rhea', 'deepthought2_cpu', 'sdg_tm76']
# Option to kill an experiment (just one experiment, not the full sweep or campaign) if one of the codes fails
kill_on_partial_failure = True
# Some pre-processing in the experiment directory
# This is performed when the campaign directory is created (before the campaign is launched)
run_dir_setup_script = None
# A post-processing script to be run in the experiment directory after the experiment completes
# For example, removing some large files after the experiment is done
run_post_process_script = None
# umask applied to your directory in the campaign so that colleagues can view files
umask = '027'
# Scheduler information: job queue, account-id etc. Leave it to None if running on a local machine
scheduler_options = {'theta': {'project': '', 'queue': 'batch'},
'summit': {'project':'csc143','reservation':'csc143_m414'}, 'rhea': {'project':'csc143'}}
# Setup your environment. Loading modules, setting the LD_LIBRARY_PATH etc.
# Ensure this script is executable
app_config_scripts = {'local': 'setup.sh', 'summit': 'env_setup.sh'}
# PARAMETER SWEEPS
#-----------------
# Setup how the workflow is run, and what values to 'sweep' over
# Use ParamCmdLineArg to setup a command line arg, ParamCmdLineOption to setup a command line option, and so on.
sweep1_parameters = [
p.ParamRunner ('producer', 'nprocs', [2]),
p.ParamRunner ('mean_calc', 'nprocs', [2]),
p.ParamCmdLineArg ('producer', 'array_size_per_pe', 1, [1024*1024,]), # 1M, 2M, 10M
p.ParamCmdLineArg ('producer', 'num_steps', 2, [10]),
p.ParamADIOS2XML ('producer', 'engine_sst', 'producer', 'engine', [ {"SST": {}} ]),
# p.ParamADIOS2XML ('producer', 'compression', 'producer', 'var_operation', [ {"U": {"zfp":{'accuracy':0.001, 'tolerance':0.9}}} ]),
]
# Summit node layout
# Create a shared node layout where the producer and mean_calc share compute nodes
shared_node_nc = SummitNode()
# place producer on the first socket
for i in range(21):
shared_node_nc.cpu[i] = 'producer:{}'.format(i)
# place analysis on the second socket
for i in range(8):
shared_node_nc.cpu[22+i] = 'mean_calc:{}'.format(i)
# This should be 'obj=machine.VirtualNode()'
shared_node_dt = DTH2CPUNode()
for i in range(10):
shared_node_dt.cpu[i] = 'producer:{}'.format(i)
shared_node_dt.cpu[11] = 'mean_calc:0'
shared_node_dt.cpu[12] = 'mean_calc:1'
# Create a sweep
# node_layout represents no. of processes per node
sweep1 = p.Sweep (node_layout = {'summit': [shared_node_nc], 'deepthought2_cpu': [shared_node_dt]}, # simulation: 16 ppn, norm_calc: 4 ppn
parameters = sweep1_parameters, rc_dependency=None)
# Create a sweep group from the above sweep. You can place multiple sweeps in the group.
# Each group is submitted as a separate job.
sweepGroup1 = p.SweepGroup ("sg-1",
walltime=300,
per_run_timeout=60,
parameter_groups=[sweep1],
launch_mode='default', # or MPMD
# optional:
tau_profiling=True,
tau_tracing=False,
# nodes=10,
# tau_profiling=True,
# tau_tracing=False,
# run_repetitions=2, <-- repeat each experiment this many times
# component_subdirs = True, <-- codes have their own separate workspace in the experiment directory
# component_inputs = {'simulation': ['some_input_file'], 'norm_calc': [SymLink('some_large_file')] } <-- inputs required by codes
# max_procs = 64 <-- max no. of procs to run concurrently. depends on 'nodes'
)
# Create a sweep group from the above sweep. You can place multiple sweeps in the group.
# Each group is submitted as a separate job.
sweepGroup2 = p.SweepGroup ("sg-2",
walltime=300,
per_run_timeout=60,
parameter_groups=[sweep1],
launch_mode='default', # or MPMD
# optional:
tau_profiling=True,
tau_tracing=False,
# nodes=10,
# tau_profiling=True,
# tau_tracing=False,
# run_repetitions=2, <-- repeat each experiment this many times
# component_subdirs = True, <-- codes have their own separate workspace in the experiment directory
# component_inputs = {'simulation': ['some_input_file'], 'norm_calc': [SymLink('some_large_file')] } <-- inputs required by codes
# max_procs = 64 <-- max no. of procs to run concurrently. depends on 'nodes'
)
# Sweep groups to be activated
sweeps = {'MACHINE_ANY':[sweepGroup1], 'summit':[sweepGroup2]}
class Brusselator(Campaign):
# A name for the campaign
name = "Brusselator"
# WORKFLOW SETUP
#---------------
# A list of the codes that will be part of the workflow
# If there is an adios xml file associated with the codes, list it here
# 'sleep_after' represents the time gap after which the next code is spawned
# Use runner_override to run the code without the default launcher (mpirun/aprun/jsrun etc.). This runs the
# code as a serial application
codes = [ ("simulation", dict(exe="simulation/Brusselator", adios_xml_file='adios2.xml', sleep_after=None)),
("norm_calc", dict(exe="analysis/norm_calc", adios_xml_file='adios2.xml', runner_override=False))
]
# CAMPAIGN SETTINGS
#------------------
# A list of machines that this campaign is supported on
supported_machines = ['local', 'titan', 'theta']
# Option to kill an experiment (just one experiment, not the full sweep or campaign) if one of the codes fails
kill_on_partial_failure = True
# Some pre-processing in the experiment directory
# This is performed when the campaign directory is created (before the campaign is launched)
run_dir_setup_script = None
# A post-processing script to be run in the experiment directory after the experiment completes
# For example, removing some large files after the experiment is done
run_post_process_script = None
# umask applied to your directory in the campaign so that colleagues can view files
umask = '027'
# Scheduler information: job queue, account-id etc. Leave it to None if running on a local machine
scheduler_options = {'titan': {'project':'CSC249ADCD01', 'queue': 'batch'}}
# Setup your environment. Loading modules, setting the LD_LIBRARY_PATH etc.
# Ensure this script is executable
app_config_scripts = {'local': 'setup.sh', 'titan': 'env_setup.sh'}
# PARAMETER SWEEPS
#-----------------
# Setup how the workflow is run, and what values to 'sweep' over
# Use ParamCmdLineArg to setup a command line arg, ParamCmdLineOption to setup a command line option, and so on.
sweep1_parameters = [
# ParamRunner with 'nprocs' sets the number of MPI processes
p.ParamRunner ('simulation', 'nprocs', [4,8]), # <-- how to sweep over values
p.ParamCmdLineArg ('simulation', 'output', 1, ['bru.bp']),
p.ParamCmdLineArg ('simulation', 'nx', 2, [32]),
p.ParamCmdLineArg ('simulation', 'ny', 3, [32]),
p.ParamCmdLineArg ('simulation', 'nz', 4, [32]),
p.ParamCmdLineArg ('simulation', 'steps', 5, [10,20,50]), # sweep over these values. creates cross-product of runs with 'nprocs' above
p.ParamCmdLineArg ('simulation', 'plotgap', 6, [1]),
p.ParamRunner ('norm_calc', 'nprocs', [1]),
p.ParamCmdLineArg ('norm_calc', 'infile', 1, ['bru.bp']),
p.ParamCmdLineArg ('norm_calc', 'outfile', 2, ['norm_calc.out.bp']),
p.ParamCmdLineArg ('norm_calc', 'write_norms_only', 3, [1]),
# ParamADIOS2XML can be used to setup a value in the ADIOS xml file for the application
# Set the transport to BPFile, as we want the codes to run serially. Set the rc_dependency
# in the Sweep to denote dependency between the codes
# To couple codes for concurrent execution, use a transport method such as SST
p.ParamADIOS2XML ('simulation', 'SimulationOutput', 'engine', [ {"BPFile": {}} ]),
p.ParamADIOS2XML ('simulation', 'AnalysisOutput', 'engine', [ {"BPFile": {}} ]),
# p.ParamADIOS2XML ('simulation', 'SimulationOutput', 'engine', [ {"BPFile": {'Threads':1}},
# p.ParamADIOS2XML ('simulation', 'SimulationOutput', 'engine', [ {"BPFile": {'Threads':1}}, {"BPFile": {"ProfileUnits": "Microseconds"}} ]),
# Use ParamCmdLineOption for named arguments
# p.ParamCmdLineOption ('plotting', 'input_stream', '-i', ['bru.bp']),
# Use ParamKeyValue to set options in a key-value configuration file. Input file can be a json file.
# File path can be relative to the path specified in '-a' or it can be absolute.
# File will be copied to the working_dir automatically by Cheetah.
# p.ParamKeyValue ('simulation', 'feed_rate', 'input.conf', 'key', ['value']),
# Sweep over environment variables
# p.ParamEnvVar ('simulation', 'openmp_stuff', 'OMP_NUM_THREADS', [4,8]),
# Pass additional scheduler arguments.
# p.ParamSchedulerArgs ('simulation', [{'-f':'hosts.txt'}])
]
# Create a sweep
# node_layout represents no. of processes per node
# rc_dependency denotes dependency between run components. Here, norm_calc will run after simulation has finished
sweep1 = p.Sweep (node_layout = {'titan': [{'simulation':16}, {'norm_calc': 4}] }, # simulation: 16 ppn, norm_calc: 4 ppn
parameters = sweep1_parameters, rc_dependency={'norm_calc':'simulation'})
# Create a sweep group from the above sweep. You can place multiple sweeps in the group.
# Each group is submitted as a separate job.
sweepGroup1 = p.SweepGroup ("sg-tmp",
walltime=300,
per_run_timeout=60,
parameter_groups=[sweep1],
launch_mode='default', # or MPMD
# optional:
# nodes=10,
# run_repetitions=2, # no. of times each experiment must be repeated (here, total runs = 3)
# component_subdirs = True, <-- codes have their own separate workspace in the experiment directory
# component_inputs = {'simulation': ['some_input_file'], 'norm_calc': [SymLink('some_large_file')] } <-- inputs required by codes
# max_procs = 64 <-- max no. of procs to run concurrently. depends on 'nodes'
)
sweepGroup2 = copy.deepcopy(sweepGroup1)
sweepGroup2.name = 'sg-2'
# Sweep groups to be activated
sweeps = [sweepGroup1, sweepGroup2]