def asynchronous_zfp(): sweep_parameters = [ p.ParamRunner('simulation', 'nprocs', [64]), p.ParamCmdLineArg('simulation', 'settings', 1, ["settings-files.json"]), p.ParamADIOS2XML('simulation', 'sim output engine', 'SimulationOutput', 'engine', [{ 'BP4': {} }]), p.ParamRunner('pdf_calc', 'nprocs', [8]), p.ParamCmdLineArg('pdf_calc', 'infile', 1, ['gs.bp']), p.ParamCmdLineArg('pdf_calc', 'outfile', 2, ['pdf.bp']), p.ParamCmdLineArg('pdf_calc', 'bins', 3, [100]), p.ParamCmdLineArg('pdf_calc', 'write_orig_data', 4, ['YES']), p.ParamADIOS2XML('pdf_calc', 'zfp compression', 'PDFAnalysisOutput', 'var_operation', [{ "U": { "zfp": { 'accuracy': 0.001 } } }, { "U": { "zfp": { 'accuracy': 0.0001 } } }]), ] sweep = p.Sweep(parameters=sweep_parameters, rc_dependency={'pdf_calc': 'simulation'}, node_layout={'summit': node_layouts.separate_nodes()}) return sweep
def get_sweep_params(adios_engine): # Setup the sweep parameters for a Sweep return [ # ParamRunner 'nprocs' specifies the no. of ranks to be spawned p.ParamRunner('simulation', 'nprocs', [1]), # Create a ParamCmdLineArg parameter to specify a command line argument to run the application p.ParamCmdLineArg('simulation', 'settings', 1, ["settings.json"]), # Edit key-value pairs in the json file # Sweep over two values for the F key in the json file. Alongwith 4 values for the nprocs property for # the pdf_calc code, this Sweep will create 2*4 = 8 experiments. p.ParamConfig('simulation', 'feed_rate_U', 'settings.json', 'F', [0.01]), p.ParamConfig('simulation', 'kill_rate_V', 'settings.json', 'k', [0.048]), # Setup an environment variable # p.ParamEnvVar ('simulation', 'openmp', 'OMP_NUM_THREADS', [4]), # Change the engine for the 'SimulationOutput' IO object in the adios xml file to SST for coupling. # As both the applications use the same xml file, you need to do this just once. p.ParamADIOS2XML('simulation', 'sim output engine', 'SimulationOutput', 'engine', [{ adios_engine: {} }]), # Now setup options for the pdf_calc application. # Sweep over four values for the nprocs p.ParamRunner('pdf_calc', 'nprocs', [1]), p.ParamCmdLineArg('pdf_calc', 'infile', 1, ['gs.bp']), p.ParamCmdLineArg('pdf_calc', 'outfile', 2, ['pdf']), ]
def insitu_analysis(sim_nprocs, analysis_nprocs, node_layout, adios_engine): sweep_parameters = [ p.ParamRunner('simulation', 'nprocs', [sim_nprocs]), p.ParamCmdLineOption('simulation', 'sim input', '-in', ["in.lj.nordf"]), p.ParamADIOS2XML('simulation', 'sim output engine', 'custom', 'engine', [{ adios_engine: {} }]), p.ParamRunner('rdf_calc', 'nprocs', [analysis_nprocs]), p.ParamCmdLineOption('rdf_calc', 'input', '-in', ["in.lj.rdf.rerun"]), p.ParamADIOS2XML('rdf_calc', 'analysis input engine', 'read_dump', 'engine', [{ adios_engine: {} }]), ] sweep = p.Sweep(parameters=sweep_parameters, rc_dependency=None, node_layout={'summit': node_layout}) return sweep
def create_experiment(writer_nprocs, reader_nprocs, config_file, adios_xml_file, engine, writer_decomposition, reader_decomposition, machine_name, node_layout): """ Creates a sweep object that tells Cheetah how to run the adios io test. Assumes 1D decomposition. """ # print(adios_xml_file) # print(engine) params = [ # ParamRunner 'nprocs' specifies the no. of ranks to be spawned p.ParamRunner ('writer', 'nprocs', [writer_nprocs]), # Create a ParamCmdLineArg parameter to specify a command line argument to run the application p.ParamCmdLineOption ('writer', 'app', '-a', [1]), p.ParamCmdLineOption ('writer', 'app-config', '-c', [config_file]), p.ParamCmdLineOption ('writer', 'adios-config', '-x', [adios_xml_file]), p.ParamCmdLineOption ('writer', 'strongscaling', '-w', [None]), p.ParamCmdLineOption ('writer', 'timing', '-t', [None]), p.ParamCmdLineOption ('writer', 'decomposition', '-d', [writer_decomposition]), # Change the engine for the 'SimulationOutput' IO object in the adios xml file p.ParamADIOS2XML ('writer', 'dump_trajectory', 'trj_dump_out', 'engine', [engine]), # Sweep over four values for the nprocs p.ParamRunner ('reader', 'nprocs', [reader_nprocs]), p.ParamCmdLineOption ('reader', 'app', '-a', [2]), p.ParamCmdLineOption ('reader', 'app-config', '-c', [config_file]), p.ParamCmdLineOption ('reader', 'adios-config', '-x', [adios_xml_file]), p.ParamCmdLineOption ('reader', 'weakscaling', '-s', [None]), p.ParamCmdLineOption ('reader', 'timing', '-t', [None]), p.ParamCmdLineOption ('reader', 'decomposition', '-d', [reader_decomposition]), # Change the engine for the 'SimulationOutput' IO object in the adios xml file p.ParamADIOS2XML ('reader', 'load_trajectory', 'trj_dump_in', 'engine', [engine]), ] sweep = p.Sweep(parameters=params) if node_layout: sweep.node_layout = {machine_name: node_layout} return sweep
def create_experiment(writer_nprocs, reader_nprocs, trj_engine, sorted_trj_engine, machine_name, node_layout): """ Creates a sweep object that tells Cheetah how to run the adios io test. Assumes 1D decomposition. """ # print(adios_xml_file) # print(engine) params = [ # ParamRunner 'nprocs' specifies the no. of ranks to be spawned p.ParamRunner ('writer', 'nprocs', [writer_nprocs]), # Create a ParamCmdLineArg parameter to specify a command line argument to run the application p.ParamCmdLineArg ('writer', 'config', 1, ['copro.nw']), # Change the engine for the 'SimulationOutput' IO object in the adios xml file p.ParamADIOS2XML ('writer', 'trajectory', 'trj', 'engine', [trj_engine]), # Sweep over four values for the nprocs p.ParamRunner ('reader', 'nprocs', [reader_nprocs]), p.ParamCmdLineArg ('reader', 'input_md', 1, ['copro_md']), p.ParamCmdLineArg ('reader', 'verbose', 2, [1]), # Change the engine for the 'SimulationOutput' IO object in the adios xml file p.ParamADIOS2XML ('reader', 'sorted_trj', 'SortingOutput', 'engine', [sorted_trj_engine]), p.ParamRunner ('analyzer', 'nprocs', [1]), p.ParamCmdLineArg ('analyzer', 'script', 1, ['pca3d.R']), p.ParamCmdLineArg ('analyzer', 'window', 2, [100]), p.ParamCmdLineArg ('analyzer', 'stride', 3, [10]), p.ParamCmdLineArg ('analyzer', 'k', 4, [5]), p.ParamCmdLineArg ('analyzer', 'sorted_trj', 5, ['copro_md_trj.bp']), p.ParamCmdLineArg ('analyzer', 'xml', 6, ['adios2.xml']), p.ParamCmdLineArg ('analyzer', 'mcCore', 7, [1]), p.ParamCmdLineArg ('analyzer', 'output', 8, ['pairs.pdf']), ] sweep = p.Sweep(parameters=params) if node_layout: sweep.node_layout = {machine_name: node_layout} return sweep
def posthoc_analysis(sim_nprocs, analysis_nprocs): sweep_parameters = [ p.ParamRunner('simulation', 'nprocs', [sim_nprocs]), p.ParamCmdLineOption('simulation', 'sim input', '-in', ["in.lj.nordf"]), p.ParamADIOS2XML('simulation', 'sim output engine', 'custom', 'engine', [{ 'BP4': {} }]), p.ParamRunner('rdf_calc', 'nprocs', [analysis_nprocs]), p.ParamCmdLineOption('rdf_calc', 'input', '-in', ["in.lj.rdf.rerun"]), ] sweep = p.Sweep(parameters=sweep_parameters, rc_dependency={'rdf_calc': 'simulation'}, node_layout={'summit': node_layouts.separate_nodes()}) return sweep
def posthoc_analysis(): sweep_parameters = [ p.ParamRunner('simulation', 'nprocs', [4]), p.ParamCmdLineArg('simulation', 'settings', 1, ["settings-files.json"]), p.ParamADIOS2XML('simulation', 'sim output engine', 'SimulationOutput', 'engine', [{ 'BP4': {} }]), p.ParamRunner('pdf_calc', 'nprocs', [4]), p.ParamCmdLineArg('pdf_calc', 'infile', 1, ['gs.bp']), p.ParamCmdLineArg('pdf_calc', 'outfile', 2, ['pdf.bp']), ] sweep = p.Sweep(parameters=sweep_parameters, rc_dependency={'pdf_calc': 'simulation'}, node_layout={'summit': node_layouts.separate_nodes()}) return sweep
def insitu_analysis(node_layout): sweep_parameters = [ p.ParamRunner('simulation', 'nprocs', [64]), p.ParamCmdLineArg('simulation', 'settings', 1, ["settings-staging.json"]), p.ParamADIOS2XML('simulation', 'sim output engine', 'SimulationOutput', 'engine', [{ 'SST': {} }]), p.ParamRunner('pdf_calc', 'nprocs', [4, 8, 16, 32]), p.ParamCmdLineArg('pdf_calc', 'infile', 1, ['gs.bp']), p.ParamCmdLineArg('pdf_calc', 'outfile', 2, ['pdf.bp']), ] sweep = p.Sweep(parameters=sweep_parameters, rc_dependency=None, node_layout={'summit': node_layout}) return sweep
class GrayScott(Campaign): # A name for the campaign name = "gray_scott" # Define your workflow. Setup the applications that form the workflow. # exe may be an absolute path. # The adios xml file is automatically copied to the campaign directory. # 'runner_override' may be used to launch the code on a login/service node as a serial code # without a runner such as aprun/srun/jsrun etc. codes = [ ("simulation", dict(exe="gray-scott", adios_xml_file='adios2.xml')), ("pdf_calc", dict(exe="pdf_calc", adios_xml_file='adios2.xml', runner_override=False)), ] # List of machines on which this code can be run supported_machines = ['local', 'titan', 'theta'] # Kill an experiment right away if any workflow components fail (just the experiment, not the whole group) kill_on_partial_failure = True # Any setup that you may need to do in an experiment directory before the experiment is run run_dir_setup_script = None # A post-process script that is run for every experiment after the experiment completes run_post_process_script = None # Directory permissions for the campaign sub-directories umask = '027' # Options for the underlying scheduler on the target system. Specify the project ID and job queue here. scheduler_options = { 'theta': { 'project': 'CSC249ADCD01', 'queue': 'default' } } # A way to setup your environment before the experiment runs. Export environment variables such as LD_LIBRARY_PATH here. app_config_scripts = {'local': 'setup.sh', 'theta': 'env_setup.sh'} # Setup the sweep parameters for a Sweep sweep1_parameters = [ # ParamRunner 'nprocs' specifies the no. of ranks to be spawned p.ParamRunner('simulation', 'nprocs', [512]), # Create a ParamCmdLineArg parameter to specify a command line argument to run the application p.ParamCmdLineArg('simulation', 'settings', 1, ["settings.json"]), # Edit key-value pairs in the json file # Sweep over two values for the F key in the json file. Alongwith 4 values for the nprocs property for # the pdf_calc code, this Sweep will create 2*4 = 8 experiments. p.ParamConfig('simulation', 'feed_rate_U', 'settings.json', 'F', [0.01, 0.02]), p.ParamConfig('simulation', 'kill_rate_V', 'settings.json', 'k', [0.048]), # Setup an environment variable # p.ParamEnvVar ('simulation', 'openmp', 'OMP_NUM_THREADS', [4]), # Change the engine for the 'SimulationOutput' IO object in the adios xml file to SST for coupling. # As both the applications use the same xml file, you need to do this just once. p.ParamADIOS2XML('simulation', 'SimulationOutput', 'engine', [{ 'SST': {} }]), # Now setup options for the pdf_calc application. # Sweep over four values for the nprocs p.ParamRunner('pdf_calc', 'nprocs', [32, 64, 128, 256]), p.ParamCmdLineArg('pdf_calc', 'infile', 1, ['gs.bp']), p.ParamCmdLineArg('pdf_calc', 'outfile', 2, ['pdf']), ] # Create a Sweep object. This one does not define a node-layout, and thus, all cores of a compute node will be # utilized and mapped to application ranks. sweep1 = p.Sweep(parameters=sweep1_parameters) # Create another Sweep object and set its node-layout to spawn 16 simulation processes per node, and # 4 processes of pdf_calc per node. On Theta, different executables reside on separate nodes as node-sharing # is not permitted on Theta. sweep2_parameters = copy.deepcopy(sweep1_parameters) sweep2 = p.Sweep( node_layout={'theta': [{ 'simulation': 16 }, { 'pdf_calc': 4 }]}, parameters=sweep2_parameters) # Create a SweepGroup and add the above Sweeps. Set batch job properties such as the no. of nodes, sweepGroup1 = p.SweepGroup( "sg-1", # A unique name for the SweepGroup walltime=3600, # Total runtime for the SweepGroup per_run_timeout= 600, # Timeout for each experiment parameter_groups=[sweep1, sweep2], # Sweeps to include in this group launch_mode='default', # Launch mode: default, or MPMD if supported nodes=128, # No. of nodes for the batch job. # rc_dependency={'pdf_calc':'simulation',}, # Specify dependencies between workflow components run_repetitions= 2, # No. of times each experiment in the group must be repeated (Total no. of runs here will be 3) ) # Activate the SweepGroup sweeps = [sweepGroup1]
class ProducerConsumer(Campaign): # A name for the campaign name = "coupling-example" # WORKFLOW SETUP #--------------- # A list of the codes that will be part of the workflow # If there is an adios xml file associated with the codes, list it here # 'sleep_after' represents the time gap after which the next code is spawned # Use runner_override to run the code without the default launcher (mpirun/aprun/jsrun etc.). This runs the # code as a serial application codes = [("producer", dict(exe="producer.py", adios_xml_file='adios2.xml', sleep_after=5)), ("mean_calc", dict(exe="mean_calculator.py", adios_xml_file='adios2.xml', runner_override=False))] # CAMPAIGN SETTINGS #------------------ # A list of machines that this campaign is supported on supported_machines = ['local', 'titan', 'theta', 'summit'] # Option to kill an experiment (just one experiment, not the full sweep or campaign) if one of the codes fails kill_on_partial_failure = True # Some pre-processing in the experiment directory # This is performed when the campaign directory is created (before the campaign is launched) run_dir_setup_script = None # A post-processing script to be run in the experiment directory after the experiment completes # For example, removing some large files after the experiment is done run_post_process_script = None # umask applied to your directory in the campaign so that colleagues can view files umask = '027' # Scheduler information: job queue, account-id etc. Leave it to None if running on a local machine scheduler_options = { 'theta': { 'project': '', 'queue': 'batch' }, 'summit': { 'project': 'CSC299' } } # Setup your environment. Loading modules, setting the LD_LIBRARY_PATH etc. # Ensure this script is executable app_config_scripts = {'local': 'setup.sh', 'summit': 'env_setup.sh'} # PARAMETER SWEEPS #----------------- # Setup how the workflow is run, and what values to 'sweep' over # Use ParamCmdLineArg to setup a command line arg, ParamCmdLineOption to setup a command line option, and so on. sweep1_parameters = [ p.ParamRunner('producer', 'nprocs', [128]), p.ParamRunner('mean_calc', 'nprocs', [36]), p.ParamCmdLineArg('producer', 'array_size_per_pe', 1, [1024 * 1024]), # 1M, 2M, 10M p.ParamCmdLineArg('producer', 'num_steps', 2, [10]), p.ParamADIOS2XML('producer', 'staging', 'producer', 'engine', [{ "SST": {} }]), ] shared_node = SummitNode() for i in range(18): shared_node.cpu[i] = "producer:{}".format(math.floor(i / 6)) shared_node.cpu[i + 21] = "producer:{}".format(math.floor( (i + 18) / 6)) for i in range(3): shared_node.cpu[i + 18] = "mean_calc:0" shared_node.cpu[i + 18 + 21] = "mean_calc:0" for i in range(6): shared_node.gpu[i] = ["producer:{}".format(i)] shared_node_layout = [shared_node] shared_node_1_per_rank = SummitNode() for i in range(18): shared_node_1_per_rank.cpu[i] = "producer:{}".format(i) shared_node_1_per_rank.cpu[i + 21] = "producer:{}".format(i + 18) for i in range(3): shared_node_1_per_rank.cpu[i + 18] = "mean_calc:{}".format(i) shared_node_1_per_rank.cpu[i + 18 + 21] = "mean_calc:{}".format(i + 3) for i in range(6): shared_node_1_per_rank.gpu[i] = ["producer:{}".format(i)] shared_node_layout_2 = [shared_node_1_per_rank] shared_node_shared_gpu = SummitNode() for i in range(18): shared_node_shared_gpu.cpu[i] = "producer:{}".format(math.floor(i / 6)) shared_node_shared_gpu.cpu[i + 21] = "producer:{}".format( math.floor((i + 18) / 6)) for i in range(3): shared_node_shared_gpu.cpu[i + 18] = "mean_calc:0" shared_node_shared_gpu.cpu[i + 18 + 21] = "mean_calc:0" shared_node_shared_gpu.gpu[0] = ["producer:0"] shared_node_shared_gpu.gpu[1] = [ "producer:0", "producer:1", ] shared_node_shared_gpu.gpu[2] = ["producer:0", "producer:1", 'mean_calc:0'] shared_node_layout_3 = [shared_node_shared_gpu] sep_node_producer = SummitNode() sep_node_mean_calc = SummitNode() for i in range(18): sep_node_producer.cpu[i] = "producer:{}".format(math.floor(i / 6)) for i in range(3): sep_node_mean_calc.cpu[i + 18] = "mean_calc:0" sep_node_mean_calc.cpu[i + 18 + 21] = "mean_calc:0" for i in range(3): sep_node_producer.gpu[i] = ["producer:{}".format(i)] sep_node_layout = [sep_node_producer, sep_node_mean_calc] # Create a sweep # node_layout represents no. of processes per node sweep1 = p.Sweep(node_layout={'summit': shared_node_layout}, parameters=sweep1_parameters, rc_dependency=None) sweep2 = p.Sweep(node_layout={'summit': shared_node_layout_2}, parameters=sweep1_parameters, rc_dependency=None) sweep3 = p.Sweep(node_layout={'summit': shared_node_layout_3}, parameters=sweep1_parameters, rc_dependency=None) sweep4 = p.Sweep(node_layout={'summit': sep_node_layout}, parameters=sweep1_parameters, rc_dependency=None) # Create a sweep group from the above sweep. You can place multiple sweeps in the group. # Each group is submitted as a separate job. sweepGroup1 = p.SweepGroup( "sg-1", walltime=300, per_run_timeout=60, parameter_groups=[sweep1, sweep2, sweep3, sweep4], launch_mode='default', # or MPMD # optional: # tau_profiling=True, # tau_tracing=False, # nodes=10, # component_subdirs = True, <-- codes have their own separate workspace in the experiment directory # component_inputs = {'producer': ['some_input_file'], 'norm_calc': [SymLink('some_large_file')] } <-- inputs required by codes # max_procs = 64 <-- max no. of procs to run concurrently. depends on 'nodes' ) # Sweep groups to be activated sweeps = {'summit': [sweepGroup1]}
class NWChem(Campaign): # A name for the campaign name = "nwchem" # Define your workflow. Setup the applications that form the workflow. # exe may be an absolute path. # The adios xml file is automatically copied to the campaign directory. # 'runner_override' may be used to launch the code on a login/service node as a serial code # without a runner such as aprun/srun/jsrun etc. codes = [ ("nwchem_main", dict( exe= "/ccs/proj/e2e/pnorbert/ADIOS/ADIOS2/build.rhea.gcc/install/bin/adios2_iotest", adios_xml_file='copro.xml')), ("sorting", dict( exe= "/ccs/proj/e2e/pnorbert/ADIOS/ADIOS2/build.rhea.gcc/install/bin/adios2_iotest", adios_xml_file='copro.xml', runner_override=False)), ] # List of machines on which this code can be run supported_machines = ['local', 'titan', 'theta', 'rhea'] # Kill an experiment right away if any workflow components fail (just the experiment, not the whole group) kill_on_partial_failure = True # Any setup that you may need to do in an experiment directory before the experiment is run run_dir_setup_script = None # A post-process script that is run for every experiment after the experiment completes run_post_process_script = None # Directory permissions for the campaign sub-directories umask = '027' # Options for the underlying scheduler on the target system. Specify the project ID and job queue here. # scheduler_options = {'theta': {'project':'CSC249ADCD01', 'queue': 'default'}} scheduler_options = {'rhea': {'project': 'csc143'}} # A way to setup your environment before the experiment runs. Export environment variables such as LD_LIBRARY_PATH here. app_config_scripts = { 'local': 'setup.sh', 'theta': 'env_setup.sh', 'rhea': 'setup_nwchem_rhea.sh' } # Setup the sweep parameters for a Sweep sweep1_parameters = [ # ParamRunner 'nprocs' specifies the no. of ranks to be spawned p.ParamRunner('nwchem_main', 'nprocs', [80]), # Create a ParamCmdLineArg parameter to specify a command line argument to run the application p.ParamCmdLineOption('nwchem_main', 'app', '-a', [1]), p.ParamCmdLineOption('nwchem_main', 'app-config', '-c', ['copro-80.txt']), p.ParamCmdLineOption('nwchem_main', 'adios-config', '-x', ['copro.xml']), p.ParamCmdLineOption('nwchem_main', 'strongscaling', '-w', [None]), p.ParamCmdLineOption('nwchem_main', 'timing', '-t', [None]), p.ParamCmdLineOption('nwchem_main', 'decomposition', '-d', [80]), # Change the engine for the 'SimulationOutput' IO object in the adios xml file to SST for coupling. p.ParamADIOS2XML('nwchem_main', 'dump_trajectory', 'trj_dump_out', 'engine', [{ 'BP4': { 'OpenTimeoutSecs': '30.0' } }]), # Now setup options for the pdf_calc application. # Sweep over four values for the nprocs p.ParamRunner('sorting', 'nprocs', [8]), p.ParamCmdLineOption('sorting', 'app', '-a', [1]), p.ParamCmdLineOption('sorting', 'app-config', '-c', ['copro-80.txt']), p.ParamCmdLineOption('sorting', 'adios-config', '-x', ['copro.xml']), p.ParamCmdLineOption('sorting', 'weakscaling', '-s', [None]), p.ParamCmdLineOption('sorting', 'timing', '-t', [None]), p.ParamCmdLineOption('sorting', 'decomposition', '-d', [8]), # Change the engine for the 'SimulationOutput' IO object in the adios xml file to SST for coupling. p.ParamADIOS2XML('sorting', 'load_trajectory', 'trj_dump_in', 'engine', [{ 'BP4': { 'OpenTimeoutSecs': '30.0' } }]), ] #print(sweep1_parameters) sweep2_parameters = sweep1_parameters.copy() sweep2_parameters[7] = p.ParamADIOS2XML('nwchem_main', 'dump_trajectory', 'trj_dump_out', 'engine', [{ 'SST': {} }]) sweep2_parameters[15] = p.ParamADIOS2XML('sorting', 'load_trajectory', 'trj_dump_in', 'engine', [{ 'SST': {} }]) sweep3_parameters = sweep1_parameters.copy() sweep3_parameters[7] = p.ParamADIOS2XML('nwchem_main', 'dump_trajectory', 'trj_dump_out', 'engine', [{ "Null": {} }]) sweep3_parameters[15] = p.ParamADIOS2XML('sorting', 'load_trajectory', 'trj_dump_in', 'engine', [{ "Null": {} }]) sweep4_parameters = sweep1_parameters.copy() sweep4_parameters[7] = p.ParamADIOS2XML('nwchem_main', 'dump_trajectory', 'trj_dump_out', 'engine', [{ 'BP4': { 'OpenTimeoutSecs': '30.0', 'BurstBufferPath': '/tmp' } }]) sweep4_parameters[15] = p.ParamADIOS2XML('sorting', 'load_trajectory', 'trj_dump_in', 'engine', [{ 'BP4': { 'OpenTimeoutSecs': '30.0', 'BurstBufferPath': '/tmp' } }]) #sweep4_parameters = sweep1_parameters.copy() #sweep4_parameters[7] = p.ParamADIOS2XML('nwchem_main', 'dump_trajectory', 'trj_dump_out', 'engine', [ {'SSC':{'DataTransport':'WAN'}} ]) #sweep4_parameters[15] = p.ParamADIOS2XML('sorting', 'load_trajectory', 'trj_dump_in', 'engine', [ {'SSC':{'DataTransport':'WAN'}} ]) # Create Sweep objects. This one does not define a node-layout, and thus, all cores of a compute node will be # utilized and mapped to application ranks. sweep1 = p.Sweep(parameters=sweep1_parameters) sweep2 = p.Sweep(parameters=sweep2_parameters) sweep3 = p.Sweep(parameters=sweep3_parameters) sweep4 = p.Sweep(parameters=sweep4_parameters) # Create a SweepGroup and add the above Sweeps. Set batch job properties such as the no. of nodes, sweepGroup1 = p.SweepGroup( "nwchem-adios", # A unique name for the SweepGroup walltime=18060, # Total runtime for the SweepGroup per_run_timeout= 500, # Timeout for each experiment parameter_groups=[sweep1, sweep2, sweep3, sweep4], # Sweeps to include in this group launch_mode='default', # Launch mode: default, or MPMD if supported nodes=6, # No. of nodes for the batch job. run_repetitions= 2, # No. of times each experiment in the group must be repeated (Total no. of runs here will be 3) component_inputs={'nwchem_main': ['copro-80.txt']}, ) # Activate the SweepGroup sweeps = [sweepGroup1]
class ProducerConsumer(Campaign): # A name for the campaign name = "coupling-example" # WORKFLOW SETUP #--------------- # A list of the codes that will be part of the workflow # If there is an adios xml file associated with the codes, list it here # 'sleep_after' represents the time gap after which the next code is spawned # Use runner_override to run the code without the default launcher (mpirun/aprun/jsrun etc.). This runs the # code as a serial application codes = [ ("producer", dict(exe="producer.py", adios_xml_file='adios2.xml', sleep_after=5)), ("mean_calc", dict(exe="mean_calculator.py", adios_xml_file='adios2.xml', runner_override=False)) ] # CAMPAIGN SETTINGS #------------------ # A list of machines that this campaign is supported on supported_machines = ['local', 'titan', 'theta', 'summit', 'rhea', 'deepthought2_cpu', 'sdg_tm76'] # Option to kill an experiment (just one experiment, not the full sweep or campaign) if one of the codes fails kill_on_partial_failure = True # Some pre-processing in the experiment directory # This is performed when the campaign directory is created (before the campaign is launched) run_dir_setup_script = None # A post-processing script to be run in the experiment directory after the experiment completes # For example, removing some large files after the experiment is done run_post_process_script = None # umask applied to your directory in the campaign so that colleagues can view files umask = '027' # Scheduler information: job queue, account-id etc. Leave it to None if running on a local machine scheduler_options = {'theta': {'project': '', 'queue': 'batch'}, 'summit': {'project':'csc143','reservation':'csc143_m414'}, 'rhea': {'project':'csc143'}} # Setup your environment. Loading modules, setting the LD_LIBRARY_PATH etc. # Ensure this script is executable app_config_scripts = {'local': 'setup.sh', 'summit': 'env_setup.sh'} # PARAMETER SWEEPS #----------------- # Setup how the workflow is run, and what values to 'sweep' over # Use ParamCmdLineArg to setup a command line arg, ParamCmdLineOption to setup a command line option, and so on. sweep1_parameters = [ p.ParamRunner ('producer', 'nprocs', [2]), p.ParamRunner ('mean_calc', 'nprocs', [2]), p.ParamCmdLineArg ('producer', 'array_size_per_pe', 1, [1024*1024,]), # 1M, 2M, 10M p.ParamCmdLineArg ('producer', 'num_steps', 2, [10]), p.ParamADIOS2XML ('producer', 'engine_sst', 'producer', 'engine', [ {"SST": {}} ]), # p.ParamADIOS2XML ('producer', 'compression', 'producer', 'var_operation', [ {"U": {"zfp":{'accuracy':0.001, 'tolerance':0.9}}} ]), ] # Summit node layout # Create a shared node layout where the producer and mean_calc share compute nodes shared_node_nc = SummitNode() # place producer on the first socket for i in range(21): shared_node_nc.cpu[i] = 'producer:{}'.format(i) # place analysis on the second socket for i in range(8): shared_node_nc.cpu[22+i] = 'mean_calc:{}'.format(i) # This should be 'obj=machine.VirtualNode()' shared_node_dt = DTH2CPUNode() for i in range(10): shared_node_dt.cpu[i] = 'producer:{}'.format(i) shared_node_dt.cpu[11] = 'mean_calc:0' shared_node_dt.cpu[12] = 'mean_calc:1' # Create a sweep # node_layout represents no. of processes per node sweep1 = p.Sweep (node_layout = {'summit': [shared_node_nc], 'deepthought2_cpu': [shared_node_dt]}, # simulation: 16 ppn, norm_calc: 4 ppn parameters = sweep1_parameters, rc_dependency=None) # Create a sweep group from the above sweep. You can place multiple sweeps in the group. # Each group is submitted as a separate job. sweepGroup1 = p.SweepGroup ("sg-1", walltime=300, per_run_timeout=60, parameter_groups=[sweep1], launch_mode='default', # or MPMD # optional: tau_profiling=True, tau_tracing=False, # nodes=10, # tau_profiling=True, # tau_tracing=False, # run_repetitions=2, <-- repeat each experiment this many times # component_subdirs = True, <-- codes have their own separate workspace in the experiment directory # component_inputs = {'simulation': ['some_input_file'], 'norm_calc': [SymLink('some_large_file')] } <-- inputs required by codes # max_procs = 64 <-- max no. of procs to run concurrently. depends on 'nodes' ) # Create a sweep group from the above sweep. You can place multiple sweeps in the group. # Each group is submitted as a separate job. sweepGroup2 = p.SweepGroup ("sg-2", walltime=300, per_run_timeout=60, parameter_groups=[sweep1], launch_mode='default', # or MPMD # optional: tau_profiling=True, tau_tracing=False, # nodes=10, # tau_profiling=True, # tau_tracing=False, # run_repetitions=2, <-- repeat each experiment this many times # component_subdirs = True, <-- codes have their own separate workspace in the experiment directory # component_inputs = {'simulation': ['some_input_file'], 'norm_calc': [SymLink('some_large_file')] } <-- inputs required by codes # max_procs = 64 <-- max no. of procs to run concurrently. depends on 'nodes' ) # Sweep groups to be activated sweeps = {'MACHINE_ANY':[sweepGroup1], 'summit':[sweepGroup2]}
class Brusselator(Campaign): # A name for the campaign name = "Brusselator" # WORKFLOW SETUP #--------------- # A list of the codes that will be part of the workflow # If there is an adios xml file associated with the codes, list it here # 'sleep_after' represents the time gap after which the next code is spawned # Use runner_override to run the code without the default launcher (mpirun/aprun/jsrun etc.). This runs the # code as a serial application codes = [ ("simulation", dict(exe="simulation/Brusselator", adios_xml_file='adios2.xml', sleep_after=None)), ("norm_calc", dict(exe="analysis/norm_calc", adios_xml_file='adios2.xml', runner_override=False)) ] # CAMPAIGN SETTINGS #------------------ # A list of machines that this campaign is supported on supported_machines = ['local', 'titan', 'theta'] # Option to kill an experiment (just one experiment, not the full sweep or campaign) if one of the codes fails kill_on_partial_failure = True # Some pre-processing in the experiment directory # This is performed when the campaign directory is created (before the campaign is launched) run_dir_setup_script = None # A post-processing script to be run in the experiment directory after the experiment completes # For example, removing some large files after the experiment is done run_post_process_script = None # umask applied to your directory in the campaign so that colleagues can view files umask = '027' # Scheduler information: job queue, account-id etc. Leave it to None if running on a local machine scheduler_options = {'titan': {'project':'CSC249ADCD01', 'queue': 'batch'}} # Setup your environment. Loading modules, setting the LD_LIBRARY_PATH etc. # Ensure this script is executable app_config_scripts = {'local': 'setup.sh', 'titan': 'env_setup.sh'} # PARAMETER SWEEPS #----------------- # Setup how the workflow is run, and what values to 'sweep' over # Use ParamCmdLineArg to setup a command line arg, ParamCmdLineOption to setup a command line option, and so on. sweep1_parameters = [ # ParamRunner with 'nprocs' sets the number of MPI processes p.ParamRunner ('simulation', 'nprocs', [4,8]), # <-- how to sweep over values p.ParamCmdLineArg ('simulation', 'output', 1, ['bru.bp']), p.ParamCmdLineArg ('simulation', 'nx', 2, [32]), p.ParamCmdLineArg ('simulation', 'ny', 3, [32]), p.ParamCmdLineArg ('simulation', 'nz', 4, [32]), p.ParamCmdLineArg ('simulation', 'steps', 5, [10,20,50]), # sweep over these values. creates cross-product of runs with 'nprocs' above p.ParamCmdLineArg ('simulation', 'plotgap', 6, [1]), p.ParamRunner ('norm_calc', 'nprocs', [1]), p.ParamCmdLineArg ('norm_calc', 'infile', 1, ['bru.bp']), p.ParamCmdLineArg ('norm_calc', 'outfile', 2, ['norm_calc.out.bp']), p.ParamCmdLineArg ('norm_calc', 'write_norms_only', 3, [1]), # ParamADIOS2XML can be used to setup a value in the ADIOS xml file for the application # Set the transport to BPFile, as we want the codes to run serially. Set the rc_dependency # in the Sweep to denote dependency between the codes # To couple codes for concurrent execution, use a transport method such as SST p.ParamADIOS2XML ('simulation', 'SimulationOutput', 'engine', [ {"BPFile": {}} ]), p.ParamADIOS2XML ('simulation', 'AnalysisOutput', 'engine', [ {"BPFile": {}} ]), # p.ParamADIOS2XML ('simulation', 'SimulationOutput', 'engine', [ {"BPFile": {'Threads':1}}, # p.ParamADIOS2XML ('simulation', 'SimulationOutput', 'engine', [ {"BPFile": {'Threads':1}}, {"BPFile": {"ProfileUnits": "Microseconds"}} ]), # Use ParamCmdLineOption for named arguments # p.ParamCmdLineOption ('plotting', 'input_stream', '-i', ['bru.bp']), # Use ParamKeyValue to set options in a key-value configuration file. Input file can be a json file. # File path can be relative to the path specified in '-a' or it can be absolute. # File will be copied to the working_dir automatically by Cheetah. # p.ParamKeyValue ('simulation', 'feed_rate', 'input.conf', 'key', ['value']), # Sweep over environment variables # p.ParamEnvVar ('simulation', 'openmp_stuff', 'OMP_NUM_THREADS', [4,8]), # Pass additional scheduler arguments. # p.ParamSchedulerArgs ('simulation', [{'-f':'hosts.txt'}]) ] # Create a sweep # node_layout represents no. of processes per node # rc_dependency denotes dependency between run components. Here, norm_calc will run after simulation has finished sweep1 = p.Sweep (node_layout = {'titan': [{'simulation':16}, {'norm_calc': 4}] }, # simulation: 16 ppn, norm_calc: 4 ppn parameters = sweep1_parameters, rc_dependency={'norm_calc':'simulation'}) # Create a sweep group from the above sweep. You can place multiple sweeps in the group. # Each group is submitted as a separate job. sweepGroup1 = p.SweepGroup ("sg-tmp", walltime=300, per_run_timeout=60, parameter_groups=[sweep1], launch_mode='default', # or MPMD # optional: # nodes=10, # run_repetitions=2, # no. of times each experiment must be repeated (here, total runs = 3) # component_subdirs = True, <-- codes have their own separate workspace in the experiment directory # component_inputs = {'simulation': ['some_input_file'], 'norm_calc': [SymLink('some_large_file')] } <-- inputs required by codes # max_procs = 64 <-- max no. of procs to run concurrently. depends on 'nodes' ) sweepGroup2 = copy.deepcopy(sweepGroup1) sweepGroup2.name = 'sg-2' # Sweep groups to be activated sweeps = [sweepGroup1, sweepGroup2]