def launcher(request): if request.param == 'launcherwrapper': # We set the registered_name for the LauncherWrapper just for # convenience for the rest of the unit tests wrapper_cls = launchers.LauncherWrapper wrapper_cls.registered_name = 'launcherwrapper' return wrapper_cls(getlauncher('alps')(), 'ddt', ['--offline']) return getlauncher(request.param)()
def prepare_run(self): if self.current_partition.fullname in ['daint:gpu', 'dom:gpu']: num_workers = 12 exec_cores = 3 else: num_workers = 36 exec_cores = 9 self.variables = { 'SPARK_WORKER_CORES': str(num_workers), 'SPARK_LOCAL_DIRS': '"/tmp"', } self.executable = 'spark-submit' self.executable_opts = [ f'--conf spark.default.parallelism={num_workers}', f'--conf spark.executor.cores={exec_cores}', f'--conf spark.executor.memory=15g', f'--master $SPARKURL' ] if self.variant == 'spark': self.executable_opts += [ '--class org.apache.spark.examples.SparkPi', '$EBROOTSPARK/examples/jars/spark-examples*.jar 10000' ] else: self.executable_opts.append('spark_pi.py') # The job launcher has to be changed since the `spark-submit` # script is not used with srun. self.job.launcher = getlauncher('local')()
def _setup_fake_check(): # A bit hacky, but we don't want to run a full test every time test = _FakeCheck() test._job = Job.create( getscheduler('local')(), getlauncher('local')(), 'fakejob') test.job._completion_time = time.time() return test
def fake_check(): class _FakeCheck(rfm.RegressionTest): pass # A bit hacky, but we don't want to run a full test every time test = _FakeCheck() test._job = Job.create( getscheduler('local')(), getlauncher('local')(), 'fakejob') test.job._completion_time = time.time() return test
def _make_flexible_job(flex_type, **jobargs): ret = Job.create(slurm_scheduler_patched, getlauncher('local')(), name='testjob', workdir=tmp_path, script_filename=str(tmp_path / 'job.sh'), stdout=str(tmp_path / 'job.out'), stderr=str(tmp_path / 'job.err'), sched_flex_alloc_nodes=flex_type, **jobargs) ret.num_tasks = 0 ret.num_tasks_per_node = 4 return ret
def set_profiler(self): # mpirun ko: orte_ess_init failed # --> Returned value No permission (-17) instead of ORTE_SUCCESS # [mpirun -np 1 <self.job.launcher.options>] ./myexe ... # https://github.com/open-mpi/ompi/issues/6981 # https://github.com/open-mpi/ompi/issues/8017 self.job.launcher = getlauncher('local')() self.executable = 'rocprof' self.executable_opts += [ '-i', self.metric_file, '--timestamp', 'on', '--basenames', 'on', f'{self.exe_path}/sedov-cuda', f"-n {self.cubeside}", f"-s {self.steps}" ]
def setup_per_env(self): envname = self.current_environ.name self.prerun_cmds = [ f'source ./shrc', f'mv {self.configs[envname]} config' ] self.executable_opts = [ f'--config={self.configs[envname]}', '--platform NVIDIA', '--tune=base', '--device GPU' ] + self.benchmarks[envname] self.reference = { 'dom:gpu': self.refs[envname], 'daint:gpu': self.refs[envname] } self.sanity_patterns = self.sanity_patterns_[envname] self.perf_patterns = self.perf_patterns_[envname] # The job launcher has to be changed since the `runspec` # script is not used with srun. self.job.launcher = getlauncher('local')()
def prepare_run(self): if self.current_partition.fullname in ['daint:gpu', 'dom:gpu']: num_workers = 12 exec_cores = 3 else: num_workers = 36 exec_cores = 9 self.variables = { 'SPARK_WORKER_CORES': '%s' % num_workers, 'SPARK_LOCAL_DIRS': '"/tmp"', } self.executable = ( 'spark-submit --conf spark.default.parallelism=%s ' '--conf spark.executor.cores=%s --conf spark.executor.memory=15g ' '--master $SPARKURL --class org.apache.spark.examples.SparkPi ' '$EBROOTSPARK/examples/jars/spark-examples_2.11-2.3.1.jar 10000;' % (num_workers, exec_cores)) # The job launcher has to be changed since the `spark-submit` # script is not used with srun. self.job.launcher = getlauncher('local')()
def fake_check(): class _FakeCheck(rfm.RegressionTest): pass @sn.deferrable def error(): raise BaseException # A bit hacky, but we don't want to run a full test every time test = _FakeCheck() test._job = Job.create( getscheduler('local')(), getlauncher('local')(), 'fakejob') test.job._completion_time = time.time() test.job._jobid = 12345 test.job._nodelist = ['localhost'] test.custom = 'hello extras' test.custom_list = ['custom', 3.0, ['hello', 'world']] test.custom_dict = {'a': 1, 'b': 2} test.deferred = sn.defer('hello') test.deferred_error = error() return test
def fake_check(): class _FakeCheck(rfm.RegressionTest): param = parameter(range(3), loggable=True, fmt=lambda x: 10 * x) custom = variable(str, value='hello extras', loggable=True) custom_list = variable(list, value=['custom', 3.0, ['hello', 'world']], loggable=True) custom_dict = variable(dict, value={'a': 1, 'b': 2}, loggable=True) # x is a variable that is loggable, but is left undefined. We want to # make sure that logging does not crash and simply reports is as # undefined x = variable(str, loggable=True) # A bit hacky, but we don't want to run a full test every time test = _FakeCheck(variant_num=1) test._job = Job.create( getscheduler('local')(), getlauncher('local')(), 'fakejob') test.job._completion_time = time.time() test.job._jobid = 12345 test.job._nodelist = ['localhost'] return test
def launcher(self): return getlauncher('srunalloc')()
def create(cls, site_config): # Create the whole system hierarchy from bottom up sysname = site_config.get('systems/0/name') partitions = [] config_save = site_config.subconfig_system for p in site_config.get('systems/0/partitions'): site_config.select_subconfig(f'{sysname}:{p["name"]}') partid = f"systems/0/partitions/@{p['name']}" part_name = site_config.get(f'{partid}/name') part_sched = getscheduler(site_config.get(f'{partid}/scheduler')) part_launcher = getlauncher(site_config.get(f'{partid}/launcher')) part_container_environs = {} for i, p in enumerate( site_config.get(f'{partid}/container_platforms')): ctype = p['type'] part_container_environs[ctype] = Environment( name=f'__rfm_env_{ctype}', modules=site_config.get( f'{partid}/container_platforms/{i}/modules'), variables=site_config.get( f'{partid}/container_platforms/{i}/variables')) part_environs = [ ProgEnvironment( name=e, modules=site_config.get(f'environments/@{e}/modules'), variables=site_config.get(f'environments/@{e}/variables'), cc=site_config.get(f'environments/@{e}/cc'), cxx=site_config.get(f'environments/@{e}/cxx'), ftn=site_config.get(f'environments/@{e}/ftn'), cppflags=site_config.get(f'environments/@{e}/cppflags'), cflags=site_config.get(f'environments/@{e}/cflags'), cxxflags=site_config.get(f'environments/@{e}/cxxflags'), fflags=site_config.get(f'environments/@{e}/fflags'), ldflags=site_config.get(f'environments/@{e}/ldflags')) for e in site_config.get(f'{partid}/environs') ] partitions.append( SystemPartition( parent=site_config.get('systems/0/name'), name=part_name, sched_type=part_sched, launcher_type=part_launcher, descr=site_config.get(f'{partid}/descr'), access=site_config.get(f'{partid}/access'), resources=site_config.get(f'{partid}/resources'), environs=part_environs, container_environs=part_container_environs, local_env=Environment( name=f'__rfm_env_{part_name}', modules=site_config.get(f'{partid}/modules'), variables=site_config.get(f'{partid}/variables')), max_jobs=site_config.get(f'{partid}/max_jobs'), prepare_cmds=site_config.get(f'{partid}/prepare_cmds'), processor=site_config.get(f'{partid}/processor'), devices=site_config.get(f'{partid}/devices'), extras=site_config.get(f'{partid}/extras'))) # Restore configuration, but ignore unresolved sections or # configuration parameters at the system level; if we came up to this # point, then all is good at the partition level, which is enough. site_config.select_subconfig(config_save, ignore_resolve_errors=True) return System( name=sysname, descr=site_config.get('systems/0/descr'), hostnames=site_config.get('systems/0/hostnames'), modules_system=site_config.get('systems/0/modules_system'), preload_env=Environment( name=f'__rfm_env_{sysname}', modules=site_config.get('systems/0/modules'), variables=site_config.get('systems/0/variables')), prefix=site_config.get('systems/0/prefix'), outputdir=site_config.get('systems/0/outputdir'), resourcesdir=site_config.get('systems/0/resourcesdir'), stagedir=site_config.get('systems/0/stagedir'), partitions=partitions)
def set_launcher(self): # The job launcher has to be changed since the `start_analytics` # script is not used with srun. self.job.launcher = getlauncher('local')()
def launcher(self): return getlauncher('ssh')()
def launcher(self): return launchers.LauncherWrapper( getlauncher('alps')(), 'ddt', ['--offline'] )
def launcher(self): return getlauncher('local')()
def set_launcher(self): # The job launcher has to be changed because # the tool can be called without srun self.job.launcher = getlauncher('local')()
def create(cls, site_config): # Create the whole system hierarchy from bottom up sysname = site_config.get('systems/0/name') partitions = [] config_save = site_config.subconfig_system for p in site_config.get('systems/0/partitions'): site_config.select_subconfig(f'{sysname}:{p["name"]}') partid = f"systems/0/partitions/@{p['name']}" part_name = site_config.get(f'{partid}/name') part_sched = getscheduler(site_config.get(f'{partid}/scheduler')) part_launcher = getlauncher(site_config.get(f'{partid}/launcher')) part_container_environs = {} for i, p in enumerate( site_config.get(f'{partid}/container_platforms')): ctype = p['type'] part_container_environs[ctype] = Environment( name=f'__rfm_env_{ctype}', modules=site_config.get( f'{partid}/container_platforms/{i}/modules'), variables=site_config.get( f'{partid}/container_platforms/{i}/variables')) part_environs = [ ProgEnvironment( name=e, modules=site_config.get(f'environments/@{e}/modules'), variables=site_config.get(f'environments/@{e}/variables'), cc=site_config.get(f'environments/@{e}/cc'), cxx=site_config.get(f'environments/@{e}/cxx'), ftn=site_config.get(f'environments/@{e}/ftn'), cppflags=site_config.get(f'environments/@{e}/cppflags'), cflags=site_config.get(f'environments/@{e}/cflags'), cxxflags=site_config.get(f'environments/@{e}/cxxflags'), fflags=site_config.get(f'environments/@{e}/fflags'), ldflags=site_config.get(f'environments/@{e}/ldflags')) for e in site_config.get(f'{partid}/environs') ] partitions.append( SystemPartition( parent=site_config.get('systems/0/name'), name=part_name, scheduler=part_sched, launcher=part_launcher, descr=site_config.get(f'{partid}/descr'), access=site_config.get(f'{partid}/access'), resources=site_config.get(f'{partid}/resources'), environs=part_environs, container_environs=part_container_environs, local_env=Environment( name=f'__rfm_env_{part_name}', modules=site_config.get(f'{partid}/modules'), variables=site_config.get(f'{partid}/variables')), max_jobs=site_config.get(f'{partid}/max_jobs'))) # Restore configuration site_config.select_subconfig(config_save) return System( name=sysname, descr=site_config.get('systems/0/descr'), hostnames=site_config.get('systems/0/hostnames'), modules_system=site_config.get('systems/0/modules_system'), preload_env=Environment( name=f'__rfm_env_{sysname}', modules=site_config.get('systems/0/modules'), variables=site_config.get('systems/0/variables')), prefix=site_config.get('systems/0/prefix'), outputdir=site_config.get('systems/0/outputdir'), resourcesdir=site_config.get('systems/0/resourcesdir'), stagedir=site_config.get('systems/0/stagedir'), partitions=partitions)
def set_job_launcher(self): # The job launcher has to be changed since the `spark-submit` # script is not used with srun. self.job.launcher = getlauncher('local')()
def prepare_run(self): # Change the job launcher since `ipython` # needs to be launched without `srun`. self.job.launcher = getlauncher('local')()
def set_launcher(self): # The job launcher has to be changed because # valgrind4hpc can not be called with srun self.job.launcher = getlauncher('local')()
def launcher(): return getlauncher('local')
def launcher(self): return getlauncher('alps')()
def set_launcher(self): # The job launcher has to be changed to local since greasy # make calls to srun self.job.launcher = getlauncher('local')()
def launcher(self): return getlauncher('mpirun')()
def launcher(self): return getlauncher('mpiexec')()
def launcher(self): return getlauncher('upcxx-run')()