def get_schedsystem_config(self, descr): # Handle the special shortcuts first if descr == 'nativeslurm': return getscheduler('slurm'), getlauncher('srun') if descr == 'local': return getscheduler('local'), getlauncher('local') try: sched_descr, launcher_descr = descr.split('+') except ValueError as e: raise ValueError('invalid syntax for the ' 'scheduling system: %s' % descr) from None return getscheduler(sched_descr), getlauncher(launcher_descr)
def _setup_job(self, **job_opts): """Setup the job related to this check.""" self.logger.debug('setting up the job descriptor') msg = 'job scheduler backend: {0}' self.logger.debug( msg.format('local' if self.is_local else self._current_partition. scheduler.registered_name)) # num_gpus_per_node is a managed resource if self.num_gpus_per_node > 0: self.extra_resources.setdefault( '_rfm_gpu', {'num_gpus_per_node': self.num_gpus_per_node}) if self.local: scheduler_type = getscheduler('local') launcher_type = getlauncher('local') else: scheduler_type = self._current_partition.scheduler launcher_type = self._current_partition.launcher job_name = '%s_%s_%s_%s' % ( self.name, self._sanitize_basename(self._current_system.name), self._sanitize_basename(self._current_partition.name), self._sanitize_basename(self._current_environ.name)) job_script_filename = os.path.join(self._stagedir, job_name + '.sh') self._job = scheduler_type( name=job_name, command=' '.join([self.executable] + self.executable_opts), launcher=launcher_type(), environs=[ self._current_partition.local_env, self._current_environ ], workdir=self._stagedir, num_tasks=self.num_tasks, num_tasks_per_node=self.num_tasks_per_node, num_tasks_per_core=self.num_tasks_per_core, num_tasks_per_socket=self.num_tasks_per_socket, num_cpus_per_task=self.num_cpus_per_task, use_smt=self.use_multithreading, time_limit=self.time_limit, script_filename=job_script_filename, stdout=self._stdout, stderr=self._stderr, pre_run=self.pre_run, post_run=self.post_run, sched_exclusive_access=self.exclusive_access, **job_opts) # Get job options from managed resources and prepend them to # job_opts. We want any user supplied options to be able to # override those set by the framework. resources_opts = [] for r, v in self.extra_resources.items(): resources_opts.extend(self._current_partition.get_resource(r, **v)) self._job.options = (self._current_partition.access + resources_opts + self._job.options)
def _setup_fake_check(): # A bit hacky, but we don't want to run a full test every time test = _FakeCheck() test._job = Job.create( getscheduler('local')(), getlauncher('local')(), 'fakejob') test.job._completion_time = time.time() return test
def setUp(self): self.workdir = tempfile.mkdtemp(dir='unittests') slurm_scheduler = getscheduler('slurm') self.testjob = slurm_scheduler( name='testjob', launcher=getlauncher('local')(), workdir=self.workdir, script_filename=os.path.join(self.workdir, 'testjob.sh'), stdout=os.path.join(self.workdir, 'testjob.out'), stderr=os.path.join(self.workdir, 'testjob.err')) # monkey patch `_get_all_nodes` to simulate extraction of # slurm nodes through the use of `scontrol show` self.testjob._get_all_nodes = self.create_dummy_nodes self.testjob._sched_flex_alloc_tasks = 'all' self.testjob._num_tasks_per_node = 4 self.testjob._num_tasks = 0
def _setup_job(self, **job_opts): """Setup the job related to this check.""" self.logger.debug('setting up the job descriptor') msg = 'job scheduler backend: {0}' self.logger.debug( msg.format('local' if self.is_local else self._current_partition.scheduler.registered_name)) # num_gpus_per_node is a managed resource if self.num_gpus_per_node > 0: self.extra_resources.setdefault( '_rfm_gpu', {'num_gpus_per_node': self.num_gpus_per_node} ) if self.local: scheduler_type = getscheduler('local') launcher_type = getlauncher('local') else: scheduler_type = self._current_partition.scheduler launcher_type = self._current_partition.launcher self._job = scheduler_type( name='rfm_%s_job' % self.name, launcher=launcher_type(), workdir=self._stagedir, num_tasks=self.num_tasks, num_tasks_per_node=self.num_tasks_per_node, num_tasks_per_core=self.num_tasks_per_core, num_tasks_per_socket=self.num_tasks_per_socket, num_cpus_per_task=self.num_cpus_per_task, use_smt=self.use_multithreading, time_limit=self.time_limit, sched_access=self._current_partition.access, sched_exclusive_access=self.exclusive_access, **job_opts) # Get job options from managed resources and prepend them to # job_opts. We want any user supplied options to be able to # override those set by the framework. resources_opts = [] for r, v in self.extra_resources.items(): resources_opts.extend( self._current_partition.get_resource(r, **v)) self._job.options = resources_opts + self._job.options
def setUp(self): # Monkey patch scheduler to simulate retrieval of nodes from Slurm patched_sched = getscheduler('slurm')() patched_sched.allnodes = self.create_dummy_nodes patched_sched._get_default_partition = lambda: 'pdef' self.workdir = tempfile.mkdtemp(dir='unittests') self.testjob = Job.create( patched_sched, getlauncher('local')(), name='testjob', workdir=self.workdir, script_filename=os.path.join(self.workdir, 'testjob.sh'), stdout=os.path.join(self.workdir, 'testjob.out'), stderr=os.path.join(self.workdir, 'testjob.err')) self.testjob._sched_flex_alloc_nodes = 'all' self.testjob.num_tasks_per_node = 4 self.testjob.num_tasks = 0
def setUp(self): self.workdir = tempfile.mkdtemp(dir='unittests') slurm_scheduler = getscheduler('slurm') self.testjob = slurm_scheduler( name='testjob', command='hostname', launcher=getlauncher('local')(), environs=[Environment(name='foo')], workdir=self.workdir, script_filename=os.path.join(self.workdir, 'testjob.sh'), stdout=os.path.join(self.workdir, 'testjob.out'), stderr=os.path.join(self.workdir, 'testjob.err')) self.builder = BashScriptBuilder() # monkey patch `_get_reservation_nodes` to simulate extraction of # slurm nodes through the use of `scontrol show` self.testjob._get_reservation_nodes = self.create_dummy_nodes self.testjob._num_tasks_per_node = 4 self.testjob._num_tasks = 0
def job_type(self): return getscheduler(self.sched_name)
def scheduler(self): return getscheduler(self.sched_name)()
def job_type(self): return getscheduler('squeue')
def job_type(self): return getscheduler('slurm')
def job_type(self): return getscheduler('local')
def compile(self): """The compilation phase of the regression test pipeline. :raises reframe.core.exceptions.ReframeError: In case of errors. """ if not self._current_environ: raise PipelineError('no programming environment set') # Copy the check's resources to the stage directory if self.sourcesdir: try: commonpath = os.path.commonpath( [self.sourcesdir, self.sourcepath]) except ValueError: commonpath = None if commonpath: self.logger.warn( "sourcepath `%s' seems to be a subdirectory of " "sourcesdir `%s', but it will be interpreted " "as relative to it." % (self.sourcepath, self.sourcesdir)) if os_ext.is_url(self.sourcesdir): self._clone_to_stagedir(self.sourcesdir) else: self._copy_to_stagedir( os.path.join(self._prefix, self.sourcesdir)) # Verify the sourcepath and determine the sourcepath in the stagedir if (os.path.isabs(self.sourcepath) or os.path.normpath(self.sourcepath).startswith('..')): raise PipelineError( 'self.sourcepath is an absolute path or does not point to a ' 'subfolder or a file contained in self.sourcesdir: ' + self.sourcepath) staged_sourcepath = os.path.join(self._stagedir, self.sourcepath) self.logger.debug('Staged sourcepath: %s' % staged_sourcepath) if os.path.isdir(staged_sourcepath): if not self.build_system: # Try to guess the build system cmakelists = os.path.join(staged_sourcepath, 'CMakeLists.txt') configure_ac = os.path.join(staged_sourcepath, 'configure.ac') configure_in = os.path.join(staged_sourcepath, 'configure.in') if os.path.exists(cmakelists): self.build_system = 'CMake' self.build_system.builddir = 'rfm_build' elif (os.path.exists(configure_ac) or os.path.exists(configure_in)): self.build_system = 'Autotools' self.build_system.builddir = 'rfm_build' else: self.build_system = 'Make' self.build_system.srcdir = self.sourcepath else: if not self.build_system: self.build_system = 'SingleSource' self.build_system.srcfile = self.sourcepath self.build_system.executable = self.executable # Prepare build job build_commands = [ *self.prebuild_cmd, *self.build_system.emit_build_commands(self._current_environ), *self.postbuild_cmd ] environs = [ self._current_partition.local_env, self._current_environ, self._user_environ ] self._build_job = getscheduler('local')( name='rfm_%s_build' % self.name, launcher=getlauncher('local')(), workdir=self._stagedir) with os_ext.change_dir(self._stagedir): try: self._build_job.prepare(build_commands, environs, login=True, trap_errors=True) except OSError as e: raise PipelineError('failed to prepare build job') from e self._build_job.submit()