def create(cls, prefix='pbox-'): """ Create a new :class:`SessionStorage` in a subdirectory of the base directory. The directory structure will be created if it does not exist and will writable by any user. :param prefix: String which should prefix all session filenames. The prefix is sluggified before use. """ WellKnownDirsHelper.populate_base() isoformat = "%Y-%m-%dT%H.%M.%S" timestamp = datetime.datetime.utcnow().strftime(isoformat) session_id = "{prefix}{timestamp}".format(prefix=slugify(prefix), timestamp=timestamp) uniq = 1 while os.path.exists(WellKnownDirsHelper.session_dir(session_id)): session_id = "{prefix}{timestamp}_({uniq})".format( prefix=slugify(prefix), timestamp=timestamp, uniq=uniq) uniq += 1 session_dir = WellKnownDirsHelper.populate_session(session_id) logger.debug(_("Created new storage in %r"), session_dir) self = cls(session_id) return self
def create(cls, base_dir, legacy_mode=False, prefix='pbox-'): """ Create a new :class:`SessionStorage` in a random subdirectory of the specified base directory. The base directory is also created if necessary. :param base_dir: Directory in which a random session directory will be created. Typically the base directory should be obtained from :meth:`SessionStorageRepository.get_default_location()` :param legacy_mode: If False (defaults to True) then the caller is expected to handle multiple sessions by itself. :param prefix: String which should prefix all session filenames. The prefix is sluggified before use. .. note:: Legacy mode is where applications using PlainBox API can only handle one session. Creating another session replaces whatever was stored before. In non-legacy mode applications can enumerate sessions, create arbitrary number of sessions at the same time and remove sessions once they are no longer necessary. Legacy mode is implemented with a symbolic link called 'last-session' that keeps track of the last session created using ``legacy_mode=True``. When a new legacy-mode session is created the target of that symlink is read and recursively removed. """ if not os.path.exists(base_dir): os.makedirs(base_dir) isoformat = "%Y-%m-%dT%H.%M.%S" timestamp = datetime.datetime.utcnow().strftime(isoformat) location = os.path.join( base_dir, "{prefix}{timestamp}{suffix}".format(prefix=slugify(prefix), timestamp=timestamp, suffix='.session')) uniq = 1 while os.path.exists(location): location = os.path.join( base_dir, "{prefix}{timestamp}_({uniq}){suffix}".format( prefix=slugify(prefix), timestamp=timestamp, uniq=uniq, suffix='.session')) uniq += 1 os.mkdir(location) logger.debug(_("Created new storage in %r"), location) self = cls(location) if legacy_mode: self._replace_legacy_session(base_dir) return self
def _run_single_job_with_session(self, ns, session, runner, job): print("[ {} ]".format(job.name).center(80, '-')) if job.description is not None: print(job.description) print("^" * len(job.description.splitlines()[-1])) print() job_state = session.job_state_map[job.name] logger.debug("Job name: %s", job.name) logger.debug("Plugin: %s", job.plugin) logger.debug("Direct dependencies: %s", job.get_direct_dependencies()) logger.debug("Resource dependencies: %s", job.get_resource_dependencies()) logger.debug("Resource program: %r", job.requires) logger.debug("Command: %r", job.command) logger.debug("Can start: %s", job_state.can_start()) logger.debug("Readiness: %s", job_state.get_readiness_description()) if job_state.can_start(): print("Running... (output in {}.*)".format( join(session.jobs_io_log_dir, slugify(job.name)))) job_result = runner.run_job(job) print("Outcome: {}".format(job_result.outcome)) print("Comments: {}".format(job_result.comments)) else: job_result = JobResult({ 'job': job, 'outcome': JobResult.OUTCOME_NOT_SUPPORTED, 'comments': job_state.get_readiness_description() }) if job_result is not None: session.update_job_result(job, job_result)
def _run_single_job_with_session(self, ns, session, runner, job): print("[ {} ]".format(job.name).center(80, '-')) if job.description is not None: print(job.description) print("^" * len(job.description.splitlines()[-1])) print() job_state = session.job_state_map[job.name] logger.debug("Job name: %s", job.name) logger.debug("Plugin: %s", job.plugin) logger.debug("Direct dependencies: %s", job.get_direct_dependencies()) logger.debug("Resource dependencies: %s", job.get_resource_dependencies()) logger.debug("Resource program: %r", job.requires) logger.debug("Command: %r", job.command) logger.debug("Can start: %s", job_state.can_start()) logger.debug("Readiness: %s", job_state.get_readiness_description()) if job_state.can_start(): print("Running... (output in {}.*)".format( join(session.jobs_io_log_dir, slugify(job.name)))) session.metadata.running_job_name = job.name session.persistent_save() job_result = runner.run_job(job) session.metadata.running_job_name = None session.persistent_save() print("Outcome: {}".format(job_result.outcome)) print("Comments: {}".format(job_result.comments)) else: job_result = MemoryJobResult({ 'outcome': IJobResult.OUTCOME_NOT_SUPPORTED, 'comments': job_state.get_readiness_description() }) if job_result is not None: session.update_job_result(job, job_result)
def _run_single_job_with_session(self, ns, session, runner, job): print("[ {} ]".format(job.name).center(80, '-')) job_state = session.job_state_map[job.name] print("Job name: {}".format(job.name)) print("Plugin: {}".format(job.plugin)) print("Direct dependencies: {}".format(job.get_direct_dependencies())) print("Resource dependencies: {}".format( job.get_resource_dependencies())) print("Resource program: {!r}".format(job.requires)) print("Command: {!r}".format(job.command)) print("Can start: {}".format(job_state.can_start())) print("Readiness: {}".format(job_state.get_readiness_description())) if job_state.can_start(): if ns.dry_run: print("Not really running anything in dry-run mode") job_result = JobResult({ 'job': job, 'outcome': 'dry-run', }) else: print("Running... (output in {}.*)".format( join(session.jobs_io_log_dir, slugify(job.name)))) job_result = runner.run_job(job) print("Outcome: {}".format(job_result.outcome)) print("Comments: {}".format(job_result.comments)) else: job_result = JobResult({ 'job': job, 'outcome': JobResult.OUTCOME_NOT_SUPPORTED }) if job_result is None and not ns.dry_run: logger.warning("Job %s did not return a result", job) if job_result is not None: session.update_job_result(job, job_result)
def _run_command(self, job, environ): start_time = time.time() slug = slugify(job.id) output_writer = CommandOutputWriter( stdout_path=os.path.join(self._jobs_io_log_dir, "{}.stdout".format(slug)), stderr_path=os.path.join(self._jobs_io_log_dir, "{}.stderr".format(slug))) io_log_gen = IOLogRecordGenerator() log = os.path.join(self._jobs_io_log_dir, "{}.record.gz".format(slug)) with gzip.open(log, mode='wb') as gzip_stream, io.TextIOWrapper( gzip_stream, encoding='UTF-8') as record_stream: writer = IOLogRecordWriter(record_stream) io_log_gen.on_new_record.connect(writer.write_record) delegate = extcmd.Chain([ self._job_runner_ui_delegate, io_log_gen, self._command_io_delegate, output_writer ]) ecmd = extcmd.ExternalCommandWithDelegate(delegate) return_code = self.execute_job(job, environ, ecmd, self._stdin) io_log_gen.on_new_record.disconnect(writer.write_record) if return_code == 0: outcome = IJobResult.OUTCOME_PASS elif return_code < 0: outcome = IJobResult.OUTCOME_CRASH else: outcome = IJobResult.OUTCOME_FAIL return JobResultBuilder(outcome=outcome, return_code=return_code, io_log_filename=log, execution_duration=time.time() - start_time)
def test_random_strings(self): self.assertEqual(slugify("A "), "A_") self.assertEqual(slugify("A-"), "A-") self.assertEqual(slugify("A_"), "A_") self.assertEqual(slugify(".b"), ".b") self.assertEqual(slugify("\z"), "_z") self.assertEqual(slugify("/z"), "_z") self.assertEqual(slugify("1k"), "1k")
def create(cls, base_dir, prefix='pbox-'): """ Create a new :class:`SessionStorage` in a random subdirectory of the specified base directory. The base directory is also created if necessary. :param base_dir: Directory in which a random session directory will be created. Typically the base directory should be obtained from :meth:`SessionStorageRepository.get_default_location()` :param prefix: String which should prefix all session filenames. The prefix is sluggified before use. """ if not os.path.exists(base_dir): os.makedirs(base_dir) isoformat = "%Y-%m-%dT%H.%M.%S" timestamp = datetime.datetime.utcnow().strftime(isoformat) location = os.path.join( base_dir, "{prefix}{timestamp}{suffix}".format(prefix=slugify(prefix), timestamp=timestamp, suffix='.session')) uniq = 1 while os.path.exists(location): location = os.path.join( base_dir, "{prefix}{timestamp}_({uniq}){suffix}".format( prefix=slugify(prefix), timestamp=timestamp, uniq=uniq, suffix='.session')) uniq += 1 os.mkdir(location) logger.debug(_("Created new storage in %r"), location) self = cls(location) return self
def _run_single_job_with_session(self, ns, manager, runner, job): if job.plugin not in ['local', 'resource']: print("[ {} ]".format(job.tr_summary()).center(80, '-')) job_state = manager.state.job_state_map[job.id] logger.debug("Job id: %s", job.id) logger.debug("Plugin: %s", job.plugin) logger.debug("Direct dependencies: %s", job.get_direct_dependencies()) logger.debug("Resource dependencies: %s", job.get_resource_dependencies()) logger.debug("Resource program: %r", job.requires) logger.debug("Command: %r", job.command) logger.debug("Can start: %s", job_state.can_start()) logger.debug("Readiness: %s", job_state.get_readiness_description()) if job_state.can_start(): if job.plugin not in ['local', 'resource']: if job.description is not None: print(job.description) print("^" * len(job.description.splitlines()[-1])) print() print("Running... (output in {}.*)".format( join(manager.storage.location, slugify(job.id)))) manager.state.metadata.running_job_name = job.id manager.checkpoint() # TODO: get a confirmation from the user for certain types of # job.plugin job_result = runner.run_job(job, self.config) if (job_result.outcome == IJobResult.OUTCOME_UNDECIDED and self.is_interactive): job_result = self._interaction_callback( runner, job, self.config) manager.state.metadata.running_job_name = None manager.checkpoint() if job.plugin not in ['local', 'resource']: print("Outcome: {}".format(job_result.outcome)) if job_result.comments is not None: print("Comments: {}".format(job_result.comments)) else: job_result = MemoryJobResult({ 'outcome': IJobResult.OUTCOME_NOT_SUPPORTED, 'comments': job_state.get_readiness_description() }) if job.plugin not in ['local', 'resource']: print("Outcome: {}".format(job_result.outcome)) if job_result is not None: manager.state.update_job_result(job, job_result)
def _run_single_job_with_session(self, ns, session, runner, job): print("[ {} ]".format(job.id).center(80, '-')) if job.description is not None: print(job.description) print("^" * len(job.description.splitlines()[-1])) print() job_state = session.job_state_map[job.id] logger.debug(_("Job id: %s"), job.id) logger.debug(_("Plugin: %s"), job.plugin) logger.debug(_("Direct dependencies: %s"), job.get_direct_dependencies()) logger.debug(_("Resource dependencies: %s"), job.get_resource_dependencies()) logger.debug(_("Resource program: %r"), job.requires) logger.debug(_("Command: %r"), job.command) logger.debug(_("Can start: %s"), job_state.can_start()) logger.debug(_("Readiness: %s"), job_state.get_readiness_description()) if job_state.can_start(): print(_("Running... (output in {}.*)").format( join(session.jobs_io_log_dir, slugify(job.id)))) session.metadata.running_job_name = job.id session.persistent_save() # TODO: get a confirmation from the user for certain types of job.plugin job_result = runner.run_job(job, self.config) if (job_result.outcome == IJobResult.OUTCOME_UNDECIDED and self.is_interactive): job_result = self._interaction_callback( runner, job, self.config) session.metadata.running_job_name = None session.persistent_save() print(_("Outcome: {}").format(job_result.outcome)) print(_("Comments: {}").format(job_result.comments)) else: job_result = MemoryJobResult({ 'outcome': IJobResult.OUTCOME_NOT_SUPPORTED, 'comments': job_state.get_readiness_description() }) if job_result is not None: session.update_job_result(job, job_result)
def get_record_path_for_job(self, job): return os.path.join(self._jobs_io_log_dir, "{}.record.gz".format(slugify(job.id)))