def create(cls, prefix='pbox-'):
"""
Create a new :class:`SessionStorage` in a subdirectory of the base
directory. The directory structure will be created if it does not exist
and will writable by any user.
:param prefix:
String which should prefix all session filenames. The prefix is
sluggified before use.
"""
WellKnownDirsHelper.populate_base()
isoformat = "%Y-%m-%dT%H.%M.%S"
timestamp = datetime.datetime.utcnow().strftime(isoformat)
session_id = "{prefix}{timestamp}".format(prefix=slugify(prefix),
timestamp=timestamp)
uniq = 1
while os.path.exists(WellKnownDirsHelper.session_dir(session_id)):
session_id = "{prefix}{timestamp}_({uniq})".format(
prefix=slugify(prefix), timestamp=timestamp, uniq=uniq)
uniq += 1
session_dir = WellKnownDirsHelper.populate_session(session_id)
logger.debug(_("Created new storage in %r"), session_dir)
self = cls(session_id)
return self
def create(cls, base_dir, legacy_mode=False, prefix='pbox-'):
"""
Create a new :class:`SessionStorage` in a random subdirectory
of the specified base directory. The base directory is also
created if necessary.
:param base_dir:
Directory in which a random session directory will be created.
Typically the base directory should be obtained from
:meth:`SessionStorageRepository.get_default_location()`
:param legacy_mode:
If False (defaults to True) then the caller is expected to
handle multiple sessions by itself.
:param prefix:
String which should prefix all session filenames. The prefix is
sluggified before use.
.. note::
Legacy mode is where applications using PlainBox API can only
handle one session. Creating another session replaces whatever was
stored before. In non-legacy mode applications can enumerate
sessions, create arbitrary number of sessions at the same time
and remove sessions once they are no longer necessary.
Legacy mode is implemented with a symbolic link called
'last-session' that keeps track of the last session created using
``legacy_mode=True``. When a new legacy-mode session is created
the target of that symlink is read and recursively removed.
"""
if not os.path.exists(base_dir):
os.makedirs(base_dir)
isoformat = "%Y-%m-%dT%H.%M.%S"
timestamp = datetime.datetime.utcnow().strftime(isoformat)
location = os.path.join(
base_dir,
"{prefix}{timestamp}{suffix}".format(prefix=slugify(prefix),
timestamp=timestamp,
suffix='.session'))
uniq = 1
while os.path.exists(location):
location = os.path.join(
base_dir, "{prefix}{timestamp}_({uniq}){suffix}".format(
prefix=slugify(prefix),
timestamp=timestamp,
uniq=uniq,
suffix='.session'))
uniq += 1
os.mkdir(location)
logger.debug(_("Created new storage in %r"), location)
self = cls(location)
if legacy_mode:
self._replace_legacy_session(base_dir)
return self
def _run_single_job_with_session(self, ns, session, runner, job):
print("[ {} ]".format(job.name).center(80, '-'))
if job.description is not None:
print(job.description)
print("^" * len(job.description.splitlines()[-1]))
print()
job_state = session.job_state_map[job.name]
logger.debug("Job name: %s", job.name)
logger.debug("Plugin: %s", job.plugin)
logger.debug("Direct dependencies: %s", job.get_direct_dependencies())
logger.debug("Resource dependencies: %s",
job.get_resource_dependencies())
logger.debug("Resource program: %r", job.requires)
logger.debug("Command: %r", job.command)
logger.debug("Can start: %s", job_state.can_start())
logger.debug("Readiness: %s", job_state.get_readiness_description())
if job_state.can_start():
print("Running... (output in {}.*)".format(
join(session.jobs_io_log_dir, slugify(job.name))))
job_result = runner.run_job(job)
print("Outcome: {}".format(job_result.outcome))
print("Comments: {}".format(job_result.comments))
else:
job_result = JobResult({
'job': job,
'outcome': JobResult.OUTCOME_NOT_SUPPORTED,
'comments': job_state.get_readiness_description()
})
if job_result is not None:
session.update_job_result(job, job_result)
def _run_single_job_with_session(self, ns, session, runner, job):
print("[ {} ]".format(job.name).center(80, '-'))
if job.description is not None:
print(job.description)
print("^" * len(job.description.splitlines()[-1]))
print()
job_state = session.job_state_map[job.name]
logger.debug("Job name: %s", job.name)
logger.debug("Plugin: %s", job.plugin)
logger.debug("Direct dependencies: %s", job.get_direct_dependencies())
logger.debug("Resource dependencies: %s",
job.get_resource_dependencies())
logger.debug("Resource program: %r", job.requires)
logger.debug("Command: %r", job.command)
logger.debug("Can start: %s", job_state.can_start())
logger.debug("Readiness: %s", job_state.get_readiness_description())
if job_state.can_start():
print("Running... (output in {}.*)".format(
join(session.jobs_io_log_dir, slugify(job.name))))
session.metadata.running_job_name = job.name
session.persistent_save()
job_result = runner.run_job(job)
session.metadata.running_job_name = None
session.persistent_save()
print("Outcome: {}".format(job_result.outcome))
print("Comments: {}".format(job_result.comments))
else:
job_result = MemoryJobResult({
'outcome':
IJobResult.OUTCOME_NOT_SUPPORTED,
'comments':
job_state.get_readiness_description()
})
if job_result is not None:
session.update_job_result(job, job_result)
def _run_single_job_with_session(self, ns, session, runner, job):
print("[ {} ]".format(job.name).center(80, '-'))
job_state = session.job_state_map[job.name]
print("Job name: {}".format(job.name))
print("Plugin: {}".format(job.plugin))
print("Direct dependencies: {}".format(job.get_direct_dependencies()))
print("Resource dependencies: {}".format(
job.get_resource_dependencies()))
print("Resource program: {!r}".format(job.requires))
print("Command: {!r}".format(job.command))
print("Can start: {}".format(job_state.can_start()))
print("Readiness: {}".format(job_state.get_readiness_description()))
if job_state.can_start():
if ns.dry_run:
print("Not really running anything in dry-run mode")
job_result = JobResult({
'job': job,
'outcome': 'dry-run',
})
else:
print("Running... (output in {}.*)".format(
join(session.jobs_io_log_dir, slugify(job.name))))
job_result = runner.run_job(job)
print("Outcome: {}".format(job_result.outcome))
print("Comments: {}".format(job_result.comments))
else:
job_result = JobResult({
'job': job,
'outcome': JobResult.OUTCOME_NOT_SUPPORTED
})
if job_result is None and not ns.dry_run:
logger.warning("Job %s did not return a result", job)
if job_result is not None:
session.update_job_result(job, job_result)
def _run_command(self, job, environ):
start_time = time.time()
slug = slugify(job.id)
output_writer = CommandOutputWriter(
stdout_path=os.path.join(self._jobs_io_log_dir,
"{}.stdout".format(slug)),
stderr_path=os.path.join(self._jobs_io_log_dir,
"{}.stderr".format(slug)))
io_log_gen = IOLogRecordGenerator()
log = os.path.join(self._jobs_io_log_dir, "{}.record.gz".format(slug))
with gzip.open(log, mode='wb') as gzip_stream, io.TextIOWrapper(
gzip_stream, encoding='UTF-8') as record_stream:
writer = IOLogRecordWriter(record_stream)
io_log_gen.on_new_record.connect(writer.write_record)
delegate = extcmd.Chain([
self._job_runner_ui_delegate, io_log_gen,
self._command_io_delegate, output_writer
])
ecmd = extcmd.ExternalCommandWithDelegate(delegate)
return_code = self.execute_job(job, environ, ecmd, self._stdin)
io_log_gen.on_new_record.disconnect(writer.write_record)
if return_code == 0:
outcome = IJobResult.OUTCOME_PASS
elif return_code < 0:
outcome = IJobResult.OUTCOME_CRASH
else:
outcome = IJobResult.OUTCOME_FAIL
return JobResultBuilder(outcome=outcome,
return_code=return_code,
io_log_filename=log,
execution_duration=time.time() - start_time)
def test_random_strings(self):
self.assertEqual(slugify("A "), "A_")
self.assertEqual(slugify("A-"), "A-")
self.assertEqual(slugify("A_"), "A_")
self.assertEqual(slugify(".b"), ".b")
self.assertEqual(slugify("\z"), "_z")
self.assertEqual(slugify("/z"), "_z")
self.assertEqual(slugify("1k"), "1k")
def test_random_strings(self):
self.assertEqual(slugify("A "), "A_")
self.assertEqual(slugify("A-"), "A-")
self.assertEqual(slugify("A_"), "A_")
self.assertEqual(slugify(".b"), ".b")
self.assertEqual(slugify("\z"), "_z")
self.assertEqual(slugify("/z"), "_z")
self.assertEqual(slugify("1k"), "1k")
def create(cls, base_dir, prefix='pbox-'):
"""
Create a new :class:`SessionStorage` in a random subdirectory
of the specified base directory. The base directory is also
created if necessary.
:param base_dir:
Directory in which a random session directory will be created.
Typically the base directory should be obtained from
:meth:`SessionStorageRepository.get_default_location()`
:param prefix:
String which should prefix all session filenames. The prefix is
sluggified before use.
"""
if not os.path.exists(base_dir):
os.makedirs(base_dir)
isoformat = "%Y-%m-%dT%H.%M.%S"
timestamp = datetime.datetime.utcnow().strftime(isoformat)
location = os.path.join(
base_dir,
"{prefix}{timestamp}{suffix}".format(prefix=slugify(prefix),
timestamp=timestamp,
suffix='.session'))
uniq = 1
while os.path.exists(location):
location = os.path.join(
base_dir, "{prefix}{timestamp}_({uniq}){suffix}".format(
prefix=slugify(prefix),
timestamp=timestamp,
uniq=uniq,
suffix='.session'))
uniq += 1
os.mkdir(location)
logger.debug(_("Created new storage in %r"), location)
self = cls(location)
return self
def _run_single_job_with_session(self, ns, manager, runner, job):
if job.plugin not in ['local', 'resource']:
print("[ {} ]".format(job.tr_summary()).center(80, '-'))
job_state = manager.state.job_state_map[job.id]
logger.debug("Job id: %s", job.id)
logger.debug("Plugin: %s", job.plugin)
logger.debug("Direct dependencies: %s", job.get_direct_dependencies())
logger.debug("Resource dependencies: %s",
job.get_resource_dependencies())
logger.debug("Resource program: %r", job.requires)
logger.debug("Command: %r", job.command)
logger.debug("Can start: %s", job_state.can_start())
logger.debug("Readiness: %s", job_state.get_readiness_description())
if job_state.can_start():
if job.plugin not in ['local', 'resource']:
if job.description is not None:
print(job.description)
print("^" * len(job.description.splitlines()[-1]))
print()
print("Running... (output in {}.*)".format(
join(manager.storage.location, slugify(job.id))))
manager.state.metadata.running_job_name = job.id
manager.checkpoint()
# TODO: get a confirmation from the user for certain types of
# job.plugin
job_result = runner.run_job(job, self.config)
if (job_result.outcome == IJobResult.OUTCOME_UNDECIDED
and self.is_interactive):
job_result = self._interaction_callback(
runner, job, self.config)
manager.state.metadata.running_job_name = None
manager.checkpoint()
if job.plugin not in ['local', 'resource']:
print("Outcome: {}".format(job_result.outcome))
if job_result.comments is not None:
print("Comments: {}".format(job_result.comments))
else:
job_result = MemoryJobResult({
'outcome':
IJobResult.OUTCOME_NOT_SUPPORTED,
'comments':
job_state.get_readiness_description()
})
if job.plugin not in ['local', 'resource']:
print("Outcome: {}".format(job_result.outcome))
if job_result is not None:
manager.state.update_job_result(job, job_result)
def _run_single_job_with_session(self, ns, manager, runner, job):
if job.plugin not in ['local', 'resource']:
print("[ {} ]".format(job.tr_summary()).center(80, '-'))
job_state = manager.state.job_state_map[job.id]
logger.debug("Job id: %s", job.id)
logger.debug("Plugin: %s", job.plugin)
logger.debug("Direct dependencies: %s", job.get_direct_dependencies())
logger.debug("Resource dependencies: %s",
job.get_resource_dependencies())
logger.debug("Resource program: %r", job.requires)
logger.debug("Command: %r", job.command)
logger.debug("Can start: %s", job_state.can_start())
logger.debug("Readiness: %s", job_state.get_readiness_description())
if job_state.can_start():
if job.plugin not in ['local', 'resource']:
if job.description is not None:
print(job.description)
print("^" * len(job.description.splitlines()[-1]))
print()
print("Running... (output in {}.*)".format(
join(manager.storage.location, slugify(job.id))))
manager.state.metadata.running_job_name = job.id
manager.checkpoint()
# TODO: get a confirmation from the user for certain types of
# job.plugin
job_result = runner.run_job(job, self.config)
if (job_result.outcome == IJobResult.OUTCOME_UNDECIDED
and self.is_interactive):
job_result = self._interaction_callback(
runner, job, self.config)
manager.state.metadata.running_job_name = None
manager.checkpoint()
if job.plugin not in ['local', 'resource']:
print("Outcome: {}".format(job_result.outcome))
if job_result.comments is not None:
print("Comments: {}".format(job_result.comments))
else:
job_result = MemoryJobResult({
'outcome': IJobResult.OUTCOME_NOT_SUPPORTED,
'comments': job_state.get_readiness_description()
})
if job.plugin not in ['local', 'resource']:
print("Outcome: {}".format(job_result.outcome))
if job_result is not None:
manager.state.update_job_result(job, job_result)
def _run_single_job_with_session(self, ns, session, runner, job):
print("[ {} ]".format(job.id).center(80, '-'))
if job.description is not None:
print(job.description)
print("^" * len(job.description.splitlines()[-1]))
print()
job_state = session.job_state_map[job.id]
logger.debug(_("Job id: %s"), job.id)
logger.debug(_("Plugin: %s"), job.plugin)
logger.debug(_("Direct dependencies: %s"),
job.get_direct_dependencies())
logger.debug(_("Resource dependencies: %s"),
job.get_resource_dependencies())
logger.debug(_("Resource program: %r"), job.requires)
logger.debug(_("Command: %r"), job.command)
logger.debug(_("Can start: %s"), job_state.can_start())
logger.debug(_("Readiness: %s"), job_state.get_readiness_description())
if job_state.can_start():
print(_("Running... (output in {}.*)").format(
join(session.jobs_io_log_dir, slugify(job.id))))
session.metadata.running_job_name = job.id
session.persistent_save()
# TODO: get a confirmation from the user for certain types of job.plugin
job_result = runner.run_job(job, self.config)
if (job_result.outcome == IJobResult.OUTCOME_UNDECIDED
and self.is_interactive):
job_result = self._interaction_callback(
runner, job, self.config)
session.metadata.running_job_name = None
session.persistent_save()
print(_("Outcome: {}").format(job_result.outcome))
print(_("Comments: {}").format(job_result.comments))
else:
job_result = MemoryJobResult({
'outcome': IJobResult.OUTCOME_NOT_SUPPORTED,
'comments': job_state.get_readiness_description()
})
if job_result is not None:
session.update_job_result(job, job_result)
def get_record_path_for_job(self, job):
return os.path.join(self._jobs_io_log_dir,
"{}.record.gz".format(slugify(job.id)))
