def test_build_IOLogRecord_non_ascii_data(self):
"""
verify that _build_IOLogRecord() checks that ``data`` is ASCII
"""
with self.assertRaises(CorruptedSessionError) as boom:
SessionResumeHelper._build_IOLogRecord([0.0, 'stdout', '\uFFFD'])
self.assertIsInstance(boom.exception.__context__, UnicodeEncodeError)
def test_simple_session(self):
"""
verify that _restore_SessionState_jobs_and_results() works when
faced with a representation of a simple session (no generated jobs
or anything "exotic").
"""
job = make_job(name='job')
session_repr = {
'jobs': {
job.name: job.get_checksum(),
},
'results': {
job.name: [{
'outcome': 'pass',
'comments': None,
'execution_duration': None,
'return_code': None,
'io_log': [],
}]
}
}
helper = SessionResumeHelper([job])
session = SessionState([job])
helper._restore_SessionState_jobs_and_results(session, session_repr)
# Session still has one job in it
self.assertEqual(session.job_list, [job])
# Resources don't have anything (no resource jobs)
self.assertEqual(session.resource_map, {})
# The result was restored correctly. This is just a smoke test
# as specific tests for restoring results are written elsewhere
self.assertEqual(
session.job_state_map[job.name].result.outcome, 'pass')
def test_build_IOLogRecord_bad_type_stream_name(self):
"""
verify that _build_IOLogRecord() checks that ``stream-name``
is a string
"""
with self.assertRaises(CorruptedSessionError):
SessionResumeHelper._build_IOLogRecord([0.0, 1])
def test_build_IOLogRecord_non_base64_ascii_data(self):
"""
verify that _build_IOLogRecord() checks that ``data`` is valid base64
"""
with self.assertRaises(CorruptedSessionError) as boom:
SessionResumeHelper._build_IOLogRecord([0.0, 'stdout', '==broken'])
# base64.standard_b64decode() raises binascii.Error
self.assertIsInstance(boom.exception.__context__, binascii.Error)
def test_build_JobResult_checks_for_missing_io_log(self):
"""
verify that _build_JobResult() checks if ``io_log`` is present
"""
with self.assertRaises(CorruptedSessionError) as boom:
obj_repr = copy.copy(self.good_repr)
del obj_repr['io_log']
SessionResumeHelper._build_JobResult(obj_repr)
self.assertEqual(
str(boom.exception), "Missing value for key 'io_log'")
def test_build_JobResult_checks_type_of_return_code(self):
"""
verify that _build_JobResult() checks if ``return_code`` is an integer
"""
with self.assertRaises(CorruptedSessionError) as boom:
obj_repr = copy.copy(self.good_repr)
obj_repr['return_code'] = "text"
SessionResumeHelper._build_JobResult(obj_repr)
self.assertEqual(
str(boom.exception),
"Value of key 'return_code' is of incorrect type str")
def test_build_JobResult_checks_type_of_outcome(self):
"""
verify that _build_JobResult() checks if ``outcome`` is a string
"""
with self.assertRaises(CorruptedSessionError) as boom:
obj_repr = copy.copy(self.good_repr)
obj_repr['outcome'] = 42
SessionResumeHelper._build_JobResult(obj_repr)
self.assertEqual(
str(boom.exception),
"Value of key 'outcome' is of incorrect type int")
def test_build_JobResult_checks_type_of_comments(self):
"""
verify that _build_JobResult() checks if ``comments`` is a string
"""
with self.assertRaises(CorruptedSessionError) as boom:
obj_repr = copy.copy(self.good_repr)
obj_repr['comments'] = False
SessionResumeHelper._build_JobResult(obj_repr)
self.assertEqual(
str(boom.exception),
"Value of key 'comments' is of incorrect type bool")
def test_build_JobResult_checks_type_of_execution_duration(self):
"""
verify that _build_JobResult() checks if ``execution_duration``
is a float
"""
with self.assertRaises(CorruptedSessionError) as boom:
obj_repr = copy.copy(self.good_repr)
obj_repr['execution_duration'] = "text"
SessionResumeHelper._build_JobResult(obj_repr)
self.assertEqual(
str(boom.exception),
"Value of key 'execution_duration' is of incorrect type str")
def test_build_JobResult_checks_for_none_io_log_filename(self):
"""
verify that _build_JobResult() checks if the value of
``io_log_filename`` is not None
"""
with self.assertRaises(CorruptedSessionError) as boom:
obj_repr = copy.copy(self.good_repr)
obj_repr['io_log_filename'] = None
SessionResumeHelper._build_JobResult(obj_repr)
self.assertEqual(
str(boom.exception),
"Value of key 'io_log_filename' cannot be None")
def test_build_JobResult_checks_value_of_outcome(self):
"""
verify that _build_JobResult() checks if the value of ``outcome`` is
in the set of known-good values.
"""
with self.assertRaises(CorruptedSessionError) as boom:
obj_repr = copy.copy(self.good_repr)
obj_repr['outcome'] = 'maybe'
SessionResumeHelper._build_JobResult(obj_repr)
self.assertEqual(
str(boom.exception), (
"Value for key 'outcome' not in allowed set [None, 'pass', "
"'fail', 'skip', 'not-supported', 'not-implemented', "
"'undecided']"))
def test_build_JobResult_does_not_check_for_missing_io_log_filename(self):
"""
verify that _build_JobResult() does not check if
``io_log_filename`` is present as that signifies that MemoryJobResult
should be recreated instead
"""
with self.assertRaises(CorruptedSessionError) as boom:
obj_repr = copy.copy(self.good_repr)
del obj_repr['io_log_filename']
SessionResumeHelper._build_JobResult(obj_repr)
# NOTE: the error message explicitly talks about 'io_log', not
# about 'io_log_filename' because we're hitting the other path
# of the restore function
self.assertEqual(
str(boom.exception), "Missing value for key 'io_log'")
def test_empty_session(self):
"""
verify that _restore_SessionState_jobs_and_results() works when
faced with a representation of an empty session. This is mostly
to do sanity checking on the 'easy' parts of the code before
testing specific cases in the rest of the code.
"""
session_repr = {
'jobs': {},
'results': {}
}
helper = SessionResumeHelper([])
session = SessionState([])
helper._restore_SessionState_jobs_and_results(session, session_repr)
self.assertEqual(session.job_list, [])
self.assertEqual(session.resource_map, {})
self.assertEqual(session.job_state_map, {})
def test_build_JobResult_restores_outcome(self):
"""
verify that _build_JobResult() restores the value of ``outcome``
"""
obj_repr = copy.copy(self.good_repr)
obj_repr['outcome'] = 'fail'
obj = SessionResumeHelper._build_JobResult(obj_repr)
self.assertEqual(obj.outcome, 'fail')
def test_build_JobResult_restores_return_code(self):
"""
verify that _build_JobResult() restores the value of ``return_code``
"""
obj_repr = copy.copy(self.good_repr)
obj_repr['return_code'] = 42
obj = SessionResumeHelper._build_JobResult(obj_repr)
self.assertEqual(obj.return_code, 42)
def test_build_JobResult_restores_comments(self):
"""
verify that _build_JobResult() restores the value of ``comments``
"""
obj_repr = copy.copy(self.good_repr)
obj_repr['comments'] = 'this is a comment'
obj = SessionResumeHelper._build_JobResult(obj_repr)
self.assertEqual(obj.comments, 'this is a comment')
def load_session(cls, unit_list, storage, early_cb=None, flags=None):
"""
Load a previously checkpointed session.
This method allows one to re-open a session that was previously
created by :meth:`SessionManager.checkpoint()`
:param unit_list:
List of all known units. This argument is used to reconstruct the
session from a dormant state. Since the suspended data cannot
capture implementation details of each unit reliably, actual units
need to be provided externally. Unlike in :meth:`create_session()`
this list really needs to be complete, it must also include any
generated units.
:param storage:
The storage that should be used for this particular session.
The storage object holds references to existing directories
in the file system. When restoring an existing dormant session
it is important to use the correct storage object, the one that
corresponds to the file system location used be the session
before it was saved.
:ptype storage:
:class:`~plainbox.impl.session.storage.SessionStorage`
:param early_cb:
A callback that allows the caller to "see" the session object
early, before the bulk of resume operation happens. This method can
be used to register callbacks on the new session before this method
call returns. The callback accepts one argument, session, which is
being resumed. This is being passed directly to
:meth:`plainbox.impl.session.resume.SessionResumeHelper.resume()`
:param flags:
An optional set of flags that may influence the resume process.
Currently this is an internal implementation detail and no "public"
flags are provided. Passing None here is a safe equvalent of using
this API before it was introduced.
:raises:
Anything that can be raised by
:meth:`~plainbox.impl.session.storage.SessionStorage.
load_checkpoint()` and :meth:`~plainbox.impl.session.suspend.
SessionResumeHelper.resume()`
:returns:
Fresh instance of :class:`SessionManager`
"""
logger.debug("SessionManager.load_session()")
try:
data = storage.load_checkpoint()
except IOError as exc:
if exc.errno == errno.ENOENT:
state = SessionState(unit_list)
else:
raise
else:
state = SessionResumeHelper(unit_list, flags,
storage.location).resume(
data, early_cb)
context = SessionDeviceContext(state)
return cls([context], storage)
def test_build_JobResult_restores_execution_duration(self):
"""
verify that _build_JobResult() restores the value of
``execution_duration``
"""
obj_repr = copy.copy(self.good_repr)
obj_repr['execution_duration'] = 5.1
obj = SessionResumeHelper._build_JobResult(obj_repr)
self.assertAlmostEqual(obj.execution_duration, 5.1)
def test_build_JobResult_allows_for_none_execution_duration(self):
"""
verify that _build_JobResult() allows for the value of
``execution_duration`` to be None
"""
obj_repr = copy.copy(self.good_repr)
obj_repr['execution_duration'] = None
obj = SessionResumeHelper._build_JobResult(obj_repr)
self.assertEqual(obj.execution_duration, None)
def test_build_JobResult_allows_for_none_return_code(self):
"""
verify that _build_JobResult() allows for the value of ``return_code``
to be None
"""
obj_repr = copy.copy(self.good_repr)
obj_repr['return_code'] = None
obj = SessionResumeHelper._build_JobResult(obj_repr)
self.assertEqual(obj.return_code, None)
def test_build_JobResult_allows_none_outcome(self):
"""
verify that _build_JobResult() allows for the value of ``outcome`` to
be None
"""
obj_repr = copy.copy(self.good_repr)
obj_repr['outcome'] = None
obj = SessionResumeHelper._build_JobResult(obj_repr)
self.assertEqual(obj.outcome, None)
def test_build_JobResult_allows_for_none_comments(self):
"""
verify that _build_JobResult() allows for the value of ``comments``
to be None
"""
obj_repr = copy.copy(self.good_repr)
obj_repr['comments'] = None
obj = SessionResumeHelper._build_JobResult(obj_repr)
self.assertEqual(obj.comments, None)
def test_build_JobResult_restores_io_log_filename(self):
"""
verify that _build_JobResult() restores the value of
``io_log_filename`` DiskJobResult representations
"""
obj_repr = copy.copy(self.good_repr)
obj_repr['io_log_filename'] = "some-file.txt"
obj = SessionResumeHelper._build_JobResult(obj_repr)
self.assertEqual(obj.io_log_filename, "some-file.txt")
def test_build_IOLogRecord_values(self):
"""
verify that _build_IOLogRecord() returns a proper IOLogRecord object
with all the values in order
"""
record = SessionResumeHelper._build_IOLogRecord(
[1.5, 'stderr', 'dGhpcyB3b3Jrcw=='])
self.assertAlmostEqual(record.delay, 1.5)
self.assertEqual(record.stream_name, 'stderr')
self.assertEqual(record.data, b"this works")
def test_session_with_generated_jobs(self):
"""
verify that _restore_SessionState_jobs_and_results() works when
faced with a representation of a non-trivial session where one
job generates another one.
"""
parent = make_job(name='parent', plugin='local')
# The child job is only here so that we can get the checksum.
# We don't actually introduce it into the resume machinery
# caveat: make_job() has a default value for
# plugin='dummy' which we don't want here
child = make_job(name='child', plugin=None)
session_repr = {
'jobs': {
parent.name: parent.get_checksum(),
child.name: child.get_checksum(),
},
'results': {
parent.name: [{
'outcome': 'pass',
'comments': None,
'execution_duration': None,
'return_code': None,
'io_log': [
# This record will generate a job identical
# to the 'child' job defined above.
[0.0, 'stdout', base64.standard_b64encode(
b'name: child\n'
).decode('ASCII')]
],
}],
child.name: [],
}
}
# We only pass the parent to the helper! Child will be re-created
helper = SessionResumeHelper([parent])
session = SessionState([parent])
helper._restore_SessionState_jobs_and_results(session, session_repr)
# We should now have two jobs, parent and child
self.assertEqual(session.job_list, [parent, child])
# Resources don't have anything (no resource jobs)
self.assertEqual(session.resource_map, {})
def test_unknown_jobs_get_reported(self):
"""
verify that _restore_SessionState_jobs_and_results() reports
all unresolved jobs (as CorruptedSessionError exception)
"""
session_repr = {
'jobs': {
'job-name': 'job-checksum',
},
'results': {
'job-name': []
}
}
helper = SessionResumeHelper([])
session = SessionState([])
with self.assertRaises(CorruptedSessionError) as boom:
helper._restore_SessionState_jobs_and_results(
session, session_repr)
self.assertEqual(
str(boom.exception), "Unknown jobs remaining: job-name")
def test_process_job_restores_jobs(self):
"""
verify that _process_job() recreates generated jobs
"""
# Set the stage for testing. Setup a session with a known
# local job, representation of the job (checksum)
# and representation of a single result, which has a single line
# that defines a 'name': 'generated' job.
job_name = 'local'
job = make_job(name=job_name, plugin='local')
jobs_repr = {
job_name: job.get_checksum()
}
results_repr = {
job_name: [{
'outcome': None,
'comments': None,
'execution_duration': None,
'return_code': None,
'io_log': [
[0.0, 'stdout', base64.standard_b64encode(
b'name: generated'
).decode('ASCII')]
],
}]
}
helper = SessionResumeHelper([job])
session = SessionState([job])
# Ensure that the 'generated' job was not there initially
self.assertNotIn('generated', session.job_state_map)
self.assertEqual(session.job_list, [job])
# Process the representation data defined above
helper._process_job(session, jobs_repr, results_repr, job_name)
# Ensure that we now have the 'generated' job in the job_state_map
self.assertIn('generated', session.job_state_map)
# And that it looks right
self.assertEqual(
session.job_state_map['generated'].job.name, 'generated')
self.assertIn(
session.job_state_map['generated'].job, session.job_list)
def test_process_job_restores_resources(self):
"""
verify that _process_job() recreates resources
"""
# Set the stage for testing. Setup a session with a known
# resource job, representation of the job (checksum)
# and representation of a single result, which has a single line
# that defines a 'key': 'value' resource record.
job_name = 'resource'
job = make_job(name=job_name, plugin='resource')
jobs_repr = {
job_name: job.get_checksum()
}
results_repr = {
job_name: [{
'outcome': None,
'comments': None,
'execution_duration': None,
'return_code': None,
'io_log': [
# A bit convoluted but this is how we encode each chunk
# of IOLogRecord
[0.0, 'stdout', base64.standard_b64encode(
b'key: value'
).decode('ASCII')]
],
}]
}
helper = SessionResumeHelper([job])
session = SessionState([job])
# Ensure that the resource was not there initially
self.assertNotIn(job_name, session.resource_map)
# Process the representation data defined above
helper._process_job(session, jobs_repr, results_repr, job_name)
# Ensure that we now have the resource in the resource map
self.assertIn(job_name, session.resource_map)
# And that it looks right
self.assertEqual(
session.resource_map[job_name],
[Resource({'key': 'value'})])
def test_build_JobResult_restores_io_log(self):
"""
verify that _build_JobResult() checks if ``io_log``
is restored for MemoryJobResult representations
"""
obj_repr = copy.copy(self.good_repr)
obj_repr['io_log'] = [[0.0, 'stdout', '']]
obj = SessionResumeHelper._build_JobResult(obj_repr)
# NOTE: MemoryJobResult.io_log is a property that converts
# whatever was stored to IOLogRecord and returns a _tuple_
# so the original list is not visible
self.assertEqual(obj.io_log, tuple([
IOLogRecord(0.0, 'stdout', b'')
]))
def load_session(cls, job_list, storage, early_cb=None):
"""
Open a previously checkpointed session.
This method allows one to re-open a session that was previously
created by :meth:`SessionManager.checkpoint()`
:param job_list:
List of all known jobs. This argument is used to reconstruct the
session from a dormant state. Since the suspended data cannot
capture implementation details of each job reliably actual jobs
need to be provided externally. Unlike in :meth:`create_session()`
this list really needs to be complete, it must also include
any generated jobs.
:param storage:
The storage that should be used for this particular session.
The storage object holds references to existing directories
in the file system. When restoring an existing dormant session
it is important to use the correct storage object, the one that
corresponds to the file system location used be the session
before it was saved.
:ptype storage:
:class:`~plainbox.impl.session.storage.SessionStorage`
:param early_cb:
A callback that allows the caller to "see" the session object
early, before the bulk of resume operation happens. This method can
be used to register callbacks on the new session before this method
call returns. The callback accepts one argument, session, which is
being resumed. This is being passed directly to
:meth:`plainbox.impl.session.resume.SessionResumeHelper.resume()`
:raises:
Anything that can be raised by
:meth:`~plainbox.impl.session.storage.SessionStorage.
load_checkpoint()` and :meth:`~plainbox.impl.session.suspend.
SessionResumeHelper.resume()`
:returns:
Fresh instance of :class:`SessionManager`
"""
logger.debug("SessionManager.open_session()")
data = storage.load_checkpoint()
state = SessionResumeHelper(job_list).resume(data, early_cb)
return cls(state, storage)
def setUp(self):
self.job_name = 'job'
self.job = make_job(name=self.job_name)
self.jobs_repr = {
self.job_name: self.job.get_checksum()
}
self.results_repr = {
self.job_name: [{
'outcome': 'fail',
'comments': None,
'execution_duration': None,
'return_code': None,
'io_log': [],
}]
}
self.helper = SessionResumeHelper([self.job])
# This object is artificial and would be constructed internally
# by the helper but having it here makes testing easier as we
# can reliably test a single method in isolation.
self.session = SessionState([self.job])
class ProcessJobTests(TestCase):
"""
Tests for :class:`~plainbox.impl.session.resume.SessionResumeHelper`
and how it handles processing jobs using _process_job() method
"""
def setUp(self):
self.job_name = 'job'
self.job = make_job(name=self.job_name)
self.jobs_repr = {
self.job_name: self.job.get_checksum()
}
self.results_repr = {
self.job_name: [{
'outcome': 'fail',
'comments': None,
'execution_duration': None,
'return_code': None,
'io_log': [],
}]
}
self.helper = SessionResumeHelper([self.job])
# This object is artificial and would be constructed internally
# by the helper but having it here makes testing easier as we
# can reliably test a single method in isolation.
self.session = SessionState([self.job])
def test_process_job_checks_type_of_job_name(self):
"""
verify that _process_job() checks the type of ``job_name``
"""
with self.assertRaises(CorruptedSessionError) as boom:
# Pass a job name of the wrong type
job_name = 1
self.helper._process_job(
self.session, self.jobs_repr, self.results_repr, job_name)
self.assertEqual(
str(boom.exception), "Value of object is of incorrect type int")
def test_process_job_checks_for_missing_checksum(self):
"""
verify that _process_job() checks if ``checksum`` is missing
"""
with self.assertRaises(CorruptedSessionError) as boom:
# Pass a jobs_repr that has no checksums (for any job)
jobs_repr = {}
self.helper._process_job(
self.session, jobs_repr, self.results_repr, self.job_name)
self.assertEqual(str(boom.exception), "Missing value for key 'job'")
def test_process_job_checks_if_job_is_known(self):
"""
verify that _process_job() checks if job is known or raises KeyError
"""
with self.assertRaises(KeyError) as boom:
# Pass a session that does not know about any jobs
session = SessionState([])
self.helper._process_job(
session, self.jobs_repr, self.results_repr, self.job_name)
self.assertEqual(boom.exception.args[0], 'job')
def test_process_job_checks_if_job_checksum_matches(self):
"""
verify that _process_job() checks if job checksum matches the
checksum of a job with the same name that was passed to the helper.
"""
with self.assertRaises(IncompatibleJobError) as boom:
# Pass a jobs_repr with a bad checksum
jobs_repr = {self.job_name: 'bad-checksum'}
self.helper._process_job(
self.session, jobs_repr, self.results_repr, self.job_name)
self.assertEqual(
str(boom.exception), "Definition of job 'job' has changed")
def test_process_job_handles_ignores_empty_results(self):
"""
verify that _process_job() does not crash if we have no results
for a particular job
"""
self.assertEqual(
self.session.job_state_map[self.job_name].result.outcome, None)
results_repr = {
self.job_name: []
}
self.helper._process_job(
self.session, self.jobs_repr, results_repr, self.job_name)
self.assertEqual(
self.session.job_state_map[self.job_name].result.outcome, None)
def test_process_job_handles_only_result_back_to_the_session(self):
"""
verify that _process_job() passes the only result to the session
"""
self.assertEqual(
self.session.job_state_map[self.job_name].result.outcome, None)
self.helper._process_job(
self.session, self.jobs_repr, self.results_repr, self.job_name)
# The result in self.results_repr is a failure so we should see it here
self.assertEqual(
self.session.job_state_map[self.job_name].result.outcome, "fail")
def test_process_job_handles_last_result_back_to_the_session(self):
"""
verify that _process_job() passes last of the results to the session
"""
self.assertEqual(
self.session.job_state_map[self.job_name].result.outcome, None)
results_repr = {
self.job_name: [{
'outcome': 'fail',
'comments': None,
'execution_duration': None,
'return_code': None,
'io_log': [],
}, {
'outcome': 'pass',
'comments': None,
'execution_duration': None,
'return_code': None,
'io_log': [],
}]
}
self.helper._process_job(
self.session, self.jobs_repr, results_repr, self.job_name)
# results_repr has two entries: [fail, pass] so we should see
# the passing entry only
self.assertEqual(
self.session.job_state_map[self.job_name].result.outcome, "pass")
def test_process_job_checks_results_repr_is_a_list(self):
"""
verify that _process_job() checks if results_repr is a dictionary
of lists.
"""
with self.assertRaises(CorruptedSessionError) as boom:
results_repr = {self.job_name: 1}
self.helper._process_job(
self.session, self.jobs_repr, results_repr, self.job_name)
self.assertEqual(
str(boom.exception),
"Value of key 'job' is of incorrect type int")
def test_process_job_checks_results_repr_values_are_dicts(self):
"""
verify that _process_job() checks if results_repr is a dictionary
of lists, each of which holds a dictionary.
"""
with self.assertRaises(CorruptedSessionError) as boom:
results_repr = {self.job_name: [1]}
self.helper._process_job(
self.session, self.jobs_repr, results_repr, self.job_name)
self.assertEqual(
str(boom.exception),
"Value of object is of incorrect type int")