def _run_jobs(self, ns, job_list, exporter, transport=None):
# Compute the run list, this can give us notification about problems in
# the selected jobs. Currently we just display each problem
matching_job_list = self._get_matching_job_list(ns, job_list)
print("[ Analyzing Jobs ]".center(80, '='))
# Create a session that handles most of the stuff needed to run jobs
try:
session = SessionState(job_list)
except DependencyDuplicateError as exc:
# Handle possible DependencyDuplicateError that can happen if
# someone is using plainbox for job development.
print("The job database you are currently using is broken")
print("At least two jobs contend for the name {0}".format(
exc.job.name))
print("First job defined in: {0}".format(exc.job.origin))
print("Second job defined in: {0}".format(
exc.duplicate_job.origin))
raise SystemExit(exc)
with session.open():
if session.previous_session_file():
if self.ask_for_resume():
session.resume()
else:
session.clean()
self._update_desired_job_list(session, matching_job_list)
if (sys.stdin.isatty() and sys.stdout.isatty() and not
ns.not_interactive):
outcome_callback = self.ask_for_outcome
else:
outcome_callback = None
runner = JobRunner(
session.session_dir,
session.jobs_io_log_dir,
outcome_callback=outcome_callback,
dry_run=ns.dry_run
)
self._run_jobs_with_session(ns, session, runner)
# Get a stream with exported session data.
exported_stream = io.BytesIO()
data_subset = exporter.get_session_data_subset(session)
exporter.dump(data_subset, exported_stream)
exported_stream.seek(0) # Need to rewind the file, puagh
# Write the stream to file if requested
self._save_results(ns.output_file, exported_stream)
# Invoke the transport?
if transport:
exported_stream.seek(0)
try:
transport.send(exported_stream.read())
except InvalidSchema as exc:
print("Invalid destination URL: {0}".format(exc))
except ConnectionError as exc:
print(("Unable to connect "
"to destination URL: {0}").format(exc))
except HTTPError as exc:
print(("Server returned an error when "
"receiving or processing: {0}").format(exc))
# FIXME: sensible return value
return 0
class SessionStateLocalStorageTests(TestCase):
def setUp(self):
# session data are kept in XDG_CACHE_HOME/plainbox/.session
# To avoid resuming a real session, we have to select a temporary
# location instead
self._sandbox = tempfile.mkdtemp()
self._env = os.environ
os.environ['XDG_CACHE_HOME'] = self._sandbox
def job_state(self, name):
# A helper function to avoid overly long expressions
return self.session.job_state_map[name]
def test_persistent_save(self):
self.job_A = make_job("A")
self.job_list = [self.job_A]
self.session = SessionState(self.job_list)
result_A = JobResult({
'job': self.job_A,
'outcome': JobResult.OUTCOME_PASS,
'comments': 'All good',
'return_code': 0,
'io_log': ((0, 'stdout', "Success !\n"),)
})
session_json_text = """{
"_job_state_map": {
"A": {
"_job": {
"data": {
"name": "A",
"plugin": "dummy",
"requires": null,
"depends": null
},
"_class_id": "JOB_DEFINITION"
},
"_result": {
"data": {
"job": {
"data": {
"name": "A",
"plugin": "dummy",
"requires": null,
"depends": null
},
"_class_id": "JOB_DEFINITION"
},
"outcome": "pass",
"return_code": 0,
"comments": "All good",
"io_log": [
[
0,
"stdout",
"Success !\\n"
]
]
},
"_class_id": "JOB_RESULT"
},
"_class_id": "JOB_STATE"
}
},
"_desired_job_list": [
{
"data": {
"name": "A",
"plugin": "dummy",
"requires": null,
"depends": null
},
"_class_id": "JOB_DEFINITION"
}
],
"_class_id": "SESSION_STATE"
}"""
self.session.open()
self.session.update_desired_job_list([self.job_A])
self.session.update_job_result(self.job_A, result_A)
self.session.persistent_save()
session_file = self.session.previous_session_file()
self.session.close()
self.assertIsNotNone(session_file)
with open(session_file) as f:
raw_json = json.load(f)
self.maxDiff = None
self.assertEqual(raw_json, json.loads(session_json_text))
def test_resume_session(self):
# All of the tests below are using one session. The session has four
# jobs, Job A depends on a resource provided by job R which has no
# dependencies at all. Both Job X and Y depend on job A.
#
# A -(resource dependency)-> R
#
# X -(direct dependency) -> A
#
# Y -(direct dependency) -> A
self.job_A = make_job("A", requires="R.attr == 'value'")
self.job_A_expr = self.job_A.get_resource_program().expression_list[0]
self.job_R = make_job("R", plugin="resource")
self.job_X = make_job("X", depends='A')
self.job_Y = make_job("Y", depends='A')
self.job_list = [self.job_A, self.job_R, self.job_X, self.job_Y]
# Create a new session (session_dir is empty)
self.session = SessionState(self.job_list)
result_R = JobResult({
'job': self.job_R,
'io_log': make_io_log(((0, 'stdout', b"attr: value\n"),),
self._sandbox)
})
result_A = JobResult({
'job': self.job_A,
'outcome': JobResult.OUTCOME_PASS
})
result_X = JobResult({
'job': self.job_X,
'outcome': JobResult.OUTCOME_PASS
})
# Job Y can't start as it requires job A
self.assertFalse(self.job_state('Y').can_start())
self.session.update_desired_job_list([self.job_X, self.job_Y])
self.session.open()
self.session.update_job_result(self.job_R, result_R)
self.session.update_job_result(self.job_A, result_A)
self.session.update_job_result(self.job_X, result_X)
self.session.persistent_save()
self.session.close()
# Create a new session (session_dir should contain session data)
self.session = SessionState(self.job_list)
self.session.open()
# Resume the previous session
self.session.resume()
# This time job Y can start
self.assertTrue(self.job_state('Y').can_start())
self.session.close()
def tearDown(self):
shutil.rmtree(self._sandbox)
os.environ = self._env
class SessionStateLocalStorageTests(TestCase):
def setUp(self):
# session data are kept in XDG_CACHE_HOME/plainbox/.session
# To avoid resuming a real session, we have to select a temporary
# location instead
self._sandbox = tempfile.mkdtemp()
self._env = os.environ
os.environ['XDG_CACHE_HOME'] = self._sandbox
def job_state(self, name):
# A helper function to avoid overly long expressions
return self.session.job_state_map[name]
def test_persistent_save(self):
self.job_A = make_job("A")
self.job_list = [self.job_A]
self.session = SessionState(self.job_list)
result_A = JobResult({
'job': self.job_A,
'outcome': JobResult.OUTCOME_PASS,
'comments': 'All good',
'return_code': 0,
'io_log': ((0, 'stdout', "Success !\n"), )
})
session_json_text = """{
"_job_state_map": {
"A": {
"_job": {
"data": {
"name": "A",
"plugin": "dummy",
"requires": null,
"depends": null
},
"_class_id": "JOB_DEFINITION"
},
"_result": {
"data": {
"job": {
"data": {
"name": "A",
"plugin": "dummy",
"requires": null,
"depends": null
},
"_class_id": "JOB_DEFINITION"
},
"outcome": "pass",
"return_code": 0,
"comments": "All good",
"io_log": [
[
0,
"stdout",
"Success !\\n"
]
]
},
"_class_id": "JOB_RESULT"
},
"_class_id": "JOB_STATE"
}
},
"_desired_job_list": [
{
"data": {
"name": "A",
"plugin": "dummy",
"requires": null,
"depends": null
},
"_class_id": "JOB_DEFINITION"
}
],
"_class_id": "SESSION_STATE"
}"""
self.session.open()
self.session.update_desired_job_list([self.job_A])
self.session.update_job_result(self.job_A, result_A)
self.session.persistent_save()
session_file = self.session.previous_session_file()
self.session.close()
self.assertIsNotNone(session_file)
with open(session_file) as f:
raw_json = json.load(f)
self.maxDiff = None
self.assertEqual(raw_json, json.loads(session_json_text))
def test_resume_session(self):
# All of the tests below are using one session. The session has four
# jobs, Job A depends on a resource provided by job R which has no
# dependencies at all. Both Job X and Y depend on job A.
#
# A -(resource dependency)-> R
#
# X -(direct dependency) -> A
#
# Y -(direct dependency) -> A
self.job_A = make_job("A", requires="R.attr == 'value'")
self.job_A_expr = self.job_A.get_resource_program().expression_list[0]
self.job_R = make_job("R", plugin="resource")
self.job_X = make_job("X", depends='A')
self.job_Y = make_job("Y", depends='A')
self.job_list = [self.job_A, self.job_R, self.job_X, self.job_Y]
# Create a new session (session_dir is empty)
self.session = SessionState(self.job_list)
result_R = JobResult({
'job':
self.job_R,
'io_log':
make_io_log(((0, 'stdout', b"attr: value\n"), ), self._sandbox)
})
result_A = JobResult({
'job': self.job_A,
'outcome': JobResult.OUTCOME_PASS
})
result_X = JobResult({
'job': self.job_X,
'outcome': JobResult.OUTCOME_PASS
})
# Job Y can't start as it requires job A
self.assertFalse(self.job_state('Y').can_start())
self.session.update_desired_job_list([self.job_X, self.job_Y])
self.session.open()
self.session.update_job_result(self.job_R, result_R)
self.session.update_job_result(self.job_A, result_A)
self.session.update_job_result(self.job_X, result_X)
self.session.persistent_save()
self.session.close()
# Create a new session (session_dir should contain session data)
self.session = SessionState(self.job_list)
self.session.open()
# Resume the previous session
self.session.resume()
# This time job Y can start
self.assertTrue(self.job_state('Y').can_start())
self.session.close()
def tearDown(self):
shutil.rmtree(self._sandbox)
os.environ = self._env
