def _run_command(self, session, job, parent):
"""
Run a Job command in a separate thread
"""
ui_io_delegate = UIOutputPrinter(self)
runner = JobRunner(session.session_dir, session.jobs_io_log_dir,
command_io_delegate=ui_io_delegate)
return_code, record_path = runner._run_command(job, None)
parent._command_callback(return_code, record_path)
def _run_local_jobs(self):
print(_("[Running Local Jobs]").center(80, '='))
manager = SessionManager.create_with_state(self.session)
try:
manager.state.metadata.title = "plainbox dev analyze session"
manager.state.metadata.flags = [SessionMetaData.FLAG_INCOMPLETE]
manager.checkpoint()
runner = JobRunner(manager.storage.location,
self.provider_list,
os.path.join(manager.storage.location,
'io-logs'),
command_io_delegate=self)
again = True
while again:
for job in self.session.run_list:
if job.plugin == 'local':
if self.session.job_state_map[
job.id].result.outcome is None:
self._run_local_job(manager, runner, job)
break
else:
again = False
manager.state.metadata.flags = []
manager.checkpoint()
finally:
manager.destroy()
def run(self):
# Compute the run list, this can give us notification about problems in
# the selected jobs. Currently we just display each problem
# Create a session that handles most of the stuff needed to run jobs
try:
self.session = SessionState(self.job_list)
except DependencyDuplicateError as exc:
# Handle possible DependencyDuplicateError that can happen if
# someone is using plainbox for job development.
print("The job database you are currently using is broken")
print("At least two jobs contend for the name {0}".format(
exc.job.id))
print("First job defined in: {0}".format(exc.job.origin))
print("Second job defined in: {0}".format(
exc.duplicate_job.origin))
raise SystemExit(exc)
with self.session.open():
self._set_job_selection()
self.runner = JobRunner(self.session.session_dir,
self.provider_list,
self.session.jobs_io_log_dir,
command_io_delegate=self,
dry_run=self.ns.dry_run)
self._run_all_jobs()
if self.config.fallback_file is not Unset:
self._save_results()
self._submit_results()
# FIXME: sensible return value
return 0
def _run_jobs(self, ns, manager):
runner = JobRunner(manager.storage.location,
self.provider_list,
os.path.join(manager.storage.location, 'io-logs'),
command_io_delegate=self)
self._run_jobs_with_session(ns, manager, runner)
if not self._local_only:
self.save_results(manager)
def run(self):
job = self._get_job()
if job is None:
print(_("There is no job called {!a}").format(self.job_id))
print(_("See `plainbox special --list-jobs` for a list of choices"))
return 126
elif job.command is None:
print(_("Selected job does not have a command"))
return 125
with TemporaryDirectory() as scratch, TemporaryDirectory() as iologs:
runner = JobRunner(scratch, self.provider_list, iologs)
bait_dir = os.path.join(scratch, "files-created-in-current-dir")
os.mkdir(bait_dir)
with TestCwd(bait_dir):
return_code, record_path = runner._run_command(job, self.config)
self._display_side_effects(scratch)
self._display_script_outcome(job, return_code)
return return_code
def run(self):
job = self._get_job()
if job is None:
print("There is no job called {!a}".format(self.job_name))
print("See `plainbox special --list-jobs` for a list of choices")
return 126
elif job.command is None:
print("Selected job does not have a command")
return 125
with TemporaryDirectory() as scratch, TemporaryDirectory() as iologs:
runner = JobRunner(scratch, iologs)
bait_dir = os.path.join(scratch, 'files-created-in-current-dir')
os.mkdir(bait_dir)
with TestCwd(bait_dir):
return_code, record_path = runner._run_command(
job, self.config)
self._display_side_effects(scratch)
self._display_script_outcome(job, return_code)
return return_code
def run(self):
job = self._get_job()
if job is None:
print(_("There is no job called {!a}").format(self.job_id))
print(
_("See `plainbox special --list-jobs` for a list of choices"))
return 126
elif job.command is None:
print(_("Selected job does not have a command"))
return 125
with TemporaryDirectory() as scratch, TemporaryDirectory() as iologs:
runner = JobRunner(scratch, self.provider_list, iologs)
job_state = JobState(job)
ctrl = runner._get_ctrl_for_job(job)
runner.log_leftovers = False
runner.on_leftover_files.connect(self._on_leftover_files)
return_code, record_path = runner._run_command(
job, job_state, self.config, ctrl)
self._display_script_outcome(job, return_code)
return return_code
def _run(self, session, job, running_job_wrapper):
"""
Start a JobRunner in a separate thread
"""
runner = JobRunner(
session.session_dir,
session.jobs_io_log_dir,
command_io_delegate=running_job_wrapper.ui_io_delegate,
interaction_callback=running_job_wrapper.emitAskForOutcomeSignal
)
job_state = session.job_state_map[job.name]
if job_state.can_start():
job_result = runner.run_job(job)
else:
job_result = MemoryJobResult({
'outcome': IJobResult.OUTCOME_NOT_SUPPORTED,
'comments': job_state.get_readiness_description()
})
if job_result is not None:
running_job_wrapper.update_job_result_callback(job, job_result)
def test_user_env_without_environ_keys(self):
with patch.dict('os.environ', {'foo': 'bar'}):
job = JobDefinition({
'name': 'name',
'plugin': 'plugin',
})
job._provider = Mock()
job._provider.extra_PYTHONPATH = None
job._provider.extra_PATH = ""
self.assertIn(
"foo",
JobRunner._get_script_env(Mock(), job, only_changes=False))
def _run(self, session, job, running_job_wrapper):
"""
Start a JobRunner in a separate thread
"""
runner = JobRunner(
session.session_dir,
session.jobs_io_log_dir,
command_io_delegate=running_job_wrapper.ui_io_delegate,
interaction_callback=running_job_wrapper.emitAskForOutcomeSignal)
job_state = session.job_state_map[job.name]
if job_state.can_start():
job_result = runner.run_job(job)
else:
job_result = MemoryJobResult({
'outcome':
IJobResult.OUTCOME_NOT_SUPPORTED,
'comments':
job_state.get_readiness_description()
})
if job_result is not None:
running_job_wrapper.update_job_result_callback(job, job_result)
def test_user_env_without_environ_keys(self):
with patch.dict('os.environ', {'foo': 'bar'}):
job = JobDefinition({
'name': 'name',
'plugin': 'plugin',
})
job._provider = Mock()
job._provider.extra_PYTHONPATH = None
job._provider.extra_PATH = ""
self.assertIn(
"foo",
JobRunner._get_script_env(Mock(), job, only_changes=False))
def test_root_env_without_environ_keys(self):
with patch.dict('os.environ', {'foo': 'bar'}):
job = JobDefinition({
'name': 'name',
'plugin': 'plugin',
'user': '******',
})
job._checkbox = Mock()
job._checkbox.extra_PYTHONPATH = None
job._checkbox.extra_PATH = ""
self.assertNotIn(
"foo",
JobRunner._get_script_env(Mock(), job, only_changes=True))
def test_get_warm_up_sequence(self):
# create a mocked execution controller
ctrl = Mock(spec_set=IExecutionController, name='ctrl')
# create a fake warm up function
warm_up_func = Mock(name='warm_up_func')
# make the execution controller accept any job
ctrl.get_score.return_value = 1
# make the execution controller return warm_up_func as warm-up
ctrl.get_warm_up_for_job.return_value = warm_up_func
# make a pair of mock jobs for our controller to see
job1 = Mock(spec_set=IJobDefinition, name='job1')
job2 = Mock(spec_set=IJobDefinition, name='job2')
with TemporaryDirectory() as session_dir:
# Create a real runner with a fake execution controller, empty list
# of providers and fake io-log directory.
runner = JobRunner(session_dir,
provider_list=[],
jobs_io_log_dir=os.path.join(
session_dir, 'io-log'),
execution_ctrl_list=[ctrl])
# Ensure that we got the warm up function we expected
self.assertEqual(runner.get_warm_up_sequence([job1, job2]),
[warm_up_func])
def create_runner(self):
"""
Create a job runner.
This sets the attr:`_runner` which enables :meth:`runner` property.
Requires the manager to be created (we need the storage object)
"""
self._runner = JobRunner(
self.storage.location,
self.provider_list,
# TODO: tie this with well-known-dirs helper
os.path.join(self.storage.location, 'io-logs'),
command_io_delegate=self,
dry_run=self.ns.dry_run)
def _run_local_jobs(self):
print("[Running Local Jobs]".center(80, '='))
with self.session.open():
runner = JobRunner(
self.session.session_dir, self.session.jobs_io_log_dir,
command_io_delegate=self, interaction_callback=None)
again = True
while again:
for job in self.session.run_list:
if job.plugin == 'local':
if self.session.job_state_map[job.name].result.outcome is None:
self._run_local_job(runner, job)
break
else:
again = False
def __init__(self, service, session, provider_list, job):
"""
Initialize a primed job.
This should not be called by applications.
Please call :meth:`Service.prime_job()` instead.
"""
self._service = service
self._session = session
self._provider_list = provider_list
self._job = job
self._runner = JobRunner(
session.session_dir,
self._provider_list,
session.jobs_io_log_dir,
# Pass a dummy IO delegate, we don't want to get any tracing here
# Later on this could be configurable but it's better if it's
# simple and limited rather than complete but broken somehow.
command_io_delegate=self)
class PlainBox:
"""
High-level plainbox object
"""
def __init__(self):
self._checkbox = CheckBox()
self._context = ResourceContext()
self._scratch = Scratch()
self._runner = JobRunner(self._checkbox, self._context, self._scratch)
def main(self, argv=None):
basicConfig(level="WARNING")
# TODO: setup sane logging system that works just as well for Joe user
# that runs checkbox from the CD as well as for checkbox developers and
# custom debugging needs. It would be perfect^Hdesirable not to create
# another broken, never-rotated, uncapped logging crap that kills my
# SSD by writing junk to ~/.cache/
parser = ArgumentParser(prog="plainbox")
parser.add_argument(
"-v", "--version", action="version",
version="{}.{}.{}".format(*version[:3]))
parser.add_argument(
"-l", "--log-level", action="store",
choices=('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'),
help="Set logging level")
group = parser.add_argument_group(title="user interface options")
group.add_argument(
"-u", "--ui", action="store",
default=None, choices=('headless', 'text', 'graphics'),
help="select the UI front-end (defaults to auto)")
group = parser.add_argument_group(title="job definition options")
group.add_argument(
"--load-extra", action="append",
metavar="FILE", default=[],
help="Load extra job definitions from FILE",
type=FileType("rt"))
group.add_argument(
'-r', '--run-pattern', action="append",
metavar='PATTERN', default=[],
help="Run jobs matching the given pattern")
group.add_argument(
'--list-jobs', help="List all jobs",
action="store_true")
ns = parser.parse_args(argv)
# Set the desired log level
if ns.log_level:
getLogger("").setLevel(ns.log_level)
# Load built-in job definitions
job_list = self.get_builtin_jobs()
# Load additional job definitions
job_list.extend(self._load_jobs(ns.load_extra))
if ns.list_jobs:
print("Available jobs:")
for job in job_list:
print(" - {}".format(job))
else:
# And run them
with self._scratch:
return self.run(
run_pattern_list=ns.run_pattern,
job_list=job_list,
ui=ns.ui)
def run(self, run_pattern_list, job_list, **kwargs):
job_map = {job.name: job for job in job_list}
matching_job_list = []
# Find jobs that matched patterns
print("[ Searching for Matching Jobs ]".center(80, '='))
for job in job_list:
for pattern in run_pattern_list:
if fnmatch(job.name, pattern):
matching_job_list.append(job)
break
print("Matching jobs: {}".format(
', '.join((job.name for job in matching_job_list))))
# Compute required resources
print("[ Analyzing Jobs ]".center(80, '='))
needed_resource_jobs = set()
resource_job_list = []
for job in matching_job_list:
prog = job.get_resource_program()
if prog is None:
continue
for resource_name in prog.required_resources:
if resource_name in needed_resource_jobs:
continue
else:
needed_resource_jobs.add(resource_name)
try:
required_job = job_map[resource_name]
except KeyError:
print("Unable to find resource {!r} required by job"
" {}".format(resource_name, job))
print("Job {} will not run".format(job))
matching_job_list.remove(job)
if required_job.plugin != "resource":
print("Job {} references resource {!r} but job {} uses"
" non-resource plugin {!r}".format(
job, resource_name, required_job,
required_job.plugin))
print("Job {} will not run".format(job))
matching_job_list.remove(job)
else:
resource_job_list.append(required_job)
# Resolve dependencies in resource jobs
# XXX: not implemented
print("Required resource jobs: {}".format(
', '.join((job.name for job in resource_job_list))))
# Run resource jobs
print("[ Gathering Resources ]".center(80, '='))
if not resource_job_list:
print("No resource jobs required")
else:
self._run_jobs(resource_job_list)
# Run non-resource jobs
result_list = []
other_job_list = [
job
for job in matching_job_list
if job.plugin != "resource"]
print("[ Testing ]".center(80, '='))
if not other_job_list:
print("No jobs selected")
else:
result_list = self._run_jobs(other_job_list)
print("[ Results ]".center(80, '='))
for result in result_list:
print(" * {}: {}".format(
result.job.name, result.outcome))
def get_builtin_jobs(self):
logger.debug("Loading built-in jobs...")
return self._load_builtin_jobs()
def save(self, something, somewhere):
raise NotImplementedError()
def load(self, somewhere):
if isinstance(somewhere, str):
# Load data from a file with the given name
filename = somewhere
with open(filename, 'rt', encoding='UTF-8') as stream:
return load(stream)
if isinstance(somewhere, TextIOWrapper):
stream = somewhere
logger.debug("Loading jobs definitions from %r...", stream.name)
record_list = load_rfc822_records(stream)
job_list = []
for record in record_list:
job = JobDefinition.from_rfc822_record(record)
logger.debug("Loaded %r", job)
job_list.append(job)
return job_list
else:
raise TypeError(
"Unsupported type of 'somewhere': {!r}".format(
type(somewhere)))
def _run_jobs(self, job_list):
result_list = []
for job in job_list:
print("[ {} ]".format(job.name).center(80, '-'))
if job.description:
print()
print(job.description)
print()
print("_" * 80)
print(" * job attributes set: {}".format(
", ".join((attr for attr in job._data))))
print(" * job type: {}".format(job.plugin))
if job.command:
print(" * job command: {!r}".format(job.command))
if job.depends is not None:
print(" * job dependencies: {}".format(', '.join(job.depends)))
prog = job.get_resource_program()
if prog:
met = prog.evaluate(self._context.resources)
print(" * job requirements: {}".format(
"met" if met else "not met"))
for expression in prog.expression_list:
print(" - {}".format(expression.text))
try:
print(" * starting job... ", end="")
result = self._runner.run_job(job)
except NotImplementedError:
print("error")
logger.exception("Something was not implemented fully")
else:
print("done")
if result is not None:
result_list.append(result)
elif job.plugin == "resource":
pass
else:
logger.warning("Job %s did not return a result", job)
return result_list
def _load_jobs(self, source_list):
"""
Load jobs from the list of sources
"""
job_list = []
for source in source_list:
job_list.extend(self.load(source))
return job_list
def _load_builtin_jobs(self):
"""
Load jobs from built into CheckBox
"""
return self._load_jobs([
join(self._checkbox.jobs_dir, name)
for name in listdir(self._checkbox.jobs_dir)
if name.endswith(".txt") or name.endswith(".txt.in")])
def __init__(self):
self._checkbox = CheckBox()
self._context = ResourceContext()
self._scratch = Scratch()
self._runner = JobRunner(self._checkbox, self._context, self._scratch)
def __init__(self):
self._checkbox = CheckBox()
self._context = ResourceContext()
self._scratch = Scratch()
self._runner = JobRunner(self._checkbox, self._context, self._scratch)
class PlainBox:
"""
High-level plainbox object
"""
def __init__(self):
self._checkbox = CheckBox()
self._context = ResourceContext()
self._scratch = Scratch()
self._runner = JobRunner(self._checkbox, self._context, self._scratch)
def main(self, argv=None):
basicConfig(level="WARNING")
# TODO: setup sane logging system that works just as well for Joe user
# that runs checkbox from the CD as well as for checkbox developers and
# custom debugging needs. It would be perfect^Hdesirable not to create
# another broken, never-rotated, uncapped logging crap that kills my
# SSD by writing junk to ~/.cache/
parser = ArgumentParser(prog="plainbox")
parser.add_argument("-v",
"--version",
action="version",
version="{}.{}.{}".format(*version[:3]))
parser.add_argument("-l",
"--log-level",
action="store",
choices=('DEBUG', 'INFO', 'WARNING', 'ERROR',
'CRITICAL'),
help="Set logging level")
group = parser.add_argument_group(title="user interface options")
group.add_argument("-u",
"--ui",
action="store",
default=None,
choices=('headless', 'text', 'graphics'),
help="select the UI front-end (defaults to auto)")
group = parser.add_argument_group(title="job definition options")
group.add_argument("--load-extra",
action="append",
metavar="FILE",
default=[],
help="Load extra job definitions from FILE",
type=FileType("rt"))
group.add_argument('-r',
'--run-pattern',
action="append",
metavar='PATTERN',
default=[],
help="Run jobs matching the given pattern")
group.add_argument('--list-jobs',
help="List all jobs",
action="store_true")
ns = parser.parse_args(argv)
# Set the desired log level
if ns.log_level:
getLogger("").setLevel(ns.log_level)
# Load built-in job definitions
job_list = self.get_builtin_jobs()
# Load additional job definitions
job_list.extend(self._load_jobs(ns.load_extra))
if ns.list_jobs:
print("Available jobs:")
for job in job_list:
print(" - {}".format(job))
else:
# And run them
with self._scratch:
return self.run(run_pattern_list=ns.run_pattern,
job_list=job_list,
ui=ns.ui)
def run(self, run_pattern_list, job_list, **kwargs):
job_map = {job.name: job for job in job_list}
matching_job_list = []
# Find jobs that matched patterns
print("[ Searching for Matching Jobs ]".center(80, '='))
for job in job_list:
for pattern in run_pattern_list:
if fnmatch(job.name, pattern):
matching_job_list.append(job)
break
print("Matching jobs: {}".format(', '.join(
(job.name for job in matching_job_list))))
# Compute required resources
print("[ Analyzing Jobs ]".center(80, '='))
needed_resource_jobs = set()
resource_job_list = []
for job in matching_job_list:
prog = job.get_resource_program()
if prog is None:
continue
for resource_name in prog.required_resources:
if resource_name in needed_resource_jobs:
continue
else:
needed_resource_jobs.add(resource_name)
try:
required_job = job_map[resource_name]
except KeyError:
print("Unable to find resource {!r} required by job"
" {}".format(resource_name, job))
print("Job {} will not run".format(job))
matching_job_list.remove(job)
if required_job.plugin != "resource":
print("Job {} references resource {!r} but job {} uses"
" non-resource plugin {!r}".format(
job, resource_name, required_job,
required_job.plugin))
print("Job {} will not run".format(job))
matching_job_list.remove(job)
else:
resource_job_list.append(required_job)
# Resolve dependencies in resource jobs
# XXX: not implemented
print("Required resource jobs: {}".format(', '.join(
(job.name for job in resource_job_list))))
# Run resource jobs
print("[ Gathering Resources ]".center(80, '='))
if not resource_job_list:
print("No resource jobs required")
else:
self._run_jobs(resource_job_list)
# Run non-resource jobs
result_list = []
other_job_list = [
job for job in matching_job_list if job.plugin != "resource"
]
print("[ Testing ]".center(80, '='))
if not other_job_list:
print("No jobs selected")
else:
result_list = self._run_jobs(other_job_list)
print("[ Results ]".center(80, '='))
for result in result_list:
print(" * {}: {}".format(result.job.name, result.outcome))
def get_builtin_jobs(self):
logger.debug("Loading built-in jobs...")
return self._load_builtin_jobs()
def save(self, something, somewhere):
raise NotImplementedError()
def load(self, somewhere):
if isinstance(somewhere, str):
# Load data from a file with the given name
filename = somewhere
with open(filename, 'rt', encoding='UTF-8') as stream:
return load(stream)
if isinstance(somewhere, TextIOWrapper):
stream = somewhere
logger.debug("Loading jobs definitions from %r...", stream.name)
record_list = load_rfc822_records(stream)
job_list = []
for record in record_list:
job = JobDefinition.from_rfc822_record(record)
logger.debug("Loaded %r", job)
job_list.append(job)
return job_list
else:
raise TypeError("Unsupported type of 'somewhere': {!r}".format(
type(somewhere)))
def _run_jobs(self, job_list):
result_list = []
for job in job_list:
print("[ {} ]".format(job.name).center(80, '-'))
if job.description:
print()
print(job.description)
print()
print("_" * 80)
print(" * job attributes set: {}".format(", ".join(
(attr for attr in job._data))))
print(" * job type: {}".format(job.plugin))
if job.command:
print(" * job command: {!r}".format(job.command))
if job.depends is not None:
print(" * job dependencies: {}".format(', '.join(job.depends)))
prog = job.get_resource_program()
if prog:
met = prog.evaluate(self._context.resources)
print(" * job requirements: {}".format(
"met" if met else "not met"))
for expression in prog.expression_list:
print(" - {}".format(expression.text))
try:
print(" * starting job... ", end="")
result = self._runner.run_job(job)
except NotImplementedError:
print("error")
logger.exception("Something was not implemented fully")
else:
print("done")
if result is not None:
result_list.append(result)
elif job.plugin == "resource":
pass
else:
logger.warning("Job %s did not return a result", job)
return result_list
def _load_jobs(self, source_list):
"""
Load jobs from the list of sources
"""
job_list = []
for source in source_list:
job_list.extend(self.load(source))
return job_list
def _load_builtin_jobs(self):
"""
Load jobs from built into CheckBox
"""
return self._load_jobs([
join(self._checkbox.jobs_dir, name)
for name in listdir(self._checkbox.jobs_dir)
if name.endswith(".txt") or name.endswith(".txt.in")
])
def _run_jobs(self, ns, job_list, exporter, transport=None):
# Compute the run list, this can give us notification about problems in
# the selected jobs. Currently we just display each problem
matching_job_list = self._get_matching_job_list(ns, job_list)
print(_("[ Analyzing Jobs ]").center(80, '='))
# Create a session that handles most of the stuff needed to run jobs
try:
session = SessionState(job_list)
except DependencyDuplicateError as exc:
# Handle possible DependencyDuplicateError that can happen if
# someone is using plainbox for job development.
print(_("The job database you are currently using is broken"))
print(_("At least two jobs contend for the id {0}").format(
exc.job.id))
print(_("First job defined in: {0}").format(exc.job.origin))
print(_("Second job defined in: {0}").format(
exc.duplicate_job.origin))
raise SystemExit(exc)
with session.open():
if session.previous_session_file():
if self.ask_for_resume():
session.resume()
self._maybe_skip_last_job_after_resume(session)
else:
session.clean()
session.metadata.title = " ".join(sys.argv)
session.persistent_save()
self._update_desired_job_list(session, matching_job_list)
# Ask the password before anything else in order to run jobs
# requiring privileges
if self._auth_warmup_needed(session):
print(_("[ Authentication ]").center(80, '='))
return_code = authenticate_warmup()
if return_code:
raise SystemExit(return_code)
runner = JobRunner(
session.session_dir, self.provider_list,
session.jobs_io_log_dir, dry_run=ns.dry_run)
self._run_jobs_with_session(ns, session, runner)
# Get a stream with exported session data.
exported_stream = io.BytesIO()
data_subset = exporter.get_session_data_subset(session)
exporter.dump(data_subset, exported_stream)
exported_stream.seek(0) # Need to rewind the file, puagh
# Write the stream to file if requested
self._save_results(ns.output_file, exported_stream)
# Invoke the transport?
if transport:
exported_stream.seek(0)
try:
transport.send(exported_stream.read())
except InvalidSchema as exc:
print(_("Invalid destination URL: {0}").format(exc))
except ConnectionError as exc:
print(_("Unable to connect "
"to destination URL: {0}").format(exc))
except HTTPError as exc:
print(_("Server returned an error when "
"receiving or processing: {0}").format(exc))
# FIXME: sensible return value
return 0