class SessionAssistant2():
"""Remote execution enabling wrapper for the SessionAssistant"""
# TODO: the name?
def __init__(self, cmd_callback):
_logger.debug("__init__()")
self._state = Idle
self._sa = SessionAssistant('service', api_flags={SA_RESTARTABLE})
self._sa.configure_application_restart(cmd_callback)
self._sudo_broker = SudoBroker()
self._sudo_password = None
self._sa.use_alternate_execution_controllers([
(RootViaSudoWithPassExecutionController, (), {
'password_provider_cls': self.get_decrypted_password
}),
(UserJobExecutionController, [], {}),
])
self._sa.select_providers('*')
self._session_change_lock = Lock()
self._operator_lock = Lock()
self._be = None
self._session_id = ""
self._jobs_count = 0
self._job_index = 0
self._currently_running_job = None # XXX: yuck!
self.buffered_ui = BufferedUI()
self._last_response = None
@property
def session_change_lock(self):
return self._session_change_lock
def allowed_when(state):
def wrap(f):
def fun(self, *args):
if self._state != state:
raise AssertionError("expected %s, is %s" %
(self._state, state))
return f(self, *args)
return fun
return wrap
@allowed_when(Idle)
def start_session(self, configuration):
_logger.debug("start_session: %r", configuration)
self._sa.start_new_session('checkbox-service')
self._session_id = self._sa.get_session_id()
tps = self._sa.get_test_plans()
response = zip(tps, [self._sa.get_test_plan(tp).name for tp in tps])
self._state = Started
self._available_testplans = sorted(
response, key=lambda x: x[1]) # sorted by name
return self._available_testplans
@allowed_when(Started)
def bootstrap(self, test_plan_id):
_logger.debug("bootstrap: %r", test_plan_id)
self._sa.update_app_blob(
json.dumps({
'testplan_id': test_plan_id,
}).encode("UTF-8"))
self._sa.select_test_plan(test_plan_id)
self._sa.bootstrap()
self._jobs_count = len(self._sa.get_static_todo_list())
self._state = Bootstrapped
return self._sa.get_static_todo_list()
@allowed_when(Bootstrapped)
def run_job(self, job_id):
"""
Depending on the type of the job, run_job can yield different number
of Interaction instances.
"""
_logger.debug("run_job: %r", job_id)
self._job_index = self._jobs_count - len(
self._sa.get_dynamic_todo_list()) + 1
self._currently_running_job = job_id
job = self._sa.get_job(job_id)
if job.plugin in ['manual', 'user-interact-verify', 'user-interact']:
self._current_interaction = Interaction('purpose',
job.tr_purpose())
yield self._current_interaction
if job.user and not self._sudo_password:
self._ephemeral_key = EphemeralKey()
self._current_interaction = Interaction(
'sudo_input', self._ephemeral_key.public_key)
yield self._current_interaction
assert (self._sudo_password is not None)
self._state = Running
self._be = BackgroundExecutor(self, job_id, self._sa.run_job)
@allowed_when(Running)
def monitor_job(self):
"""
Check the state of the currently running job.
:returns:
(state, payload) tuple.
Payload conveys detailed info that's characteristic
to the current state.
"""
_logger.debug("monitor_job()")
# either return [done, running, awaiting response]
# TODO: handle awaiting_response (reading from stdin by the job)
if self._be.is_alive():
return ('running', self.buffered_ui.get_output())
else:
return ('done', self._be.outcome())
return 'running' if self._be.is_alive() else 'done'
def whats_up(self):
"""
Check what is remote-service up to
:returns:
(state, payload) tuple.
"""
_logger.debug("whats_up()")
payload = None
if self._state == Running:
payload = (self._job_index, self._jobs_count,
self._currently_running_job)
if self._state == Started:
payload = self._available_testplans
return self._state, payload
def get_session_progress(self):
"""Return list of completed and not completed jobs in a dict."""
_logger.debug("get_session_progress()")
return {
"done": [],
"todo": self._sa.get_dynamic_todo_list(),
}
def get_master_public_key(self):
"""Expose the master public key"""
return self._sudo_broker.master_public
def save_password(self, cyphertext):
"""Store encrypted password"""
self._sudo_password = cyphertext
def get_decrypted_password(self):
"""Return decrypted password"""
assert (self._sudo_password)
return self._sudo_broker.decrypt_password(self._sudo_password)
def finish_job(self):
# assert the thread completed
self._sa.use_job_result(self._currently_running_job,
self._be.wait().get_result())
self._session_change_lock.release()
if not self._sa.get_dynamic_todo_list():
self._state = Idle
else:
self._state = Bootstrapped
def get_jobs_repr(self, job_ids):
"""
Translate jobs into a {'field': 'val'} representations.
:param job_ids:
list of job ids to get and translate
:returns:
list of dicts representing jobs
"""
test_info_list = tuple()
for job_id in job_ids:
job = self._sa.get_job(job_id)
cat_id = self._sa.get_job_state(job.id).effective_category_id
duration_txt = _('No estimated duration provided for this job')
if job.estimated_duration is not None:
duration_txt = '{} {}'.format(job.estimated_duration,
_('seconds'))
# the next dict is only to get test_info generating code tidier
automated_desc = {
True: _('this job is fully automated'),
False: _('this job requires some manual interaction')
}
test_info = {
"id":
job.id,
"partial_id":
job.partial_id,
"name":
job.tr_summary(),
"category_id":
cat_id,
"category_name":
self._sa.get_category(cat_id).tr_name(),
"automated":
automated_desc[job.automated],
"duration":
duration_txt,
"description": (job.tr_description()
or _('No description provided for this job')),
"outcome":
self._sa.get_job_state(job.id).result.outcome,
}
test_info_list = test_info_list + ((test_info, ))
return test_info_list
def resume_last(self):
last = next(self._sa.get_resumable_sessions())
meta = self._sa.resume_session(last.id)
app_blob = json.loads(meta.app_blob.decode("UTF-8"))
test_plan_id = app_blob['testplan_id']
self._sa.select_test_plan(test_plan_id)
self._sa.bootstrap()
result = MemoryJobResult({
'outcome':
IJobResult.OUTCOME_PASS,
'comments':
_("Passed after resuming execution")
})
last_job = meta.running_job_name
if last_job:
try:
self._sa.use_job_result(last_job, result)
except KeyError:
raise SystemExit(last_job)
self._state = Bootstrapped
def finalize_session(self):
self._sa.finalize_session()
self._state = Idle
class RemoteSessionAssistant():
"""Remote execution enabling wrapper for the SessionAssistant"""
REMOTE_API_VERSION = 9
def __init__(self, cmd_callback):
_logger.debug("__init__()")
self._cmd_callback = cmd_callback
self._sudo_broker = SudoBroker()
self._sudo_password = None
self._session_change_lock = Lock()
self._operator_lock = Lock()
self.buffered_ui = BufferedUI()
self._input_piping = os.pipe()
self._passwordless_sudo = is_passwordless_sudo()
self.terminate_cb = None
self._pipe_from_master = open(self._input_piping[1], 'w')
self._pipe_to_subproc = open(self._input_piping[0])
self._reset_sa()
self._currently_running_job = None
def _reset_sa(self):
_logger.info("Resetting RSA")
self._state = Idle
self._sa = SessionAssistant('service', api_flags={SA_RESTARTABLE})
self._sa.configure_application_restart(self._cmd_callback)
self._be = None
self._session_id = ""
self._jobs_count = 0
self._job_index = 0
self._currently_running_job = None # XXX: yuck!
self._last_job = None
self._current_comments = ""
self._last_response = None
self._normal_user = ''
self.session_change_lock.acquire(blocking=False)
self.session_change_lock.release()
@property
def session_change_lock(self):
return self._session_change_lock
@property
def config(self):
return self._sa.config
def allowed_when(*states):
def wrap(f):
def fun(self, *args):
if self._state not in states:
raise AssertionError("expected %s, is %s" %
(states, self._state))
return f(self, *args)
return fun
return wrap
def interact(self, interaction):
self._state = Interacting
self._current_interaction = interaction
yield self._current_interaction
@allowed_when(Interacting)
def remember_users_response(self, response):
if response == 'rollback':
self._currently_running_job = None
self.session_change_lock.acquire(blocking=False)
self.session_change_lock.release()
self._current_comments = ""
self._state = TestsSelected
return
self._last_response = response
self._state = Running
def _prepare_display_without_psutil(self):
try:
value = check_output(
'strings /proc/*/environ 2>/dev/null | '
'grep -m 1 -oP "(?<=DISPLAY=).*"',
shell=True,
universal_newlines=True).rstrip()
return {'DISPLAY': value}
except CalledProcessError:
return None
def prepare_extra_env(self):
# If possible also set the DISPLAY env var
# i.e when a user desktop session is running
for p in psutil.pids():
try:
p_environ = psutil.Process(p).environ()
p_user = psutil.Process(p).username()
except psutil.AccessDenied:
continue
except AttributeError:
# psutil < 4.0.0 doesn't provide Process.environ()
return self._prepare_display_without_psutil()
except psutil.NoSuchProcess:
# quietly ignore the process that died before we had a chance to
# read the environment from them
continue
if ("DISPLAY" in p_environ and p_user != 'gdm'): # gdm uses :1024
return {'DISPLAY': p_environ['DISPLAY']}
@allowed_when(Idle)
def start_session(self, configuration):
self._reset_sa()
_logger.debug("start_session: %r", configuration)
session_title = 'checkbox-slave'
session_desc = 'checkbox-slave session'
self._launcher = load_configs()
if configuration['launcher']:
self._launcher.read_string(configuration['launcher'], False)
session_title = self._launcher.session_title
session_desc = self._launcher.session_desc
self._sa.use_alternate_configuration(self._launcher)
self._normal_user = self._launcher.normal_user
if configuration['normal_user']:
self._normal_user = configuration['normal_user']
pass_provider = (None if self._passwordless_sudo else
self.get_decrypted_password)
runner_kwargs = {
'normal_user_provider': lambda: self._normal_user,
'password_provider': pass_provider,
'stdin': self._pipe_to_subproc,
'extra_env': self.prepare_extra_env(),
}
self._sa.start_new_session(session_title, UnifiedRunner, runner_kwargs)
self._sa.update_app_blob(
json.dumps({
'description': session_desc,
}).encode("UTF-8"))
self._sa.update_app_blob(
json.dumps({
'launcher': configuration['launcher'],
}).encode("UTF-8"))
self._session_id = self._sa.get_session_id()
tps = self._sa.get_test_plans()
filtered_tps = set()
for filter in self._launcher.test_plan_filters:
filtered_tps.update(fnmatch.filter(tps, filter))
filtered_tps = list(filtered_tps)
response = zip(
filtered_tps,
[self._sa.get_test_plan(tp).name for tp in filtered_tps])
self._state = Started
self._available_testplans = sorted(
response, key=lambda x: x[1]) # sorted by name
return self._available_testplans
@allowed_when(Started)
def prepare_bootstrapping(self, test_plan_id):
"""
Go through the list of bootstrapping jobs, and return True
if sudo password will be needed for any bootstrapping job.
"""
_logger.debug("prepare_bootstrapping: %r", test_plan_id)
self._sa.update_app_blob(
json.dumps({
'testplan_id': test_plan_id,
}).encode("UTF-8"))
self._sa.select_test_plan(test_plan_id)
for job_id in self._sa.get_bootstrap_todo_list():
job = self._sa.get_job(job_id)
if job.user is not None:
# job requires sudo controller
return True
return False
@allowed_when(Started)
def get_bootstrapping_todo_list(self):
return self._sa.get_bootstrap_todo_list()
def finish_bootstrap(self):
self._sa.finish_bootstrap()
self._state = Bootstrapped
if self._launcher.auto_retry:
for job_id in self._sa.get_static_todo_list():
job_state = self._sa.get_job_state(job_id)
job_state.attempts = self._launcher.max_attempts
return self._sa.get_static_todo_list()
def get_manifest_repr(self):
return self._sa.get_manifest_repr()
def save_manifest(self, manifest_answers):
return self._sa.save_manifest(manifest_answers)
def modify_todo_list(self, chosen_jobs):
self._sa.use_alternate_selection(chosen_jobs)
def finish_job_selection(self):
self._jobs_count = len(self._sa.get_dynamic_todo_list())
self._state = TestsSelected
@allowed_when(Interacting)
def rerun_job(self, job_id, result):
self._sa.use_job_result(job_id, result)
self.session_change_lock.acquire(blocking=False)
self.session_change_lock.release()
self._state = TestsSelected
@allowed_when(TestsSelected)
def run_job(self, job_id):
"""
Depending on the type of the job, run_job can yield different number
of Interaction instances.
"""
_logger.debug("run_job: %r", job_id)
self._job_index = self._jobs_count - len(
self._sa.get_dynamic_todo_list()) + 1
self._currently_running_job = job_id
self._current_comments = ""
job = self._sa.get_job(job_id)
if job.plugin in ['manual', 'user-interact-verify', 'user-interact']:
may_comment = True
while may_comment:
may_comment = False
if job.tr_description() and not job.tr_purpose():
yield from self.interact(
Interaction('description', job.tr_description()))
if job.tr_purpose():
yield from self.interact(
Interaction('purpose', job.tr_purpose()))
if job.tr_steps():
yield from self.interact(
Interaction('steps', job.tr_steps()))
if self._last_response == 'comment':
yield from self.interact(Interaction('comment'))
if self._last_response:
self._current_comments += self._last_response
may_comment = True
continue
if self._last_response == 'skip':
def skipped_builder(*args, **kwargs):
result_builder = JobResultBuilder(
outcome=IJobResult.OUTCOME_SKIP,
comments=_("Explicitly skipped before execution"))
if self._current_comments != "":
result_builder.comments = self._current_comments
return result_builder
self._be = BackgroundExecutor(self, job_id, skipped_builder)
yield from self.interact(
Interaction('skip', job.verification, self._be))
if job.command:
if (job.user and not self._passwordless_sudo
and not self._sudo_password):
self._ephemeral_key = EphemeralKey()
self._current_interaction = Interaction(
'sudo_input', self._ephemeral_key.public_key)
pass_is_correct = False
while not pass_is_correct:
self.state = Interacting
yield self._current_interaction
pass_is_correct = validate_pass(
self._sudo_broker.decrypt_password(
self._sudo_password))
if not pass_is_correct:
print(_('Sorry, try again.'))
assert (self._sudo_password is not None)
self._state = Running
self._be = BackgroundExecutor(self, job_id, self._sa.run_job)
else:
def undecided_builder(*args, **kwargs):
return JobResultBuilder(outcome=IJobResult.OUTCOME_UNDECIDED)
self._be = BackgroundExecutor(self, job_id, undecided_builder)
if self._sa.get_job(self._currently_running_job).plugin in [
'manual', 'user-interact-verify'
]:
yield from self.interact(
Interaction('verification', job.verification, self._be))
@allowed_when(Started, Bootstrapping)
def run_bootstrapping_job(self, job_id):
self._currently_running_job = job_id
self._state = Bootstrapping
self._be = BackgroundExecutor(self, job_id, self._sa.run_job)
@allowed_when(Running, Bootstrapping, Interacting, TestsSelected)
def monitor_job(self):
"""
Check the state of the currently running job.
:returns:
(state, payload) tuple.
Payload conveys detailed info that's characteristic
to the current state.
"""
_logger.debug("monitor_job()")
# either return [done, running, awaiting response]
# TODO: handle awaiting_response (reading from stdin by the job)
if self._be and self._be.is_alive():
return ('running', self.buffered_ui.get_output())
else:
return ('done', self.buffered_ui.get_output())
def get_remote_api_version(self):
return self.REMOTE_API_VERSION
def whats_up(self):
"""
Check what is remote-service up to
:returns:
(state, payload) tuple.
"""
_logger.debug("whats_up()")
payload = None
if self._state == Running:
payload = (self._job_index, self._jobs_count,
self._currently_running_job)
if self._state == TestsSelected and not self._currently_running_job:
payload = {'last_job': self._last_job}
elif self._state == Started:
payload = self._available_testplans
elif self._state == Interacting:
payload = self._current_interaction
elif self._state == Bootstrapped:
payload = self._sa.get_static_todo_list()
return self._state, payload
def terminate(self):
if self.terminate_cb:
self.terminate_cb()
def get_session_progress(self):
"""Return list of completed and not completed jobs in a dict."""
_logger.debug("get_session_progress()")
return {
"done": self._sa.get_dynamic_done_list(),
"todo": self._sa.get_dynamic_todo_list(),
}
def get_master_public_key(self):
"""Expose the master public key"""
return self._sudo_broker.master_public
def save_password(self, cyphertext):
"""Store encrypted password"""
if validate_pass(self._sudo_broker.decrypt_password(cyphertext)):
self._sudo_password = cyphertext
return True
return False
def get_decrypted_password(self):
"""Return decrypted password"""
if self._passwordless_sudo:
return ''
assert (self._sudo_password)
return self._sudo_broker.decrypt_password(self._sudo_password)
def finish_job(self, result=None):
# assert the thread completed
self.session_change_lock.acquire(blocking=False)
self.session_change_lock.release()
if self._sa.get_job(self._currently_running_job).plugin in [
'manual', 'user-interact-verify'
] and not result:
# for manually verified jobs we don't set the outcome here
# it is already determined
return
if not result:
result = self._be.wait().get_result()
self._sa.use_job_result(self._currently_running_job, result)
if self._state != Bootstrapping:
if not self._sa.get_dynamic_todo_list():
if (self._launcher.auto_retry
and self.get_rerun_candidates('auto')):
self._state = TestsSelected
else:
self._state = Idle
else:
self._state = TestsSelected
return result
def get_rerun_candidates(self, session_type='manual'):
return self._sa.get_rerun_candidates(session_type)
def prepare_rerun_candidates(self, rerun_candidates):
candidates = self._sa.prepare_rerun_candidates(rerun_candidates)
self._state = TestsSelected
return candidates
def get_job_result(self, job_id):
return self._sa.get_job_state(job_id).result
def get_jobs_repr(self, job_ids, offset=0):
"""
Translate jobs into a {'field': 'val'} representations.
:param job_ids:
list of job ids to get and translate
:param offset:
apply an offset to the job number if for instance the job list
is being requested part way through a session
:returns:
list of dicts representing jobs
"""
test_info_list = tuple()
for job_no, job_id in enumerate(job_ids, start=offset + 1):
job = self._sa.get_job(job_id)
cat_id = self._sa.get_job_state(job.id).effective_category_id
duration_txt = _('No estimated duration provided for this job')
if job.estimated_duration is not None:
duration_txt = '{} {}'.format(job.estimated_duration,
_('seconds'))
# the next dict is only to get test_info generating code tidier
automated_desc = {
True: _('this job is fully automated'),
False: _('this job requires some manual interaction')
}
test_info = {
"id":
job.id,
"partial_id":
job.partial_id,
"name":
job.tr_summary(),
"category_id":
cat_id,
"category_name":
self._sa.get_category(cat_id).tr_name(),
"automated":
automated_desc[job.automated],
"duration":
duration_txt,
"description": (job.tr_description()
or _('No description provided for this job')),
"outcome":
self._sa.get_job_state(job.id).result.outcome,
"user":
job.user,
"command":
job.command,
"num":
job_no,
"plugin":
job.plugin,
}
test_info_list = test_info_list + ((test_info, ))
return test_info_list
def resume_by_id(self, session_id=None):
self._launcher = load_configs()
resume_candidates = list(self._sa.get_resumable_sessions())
if not session_id:
if not resume_candidates:
print('No session to resume')
return
session_id = resume_candidates[0].id
if session_id not in [s.id for s in resume_candidates]:
print("Requested session not found")
return
_logger.warning("Resuming session: %r", session_id)
self._normal_user = self._launcher.normal_user
pass_provider = (None if self._passwordless_sudo else
self.get_decrypted_password)
runner_kwargs = {
'normal_user_provider': lambda: self._normal_user,
'password_provider': pass_provider,
'stdin': self._pipe_to_subproc,
'extra_env': self.prepare_extra_env(),
}
meta = self._sa.resume_session(session_id, runner_kwargs=runner_kwargs)
app_blob = json.loads(meta.app_blob.decode("UTF-8"))
launcher = app_blob['launcher']
self._launcher.read_string(launcher, False)
self._sa.use_alternate_configuration(self._launcher)
test_plan_id = app_blob['testplan_id']
self._sa.select_test_plan(test_plan_id)
self._sa.bootstrap()
self._last_job = meta.running_job_name
result_dict = {
'outcome': IJobResult.OUTCOME_PASS,
'comments': _("Automatically passed after resuming execution"),
}
result_path = os.path.join(self._sa.get_session_dir(), 'CHECKBOX_DATA',
'__result')
if os.path.exists(result_path):
try:
with open(result_path, 'rt') as f:
result_dict = json.load(f)
# the only really important field in the result is
# 'outcome' so let's make sure it doesn't contain
# anything stupid
if result_dict.get('outcome') not in [
'pass', 'fail', 'skip'
]:
result_dict['outcome'] = IJobResult.OUTCOME_PASS
except json.JSONDecodeError as e:
pass
result = MemoryJobResult(result_dict)
if self._last_job:
try:
self._sa.use_job_result(self._last_job, result, True)
except KeyError:
raise SystemExit(self._last_job)
# some jobs have already been run, so we need to update the attempts
# count for future auto-rerunning
if self._launcher.auto_retry:
for job_id in [
job.id for job in self.get_rerun_candidates('auto')
]:
job_state = self._sa.get_job_state(job_id)
job_state.attempts = self._launcher.max_attempts - len(
job_state.result_history)
self._state = TestsSelected
def finalize_session(self):
self._sa.finalize_session()
self._reset_sa()
def transmit_input(self, text):
self._pipe_from_master.write(text)
self._pipe_from_master.flush()
def send_signal(self, signal):
if not self._currently_running_job:
return
target_user = self._sa.get_job(self._currently_running_job).user
self._sa.send_signal(signal, target_user)
@property
def manager(self):
return self._sa._manager
@property
def passwordless_sudo(self):
return self._passwordless_sudo
@property
def sideloaded_providers(self):
return self._sa.sideloaded_providers
class CheckboxConvergedApplication(PlainboxApplication):
"""
Class implementing the whole checkbox-converged application logic.
This class exposes methods that can be called by the javascript embedded
into many of the QML views. Each method implements a request / response
semantics where the request is the set of data passed to python from
javascript and the response is the python dictionary returned and processed
back on the javascript side.
This model follows the similar web development mechanics where the browser
can issue asynchronous requests in reaction to user interactions and uses
response data to alter the user interface.
"""
__version__ = (1, 5, 0, 'dev', 0)
def __init__(self, launcher_definition=None):
if plainbox.__version__ < (0, 22):
raise SystemExit("plainbox 0.22 required, you have {}".format(
ToolBase.format_version_tuple(plainbox.__version__)))
self.assistant = SessionAssistant('checkbox-converged')
self.ui = CheckboxConvergedUI()
self.index = 0
self._password = None
self._timestamp = None
self._latest_session = None
self._available_test_plans = []
self.test_plan_id = None
self.resume_candidate_storage = None
self.launcher = None
self.assistant.use_alternate_repository(
self._get_app_cache_directory())
# Prepare custom execution controller list
from plainbox.impl.ctrl import UserJobExecutionController
from sudo_with_pass_ctrl import RootViaSudoWithPassExecutionController
ctrl_setup_list = [
(RootViaSudoWithPassExecutionController, [self._password_provider],
{}),
(UserJobExecutionController, [], {}),
]
self.assistant.use_alternate_execution_controllers(ctrl_setup_list)
if launcher_definition:
generic_launcher = LauncherDefinition()
generic_launcher.read([launcher_definition])
config_filename = os.path.expandvars(
generic_launcher.config_filename)
if not os.path.split(config_filename)[0]:
configs = [
'/etc/xdg/{}'.format(config_filename),
os.path.expanduser('~/.config/{}'.format(config_filename))
]
else:
configs = [config_filename]
self.launcher = generic_launcher.get_concrete_launcher()
configs.append(launcher_definition)
self.launcher.read(configs)
# Checkbox-Converged supports new launcher syntax, so if we have
# LauncherDefinitionLegacy as launcher right now, let's replace it
# with a default one
if type(self.launcher) == LauncherDefinitionLegacy:
self.launcher = DefaultLauncherDefinition()
self.assistant.use_alternate_configuration(self.launcher)
self._prepare_transports()
else:
self.launcher = DefaultLauncherDefinition()
def __repr__(self):
return "app"
@view
def get_launcher_settings(self):
# this pseudo-adapter exists so qml can now know some bits about the
# launcher, if you need another setting in the QML fron-end, just add
# it to the returned dict below
return {
'ui_type': self.launcher.ui_type,
}
@view
def load_providers(self, providers_dir):
if self.launcher:
self.assistant.select_providers(
*self.launcher.providers,
additional_providers=self._get_embedded_providers(
providers_dir))
else:
self.assistant.select_providers(
'*',
additional_providers=self._get_embedded_providers(
providers_dir))
@view
def get_version_pair(self):
return {
'plainbox_version':
ToolBase.format_version_tuple(plainbox.__version__),
'application_version':
ToolBase.format_version_tuple(self.__version__)
}
@view
def start_session(self):
"""Start a new session."""
self.assistant.start_new_session('Checkbox Converged session')
self._timestamp = datetime.datetime.utcnow().isoformat()
return {
'session_id': self.assistant.get_session_id(),
'session_dir': self.assistant.get_session_dir()
}
@view
def resume_session(self, rerun_last_test, outcome='skip'):
"""
Resume latest sesssion.
:param rerun_last_test:
A bool stating whether runtime should repeat the test, that the app
was executing when it was interrupted.
:param outcome:
Outcome to set to the last job run. Option useless when rerunning.
"""
assert outcome in ['pass', 'skip', 'fail', None]
metadata = self.assistant.resume_session(self._latest_session)
app_blob = json.loads(metadata.app_blob.decode("UTF-8"))
self.index = app_blob['index_in_run_list']
self.test_plan_id = app_blob['test_plan_id']
self.assistant.select_test_plan(self.test_plan_id)
self.assistant.bootstrap()
if not rerun_last_test:
# Skip current test
test = self.get_next_test()['result']
test['outcome'] = outcome
self.register_test_result(test)
return {
'session_id': self._latest_session,
'session_dir': self.assistant.get_session_dir()
}
@view
def clear_session(self):
"""Reset app-custom state info about the session."""
self.index = 0
self._timestamp = datetime.datetime.utcnow().isoformat()
self._finalize_session()
@view
def is_session_resumable(self):
"""Check whether there is a session that can be resumed."""
for session_id, session_md in self.assistant.get_resumable_sessions():
if session_md.app_blob is None:
continue
# we're interested in the latest session only, this is why we
# return early
self._latest_session = session_id
return {
'resumable': True,
'running_job_name': session_md.running_job_name,
'error_encountered': False,
}
else:
return {
'resumable': False,
'error_encountered': False,
}
@view
def get_testplans(self):
"""Get the list of available test plans."""
if not self._available_test_plans:
if self.launcher:
if self.launcher.test_plan_forced:
self._available_test_plans = [
self.assistant.get_test_plan(
self.launcher.test_plan_default_selection)
]
else:
test_plan_ids = self.assistant.get_test_plans()
filtered_tp_ids = set()
for filter in self.launcher.test_plan_filters:
filtered_tp_ids.update(
fnmatch.filter(test_plan_ids, filter))
filtered_tp_ids = list(filtered_tp_ids)
filtered_tp_ids.sort(key=lambda tp_id: self.assistant.
get_test_plan(tp_id).name)
self._available_test_plans = [
self.assistant.get_test_plan(tp_id)
for tp_id in filtered_tp_ids
]
return {
'testplan_info_list': [{
"mod_id":
tp.id,
"mod_name":
tp.name,
"mod_selected":
tp.id == self.launcher.test_plan_default_selection,
"mod_disabled":
False,
} for tp in self._available_test_plans],
'forced_selection':
self.launcher.test_plan_forced
}
else:
self._available_test_plans = [
self.assistant.get_test_plan(tp_id)
for tp_id in self.assistant.get_test_plans()
]
return {
'testplan_info_list': [{
"mod_id": tp.id,
"mod_name": tp.name,
"mod_selected": False,
"mod_disabled": False,
} for tp in self._available_test_plans],
'forced_selection':
False
}
@view
def remember_testplan(self, test_plan_id):
"""Pick the test plan as the one in force."""
if self.test_plan_id:
# test plan has been previously selected. User changed mind, we
# have to abandon the session
self._finalize_session()
self.assistant.start_new_session('Checkbox Converged session')
self._timestamp = datetime.datetime.utcnow().isoformat()
self.test_plan_id = test_plan_id
self.assistant.select_test_plan(test_plan_id)
self.assistant.bootstrap()
# because session id (and storage) might have changed, let's share this
# info with the qml side
return {
'session_id': self.assistant.get_session_id(),
'session_dir': self.assistant.get_session_dir()
}
@view
def get_categories(self):
"""Get categories selection data."""
category_info_list = [{
"mod_id": category.id,
"mod_name": category.tr_name(),
"mod_selected": True,
"mod_disabled": False,
} for category in (
self.assistant.get_category(category_id)
for category_id in self.assistant.get_participating_categories())]
category_info_list.sort(key=lambda ci: (ci['mod_name']))
return {
'category_info_list': category_info_list,
'forced_selection': self.launcher.test_selection_forced
}
@view
def remember_categories(self, selected_id_list):
"""Save category selection."""
_logger.info("Selected categories: %s", selected_id_list)
# Remove previously set filters
self.assistant.remove_all_filters()
self.assistant.filter_jobs_by_categories(selected_id_list)
@view
def get_available_tests(self):
"""
Get all tests for selection purposes.
The response object will contain only tests with category matching
previously set list. Tests are sorted by (category, name)
"""
category_names = {
cat_id: self.assistant.get_category(cat_id).tr_name()
for cat_id in self.assistant.get_participating_categories()
}
job_units = [
self.assistant.get_job(job_id)
for job_id in self.assistant.get_static_todo_list()
]
mandatory_jobs = self.assistant.get_mandatory_jobs()
test_info_list = [{
"mod_id": job.id,
"mod_name": job.tr_summary(),
"mod_group": category_names[job.category_id],
"mod_selected": True,
"mod_disabled": job.id in mandatory_jobs,
} for job in job_units]
test_info_list.sort(key=lambda ti: (ti['mod_group'], ti['mod_name']))
return {
'test_info_list': test_info_list,
'forced_selection': self.launcher.test_selection_forced
}
@view
def get_rerun_candidates(self):
"""Get all the tests that might be selected for rerunning."""
def rerun_predicate(job_state):
return job_state.result.outcome in (
IJobResult.OUTCOME_FAIL, IJobResult.OUTCOME_CRASH,
IJobResult.OUTCOME_NOT_SUPPORTED, IJobResult.OUTCOME_SKIP)
rerun_candidates = []
todo_list = self.assistant.get_static_todo_list()
job_units = {
job_id: self.assistant.get_job(job_id)
for job_id in todo_list
}
job_states = {
job_id: self.assistant.get_job_state(job_id)
for job_id in todo_list
}
category_names = {
cat_id: self.assistant.get_category(cat_id).tr_name()
for cat_id in self.assistant.get_participating_categories()
}
for job_id, job_state in job_states.items():
if rerun_predicate(job_state):
rerun_candidates.append({
"mod_id":
job_id,
"mod_name":
job_units[job_id].tr_summary(),
"mod_group":
category_names[job_units[job_id].category_id],
"mod_selected":
False
})
return rerun_candidates
@view
def remember_tests(self, selected_id_list):
"""Save test selection."""
self.index = 0
self.assistant.use_alternate_selection(selected_id_list)
self.assistant.update_app_blob(self._get_app_blob())
_logger.info("Selected tests: %s", selected_id_list)
return
@view
def get_next_test(self):
"""
Get next text that is scheduled to run.
:returns:
Dictionary resembling JobDefinition or None if all tests are completed
"""
todo_list = self.assistant.get_static_todo_list()
if self.index < len(todo_list):
job = self.assistant.get_job(todo_list[self.index])
description = ""
if job.tr_purpose() is not None:
description = job.tr_purpose() + "\n"
if job.tr_steps() is not None:
description += job.tr_steps()
if not description:
description = job.tr_description()
test = {
"name":
job.tr_summary(),
"description":
description,
"verificationDescription":
job.tr_verification()
if job.tr_verification() is not None else description,
"plugin":
job.plugin,
"id":
job.id,
"partial_id":
job.partial_id,
"user":
job.user,
"qml_file":
job.qml_file,
"start_time":
time.time(),
"test_number":
todo_list.index(job.id),
"tests_count":
len(todo_list),
"command":
job.command,
"flags":
job.get_flag_set()
}
return test
else:
return {}
@view
def register_test_result(self, test):
"""Registers outcome of a test."""
_logger.info("Storing test result: %s", test)
job_id = test['id']
builder_kwargs = {
'outcome': test['outcome'],
'comments': test.get('comments', pod.UNSET),
'execution_duration': time.time() - test['start_time'],
}
if 'result' in test:
# if we're registering skipped test as an outcome of resuming
# session, the result field of the test object will be missing
builder_kwargs['return_code'] = test['result'].return_code
builder_kwargs['io_log_filename'] = test['result'].io_log_filename
builder_kwargs['io_log'] = test['result'].io_log
else:
builder_kwargs['return_code'] = 0
result = JobResultBuilder(**builder_kwargs).get_result()
self.assistant.use_job_result(job_id, result)
self.index += 1
self.assistant.update_app_blob(self._get_app_blob())
@view
def run_test_activity(self, test):
"""Run command associated with given test."""
plugins_handled_natively = ['qml']
res_builder = self.assistant.run_job(
test['id'], self.ui, test['plugin'] in plugins_handled_natively)
test['outcome'] = res_builder.outcome
test['result'] = res_builder
return test
@view
def get_results(self):
"""Get results object."""
stats = self.assistant.get_summary()
return {
'totalPassed':
stats[IJobResult.OUTCOME_PASS],
'totalFailed':
stats[IJobResult.OUTCOME_FAIL],
'totalSkipped':
stats[IJobResult.OUTCOME_SKIP] +
stats[IJobResult.OUTCOME_NOT_SUPPORTED] +
stats[IJobResult.OUTCOME_UNDECIDED]
}
@view
def export_results(self, output_format, option_list):
"""Export results to file(s) in the user's 'Documents' directory."""
self.assistant.finalize_session()
dirname = self._get_user_directory_documents()
return self.assistant.export_to_file(output_format, option_list,
dirname)
@view
def submit_results(self, config):
"""Submit results to a service configured by config."""
self.assistant.finalize_session()
transport = {
'hexr':
self.assistant.get_canonical_hexr_transport,
'hexr-staging':
(lambda: self.assistant.get_canonical_hexr_transport(staging=True)
),
'c3': (lambda: self.assistant.
get_canonical_certification_transport(config['secure_id'])),
'c3-staging':
(lambda: self.assistant.get_canonical_certification_transport(
config['secure_id'], staging=True)),
'oauth':
lambda: self.assistant.get_ubuntu_sso_oauth_transport(config),
}[config['type']]()
# Default to 'hexr' exporter as it provides xml submission format
# (CertificationTransport expects xml format for instance.)
submission_format = config.get('submission_format',
'2013.com.canonical.plainbox::hexr')
submission_options = config.get('submission_options', [])
return self.assistant.export_to_transport(submission_format, transport,
submission_options)
def _prepare_transports(self):
self._available_transports = get_all_transports()
self.transports = dict()
@view
def get_certification_transport_config(self):
"""Returns the c3 (certification) transport configuration."""
for report in self.launcher.stock_reports:
self._prepare_stock_report(report)
if 'c3' in self.launcher.transports:
return self.launcher.transports['c3']
elif 'c3-staging' in self.launcher.transports:
return self.launcher.transports['c3-staging']
return {}
@view
def export_results_with_launcher_settings(self):
"""
Export results to file(s) in the user's 'Documents' directory.
This method follows the launcher reports configuration.
"""
self.assistant.finalize_session()
for report in self.launcher.stock_reports:
self._prepare_stock_report(report)
# reports are stored in an ordinary dict(), so sorting them ensures
# the same order of submitting them between runs.
html_url = ""
for name, params in sorted(self.launcher.reports.items()):
exporter_id = self.launcher.exporters[params['exporter']]['unit']
if self.launcher.transports[params['transport']]['type'] == 'file':
path = self.launcher.transports[params['transport']]['path']
cls = self._available_transports['file']
self.transports[params['transport']] = cls(path)
transport = self.transports[params['transport']]
result = self.assistant.export_to_transport(
exporter_id, transport)
if (result and 'url' in result
and result['url'].endswith('html')):
html_url = result['url']
return html_url
def _prepare_stock_report(self, report):
# this is purposefully not using pythonic dict-keying for better
# readability
if not self.launcher.transports:
self.launcher.transports = dict()
if not self.launcher.exporters:
self.launcher.exporters = dict()
if not self.launcher.reports:
self.launcher.reports = dict()
if report == 'certification':
self.launcher.exporters['hexr'] = {
'unit': '2013.com.canonical.plainbox::hexr'
}
self.launcher.transports['c3'] = {
'type':
'certification',
'secure_id':
self.launcher.transports.get('c3', {}).get('secure_id', None)
}
self.launcher.reports['upload to certification'] = {
'transport': 'c3',
'exporter': 'hexr'
}
elif report == 'certification-staging':
self.launcher.exporters['hexr'] = {
'unit': '2013.com.canonical.plainbox::hexr'
}
self.launcher.transports['c3-staging'] = {
'type':
'certification',
'secure_id':
self.launcher.transports.get('c3', {}).get('secure_id', None),
'staging':
'yes'
}
self.launcher.reports['upload to certification-staging'] = {
'transport': 'c3-staging',
'exporter': 'hexr'
}
elif report == 'submission_files':
timestamp = datetime.datetime.utcnow().isoformat()
base_dir = self._get_user_directory_documents()
for exporter, file_ext in [('hexr', '.xml'), ('html', '.html'),
('xlsx', '.xlsx'), ('tar', '.tar.xz')]:
path = os.path.join(
base_dir, ''.join(['submission_', timestamp, file_ext]))
self.launcher.transports['{}_file'.format(exporter)] = {
'type': 'file',
'path': path
}
if exporter not in self.launcher.exporters:
self.launcher.exporters[exporter] = {
'unit':
'2013.com.canonical.plainbox::{}'.format(exporter)
}
self.launcher.reports['2_{}_file'.format(exporter)] = {
'transport': '{}_file'.format(exporter),
'exporter': '{}'.format(exporter)
}
@view
def drop_permissions(self, app_id, services):
# TODO: use XDG once available
trust_dbs = {
'camera': '~/.local/share/CameraService/trust.db',
'audio': '~/.local/share/PulseAudio/trust.db',
'location': '~/.local/share/UbuntuLocationServices/trust.db',
}
sql = 'delete from requests where ApplicationId = (?);'
for service in services:
conn = None
try:
conn = sqlite3.connect(os.path.expanduser(trust_dbs[service]),
isolation_level='EXCLUSIVE')
conn.execute(sql, (app_id, ))
conn.commit()
finally:
if conn:
conn.close()
@view
def get_incomplete_sessions(self):
"""Get ids of sessions with an 'incomplete' flag."""
self._incomplete_sessions = [
s[0]
for s in self.assistant.get_old_sessions(flags={'incomplete'},
allow_not_flagged=False)
]
return self._incomplete_sessions
@view
def delete_old_sessions(self, additional_sessions):
"""
Delete session storages.
:param additional_sessions:
List of ids of sessions that should be removed.
This function removes all complete sessions (i.e. the ones that session
assistant returns when get_old_sessions is run with the default params)
with the addition of the ones specified in the ``additional_sessions``
param.
"""
garbage = [s[0] for s in self.assistant.get_old_sessions()]
garbage += additional_sessions
self.assistant.delete_sessions(garbage)
def _get_user_directory_documents(self):
xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \
os.path.expanduser('~/.config')
with open(os.path.join(xdg_config_home, 'user-dirs.dirs')) as f:
match = re.search(r'XDG_DOCUMENTS_DIR="(.*)"\n', f.read())
if match:
return match.group(1).replace("$HOME", os.getenv("HOME"))
else:
return os.path.expanduser('~/Documents')
def _get_app_cache_directory(self):
xdg_cache_home = os.environ.get('XDG_CACHE_HOME') or \
os.path.expanduser('~/.cache')
app_id = os.environ.get('APP_ID')
if app_id:
# Applications will always have write access to directories they
# own as determined by the XDG base directory specification.
# Specifically: XDG_CACHE_HOME/<APP_PKGNAME>
# XDG_RUNTIME_DIR/<APP_PKGNAME>
# XDG_RUNTIME_DIR/confined/<APP_PKGNAME> (for TMPDIR)
# XDG_DATA_HOME/<APP_PKGNAME>
# XDG_CONFIG_HOME/<APP_PKGNAME>
# Note that <APP_PKGNAME> is not the APP_ID. In order to easily
# handle upgrades and sharing data between executables in the same
# app, we use the unversioned app package name for the writable
# directories.
return os.path.join(xdg_cache_home, app_id.split('_')[0])
else:
path = os.path.join(xdg_cache_home, "com.ubuntu.checkbox")
if not os.path.exists(path):
os.makedirs(path)
elif not os.path.isdir(path):
# as unlikely as it is, situation where path exists and is a
# regular file neeeds to be signalled
raise IOError("{} exists and is not a directory".format(path))
return path
def _get_app_blob(self):
"""
Get json dump of with app-specific blob
"""
return json.dumps({
'version': 1,
'test_plan_id': self.test_plan_id,
'index_in_run_list': self.index,
'session_timestamp': self._timestamp,
}).encode("UTF-8")
def _get_embedded_providers(self, providers_dir):
"""
Get providers included with the app
:param providers_dir:
Path within application tree from which to load providers
:returns:
list of loaded providers
"""
provider_list = []
app_root_dir = os.path.normpath(
os.getenv('APP_DIR', os.path.join(os.path.dirname(__file__),
'..')))
path = os.path.join(app_root_dir, os.path.normpath(providers_dir))
_logger.info("Loading all providers from %s", path)
if os.path.exists(path):
embedded_providers = EmbeddedProvider1PlugInCollection(path)
provider_list += embedded_providers.get_all_plugin_objects()
return provider_list
def _password_provider(self):
if self._password is None:
raise RuntimeError("execute_job called without providing password"
" first")
return self._password
def _finalize_session(self):
self.test_plan_id = ""
self.assistant.finalize_session()
def remember_password(self, password):
"""
Save password in app instance
It deliberately doesn't use view decorator to omit all logging that
might happen
"""
self._password = password
class CheckboxTool:
def __init__(self):
self.sa = SessionAssistant('CheckboxTool')
self.sa.select_providers('*')
self.sa.start_new_session('print_sequence')
self.all_tps = self.sa.get_test_plans()
self.units = dict()
self.template_units = dict()
for unit in self.sa._context._unit_list:
u_type = type(unit)
if u_type == TemplateUnit:
unit.re_matcher, unit.re_greedy = template_to_re(unit.id)
self.template_units[unit.id] = unit
if not (issubclass(u_type, JobDefinition)):
continue
self.units[unit.id] = unit
def get_unit_info(self, line, qualifier_unit):
kind = 'unknown'
extras = ''
full_id = line
ret_plugin = None
if "::" not in full_id:
qid = qualifier_unit.qualify_id(full_id)
full_id = qid
if '::after-suspend' in full_id:
full_id = full_id.replace('::after-suspend-', '::')
extras = ' <= {} REMOVED "after-suspend-" PREFIX'.format(full_id)
if full_id in self.units.keys():
plugin = self.units[full_id].plugin
ret_plugin = plugin
if plugin in ['shell', 'resource', 'attachment']:
kind = 'automatic'
else:
kind = 'manual'
if self.units[full_id].depends:
for dep_id in self.units[full_id].depends.split():
dep_kind, _ = self.get_kind_for_unit(
dep_id, self.units[full_id])
if dep_kind == 'manual':
extras += 'deps on manual'
if kind == 'unknown':
candidates = []
for tid, tunit in self.template_units.items():
matches = re.match(tunit.re_matcher, full_id)
if matches:
candidates.append(tunit)
if not candidates:
for tid, tunit in self.template_units.items():
matches = re.match(tunit.re_greedy, full_id)
if matches:
candidates.append(tunit)
if not candidates:
# ordinary matching: from include regex to template_ids
for tid, tunit in self.template_units.items():
matches = re.match(full_id, tid)
if matches:
candidates.append(tunit)
tunit = None
if len(candidates) == 1:
tunit = candidates[0]
elif len(candidates) > 1:
from difflib import SequenceMatcher
tid = sorted(
[c.id for c in candidates],
key=lambda a: SequenceMatcher(None, a, line).ratio(),
reverse=True)[0]
tunit = self.template_units[tid]
if tunit:
ret_plugin = tunit.get_record_value('plugin')
if tunit.get_record_value('plugin') in [
'shell', 'resource', 'attachment'
]:
kind = 'automatic'
else:
kind = 'manual'
if tunit.get_record_value('depends'):
for dep_id in tunit.get_record_value('depends').split():
dep_kind, _ = self.get_kind_for_unit(dep_id, tunit)
if dep_kind == 'manual':
extras += 'deps on man'
kind = 'manual'
if not ret_plugin:
ret_plugin = 'UNKNOWN'
return ret_plugin, extras
def get_kind_for_unit(self, line, qualifier_unit):
kind = 'unknown'
extras = ''
full_id = line
if "::" not in full_id:
qid = qualifier_unit.qualify_id(full_id)
full_id = qid
if '::after-suspend' in full_id:
full_id = full_id.replace('::after-suspend-', '::')
extras = ' <= {} REMOVED "after-suspend-" PREFIX'.format(full_id)
if full_id in self.units.keys():
plugin = self.units[full_id].plugin
if plugin in ['shell', 'resource', 'attachment']:
kind = 'automatic'
else:
kind = 'manual'
if self.units[full_id].depends:
for dep_id in self.units[full_id].depends.split():
dep_kind, _ = self.get_kind_for_unit(
dep_id, self.units[full_id])
if dep_kind == 'manual':
if kind == 'automatic':
extras = 'manual b/c of dep on {}'.format(
dep_id) + extras
kind = 'manual'
if kind == 'unknown':
candidates = []
for tid, tunit in self.template_units.items():
matches = re.match(tunit.re_matcher, full_id)
if matches:
candidates.append(tunit)
if not candidates:
for tid, tunit in self.template_units.items():
matches = re.match(tunit.re_greedy, full_id)
if matches:
candidates.append(tunit)
if not candidates:
# ordinary matching: from include regex to template_ids
for tid, tunit in self.template_units.items():
matches = re.match(full_id, tid)
if matches:
candidates.append(tunit)
tunit = None
if len(candidates) == 1:
tunit = candidates[0]
elif len(candidates) > 1:
from difflib import SequenceMatcher
tid = sorted(
[c.id for c in candidates],
key=lambda a: SequenceMatcher(None, a, line).ratio(),
reverse=True)[0]
tunit = self.template_units[tid]
if tunit:
if tunit.get_record_value('plugin') in [
'shell', 'resource', 'attachment'
]:
kind = 'automatic'
else:
kind = 'manual'
if tunit.get_record_value('depends'):
for dep_id in tunit.get_record_value('depends').split():
dep_kind, _ = self.get_kind_for_unit(dep_id, tunit)
if dep_kind == 'manual':
if kind == 'automatic':
extras = 'manual b/c of dep on {}'.format(
dep_id) + extras
kind = 'manual'
extras = ' <= {}'.format(tunit.id)
return kind, extras
def get_run_sequence(self, tp_id, include_nested=True):
tp_unit = self.sa.get_test_plan(tp_id)
result = []
if include_nested:
for tp in tp_unit.get_nested_part():
# print("NESTED {}".format(tp))
result += self.get_run_sequence(tp.id)
for line in tp_unit.include.split('\n'):
if not line:
continue
if line.startswith('#'):
continue
sections = line.split()
annotations = line[len(sections[0]):]
line = sections[0]
line = line.split()[0]
kind, extras = self.get_kind_for_unit(line, tp_unit)
result.append(UnitProxy(line, kind, extras, annotations))
return result
def split_tp(self, tpid):
tp_unit = self.sa.get_test_plan(tpid)
manuals = []
autos = []
unknowns = []
for unit in self.get_run_sequence(tpid, True):
if unit.kind == 'manual':
manuals.append(unit)
if unit.kind == 'automatic':
autos.append(unit)
if unit.kind == 'unknown':
unknowns.append(unit)
manual_tp = tpid[:-4] + 'manual'
new_man_pxu = ""
new_auto_pxu = ""
if manual_tp not in self.all_tps:
print("missing {}".format(manual_tp))
new_man_tp = TestPlanUnit(tp_unit._raw_data)
# remove the namespace prefix
include_entries = []
for unit in manuals:
namespace = tp_unit.qualify_id('')
if unit.id.startswith(namespace):
unit.id = unit.id.replace(namespace, '')
include_entries.append(unit.id + unit.annotations)
new_man_tp.id = unqualify_id(tp_unit, manual_tp)
new_man_tp.include = "\n".join(include_entries)
new_man_tp.name += ' (Manual)'
new_man_tp.description += ' (Manual)'
new_man_pxu = generate_tp_unit(new_man_tp)
else:
print("{} already there".format(manual_tp))
man_seq = self.get_run_sequence(manual_tp)
man_seq_ids = [unit.id for unit in man_seq]
new_seq = [unit.id for unit in manuals]
if man_seq_ids != new_seq:
print("BUT HAS WRONG INCLUDE")
print("FULL:\n{}\n\n VS \n\nMANUAL:\n{}".format(
"\n".join(new_seq), "\n".join(man_seq_ids)))
auto_tp = tpid[:-4] + 'automated'
if auto_tp not in self.all_tps:
print("missing {}".format(auto_tp))
new_auto_tp = TestPlanUnit(tp_unit._raw_data)
# remove the namespace prefix
include_entries = []
for unit in autos:
namespace = tp_unit.qualify_id('')
if unit.id.startswith(namespace):
unit.id = unit.id.replace(namespace, '')
include_entries.append(unit.id + unit.annotations)
new_auto_tp.id = unqualify_id(tp_unit, auto_tp)
new_auto_tp.include = "\n".join(include_entries)
new_auto_tp.name += ' (Automated)'
new_auto_tp.description += ' (Automated)'
new_auto_pxu = generate_tp_unit(new_auto_tp)
else:
print("{} already there".format(auto_tp))
auto_seq = self.get_run_sequence(auto_tp)
auto_seq_ids = [unit.id for unit in auto_seq]
new_seq = [unit.id for unit in autos]
if auto_seq_ids != new_seq:
print("BUT HAS DIFFERENT INCLUDE")
print("FULL:\n{}\n\n VS \n\AUTOMATED:\n{}".format(
"\n".join(new_seq), "\n".join(auto_seq_ids)))
if not new_man_pxu and not new_auto_pxu:
return
tp_unit.include = ""
tp_unit.id = unqualify_id(tp_unit, tp_unit.id)
tp_unit.nested_part = "\n".join([manual_tp, auto_tp])
new_full_pxu = generate_tp_unit(tp_unit)
with open(tp_unit.origin.source.filename, 'rt') as f:
pxu = f.readlines()
backup_path = tp_unit.origin.source.filename + '.bkp'
if os.path.exists(backup_path):
print("Backup file already present. Not overwriting: {}".format(
backup_path))
else:
shutil.copyfile(tp_unit.origin.source.filename, backup_path)
print("Backup made: {}".format(backup_path))
new_pxu = pxu[:tp_unit.origin.line_start - 1]
new_pxu.append(new_full_pxu)
new_pxu.append("\n")
if new_man_pxu:
new_pxu.append(new_man_pxu)
new_pxu.append("\n")
if new_auto_pxu:
new_pxu.append(new_auto_pxu)
new_pxu.append("\n")
new_pxu += pxu[tp_unit.origin.line_end:]
with open(tp_unit.origin.source.filename, 'wt') as f:
f.write("".join(new_pxu))
print("{} rewritten!".format(tp_unit.origin.source.filename))
def annotated_tp(self, tp_id):
tp_unit = self.sa.get_test_plan(tp_id)
new_include = ''
for line in tp_unit.include.split('\n'):
if not line:
new_include += line + '\n'
continue
if line.startswith('#'):
new_include += line + '\n'
continue
if '#!-' in line:
new_include += line + '\n'
continue
sections = line.split()
annotations = line[len(sections[0]):]
pattern = sections[0]
pattern = pattern.split()[0]
plugin, extras = self.get_unit_info(pattern, tp_unit)
new_include += '{} #!- {} - {}\n'.format(line, plugin, extras)
tp_unit.include = new_include
return generate_tp_unit(tp_unit)
class RemoteSessionAssistant():
"""Remote execution enabling wrapper for the SessionAssistant"""
REMOTE_API_VERSION = 11
def __init__(self, cmd_callback):
_logger.debug("__init__()")
self._cmd_callback = cmd_callback
self._session_change_lock = Lock()
self._operator_lock = Lock()
self._ui = BufferedUI()
self._input_piping = os.pipe()
self._passwordless_sudo = is_passwordless_sudo()
self.terminate_cb = None
self._pipe_from_master = open(self._input_piping[1], 'w')
self._pipe_to_subproc = open(self._input_piping[0])
self._reset_sa()
self._currently_running_job = None
def _reset_sa(self):
_logger.info("Resetting RSA")
self._state = Idle
self._sa = SessionAssistant('service', api_flags={SA_RESTARTABLE})
self._be = None
self._session_id = ""
self._jobs_count = 0
self._job_index = 0
self._currently_running_job = None # XXX: yuck!
self._last_job = None
self._current_comments = ""
self._last_response = None
self._normal_user = ''
self.session_change_lock.acquire(blocking=False)
self.session_change_lock.release()
@property
def session_change_lock(self):
return self._session_change_lock
@property
def config(self):
return self._sa.config
def update_app_blob(self, app_blob):
self._sa.update_app_blob(app_blob)
def allowed_when(*states):
def wrap(f):
def fun(self, *args):
if self._state not in states:
raise AssertionError(
"expected %s, is %s" % (states, self._state))
return f(self, *args)
return fun
return wrap
def interact(self, interaction):
self._state = Interacting
self._current_interaction = interaction
yield self._current_interaction
@allowed_when(Interacting)
def remember_users_response(self, response):
if response == 'rollback':
self._currently_running_job = None
self.session_change_lock.acquire(blocking=False)
self.session_change_lock.release()
self._current_comments = ""
self._state = TestsSelected
return
self._last_response = response
self._state = Running
def _prepare_display_without_psutil(self):
try:
display_value = check_output(
'strings /proc/*/environ 2>/dev/null | '
'grep -m 1 -oP "(?<=DISPLAY=).*"',
shell=True, universal_newlines=True).rstrip()
xauth_value = check_output(
'strings /proc/*/environ 2>/dev/null | '
'grep -m 1 -oP "(?<=XAUTHORITY=).*"',
shell=True, universal_newlines=True).rstrip()
return {'DISPLAY': display_value, 'XAUTHORITY': xauth_value}
except CalledProcessError:
return None
def prepare_extra_env(self):
# If possible also set the DISPLAY env var
# i.e when a user desktop session is running
for p in psutil.pids():
try:
p_environ = psutil.Process(p).environ()
p_user = psutil.Process(p).username()
except psutil.AccessDenied:
continue
except AttributeError:
# psutil < 4.0.0 doesn't provide Process.environ()
return self._prepare_display_without_psutil()
except psutil.NoSuchProcess:
# quietly ignore the process that died before we had a chance
# to read the environment from them
continue
if (
"DISPLAY" in p_environ and
"XAUTHORITY" in p_environ and
p_user != 'gdm'
): # gdm uses :1024
return {
'DISPLAY': p_environ['DISPLAY'],
'XAUTHORITY': p_environ['XAUTHORITY']
}
@allowed_when(Idle)
def start_session(self, configuration):
self._reset_sa()
_logger.info("start_session: %r", configuration)
session_title = 'checkbox-slave'
session_desc = 'checkbox-slave session'
session_type = 'checkbox-slave'
self._launcher = load_configs()
if configuration['launcher']:
self._launcher.read_string(configuration['launcher'], False)
if self._launcher.session_title:
session_title = self._launcher.session_title
if self._launcher.session_desc:
session_desc = self._launcher.session_desc
self._sa.use_alternate_configuration(self._launcher)
if configuration['normal_user']:
self._normal_user = configuration['normal_user']
else:
self._normal_user = self._launcher.normal_user
if not self._normal_user:
import pwd
try:
self._normal_user = pwd.getpwuid(1000).pw_name
_logger.warning(
("normal_user not supplied via config(s). "
"non-root jobs will run as %s"), self._normal_user)
except KeyError:
raise RuntimeError(
("normal_user not supplied via config(s). "
"Username for uid 1000 not found"))
runner_kwargs = {
'normal_user_provider': lambda: self._normal_user,
'stdin': self._pipe_to_subproc,
'extra_env': self.prepare_extra_env(),
}
self._sa.start_new_session(session_title, UnifiedRunner, runner_kwargs)
new_blob = json.dumps({
'description': session_desc,
'type': session_type,
'launcher': configuration['launcher'],
'effective_normal_user': self._normal_user,
}).encode("UTF-8")
self._sa.update_app_blob(new_blob)
self._sa.configure_application_restart(self._cmd_callback)
self._session_id = self._sa.get_session_id()
tps = self._sa.get_test_plans()
filtered_tps = set()
for filter in self._launcher.test_plan_filters:
filtered_tps.update(fnmatch.filter(tps, filter))
filtered_tps = list(filtered_tps)
response = zip(filtered_tps, [self._sa.get_test_plan(
tp).name for tp in filtered_tps])
self._state = Started
self._available_testplans = sorted(
response, key=lambda x: x[1]) # sorted by name
return self._available_testplans
@allowed_when(Started)
def prepare_bootstrapping(self, test_plan_id):
"""Save picked test plan to the app blob."""
_logger.debug("prepare_bootstrapping: %r", test_plan_id)
self._sa.update_app_blob(json.dumps(
{'testplan_id': test_plan_id, }).encode("UTF-8"))
self._sa.select_test_plan(test_plan_id)
# TODO: REMOTE API RAPI: Change this API on the next RAPI bump
# previously the function returned bool signifying the need for sudo
# password. With slave being guaranteed to never need it anymor
# we can make this funciton return nothing
return False
@allowed_when(Started)
def get_bootstrapping_todo_list(self):
return self._sa.get_bootstrap_todo_list()
def finish_bootstrap(self):
self._sa.finish_bootstrap()
self._state = Bootstrapped
if self._launcher.auto_retry:
for job_id in self._sa.get_static_todo_list():
job_state = self._sa.get_job_state(job_id)
job_state.attempts = self._launcher.max_attempts
return self._sa.get_static_todo_list()
def get_manifest_repr(self):
return self._sa.get_manifest_repr()
def save_manifest(self, manifest_answers):
return self._sa.save_manifest(manifest_answers)
def modify_todo_list(self, chosen_jobs):
self._sa.use_alternate_selection(chosen_jobs)
def finish_job_selection(self):
self._jobs_count = len(self._sa.get_dynamic_todo_list())
self._state = TestsSelected
@allowed_when(Interacting)
def rerun_job(self, job_id, result):
self._sa.use_job_result(job_id, result)
self.session_change_lock.acquire(blocking=False)
self.session_change_lock.release()
self._state = TestsSelected
def _get_ui_for_job(self, job):
show_out = True
if self._launcher.output == 'hide-resource-and-attachment':
if job.plugin in ('local', 'resource', 'attachment'):
show_out = False
elif self._launcher.output in ['hide', 'hide-automated']:
if job.plugin in ('shell', 'local', 'resource', 'attachment'):
show_out = False
if 'suppress-output' in job.get_flag_set():
show_out = False
if show_out:
self._ui = BufferedUI()
else:
self._ui = RemoteSilentUI()
return self._ui
@allowed_when(TestsSelected)
def run_job(self, job_id):
"""
Depending on the type of the job, run_job can yield different number
of Interaction instances.
"""
_logger.debug("run_job: %r", job_id)
self._job_index = self._jobs_count - len(
self._sa.get_dynamic_todo_list()) + 1
self._currently_running_job = job_id
self._current_comments = ""
job = self._sa.get_job(job_id)
job_state = self._sa.get_job_state(job_id)
if not job_state.can_start():
outcome = IJobResult.OUTCOME_NOT_SUPPORTED
for inhibitor in job_state.readiness_inhibitor_list:
if (
inhibitor.cause == InhibitionCause.FAILED_RESOURCE and
'fail-on-resource' in job.get_flag_set()
):
outcome = IJobResult.OUTCOME_FAIL
break
elif inhibitor.cause != InhibitionCause.FAILED_DEP:
continue
related_job_state = self._sa._context.state.job_state_map[
inhibitor.related_job.id]
if related_job_state.result.outcome == IJobResult.OUTCOME_SKIP:
outcome = IJobResult.OUTCOME_SKIP
def cant_start_builder(*args, **kwargs):
result_builder = JobResultBuilder(
outcome=outcome,
comments=job_state.get_readiness_description())
return result_builder
self._be = BackgroundExecutor(self, job_id, cant_start_builder)
yield from self.interact(Interaction('skip', None, self._be))
if job.plugin in [
'manual', 'user-interact-verify', 'user-interact']:
may_comment = True
while may_comment:
may_comment = False
if job.tr_description() and not job.tr_purpose():
yield from self.interact(
Interaction('description', job.tr_description()))
if job.tr_purpose():
yield from self.interact(
Interaction('purpose', job.tr_purpose()))
if job.tr_steps():
yield from self.interact(
Interaction('steps', job.tr_steps()))
if self._last_response == 'comment':
yield from self.interact(Interaction('comment'))
if self._last_response:
self._current_comments += self._last_response
may_comment = True
continue
if self._last_response == 'skip':
def skipped_builder(*args, **kwargs):
result_builder = JobResultBuilder(
outcome=IJobResult.OUTCOME_SKIP,
comments=_("Explicitly skipped before execution"))
if self._current_comments != "":
result_builder.comments = self._current_comments
return result_builder
self._be = BackgroundExecutor(
self, job_id, skipped_builder)
yield from self.interact(
Interaction('skip', job.verification, self._be))
if job.command:
self._state = Running
ui = self._get_ui_for_job(job)
self._be = BackgroundExecutor(self, job_id, self._sa.run_job, ui)
else:
def undecided_builder(*args, **kwargs):
return JobResultBuilder(outcome=IJobResult.OUTCOME_UNDECIDED)
self._be = BackgroundExecutor(self, job_id, undecided_builder)
if self._sa.get_job(self._currently_running_job).plugin in [
'manual', 'user-interact-verify']:
yield from self.interact(
Interaction('verification', job.verification, self._be))
@allowed_when(Started, Bootstrapping)
def run_bootstrapping_job(self, job_id):
self._currently_running_job = job_id
self._state = Bootstrapping
self._be = BackgroundExecutor(self, job_id, self._sa.run_job)
@allowed_when(Running, Bootstrapping, Interacting, TestsSelected)
def monitor_job(self):
"""
Check the state of the currently running job.
:returns:
(state, payload) tuple.
Payload conveys detailed info that's characteristic
to the current state.
"""
_logger.debug("monitor_job()")
# either return [done, running, awaiting response]
# TODO: handle awaiting_response (reading from stdin by the job)
if self._be and self._be.is_alive():
return ('running', self._ui.get_output())
else:
return ('done', self._ui.get_output())
def get_remote_api_version(self):
return self.REMOTE_API_VERSION
def whats_up(self):
"""
Check what is remote-service up to
:returns:
(state, payload) tuple.
"""
_logger.debug("whats_up() -> %r", self._state)
payload = None
if self._state == Running:
payload = (
self._job_index, self._jobs_count, self._currently_running_job
)
if self._state == TestsSelected and not self._currently_running_job:
payload = {'last_job': self._last_job}
elif self._state == Started:
payload = self._available_testplans
elif self._state == Interacting:
payload = self._current_interaction
elif self._state == Bootstrapped:
payload = self._sa.get_static_todo_list()
return self._state, payload
def terminate(self):
if self.terminate_cb:
self.terminate_cb()
def get_session_progress(self):
"""Return list of completed and not completed jobs in a dict."""
_logger.debug("get_session_progress()")
return {
"done": self._sa.get_dynamic_done_list(),
"todo": self._sa.get_dynamic_todo_list(),
}
def get_master_public_key(self):
# TODO: REMOTE API RAPI: Remove this API on the next RAPI bump
# this key is only for RAPI compliance. It will never be used as
# this master requires slave to be completely sudoless
return (
b'-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMII'
b'BCgKCAQEA5r0bjOA+IH5lDKkW3OYb\nDuEjf5VKgUlDSJJuyBlfLTBIXZ8j3s98'
b'6AbV0zB62rAcgiFrBOzx51IzBDBmHI8V\nYYpEa+q4OP4yprYpSg6xzX6LRQapC'
b'Iv9BAqN4MWrKBukGMzJyemIVEPv4BSHL5L/\nLY98Mwh4dAXxj5ZdsoVPqgeMo8'
b'dxfYEOwVRJvSkseIhxRL6tvgP37c48ApUyjdUO\n3C2YgqJRx7mKKDyLOvhDVEl'
b'MqkAfp6qS/8xcGBTEqn08dDQIgPl8KofpC9GXMGbK\nV9FGP+c1bpA3vMOfnpsE'
b'WCju2qDoTSKJTm3VMZj88mqH7nOpbk7JI/Yz0EmtNXOM\n6QIDAQAB\n-----EN'
b'D PUBLIC KEY-----')
def save_password(self, password):
"""Store sudo password"""
# TODO: REMOTE API RAPI: Remove this API on the next RAPI bump
# if the slave is running it means we don't need password
# so we can consider call to this function as passing
return True
def finish_job(self, result=None):
# assert the thread completed
self.session_change_lock.acquire(blocking=False)
self.session_change_lock.release()
if self._sa.get_job(self._currently_running_job).plugin in [
'manual', 'user-interact-verify'] and not result:
# for manually verified jobs we don't set the outcome here
# it is already determined
return
if not result:
result = self._be.wait().get_result()
self._sa.use_job_result(self._currently_running_job, result)
if self._state != Bootstrapping:
if not self._sa.get_dynamic_todo_list():
if (
self._launcher.auto_retry and
self.get_rerun_candidates('auto')
):
self._state = TestsSelected
else:
self._state = Idle
else:
self._state = TestsSelected
return result
def get_rerun_candidates(self, session_type='manual'):
return self._sa.get_rerun_candidates(session_type)
def prepare_rerun_candidates(self, rerun_candidates):
candidates = self._sa.prepare_rerun_candidates(rerun_candidates)
self._state = TestsSelected
return candidates
def get_job_result(self, job_id):
return self._sa.get_job_state(job_id).result
def get_jobs_repr(self, job_ids, offset=0):
"""
Translate jobs into a {'field': 'val'} representations.
:param job_ids:
list of job ids to get and translate
:param offset:
apply an offset to the job number if for instance the job list
is being requested part way through a session
:returns:
list of dicts representing jobs
"""
test_info_list = tuple()
for job_no, job_id in enumerate(job_ids, start=offset + 1):
job = self._sa.get_job(job_id)
cat_id = self._sa.get_job_state(job.id).effective_category_id
duration_txt = _('No estimated duration provided for this job')
if job.estimated_duration is not None:
duration_txt = '{} {}'.format(job.estimated_duration, _(
'seconds'))
# the next dict is only to get test_info generating code tidier
automated_desc = {
True: _('this job is fully automated'),
False: _('this job requires some manual interaction')
}
test_info = {
"id": job.id,
"partial_id": job.partial_id,
"name": job.tr_summary(),
"category_id": cat_id,
"category_name": self._sa.get_category(cat_id).tr_name(),
"automated": automated_desc[job.automated],
"duration": duration_txt,
"description": (job.tr_description() or
_('No description provided for this job')),
"outcome": self._sa.get_job_state(job.id).result.outcome,
"user": job.user,
"command": job.command,
"num": job_no,
"plugin": job.plugin,
}
test_info_list = test_info_list + ((test_info, ))
return json.dumps(test_info_list)
def resume_by_id(self, session_id=None):
_logger.info("resume_by_id: %r", session_id)
self._launcher = load_configs()
resume_candidates = list(self._sa.get_resumable_sessions())
if not session_id:
if not resume_candidates:
print('No session to resume')
return
session_id = resume_candidates[0].id
if session_id not in [s.id for s in resume_candidates]:
print("Requested session not found")
return
_logger.warning("Resuming session: %r", session_id)
runner_kwargs = {
'normal_user_provider': lambda: self._normal_user,
'stdin': self._pipe_to_subproc,
'extra_env': self.prepare_extra_env(),
}
meta = self._sa.resume_session(session_id, runner_kwargs=runner_kwargs)
app_blob = json.loads(meta.app_blob.decode("UTF-8"))
launcher = app_blob['launcher']
self._launcher.read_string(launcher, False)
self._sa.use_alternate_configuration(self._launcher)
self._normal_user = app_blob.get(
'effective_normal_user', self._launcher.normal_user)
_logger.info(
"normal_user after loading metadata: %r", self._normal_user)
test_plan_id = app_blob['testplan_id']
self._sa.select_test_plan(test_plan_id)
self._sa.bootstrap()
self._last_job = meta.running_job_name
result_dict = {
'outcome': IJobResult.OUTCOME_PASS,
'comments': _("Automatically passed after resuming execution"),
}
session_share = WellKnownDirsHelper.session_share(
self._sa._manager.storage.id)
result_path = os.path.join(session_share, '__result')
if os.path.exists(result_path):
try:
with open(result_path, 'rt') as f:
result_dict = json.load(f)
# the only really important field in the result is
# 'outcome' so let's make sure it doesn't contain
# anything stupid
if result_dict.get('outcome') not in [
'pass', 'fail', 'skip']:
result_dict['outcome'] = IJobResult.OUTCOME_PASS
except json.JSONDecodeError as e:
pass
result = MemoryJobResult(result_dict)
if self._last_job:
try:
self._sa.use_job_result(self._last_job, result, True)
except KeyError:
raise SystemExit(self._last_job)
# some jobs have already been run, so we need to update the attempts
# count for future auto-rerunning
if self._launcher.auto_retry:
for job_id in [
job.id for job in self.get_rerun_candidates('auto')]:
job_state = self._sa.get_job_state(job_id)
job_state.attempts = self._launcher.max_attempts - len(
job_state.result_history)
self._state = TestsSelected
def finalize_session(self):
self._sa.finalize_session()
self._reset_sa()
def transmit_input(self, text):
self._pipe_from_master.write(text)
self._pipe_from_master.flush()
def send_signal(self, signal):
if not self._currently_running_job:
return
target_user = self._sa.get_job(self._currently_running_job).user
self._sa.send_signal(signal, target_user)
@property
def manager(self):
return self._sa._manager
@property
def passwordless_sudo(self):
# TODO: REMOTE API RAPI: Remove this API on the next RAPI bump
# if the slave is still running it means it's very passwordless
return True
@property
def sideloaded_providers(self):
return self._sa.sideloaded_providers
def exposed_cache_report(self, exporter_id, options):
exporter = self._sa._manager.create_exporter(exporter_id, options)
exported_stream = SpooledTemporaryFile(max_size=102400, mode='w+b')
exporter.dump_from_session_manager(self._sa._manager, exported_stream)
exported_stream.flush()
return exported_stream