def test_schedule_retry(self):
"""Test scheduling a retry of a failed install."""
updater = SkillUpdater(self.message_bus_mock)
updater._schedule_retry()
self.assertEqual(1, updater.install_retries)
self.assertEqual(400, updater.next_download)
self.assertFalse(updater.default_skill_install_error)
def __init__(self, bus, watchdog=None):
"""Constructor
Arguments:
bus (event emitter): Mycroft messagebus connection
watchdog (callable): optional watchdog function
"""
super(SkillManager, self).__init__()
self.bus = bus
# Set watchdog to argument or function returning None
self._watchdog = watchdog or (lambda: None)
self._stop_event = Event()
self._connected_event = Event()
self.config = Configuration.get()
self.upload_queue = UploadQueue()
self.skill_loaders = {}
self.enclosure = EnclosureAPI(bus)
self.initial_load_complete = False
self.num_install_retries = 0
self.settings_downloader = SkillSettingsDownloader(self.bus)
self.empty_skill_dirs = set() # Save a record of empty skill dirs.
# Statuses
self._alive_status = False # True after priority skills has loaded
self._loaded_status = False # True after all skills has loaded
self.skill_updater = SkillUpdater()
self._define_message_bus_events()
self.daemon = True
self.lang_code = self.config.get("lang", 'en-us')
load_languages([self.lang_code, 'en-us'])
def test_install_or_update_local(self):
"""Test calling install_or_update with a local skill"""
skill = self._build_mock_msm_skill_list()
updater = SkillUpdater(self.message_bus_mock)
updater.install_or_update(skill)
self.assertIn('foobar', updater.installed_skills)
skill.update.assert_called_once_with()
skill.update_deps.assert_called_once_with()
self.msm_mock.install.assert_not_called()
def test_install_or_update_beta(self):
"""Test calling install_or_update with a beta skill."""
self.msm_mock.device_skill_state['skills'][0]['beta'] = True
skill = self._build_mock_msm_skill_list()
skill.is_local = False
updater = SkillUpdater(self.message_bus_mock)
updater.install_or_update(skill)
self.assertIn('foobar', updater.installed_skills)
self.assertIsNone(skill.sha)
def test_install_or_update_default(self):
"""Test calling install_or_update with a default skill"""
skill = self._build_mock_msm_skill_list()
skill.name = 'test_skill'
skill.is_local = False
updater = SkillUpdater(self.message_bus_mock)
updater.install_or_update(skill)
self.assertIn('test_skill', updater.installed_skills)
self.assertTrue(not skill.update.called)
self.msm_mock.install.assert_called_once_with(skill, origin='default')
def test_apply_install_or_update_quick(self):
"""Test invoking MSM to install or update skills quickly"""
skill = self._build_mock_msm_skill_list()
self.msm_mock.list_all_defaults.return_value = [skill]
updater = SkillUpdater(self.message_bus_mock)
updater._apply_install_or_update(quick=True)
self.msm_mock.apply.assert_called_once_with(updater.install_or_update,
self.msm_mock.list(),
max_threads=20)
def test_download_skills_not_connected(self):
"""Test the error that occurs when the device is not connected."""
with patch(self.mock_package + 'connected') as connected_mock:
connected_mock.return_value = False
with patch(self.mock_package + 'time', spec=True) as time_mock:
time_mock.return_value = 100
updater = SkillUpdater(self.message_bus_mock)
result = updater.update_skills()
self.assertFalse(result)
self.assertEqual(400, updater.next_download)
def test_install_or_update_default_fail(self):
"""Test calling install_or_update with a failed install result"""
skill = self._build_mock_msm_skill_list()
skill.name = 'test_skill'
skill.is_local = False
self.msm_mock.install.side_effect = ValueError
updater = SkillUpdater(self.message_bus_mock)
with self.assertRaises(ValueError):
updater.install_or_update(skill)
self.assertNotIn('test_skill', updater.installed_skills)
self.assertTrue(not skill.update.called)
self.msm_mock.install.assert_called_once_with(skill, origin='default')
self.assertTrue(updater.default_skill_install_error)
def test_load_installed_skills(self):
"""Test loading a set of installed skills into an instance attribute"""
skill_file_path = str(self.temp_dir.joinpath('.mycroft_skills'))
with open(skill_file_path, 'w') as skill_file:
skill_file.write('FooSkill\n')
skill_file.write('BarSkill\n')
patch_path = (self.mock_package +
'SkillUpdater.installed_skills_file_path')
with patch(patch_path, new_callable=PropertyMock) as mock_file_path:
mock_file_path.return_value = skill_file_path
updater = SkillUpdater(self.message_bus_mock)
updater._load_installed_skills()
self.assertEqual({'FooSkill', 'BarSkill'}, updater.installed_skills)
def test_save_installed_skills(self):
"""Test saving list of installed skills to a file."""
skill_file_path = str(self.temp_dir.joinpath('.mycroft_skills'))
patch_path = (self.mock_package +
'SkillUpdater.installed_skills_file_path')
with patch(patch_path, new_callable=PropertyMock) as mock_file:
mock_file.return_value = skill_file_path
updater = SkillUpdater(self.message_bus_mock)
updater.installed_skills = ['FooSkill', 'BarSkill']
updater._save_installed_skills()
with open(skill_file_path) as skill_file:
skills = skill_file.readlines()
self.assertListEqual(['FooSkill\n', 'BarSkill\n'], skills)
def test_installed_skills_path_not_virtual_env(self):
"""Test the property representing the installed skill file path."""
with patch(self.mock_package + 'os.access') as os_patch:
os_patch.return_value = False
updater = SkillUpdater(self.message_bus_mock)
self.assertEqual(path.expanduser('~/.mycroft/.mycroft-skills'),
updater.installed_skills_file_path)
def test_update_download_time(self):
"""Test updating the next time a download will occur."""
dot_msm_path = self.temp_dir.joinpath('.msm')
dot_msm_path.touch()
dot_msm_mtime_before = dot_msm_path.stat().st_mtime
sleep(0.5)
SkillUpdater(self.message_bus_mock)._update_download_time()
dot_msm_mtime_after = dot_msm_path.stat().st_mtime
self.assertLess(dot_msm_mtime_before, dot_msm_mtime_after)
def test_installed_skills_path_virtual_env(self):
"""Test the property representing the installed skill file path."""
with patch(self.mock_package + 'sys', spec=True) as sys_mock:
sys_mock.executable = 'path/to/the/virtual_env/bin/python'
with patch(self.mock_package + 'os.access') as os_patch:
os_patch.return_value = True
updater = SkillUpdater(self.message_bus_mock)
self.assertEqual('path/to/the/virtual_env/.mycroft-skills',
updater.installed_skills_file_path)
def test_post_manifest_allowed(self):
"""Test calling the skill manifest API endpoint"""
self.msm_mock.device_skill_state = 'foo'
with patch(self.mock_package + 'is_paired') as paired_mock:
paired_mock.return_value = True
with patch(self.mock_package + 'DeviceApi', spec=True) as api_mock:
SkillUpdater(self.message_bus_mock).post_manifest()
api_instance = api_mock.return_value
api_instance.upload_skills_data.assert_called_once_with('foo')
paired_mock.assert_called_once_with()
def test_installed_skills_path_not_virtual_env(self):
"""Test the property representing the installed skill file path."""
with patch(self.mock_package + 'os.access') as os_patch:
os_patch.return_value = False
updater = SkillUpdater(self.message_bus_mock)
self.assertEqual(
os.path.join(BaseDirectory.save_data_path('mycroft'),
'.mycroft-skills'),
updater.installed_skills_file_path
)
class SkillManager(Thread):
_msm = None
def __init__(self, bus, watchdog=None):
"""Constructor
Arguments:
bus (event emitter): Mycroft messagebus connection
watchdog (callable): optional watchdog function
"""
super(SkillManager, self).__init__()
self.bus = bus
# Set watchdog to argument or function returning None
self._watchdog = watchdog or (lambda: None)
self._stop_event = Event()
self._connected_event = Event()
self.config = Configuration.get()
self.upload_queue = UploadQueue()
self.skill_loaders = {}
self.enclosure = EnclosureAPI(bus)
self.initial_load_complete = False
self.num_install_retries = 0
self.settings_downloader = SkillSettingsDownloader(self.bus)
self.empty_skill_dirs = set() # Save a record of empty skill dirs.
# Statuses
self._alive_status = False # True after priority skills has loaded
self._loaded_status = False # True after all skills has loaded
self.skill_updater = SkillUpdater()
self._define_message_bus_events()
self.daemon = True
self.lang_code = self.config.get("lang", 'en-us')
load_languages([self.lang_code, 'en-us'])
def _define_message_bus_events(self):
"""Define message bus events with handlers defined in this class."""
# Conversation management
self.bus.on('skill.converse.request', self.handle_converse_request)
# Update on initial connection
self.bus.on('mycroft.internet.connected',
lambda x: self._connected_event.set())
# Update upon request
self.bus.on('skillmanager.update', self.schedule_now)
self.bus.on('skillmanager.list', self.send_skill_list)
self.bus.on('skillmanager.deactivate', self.deactivate_skill)
self.bus.on('skillmanager.keep', self.deactivate_except)
self.bus.on('skillmanager.activate', self.activate_skill)
self.bus.on('mycroft.paired', self.handle_paired)
self.bus.on('mycroft.skills.settings.update',
self.settings_downloader.download)
@property
def skills_config(self):
return self.config['skills']
@property
def msm(self):
if self._msm is None:
msm_config = build_msm_config(self.config)
self._msm = msm_creator(msm_config)
return self._msm
@staticmethod
def create_msm():
LOG.debug('instantiating msm via static method...')
msm_config = build_msm_config(Configuration.get())
msm_instance = msm_creator(msm_config)
return msm_instance
def schedule_now(self, _):
self.skill_updater.next_download = time() - 1
def _start_settings_update(self):
LOG.info('Start settings update')
self.skill_updater.post_manifest(reload_skills_manifest=True)
self.upload_queue.start()
LOG.info('All settings meta has been processed or upload has started')
self.settings_downloader.download()
LOG.info('Skill settings downloading has started')
def handle_paired(self, _):
"""Trigger upload of skills manifest after pairing."""
self._start_settings_update()
def load_priority(self):
skills = {skill.name: skill for skill in self.msm.all_skills}
priority_skills = self.skills_config.get("priority_skills", [])
for skill_name in priority_skills:
skill = skills.get(skill_name)
if skill is not None:
if not skill.is_local:
try:
self.msm.install(skill)
except Exception:
log_msg = 'Downloading priority skill: {} failed'
LOG.exception(log_msg.format(skill_name))
continue
loader = self._load_skill(skill.path)
if loader:
self.upload_queue.put(loader)
else:
LOG.error(
'Priority skill {} can\'t be found'.format(skill_name))
self._alive_status = True
def run(self):
"""Load skills and update periodically from disk and internet."""
self._remove_git_locks()
self._connected_event.wait()
if (not self.skill_updater.defaults_installed()
and self.skills_config["auto_update"]):
LOG.info('Not all default skills are installed, '
'performing skill update...')
self.skill_updater.update_skills()
self._load_on_startup()
# Sync backend and skills.
if is_paired() and not self.upload_queue.started:
self._start_settings_update()
# Scan the file folder that contains Skills. If a Skill is updated,
# unload the existing version from memory and reload from the disk.
while not self._stop_event.is_set():
try:
self._unload_removed_skills()
self._reload_modified_skills()
self._load_new_skills()
self._update_skills()
if (is_paired() and self.upload_queue.started
and len(self.upload_queue) > 0):
self.msm.clear_cache()
self.skill_updater.post_manifest()
self.upload_queue.send()
self._watchdog()
sleep(2) # Pause briefly before beginning next scan
except Exception:
LOG.exception('Something really unexpected has occured '
'and the skill manager loop safety harness was '
'hit.')
sleep(30)
def _remove_git_locks(self):
"""If git gets killed from an abrupt shutdown it leaves lock files."""
for i in glob(os.path.join(self.msm.skills_dir, '*/.git/index.lock')):
LOG.warning('Found and removed git lock file: ' + i)
os.remove(i)
def _load_on_startup(self):
"""Handle initial skill load."""
LOG.info('Loading installed skills...')
self._load_new_skills()
LOG.info("Skills all loaded!")
self.bus.emit(Message('mycroft.skills.initialized'))
self._loaded_status = True
def _reload_modified_skills(self):
"""Handle reload of recently changed skill(s)"""
for skill_dir in self._get_skill_directories():
try:
skill_loader = self.skill_loaders.get(skill_dir)
if skill_loader is not None and skill_loader.reload_needed():
# If reload succeed add settingsmeta to upload queue
if skill_loader.reload():
self.upload_queue.put(skill_loader)
except Exception:
LOG.exception('Unhandled exception occured while '
'reloading {}'.format(skill_dir))
def _load_new_skills(self):
"""Handle load of skills installed since startup."""
for skill_dir in self._get_skill_directories():
if skill_dir not in self.skill_loaders:
loader = self._load_skill(skill_dir)
if loader:
self.upload_queue.put(loader)
def _load_skill(self, skill_directory):
skill_loader = SkillLoader(self.bus, skill_directory)
try:
load_status = skill_loader.load()
except Exception:
LOG.exception('Load of skill {} failed!'.format(skill_directory))
load_status = False
finally:
self.skill_loaders[skill_directory] = skill_loader
return skill_loader if load_status else None
def _get_skill_directories(self):
skill_glob = glob(os.path.join(self.msm.skills_dir, '*/'))
skill_directories = []
for skill_dir in skill_glob:
# TODO: all python packages must have __init__.py! Better way?
# check if folder is a skill (must have __init__.py)
if SKILL_MAIN_MODULE in os.listdir(skill_dir):
skill_directories.append(skill_dir.rstrip('/'))
if skill_dir in self.empty_skill_dirs:
self.empty_skill_dirs.discard(skill_dir)
else:
if skill_dir not in self.empty_skill_dirs:
self.empty_skill_dirs.add(skill_dir)
LOG.debug('Found skills directory with no skill: ' +
skill_dir)
return skill_directories
def _unload_removed_skills(self):
"""Shutdown removed skills."""
skill_dirs = self._get_skill_directories()
# Find loaded skills that don't exist on disk
removed_skills = [
s for s in self.skill_loaders.keys() if s not in skill_dirs
]
for skill_dir in removed_skills:
skill = self.skill_loaders[skill_dir]
LOG.info('removing {}'.format(skill.skill_id))
try:
skill.unload()
except Exception:
LOG.exception('Failed to shutdown skill ' + skill.id)
del self.skill_loaders[skill_dir]
# If skills were removed make sure to update the manifest on the
# mycroft backend.
if removed_skills:
self.skill_updater.post_manifest(reload_skills_manifest=True)
def _update_skills(self):
"""Update skills once an hour if update is enabled"""
do_skill_update = (time() >= self.skill_updater.next_download
and self.skills_config["auto_update"])
if do_skill_update:
self.skill_updater.update_skills()
def is_alive(self, message=None):
"""Respond to is_alive status request."""
return self._alive_status
def is_all_loaded(self, message=None):
""" Respond to all_loaded status request."""
return self._loaded_status
def send_skill_list(self, _):
"""Send list of loaded skills."""
try:
message_data = {}
for skill_dir, skill_loader in self.skill_loaders.items():
message_data[skill_loader.skill_id] = dict(
active=skill_loader.active and skill_loader.loaded,
id=skill_loader.skill_id)
self.bus.emit(Message('mycroft.skills.list', data=message_data))
except Exception:
LOG.exception('Failed to send skill list')
def deactivate_skill(self, message):
"""Deactivate a skill."""
try:
for skill_loader in self.skill_loaders.values():
if message.data['skill'] == skill_loader.skill_id:
skill_loader.deactivate()
except Exception:
LOG.exception('Failed to deactivate ' + message.data['skill'])
def deactivate_except(self, message):
"""Deactivate all skills except the provided."""
try:
skill_to_keep = message.data['skill']
LOG.info('Deactivating all skills except {}'.format(skill_to_keep))
loaded_skill_file_names = [
os.path.basename(skill_dir) for skill_dir in self.skill_loaders
]
if skill_to_keep in loaded_skill_file_names:
for skill in self.skill_loaders.values():
if skill.skill_id != skill_to_keep:
skill.deactivate()
else:
LOG.info('Couldn\'t find skill ' + message.data['skill'])
except Exception:
LOG.exception('An error occurred during skill deactivation!')
def activate_skill(self, message):
"""Activate a deactivated skill."""
try:
for skill_loader in self.skill_loaders.values():
if (message.data['skill'] in ('all', skill_loader.skill_id)
and not skill_loader.active):
skill_loader.activate()
except Exception:
LOG.exception('Couldn\'t activate skill')
def stop(self):
"""Tell the manager to shutdown."""
self._stop_event.set()
self.settings_downloader.stop_downloading()
self.upload_queue.stop()
# Do a clean shutdown of all skills
for skill_loader in self.skill_loaders.values():
if skill_loader.instance is not None:
_shutdown_skill(skill_loader.instance)
def handle_converse_request(self, message):
"""Check if the targeted skill id can handle conversation
If supported, the conversation is invoked.
"""
skill_id = message.data['skill_id']
# loop trough skills list and call converse for skill with skill_id
skill_found = False
for skill_loader in self.skill_loaders.values():
if skill_loader.skill_id == skill_id:
skill_found = True
if not skill_loader.loaded:
error_message = 'converse requested but skill not loaded'
self._emit_converse_error(message, skill_id, error_message)
break
try:
# check the signature of a converse method
# to either pass a message or not
if len(
signature(skill_loader.instance.converse).
parameters) == 1:
result = skill_loader.instance.converse(
message=message)
else:
utterances = message.data['utterances']
lang = message.data['lang']
result = skill_loader.instance.converse(
utterances=utterances, lang=lang)
self._emit_converse_response(result, message, skill_loader)
except Exception:
error_message = 'exception in converse method'
LOG.exception(error_message)
self._emit_converse_error(message, skill_id, error_message)
finally:
break
if not skill_found:
error_message = 'skill id does not exist'
self._emit_converse_error(message, skill_id, error_message)
def _emit_converse_error(self, message, skill_id, error_msg):
"""Emit a message reporting the error back to the intent service."""
reply = message.reply('skill.converse.response',
data=dict(skill_id=skill_id, error=error_msg))
self.bus.emit(reply)
def _emit_converse_response(self, result, message, skill_loader):
reply = message.reply('skill.converse.response',
data=dict(skill_id=skill_loader.skill_id,
result=result))
self.bus.emit(reply)
def test_default_skill_names(self):
"""Test the property representing the list of default skills."""
updater = SkillUpdater(self.message_bus_mock)
self.assertIn('time', updater.default_skill_names)
self.assertIn('weather', updater.default_skill_names)
self.assertIn('test_skill', updater.default_skill_names)