def loadStoreData(self):
installers = dict()
updateSource = self.ConfigManager.getModulesUpdateSource()
req = requests.get(
url=
'https://api.github.com/search/code?q=extension:install+repo:project-alice-powered-by-snips/ProjectAliceModules/',
auth=GithubCloner.getGithubAuth())
results = req.json()
if results:
for module in results['items']:
try:
req = requests.get(
url=f"{module['url'].split('?')[0]}?ref={updateSource}",
headers={
'Accept': 'application/vnd.github.VERSION.raw'
},
auth=GithubCloner.getGithubAuth())
installer = req.json()
if installer:
installers[installer['name']] = installer
except Exception:
continue
actualVersion = Version(constants.VERSION)
return {
moduleName: moduleInfo
for moduleName, moduleInfo in installers.items()
if self.ModuleManager.getModuleInstance(moduleName=moduleName,
silent=True) is None
and actualVersion >= Version(moduleInfo['aliceMinVersion'])
}
def revert(self, skillName: str) -> Response:
"""
reverts the skill to its official state. Removes the "modified" status and runs an update.
:param skillName:
:return:
"""
if skillName not in self.SkillManager.allSkills:
return self.skillNotFound()
self.SkillManager.getSkillInstance(skillName=skillName).modified = False
gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git',
dest=Path(self.Commons.rootDir()) / 'skills' / skillName,
skillName=skillName)
gitCloner.checkoutMaster()
return self.checkUpdate(skillName)
def getModulesUpdateSource(self) -> str:
updateSource = 'master'
if self.getAliceConfigByName('updateChannel') == 'master':
return updateSource
req = requests.get(
'https://api.github.com/repos/project-alice-powered-by-snips/ProjectAliceModules/branches',
auth=GithubCloner.getGithubAuth())
result = req.json()
if result:
userUpdatePref = self.getAliceConfigByName('updateChannel')
versions = list()
for branch in result:
repoVersion = Version(branch['name'])
if not repoVersion.isVersionNumber:
continue
if userUpdatePref == 'alpha' and repoVersion.infos[
'releaseType'] in ('master', 'rc', 'b', 'a'):
versions.append(repoVersion)
elif userUpdatePref == 'beta' and repoVersion.infos[
'releaseType'] in ('master', 'rc', 'b'):
versions.append(repoVersion)
elif userUpdatePref == 'rc' and repoVersion.infos[
'releaseType'] in ('master', 'rc'):
versions.append(repoVersion)
if len(versions) > 0:
versions.sort(reverse=True)
updateSource = versions[0]
return updateSource
def uploadToGithub(self):
try:
skillName = request.form.get('skillName', '')
skillDesc = request.form.get('skillDesc', '')
if not skillName:
raise Exception
skillName = skillName[0].upper() + skillName[1:]
data = {
'name': skillName,
'description': skillDesc,
'has-issues': True,
'has-wiki': False
}
req = requests.post('https://api.github.com/user/repos',
data=json.dumps(data),
auth=GithubCloner.getGithubAuth())
if req.status_code == 201:
return jsonify(success=True)
else:
raise Exception
except Exception as e:
self.logError(f'Failed uploading to github: {e}')
return jsonify(success=False)
def setModified(self, skillName: str) -> Response:
"""
sets the modified status for that skill.
creates a private repository for the user and checks out the fork
:param skillName:
:return:
"""
if skillName not in self.SkillManager.allSkills:
return self.skillNotFound()
if not GithubCloner.hasAuth():
return self.githubMissing()
self.SkillManager.getSkillInstance(skillName=skillName).modified = True
gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git',
dest=Path(self.Commons.rootDir()) / 'skills' / skillName,
skillName=skillName)
if not gitCloner.checkOwnRepoAvailable(skillName=skillName):
gitCloner.createForkForSkill(skillName=skillName)
gitCloner.checkoutOwnFork()
return jsonify(success=True)
def upload(self, skillName: str) -> Response:
"""
upload the skill to the private repository.
Will create a repository if required, add all untracked changes, create a commit and push
:param skillName:
:return:
"""
if skillName not in self.SkillManager.allSkills:
return self.skillNotFound()
self.SkillManager.getSkillInstance(skillName=skillName).modified = True
gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git',
dest=Path(self.Commons.rootDir()) / 'skills' / skillName,
skillName=skillName)
if not gitCloner.isRepo() and not gitCloner.createRepo():
return jsonify(success=False, message="Failed creating repository")
elif not gitCloner.add():
return jsonify(success=False, message="Failed adding to git")
elif not gitCloner.commit(message="pushed via API"):
return jsonify(success=False, message="Failed creating commit")
elif not gitCloner.push():
return jsonify(success=False, message="Failed pushing to git")
else:
return jsonify(success=True)
def getGitStatus(self, skillName: str) -> Response:
"""
returns a list containing the public and private github URL of that skill.
The repository does not have to exist yet!
The current status of the repository is included as well
Currently possible status: True/False
:param skillName:
:return:
"""
if skillName not in self.SkillManager.allSkills:
return self.skillNotFound()
gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git',
dest=Path(self.Commons.rootDir()) / 'skills' / skillName,
skillName=skillName)
return jsonify(success=True,
result={'Public' : {'name' : 'Public',
'url' : gitCloner.getRemote(origin=True, noToken=True),
'status': gitCloner.checkRemote(origin=True)},
'Private': {'name' : 'Private',
'url' : gitCloner.getRemote(AliceSK=True, noToken=True),
'status': gitCloner.checkRemote(AliceSK=True)}})
def _installModules(self, modules: list) -> dict:
root = Path(commons.rootDir(), 'system/moduleInstallTickets')
availableModules = self.ConfigManager.modulesConfigurations
modulesToBoot = dict()
self.MqttManager.broadcast(topic='hermes/leds/systemUpdate',
payload={'sticky': True})
for file in modules:
moduleName = Path(file).with_suffix('')
self._logger.info('[{}] Now taking care of module {}'.format(
self.name, moduleName.stem))
res = root / file
try:
updating = False
installFile = json.loads(res.read_text())
moduleName = installFile['name']
path = Path(installFile['author'], moduleName)
if not moduleName:
self._logger.error(
'[{}] Module name to install not found, aborting to avoid casualties!'
.format(self.name))
continue
directory = Path(commons.rootDir()) / 'modules' / moduleName
if moduleName in availableModules:
localVersionDirExists = directory.is_dir()
localVersionAttributeExists: bool = 'version' in availableModules[
moduleName]
localVersionIsLatest: bool = \
localVersionDirExists and \
localVersionAttributeExists and \
float(availableModules[moduleName]['version']) >= float(installFile['version'])
if localVersionIsLatest:
self._logger.warning(
'[{}] Module "{}" is already installed, skipping'.
format(self.name, moduleName))
subprocess.run(['sudo', 'rm', res])
continue
else:
self._logger.warning(
'[{}] Module "{}" needs updating'.format(
self.name, moduleName))
updating = True
if moduleName in self._modules:
try:
self._modules[moduleName]['instance'].onStop()
except Exception as e:
self._logger.error(
'[{}] Error stopping "{}" for update: {}'.format(
self.name, moduleName, e))
gitCloner = GithubCloner(baseUrl=self.GITHUB_API_BASE_URL,
path=path,
dest=directory)
if gitCloner.clone():
self._logger.info(
'[{}] Module successfully downloaded'.format(
self.name))
try:
pipReq = installFile.get('pipRequirements', None)
sysReq = installFile.get('systemRequirements', None)
scriptReq = installFile.get('script', None)
if pipReq:
for requirement in pipReq:
subprocess.run([
'./venv/bin/pip3', 'install', requirement
])
if sysReq:
for requirement in sysReq:
subprocess.run([
'sudo', 'apt-get', 'install', '-y',
requirement
])
if scriptReq:
subprocess.run([
'sudo', 'chmod', '+x',
str(directory / scriptReq)
])
subprocess.run(
['sudo', str(directory / scriptReq)])
node = {
'active': True,
'version': installFile['version'],
'author': installFile['author'],
'conditions': installFile['conditions']
}
self.ConfigManager.addModuleToAliceConfig(
installFile['name'], node)
subprocess.run(['mv', res, directory])
modulesToBoot[moduleName] = {'update': updating}
except Exception as e:
self._logger.error(
'[{}] Failed installing module "{}": {}'.format(
self.name, moduleName, e))
res.unlink()
else:
self._logger.error('[{}] Failed cloning module'.format(
self.name))
res.unlink()
except Exception as e:
self._logger.error(
'[{}] Failed installing module "{}": {}'.format(
self.name, moduleName, e))
res.unlink()
self.MqttManager.broadcast(topic='hermes/leds/clear')
return modulesToBoot
def uploadSkillToGithub(self, skillName: str, skillDesc: str) -> bool:
try:
self.logInfo(f'Uploading {skillName} to Github')
skillName = skillName[0].upper() + skillName[1:]
localDirectory = Path('/home', getpass.getuser(), f'ProjectAlice/skills/{skillName}')
if not localDirectory.exists():
raise Exception("Local skill doesn't exist")
data = {
'name' : f'skill_{skillName}',
'description': skillDesc,
'has-issues' : True,
'has-wiki' : False
}
req = requests.post('https://api.github.com/user/repos', data=json.dumps(data), auth=GithubCloner.getGithubAuth())
if req.status_code != 201:
raise Exception("Couldn't create the repository on Github")
self.Commons.runSystemCommand(['rm', '-rf', f'{str(localDirectory)}/.git'])
self.Commons.runSystemCommand(['git', '-C', str(localDirectory), 'init'])
self.Commons.runSystemCommand(['git', 'config', '--global', 'user.email', '*****@*****.**'])
self.Commons.runSystemCommand(['git', 'config', '--global', 'user.name', '*****@*****.**'])
remote = f'https://{self.ConfigManager.getAliceConfigByName("githubUsername")}:{self.ConfigManager.getAliceConfigByName("githubToken")}@github.com/{self.ConfigManager.getAliceConfigByName("githubUsername")}/skill_{skillName}.git'
self.Commons.runSystemCommand(['git', '-C', str(localDirectory), 'remote', 'add', 'origin', remote])
self.Commons.runSystemCommand(['git', '-C', str(localDirectory), 'add', '--all'])
self.Commons.runSystemCommand(['git', '-C', str(localDirectory), 'commit', '-m', '"Initial upload"'])
self.Commons.runSystemCommand(['git', '-C', str(localDirectory), 'push', '--set-upstream', 'origin', 'master'])
url = f'https://github.com/{self.ConfigManager.getAliceConfigByName("githubUsername")}/skill_{skillName}.git'
self.logInfo(f'Skill uploaded! You can find it on {url}')
return True
except Exception as e:
self.logWarning(f'Something went wrong uploading skill to Github: {e}')
return False
def _installSkills(self, skills: list) -> dict:
root = Path(self.Commons.rootDir(), constants.SKILL_INSTALL_TICKET_PATH)
availableSkills = self.ConfigManager.skillsConfigurations
skillsToBoot = dict()
self.MqttManager.mqttBroadcast(topic='hermes/leds/systemUpdate', payload={'sticky': True})
for file in skills:
skillName = Path(file).with_suffix('')
self.logInfo(f'Now taking care of skill {skillName.stem}')
res = root / file
try:
updating = False
installFile = json.loads(res.read_text())
skillName = installFile['name']
path = Path(installFile['author'], skillName)
if not skillName:
self.logError('Skill name to install not found, aborting to avoid casualties!')
continue
directory = Path(self.Commons.rootDir()) / 'skills' / skillName
conditions = {
'aliceMinVersion': installFile['aliceMinVersion'],
**installFile.get('conditions', dict())
}
self.checkSkillConditions(skillName, conditions, availableSkills)
if skillName in availableSkills:
installedVersion = Version.fromString(availableSkills[skillName]['version'])
remoteVersion = Version.fromString(installFile['version'])
localVersionIsLatest: bool = \
directory.is_dir() and \
'version' in availableSkills[skillName] and \
installedVersion >= remoteVersion
if localVersionIsLatest:
self.logWarning(f'Skill "{skillName}" is already installed, skipping')
self.Commons.runRootSystemCommand(['rm', res])
continue
else:
self.logWarning(f'Skill "{skillName}" needs updating')
updating = True
if skillName in self._activeSkills:
try:
self._activeSkills[skillName].onStop()
except Exception as e:
self.logError(f'Error stopping "{skillName}" for update: {e}')
raise
gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git', path=path, dest=directory)
try:
gitCloner.clone(skillName=skillName)
self.logInfo('Skill successfully downloaded')
self._installSkill(res)
skillsToBoot[skillName] = {
'update': updating
}
except (GithubTokenFailed, GithubRateLimit):
self.logError('Failed cloning skill')
raise
except GithubNotFound:
if self.ConfigManager.getAliceConfigByName('devMode'):
if not Path(f'{self.Commons.rootDir}/skills/{skillName}').exists() or not \
Path(f'{self.Commons.rootDir}/skills/{skillName}/{skillName.py}').exists() or not \
Path(f'{self.Commons.rootDir}/skills/{skillName}/dialogTemplate').exists() or not \
Path(f'{self.Commons.rootDir}/skills/{skillName}/talks').exists():
self.logWarning(f'Skill "{skillName}" cannot be installed in dev mode due to missing base files')
else:
self._installSkill(res)
skillsToBoot[skillName] = {
'update': updating
}
continue
else:
self.logWarning(f'Skill "{skillName}" is not available on Github, cannot install')
raise
except SkillNotConditionCompliant as e:
self.logInfo(f'Skill "{skillName}" does not comply to "{e.condition}" condition, required "{e.conditionValue}"')
if res.exists():
res.unlink()
self.broadcast(
method=constants.EVENT_SKILL_INSTALL_FAILED,
exceptions=self._name,
skill=skillName
)
except Exception as e:
self.logError(f'Failed installing skill "{skillName}": {e}')
if res.exists():
res.unlink()
self.broadcast(
method=constants.EVENT_SKILL_INSTALL_FAILED,
exceptions=self.name,
skill=skillName
)
raise
return skillsToBoot