def create_merged_channel(base, delta, precedence="longest"):
if is_compatible_dict(base, delta):
if precedence == "base":
return merge_dict(base,
delta,
merge_lists=True,
new_only=True,
no_dupes=True,
skip_empty=True)
elif precedence == "delta":
return merge_dict(base,
delta,
merge_lists=True,
no_dupes=True,
skip_empty=True)
elif precedence == "longest":
for key in [k for k in delta if k in base]:
if isinstance(base[key], str) and isinstance(
delta[key], str):
if len(base[key]) > len(delta[key]):
delta[key] = base[key]
if delta.get("skill_id") and base.get("skill_id") \
and base.get("skill_id") not in delta.get("skill_id"):
delta["skill_id"] = delta["skill_id"] + "/" + base[
"skill_id"]
return merge_dict(base,
delta,
merge_lists=True,
no_dupes=True,
skip_empty=True)
else:
return None
def test_merge_dict(self):
self.assertEqual(
merge_dict(deepcopy(self.base_dict), deepcopy(self.delta_dict)), {
"one": 1,
"two": 2,
"three": 30,
"four": [4, 5, 6, "foo"],
"five": None
})
def get_config_object() -> Union[JsonStorageXDG, JsonConfigXDG]:
"""Locates or creates the OSM config file, and ensures that all required
values are present, inserting defaults as needed
Returns:
json_database.JsonConfigXDG: the OSM config object
"""
config = _existing_osm_config() or \
JsonConfigXDG("OVOS-SkillsManager", subfolder="OpenVoiceOS")
default_appstores = {
"local": {
"active": True,
"url": safe_get_skills_folder(),
"parse_github": False,
"priority": 1},
"ovos": {
"active": True,
"url": "https://github.com/OpenVoiceOS/OVOS-appstore",
"parse_github": False,
"priority": 2},
"mycroft_marketplace": {
"active": False,
"url": "https://market.mycroft.ai/",
"parse_github": False,
"priority": 5},
"pling": {
"active": False,
"url": "https://apps.plasma-bigscreen.org/",
"parse_github": False,
"priority": 10},
"neon": {
"active": False,
"url": "https://github.com/NeonGeckoCom/neon-skills-submodules/",
"parse_github": False,
"auth_token": None,
"priority": 50},
"andlo_skill_list": {
"active": False,
"url": "https://andlo.gitbook.io/mycroft-skills-list/",
"parse_github": False,
"priority": 100}
}
if "appstores" not in config:
# NOTE, below should match Appstore.appstore_id
config["appstores"] = default_appstores
config["appstores"] = merge_dict(config["appstores"],
default_appstores,
new_only=True,
no_dupes=True)
if "version" not in config:
# This stuff can really only happen on first run
config["version"] = CURRENT_OSM_VERSION
config["last_upgrade"] = CURRENT_OSM_VERSION
config.store()
return config
def __init__(self):
self.config = JsonStorageXDG("OVOS-SkillsManager")
default_config = {
"local": {
"active": True,
"url": get_skills_folder(),
"parse_github": False,
"priority": 1
},
"ovos": {
"active": True,
"url": "https://github.com/OpenVoiceOS/OVOS-appstore",
"parse_github": False,
"priority": 2
},
"mycroft_marketplace": {
"active": False,
"url": "https://market.mycroft.ai/",
"parse_github": False,
"priority": 5
},
"pling": {
"active": False,
"url": "https://apps.plasma-bigscreen.org/",
"parse_github": False,
"priority": 10
},
"neon": {
"active": False,
"url":
"https://github.com/NeonGeckoCom/neon-skills-submodules/",
"parse_github": False,
"auth_token": None,
"priority": 50
},
"andlo_skill_list": {
"active": False,
"url": "https://andlo.gitbook.io/mycroft-skills-list/",
"parse_github": False,
"priority": 100
}
}
if "appstores" not in self.config:
# NOTE, below should match Appstore.appstore_id
self.config["appstores"] = default_config
self.save_config()
self.config["appstores"] = merge_dict(self.config["appstores"],
default_config,
new_only=True,
no_dupes=True)
self._boostrap_tracker = {}
self.save_config()
self._threads = []
def test_merge_dict_lists(self):
self.assertEqual(
merge_dict(deepcopy(self.base_dict),
deepcopy(self.delta_dict),
merge_lists=True,
no_dupes=False), {
"one": 1,
"two": 2,
"three": 30,
"four": ["foo", "bar", "baz", 4, 5, 6, "foo"],
"five": None
})
def test_merge_dict_no_dupes(self):
self.assertEqual(
merge_dict(deepcopy(self.base_dict),
deepcopy(self.delta_dict),
merge_lists=True,
skip_empty=True,
no_dupes=True), {
"one": 1,
"two": 2,
"three": 30,
"four": ["foo", "bar", "baz", 4, 5, 6],
"five": 50
})
def play_video_event(self, message):
video_data = message.data["modelData"]
if video_data["skill_id"] == self.skill_id:
# ensure all data fields present
video_data = merge_dict(video_data, {
"match_confidence": 100,
"media_type": self.media_type,
"playback": self.playback_type,
"skill_icon": self.skill_icon,
"skill_logo": self.skill_logo,
"bg_image": video_data.get("logo") or self.default_bg,
"image": video_data.get("logo") or self.default_image,
"author": self.name
})
self.bus.emit(Message("better_cps.play", {
"tracks": [video_data]
}))
def handle_utterance(self, event):
LOG.info("Utterance: " + str(event['utterances']))
context = event["context"] # from audio transformers
context.update({'client_name': 'mycroft_listener',
'source': 'audio',
'ident': event.pop('ident', str(round(time()))),
'raw_audio': event.pop('raw_audio', None),
'destination': ["skills"],
"timing": event.pop("timing", {}),
'username': self.user_config["user"]["username"] or
"local",
'user_profiles': [self.user_config.content]
})
if "data" in event:
data = event.pop("data")
context = merge_dict(context, data)
self._emit_utterance_to_skills(Message('recognizer_loop:utterance',
event, context))
def from_json(data: Union[str, dict], parse_github: bool = True):
if isinstance(data, str):
if data.startswith("http"):
url = data
if "github" in url:
data = {"url": url}
# repo is parsed in github info step below,
# branch detected when parsing data dict
else:
try:
res = requests.get(url).text
data = json.loads(res)
except JSONDecodeError:
raise GithubFileNotFound
elif isfile(data):
with open(data) as f:
data = json.load(f)
else:
data = json.loads(data)
if not isinstance(data, dict):
# TODO new exception
raise ValueError("unrecognized format")
# augment with github info
if parse_github:
url = data.get("url", "")
if "github" in url:
try:
github_data = get_skill_data(url, data.get("branch"))
data = merge_dict(github_data,
data,
merge_lists=True,
skip_empty=True,
no_dupes=True)
parse_python_dependencies(
data["requirements"].get("python"),
requests.headers.get("Authorization"))
except GithubInvalidUrl as e:
raise e
return SkillEntry(data)
def skill_entry_from_url(url: str):
"""
Builds a minimal SkillEntry object from the passed GitHub URL to use for skill installation
:param url: URL of skill to install
:return: SkillEntry object with url, branch, requirements, and authorname populated
"""
from ovos_skills_manager.exceptions import GithubInvalidBranch, GithubFileNotFound
from ovos_skills_manager.github import get_branch_from_github_url, normalize_github_url, get_requirements_json,\
get_skill_json
from ovos_skills_manager.skill_entry import SkillEntry
try:
branch = get_branch_from_github_url(url)
except GithubInvalidBranch:
branch = None
url = normalize_github_url(url)
requirements = get_requirements_json(url, branch)
requirements["system"] = {
k: v.split()
for k, v in requirements.get("system", {}).items()
}
try:
json = get_skill_json(url, branch)
requirements = merge_dict(requirements,
json.get("requirements", {}),
merge_lists=True,
skip_empty=True,
no_dupes=True)
except GithubFileNotFound:
json = {"authorname": author_repo_from_github_url(url)[0]}
return SkillEntry.from_json(
{
"url": url,
"branch": branch,
"requirements": requirements,
"authorname": json.get("authorname")
}, False)
def CPS_search(self, phrase, media_type):
base_score = self.match_media_type(phrase, media_type)
# penalty for generic searches, they tend to overmatch
if media_type == CPSMatchType.GENERIC:
base_score -= 20
# match titles and sort
# then match all the metadata up to self.settings["search_depth"]
videos = sorted(self.videos,
key=lambda k: fuzzy_match(k["title"], phrase),
reverse=True)
cps_results = []
for idx, video in enumerate(videos[:self.settings["search_depth"]]):
score = base_score + fuzzy_match(video["title"], phrase) * 30
if self.settings["match_tags"]:
score += self.match_tags(video, phrase, media_type)
if self.settings["match_title"]:
score += self.match_title(video, phrase, media_type)
if self.settings["match_description"]:
score += self.match_description(video, phrase, media_type)
if score < self.settings["min_score"]:
continue
cps_results.append(merge_dict(video, {
"match_confidence": min(100, score),
"media_type": self.media_type,
"playback": self.playback_type,
"skill_icon": self.skill_icon,
"skill_logo": self.skill_logo,
"bg_image": video.get("logo") or self.default_bg,
"image": video.get("logo") or self.default_image,
"author": self.name
}))
cps_results = sorted(cps_results,
key=lambda k: k["match_confidence"],
reverse=True)
return cps_results
def get_local_skills(parse_github=False, skiplist=None):
skills = get_skills_folder()
skiplist = skiplist or []
folders = listdir(skills)
for fold in folders:
path = join(skills, fold)
if not isdir(path) or fold in skiplist:
continue
skill = {
"appstore": "InstalledSkills",
"appstore_url": skills,
"skill_id": fold,
"foldername": fold,
"requirements": {
"python": [],
"system": [],
"skill": []
}
}
# if installed by msm/osm will obey this convention
if "." in fold:
try:
repo, author = fold.split(".")
skill["skillname"] = repo
skill["authorname"] = author
skill["url"] = f'https://github.com/{author}/{repo}'
except: # TODO replace with some clever check ?
pass
# parse git info
gitinfo = join(path, ".git/config")
if isfile(gitinfo):
with open(gitinfo) as f:
for l in f.readlines():
if l.strip().startswith("url ="):
skill["url"] = l.split("url =")[-1].strip()
skill["authorname"], skill["skillname"] = \
author_repo_from_github_url(skill["url"])
if l.strip().startswith("[branch "):
skill["branch"] = l.split("branch")[-1]\
.replace('"', "").strip()
for rtdir, foldrs, files in walk(join(skills, fold)):
for f in files:
if f in GITHUB_JSON_FILES:
with open(join(rtdir, f)) as fi:
skill_meta = json.load(fi)
skill = merge_dict(skill, skill_meta, merge_lists=True)
elif f in GITHUB_README_FILES:
with open(join(rtdir, f)) as fi:
readme = readme_to_json(fi.read())
skill = merge_dict(skill,
readme,
new_only=True,
merge_lists=True)
elif f in GITHUB_DESKTOP_FILES:
skill['desktopFile'] = True
elif f in GITHUB_ICON_FILES:
skill["icon"] = join(rtdir, f)
elif f in GITHUB_LICENSE_FILES:
with open(join(rtdir, f)) as fi:
lic = fi.read()
skill["license"] = parse_license_type(lic)
elif f in GITHUB_LOGO_FILES:
skill["logo"] = join(rtdir, f)
elif f in GITHUB_MANIFEST_FILES:
with open(join(rtdir, f)) as fi:
manifest = validate_manifest(fi.read())
skill["requirements"]["python"] += manifest.get(
"python") or []
skill["requirements"]["system"] += manifest.get(
"system") or []
skill["requirements"]["skill"] += manifest.get(
"skill") or []
elif f in GITHUB_REQUIREMENTS_FILES:
with open(join(rtdir, f)) as fi:
reqs = [r for r in fi.read().split("\n") if r.strip()]
skill["requirements"]["python"] += reqs
elif f in GITHUB_SKILL_REQUIREMENTS_FILES:
with open(join(rtdir, f)) as fi:
reqs = [r for r in fi.read().split("\n") if r.strip()]
skill["requirements"]["skill"] += reqs
yield SkillEntry.from_json(skill, parse_github=parse_github)
def get_skill_from_api(url, branch=None, strict=False):
data = {}
# extract branch from .json, should branch take precedence?
# i think so because user explicitly requested it
branch = get_branch_from_skill_json_github_api(url, branch)
try:
api_data = get_repo_data_from_github_api(url, branch)
data["branch"] = branch = api_data['default_branch']
data["short_description"] = api_data['description']
data["license"] = api_data["license"]["key"]
data["foldername"] = api_data["name"]
data["last_updated"] = api_data['updated_at']
data["url"] = api_data["html_url"]
data["authorname"] = api_data["owner"]["login"]
except GithubAPIException as e:
LOG.error("Failed to retrieve repo data from github api")
raise
try:
releases = get_repo_releases_from_github_api(url, branch)
if branch:
for r in releases:
if r["name"] == branch or r["commit"]["sha"] == branch:
data["version"] = r["name"]
#data["download_url"] = r["tarball_url"]
break
else:
data["version"] = releases[0]["name"]
#data["download_url"] = releases[0]["tarball_url"]
except GithubAPIException as e:
LOG.error("Failed to retrieve releases data from github api")
if strict:
raise GithubAPIReleasesNotFound
# augment with readme data
try:
data = merge_dict(data, get_readme_json_from_api(url, branch),
merge_lists=True, skip_empty=True, no_dupes=True)
except GithubAPIReadmeNotFound:
pass
data["requirements"] = get_requirements_json_from_github_api(url, branch)
# find logo
try:
data["logo"] = get_logo_url_from_github_api(url, branch)
except GithubAPIFileNotFound as e:
pass
# find icon
try:
data["icon"] = icon = get_icon_url_from_github_api(url, branch)
except GithubAPIFileNotFound:
icon = None
# augment with android data
try:
data["android"] = get_android_json_from_github_api(url, branch)
except GithubAPIFileNotFound:
# best guess or throw exception?
author, repo = author_repo_from_github_url(url)
data["android"] = {
'android_icon': icon,
'android_name': skill_name_from_github_url(url),
'android_handler': '{repo}.{author}.home'.format(repo=repo,
author=author.lower())}
# augment with desktop data
try:
data["desktop"] = get_desktop_json_from_github_api(url, branch)
data["desktopFile"] = True
except GithubFileNotFound:
data["desktopFile"] = False
# augment tags
if "tags" not in data:
data["tags"] = []
if is_viral(data["license"]):
data["tags"].append("viral-license")
elif is_permissive(data["license"]):
data["tags"].append("permissive-license")
elif "unknown" in data["license"]:
data["tags"].append("no-license")
# augment with json data
# this should take precedence over everything else
try:
data = merge_dict(data, get_skill_json_from_github_api(url, branch),
merge_lists=True, skip_empty=True, no_dupes=True)
except GithubFileNotFound:
pass
return data
def get_skill_from_github_url(url: str,
branch: Optional[str] = None) -> str:
"""
Get skill icon file URL for the specified repository
@param url: Repository URL to query
@param branch: Optional branch spec, otherwise default branch will be used
@return: URL of skill icon
"""
# cache_repo_requests(url) # speed up requests TODO avoid rate limit
author, repo = author_repo_from_github_url(url)
data = {
"authorname": author,
"foldername": repo,
"branch": branch,
"license": "unknown",
"tags": []
}
if not branch:
try:
# check if branch is in the url itself
data["branch"] = branch = get_branch_from_github_url(url)
except GithubInvalidBranch:
# let's assume latest release
try:
release = get_repo_releases_from_github_url(url)[0]
data["branch"] = data["version"] = branch = release["name"]
#data["download_url"] = release["tarball_url"]
except GithubInvalidBranch:
pass # unknown branch...
url = normalize_github_url(url)
data["url"] = url
data["skillname"] = skill_name_from_github_url(url)
data["requirements"] = get_requirements_json_from_github_url(url, branch)
# extract branch from .json, should branch take precedence?
# i think so because user explicitly requested it
branch = get_branch_from_skill_json_github_url(url, branch)
# augment with readme data
try:
readme_data = get_readme_json_from_github_url(url, branch)
data = merge_dict(data, readme_data,
merge_lists=True, skip_empty=True, no_dupes=True)
except GithubReadmeNotFound:
pass
if branch: # final, all sources checked by priority order
data["branch"] = branch
#data["download_url"] = GithubUrls.DOWNLOAD.format(author=author,
# repo=repo,
# branch=branch)
try:
data["license"] = get_license_type_from_github_url(url, branch)
except GithubLicenseNotFound:
pass
try:
data["icon"] = get_icon_url_from_github_url(url, branch)
except GithubFileNotFound:
pass
# parse bigscreen flags
if data["requirements"].get("system"):
data['systemDeps'] = True
else:
data['systemDeps'] = False
# find logo
try:
data["logo"] = get_logo_url_from_github_url(url, branch)
except GithubFileNotFound as e:
pass
# augment with android data
data["android"] = get_android_json_from_github_url(url, branch)
# augment with desktop data
try:
data["desktop"] = get_desktop_json_from_github_url(url, branch)
data["desktopFile"] = True
except GithubFileNotFound:
data["desktopFile"] = False
# augment tags
if "tags" not in data:
data["tags"] = []
if is_viral(data["license"]):
data["tags"].append("viral-license")
elif is_permissive(data["license"]):
data["tags"].append("permissive-license")
elif "unknown" in data["license"]:
data["tags"].append("no-license")
# augment with json data
# this should take precedence over everything else
try:
data = merge_dict(data, get_skill_json_from_github_url(url, branch),
merge_lists=True, skip_empty=True, no_dupes=True)
except GithubFileNotFound:
pass
return data
def get_skill_data_from_directory(skill_dir: str):
"""
Parse the specified skill directory and return a dict representation of a
SkillEntry.
@param skill_dir: path to skill directory
@return: dict parsed skill data
"""
skills, fold = skill_dir.rsplit('/', 1)
skill_data = {
"appstore": "InstalledSkills",
"appstore_url": skills,
"skill_id": fold,
"requirements": {
"python": [],
"system": {},
"skill": []
}
}
# if installed by msm/osm will obey this convention
if "." in fold:
try:
repo, author = fold.split(".")
skill_data["skillname"] = repo
skill_data["authorname"] = author
skill_data["url"] = f'https://github.com/{author}/{repo}'
except: # TODO replace with some clever check ?
pass
# parse git info
gitinfo = join(skill_dir, ".git/config")
if isfile(gitinfo):
with open(gitinfo) as f:
for l in f.readlines():
if l.strip().startswith("url ="):
skill_data["url"] = l.split("url =")[-1].strip()
skill_data["authorname"], skill_data["skillname"] = \
author_repo_from_github_url(skill_data["url"])
if l.strip().startswith("[branch "):
skill_data["branch"] = l.split("branch")[-1] \
.replace('"', "").strip()
# parse skill files
for root_dir, _, files in walk(skill_dir):
for f in files:
if f in GITHUB_JSON_FILES: # skill.json
with open(join(root_dir, f)) as fi:
skill_meta = json.load(fi)
skill_data = merge_dict(skill_data,
skill_meta,
merge_lists=True)
elif f in GITHUB_README_FILES:
with open(join(root_dir, f)) as fi:
readme = readme_to_json(fi.read())
skill_data = merge_dict(skill_data,
readme,
new_only=True,
merge_lists=True)
elif f in GITHUB_DESKTOP_FILES:
skill_data['desktopFile'] = True
elif f in GITHUB_ICON_FILES:
skill_data["icon"] = join(root_dir, f)
elif f in GITHUB_LICENSE_FILES:
with open(join(root_dir, f)) as fi:
lic = fi.read()
skill_data["license"] = parse_license_type(lic)
elif f in GITHUB_LOGO_FILES:
skill_data["logo"] = join(root_dir, f)
elif f in GITHUB_MANIFEST_FILES:
with open(join(root_dir, f)) as fi:
manifest = validate_manifest(fi.read()).get(
"dependencies", {})
skill_data["requirements"]["python"] += \
manifest.get("python") or []
skill_data["requirements"]["system"] = \
merge_dict(skill_data["requirements"]["system"],
manifest.get("system") or {}, merge_lists=True)
skill_data["requirements"]["skill"] += \
manifest.get("skill") or []
elif f in GITHUB_REQUIREMENTS_FILES:
with open(join(root_dir, f)) as fi:
reqs = [r for r in fi.read().split("\n") if r.strip()]
skill_data["requirements"]["python"] += reqs
elif f in GITHUB_SKILL_REQUIREMENTS_FILES:
with open(join(root_dir, f)) as fi:
reqs = [r for r in fi.read().split("\n") if r.strip()]
skill_data["requirements"]["skill"] += reqs
# de-dupe requirements
skill_data["requirements"]["python"] = \
list(set(skill_data["requirements"]["python"]))
skill_data["requirements"]["skill"] = \
list(set(skill_data["requirements"]["skill"]))
skill_data['foldername'] = fold # Override what the config specifies
skill_data['authorname'] = skill_data.get('authorname') or "local"
return skill_data
def merge(self, conf):
merge_dict(self, conf)
return self