def test_equivalent_branch_specs(self):
from ovos_skills_manager.skill_entry import SkillEntry
tree_spec = SkillEntry.from_github_url(
"https://github.com/OpenVoiceOS/tskill-osm_parsing/tree/dev")
at_spec = SkillEntry.from_github_url(
"https://github.com/OpenVoiceOS/tskill-osm_parsing@dev")
self.assertEqual(tree_spec, at_spec)
def test_skill_entry_properties_invalid_entry(self):
from ovos_skills_manager.skill_entry import SkillEntry
entry = SkillEntry({})
self.assertFalse(entry.uuid)
self.assertIsInstance(entry.json, dict)
self.assertIsInstance(repr(entry), str)
self.assertEqual(entry, SkillEntry({}))
def test_equivalent_default(self):
from ovos_skills_manager.skill_entry import SkillEntry
implicit = SkillEntry.from_github_url(
"https://github.com/OpenVoiceOS/tskill-osm_parsing")
explicit = SkillEntry.from_github_url(
"https://github.com/OpenVoiceOS/tskill-osm_parsing/archive/v0.2.1.zip"
)
self.assertEqual(implicit, explicit)
def test_skill_entry_uuid(self):
from ovos_skills_manager.skill_entry import SkillEntry
entry = SkillEntry.from_github_url(
"https://github.com/OpenVoiceOS/tskill-osm_parsing@dev")
self.assertEqual(entry.uuid, "tskill-osm_parsing.openvoiceos")
entry = SkillEntry.from_github_url(
"https://github.com/NeonDaniel/Tskill-osm_parsing@dev")
self.assertEqual(entry.uuid, "tskill-osm_parsing.neondaniel")
def test_skill_entry_from_directory(self):
from ovos_skills_manager.skill_entry import SkillEntry
test_dir = os.path.join(os.path.dirname(__file__), "skill_dirs")
complete_dir = os.path.join(test_dir, "tskill-osm_parsing-complete")
minimal_dir = os.path.join(test_dir, "tskill-osm_parsing-minimal")
no_git_dir = os.path.join(test_dir, "tskill-osm_parsing-no_git")
no_json_dir = os.path.join(test_dir, "tskill-osm_parsing-no_json")
with self.assertRaises(ValueError):
SkillEntry.from_directory(__file__)
with self.assertRaises(ValueError):
SkillEntry.from_directory("/invalid")
for test_skill_dir in (complete_dir, minimal_dir, no_git_dir,
no_json_dir):
skill = SkillEntry.from_directory(test_skill_dir)
self.assertIsInstance(skill, SkillEntry)
self.assertIsInstance(skill.uuid, str)
self.assertEqual(skill.appstore, "InstalledSkills")
self.assertIsInstance(skill.skill_name, str)
self.assertEqual(skill.skill_folder, basename(test_skill_dir))
self.assertIsInstance(skill.skill_examples, list)
self.assertIsInstance(skill.requirements, dict)
self.assertIsInstance(skill.url, str)
if test_skill_dir == complete_dir:
self.assertTrue(skill.url.startswith("https://github.com"))
self.assertEqual(
set(skill.requirements["python"]), {
"text_requirements", "manifest_requirement",
"json-requirements"
})
self.assertIsInstance(skill.requirements["system"], dict)
self.assertEqual(set(skill.requirements["skill"]),
{"manifest-skill", "json-skill"})
elif test_skill_dir == minimal_dir:
self.assertEqual(set(skill.requirements["python"]),
{"text_requirements"})
self.assertIsInstance(skill.requirements["system"], dict)
elif test_skill_dir == no_git_dir:
self.assertEqual(
set(skill.requirements["python"]), {
"text_requirements", "manifest_requirement",
"json-requirements"
})
self.assertIsInstance(skill.requirements["system"], dict)
self.assertEqual(set(skill.requirements["skill"]),
{"manifest-skill", "json-skill"})
elif test_skill_dir == no_json_dir:
self.assertEqual(set(skill.requirements["python"]),
{"text_requirements", "manifest_requirement"})
self.assertIsInstance(skill.requirements["system"], dict)
self.assertEqual(set(skill.requirements["skill"]),
{"manifest-skill"})
def get_mycroft_marketplace_skills(branch:str=None, parse_github:bool=False,
skiplist=None):
skiplist = skiplist or []
data = get_marketplace_json(branch)
for _, skill in data.items():
url = skill["repo"]
branch = skill["tree"]
if normalize_github_url(url) in skiplist:
continue
data = {
"skillname": skill["display_name"],
"foldername": skill["name"],
"url": url,
"branch": branch,
"description": skill["description"],
"authorname": skill["github_username"],
"examples": skill["examples"],
"category": skill["categories"][0],
"tags": list(set(skill["tags"] + skill["categories"])),
"platforms": skill["platforms"],
"short_description": skill["short_desc"]
}
if parse_github:
try:
validate_branch(branch, url)
except GithubInvalidBranch:
LOG.error("branch : {branch} not available for skill: {skill}".format(branch=branch, skill=url))
continue
if not is_valid_github_skill_url(url, branch):
LOG.error("{skill} does not seem like a valid skill".format(skill=url))
continue
yield SkillEntry.from_json(data, parse_github=parse_github)
def test_requirements_null_json(self):
from ovos_skills_manager.skill_entry import SkillEntry
entry = SkillEntry.from_github_url(
"https://github.com/OpenVoiceOS/tskill-osm_parsing@commented_requirements"
)
requirements = entry.json.pop("requirements")
self.assertEqual(requirements, entry.requirements)
entry = SkillEntry.from_github_url(
"https://github.com/OpenVoiceOS/[email protected]")
requirements = entry.json.pop("requirements")
self.assertEqual(requirements, entry.requirements)
entry = SkillEntry.from_github_url(
"https://github.com/OpenVoiceOS/tskill-osm_parsing")
requirements = entry.json.pop("requirements")
self.assertEqual(requirements, entry.requirements)
def get_neon_skills(parse_github=False, skiplist=None):
skiplist = skiplist or []
skills_url = "https://raw.githubusercontent.com/NeonGeckoCom/neon-skills-submodules/master/skill-metadata.json"
skill_json = requests.get(skills_url).json()
for skill in skill_json.values():
if skill["url"] in skiplist:
continue
skill["appstore"] = "Neon"
skill["appstore_url"] = skills_url
yield SkillEntry.from_json(skill, parse_github=parse_github)
def test_url(self):
branch_from_json = "https://github.com/JarbasSkills/skill-bandcamp"
branch_from_tree = "https://github.com/JarbasSkills/skill-ddg/tree/v0.1.0"
commit_from_blob = "https://github.com/OpenVoiceOS/OVOS-skills-store/blob" \
"/f4ab4ea00e47955798c9906c8c03807391bc20f0/skill-icanhazdadjokes.json"
branch_from_git = "https://github.com/NeonGeckoCom/caffeinewiz.neon@dev"
# should get branch defined in https://github.com/JarbasSkills/skill-ddg/blob/master/res/desktop/skill.json
entry = SkillEntry.from_github_url(branch_from_json)
self.assertEqual(entry.branch, "v0.3.1")
# should match commit pinned in url
entry = SkillEntry.from_github_url(commit_from_blob)
self.assertEqual(entry.branch,
"f4ab4ea00e47955798c9906c8c03807391bc20f0")
# dev branch implicit in url
entry = SkillEntry.from_github_url(branch_from_git)
self.assertEqual(entry.branch, "dev")
# github release implicit in url
entry = SkillEntry.from_github_url(branch_from_tree)
self.assertEqual(entry.branch, "v0.1.0")
def test_requirements_commented(self):
from ovos_skills_manager.skill_entry import SkillEntry
entry = SkillEntry.from_github_url(
"https://github.com/OpenVoiceOS/tskill-osm_parsing@commented_requirements"
)
self.assertIsInstance(entry.requirements, dict)
self.assertEqual(set(entry.requirements.keys()),
{"python", "system", "skill"})
self.assertIsInstance(entry.requirements["python"], list)
self.assertIsInstance(entry.requirements["system"], dict)
self.assertIsInstance(entry.requirements["skill"], list)
self.assertEqual(
set(entry.requirements["python"]),
{"json-requirements", "manifest_requirement", "text_requirements"})
def get_ovos_skills(parse_github: bool = False, skiplist=None):
skiplist = skiplist or []
path = join(gettempdir(), "ovos")
dl_url = "https://github.com/OpenVoiceOS/OVOS-skills-store/archive/main.zip"
download_extract_zip(dl_url, path, join(path, "ovos-appstore.zip"))
for root, folders, files in walk(path):
files = [f for f in files if f.endswith(".json")]
for f in files:
with open(join(root, f)) as j:
data = json.load(j)
if data["url"] in skiplist:
continue
data["appstore"] = "OpenVoiceOS"
data["appstore_url"] = \
join("https://openvoiceos.github.io/OVOS-skills-store", f)
yield SkillEntry.from_json(data, parse_github=parse_github)
def get_neon_skills_from_api(parse_github=False, skiplist=None):
skiplist = skiplist or []
skills_url = "https://api.github.com/repos/NeonGeckoCom/neon-skills-submodules/contents/skill-metadata.json"
skill_json = requests.get(skills_url).json()
if skill_json.get("message") == 'Not Found' or "API rate limit exceeded" \
in skill_json.get("message", ""):
raise AuthenticationError
if skill_json.get("encoding") == "base64":
json_str = base64.b64decode(skill_json["content"]).decode("utf-8")
skill_json = json.loads(json_str)
for skill in skill_json.values():
if skill["url"] in skiplist:
continue
skill["appstore"] = "Neon"
skill["appstore_url"] = skills_url
yield SkillEntry.from_json(skill, parse_github=parse_github)
def get_local_skills(parse_github: bool = False,
skiplist: Optional[list] = None):
try:
skills = get_skills_folder()
except FileNotFoundError:
return
except KeyError:
# TODO: Patching config error in ovos_utils
return
skiplist = skiplist or []
folders = listdir(skills)
for fold in folders:
path = join(skills, fold)
if not isdir(path) or fold in skiplist:
continue
skill_dir = join(skills, fold)
skill = get_skill_data_from_directory(skill_dir)
yield SkillEntry.from_json(skill, parse_github=parse_github)
def test_explicit_branch(self):
from ovos_skills_manager.skill_entry import SkillEntry
entry = SkillEntry.from_github_url(
"https://github.com/OpenVoiceOS/tskill-osm_parsing@dev")
self.assertIsInstance(entry.requirements, dict)
self.assertEqual(set(entry.requirements.keys()),
{"python", "system", "skill"})
self.assertIsInstance(entry.requirements["python"], list)
self.assertIsInstance(entry.requirements["system"], dict)
self.assertIsInstance(entry.requirements["skill"], list)
self.assertEqual(entry.branch, "dev")
self.assertEqual(
entry.download_url,
"https://github.com/OpenVoiceOS/tskill-osm_parsing/archive/dev.zip"
)
self.assertIsInstance(entry.uuid, str)
self.assertEqual(entry.uuid, "tskill-osm_parsing.openvoiceos")
def get_pling_skills(parse_github=False, skiplist=None):
skiplist = skiplist or []
url = "https://api.kde-look.org/ocs/v1/content/data"
params = {"categories": "608", "page": 0}
xml = requests.get(url, params=params).text
data = xml2dict(xml)
meta = data["ocs"]["meta"]
n_pages = int(meta["totalitems"]) // int(meta["itemsperpage"])
for n in range(0, n_pages + 1):
LOG.debug("Parsing pling page {i} out of {n}".format(i=n, n=n_pages))
params = {"categories": "608", "page": n}
xml = requests.get(url, params=params).text
for skill in xml2dict(xml)["ocs"]["data"]["content"]:
skill_json = _parse_pling(skill)
if skill_json.get("url", "") in skiplist or not skill_json.get("url"):
continue
yield SkillEntry.from_json(skill_json, parse_github=parse_github)
def test_requirements_json_manifest_txt(self):
from ovos_skills_manager.skill_entry import SkillEntry
entry = SkillEntry.from_github_url(
"https://github.com/OpenVoiceOS/tskill-osm_parsing/tree/main")
self.assertIsInstance(entry.requirements, dict)
self.assertEqual(set(entry.requirements.keys()),
{"python", "system", "skill"})
self.assertEqual(
set(entry.requirements["python"]),
{"json-requirements", "manifest_requirement", "text_requirements"})
self.assertEqual(set(entry.requirements["system"]["all"]),
{"json-pkg", "system-manifest-pkg"})
self.assertEqual(set(entry.requirements["skill"]),
{"json-skill", "manifest-skill"})
self.assertEqual(entry.branch, "main")
self.assertEqual(
entry.download_url,
"https://github.com/OpenVoiceOS/tskill-osm_parsing/archive/main.zip"
)
def get_local_skills(parse_github=False, skiplist=None):
skills = get_skills_folder()
skiplist = skiplist or []
folders = listdir(skills)
for fold in folders:
path = join(skills, fold)
if not isdir(path) or fold in skiplist:
continue
skill = {
"appstore": "InstalledSkills",
"appstore_url": skills,
"skill_id": fold,
"foldername": fold,
"requirements": {
"python": [],
"system": [],
"skill": []
}
}
# if installed by msm/osm will obey this convention
if "." in fold:
try:
repo, author = fold.split(".")
skill["skillname"] = repo
skill["authorname"] = author
skill["url"] = f'https://github.com/{author}/{repo}'
except: # TODO replace with some clever check ?
pass
# parse git info
gitinfo = join(path, ".git/config")
if isfile(gitinfo):
with open(gitinfo) as f:
for l in f.readlines():
if l.strip().startswith("url ="):
skill["url"] = l.split("url =")[-1].strip()
skill["authorname"], skill["skillname"] = \
author_repo_from_github_url(skill["url"])
if l.strip().startswith("[branch "):
skill["branch"] = l.split("branch")[-1]\
.replace('"', "").strip()
for rtdir, foldrs, files in walk(join(skills, fold)):
for f in files:
if f in GITHUB_JSON_FILES:
with open(join(rtdir, f)) as fi:
skill_meta = json.load(fi)
skill = merge_dict(skill, skill_meta, merge_lists=True)
elif f in GITHUB_README_FILES:
with open(join(rtdir, f)) as fi:
readme = readme_to_json(fi.read())
skill = merge_dict(skill,
readme,
new_only=True,
merge_lists=True)
elif f in GITHUB_DESKTOP_FILES:
skill['desktopFile'] = True
elif f in GITHUB_ICON_FILES:
skill["icon"] = join(rtdir, f)
elif f in GITHUB_LICENSE_FILES:
with open(join(rtdir, f)) as fi:
lic = fi.read()
skill["license"] = parse_license_type(lic)
elif f in GITHUB_LOGO_FILES:
skill["logo"] = join(rtdir, f)
elif f in GITHUB_MANIFEST_FILES:
with open(join(rtdir, f)) as fi:
manifest = validate_manifest(fi.read())
skill["requirements"]["python"] += manifest.get(
"python") or []
skill["requirements"]["system"] += manifest.get(
"system") or []
skill["requirements"]["skill"] += manifest.get(
"skill") or []
elif f in GITHUB_REQUIREMENTS_FILES:
with open(join(rtdir, f)) as fi:
reqs = [r for r in fi.read().split("\n") if r.strip()]
skill["requirements"]["python"] += reqs
elif f in GITHUB_SKILL_REQUIREMENTS_FILES:
with open(join(rtdir, f)) as fi:
reqs = [r for r in fi.read().split("\n") if r.strip()]
skill["requirements"]["skill"] += reqs
yield SkillEntry.from_json(skill, parse_github=parse_github)
def test_requirements_from_txt(self):
from ovos_skills_manager.skill_entry import SkillEntry
entry = SkillEntry.from_github_url(
"https://github.com/OpenVoiceOS/tskill-osm_parsing/tree/dev")
self.assertIsInstance(entry.requirements, dict)
self.assertIsInstance(entry.requirements["python"], list)