def get_license_type_from_github_api(url, branch=None):
try:
data = get_repo_data_from_github_api(url, branch)
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
return data["license"]["key"]
except Exception as e:
pass
text = get_license_from_github_api(url, branch)
return parse_license_type(text)
def get_license_type_from_github_url(url: str,
branch: Optional[str] = None) -> str:
"""
Determine the License Name for the license of a given repository
https://docs.github.com/en/rest/reference/licenses#get-the-license-for-a-repository
@param url: Repository URL to query
@param branch: Optional branch spec, otherwise default branch will be used
@return: License Name
"""
branch = branch or get_branch_from_github_url(url)
license = get_license_from_github_url(url, branch).lower()
return parse_license_type(license)
def get_license_data_from_github_url(url: str,
branch: Optional[str] = None) -> dict:
"""
Get license data for the repository at the given URL and branch
https://docs.github.com/en/rest/reference/licenses#get-the-license-for-a-repository
@param url: Repository URL
@param branch: Optional branch spec, otherwise default branch will be used
@return: dict license data
"""
branch = branch or get_branch_from_github_url(url)
lic = get_license_from_github_url(url, branch)
return {
"license_type": parse_license_type(lic),
"license_text": lic
}
def get_license_type_from_github_api(url: str,
branch: Optional[str] = None) -> str:
"""
Determine the License Name for the license of a given repository
https://docs.github.com/en/rest/reference/licenses#get-the-license-for-a-repository
@param url: Repository URL to query
@param branch: Optional branch spec, otherwise default branch will be used
@return: License Name
"""
try:
data = get_repo_data_from_github_api(url, branch)
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
return data["license"]["key"]
except Exception as e:
pass
text = get_license_from_github_api(url, branch)
return parse_license_type(text)
def test_detect_viral(self):
for lic in [
self.mit_template_noheader, self.isc_template,
self.bsd0_template
]:
lic = parse_license_type(lic)
self.assertFalse(is_viral(lic))
self.assertTrue(is_permissive(lic))
for url in self.apaches + self.apache_fails + self.mits + \
self.mit_fails + self.isc + self.unlicense:
# TODO license text examples, not url
lic = get_license_type(url, "master")
self.assertFalse(is_viral(lic))
self.assertTrue(is_permissive(lic))
for url in self.agpls + self.lgpls + self.gpls + self.gpl2 + \
self.epl1 + self.epl2:
# TODO license text examples, not url
lic = get_license_type(url, "master")
self.assertTrue(is_viral(lic))
self.assertFalse(is_permissive(lic))
def test_parse(self):
from ovos_skills_manager.licenses import parse_license_type
self.assertEqual(parse_license_type(self.mit_template_noheader), "mit")
self.assertEqual(parse_license_type(self.isc_template), "isc")
self.assertEqual(parse_license_type(self.bsd0_template), "0bsd")
def get_local_skills(parse_github=False, skiplist=None):
skills = get_skills_folder()
skiplist = skiplist or []
folders = listdir(skills)
for fold in folders:
path = join(skills, fold)
if not isdir(path) or fold in skiplist:
continue
skill = {
"appstore": "InstalledSkills",
"appstore_url": skills,
"skill_id": fold,
"foldername": fold,
"requirements": {
"python": [],
"system": [],
"skill": []
}
}
# if installed by msm/osm will obey this convention
if "." in fold:
try:
repo, author = fold.split(".")
skill["skillname"] = repo
skill["authorname"] = author
skill["url"] = f'https://github.com/{author}/{repo}'
except: # TODO replace with some clever check ?
pass
# parse git info
gitinfo = join(path, ".git/config")
if isfile(gitinfo):
with open(gitinfo) as f:
for l in f.readlines():
if l.strip().startswith("url ="):
skill["url"] = l.split("url =")[-1].strip()
skill["authorname"], skill["skillname"] = \
author_repo_from_github_url(skill["url"])
if l.strip().startswith("[branch "):
skill["branch"] = l.split("branch")[-1]\
.replace('"', "").strip()
for rtdir, foldrs, files in walk(join(skills, fold)):
for f in files:
if f in GITHUB_JSON_FILES:
with open(join(rtdir, f)) as fi:
skill_meta = json.load(fi)
skill = merge_dict(skill, skill_meta, merge_lists=True)
elif f in GITHUB_README_FILES:
with open(join(rtdir, f)) as fi:
readme = readme_to_json(fi.read())
skill = merge_dict(skill,
readme,
new_only=True,
merge_lists=True)
elif f in GITHUB_DESKTOP_FILES:
skill['desktopFile'] = True
elif f in GITHUB_ICON_FILES:
skill["icon"] = join(rtdir, f)
elif f in GITHUB_LICENSE_FILES:
with open(join(rtdir, f)) as fi:
lic = fi.read()
skill["license"] = parse_license_type(lic)
elif f in GITHUB_LOGO_FILES:
skill["logo"] = join(rtdir, f)
elif f in GITHUB_MANIFEST_FILES:
with open(join(rtdir, f)) as fi:
manifest = validate_manifest(fi.read())
skill["requirements"]["python"] += manifest.get(
"python") or []
skill["requirements"]["system"] += manifest.get(
"system") or []
skill["requirements"]["skill"] += manifest.get(
"skill") or []
elif f in GITHUB_REQUIREMENTS_FILES:
with open(join(rtdir, f)) as fi:
reqs = [r for r in fi.read().split("\n") if r.strip()]
skill["requirements"]["python"] += reqs
elif f in GITHUB_SKILL_REQUIREMENTS_FILES:
with open(join(rtdir, f)) as fi:
reqs = [r for r in fi.read().split("\n") if r.strip()]
skill["requirements"]["skill"] += reqs
yield SkillEntry.from_json(skill, parse_github=parse_github)
def test_parse(self):
self.assertEqual(parse_license_type(self.mit_template_noheader), "mit")
self.assertEqual(parse_license_type(self.isc_template), "isc")
self.assertEqual(parse_license_type(self.bsd0_template), "0bsd")
def get_skill_data_from_directory(skill_dir: str):
"""
Parse the specified skill directory and return a dict representation of a
SkillEntry.
@param skill_dir: path to skill directory
@return: dict parsed skill data
"""
skills, fold = skill_dir.rsplit('/', 1)
skill_data = {
"appstore": "InstalledSkills",
"appstore_url": skills,
"skill_id": fold,
"requirements": {
"python": [],
"system": {},
"skill": []
}
}
# if installed by msm/osm will obey this convention
if "." in fold:
try:
repo, author = fold.split(".")
skill_data["skillname"] = repo
skill_data["authorname"] = author
skill_data["url"] = f'https://github.com/{author}/{repo}'
except: # TODO replace with some clever check ?
pass
# parse git info
gitinfo = join(skill_dir, ".git/config")
if isfile(gitinfo):
with open(gitinfo) as f:
for l in f.readlines():
if l.strip().startswith("url ="):
skill_data["url"] = l.split("url =")[-1].strip()
skill_data["authorname"], skill_data["skillname"] = \
author_repo_from_github_url(skill_data["url"])
if l.strip().startswith("[branch "):
skill_data["branch"] = l.split("branch")[-1] \
.replace('"', "").strip()
# parse skill files
for root_dir, _, files in walk(skill_dir):
for f in files:
if f in GITHUB_JSON_FILES: # skill.json
with open(join(root_dir, f)) as fi:
skill_meta = json.load(fi)
skill_data = merge_dict(skill_data,
skill_meta,
merge_lists=True)
elif f in GITHUB_README_FILES:
with open(join(root_dir, f)) as fi:
readme = readme_to_json(fi.read())
skill_data = merge_dict(skill_data,
readme,
new_only=True,
merge_lists=True)
elif f in GITHUB_DESKTOP_FILES:
skill_data['desktopFile'] = True
elif f in GITHUB_ICON_FILES:
skill_data["icon"] = join(root_dir, f)
elif f in GITHUB_LICENSE_FILES:
with open(join(root_dir, f)) as fi:
lic = fi.read()
skill_data["license"] = parse_license_type(lic)
elif f in GITHUB_LOGO_FILES:
skill_data["logo"] = join(root_dir, f)
elif f in GITHUB_MANIFEST_FILES:
with open(join(root_dir, f)) as fi:
manifest = validate_manifest(fi.read()).get(
"dependencies", {})
skill_data["requirements"]["python"] += \
manifest.get("python") or []
skill_data["requirements"]["system"] = \
merge_dict(skill_data["requirements"]["system"],
manifest.get("system") or {}, merge_lists=True)
skill_data["requirements"]["skill"] += \
manifest.get("skill") or []
elif f in GITHUB_REQUIREMENTS_FILES:
with open(join(root_dir, f)) as fi:
reqs = [r for r in fi.read().split("\n") if r.strip()]
skill_data["requirements"]["python"] += reqs
elif f in GITHUB_SKILL_REQUIREMENTS_FILES:
with open(join(root_dir, f)) as fi:
reqs = [r for r in fi.read().split("\n") if r.strip()]
skill_data["requirements"]["skill"] += reqs
# de-dupe requirements
skill_data["requirements"]["python"] = \
list(set(skill_data["requirements"]["python"]))
skill_data["requirements"]["skill"] = \
list(set(skill_data["requirements"]["skill"]))
skill_data['foldername'] = fold # Override what the config specifies
skill_data['authorname'] = skill_data.get('authorname') or "local"
return skill_data
def get_license_data_from_github_url(url, branch=None):
branch = branch or get_branch_from_github_url(url)
lic = get_license_from_github_url(url, branch)
return {"license_type": parse_license_type(lic), "license_text": lic}
def get_license_type_from_github_url(url, branch=None):
branch = branch or get_branch_from_github_url(url)
license = get_license_from_github_url(url, branch).lower()
return parse_license_type(license)
