def test_author_repo_from_url(self):
from ovos_skills_manager.github.utils import author_repo_from_github_url
url = "https://github.com/JarbasSkills/skill-wolfie"
self.assertEqual(author_repo_from_github_url(url),
["JarbasSkills", "skill-wolfie"])
self.assertEqual(author_repo_from_github_url(url + "/tree/v0.1"),
["JarbasSkills", "skill-wolfie"])
url = "https://github.com/MycroftAI/skill-hello-world"
self.assertEqual(author_repo_from_github_url(url + "/tree/20.08"),
["MycroftAI", "skill-hello-world"])
url = "https://raw.githubusercontent.com/JarbasSkills/skill-wolfie"
self.assertEqual(author_repo_from_github_url(url),
["JarbasSkills", "skill-wolfie"])
def get_repo_releases_from_github_api(url: str,
branch: Optional[str] = None) -> list:
"""
Get releases data for the repository at the specified URL and branch
https://docs.github.com/en/rest/reference/repos#list-repository-tags
@param url: Repository URL
@param branch: Optional branch spec, otherwise default branch will be used
@return: repo tag data
"""
# TODO: There is a releases API, but this uses the tags API
try:
author, repo = author_repo_from_github_url(url)
url = GithubAPI.REPO_RELEASES.format(owner=author, repo=repo)
data = requests.get(url, params={"ref": branch}).json()
except Exception as e:
raise GithubAPIReleasesNotFound(str(e))
if isinstance(data, dict):
# result is usually a list, unless api call fails
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
# let's fix api urls
for idx, r in enumerate(data):
data[idx]["tarball_url"] = GithubUrls.DOWNLOAD_TARBALL.format(
author=author, repo=repo, branch=r["name"])
data[idx]["zipball_url"] = GithubUrls.DOWNLOAD.format(author=author,
repo=repo,
branch=r["name"])
data[idx].pop('node_id')
return data
def get_skill_json_from_github_api(url: str,
branch: Optional[str] = None) -> dict:
"""
Get skill.json file contents for the specified repository
@param url: Repository URL to query
@param branch: Optional branch spec, otherwise default branch will be used
@return: data parsed from skill.json
"""
author, repo = author_repo_from_github_url(url)
for dst in GITHUB_JSON_FILES:
try:
data = get_file_from_github_api(url, dst.format(repo=repo), branch)
except GithubAPIFileNotFound:
continue
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
if data.get("content"):
content = data["content"]
if data["encoding"] == "base64":
json_data = base64.b64decode(content).decode("utf-8")
else:
# TODO Raise UnknownEncoding?
json_data = content
return json.loads(json_data)
raise GithubAPIFileNotFound
def get_manifest_from_github_api(url: str,
branch: Optional[str] = None) -> dict:
"""
Get requirements specified in the repository manifest file
@param url: Repository URL to query
@param branch: Optional branch spec, otherwise default branch will be used
@return: dict parsed requirements
"""
author, repo = author_repo_from_github_url(url)
content = None
for dst in GITHUB_MANIFEST_FILES:
try:
data = get_file_from_github_api(url, dst.format(repo=repo), branch)
except GithubAPIFileNotFound:
continue
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
if data.get("content"):
content = data["content"]
if data["encoding"] == "base64":
content = base64.b64decode(content).decode("utf-8")
# TODO Raise UnknownEncoding?
if not content:
raise GithubAPIFileNotFound
return validate_manifest(content)
def get_skill_requirements_from_github_api(url: str,
branch: Optional[str] = None
) -> list:
"""
Get Skill requirements from text files in the specified repository
@param url: Repository URL to query
@param branch: Optional branch spec, otherwise default branch will be used
@return: List parsed skill requirements
"""
author, repo = author_repo_from_github_url(url)
content = None
for dst in GITHUB_SKILL_REQUIREMENTS_FILES:
try:
data = get_file_from_github_api(url, dst.format(repo=repo), branch)
except GithubAPIFileNotFound:
continue
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
if data.get("content"):
content = data["content"]
if data["encoding"] == "base64":
content = base64.b64decode(content).decode("utf-8")
# TODO Raise UnknownEncoding?
if not content:
raise GithubAPIFileNotFound
return [
t for t in content.split("\n")
if t.strip() and not t.strip().startswith("#")
]
def api_zip_url_from_github_url(url: str,
branch: Optional[str] = None,
token: Optional[str] = None) -> str:
"""
Get an API URL to download the repository as a zip archive
https://docs.github.com/en/rest/reference/repos#download-a-repository-archive-zip
@param url: Repository URL, optionally containing a branch spec
@param branch: Optional branch spec, otherwise branch from `url` will be used
@param token: Optional GitHub token to include with request
@return: GitHub API URL to query for a zip archive
"""
# TODO: `token` is not used?
# specific file
try:
url = blob2raw(url)
if requests.get(url).ok:
return url
except GithubInvalidUrl:
pass
# full git repo
branch = branch or get_branch_from_github_url(url)
owner, repo = author_repo_from_github_url(url)
url = GithubAPI.REPO_ZIP.format(owner=owner, branch=branch, repo=repo)
if requests.get(url).status_code == 200:
return url
raise GithubInvalidUrl
def get_readme_url_from_github_api(url: str,
branch: Optional[str] = None) -> str:
"""
Get the readme file url for the specified repository
https://docs.github.com/en/rest/reference/repos#get-a-repository-readme
@param url: Repository URL
@param branch: Optional branch to query, otherwise default branch will be used
@return: url of repository README file
"""
author, repo = author_repo_from_github_url(url)
default_url = GithubAPI.REPO_README.format(owner=author, repo=repo)
try:
data = requests.get(default_url, params={"ref": branch}).json()
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
return data["html_url"]
except Exception as e:
pass # check files individually
for dst in GITHUB_README_FILES:
try:
data = get_file_from_github_api(url, dst, branch)
except GithubAPIFileNotFound:
continue
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
if data.get("html_url"):
return data["html_url"]
raise GithubAPIReadmeNotFound
def get_readme_from_github_api(url, branch=None):
author, repo = author_repo_from_github_url(url)
default_url = GithubAPI.REPO_README.format(owner=author, repo=repo)
try:
data = requests.get(default_url, params={"ref": branch}).json()
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
readme = data["content"]
if data["encoding"] == "base64":
return base64.b64decode(readme).decode("utf-8")
# TODO Raise UnknownEncoding?
return readme
except GithubAPIRateLimited:
raise
except Exception as e:
pass
# check files individually
for dst in GITHUB_README_FILES:
try:
data = get_file_from_github_api(url, dst, branch)
except GithubAPIFileNotFound:
continue
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
readme = data.get("content")
if readme:
if data["encoding"] == "base64":
return base64.b64decode(readme).decode("utf-8")
# TODO Raise UnknownEncoding?
return readme
raise GithubAPIReadmeNotFound
def get_license_data_from_github_api(url, branch=None):
author, repo = author_repo_from_github_url(url)
url = GithubAPI.REPO_LICENSE.format(owner=author, repo=repo)
try:
data = requests.get(url, params={"ref": branch}).json()
except Exception as e:
raise GithubAPILicenseNotFound
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
return data
def get_file_from_github_api(url, filepath, branch=None):
author, repo = author_repo_from_github_url(url)
branch = branch or get_main_branch_from_github_api(url)
url = GithubAPI.REPO_FILE.format(owner=author, repo=repo, file=filepath)
data = requests.get(url, params={"ref": branch}).json()
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
if data.get("message", "") != 'Not Found':
return data
raise GithubAPIFileNotFound
def get_logo_url_from_github_api(url, branch=None):
author, repo = author_repo_from_github_url(url)
for dst in GITHUB_LOGO_FILES:
try:
data = get_file_from_github_api(url, dst.format(repo=repo), branch)
except GithubAPIFileNotFound:
continue
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
if data.get("download_url"):
return data["download_url"]
raise GithubAPIFileNotFound
def get_desktop_from_github_api(url, branch=None):
author, repo = author_repo_from_github_url(url)
for dst in GITHUB_DESKTOP_FILES:
try:
data = get_file_from_github_api(url, dst.format(repo=repo), branch)
except GithubAPIFileNotFound:
continue
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
if data.get("content"):
readme = data["content"]
if data["encoding"] == "base64":
return base64.b64decode(readme).decode("utf-8")
# TODO Raise UnknownEncoding?
return readme
raise GithubAPIFileNotFound
def api_zip_url_from_github_url(url, branch=None, token=None):
# specific file
try:
url = blob2raw(url)
if requests.get(url).status_code == 200:
return url
except GithubInvalidUrl:
pass
# full git repo
branch = branch or get_branch_from_github_url(url)
owner, repo = author_repo_from_github_url(url)
url = GithubAPI.REPO_ZIP.format(owner=owner, branch=branch, repo=repo)
if requests.get(url).status_code == 200:
return url
raise GithubInvalidUrl
def get_license_data_from_github_api(url: str,
branch: Optional[str] = None) -> dict:
"""
Get license data for the repository at the given URL and branch
https://docs.github.com/en/rest/reference/licenses#get-the-license-for-a-repository
@param url: Repository URL
@param branch: Optional branch spec, otherwise default branch will be used
@return: dict license data
"""
author, repo = author_repo_from_github_url(url)
url = GithubAPI.REPO_LICENSE.format(owner=author, repo=repo)
try:
data = requests.get(url, params={"ref": branch}).json()
except Exception as e:
raise GithubAPILicenseNotFound
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
return data
def get_manifest_from_github_api(url, branch=None):
author, repo = author_repo_from_github_url(url)
content = None
for dst in GITHUB_MANIFEST_FILES:
try:
data = get_file_from_github_api(url, dst.format(repo=repo), branch)
except GithubAPIFileNotFound:
continue
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
if data.get("content"):
content = data["content"]
if data["encoding"] == "base64":
content = base64.b64decode(content).decode("utf-8")
# TODO Raise UnknownEncoding?
if not content:
raise GithubAPIFileNotFound
return validate_manifest(content)
def get_skill_json_from_github_api(url, branch=None):
author, repo = author_repo_from_github_url(url)
for dst in GITHUB_JSON_FILES:
try:
data = get_file_from_github_api(url, dst.format(repo=repo), branch)
except GithubAPIFileNotFound:
continue
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
if data.get("content"):
content = data["content"]
if data["encoding"] == "base64":
json_data = base64.b64decode(content).decode("utf-8")
else:
# TODO Raise UnknownEncoding?
json_data = content
return json.loads(json_data)
raise GithubAPIFileNotFound
def get_logo_url_from_github_api(url: str,
branch: Optional[str] = None) -> str:
"""
Get skill logo file URL for the specified repository
@param url: Repository URL to query
@param branch: Optional branch spec, otherwise default branch will be used
@return: URL of skill logo
"""
author, repo = author_repo_from_github_url(url)
for dst in GITHUB_LOGO_FILES:
try:
data = get_file_from_github_api(url, dst.format(repo=repo), branch)
except GithubAPIFileNotFound:
continue
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
if data.get("download_url"):
return data["download_url"]
raise GithubAPIFileNotFound
def get_skill_requirements_from_github_api(url, branch=None):
author, repo = author_repo_from_github_url(url)
content = None
for dst in GITHUB_SKILL_REQUIREMENTS_FILES:
try:
data = get_file_from_github_api(url, dst.format(repo=repo), branch)
except GithubAPIFileNotFound:
continue
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
if data.get("content"):
content = data["content"]
if data["encoding"] == "base64":
content = base64.b64decode(content).decode("utf-8")
# TODO Raise UnknownEncoding?
if not content:
raise GithubAPIFileNotFound
return [t for t in content.split("\n")
if t.strip() and not t.strip().startswith("#")]
def get_repo_releases_from_github_api(url, branch=None):
try:
author, repo = author_repo_from_github_url(url)
url = GithubAPI.REPO_RELEASES.format(owner=author, repo=repo)
data = requests.get(url, params={"ref": branch}).json()
except Exception as e:
raise GithubAPIReleasesNotFound(str(e))
if isinstance(data, dict):
# result is usually a list, unless api call fails
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
# let's fix api urls
for idx, r in enumerate(data):
data[idx]["tarball_url"] = GithubUrls.DOWNLOAD_TARBALL.format(
author=author, repo=repo, branch=r["name"])
data[idx]["zipball_url"] = GithubUrls.DOWNLOAD.format(
author=author, repo=repo, branch=r["name"])
data[idx].pop('node_id')
return data
def get_file_from_github_api(url: str,
filepath: str,
branch: Optional[str] = None) -> dict:
"""
Get information for a file in a repository.
https://docs.github.com/en/rest/reference/repos#get-repository-content
@param url: Repository URL
@param filepath: path to a file in the repository
@param branch: Optional branch to query, otherwise branch from `url` will be used
@return: parsed API data
"""
author, repo = author_repo_from_github_url(url)
branch = branch or get_main_branch_from_github_api(url)
url = GithubAPI.REPO_FILE.format(owner=author, repo=repo, file=filepath)
data = requests.get(url, params={"ref": branch}).json()
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
if data.get("message", "") != 'Not Found':
return data
raise GithubAPIFileNotFound
def get_readme_url_from_github_api(url, branch=None):
author, repo = author_repo_from_github_url(url)
default_url = GithubAPI.REPO_README.format(owner=author, repo=repo)
try:
data = requests.get(default_url, params={"ref": branch}).json()
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
return data["html_url"]
except Exception as e:
pass # check files individually
for dst in GITHUB_README_FILES:
try:
data = get_file_from_github_api(url, dst, branch)
except GithubAPIFileNotFound:
continue
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
if data.get("html_url"):
return data["html_url"]
raise GithubAPIReadmeNotFound
def get_desktop_from_github_api(url: str, branch: Optional[str] = None) -> str:
"""
Get skill.desktop file contents for the specified repository
@param url: Repository URL to query
@param branch: Optional branch spec, otherwise default branch will be used
@return: skill.desktop string contents
"""
author, repo = author_repo_from_github_url(url)
for dst in GITHUB_DESKTOP_FILES:
try:
data = get_file_from_github_api(url, dst.format(repo=repo), branch)
except GithubAPIFileNotFound:
continue
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
if data.get("content"):
readme = data["content"]
if data["encoding"] == "base64":
return base64.b64decode(readme).decode("utf-8")
# TODO Raise UnknownEncoding?
return readme
raise GithubAPIFileNotFound
def get_readme_from_github_api(url: str, branch: Optional[str] = None) -> str:
"""
Get the readme file contents for the specified repository
@param url: Repository URL
@param branch: Optional branch to query, otherwise default branch will be used
@return: contents of repository README file
"""
author, repo = author_repo_from_github_url(url)
default_url = GithubAPI.REPO_README.format(owner=author, repo=repo)
try:
data = requests.get(default_url, params={"ref": branch}).json()
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
readme = data["content"]
if data["encoding"] == "base64":
return base64.b64decode(readme).decode("utf-8")
# TODO Raise UnknownEncoding?
return readme
except GithubAPIRateLimited:
raise
except Exception as e:
pass
# check files individually
for dst in GITHUB_README_FILES:
try:
data = get_file_from_github_api(url, dst, branch)
except GithubAPIFileNotFound:
continue
if "API rate limit exceeded" in data.get("message", ""):
raise GithubAPIRateLimited
readme = data.get("content")
if readme:
if data["encoding"] == "base64":
return base64.b64decode(readme).decode("utf-8")
# TODO Raise UnknownEncoding?
return readme
raise GithubAPIReadmeNotFound
def build_store_model(self, store_id):
store = self.osm_manager.get_appstore(store_id)
skills_model = []
for skill in store:
skill_icon = skill.skill_icon or self.default_icon
if not validators.url(skill.skill_icon):
# TODO osm should transform the relative paths!
# discard bad paths for now
skill_icon = self.default_icon
author, repo = author_repo_from_github_url(skill.url)
desc = skill.skill_short_description or \
skill.skill_description or \
f"{repo} by {author}"
skills_model.append({
"title": skill.skill_name or repo,
"description": desc,
"logo": skill.json.get("logo") or skill_icon,
"author": skill.skill_author,
"category": skill.skill_category,
"url": skill.url
})
return skills_model
def get_local_skills(parse_github=False, skiplist=None):
skills = get_skills_folder()
skiplist = skiplist or []
folders = listdir(skills)
for fold in folders:
path = join(skills, fold)
if not isdir(path) or fold in skiplist:
continue
skill = {
"appstore": "InstalledSkills",
"appstore_url": skills,
"skill_id": fold,
"foldername": fold,
"requirements": {
"python": [],
"system": [],
"skill": []
}
}
# if installed by msm/osm will obey this convention
if "." in fold:
try:
repo, author = fold.split(".")
skill["skillname"] = repo
skill["authorname"] = author
skill["url"] = f'https://github.com/{author}/{repo}'
except: # TODO replace with some clever check ?
pass
# parse git info
gitinfo = join(path, ".git/config")
if isfile(gitinfo):
with open(gitinfo) as f:
for l in f.readlines():
if l.strip().startswith("url ="):
skill["url"] = l.split("url =")[-1].strip()
skill["authorname"], skill["skillname"] = \
author_repo_from_github_url(skill["url"])
if l.strip().startswith("[branch "):
skill["branch"] = l.split("branch")[-1]\
.replace('"', "").strip()
for rtdir, foldrs, files in walk(join(skills, fold)):
for f in files:
if f in GITHUB_JSON_FILES:
with open(join(rtdir, f)) as fi:
skill_meta = json.load(fi)
skill = merge_dict(skill, skill_meta, merge_lists=True)
elif f in GITHUB_README_FILES:
with open(join(rtdir, f)) as fi:
readme = readme_to_json(fi.read())
skill = merge_dict(skill,
readme,
new_only=True,
merge_lists=True)
elif f in GITHUB_DESKTOP_FILES:
skill['desktopFile'] = True
elif f in GITHUB_ICON_FILES:
skill["icon"] = join(rtdir, f)
elif f in GITHUB_LICENSE_FILES:
with open(join(rtdir, f)) as fi:
lic = fi.read()
skill["license"] = parse_license_type(lic)
elif f in GITHUB_LOGO_FILES:
skill["logo"] = join(rtdir, f)
elif f in GITHUB_MANIFEST_FILES:
with open(join(rtdir, f)) as fi:
manifest = validate_manifest(fi.read())
skill["requirements"]["python"] += manifest.get(
"python") or []
skill["requirements"]["system"] += manifest.get(
"system") or []
skill["requirements"]["skill"] += manifest.get(
"skill") or []
elif f in GITHUB_REQUIREMENTS_FILES:
with open(join(rtdir, f)) as fi:
reqs = [r for r in fi.read().split("\n") if r.strip()]
skill["requirements"]["python"] += reqs
elif f in GITHUB_SKILL_REQUIREMENTS_FILES:
with open(join(rtdir, f)) as fi:
reqs = [r for r in fi.read().split("\n") if r.strip()]
skill["requirements"]["skill"] += reqs
yield SkillEntry.from_json(skill, parse_github=parse_github)
def get_skill_data_from_directory(skill_dir: str):
"""
Parse the specified skill directory and return a dict representation of a
SkillEntry.
@param skill_dir: path to skill directory
@return: dict parsed skill data
"""
skills, fold = skill_dir.rsplit('/', 1)
skill_data = {
"appstore": "InstalledSkills",
"appstore_url": skills,
"skill_id": fold,
"requirements": {
"python": [],
"system": {},
"skill": []
}
}
# if installed by msm/osm will obey this convention
if "." in fold:
try:
repo, author = fold.split(".")
skill_data["skillname"] = repo
skill_data["authorname"] = author
skill_data["url"] = f'https://github.com/{author}/{repo}'
except: # TODO replace with some clever check ?
pass
# parse git info
gitinfo = join(skill_dir, ".git/config")
if isfile(gitinfo):
with open(gitinfo) as f:
for l in f.readlines():
if l.strip().startswith("url ="):
skill_data["url"] = l.split("url =")[-1].strip()
skill_data["authorname"], skill_data["skillname"] = \
author_repo_from_github_url(skill_data["url"])
if l.strip().startswith("[branch "):
skill_data["branch"] = l.split("branch")[-1] \
.replace('"', "").strip()
# parse skill files
for root_dir, _, files in walk(skill_dir):
for f in files:
if f in GITHUB_JSON_FILES: # skill.json
with open(join(root_dir, f)) as fi:
skill_meta = json.load(fi)
skill_data = merge_dict(skill_data,
skill_meta,
merge_lists=True)
elif f in GITHUB_README_FILES:
with open(join(root_dir, f)) as fi:
readme = readme_to_json(fi.read())
skill_data = merge_dict(skill_data,
readme,
new_only=True,
merge_lists=True)
elif f in GITHUB_DESKTOP_FILES:
skill_data['desktopFile'] = True
elif f in GITHUB_ICON_FILES:
skill_data["icon"] = join(root_dir, f)
elif f in GITHUB_LICENSE_FILES:
with open(join(root_dir, f)) as fi:
lic = fi.read()
skill_data["license"] = parse_license_type(lic)
elif f in GITHUB_LOGO_FILES:
skill_data["logo"] = join(root_dir, f)
elif f in GITHUB_MANIFEST_FILES:
with open(join(root_dir, f)) as fi:
manifest = validate_manifest(fi.read()).get(
"dependencies", {})
skill_data["requirements"]["python"] += \
manifest.get("python") or []
skill_data["requirements"]["system"] = \
merge_dict(skill_data["requirements"]["system"],
manifest.get("system") or {}, merge_lists=True)
skill_data["requirements"]["skill"] += \
manifest.get("skill") or []
elif f in GITHUB_REQUIREMENTS_FILES:
with open(join(root_dir, f)) as fi:
reqs = [r for r in fi.read().split("\n") if r.strip()]
skill_data["requirements"]["python"] += reqs
elif f in GITHUB_SKILL_REQUIREMENTS_FILES:
with open(join(root_dir, f)) as fi:
reqs = [r for r in fi.read().split("\n") if r.strip()]
skill_data["requirements"]["skill"] += reqs
# de-dupe requirements
skill_data["requirements"]["python"] = \
list(set(skill_data["requirements"]["python"]))
skill_data["requirements"]["skill"] = \
list(set(skill_data["requirements"]["skill"]))
skill_data['foldername'] = fold # Override what the config specifies
skill_data['authorname'] = skill_data.get('authorname') or "local"
return skill_data
def get_skill_data(url, branch=None):
author, repo = author_repo_from_github_url(url)
explicit_branch = branch
if not branch:
try:
branch = get_branch_from_github_url(url)
explicit_branch = branch
except GithubInvalidBranch:
branch = get_main_branch(url)
data = {
"authorname": author,
"foldername": repo,
"url": normalize_github_url(url),
"branch": branch,
"license": "unknown",
"tags": []
}
# augment with repo data
try:
api_data = get_repo_data(url)
# only replace branch if not found by prev methods
branch = branch or api_data['default_branch']
LOG.debug(f"default branch extracted from github api: {branch}")
data["branch"] = branch
data["short_description"] = api_data['description']
data["foldername"] = api_data["name"]
data["last_updated"] = api_data['updated_at']
data["url"] = api_data["html_url"]
data["authorname"] = api_data["owner"]["login"]
branch = branch or api_data['default_branch']
if "license" in data:
data["license"] = api_data["license"]["key"]
except:
pass
# augment with releases data
try:
releases = get_releases(url)
# search release that matches branch
if branch:
for r in releases:
if r["name"] == branch or r["commit"]["sha"] == branch:
data["version"] = r["name"]
# data["download_url"] = r["tarball_url"]
break
# just pick latest release if branch not defined
elif len(releases) > 0:
data["version"] = data["branch"] = branch = releases[0]["name"]
# data["download_url"] = releases[0]["tarball_url"]
LOG.debug(f"default branch extracted from github releases:"
f" {branch}")
except GithubSkillEntryError as e:
pass
# augment with readme data
try:
data = merge_dict(data, get_readme_json(url, branch),
merge_lists=True, skip_empty=True, no_dupes=True)
except GithubSkillEntryError as e:
pass
# find logo
try:
data["logo"] = get_logo_url(url, branch)
except GithubFileNotFound as e:
pass
# augment with requirements
data["requirements"] = get_requirements_json(url, branch)
# augment with android data
data["android"] = get_android_json(url, branch)
# augment with desktop data
try:
data["desktop"] = get_desktop_json(url, branch)
data["desktopFile"] = True
except GithubFileNotFound:
data["desktopFile"] = False
# augment tags
if is_viral(data["license"]):
data["tags"].append("viral-license")
elif is_permissive(data["license"]):
data["tags"].append("permissive-license")
elif "unknown" in data["license"]:
data["tags"].append("no-license")
try:
data["license"] = get_license_type(url, branch)
except GithubLicenseNotFound:
pass
# augment with json data
# this should take precedence over everything else
try:
data = merge_dict(data, get_skill_json(url, branch),
merge_lists=True, skip_empty=True, no_dupes=True)
branch = data.get("branch")
LOG.debug(f"default branch extracted from json: {branch}")
except GithubFileNotFound:
pass
if explicit_branch:
LOG.debug(f"explicit branch in url selected: {branch}")
data["branch"] = explicit_branch
if data.get("download_url"):
# TODO deprecate this warning
LOG.warning("the skill.json provides a download url, this used to be "
"supported but it is dangerous and OSM will discard it, "
"skill.json should only define github url and branch, "
"the download_url will be auto-generated")
LOG.debug(f"discarded url: {data['download_url']}")
data.pop("download_url")
return data
def get_skill_from_api(url, branch=None, strict=False):
data = {}
# extract branch from .json, should branch take precedence?
# i think so because user explicitly requested it
branch = get_branch_from_skill_json_github_api(url, branch)
try:
api_data = get_repo_data_from_github_api(url, branch)
data["branch"] = branch = api_data['default_branch']
data["short_description"] = api_data['description']
data["license"] = api_data["license"]["key"]
data["foldername"] = api_data["name"]
data["last_updated"] = api_data['updated_at']
data["url"] = api_data["html_url"]
data["authorname"] = api_data["owner"]["login"]
except GithubAPIException as e:
LOG.error("Failed to retrieve repo data from github api")
raise
try:
releases = get_repo_releases_from_github_api(url, branch)
if branch:
for r in releases:
if r["name"] == branch or r["commit"]["sha"] == branch:
data["version"] = r["name"]
#data["download_url"] = r["tarball_url"]
break
else:
data["version"] = releases[0]["name"]
#data["download_url"] = releases[0]["tarball_url"]
except GithubAPIException as e:
LOG.error("Failed to retrieve releases data from github api")
if strict:
raise GithubAPIReleasesNotFound
# augment with readme data
try:
data = merge_dict(data, get_readme_json_from_api(url, branch),
merge_lists=True, skip_empty=True, no_dupes=True)
except GithubAPIReadmeNotFound:
pass
data["requirements"] = get_requirements_json_from_github_api(url, branch)
# find logo
try:
data["logo"] = get_logo_url_from_github_api(url, branch)
except GithubAPIFileNotFound as e:
pass
# find icon
try:
data["icon"] = icon = get_icon_url_from_github_api(url, branch)
except GithubAPIFileNotFound:
icon = None
# augment with android data
try:
data["android"] = get_android_json_from_github_api(url, branch)
except GithubAPIFileNotFound:
# best guess or throw exception?
author, repo = author_repo_from_github_url(url)
data["android"] = {
'android_icon': icon,
'android_name': skill_name_from_github_url(url),
'android_handler': '{repo}.{author}.home'.format(repo=repo,
author=author.lower())}
# augment with desktop data
try:
data["desktop"] = get_desktop_json_from_github_api(url, branch)
data["desktopFile"] = True
except GithubFileNotFound:
data["desktopFile"] = False
# augment tags
if "tags" not in data:
data["tags"] = []
if is_viral(data["license"]):
data["tags"].append("viral-license")
elif is_permissive(data["license"]):
data["tags"].append("permissive-license")
elif "unknown" in data["license"]:
data["tags"].append("no-license")
# augment with json data
# this should take precedence over everything else
try:
data = merge_dict(data, get_skill_json_from_github_api(url, branch),
merge_lists=True, skip_empty=True, no_dupes=True)
except GithubFileNotFound:
pass
return data
