def test_author_repo_from_url(self): from ovos_skills_manager.github.utils import author_repo_from_github_url url = "https://github.com/JarbasSkills/skill-wolfie" self.assertEqual(author_repo_from_github_url(url), ["JarbasSkills", "skill-wolfie"]) self.assertEqual(author_repo_from_github_url(url + "/tree/v0.1"), ["JarbasSkills", "skill-wolfie"]) url = "https://github.com/MycroftAI/skill-hello-world" self.assertEqual(author_repo_from_github_url(url + "/tree/20.08"), ["MycroftAI", "skill-hello-world"]) url = "https://raw.githubusercontent.com/JarbasSkills/skill-wolfie" self.assertEqual(author_repo_from_github_url(url), ["JarbasSkills", "skill-wolfie"])
def get_repo_releases_from_github_api(url: str, branch: Optional[str] = None) -> list: """ Get releases data for the repository at the specified URL and branch https://docs.github.com/en/rest/reference/repos#list-repository-tags @param url: Repository URL @param branch: Optional branch spec, otherwise default branch will be used @return: repo tag data """ # TODO: There is a releases API, but this uses the tags API try: author, repo = author_repo_from_github_url(url) url = GithubAPI.REPO_RELEASES.format(owner=author, repo=repo) data = requests.get(url, params={"ref": branch}).json() except Exception as e: raise GithubAPIReleasesNotFound(str(e)) if isinstance(data, dict): # result is usually a list, unless api call fails if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited # let's fix api urls for idx, r in enumerate(data): data[idx]["tarball_url"] = GithubUrls.DOWNLOAD_TARBALL.format( author=author, repo=repo, branch=r["name"]) data[idx]["zipball_url"] = GithubUrls.DOWNLOAD.format(author=author, repo=repo, branch=r["name"]) data[idx].pop('node_id') return data
def get_skill_json_from_github_api(url: str, branch: Optional[str] = None) -> dict: """ Get skill.json file contents for the specified repository @param url: Repository URL to query @param branch: Optional branch spec, otherwise default branch will be used @return: data parsed from skill.json """ author, repo = author_repo_from_github_url(url) for dst in GITHUB_JSON_FILES: try: data = get_file_from_github_api(url, dst.format(repo=repo), branch) except GithubAPIFileNotFound: continue if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited if data.get("content"): content = data["content"] if data["encoding"] == "base64": json_data = base64.b64decode(content).decode("utf-8") else: # TODO Raise UnknownEncoding? json_data = content return json.loads(json_data) raise GithubAPIFileNotFound
def get_manifest_from_github_api(url: str, branch: Optional[str] = None) -> dict: """ Get requirements specified in the repository manifest file @param url: Repository URL to query @param branch: Optional branch spec, otherwise default branch will be used @return: dict parsed requirements """ author, repo = author_repo_from_github_url(url) content = None for dst in GITHUB_MANIFEST_FILES: try: data = get_file_from_github_api(url, dst.format(repo=repo), branch) except GithubAPIFileNotFound: continue if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited if data.get("content"): content = data["content"] if data["encoding"] == "base64": content = base64.b64decode(content).decode("utf-8") # TODO Raise UnknownEncoding? if not content: raise GithubAPIFileNotFound return validate_manifest(content)
def get_skill_requirements_from_github_api(url: str, branch: Optional[str] = None ) -> list: """ Get Skill requirements from text files in the specified repository @param url: Repository URL to query @param branch: Optional branch spec, otherwise default branch will be used @return: List parsed skill requirements """ author, repo = author_repo_from_github_url(url) content = None for dst in GITHUB_SKILL_REQUIREMENTS_FILES: try: data = get_file_from_github_api(url, dst.format(repo=repo), branch) except GithubAPIFileNotFound: continue if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited if data.get("content"): content = data["content"] if data["encoding"] == "base64": content = base64.b64decode(content).decode("utf-8") # TODO Raise UnknownEncoding? if not content: raise GithubAPIFileNotFound return [ t for t in content.split("\n") if t.strip() and not t.strip().startswith("#") ]
def api_zip_url_from_github_url(url: str, branch: Optional[str] = None, token: Optional[str] = None) -> str: """ Get an API URL to download the repository as a zip archive https://docs.github.com/en/rest/reference/repos#download-a-repository-archive-zip @param url: Repository URL, optionally containing a branch spec @param branch: Optional branch spec, otherwise branch from `url` will be used @param token: Optional GitHub token to include with request @return: GitHub API URL to query for a zip archive """ # TODO: `token` is not used? # specific file try: url = blob2raw(url) if requests.get(url).ok: return url except GithubInvalidUrl: pass # full git repo branch = branch or get_branch_from_github_url(url) owner, repo = author_repo_from_github_url(url) url = GithubAPI.REPO_ZIP.format(owner=owner, branch=branch, repo=repo) if requests.get(url).status_code == 200: return url raise GithubInvalidUrl
def get_readme_url_from_github_api(url: str, branch: Optional[str] = None) -> str: """ Get the readme file url for the specified repository https://docs.github.com/en/rest/reference/repos#get-a-repository-readme @param url: Repository URL @param branch: Optional branch to query, otherwise default branch will be used @return: url of repository README file """ author, repo = author_repo_from_github_url(url) default_url = GithubAPI.REPO_README.format(owner=author, repo=repo) try: data = requests.get(default_url, params={"ref": branch}).json() if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited return data["html_url"] except Exception as e: pass # check files individually for dst in GITHUB_README_FILES: try: data = get_file_from_github_api(url, dst, branch) except GithubAPIFileNotFound: continue if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited if data.get("html_url"): return data["html_url"] raise GithubAPIReadmeNotFound
def get_readme_from_github_api(url, branch=None): author, repo = author_repo_from_github_url(url) default_url = GithubAPI.REPO_README.format(owner=author, repo=repo) try: data = requests.get(default_url, params={"ref": branch}).json() if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited readme = data["content"] if data["encoding"] == "base64": return base64.b64decode(readme).decode("utf-8") # TODO Raise UnknownEncoding? return readme except GithubAPIRateLimited: raise except Exception as e: pass # check files individually for dst in GITHUB_README_FILES: try: data = get_file_from_github_api(url, dst, branch) except GithubAPIFileNotFound: continue if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited readme = data.get("content") if readme: if data["encoding"] == "base64": return base64.b64decode(readme).decode("utf-8") # TODO Raise UnknownEncoding? return readme raise GithubAPIReadmeNotFound
def get_license_data_from_github_api(url, branch=None): author, repo = author_repo_from_github_url(url) url = GithubAPI.REPO_LICENSE.format(owner=author, repo=repo) try: data = requests.get(url, params={"ref": branch}).json() except Exception as e: raise GithubAPILicenseNotFound if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited return data
def get_file_from_github_api(url, filepath, branch=None): author, repo = author_repo_from_github_url(url) branch = branch or get_main_branch_from_github_api(url) url = GithubAPI.REPO_FILE.format(owner=author, repo=repo, file=filepath) data = requests.get(url, params={"ref": branch}).json() if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited if data.get("message", "") != 'Not Found': return data raise GithubAPIFileNotFound
def get_logo_url_from_github_api(url, branch=None): author, repo = author_repo_from_github_url(url) for dst in GITHUB_LOGO_FILES: try: data = get_file_from_github_api(url, dst.format(repo=repo), branch) except GithubAPIFileNotFound: continue if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited if data.get("download_url"): return data["download_url"] raise GithubAPIFileNotFound
def get_desktop_from_github_api(url, branch=None): author, repo = author_repo_from_github_url(url) for dst in GITHUB_DESKTOP_FILES: try: data = get_file_from_github_api(url, dst.format(repo=repo), branch) except GithubAPIFileNotFound: continue if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited if data.get("content"): readme = data["content"] if data["encoding"] == "base64": return base64.b64decode(readme).decode("utf-8") # TODO Raise UnknownEncoding? return readme raise GithubAPIFileNotFound
def api_zip_url_from_github_url(url, branch=None, token=None): # specific file try: url = blob2raw(url) if requests.get(url).status_code == 200: return url except GithubInvalidUrl: pass # full git repo branch = branch or get_branch_from_github_url(url) owner, repo = author_repo_from_github_url(url) url = GithubAPI.REPO_ZIP.format(owner=owner, branch=branch, repo=repo) if requests.get(url).status_code == 200: return url raise GithubInvalidUrl
def get_license_data_from_github_api(url: str, branch: Optional[str] = None) -> dict: """ Get license data for the repository at the given URL and branch https://docs.github.com/en/rest/reference/licenses#get-the-license-for-a-repository @param url: Repository URL @param branch: Optional branch spec, otherwise default branch will be used @return: dict license data """ author, repo = author_repo_from_github_url(url) url = GithubAPI.REPO_LICENSE.format(owner=author, repo=repo) try: data = requests.get(url, params={"ref": branch}).json() except Exception as e: raise GithubAPILicenseNotFound if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited return data
def get_manifest_from_github_api(url, branch=None): author, repo = author_repo_from_github_url(url) content = None for dst in GITHUB_MANIFEST_FILES: try: data = get_file_from_github_api(url, dst.format(repo=repo), branch) except GithubAPIFileNotFound: continue if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited if data.get("content"): content = data["content"] if data["encoding"] == "base64": content = base64.b64decode(content).decode("utf-8") # TODO Raise UnknownEncoding? if not content: raise GithubAPIFileNotFound return validate_manifest(content)
def get_skill_json_from_github_api(url, branch=None): author, repo = author_repo_from_github_url(url) for dst in GITHUB_JSON_FILES: try: data = get_file_from_github_api(url, dst.format(repo=repo), branch) except GithubAPIFileNotFound: continue if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited if data.get("content"): content = data["content"] if data["encoding"] == "base64": json_data = base64.b64decode(content).decode("utf-8") else: # TODO Raise UnknownEncoding? json_data = content return json.loads(json_data) raise GithubAPIFileNotFound
def get_logo_url_from_github_api(url: str, branch: Optional[str] = None) -> str: """ Get skill logo file URL for the specified repository @param url: Repository URL to query @param branch: Optional branch spec, otherwise default branch will be used @return: URL of skill logo """ author, repo = author_repo_from_github_url(url) for dst in GITHUB_LOGO_FILES: try: data = get_file_from_github_api(url, dst.format(repo=repo), branch) except GithubAPIFileNotFound: continue if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited if data.get("download_url"): return data["download_url"] raise GithubAPIFileNotFound
def get_skill_requirements_from_github_api(url, branch=None): author, repo = author_repo_from_github_url(url) content = None for dst in GITHUB_SKILL_REQUIREMENTS_FILES: try: data = get_file_from_github_api(url, dst.format(repo=repo), branch) except GithubAPIFileNotFound: continue if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited if data.get("content"): content = data["content"] if data["encoding"] == "base64": content = base64.b64decode(content).decode("utf-8") # TODO Raise UnknownEncoding? if not content: raise GithubAPIFileNotFound return [t for t in content.split("\n") if t.strip() and not t.strip().startswith("#")]
def get_repo_releases_from_github_api(url, branch=None): try: author, repo = author_repo_from_github_url(url) url = GithubAPI.REPO_RELEASES.format(owner=author, repo=repo) data = requests.get(url, params={"ref": branch}).json() except Exception as e: raise GithubAPIReleasesNotFound(str(e)) if isinstance(data, dict): # result is usually a list, unless api call fails if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited # let's fix api urls for idx, r in enumerate(data): data[idx]["tarball_url"] = GithubUrls.DOWNLOAD_TARBALL.format( author=author, repo=repo, branch=r["name"]) data[idx]["zipball_url"] = GithubUrls.DOWNLOAD.format( author=author, repo=repo, branch=r["name"]) data[idx].pop('node_id') return data
def get_file_from_github_api(url: str, filepath: str, branch: Optional[str] = None) -> dict: """ Get information for a file in a repository. https://docs.github.com/en/rest/reference/repos#get-repository-content @param url: Repository URL @param filepath: path to a file in the repository @param branch: Optional branch to query, otherwise branch from `url` will be used @return: parsed API data """ author, repo = author_repo_from_github_url(url) branch = branch or get_main_branch_from_github_api(url) url = GithubAPI.REPO_FILE.format(owner=author, repo=repo, file=filepath) data = requests.get(url, params={"ref": branch}).json() if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited if data.get("message", "") != 'Not Found': return data raise GithubAPIFileNotFound
def get_readme_url_from_github_api(url, branch=None): author, repo = author_repo_from_github_url(url) default_url = GithubAPI.REPO_README.format(owner=author, repo=repo) try: data = requests.get(default_url, params={"ref": branch}).json() if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited return data["html_url"] except Exception as e: pass # check files individually for dst in GITHUB_README_FILES: try: data = get_file_from_github_api(url, dst, branch) except GithubAPIFileNotFound: continue if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited if data.get("html_url"): return data["html_url"] raise GithubAPIReadmeNotFound
def get_desktop_from_github_api(url: str, branch: Optional[str] = None) -> str: """ Get skill.desktop file contents for the specified repository @param url: Repository URL to query @param branch: Optional branch spec, otherwise default branch will be used @return: skill.desktop string contents """ author, repo = author_repo_from_github_url(url) for dst in GITHUB_DESKTOP_FILES: try: data = get_file_from_github_api(url, dst.format(repo=repo), branch) except GithubAPIFileNotFound: continue if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited if data.get("content"): readme = data["content"] if data["encoding"] == "base64": return base64.b64decode(readme).decode("utf-8") # TODO Raise UnknownEncoding? return readme raise GithubAPIFileNotFound
def get_readme_from_github_api(url: str, branch: Optional[str] = None) -> str: """ Get the readme file contents for the specified repository @param url: Repository URL @param branch: Optional branch to query, otherwise default branch will be used @return: contents of repository README file """ author, repo = author_repo_from_github_url(url) default_url = GithubAPI.REPO_README.format(owner=author, repo=repo) try: data = requests.get(default_url, params={"ref": branch}).json() if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited readme = data["content"] if data["encoding"] == "base64": return base64.b64decode(readme).decode("utf-8") # TODO Raise UnknownEncoding? return readme except GithubAPIRateLimited: raise except Exception as e: pass # check files individually for dst in GITHUB_README_FILES: try: data = get_file_from_github_api(url, dst, branch) except GithubAPIFileNotFound: continue if "API rate limit exceeded" in data.get("message", ""): raise GithubAPIRateLimited readme = data.get("content") if readme: if data["encoding"] == "base64": return base64.b64decode(readme).decode("utf-8") # TODO Raise UnknownEncoding? return readme raise GithubAPIReadmeNotFound
def build_store_model(self, store_id): store = self.osm_manager.get_appstore(store_id) skills_model = [] for skill in store: skill_icon = skill.skill_icon or self.default_icon if not validators.url(skill.skill_icon): # TODO osm should transform the relative paths! # discard bad paths for now skill_icon = self.default_icon author, repo = author_repo_from_github_url(skill.url) desc = skill.skill_short_description or \ skill.skill_description or \ f"{repo} by {author}" skills_model.append({ "title": skill.skill_name or repo, "description": desc, "logo": skill.json.get("logo") or skill_icon, "author": skill.skill_author, "category": skill.skill_category, "url": skill.url }) return skills_model
def get_local_skills(parse_github=False, skiplist=None): skills = get_skills_folder() skiplist = skiplist or [] folders = listdir(skills) for fold in folders: path = join(skills, fold) if not isdir(path) or fold in skiplist: continue skill = { "appstore": "InstalledSkills", "appstore_url": skills, "skill_id": fold, "foldername": fold, "requirements": { "python": [], "system": [], "skill": [] } } # if installed by msm/osm will obey this convention if "." in fold: try: repo, author = fold.split(".") skill["skillname"] = repo skill["authorname"] = author skill["url"] = f'https://github.com/{author}/{repo}' except: # TODO replace with some clever check ? pass # parse git info gitinfo = join(path, ".git/config") if isfile(gitinfo): with open(gitinfo) as f: for l in f.readlines(): if l.strip().startswith("url ="): skill["url"] = l.split("url =")[-1].strip() skill["authorname"], skill["skillname"] = \ author_repo_from_github_url(skill["url"]) if l.strip().startswith("[branch "): skill["branch"] = l.split("branch")[-1]\ .replace('"', "").strip() for rtdir, foldrs, files in walk(join(skills, fold)): for f in files: if f in GITHUB_JSON_FILES: with open(join(rtdir, f)) as fi: skill_meta = json.load(fi) skill = merge_dict(skill, skill_meta, merge_lists=True) elif f in GITHUB_README_FILES: with open(join(rtdir, f)) as fi: readme = readme_to_json(fi.read()) skill = merge_dict(skill, readme, new_only=True, merge_lists=True) elif f in GITHUB_DESKTOP_FILES: skill['desktopFile'] = True elif f in GITHUB_ICON_FILES: skill["icon"] = join(rtdir, f) elif f in GITHUB_LICENSE_FILES: with open(join(rtdir, f)) as fi: lic = fi.read() skill["license"] = parse_license_type(lic) elif f in GITHUB_LOGO_FILES: skill["logo"] = join(rtdir, f) elif f in GITHUB_MANIFEST_FILES: with open(join(rtdir, f)) as fi: manifest = validate_manifest(fi.read()) skill["requirements"]["python"] += manifest.get( "python") or [] skill["requirements"]["system"] += manifest.get( "system") or [] skill["requirements"]["skill"] += manifest.get( "skill") or [] elif f in GITHUB_REQUIREMENTS_FILES: with open(join(rtdir, f)) as fi: reqs = [r for r in fi.read().split("\n") if r.strip()] skill["requirements"]["python"] += reqs elif f in GITHUB_SKILL_REQUIREMENTS_FILES: with open(join(rtdir, f)) as fi: reqs = [r for r in fi.read().split("\n") if r.strip()] skill["requirements"]["skill"] += reqs yield SkillEntry.from_json(skill, parse_github=parse_github)
def get_skill_data_from_directory(skill_dir: str): """ Parse the specified skill directory and return a dict representation of a SkillEntry. @param skill_dir: path to skill directory @return: dict parsed skill data """ skills, fold = skill_dir.rsplit('/', 1) skill_data = { "appstore": "InstalledSkills", "appstore_url": skills, "skill_id": fold, "requirements": { "python": [], "system": {}, "skill": [] } } # if installed by msm/osm will obey this convention if "." in fold: try: repo, author = fold.split(".") skill_data["skillname"] = repo skill_data["authorname"] = author skill_data["url"] = f'https://github.com/{author}/{repo}' except: # TODO replace with some clever check ? pass # parse git info gitinfo = join(skill_dir, ".git/config") if isfile(gitinfo): with open(gitinfo) as f: for l in f.readlines(): if l.strip().startswith("url ="): skill_data["url"] = l.split("url =")[-1].strip() skill_data["authorname"], skill_data["skillname"] = \ author_repo_from_github_url(skill_data["url"]) if l.strip().startswith("[branch "): skill_data["branch"] = l.split("branch")[-1] \ .replace('"', "").strip() # parse skill files for root_dir, _, files in walk(skill_dir): for f in files: if f in GITHUB_JSON_FILES: # skill.json with open(join(root_dir, f)) as fi: skill_meta = json.load(fi) skill_data = merge_dict(skill_data, skill_meta, merge_lists=True) elif f in GITHUB_README_FILES: with open(join(root_dir, f)) as fi: readme = readme_to_json(fi.read()) skill_data = merge_dict(skill_data, readme, new_only=True, merge_lists=True) elif f in GITHUB_DESKTOP_FILES: skill_data['desktopFile'] = True elif f in GITHUB_ICON_FILES: skill_data["icon"] = join(root_dir, f) elif f in GITHUB_LICENSE_FILES: with open(join(root_dir, f)) as fi: lic = fi.read() skill_data["license"] = parse_license_type(lic) elif f in GITHUB_LOGO_FILES: skill_data["logo"] = join(root_dir, f) elif f in GITHUB_MANIFEST_FILES: with open(join(root_dir, f)) as fi: manifest = validate_manifest(fi.read()).get( "dependencies", {}) skill_data["requirements"]["python"] += \ manifest.get("python") or [] skill_data["requirements"]["system"] = \ merge_dict(skill_data["requirements"]["system"], manifest.get("system") or {}, merge_lists=True) skill_data["requirements"]["skill"] += \ manifest.get("skill") or [] elif f in GITHUB_REQUIREMENTS_FILES: with open(join(root_dir, f)) as fi: reqs = [r for r in fi.read().split("\n") if r.strip()] skill_data["requirements"]["python"] += reqs elif f in GITHUB_SKILL_REQUIREMENTS_FILES: with open(join(root_dir, f)) as fi: reqs = [r for r in fi.read().split("\n") if r.strip()] skill_data["requirements"]["skill"] += reqs # de-dupe requirements skill_data["requirements"]["python"] = \ list(set(skill_data["requirements"]["python"])) skill_data["requirements"]["skill"] = \ list(set(skill_data["requirements"]["skill"])) skill_data['foldername'] = fold # Override what the config specifies skill_data['authorname'] = skill_data.get('authorname') or "local" return skill_data
def get_skill_data(url, branch=None): author, repo = author_repo_from_github_url(url) explicit_branch = branch if not branch: try: branch = get_branch_from_github_url(url) explicit_branch = branch except GithubInvalidBranch: branch = get_main_branch(url) data = { "authorname": author, "foldername": repo, "url": normalize_github_url(url), "branch": branch, "license": "unknown", "tags": [] } # augment with repo data try: api_data = get_repo_data(url) # only replace branch if not found by prev methods branch = branch or api_data['default_branch'] LOG.debug(f"default branch extracted from github api: {branch}") data["branch"] = branch data["short_description"] = api_data['description'] data["foldername"] = api_data["name"] data["last_updated"] = api_data['updated_at'] data["url"] = api_data["html_url"] data["authorname"] = api_data["owner"]["login"] branch = branch or api_data['default_branch'] if "license" in data: data["license"] = api_data["license"]["key"] except: pass # augment with releases data try: releases = get_releases(url) # search release that matches branch if branch: for r in releases: if r["name"] == branch or r["commit"]["sha"] == branch: data["version"] = r["name"] # data["download_url"] = r["tarball_url"] break # just pick latest release if branch not defined elif len(releases) > 0: data["version"] = data["branch"] = branch = releases[0]["name"] # data["download_url"] = releases[0]["tarball_url"] LOG.debug(f"default branch extracted from github releases:" f" {branch}") except GithubSkillEntryError as e: pass # augment with readme data try: data = merge_dict(data, get_readme_json(url, branch), merge_lists=True, skip_empty=True, no_dupes=True) except GithubSkillEntryError as e: pass # find logo try: data["logo"] = get_logo_url(url, branch) except GithubFileNotFound as e: pass # augment with requirements data["requirements"] = get_requirements_json(url, branch) # augment with android data data["android"] = get_android_json(url, branch) # augment with desktop data try: data["desktop"] = get_desktop_json(url, branch) data["desktopFile"] = True except GithubFileNotFound: data["desktopFile"] = False # augment tags if is_viral(data["license"]): data["tags"].append("viral-license") elif is_permissive(data["license"]): data["tags"].append("permissive-license") elif "unknown" in data["license"]: data["tags"].append("no-license") try: data["license"] = get_license_type(url, branch) except GithubLicenseNotFound: pass # augment with json data # this should take precedence over everything else try: data = merge_dict(data, get_skill_json(url, branch), merge_lists=True, skip_empty=True, no_dupes=True) branch = data.get("branch") LOG.debug(f"default branch extracted from json: {branch}") except GithubFileNotFound: pass if explicit_branch: LOG.debug(f"explicit branch in url selected: {branch}") data["branch"] = explicit_branch if data.get("download_url"): # TODO deprecate this warning LOG.warning("the skill.json provides a download url, this used to be " "supported but it is dangerous and OSM will discard it, " "skill.json should only define github url and branch, " "the download_url will be auto-generated") LOG.debug(f"discarded url: {data['download_url']}") data.pop("download_url") return data
def get_skill_from_api(url, branch=None, strict=False): data = {} # extract branch from .json, should branch take precedence? # i think so because user explicitly requested it branch = get_branch_from_skill_json_github_api(url, branch) try: api_data = get_repo_data_from_github_api(url, branch) data["branch"] = branch = api_data['default_branch'] data["short_description"] = api_data['description'] data["license"] = api_data["license"]["key"] data["foldername"] = api_data["name"] data["last_updated"] = api_data['updated_at'] data["url"] = api_data["html_url"] data["authorname"] = api_data["owner"]["login"] except GithubAPIException as e: LOG.error("Failed to retrieve repo data from github api") raise try: releases = get_repo_releases_from_github_api(url, branch) if branch: for r in releases: if r["name"] == branch or r["commit"]["sha"] == branch: data["version"] = r["name"] #data["download_url"] = r["tarball_url"] break else: data["version"] = releases[0]["name"] #data["download_url"] = releases[0]["tarball_url"] except GithubAPIException as e: LOG.error("Failed to retrieve releases data from github api") if strict: raise GithubAPIReleasesNotFound # augment with readme data try: data = merge_dict(data, get_readme_json_from_api(url, branch), merge_lists=True, skip_empty=True, no_dupes=True) except GithubAPIReadmeNotFound: pass data["requirements"] = get_requirements_json_from_github_api(url, branch) # find logo try: data["logo"] = get_logo_url_from_github_api(url, branch) except GithubAPIFileNotFound as e: pass # find icon try: data["icon"] = icon = get_icon_url_from_github_api(url, branch) except GithubAPIFileNotFound: icon = None # augment with android data try: data["android"] = get_android_json_from_github_api(url, branch) except GithubAPIFileNotFound: # best guess or throw exception? author, repo = author_repo_from_github_url(url) data["android"] = { 'android_icon': icon, 'android_name': skill_name_from_github_url(url), 'android_handler': '{repo}.{author}.home'.format(repo=repo, author=author.lower())} # augment with desktop data try: data["desktop"] = get_desktop_json_from_github_api(url, branch) data["desktopFile"] = True except GithubFileNotFound: data["desktopFile"] = False # augment tags if "tags" not in data: data["tags"] = [] if is_viral(data["license"]): data["tags"].append("viral-license") elif is_permissive(data["license"]): data["tags"].append("permissive-license") elif "unknown" in data["license"]: data["tags"].append("no-license") # augment with json data # this should take precedence over everything else try: data = merge_dict(data, get_skill_json_from_github_api(url, branch), merge_lists=True, skip_empty=True, no_dupes=True) except GithubFileNotFound: pass return data