def get_versions_by_regex(url, regex, project, insecure=False): ''' For the provided url, return all the version retrieved via the specified regular expression. ''' try: req = BaseBackend.call_url(url, insecure=insecure) except Exception as err: _log.debug('%s ERROR: %s' % (project.name, str(err))) raise AnityaPluginException( 'Could not call : "%s" of "%s", with error: %s' % (url, project.name, str(err))) if not isinstance(req, six.string_types): req = req.text return get_versions_by_regex_for_text(req, url, regex, project)
def get_versions(cls, project): """ Method called to retrieve all the versions (that can be found) of the projects provided, project that relies on the backend of this plugin. :arg Project project: a :class:`anitya.db.models.Project` object whose backend corresponds to the current plugin. :return: a list of all the possible releases found :return type: list :raise AnityaPluginException: a :class:`anitya.lib.exceptions.AnityaPluginException` exception when the versions cannot be retrieved correctly """ url = cls.get_version_url(project) last_change = project.get_time_last_created_version() try: req = cls.call_url(url, last_change=last_change, insecure=project.insecure) except Exception as err: raise AnityaPluginException( 'Could not call : "%s" of "%s", with error: %s' % (url, project.name, str(err))) versions = [] if not isinstance(req, six.string_types): # Not modified if req.status_code == 304: return versions req = req.text try: regex = REGEX % {"name": project.name.replace("+", r"\+")} versions = get_versions_by_regex_for_text(req, url, regex, project) except AnityaPluginException: versions = get_versions_by_regex_for_text(req, url, DEFAULT_REGEX, project) return versions
def get_versions(cls, project): """Method called to retrieve all the versions (that can be found) of the projects provided, project that relies on the backend of this plugin. :arg Project project: a :class:`anitya.db.models.Project` object whose backend corresponds to the current plugin. :return: a list of all the possible releases found :return type: list :raise AnityaPluginException: a :class:`anitya.lib.exceptions.AnityaPluginException` exception when the versions cannot be retrieved correctly """ url = cls.get_version_url(project) if not url: raise AnityaPluginException("Project %s was incorrectly set up." % project.name) return get_versions_by_regex(url, REGEX, project)
def get_versions(cls, project): ''' Method called to retrieve all the versions (that can be found) of the projects provided, project that relies on the backend of this plugin. :arg Project project: a :class:`anitya.db.models.Project` object whose backend corresponds to the current plugin. :return: a list of all the possible releases found :return type: list :raise AnityaPluginException: a :class:`anitya.lib.exceptions.AnityaPluginException` exception when the versions cannot be retrieved correctly ''' url = cls.get_version_url(project) if not url: raise AnityaPluginException( "Aritfact needs to be in format groupId:artifactId") return get_versions_by_regex(url, VERSION_REGEX, project)
def get_versions(cls, project): """Method called to retrieve all the versions (that can be found) of the projects provided, project that relies on the backend of this plugin. :arg Project project: a :class:`anitya.db.models.Project` object whose backend corresponds to the current plugin. :return: a list of all the possible releases found :return type: list :raise AnityaPluginException: a :class:`anitya.lib.exceptions.AnityaPluginException` exception when the versions cannot be retrieved correctly """ namespace, repo = cls.get_namespace_repo(project) url = project.get_version_url() git_tag_request = requests.get(url) if git_tag_request.status_code == 404: raise AnityaPluginException( 'Could not call : "%s" of "%s", with error: %s' % (url, project.name, git_tag_request.status_code)) soup = BeautifulSoup(git_tag_request.content, "html.parser") def is_release_tag_link(tag): link_prefix = f"/p/{namespace}/{repo}/ci/" return (tag.has_attr("href") and tag.attrs["href"].startswith(link_prefix) and len(tag.contents) == 1) release_tags = [] for release_tag in soup.find_all(is_release_tag_link): release_tag_text = release_tag.contents[0] release_tags.append(release_tag_text) return release_tags
def get_versions(cls, project): """ Method called to retrieve all the versions (that can be found) of the projects provided, project that relies on the backend of this plugin. :arg Project project: a :class:`anitya.db.models.Project` object whose backend corresponds to the current plugin. :return: a list of all the possible releases found :return type: list :raise AnityaPluginException: a :class:`anitya.lib.exceptions.AnityaPluginException` exception when the versions cannot be retrieved correctly """ url = cls.get_version_url(project) versions = [] versions = _get_versions(url) if not versions: raise AnityaPluginException("No versions found for %s" % project.name.lower()) return versions
def get_versions_by_regex(url, regex, project, insecure=False): """For the provided url, return all the version retrieved via the specified regular expression. """ last_change = project.get_time_last_created_version() try: req = BaseBackend.call_url(url, last_change=last_change, insecure=insecure) except Exception as err: _log.debug("%s ERROR: %s" % (project.name, str(err))) raise AnityaPluginException( 'Could not call : "%s" of "%s", with error: %s' % (url, project.name, str(err)) ) if not isinstance(req, six.string_types): # Not modified if req.status_code == 304: return [] req = req.text return get_versions_by_regex_for_text(req, url, regex, project)
def get_versions(cls, project): ''' Method called to retrieve all the versions (that can be found) of the projects provided, project that relies on the backend of this plugin. :arg Project project: a :class:`anitya.db.models.Project` object whose backend corresponds to the current plugin. :return: a list of all the possible releases found :return type: list :raise AnityaPluginException: a :class:`anitya.lib.exceptions.AnityaPluginException` exception when the versions cannot be retrieved correctly ''' url = project.version_url try: req = cls.call_url(url, insecure=project.insecure) except Exception as err: raise AnityaPluginException( 'Could not call : "%s" of "%s", with error: %s' % ( url, project.name, str(err))) versions = None if not isinstance(req, six.string_types): req = req.text try: regex = REGEX % {'name': project.name.replace('+', '\+')} versions = get_versions_by_regex_for_text( req, url, regex, project) except AnityaPluginException: versions = get_versions_by_regex_for_text( req, url, DEFAULT_REGEX, project) return versions
def test_existing_project_check_failure(self, mock_check): """Assert that when an existing project fails a version check nothing happens""" mock_check.side_effect = AnityaPluginException() project = Project( name="ImageMetaTag", homepage="https://pypi.python.org/pypi/ImageMetaTag", ecosystem_name="pypi", backend="PyPI", ) self.session.add(project) self.session.commit() event = sseclient.Event( data="{" '"name": "ImageMetaTag",' '"platform": "PyPi",' '"version": "0.6.9",' '"package_manager_url": "https://pypi.org/project/ImageMetaTag/"' "}") self.assertEqual(1, self.session.query(Project).count()) self.client.process_message(event) self.assertEqual(1, self.session.query(Project).count()) project = self.session.query(Project).first() self.assertIs(project.latest_version, None)
def parse_json(json, project): """Function for parsing json response Args: json (dict): Json dictionary to parse. project (:obj:`anitya.db.models.Project`): Project object whose backend corresponds to the current plugin. Returns: :obj:`list`: A list of all the possible releases found. Raises: AnityaPluginException: A :obj:`anitya.lib.exceptions.AnityaPluginException` exception when the versions cannot be retrieved correctly. RateLimitException: A :obj:`anitya.lib.exceptions.RateLimitException` exception when rate limit threshold is reached. """ global reset_time # We need to check limit first, # because exceeding the limit will also return error try: limit = json["data"]["rateLimit"]["limit"] remaining = json["data"]["rateLimit"]["remaining"] reset_time = json["data"]["rateLimit"]["resetAt"] _log.debug("Github API ratelimit remains %s, will reset at %s UTC" % (remaining, reset_time)) if (remaining / limit) <= RATE_LIMIT_THRESHOLD: raise RateLimitException(reset_time) except KeyError: _log.info("Github API ratelimit key is missing. Checking for errors.") if "errors" in json: error_str = "" for error in json["errors"]: error_str += '"%s": "%s"\n' % (error["type"], error["message"]) raise AnityaPluginException( "%s: Server responded with following errors\n%s" % (project.name, error_str)) if project.releases_only: json_data = json["data"]["repository"]["releases"] else: json_data = json["data"]["repository"]["refs"] total_count = json_data["totalCount"] if project.releases_only: _log.debug("Received %s releases for %s" % (total_count, project.name)) else: _log.debug("Received %s tags for %s" % (total_count, project.name)) versions = [] for edge in json_data["edges"]: version = {"cursor": edge["cursor"]} if project.releases_only: hook = edge["node"]["tag"] else: hook = edge["node"] version["version"] = hook["name"] version["commit_url"] = hook["target"]["commitUrl"] versions.append(version) return versions
def call_url(self, url, insecure=False): """ Dedicated method to query a URL. It is important to use this method as it allows to query them with a defined user-agent header thus informing the projects we are querying what our intentions are. Attributes: url (str): The url to request (get). insecure (bool, optional): Flag for secure/insecure connection. Defaults to False. Returns: In case of FTP url it returns binary encoded string otherwise :obj:`requests.Response` object. """ headers = REQUEST_HEADERS.copy() if "*" in url: url = self.expand_subdirs(url) if url.startswith("ftp://") or url.startswith("ftps://"): socket.setdefaulttimeout(30) req = urllib.Request(url) req.add_header("User-Agent", headers["User-Agent"]) req.add_header("From", headers["From"]) try: # Ignore this bandit issue, the url is checked above resp = urllib.urlopen(req) # nosec content = resp.read().decode() except URLError as e: raise AnityaPluginException( 'Could not call "%s" with error: %s' % (url, e.reason) ) except UnicodeDecodeError: raise AnityaPluginException( "FTP response cannot be decoded with UTF-8: %s" % url ) return content else: # Works around https://github.com/kennethreitz/requests/issues/2863 # Currently, requests does not start new TCP connections based on # TLS settings. This means that if a connection is ever started to # a host with `verify=False`, further requests to that # (scheme, host, port) combination will also be insecure, even if # `verify=True` is passed to requests. # # This starts a new session which is immediately discarded when the # request is insecure. We don't get to pool connections for these # requests, but it stops us from making insecure requests by # accident. This can be removed in requests-3.0. if insecure: with requests.Session() as r_session: resp = r_session.get(url, headers=headers, timeout=60, verify=False) else: resp = http_session.get(url, headers=headers, timeout=60, verify=True) return resp
def get_versions(cls, project): ''' Method called to retrieve all the versions (that can be found) of the projects provided, project that relies on the backend of this plugin. Args: project (:obj:`anitya.db.models.Project`): Project object whose backend corresponds to the current plugin. Returns: :obj:`list`: A list of all the possible releases found Raises: AnityaPluginException: A :obj:`anitya.lib.exceptions.AnityaPluginException` exception when the versions cannot be retrieved correctly ''' owner = None repo = None if project.version_url: url = project.version_url elif project.homepage.startswith('https://github.com'): url = project.homepage.replace('https://github.com/', '') if url.endswith('/'): url = url[:-1] else: raise AnityaPluginException('Project %s was incorrectly set-up' % project.name) try: (owner, repo) = url.split('/') except ValueError: raise AnityaPluginException("""Project {} was incorrectly set-up. Can\'t parse owner and repo.""".format(project.name)) query = prepare_query(owner, repo) try: headers = REQUEST_HEADERS.copy() token = config['GITHUB_ACCESS_TOKEN'] if token: headers['Authorization'] = 'bearer %s' % token resp = http_session.post(API_URL, json={'query': query}, headers=headers, timeout=60, verify=True) except Exception as err: _log.debug('%s ERROR: %s' % (project.name, str(err))) raise AnityaPluginException( 'Could not call : "%s" of "%s", with error: %s' % (API_URL, project.name, str(err))) if resp.ok: json = resp.json() else: raise AnityaPluginException( '%s: Server responded with status "%s": "%s"' % (project.name, resp.status_code, resp.reason)) versions = parse_json(json, project) if len(versions) == 0: raise AnityaPluginException('%s: No upstream version found.' % (project.name)) return versions
def get_versions(cls, project): """ Method called to retrieve all the versions (that can be found) of the projects provided, project that relies on the backend of this plugin. Args: project (:obj:`anitya.db.models.Project`): Project object whose backend corresponds to the current plugin. Returns: :obj:`list`: A list of all the possible releases found Raises: AnityaPluginException: A :obj:`anitya.lib.exceptions.AnityaPluginException` exception when the versions cannot be retrieved correctly """ owner = None repo = None url = cls.get_version_url(project) if url: url = url.replace("https://github.com/", "") url = url.replace("/tags", "") else: raise AnityaPluginException("Project %s was incorrectly set-up" % project.name) try: (owner, repo) = url.split("/") except ValueError: raise AnityaPluginException("""Project {} was incorrectly set-up. Can\'t parse owner and repo.""".format(project.name)) query = prepare_query(owner, repo, project.releases_only) try: headers = REQUEST_HEADERS.copy() token = config["GITHUB_ACCESS_TOKEN"] if token: headers["Authorization"] = "bearer %s" % token resp = http_session.post(API_URL, json={"query": query}, headers=headers, timeout=60, verify=True) except Exception as err: _log.debug("%s ERROR: %s" % (project.name, str(err))) raise AnityaPluginException( 'Could not call : "%s" of "%s", with error: %s' % (API_URL, project.name, str(err))) if resp.ok: json = resp.json() elif resp.status_code == 403: _log.info("Github API ratelimit reached.") raise RateLimitException(reset_time) else: raise AnityaPluginException( '%s: Server responded with status "%s": "%s"' % (project.name, resp.status_code, resp.reason)) versions = parse_json(json, project) _log.debug(f"Retrieved versions: {versions}") if len(versions) == 0: raise AnityaPluginException("%s: No upstream version found." % (project.name)) return versions