def get_versions_by_regex(url, regex, project, insecure=False):
''' For the provided url, return all the version retrieved via the
specified regular expression.
'''
try:
req = BaseBackend.call_url(url, insecure=insecure)
except Exception as err:
_log.debug('%s ERROR: %s' % (project.name, str(err)))
raise AnityaPluginException(
'Could not call : "%s" of "%s", with error: %s' %
(url, project.name, str(err)))
if not isinstance(req, six.string_types):
req = req.text
return get_versions_by_regex_for_text(req, url, regex, project)
def get_versions(cls, project):
""" Method called to retrieve all the versions (that can be found)
of the projects provided, project that relies on the backend of
this plugin.
:arg Project project: a :class:`anitya.db.models.Project` object whose backend
corresponds to the current plugin.
:return: a list of all the possible releases found
:return type: list
:raise AnityaPluginException: a
:class:`anitya.lib.exceptions.AnityaPluginException` exception
when the versions cannot be retrieved correctly
"""
url = cls.get_version_url(project)
last_change = project.get_time_last_created_version()
try:
req = cls.call_url(url,
last_change=last_change,
insecure=project.insecure)
except Exception as err:
raise AnityaPluginException(
'Could not call : "%s" of "%s", with error: %s' %
(url, project.name, str(err)))
versions = []
if not isinstance(req, six.string_types):
# Not modified
if req.status_code == 304:
return versions
req = req.text
try:
regex = REGEX % {"name": project.name.replace("+", r"\+")}
versions = get_versions_by_regex_for_text(req, url, regex, project)
except AnityaPluginException:
versions = get_versions_by_regex_for_text(req, url, DEFAULT_REGEX,
project)
return versions
def get_versions(cls, project):
"""Method called to retrieve all the versions (that can be found)
of the projects provided, project that relies on the backend of
this plugin.
:arg Project project: a :class:`anitya.db.models.Project` object whose backend
corresponds to the current plugin.
:return: a list of all the possible releases found
:return type: list
:raise AnityaPluginException: a
:class:`anitya.lib.exceptions.AnityaPluginException` exception
when the versions cannot be retrieved correctly
"""
url = cls.get_version_url(project)
if not url:
raise AnityaPluginException("Project %s was incorrectly set up." %
project.name)
return get_versions_by_regex(url, REGEX, project)
def get_versions(cls, project):
''' Method called to retrieve all the versions (that can be found)
of the projects provided, project that relies on the backend of
this plugin.
:arg Project project: a :class:`anitya.db.models.Project` object whose backend
corresponds to the current plugin.
:return: a list of all the possible releases found
:return type: list
:raise AnityaPluginException: a
:class:`anitya.lib.exceptions.AnityaPluginException` exception
when the versions cannot be retrieved correctly
'''
url = cls.get_version_url(project)
if not url:
raise AnityaPluginException(
"Aritfact needs to be in format groupId:artifactId")
return get_versions_by_regex(url, VERSION_REGEX, project)
def get_versions(cls, project):
"""Method called to retrieve all the versions (that can be found)
of the projects provided, project that relies on the backend of
this plugin.
:arg Project project: a :class:`anitya.db.models.Project` object whose backend
corresponds to the current plugin.
:return: a list of all the possible releases found
:return type: list
:raise AnityaPluginException: a
:class:`anitya.lib.exceptions.AnityaPluginException` exception
when the versions cannot be retrieved correctly
"""
namespace, repo = cls.get_namespace_repo(project)
url = project.get_version_url()
git_tag_request = requests.get(url)
if git_tag_request.status_code == 404:
raise AnityaPluginException(
'Could not call : "%s" of "%s", with error: %s' %
(url, project.name, git_tag_request.status_code))
soup = BeautifulSoup(git_tag_request.content, "html.parser")
def is_release_tag_link(tag):
link_prefix = f"/p/{namespace}/{repo}/ci/"
return (tag.has_attr("href")
and tag.attrs["href"].startswith(link_prefix)
and len(tag.contents) == 1)
release_tags = []
for release_tag in soup.find_all(is_release_tag_link):
release_tag_text = release_tag.contents[0]
release_tags.append(release_tag_text)
return release_tags
def get_versions(cls, project):
""" Method called to retrieve all the versions (that can be found)
of the projects provided, project that relies on the backend of
this plugin.
:arg Project project: a :class:`anitya.db.models.Project` object whose backend
corresponds to the current plugin.
:return: a list of all the possible releases found
:return type: list
:raise AnityaPluginException: a
:class:`anitya.lib.exceptions.AnityaPluginException` exception
when the versions cannot be retrieved correctly
"""
url = cls.get_version_url(project)
versions = []
versions = _get_versions(url)
if not versions:
raise AnityaPluginException("No versions found for %s" %
project.name.lower())
return versions
def get_versions_by_regex(url, regex, project, insecure=False):
"""For the provided url, return all the version retrieved via the
specified regular expression.
"""
last_change = project.get_time_last_created_version()
try:
req = BaseBackend.call_url(url, last_change=last_change, insecure=insecure)
except Exception as err:
_log.debug("%s ERROR: %s" % (project.name, str(err)))
raise AnityaPluginException(
'Could not call : "%s" of "%s", with error: %s'
% (url, project.name, str(err))
)
if not isinstance(req, six.string_types):
# Not modified
if req.status_code == 304:
return []
req = req.text
return get_versions_by_regex_for_text(req, url, regex, project)
def get_versions(cls, project):
''' Method called to retrieve all the versions (that can be found)
of the projects provided, project that relies on the backend of
this plugin.
:arg Project project: a :class:`anitya.db.models.Project` object whose backend
corresponds to the current plugin.
:return: a list of all the possible releases found
:return type: list
:raise AnityaPluginException: a
:class:`anitya.lib.exceptions.AnityaPluginException` exception
when the versions cannot be retrieved correctly
'''
url = project.version_url
try:
req = cls.call_url(url, insecure=project.insecure)
except Exception as err:
raise AnityaPluginException(
'Could not call : "%s" of "%s", with error: %s' % (
url, project.name, str(err)))
versions = None
if not isinstance(req, six.string_types):
req = req.text
try:
regex = REGEX % {'name': project.name.replace('+', '\+')}
versions = get_versions_by_regex_for_text(
req, url, regex, project)
except AnityaPluginException:
versions = get_versions_by_regex_for_text(
req, url, DEFAULT_REGEX, project)
return versions
def test_existing_project_check_failure(self, mock_check):
"""Assert that when an existing project fails a version check nothing happens"""
mock_check.side_effect = AnityaPluginException()
project = Project(
name="ImageMetaTag",
homepage="https://pypi.python.org/pypi/ImageMetaTag",
ecosystem_name="pypi",
backend="PyPI",
)
self.session.add(project)
self.session.commit()
event = sseclient.Event(
data="{"
'"name": "ImageMetaTag",'
'"platform": "PyPi",'
'"version": "0.6.9",'
'"package_manager_url": "https://pypi.org/project/ImageMetaTag/"'
"}")
self.assertEqual(1, self.session.query(Project).count())
self.client.process_message(event)
self.assertEqual(1, self.session.query(Project).count())
project = self.session.query(Project).first()
self.assertIs(project.latest_version, None)
def parse_json(json, project):
"""Function for parsing json response
Args:
json (dict): Json dictionary to parse.
project (:obj:`anitya.db.models.Project`): Project object whose backend
corresponds to the current plugin.
Returns:
:obj:`list`: A list of all the possible releases found.
Raises:
AnityaPluginException: A
:obj:`anitya.lib.exceptions.AnityaPluginException` exception
when the versions cannot be retrieved correctly.
RateLimitException: A
:obj:`anitya.lib.exceptions.RateLimitException` exception
when rate limit threshold is reached.
"""
global reset_time
# We need to check limit first,
# because exceeding the limit will also return error
try:
limit = json["data"]["rateLimit"]["limit"]
remaining = json["data"]["rateLimit"]["remaining"]
reset_time = json["data"]["rateLimit"]["resetAt"]
_log.debug("Github API ratelimit remains %s, will reset at %s UTC" %
(remaining, reset_time))
if (remaining / limit) <= RATE_LIMIT_THRESHOLD:
raise RateLimitException(reset_time)
except KeyError:
_log.info("Github API ratelimit key is missing. Checking for errors.")
if "errors" in json:
error_str = ""
for error in json["errors"]:
error_str += '"%s": "%s"\n' % (error["type"], error["message"])
raise AnityaPluginException(
"%s: Server responded with following errors\n%s" %
(project.name, error_str))
if project.releases_only:
json_data = json["data"]["repository"]["releases"]
else:
json_data = json["data"]["repository"]["refs"]
total_count = json_data["totalCount"]
if project.releases_only:
_log.debug("Received %s releases for %s" % (total_count, project.name))
else:
_log.debug("Received %s tags for %s" % (total_count, project.name))
versions = []
for edge in json_data["edges"]:
version = {"cursor": edge["cursor"]}
if project.releases_only:
hook = edge["node"]["tag"]
else:
hook = edge["node"]
version["version"] = hook["name"]
version["commit_url"] = hook["target"]["commitUrl"]
versions.append(version)
return versions
def call_url(self, url, insecure=False):
""" Dedicated method to query a URL.
It is important to use this method as it allows to query them with
a defined user-agent header thus informing the projects we are
querying what our intentions are.
Attributes:
url (str): The url to request (get).
insecure (bool, optional): Flag for secure/insecure connection.
Defaults to False.
Returns:
In case of FTP url it returns binary encoded string
otherwise :obj:`requests.Response` object.
"""
headers = REQUEST_HEADERS.copy()
if "*" in url:
url = self.expand_subdirs(url)
if url.startswith("ftp://") or url.startswith("ftps://"):
socket.setdefaulttimeout(30)
req = urllib.Request(url)
req.add_header("User-Agent", headers["User-Agent"])
req.add_header("From", headers["From"])
try:
# Ignore this bandit issue, the url is checked above
resp = urllib.urlopen(req) # nosec
content = resp.read().decode()
except URLError as e:
raise AnityaPluginException(
'Could not call "%s" with error: %s' % (url, e.reason)
)
except UnicodeDecodeError:
raise AnityaPluginException(
"FTP response cannot be decoded with UTF-8: %s" % url
)
return content
else:
# Works around https://github.com/kennethreitz/requests/issues/2863
# Currently, requests does not start new TCP connections based on
# TLS settings. This means that if a connection is ever started to
# a host with `verify=False`, further requests to that
# (scheme, host, port) combination will also be insecure, even if
# `verify=True` is passed to requests.
#
# This starts a new session which is immediately discarded when the
# request is insecure. We don't get to pool connections for these
# requests, but it stops us from making insecure requests by
# accident. This can be removed in requests-3.0.
if insecure:
with requests.Session() as r_session:
resp = r_session.get(url, headers=headers, timeout=60, verify=False)
else:
resp = http_session.get(url, headers=headers, timeout=60, verify=True)
return resp
def get_versions(cls, project):
''' Method called to retrieve all the versions (that can be found)
of the projects provided, project that relies on the backend of
this plugin.
Args:
project (:obj:`anitya.db.models.Project`): Project object whose backend
corresponds to the current plugin.
Returns:
:obj:`list`: A list of all the possible releases found
Raises:
AnityaPluginException: A
:obj:`anitya.lib.exceptions.AnityaPluginException` exception
when the versions cannot be retrieved correctly
'''
owner = None
repo = None
if project.version_url:
url = project.version_url
elif project.homepage.startswith('https://github.com'):
url = project.homepage.replace('https://github.com/', '')
if url.endswith('/'):
url = url[:-1]
else:
raise AnityaPluginException('Project %s was incorrectly set-up' %
project.name)
try:
(owner, repo) = url.split('/')
except ValueError:
raise AnityaPluginException("""Project {} was incorrectly set-up.
Can\'t parse owner and repo.""".format(project.name))
query = prepare_query(owner, repo)
try:
headers = REQUEST_HEADERS.copy()
token = config['GITHUB_ACCESS_TOKEN']
if token:
headers['Authorization'] = 'bearer %s' % token
resp = http_session.post(API_URL,
json={'query': query},
headers=headers,
timeout=60,
verify=True)
except Exception as err:
_log.debug('%s ERROR: %s' % (project.name, str(err)))
raise AnityaPluginException(
'Could not call : "%s" of "%s", with error: %s' %
(API_URL, project.name, str(err)))
if resp.ok:
json = resp.json()
else:
raise AnityaPluginException(
'%s: Server responded with status "%s": "%s"' %
(project.name, resp.status_code, resp.reason))
versions = parse_json(json, project)
if len(versions) == 0:
raise AnityaPluginException('%s: No upstream version found.' %
(project.name))
return versions
def get_versions(cls, project):
""" Method called to retrieve all the versions (that can be found)
of the projects provided, project that relies on the backend of
this plugin.
Args:
project (:obj:`anitya.db.models.Project`): Project object whose backend
corresponds to the current plugin.
Returns:
:obj:`list`: A list of all the possible releases found
Raises:
AnityaPluginException: A
:obj:`anitya.lib.exceptions.AnityaPluginException` exception
when the versions cannot be retrieved correctly
"""
owner = None
repo = None
url = cls.get_version_url(project)
if url:
url = url.replace("https://github.com/", "")
url = url.replace("/tags", "")
else:
raise AnityaPluginException("Project %s was incorrectly set-up" %
project.name)
try:
(owner, repo) = url.split("/")
except ValueError:
raise AnityaPluginException("""Project {} was incorrectly set-up.
Can\'t parse owner and repo.""".format(project.name))
query = prepare_query(owner, repo, project.releases_only)
try:
headers = REQUEST_HEADERS.copy()
token = config["GITHUB_ACCESS_TOKEN"]
if token:
headers["Authorization"] = "bearer %s" % token
resp = http_session.post(API_URL,
json={"query": query},
headers=headers,
timeout=60,
verify=True)
except Exception as err:
_log.debug("%s ERROR: %s" % (project.name, str(err)))
raise AnityaPluginException(
'Could not call : "%s" of "%s", with error: %s' %
(API_URL, project.name, str(err)))
if resp.ok:
json = resp.json()
elif resp.status_code == 403:
_log.info("Github API ratelimit reached.")
raise RateLimitException(reset_time)
else:
raise AnityaPluginException(
'%s: Server responded with status "%s": "%s"' %
(project.name, resp.status_code, resp.reason))
versions = parse_json(json, project)
_log.debug(f"Retrieved versions: {versions}")
if len(versions) == 0:
raise AnityaPluginException("%s: No upstream version found." %
(project.name))
return versions
