def test_create_project_dry_run(self):
""" Test the create_project dry_run parameter. """
create_distro(self.session)
self.assertEqual(2, models.Distro.all(self.session, count=True))
with fml_testing.mock_sends():
utilities.create_project(
self.session,
name="geany",
homepage="https://www.geany.org/",
version_url="https://www.geany.org/Download/Releases",
regex="DEFAULT",
user_id="*****@*****.**",
dry_run=True,
)
project_objs = models.Project.all(self.session)
self.assertEqual(len(project_objs), 1)
self.assertEqual(project_objs[0].name, "geany")
self.assertEqual(project_objs[0].homepage, "https://www.geany.org/")
# This should be still a running transaction
self.session.rollback()
project_objs = models.Project.all(self.session)
self.assertEqual(len(project_objs), 0)
def test_create_project_duplicate(self):
""" Assert that duplicate can't be created. """
create_distro(self.session)
self.assertEqual(2, models.Distro.all(self.session, count=True))
with fml_testing.mock_sends(anitya_schema.ProjectCreated):
utilities.create_project(
self.session,
name="geany",
homepage="https://www.geany.org/",
version_url="https://www.geany.org/Download/Releases",
regex="DEFAULT",
user_id="*****@*****.**",
)
project_objs = models.Project.all(self.session)
self.assertEqual(len(project_objs), 1)
self.assertEqual(project_objs[0].name, "geany")
self.assertEqual(project_objs[0].homepage, "https://www.geany.org/")
self.assertRaises(
ProjectExists,
utilities.create_project,
self.session,
name="geany",
homepage="https://www.geany.org/",
version_url="https://www.geany.org/Download/Releases",
regex="DEFAULT",
user_id="*****@*****.**",
)
project_objs = models.Project.all(self.session)
self.assertEqual(len(project_objs), 1)
self.assertEqual(project_objs[0].name, "geany")
self.assertEqual(project_objs[0].homepage, "https://www.geany.org/")
def test_create_project(self):
""" Test the create_project function of Distro. """
create_distro(self.session)
self.assertEqual(2, models.Distro.all(self.session, count=True))
utilities.create_project(
self.session,
name="geany",
homepage="https://www.geany.org/",
version_url="https://www.geany.org/Download/Releases",
regex="DEFAULT",
user_id="*****@*****.**",
)
project_objs = models.Project.all(self.session)
self.assertEqual(len(project_objs), 1)
self.assertEqual(project_objs[0].name, "geany")
self.assertEqual(project_objs[0].homepage, "https://www.geany.org/")
self.assertRaises(
ProjectExists,
utilities.create_project,
self.session,
name="geany",
homepage="https://www.geany.org/",
version_url="https://www.geany.org/Download/Releases",
regex="DEFAULT",
user_id="*****@*****.**",
)
project_objs = models.Project.all(self.session)
self.assertEqual(len(project_objs), 1)
self.assertEqual(project_objs[0].name, "geany")
self.assertEqual(project_objs[0].homepage, "https://www.geany.org/")
def test_create_project(self):
""" Test the create_project function of Distro. """
create_distro(self.session)
self.assertEqual(2, models.Distro.all(self.session, count=True))
utilities.create_project(
self.session,
name="geany",
homepage="https://www.geany.org/",
version_url="https://www.geany.org/Download/Releases",
regex="DEFAULT",
user_id="*****@*****.**",
)
project_objs = models.Project.all(self.session)
self.assertEqual(len(project_objs), 1)
self.assertEqual(project_objs[0].name, "geany")
self.assertEqual(project_objs[0].homepage, "https://www.geany.org/")
self.assertRaises(
ProjectExists,
utilities.create_project,
self.session,
name="geany",
homepage="https://www.geany.org/",
version_url="https://www.geany.org/Download/Releases",
regex="DEFAULT",
user_id="*****@*****.**",
)
project_objs = models.Project.all(self.session)
self.assertEqual(len(project_objs), 1)
self.assertEqual(project_objs[0].name, "geany")
self.assertEqual(project_objs[0].homepage, "https://www.geany.org/")
def test_create_project(self):
""" Test the create_project function of Distro. """
create_distro(self.session)
self.assertEqual(2, model.Distro.all(self.session, count=True))
utilities.create_project(
self.session,
name='geany',
homepage='http://www.geany.org/',
version_url='http://www.geany.org/Download/Releases',
regex='DEFAULT',
user_id='*****@*****.**',
)
project_objs = model.Project.all(self.session)
self.assertEqual(len(project_objs), 1)
self.assertEqual(project_objs[0].name, 'geany')
self.assertEqual(project_objs[0].homepage, 'http://www.geany.org/')
self.assertRaises(
ProjectExists,
utilities.create_project,
self.session,
name='geany',
homepage='http://www.geany.org/',
version_url='http://www.geany.org/Download/Releases',
regex='DEFAULT',
user_id='*****@*****.**',
)
project_objs = model.Project.all(self.session)
self.assertEqual(len(project_objs), 1)
self.assertEqual(project_objs[0].name, 'geany')
self.assertEqual(project_objs[0].homepage, 'http://www.geany.org/')
def create_ecosystem_projects(session):
""" Create some fake projects from particular upstream ecosystems
Each project name is used in two different ecosystems
"""
utilities.create_project(
session,
name='pypi_and_npm',
homepage='https://example.com/not-a-real-pypi-project',
backend='PyPI',
user_id='*****@*****.**')
utilities.create_project(
session,
name='pypi_and_npm',
homepage='https://example.com/not-a-real-npmjs-project',
backend='npmjs',
user_id='*****@*****.**')
utilities.create_project(
session,
name='rubygems_and_maven',
homepage='https://example.com/not-a-real-rubygems-project',
backend='Rubygems',
user_id='*****@*****.**')
utilities.create_project(
session,
name='rubygems_and_maven',
homepage='https://example.com/not-a-real-maven-project',
backend='Maven Central',
user_id='*****@*****.**')
def create_flagged_project(session):
""" Create and flag a project. Returns the ProjectFlag. """
project = utilities.create_project(
session,
name='geany',
homepage='http://www.geany.org/',
version_url='http://www.geany.org/Download/Releases',
regex='DEFAULT',
user_id='*****@*****.**',
)
session.add(project)
flag = utilities.flag_project(
session,
project,
"This is a duplicate.",
"*****@*****.**",
"user_openid_id",
)
session.add(flag)
session.commit()
return flag
def test_check_project_check_times_exception(self):
""" Test if check times are set if `exceptions.RateLimitException` is raised. """
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
)
last_check_orig = project.last_check
next_check = arrow.get("2008-09-03T20:56:35.450686").naive
with mock.patch(
"anitya.lib.backends.npmjs.NpmjsBackend.get_versions",
return_value=["1.0.0"],
) as mock_object:
mock_object.side_effect = exceptions.RateLimitException(
"2008-09-03T20:56:35.450686")
self.assertRaises(
exceptions.RateLimitException,
utilities.check_project_release,
project,
self.session,
)
self.assertTrue(last_check_orig < project.last_check)
self.assertEqual(next_check, project.next_check)
def test_check_project_check_times_exception(self):
""" Test if check times are set if `exceptions.RateLimitException` is raised. """
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
)
last_check_orig = project.last_check
next_check = arrow.get("2008-09-03T20:56:35.450686").naive
with mock.patch(
"anitya.lib.backends.npmjs.NpmjsBackend.get_versions",
return_value=["1.0.0"],
) as mock_object:
mock_object.side_effect = exceptions.RateLimitException(
"2008-09-03T20:56:35.450686"
)
self.assertRaises(
exceptions.RateLimitException,
utilities.check_project_release,
project,
self.session,
)
self.assertTrue(last_check_orig < project.last_check)
self.assertEqual(next_check, project.next_check)
def test_check_project_release_old_version(self, mock_method):
"""
Test the check_project_release function for Project.
This test will test if the fedora message is sent even if
the version is not the newest.
"""
with fml_testing.mock_sends(anitya_schema.ProjectCreated):
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
version_scheme="RPM",
)
# Create version
version = models.ProjectVersion(project_id=project.id, version="1.0.0")
self.session.add(version)
self.session.commit()
with fml_testing.mock_sends(anitya_schema.ProjectVersionUpdated,
anitya_schema.ProjectVersionUpdatedV2):
utilities.check_project_release(project, self.session)
versions = project.get_sorted_version_objects()
self.assertEqual(len(versions), 2)
self.assertEqual(versions[0].version, "1.0.0")
self.assertEqual(versions[1].version, "0.9.9")
def test_check_project_release_backend(self, mock_method):
""" Test the check_project_release function for Project. """
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
)
versions = utilities.check_project_release(project, self.session, test=True)
self.assertEqual(versions, ["0.9.8", "0.9.9", "1.0.0"])
def test_check_project_release_backend(self, mock_method):
""" Test the check_project_release function for Project. """
project = utilities.create_project(
self.session,
name='pypi_and_npm',
homepage='https://example.com/not-a-real-npmjs-project',
backend='npmjs',
user_id='*****@*****.**')
versions = utilities.check_project_release(project,
self.session,
test=True)
self.assertEqual(versions, ['0.9.8', '0.9.9', '1.0.0'])
def test_check_project_release_backend(self, mock_method):
""" Test the check_project_release function for Project. """
with fml_testing.mock_sends(anitya_schema.ProjectCreated):
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
)
versions = utilities.check_project_release(project, self.session, test=True)
self.assertEqual(versions, ["1.0.0", "0.9.9", "0.9.8"])
def create_project(session):
""" Create some basic projects to work with. """
utilities.create_project(
session,
name='geany',
homepage='http://www.geany.org/',
version_url='http://www.geany.org/Download/Releases',
regex='DEFAULT',
user_id='*****@*****.**',
)
utilities.create_project(
session,
name='subsurface',
homepage='http://subsurface.hohndel.org/',
version_url='http://subsurface.hohndel.org/downloads/',
regex='DEFAULT',
user_id='*****@*****.**',
)
utilities.create_project(
session,
name='R2spec',
homepage='https://fedorahosted.org/r2spec/',
user_id='*****@*****.**',
)
def test_check_project_release_new_version(self, mock_method):
""" Test the check_project_release function for Project. """
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
version_scheme="RPM",
)
utilities.check_project_release(project, self.session)
versions = project.get_sorted_version_objects()
self.assertEqual(len(versions), 3)
self.assertEqual(versions[0].version, "1.0.0")
def test_check_project_release_plugin_exception(self):
""" Test the check_project_release function for Project. """
project = utilities.create_project(
self.session,
name='pypi_and_npm',
homepage='https://example.com/not-a-real-npmjs-project',
backend='npmjs',
user_id='*****@*****.**')
self.assertRaises(
exceptions.AnityaPluginException,
utilities.check_project_release,
project,
self.session,
)
def test_check_project_release_new_version(self, mock_method):
""" Test the check_project_release function for Project. """
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
version_scheme="RPM",
)
utilities.check_project_release(project, self.session)
versions = project.get_sorted_version_objects()
self.assertEqual(len(versions), 3)
self.assertEqual(versions[0].version, "1.0.0")
def test_check_project_release_versions_with_cursor(self, mock_method):
"""Test check_project_release() with versions with cursor."""
with fml_testing.mock_sends(anitya_schema.ProjectCreated):
project = utilities.create_project(
self.session,
name="project_name",
homepage="https://not-a-real-homepage.com",
backend="GitHub",
user_id="*****@*****.**",
)
with fml_testing.mock_sends(anitya_schema.ProjectVersionUpdated):
utilities.check_project_release(project, self.session)
self.assertEqual(project.latest_version_cursor, "Hbgf")
def test_check_project_check_times_test(self, mock_method):
""" Test if check times aren't set in test mode. """
project = utilities.create_project(
self.session,
name='pypi_and_npm',
homepage='https://example.com/not-a-real-npmjs-project',
backend='npmjs',
user_id='*****@*****.**')
last_check_orig = project.last_check
next_check_orig = project.next_check
utilities.check_project_release(project, self.session, test=True)
self.assertEqual(last_check_orig, project.last_check)
self.assertEqual(next_check_orig, project.next_check)
def test_check_project_check_times(self, mock_method):
""" Test if check times are set. """
project = utilities.create_project(
self.session,
name='pypi_and_npm',
homepage='https://example.com/not-a-real-npmjs-project',
backend='npmjs',
user_id='*****@*****.**')
last_check_orig = project.last_check
utilities.check_project_release(project, self.session)
next_check = plugins.get_plugin(
project.backend).check_interval + project.last_check
self.assertTrue(last_check_orig < project.last_check)
self.assertEqual(next_check, project.next_check)
def test_check_project_release_plugin_exception(self):
""" Test the check_project_release function for Project. """
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
)
self.assertRaises(
exceptions.AnityaPluginException,
utilities.check_project_release,
project,
self.session,
)
def test_check_project_check_times_test(self, mock_method):
""" Test if check times aren't set in test mode. """
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
)
last_check_orig = project.last_check
next_check_orig = project.next_check
utilities.check_project_release(project, self.session, test=True)
self.assertEqual(last_check_orig, project.last_check)
self.assertEqual(next_check_orig, project.next_check)
def test_check_project_release_with_versions_with_urls(self, mock_method):
"""Test check_project_release() with versions with URLs."""
with fml_testing.mock_sends(anitya_schema.ProjectCreated):
project = utilities.create_project(
self.session,
name="project_name",
homepage="https://not-a-real-homepage.com",
backend="GitHub",
user_id="*****@*****.**",
)
with fml_testing.mock_sends(anitya_schema.ProjectVersionUpdated):
utilities.check_project_release(project, self.session)
for vo in project.versions_obj:
self.assertEqual(vo.commit_url,
"https://example.com/tags/" + vo.version)
def test_check_project_release_prefix_remove(self, mock_method):
""" Test the check_project_release function for Project. """
project = utilities.create_project(
self.session,
name='pypi_and_npm',
homepage='https://example.com/not-a-real-npmjs-project',
backend='npmjs',
user_id='*****@*****.**',
version_scheme='RPM',
)
utilities.check_project_release(project, self.session)
versions = project.get_sorted_version_objects()
self.assertEqual(len(versions), 3)
self.assertEqual(versions[0].version, 'v1.0.0')
self.assertEqual(project.latest_version, '1.0.0')
def test_check_project_release_error_counter_reset(self, mock_method):
""" Test if error_counter reset to 0, when successful check is done. """
with fml_testing.mock_sends(anitya_schema.ProjectCreated):
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
)
project.error_counter = 30
self.session.add(project)
self.session.commit()
utilities.check_project_release(project, self.session)
self.assertEqual(project.error_counter, 0)
def test_check_project_check_times_test(self, mock_method):
""" Test if check times aren't set in test mode. """
with fml_testing.mock_sends(anitya_schema.ProjectCreated):
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
)
last_check_orig = project.last_check
next_check_orig = project.next_check
with fml_testing.mock_sends():
utilities.check_project_release(project, self.session, test=True)
self.assertEqual(last_check_orig, project.last_check)
self.assertEqual(next_check_orig, project.next_check)
def test_check_project_check_times(self, mock_method):
""" Test if check times are set. """
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
)
last_check_orig = project.last_check
utilities.check_project_release(project, self.session)
next_check = (
plugins.get_plugin(project.backend).check_interval + project.last_check
)
self.assertTrue(last_check_orig < project.last_check)
self.assertEqual(next_check, project.next_check)
def test_check_project_release_plugin_exception(self):
""" Test the check_project_release function for Project. """
with fml_testing.mock_sends(anitya_schema.ProjectCreated):
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
)
self.assertRaises(
exceptions.AnityaPluginException,
utilities.check_project_release,
project,
self.session,
)
self.assertEqual(project.error_counter, 1)
def test_check_project_check_times(self, mock_method):
""" Test if check times are set. """
with fml_testing.mock_sends(anitya_schema.ProjectCreated):
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
)
last_check_orig = project.last_check
with fml_testing.mock_sends(anitya_schema.ProjectVersionUpdated):
utilities.check_project_release(project, self.session)
next_check = (plugins.get_plugin(project.backend).check_interval +
project.last_check)
self.assertTrue(last_check_orig < project.last_check)
self.assertEqual(next_check, project.next_check)
def test_check_project_release_prefix_remove(self, mock_method):
""" Test the check_project_release function for Project. """
with fml_testing.mock_sends(anitya_schema.ProjectCreated):
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
version_scheme="RPM",
)
with fml_testing.mock_sends(anitya_schema.ProjectVersionUpdated):
utilities.check_project_release(project, self.session)
versions = project.get_sorted_version_objects()
self.assertEqual(len(versions), 3)
self.assertEqual(versions[0].version, "v1.0.0")
self.assertEqual(project.latest_version, "1.0.0")
def test_check_project_release_no_new_version(self, mock_method):
""" Test the check_project_release function for Project. """
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
)
project.latest_version = "1.0.0"
version = models.ProjectVersion(version="1.0.0", project_id=project.id)
self.session.add(version)
self.session.commit()
utilities.check_project_release(project, self.session)
self.assertEqual(project.latest_version, "1.0.0")
self.assertEqual(project.logs, "No new version found")
def test_check_project_release_no_new_version(self, mock_method):
""" Test the check_project_release function for Project. """
project = utilities.create_project(
self.session,
name='pypi_and_npm',
homepage='https://example.com/not-a-real-npmjs-project',
backend='npmjs',
user_id='*****@*****.**',
)
project.latest_version = '1.0.0'
version = models.ProjectVersion(version='1.0.0', project_id=project.id)
self.session.add(version)
self.session.commit()
utilities.check_project_release(project, self.session)
self.assertEqual(project.latest_version, '1.0.0')
self.assertEqual(project.logs, 'No new version found')
def test_check_project_release_no_new_version(self, mock_method):
""" Test the check_project_release function for Project. """
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
)
project.latest_version = "1.0.0"
version = models.ProjectVersion(version="1.0.0", project_id=project.id)
self.session.add(version)
self.session.commit()
utilities.check_project_release(project, self.session)
self.assertEqual(project.latest_version, "1.0.0")
self.assertEqual(project.logs, "No new version found")
def process_message(self, message):
"""
This method is called when a incoming SSE message is received.
If the project is unknown to Anitya, a new :class:`Project` is created,
but only if the platform is whitelisted. See `self.whitelist`.
If it's an existing project, this will call Anitya's ``check_project_release``
API to update the latest version information. This is because we are
subscribed to libraries.io over HTTP and therefore have no
authentication.
Checking the version ourselves serves two purposes. Firstly, we connect
to libraries.io over HTTP so we don't have any authentication.
Secondly, we might map the libraries.io project to the wrong Anitya
project, so this is a good opportunity to catch those problems.
Args:
message (dict): The fedmsg to process.
"""
# The SSE spec requires all data to be UTF-8 encoded
try:
librariesio_msg = json.loads(message.data, encoding="utf-8")
except json.JSONDecodeError:
_log.warning(
"Dropping librariesio update message. Invalid json '{}'.".
format(message.data))
return
name = librariesio_msg["name"]
platform = librariesio_msg["platform"].lower()
version = librariesio_msg["version"]
homepage = librariesio_msg["package_manager_url"]
for ecosystem in plugins.ECOSYSTEM_PLUGINS.get_plugins():
if platform == ecosystem.name or platform in ecosystem.aliases:
break
else:
_log.debug("Dropped librariesio update to {} for {} ({}) since"
" it is on the unsupported {} platform".format(
version, name, homepage, platform))
return
session = Session()
project = models.Project.query.filter_by(
name=name, ecosystem_name=ecosystem.name).one_or_none()
if project is None and platform in self.whitelist:
try:
project = utilities.create_project(
session,
name,
homepage,
"anitya",
backend=ecosystem.default_backend)
utilities.check_project_release(project, Session)
_log.info(
"Discovered new project at version {} via libraries.io: {}"
.format(version, project))
except exceptions.AnityaException as e:
_log.error(
"A new project was discovered via libraries.io, {}, "
'but we failed with "{}"'.format(name, str(e)))
elif project is None:
_log.info(
"Discovered new project, {}, on the {} platform, but anitya is "
"configured to not create the project".format(name, platform))
else:
_log.info(
"libraries.io has found an update (version {}) for project {}".
format(version, project.name))
# This will fetch the version, emit fedmsgs, add log entries, and
# commit the transaction.
try:
utilities.check_project_release(project, session)
except exceptions.AnityaPluginException as e:
_log.warning(
"libraries.io found an update for {}, but we failed with {}"
.format(project, str(e)))
# Refresh the project object that was committed by either
# ``create_project`` or ``check_project_release`` above
project = models.Project.query.filter_by(
name=name, ecosystem_name=ecosystem.name).one_or_none()
if project and project.latest_version != version:
_log.info(
"libraries.io found {} had a latest version of {}, but Anitya found {}"
.format(project, version, project.latest_version))
Session.remove()
def post(self):
"""
Create a new project.
**Example Request**:
.. sourcecode:: http
POST /api/v2/projects/ HTTP/1.1
Authorization: token hxPpKow7nnT6UTAEKMtQwl310P6GtyqV8DDbexnk
Accept: application/json
Accept-Encoding: gzip, deflate
Connection: keep-alive
Content-Length: 114
Content-Type: application/json
Host: localhost:5000
User-Agent: HTTPie/0.9.4
{
"backend": "custom",
"homepage": "https://example.com/test",
"name": "test_project",
"version_prefix": "release-"
}
**Example Response**:
.. sourcecode:: http
HTTP/1.0 201 CREATED
Content-Length: 276
Content-Type: application/json
Date: Sun, 26 Mar 2017 15:56:30 GMT
Server: Werkzeug/0.12.1 Python/2.7.13
{
"backend": "PyPI",
"created_on": 1490543790.0,
"homepage": "http://python-requests.org",
"id": 13857,
"name": "requests",
"regex": null,
"updated_on": 1490543790.0,
"version": null,
"version_url": null,
"versions": []
}
:query string access_token: Your API access token.
:reqjson string name: The project name
:reqjson string homepage: The project homepage URL
:reqjson string backend: The project backend (github, folder, etc.).
:reqjson string version_url: The URL to fetch when determining the
project version (defaults to null).
:reqjson string version_prefix: The project version prefix, if any. For
example, some projects prefix with "v".
:reqjson string regex: The regex to use when searching the
``version_url`` page.
:reqjson bool insecure: When retrieving the versions via HTTPS, do not
validate the certificate (defaults to false).
:reqjson bool check_release: Check the release immediately after
creating the project.
:statuscode 201: When the project was successfully created.
:statuscode 400: When required arguments are missing or malformed.
:statuscode 401: When your access token is missing or invalid, or when
the server is not configured for OpenID Connect. The
response will include a JSON body describing the exact
problem.
:statuscode 409: When the project already exists.
"""
name_help = _("The project name")
homepage_help = _("The project homepage URL")
backend_help = _("The project backend (github, folder, etc.)")
version_url_help = _(
"The URL to fetch when determining the project "
"version (defaults to null)"
)
version_prefix_help = _(
"The project version prefix, if any. For "
'example, some projects prefix with "v"'
)
regex_help = _("The regex to use when searching the version_url page")
insecure_help = _(
"When retrieving the versions via HTTPS, do not "
"validate the certificate (defaults to false)"
)
check_release_help = _(
"Check the release immediately after creating " "the project."
)
parser = _BASE_ARG_PARSER.copy()
parser.add_argument("name", type=str, help=name_help, required=True)
parser.add_argument("homepage", type=str, help=homepage_help, required=True)
parser.add_argument("backend", type=str, help=backend_help, required=True)
parser.add_argument(
"version_url", type=str, help=version_url_help, default=None
)
parser.add_argument(
"version_prefix", type=str, help=version_prefix_help, default=None
)
parser.add_argument("regex", type=str, help=regex_help, default=None)
parser.add_argument("insecure", type=bool, help=insecure_help, default=False)
parser.add_argument("check_release", type=bool, help=check_release_help)
args = parser.parse_args(strict=True)
try:
project = utilities.create_project(
Session, user_id=flask_login.current_user.email, **args
)
Session.commit()
return project.__json__(), 201
except ProjectExists as e:
response = jsonify(e.to_dict())
response.status_code = 409
return response
def post(self):
"""
Create a new project.
**Example Request**:
.. sourcecode:: http
POST /api/v2/projects/?access_token=MYAPIACCESSTOKEN HTTP/1.1
Accept: application/json
Accept-Encoding: gzip, deflate
Connection: keep-alive
Content-Length: 114
Content-Type: application/json
Host: localhost:5000
User-Agent: HTTPie/0.9.4
{
"backend": "custom",
"homepage": "http://example.com/test",
"name": "test_project",
"version_prefix": "release-"
}
**Example Response**:
.. sourcecode:: http
HTTP/1.0 201 CREATED
Content-Length: 276
Content-Type: application/json
Date: Sun, 26 Mar 2017 15:56:30 GMT
Server: Werkzeug/0.12.1 Python/2.7.13
{
"backend": "PyPI",
"created_on": 1490543790.0,
"homepage": "http://python-requests.org",
"id": 13857,
"name": "requests",
"regex": null,
"updated_on": 1490543790.0,
"version": null,
"version_url": null,
"versions": []
}
:query string access_token: Your API access token.
:reqjson string name: The project name
:reqjson string homepage: The project homepage URL
:reqjson string backend: The project backend (github, folder, etc.).
:reqjson string version_url: The URL to fetch when determining the
project version (defaults to null).
:reqjson string version_prefix: The project version prefix, if any. For
example, some projects prefix with "v".
:reqjson string regex: The regex to use when searching the
``version_url`` page.
:reqjson bool insecure: When retrieving the versions via HTTPS, do not
validate the certificate (defaults to false).
:reqjson bool check_release: Check the release immediately after
creating the project.
:statuscode 201: When the project was successfully created.
:statuscode 400: When required arguments are missing or malformed.
:statuscode 401: When your access token is missing or invalid, or when
the server is not configured for OpenID Connect. The
response will include a JSON body describing the exact
problem.
:statuscode 409: When the project already exists.
"""
name_help = _('The project name')
homepage_help = _('The project homepage URL')
backend_help = _('The project backend (github, folder, etc.)')
version_url_help = _('The URL to fetch when determining the project '
'version (defaults to null)')
version_prefix_help = _('The project version prefix, if any. For '
'example, some projects prefix with "v"')
regex_help = _('The regex to use when searching the version_url page')
insecure_help = _('When retrieving the versions via HTTPS, do not '
'validate the certificate (defaults to false)')
check_release_help = _('Check the release immediately after creating '
'the project.')
parser = _BASE_ARG_PARSER.copy()
parser.add_argument('name', type=str, help=name_help, required=True)
parser.add_argument('homepage',
type=str,
help=homepage_help,
required=True)
parser.add_argument('backend',
type=str,
help=backend_help,
required=True)
parser.add_argument('version_url',
type=str,
help=version_url_help,
default=None)
parser.add_argument('version_prefix',
type=str,
help=version_prefix_help,
default=None)
parser.add_argument('regex', type=str, help=regex_help, default=None)
parser.add_argument('insecure',
type=bool,
help=insecure_help,
default=False)
parser.add_argument('check_release',
type=bool,
help=check_release_help)
args = parser.parse_args(strict=True)
access_token = args.pop('access_token')
try:
project = utilities.create_project(
SESSION,
user_id=authentication.oidc.user_getfield(
'email', access_token),
**args)
SESSION.commit()
return project.__json__(), 201
except ProjectExists as e:
response = jsonify(e.to_dict())
response.status_code = 409
return response
def new_project():
"""
View for creating a new project.
This function accepts GET and POST requests. POST requests can result in
a HTTP 400 for invalid forms, a HTTP 409 if the request conflicts with an
existing project, or a HTTP 302 redirect to the new project.
"""
backend_plugins = anitya_plugins.load_plugins(Session)
plg_names = [plugin.name for plugin in backend_plugins]
version_plugins = anitya_plugins.load_plugins(Session, family="versions")
version_plg_names = [plugin.name for plugin in version_plugins]
# Get all available distros name
distros = models.Distro.all(Session)
distro_names = []
for distro in distros:
distro_names.append(distro.name)
form = anitya.forms.ProjectForm(backends=plg_names,
version_schemes=version_plg_names,
distros=distro_names)
if flask.request.method == "GET":
form.name.data = flask.request.args.get("name", "")
form.homepage.data = flask.request.args.get("homepage", "")
form.backend.data = flask.request.args.get("backend", "")
form.version_scheme.data = flask.request.args.get("version_scheme", "")
form.distro.data = flask.request.args.get("distro", "")
form.package_name.data = flask.request.args.get("package_name", "")
return flask.render_template(
"project_new.html",
context="Add",
current="Add projects",
form=form,
plugins=backend_plugins,
)
elif form.validate_on_submit():
project = None
try:
project = utilities.create_project(
Session,
name=form.name.data.strip(),
homepage=form.homepage.data.strip(),
backend=form.backend.data.strip(),
version_scheme=form.version_scheme.data.strip(),
version_url=form.version_url.data.strip() or None,
version_prefix=form.version_prefix.data.strip() or None,
pre_release_filter=form.pre_release_filter.data.strip()
or None,
version_filter=form.version_filter.data.strip() or None,
regex=form.regex.data.strip() or None,
user_id=flask.g.user.username,
releases_only=form.releases_only.data,
)
Session.commit()
# Optionally, the user can also map a distro when creating a proj.
if form.distro.data and form.package_name.data:
utilities.map_project(
Session,
project=project,
package_name=form.package_name.data,
distribution=form.distro.data,
user_id=flask.g.user.username,
)
Session.commit()
flask.flash("Project created")
except exceptions.AnityaException as err:
flask.flash(str(err))
return (
flask.render_template(
"project_new.html",
context="Add",
current="Add projects",
form=form,
plugins=backend_plugins,
),
409,
)
if form.check_release.data is True:
try:
utilities.check_project_release(project, Session)
except exceptions.AnityaException:
flask.flash("Check failed")
return flask.redirect(
flask.url_for("anitya_ui.project", project_id=project.id))
return (
flask.render_template(
"project_new.html",
context="Add",
current="Add projects",
form=form,
plugins=backend_plugins,
),
400,
)
def post(self):
"""
Create a new project.
**Example Request**:
.. sourcecode:: http
POST /api/v2/projects/ HTTP/1.1
Authorization: token hxPpKow7nnT6UTAEKMtQwl310P6GtyqV8DDbexnk
Accept: application/json
Accept-Encoding: gzip, deflate
Connection: keep-alive
Content-Length: 114
Content-Type: application/json
Host: localhost:5000
User-Agent: HTTPie/0.9.4
{
"backend": "custom",
"homepage": "https://example.com/test",
"name": "test_project",
"version_prefix": "release-"
}
**Example Response**:
.. sourcecode:: http
HTTP/1.0 201 CREATED
Content-Length: 276
Content-Type: application/json
Date: Sun, 26 Mar 2017 15:56:30 GMT
Server: Werkzeug/0.12.1 Python/2.7.13
{
"backend": "PyPI",
"created_on": 1490543790.0,
"homepage": "http://python-requests.org",
"id": 13857,
"name": "requests",
"regex": null,
"updated_on": 1490543790.0,
"version": null,
"version_url": null,
"versions": [],
"stable_versions": []
}
:query string access_token: Your API access token.
:reqjson string name: The project name
:reqjson string homepage: The project homepage URL
:reqjson string backend: The project backend (github, folder, etc.).
:reqjson string version_url: The URL to fetch when determining the
project version (defaults to null).
:reqjson string version_prefix: The project version prefix, if any. For
example, some projects prefix with "v".
:reqjson string regex: The regex to use when searching the
``version_url`` page.
:reqjson bool insecure: When retrieving the versions via HTTPS, do not
validate the certificate (defaults to false).
:reqjson bool check_release: Check the release immediately after
creating the project.
:statuscode 201: When the project was successfully created.
:statuscode 400: When required arguments are missing or malformed.
:statuscode 401: When your access token is missing or invalid, or when
the server is not configured for OpenID Connect. The
response will include a JSON body describing the exact
problem.
:statuscode 409: When the project already exists.
"""
name_help = _("The project name")
homepage_help = _("The project homepage URL")
backend_help = _("The project backend (github, folder, etc.)")
version_url_help = _("The URL to fetch when determining the project "
"version (defaults to null)")
version_prefix_help = _("The project version prefix, if any. For "
'example, some projects prefix with "v"')
regex_help = _("The regex to use when searching the version_url page")
insecure_help = _("When retrieving the versions via HTTPS, do not "
"validate the certificate (defaults to false)")
check_release_help = _("Check the release immediately after creating "
"the project.")
parser = _BASE_ARG_PARSER.copy()
parser.add_argument("name", type=str, help=name_help, required=True)
parser.add_argument("homepage",
type=str,
help=homepage_help,
required=True)
parser.add_argument("backend",
type=str,
help=backend_help,
required=True)
parser.add_argument("version_url",
type=str,
help=version_url_help,
default=None)
parser.add_argument("version_prefix",
type=str,
help=version_prefix_help,
default=None)
parser.add_argument("regex", type=str, help=regex_help, default=None)
parser.add_argument("insecure",
type=bool,
help=insecure_help,
default=False)
parser.add_argument("check_release",
type=bool,
help=check_release_help)
args = parser.parse_args(strict=True)
try:
project = utilities.create_project(
Session,
user_id=flask_login.current_user.email,
name=args.name,
homepage=args.homepage,
backend=args.backend,
version_url=args.version_url,
version_prefix=args.version_prefix,
regex=args.regex,
insecure=args.insecure,
)
Session.commit()
if args.check_release:
try:
utilities.check_project_release(project, Session)
except AnityaException as err:
_log.error(str(err))
return project.__json__(), 201
except ProjectExists as e:
response = jsonify(e.to_dict())
response.status_code = 409
return response
def post(self):
"""
Check for new versions on the project. The API first checks if the project exists,
if exists it will do check on it while applying any changes.
If not it will create a temporary object in memory and do a check on the temporary object.
**Example Request**:
.. sourcecode:: http
POST /api/v2/versions/ HTTP/1.1
Accept: application/json, */*
Accept-Encoding: gzip, deflate
Authorization: token s12p01zUiVdEOZIhVf0jyZqtyYXfo2DECi6YdqqV
Connection: keep-alive
Content-Length: 15
Content-Type: application/json
Host: localhost:5000
User-Agent: HTTPie/1.0.3
{
"id": "55612"
}
**Example Response**:
.. sourcecode:: http
HTTP/1.0 200 OK
Content-Length: 118
Content-Type: application/json
Date: Tue, 20 Oct 2020 08:49:01 GMT
Server: Werkzeug/0.16.0 Python/3.8.6
{
"found_versions": [],
"latest_version": "0.0.2",
"versions": [
"0.0.2",
"0.0.1"
],
"stable_versions": [
"0.0.2",
"0.0.1"
]
}
:query string access_token: Your API access token.
:reqjson int id: Id of the project.
If provided the check is done above existing project.
:reqjson string name: The project name. Used as a filter to find existing project,
if id not provided.
:reqjson string homepage: The project homepage URL. Used as a filter to find
existing project, if id not provided.
:reqjson string backend: The project backend (github, folder, etc.).
:reqjson string version_url: The URL to fetch when determining the
project version.
:reqjson string version_scheme: The project version scheme
(defaults to "RPM" for temporary project).
:reqjson string version_pattern: The version pattern for calendar version scheme.
:reqjson string version_prefix: The project version prefix, if any.
:reqjson string pre_release_filter: Filter for unstable versions.
:reqjson string version_filter: Filter for blacklisted versions.
:reqjson string regex: The regex to use when searching the
``version_url`` page
(defaults to none for temporary project).
:reqjson bool insecure: When retrieving the versions via HTTPS, do not
validate the certificate
(defaults to false for temporary project).
:reqjson bool releases_only: When retrieving the versions, use releases
instead of tags (defaults to false
for temporary project).
Only available for GitHub backend.
:reqjson bool dry_run: If set, doesn't save anything (defaults to true).
Can't be set to False for temporary project.
:statuscode 200: When the versions were successfully retrieved.
:statuscode 400: When required arguments are missing or malformed.
:statuscode 401: When your access token is missing or invalid, or when
the server is not configured for OpenID Connect. The
response will include a JSON body describing the exact
problem.
:statuscode 404: When id is provided and the project doesn't exist or is archived.
:statuscode 500: If there is error during the check
"""
id_help = _(
"Id of the project. If provided the check is done above existing project."
)
name_help = _(
"The project name. Used as a filter to find existing project, "
"if id not provided.")
homepage_help = _("The project homepage URL. Used as a filter to find "
"existing project, if id not provided.")
backend_help = _("The project backend (github, folder, etc.).")
version_url_help = _("The URL to fetch when determining the "
"project version.")
version_scheme_help = _("The project version scheme "
"(defaults to 'RPM' for temporary project).")
version_pattern_help = _(
"The version pattern for calendar version scheme.")
version_prefix_help = _("The project version prefix, if any.")
pre_release_filter_help = _("Filter for unstable versions.")
version_filter_help = _("Filter for blacklisted versions.")
regex_help = _("The regex to use when searching the version_url page "
"(defaults to none for temporary project).")
insecure_help = _(
"When retrieving the versions via HTTPS, do not "
"validate the certificate (defaults to false for temporary project)."
)
releases_only_help = _(
"When retrieving the versions, use releases "
"instead of tags (defaults to false for temporary project). "
"Only available for GitHub backend.")
dry_run_help = _("If set, doesn't save anything (defaults to true). "
"Can't be set to False for temporary project.")
parser = _BASE_ARG_PARSER.copy()
parser.add_argument("id", type=int, help=id_help)
parser.add_argument("name", type=str, help=name_help)
parser.add_argument("homepage", type=str, help=homepage_help)
parser.add_argument("backend", type=str, help=backend_help)
parser.add_argument("version_url", type=str, help=version_url_help)
parser.add_argument("version_scheme",
type=str,
help=version_scheme_help)
parser.add_argument("version_pattern",
type=str,
help=version_pattern_help)
parser.add_argument("version_prefix",
type=str,
help=version_prefix_help)
parser.add_argument("pre_release_filter",
type=str,
help=pre_release_filter_help)
parser.add_argument("version_filter",
type=str,
help=version_filter_help)
parser.add_argument("regex", type=str, help=regex_help, default=None)
parser.add_argument("insecure",
type=inputs.boolean,
help=insecure_help,
default=False)
parser.add_argument("releases_only",
type=inputs.boolean,
help=releases_only_help,
default=False)
parser.add_argument("dry_run",
type=inputs.boolean,
help=dry_run_help,
default=True)
args = parser.parse_args(strict=True)
project = None
dry_run = args.get("dry_run")
# If we have id, try to get the project
if args.id:
project = models.Project.get(Session, project_id=args.pop("id"))
if not project:
response = "No such project", 404
return response
# Don't look for the project if already retrieved
if not project:
homepage = args.get("homepage")
name = args.get("name")
q = models.Project.query
if homepage:
q = q.filter(
func.lower(models.Project.homepage) == func.lower(
homepage))
if name:
q = q.filter(
func.lower(models.Project.name) == func.lower(name))
query_result = q.all()
if len(query_result) > 1:
response = (
"More than one project found",
400,
)
return response
elif len(query_result) == 1:
project = query_result[0]
# If we still don't have project create temporary one
if not project:
# Check if we have all the required parameters
missing_parameters = []
name = args.get("name")
if not name:
missing_parameters.append("name")
homepage = args.get("homepage")
if not homepage:
missing_parameters.append("homepage")
backend = args.get("backend")
if not backend:
missing_parameters.append("backend")
version_url = args.get("version_url")
if not version_url:
version_url = homepage
if missing_parameters:
response = (
"Can't create temporary project. Missing arguments: " +
str(missing_parameters),
400,
)
return response
# Before we create a temporary project, check the dry_run parameter
if not dry_run:
dry_run = True
project = utilities.create_project(
Session,
user_id=flask_login.current_user.email,
name=name,
homepage=homepage,
backend=backend,
version_url=version_url,
version_pattern=args.version_pattern,
version_scheme=args.version_scheme,
version_prefix=args.version_prefix,
pre_release_filter=args.pre_release_filter,
version_filter=args.version_filter,
regex=args.regex,
insecure=args.insecure,
dry_run=dry_run,
)
# If the project was retrieved, update it with provided arguments
else:
# If argument is missing, use the actual one
name = args.get("name")
if not name:
name = project.name
homepage = args.get("homepage")
if not homepage:
homepage = project.homepage
backend = args.get("backend")
if not backend:
backend = project.backend
version_scheme = args.get("version_scheme")
if not version_scheme:
version_scheme = project.version_scheme
version_pattern = args.get("version_pattern")
if not version_pattern:
version_pattern = project.version_pattern
version_url = args.get("version_url")
if not version_url:
version_url = project.version_url
version_prefix = args.get("version_prefix")
if not version_prefix:
version_prefix = project.version_prefix
pre_release_filter = args.get("pre_release_filter")
if not pre_release_filter:
pre_release_filter = project.pre_release_filter
version_filter = args.get("version_filter")
if not version_filter:
version_filter = project.version_filter
regex = args.get("regex")
if not regex:
regex = project.regex
insecure = args.get("insecure")
if not insecure:
insecure = project.insecure
releases_only = args.get("releases_only")
if not releases_only:
releases_only = project.releases_only
try:
utilities.edit_project(
Session,
user_id=flask_login.current_user.email,
project=project,
name=name,
homepage=homepage,
backend=backend,
version_scheme=version_scheme,
version_pattern=version_pattern,
version_url=version_url,
version_prefix=version_prefix,
pre_release_filter=pre_release_filter,
version_filter=version_filter,
regex=regex,
insecure=insecure,
releases_only=releases_only,
dry_run=dry_run,
)
except AnityaException as err:
response = str(err), 500
return response
if project:
try:
versions = utilities.check_project_release(project,
Session,
test=dry_run)
except AnityaException as err:
response = "Error when checking for new version: " + str(
err), 500
return response
response = {
"latest_version": project.latest_version,
"found_versions": versions,
"versions": project.versions,
"stable_versions": [str(v) for v in project.stable_versions],
}
return response
def consume(self, message):
"""
This method is called when a message with a topic this consumer
subscribes to arrives.
If the project is unknown to Anitya, a new :class:`Project` is created,
but only if the 'create_librariesio_projects' configuration key is set.
If it's an existing project, this will call Anitya's ``check_project_release``
API to update the latest version information. This is because we are
subscribed to libraries.io over HTTP and therefore have no
authentication.
Checking the version ourselves serves two purposes. Firstly, we connect
to libraries.io over HTTP so we don't have any authentication.
Secondly, we might map the libraries.io project to the wrong Anitya
project, so this is a good opportunity to catch those problems.
Args:
message (dict): The fedmsg to process.
"""
librariesio_msg = message["body"]["msg"]["data"]
name = librariesio_msg["name"]
platform = librariesio_msg["platform"].lower()
version = librariesio_msg["version"]
homepage = librariesio_msg["package_manager_url"]
for ecosystem in plugins.ECOSYSTEM_PLUGINS.get_plugins():
if platform == ecosystem.name or platform in ecosystem.aliases:
break
else:
_log.debug(
"Dropped librariesio update to %s for %s (%s) since it is on the "
"unsupported %s platform",
version,
name,
homepage,
platform,
)
return
session = Session()
project = models.Project.by_name_and_ecosystem(session, name, ecosystem.name)
if (
project is None
and platform in config.config["LIBRARIESIO_PLATFORM_WHITELIST"]
):
try:
project = utilities.create_project(
session,
name,
homepage,
"anitya",
backend=ecosystem.default_backend,
check_release=True,
)
_log.info(
"Discovered new project at version %s via libraries.io: %r",
version,
project,
)
except exceptions.AnityaException as e:
_log.error(
"A new project was discovered via libraries.io, %r, "
'but we failed with "%s"',
project,
str(e),
)
elif project is None:
_log.info(
"Discovered new project, %s, on the %s platform, but anitya is "
"configured to not create the project",
name,
platform,
)
else:
_log.info(
"libraries.io has found an update (version %s) for project %r",
version,
project,
)
# This will fetch the version, emit fedmsgs, add log entries, and
# commit the transaction.
try:
utilities.check_project_release(project, session)
except exceptions.AnityaPluginException as e:
_log.warning(
"libraries.io found an update for %r, but we failed with %s",
project,
str(e),
)
# Refresh the project object that was committed by either
# ``create_project`` or ``check_project_release`` above
project = models.Project.by_name_and_ecosystem(session, name, ecosystem.name)
if project and project.latest_version != version:
_log.info(
"libraries.io found %r had a latest version of %s, but Anitya found %s",
project,
version,
project.latest_version,
)
Session.remove()
