def test_check_project_check_times_exception(self):
""" Test if check times are set if `exceptions.RateLimitException` is raised. """
project = utilities.create_project(
self.session,
name="pypi_and_npm",
homepage="https://example.com/not-a-real-npmjs-project",
backend="npmjs",
user_id="*****@*****.**",
)
last_check_orig = project.last_check
next_check = arrow.get("2008-09-03T20:56:35.450686").naive
with mock.patch(
"anitya.lib.backends.npmjs.NpmjsBackend.get_versions",
return_value=["1.0.0"],
) as mock_object:
mock_object.side_effect = exceptions.RateLimitException(
"2008-09-03T20:56:35.450686")
self.assertRaises(
exceptions.RateLimitException,
utilities.check_project_release,
project,
self.session,
)
self.assertTrue(last_check_orig < project.last_check)
self.assertEqual(next_check, project.next_check)
def test_str(self):
"""Assert the __str__ method provides a human-readable value."""
time = "2018-08-24T09:36:15Z"
exp = 'Rate limit was reached. Will be reset in "2018-08-24T09:36:15+00:00".'
e = exceptions.RateLimitException(time)
self.assertEqual(exp, str(e))
class CheckerTests(DatabaseTestCase):
""" Checker class tests. """
def setUp(self):
"""
Prepare the Checker object.
"""
super(CheckerTests, self).setUp()
self.checker = Checker()
@mock.patch("anitya.lib.utilities.check_project_release")
def test_update_project_backend_blacklist(self,
mock_check_project_release):
"""
Assert that project next_check is set to correct_time when backend
is blacklisted.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
reset_time = project.next_check + timedelta(hours=1)
with mock.patch.dict(self.checker.blacklist_dict,
{"GitHub": reset_time}):
self.checker.update_project(project.id)
self.assertEqual(project.next_check, reset_time)
self.assertEqual(self.checker.ratelimit_counter, 1)
self.assertEqual(self.checker.success_counter, 0)
self.assertEqual(self.checker.error_counter, 0)
self.assertEqual(self.checker.ratelimit_queue["GitHub"][0], project.id)
@mock.patch(
"anitya.lib.utilities.check_project_release",
mock.Mock(side_effect=exceptions.AnityaException("")),
)
def test_update_project_anitya_plugin_exception(self):
"""
Assert that error counter is incremented when `exceptions.AnityaException`
happens.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
self.checker.update_project(project.id)
self.assertEqual(self.checker.ratelimit_counter, 0)
self.assertEqual(self.checker.success_counter, 0)
self.assertEqual(self.checker.error_counter, 1)
@mock.patch(
"anitya.lib.utilities.check_project_release",
mock.Mock(side_effect=exceptions.RateLimitException(
"2008-09-03T20:56:35.450686")),
)
def test_update_project_ratelimit_exception(self):
"""
Assert that project is blacklisted when `exceptions.RatelimitException`
happens.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
reset_time = arrow.get("2008-09-03T20:56:35.450686").datetime
self.checker.update_project(project.id)
self.assertEqual(self.checker.ratelimit_counter, 1)
self.assertEqual(self.checker.success_counter, 0)
self.assertEqual(self.checker.error_counter, 0)
self.assertEqual(self.checker.blacklist_dict["GitHub"], reset_time)
self.assertEqual(self.checker.ratelimit_queue["GitHub"][0], project.id)
@mock.patch("anitya.lib.utilities.check_project_release")
def test_update_project_success(self, mock_check_project_release):
"""
Assert that correct counter is increased when the project is successfully
checked.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
self.checker.update_project(project.id)
self.assertEqual(self.checker.ratelimit_counter, 0)
self.assertEqual(self.checker.success_counter, 1)
self.assertEqual(self.checker.error_counter, 0)
@mock.patch(
"anitya.lib.utilities.check_project_release",
mock.Mock(side_effect=exceptions.AnityaException("")),
)
def test_update_project_error_threshold_reached(self):
"""
Assert that project is deleted when error threshold is reached.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
error_counter=100,
)
self.session.add(project)
self.session.commit()
with fml_testing.mock_sends(anitya_schema.ProjectDeleted):
self.checker.update_project(project.id)
projects = self.session.query(models.Project).all()
self.assertEqual(len(projects), 0)
def test_is_delete_candidate_not_reached(self):
"""
Assert that project is not marked as delete candidate,
if error threshold was not reached.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
result = self.checker.is_delete_candidate(project)
self.assertFalse(result)
def test_is_delete_candidate_mapping_version_exists(self):
"""
Assert that project is not marked as delete candidate,
if it has mapping added and at least one version.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
error_counter=100,
)
self.session.add(project)
self.session.commit()
distro = models.Distro(name="Fedora")
self.session.add(distro)
self.session.commit()
mapping = models.Packages(distro_name="Fedora", project_id=project.id)
self.session.add(mapping)
self.session.commit()
version = models.ProjectVersion(version="1.0.0", project_id=project.id)
self.session.add(version)
self.session.commit()
result = self.checker.is_delete_candidate(project)
self.assertFalse(result)
def test_is_delete_candidate_mapping_no_version(self):
"""
Assert that project is marked as delete candidate,
if it has mapping added and no version.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
error_counter=100,
)
self.session.add(project)
self.session.commit()
distro = models.Distro(name="Fedora")
self.session.add(distro)
self.session.commit()
mapping = models.Packages(distro_name="Fedora", project_id=project.id)
self.session.add(mapping)
self.session.commit()
result = self.checker.is_delete_candidate(project)
self.assertTrue(result)
def test_is_delete_candidate_ready(self):
"""
Assert that project is marked as delete candidate,
if it has reached error threshold and has no mapping.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
error_counter=100,
)
self.session.add(project)
self.session.commit()
result = self.checker.is_delete_candidate(project)
self.assertTrue(result)
def test_blacklist_project(self):
"""
Assert that project is correctly blacklisted.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
reset_time = arrow.get(project.next_check + timedelta(hours=1))
self.checker.blacklist_project(project, reset_time)
self.assertEqual(self.checker.blacklist_dict["GitHub"], reset_time)
self.assertEqual(self.checker.ratelimit_counter, 1)
self.assertEqual(self.checker.success_counter, 0)
self.assertEqual(self.checker.error_counter, 0)
self.assertEqual(self.checker.ratelimit_queue["GitHub"][0], project.id)
def test_run(self):
"""
Assert that `db.Run` is created at the end of run with success.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
def increment(project_id):
self.checker.success_counter = self.checker.success_counter + 1
self.checker.update_project = increment
self.checker.run()
run_objects = models.Run.query.all()
self.assertEqual(len(run_objects), 1)
self.assertEqual(run_objects[0].total_count, 1)
self.assertEqual(run_objects[0].error_count, 0)
self.assertEqual(run_objects[0].ratelimit_count, 0)
self.assertEqual(run_objects[0].success_count, 1)
@mock.patch("anitya.lib.utilities.check_project_release")
@mock.patch("concurrent.futures.Future.exception", return_value=True)
@mock.patch("concurrent.futures.Future.result", side_effect=Exception())
def test_run_thread_exception(self, mock_result, mock_exception,
mock_check_release):
"""
Assert that exception is logged when thread crashed.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
self.checker.run()
mock_exception.assert_called_once()
mock_result.assert_called_once()
@mock.patch("anitya.lib.utilities.check_project_release")
def test_run_nothing_to_check(self, mock_check_project_release):
"""
Assert that nothing is done if no project is ready to check.
"""
self.checker.run()
run_objects = models.Run.query.all()
self.assertEqual(len(run_objects), 0)
mock_check_project_release.assert_not_called()
@mock.patch("anitya.lib.utilities.check_project_release")
@mock.patch.dict("anitya.config.config", {"CRON_POOL": 1})
def test_run_small_pool_size(self, mock_check_project_release):
"""
Assert that small pool size is handled correctly.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
project = models.Project(
name="Fake",
backend="GitHub",
homepage="www.fakeproject1.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
self.checker.run()
run_objects = models.Run.query.all()
self.assertEqual(len(run_objects), 1)
self.assertEqual(run_objects[0].total_count, 2)
@mock.patch("anitya.check_service.as_completed",
side_effect=TimeoutError())
def test_run_timeout(self, mock_as_completed):
"""
Assert that TimeoutError is thrown when TIMEOUT is reached.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
project = models.Project(
name="Fake",
backend="GitHub",
homepage="www.fakeproject1.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
# Don't call actual project_update,
# SQLite doesn't like working with SQLAlchemy objects over multiple threads
def increment(project_id):
pass
self.checker.update_project = increment
self.checker.run()
run_objects = models.Run.query.all()
self.assertEqual(len(run_objects), 1)
self.assertEqual(run_objects[0].total_count, 2)
self.assertEqual(run_objects[0].error_count, 2)
def test_clear_counters(self):
"""
Assert that counters are cleared.
"""
self.checker.error_counter = 10
self.checker.ratelimit_counter = 10
self.checker.success_counter = 10
self.checker.clear_counters()
self.assertEqual(self.checker.error_counter, 0)
self.assertEqual(self.checker.ratelimit_counter, 0)
self.assertEqual(self.checker.success_counter, 0)
def test_construct_queue_not_ready(self):
"""
Assert that no project is added to queue when no project
is ready.
"""
time = arrow.utcnow().datetime
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=time + timedelta(hours=1),
)
self.session.add(project)
self.session.commit()
queue = self.checker.construct_queue(time)
self.assertEqual(len(queue), 0)
def test_construct_queue_duplicates(self):
"""
Assert that duplicate projects aren't added to queue.
"""
time = arrow.utcnow().datetime
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=time,
)
self.session.add(project)
self.session.commit()
self.checker.ratelimit_queue = {"GitHub": [project.id]}
self.checker.blacklist_dict = {"GitHub": time}
queue = self.checker.construct_queue(time + timedelta(hours=1))
self.assertEqual(len(queue), 1)
self.assertEqual(queue[0], project.id)
def test_construct_queue_order(self):
"""
Assert that order of projects is kept in the queue.
"""
time = arrow.utcnow().datetime
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=time,
)
self.session.add(project)
project2 = models.Project(
name="Foobar2",
backend="GitHub",
homepage="www.fakeproject2.com",
next_check=time,
)
self.session.add(project2)
self.session.commit()
self.checker.ratelimit_queue = {"GitHub": [project.id]}
self.checker.blacklist_dict = {"GitHub": time}
queue = self.checker.construct_queue(time + timedelta(hours=1))
self.assertEqual(len(queue), 2)
self.assertEqual(queue[0], project.id)
self.assertEqual(queue[1], project2.id)
def test_construct_queue_archived(self):
"""
Assert that archived project is not added to queue.
"""
time = arrow.utcnow().datetime
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=time,
archived=True,
)
self.session.add(project)
self.session.commit()
queue = self.checker.construct_queue(time + timedelta(hours=1))
self.assertEqual(len(queue), 0)
def test_construct_ratelimit_clean(self):
"""
Assert that ratelimit queue and dictionary is correctly cleared.
"""
time = arrow.utcnow().datetime
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=time,
)
self.session.add(project)
self.session.commit()
self.checker.ratelimit_queue = {"GitHub": [project.id]}
self.checker.blacklist_dict = {"GitHub": time}
self.checker.construct_queue(time + timedelta(hours=1))
self.assertEqual(self.checker.ratelimit_queue.get("GitHub", None),
None)
self.assertEqual(self.checker.blacklist_dict.get("GitHub", None), None)
def test_construct_blacklisted_backend(self):
"""
Assert that project is not added to queue if ready to check, but belonging
to blacklisted backend.
"""
time = arrow.utcnow().datetime
reset_time = time + timedelta(hours=2)
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=time,
)
self.session.add(project)
self.session.commit()
self.checker.blacklist_dict = {"GitHub": reset_time}
queue = self.checker.construct_queue(time + timedelta(hours=1))
self.assertEqual(queue, [])
self.assertEqual(
self.checker.blacklist_dict.get("GitHub", reset_time),
reset_time,
)
class CheckerTests(DatabaseTestCase):
""" Checker class tests. """
def setUp(self):
"""
Prepare the Checker object.
"""
super(CheckerTests, self).setUp()
self.checker = Checker()
@mock.patch("anitya.lib.utilities.check_project_release")
def test_update_project_backend_blacklist(self, mock_check_project_release):
"""
Assert that project next_check is set to correct_time when backend
is blacklisted.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
reset_time = project.next_check + timedelta(hours=1)
with mock.patch.dict(self.checker.blacklist_dict, {"GitHub": reset_time}):
self.checker.update_project(project.id)
self.assertEqual(project.next_check, reset_time)
self.assertEqual(self.checker.ratelimit_counter, 1)
self.assertEqual(self.checker.success_counter, 0)
self.assertEqual(self.checker.error_counter, 0)
self.assertEqual(self.checker.ratelimit_queue["GitHub"][0], project.id)
@mock.patch(
"anitya.lib.utilities.check_project_release",
mock.Mock(side_effect=exceptions.AnityaException("")),
)
def test_update_project_anitya_plugin_exception(self):
"""
Assert that error counter is incremented when `exceptions.AnityaException`
happens.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
self.checker.update_project(project.id)
self.assertEqual(self.checker.ratelimit_counter, 0)
self.assertEqual(self.checker.success_counter, 0)
self.assertEqual(self.checker.error_counter, 1)
@mock.patch(
"anitya.lib.utilities.check_project_release",
mock.Mock(
side_effect=exceptions.RateLimitException("2008-09-03T20:56:35.450686")
),
)
def test_update_project_ratelimit_exception(self):
"""
Assert that project is blacklisted when `exceptions.RatelimitException`
happens.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
reset_time = arrow.get("2008-09-03T20:56:35.450686").datetime
self.checker.update_project(project.id)
self.assertEqual(self.checker.ratelimit_counter, 1)
self.assertEqual(self.checker.success_counter, 0)
self.assertEqual(self.checker.error_counter, 0)
self.assertEqual(self.checker.blacklist_dict["GitHub"], reset_time)
self.assertEqual(self.checker.ratelimit_queue["GitHub"][0], project.id)
@mock.patch("anitya.lib.utilities.check_project_release")
def test_update_project_success(self, mock_check_project_release):
"""
Assert that correct counter is increased when the project is successfully
checked.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
self.checker.update_project(project.id)
self.assertEqual(self.checker.ratelimit_counter, 0)
self.assertEqual(self.checker.success_counter, 1)
self.assertEqual(self.checker.error_counter, 0)
def test_blacklist_project(self):
"""
Assert that project is correctly blacklisted.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
reset_time = arrow.get(project.next_check + timedelta(hours=1))
self.checker.blacklist_project(project, reset_time)
self.assertEqual(self.checker.blacklist_dict["GitHub"], reset_time)
self.assertEqual(self.checker.ratelimit_counter, 1)
self.assertEqual(self.checker.success_counter, 0)
self.assertEqual(self.checker.error_counter, 0)
self.assertEqual(self.checker.ratelimit_queue["GitHub"][0], project.id)
def test_run(self):
"""
Assert that `db.Run` is created at the end of run with success.
"""
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=arrow.utcnow().datetime,
)
self.session.add(project)
self.session.commit()
def increment(project_id):
self.checker.success_counter = self.checker.success_counter + 1
self.checker.update_project = increment
self.checker.run()
run_objects = models.Run.query.all()
self.assertEqual(len(run_objects), 1)
self.assertEqual(run_objects[0].total_count, 1)
self.assertEqual(run_objects[0].error_count, 0)
self.assertEqual(run_objects[0].ratelimit_count, 0)
self.assertEqual(run_objects[0].success_count, 1)
@mock.patch("anitya.lib.utilities.check_project_release")
def test_run_nothing_to_check(self, mock_check_project_release):
"""
Assert that nothing is done if no project is ready to check.
"""
self.checker.run()
run_objects = models.Run.query.all()
self.assertEqual(len(run_objects), 0)
mock_check_project_release.assert_not_called()
def test_clear_counters(self):
"""
Assert that counters are cleared.
"""
self.checker.error_counter = 10
self.checker.ratelimit_counter = 10
self.checker.success_counter = 10
self.checker.clear_counters()
self.assertEqual(self.checker.error_counter, 0)
self.assertEqual(self.checker.ratelimit_counter, 0)
self.assertEqual(self.checker.success_counter, 0)
def test_construct_queue_not_ready(self):
"""
Assert that no project is added to queue when no project
is ready.
"""
time = arrow.utcnow().datetime
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=time + timedelta(hours=1),
)
self.session.add(project)
self.session.commit()
queue = self.checker.construct_queue(time)
self.assertEqual(len(queue), 0)
def test_construct_queue_duplicates(self):
"""
Assert that duplicate projects aren't added to queue.
"""
time = arrow.utcnow().datetime
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=time,
)
self.session.add(project)
self.session.commit()
self.checker.ratelimit_queue = {"Github": [project.id]}
self.checker.blacklist_dict = {"Github": time}
queue = self.checker.construct_queue(time + timedelta(hours=1))
self.assertEqual(len(queue), 1)
self.assertEqual(queue[0], project.id)
def test_construct_queue_order(self):
"""
Assert that order of projects is kept in the queue.
"""
time = arrow.utcnow().datetime
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=time,
)
self.session.add(project)
project2 = models.Project(
name="Foobar2",
backend="GitHub",
homepage="www.fakeproject2.com",
next_check=time,
)
self.session.add(project2)
self.session.commit()
self.checker.ratelimit_queue = {"Github": [project.id]}
self.checker.blacklist_dict = {"Github": time}
queue = self.checker.construct_queue(time + timedelta(hours=1))
self.assertEqual(len(queue), 2)
self.assertEqual(queue[0], project.id)
self.assertEqual(queue[1], project2.id)
def test_construct_ratelimit_clean(self):
"""
Assert that ratelimit queue and dictionary is correctly cleared.
"""
time = arrow.utcnow().datetime
project = models.Project(
name="Foobar",
backend="GitHub",
homepage="www.fakeproject.com",
next_check=time,
)
self.session.add(project)
self.session.commit()
self.checker.ratelimit_queue = {"Github": [project.id]}
self.checker.blacklist_dict = {"Github": time}
self.checker.construct_queue(time + timedelta(hours=1))
self.assertEqual(self.checker.ratelimit_queue.get("Github", None), None)
self.assertEqual(self.checker.blacklist_dict.get("Github", None), None)
def test_reset_time(self):
"""Assert the property returns valid value."""
time = "2018-08-24T09:36:15Z"
exp = arrow.get(time)
e = exceptions.RateLimitException(time)
self.assertEqual(exp, e.reset_time)
