def test_with_some_resources_checked_recently_and_some_never(self):
"""
If there are 5 resources that have been checked in last 24 hours and 5
that have never been checked and 10 resources are requested, it should
return the 5 that have not been checked, sorted oldest-resource-first.
"""
resource_1 = factories.Resource()['id']
resource_2 = factories.Resource()['id']
resource_3 = factories.Resource()['id']
resource_4 = factories.Resource()['id']
resource_5 = factories.Resource()['id']
twenty_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=23)
results.upsert(resource_1, True, last_checked=twenty_hours_ago)
results.upsert(resource_2, True, last_checked=twenty_hours_ago)
results.upsert(resource_3, True, last_checked=twenty_hours_ago)
results.upsert(resource_4, True, last_checked=twenty_hours_ago)
results.upsert(resource_5, True, last_checked=twenty_hours_ago)
resource_6 = factories.Resource()['id']
resource_7 = factories.Resource()['id']
resource_8 = factories.Resource()['id']
resource_9 = factories.Resource()['id']
resource_10 = factories.Resource()['id']
resources_to_check = results.get_resources_to_check(10)
assert resources_to_check == [resource_6, resource_7, resource_8,
resource_9, resource_10]
def test_with_some_resources_checked_recently_and_some_never(self):
"""
If there are 5 resources that have been checked in last 24 hours and 5
that have never been checked and 10 resources are requested, it should
return the 5 that have not been checked, sorted oldest-resource-first.
"""
resource_1 = factories.Resource()['id']
resource_2 = factories.Resource()['id']
resource_3 = factories.Resource()['id']
resource_4 = factories.Resource()['id']
resource_5 = factories.Resource()['id']
twenty_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=23)
results.upsert(resource_1, True, last_checked=twenty_hours_ago)
results.upsert(resource_2, True, last_checked=twenty_hours_ago)
results.upsert(resource_3, True, last_checked=twenty_hours_ago)
results.upsert(resource_4, True, last_checked=twenty_hours_ago)
results.upsert(resource_5, True, last_checked=twenty_hours_ago)
resource_6 = factories.Resource()['id']
resource_7 = factories.Resource()['id']
resource_8 = factories.Resource()['id']
resource_9 = factories.Resource()['id']
resource_10 = factories.Resource()['id']
resources_to_check = results.get_resources_to_check(10)
assert resources_to_check == [
resource_6, resource_7, resource_8, resource_9, resource_10
]
def test_that_it_does_not_return_resources_with_pending_checks(self):
"""Resources with pending checks < 2 hours old should not be returned.
"""
now = datetime.datetime.utcnow()
# Create 5 resources that have been checked in the last 24 hours.
resource_1 = factories.Resource()['id']
resource_2 = factories.Resource()['id']
resource_3 = factories.Resource()['id']
resource_4 = factories.Resource()['id']
resource_5 = factories.Resource()['id']
twenty_hours_ago = now - datetime.timedelta(hours=20)
results.upsert(resource_1, True, last_checked=twenty_hours_ago)
results.upsert(resource_2, True, last_checked=twenty_hours_ago)
results.upsert(resource_3, True, last_checked=twenty_hours_ago)
results.upsert(resource_4, True, last_checked=twenty_hours_ago)
results.upsert(resource_5, True, last_checked=twenty_hours_ago)
# Create 5 resources with pending checks from < 2 hours ago.
resource_6 = factories.Resource()['id']
resource_7 = factories.Resource()['id']
resource_8 = factories.Resource()['id']
resource_9 = factories.Resource()['id']
resource_10 = factories.Resource()['id']
one_hour_ago = now - datetime.timedelta(hours=1)
results._make_pending(
[resource_6, resource_7, resource_8, resource_9, resource_10],
one_hour_ago)
# Create 5 resources that were last checked more than 24 hours ago.
resource_11 = factories.Resource()['id']
resource_12 = factories.Resource()['id']
resource_13 = factories.Resource()['id']
resource_14 = factories.Resource()['id']
resource_15 = factories.Resource()['id']
results.upsert(resource_11,
True,
last_checked=now - datetime.timedelta(hours=35))
results.upsert(resource_12,
True,
last_checked=now - datetime.timedelta(hours=34))
results.upsert(resource_13,
True,
last_checked=now - datetime.timedelta(hours=33))
results.upsert(resource_14,
True,
last_checked=now - datetime.timedelta(hours=32))
results.upsert(resource_15,
True,
last_checked=now - datetime.timedelta(hours=31))
resources_to_check = results.get_resources_to_check(10)
assert resources_to_check == [
resource_11, resource_12, resource_13, resource_14, resource_15
]
def test_custom_longer_pending_since(self):
"""If given a longer ``pending_since`` time it should not return
resources that have more recent pending checks."""
test_resource = factories.Resource()['id']
three_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=3)
results._make_pending([test_resource], three_hours_ago)
results_ = results.get_resources_to_check(
10, pending_since=datetime.timedelta(hours=4))
assert results_ == []
def test_custom_longer_since(self):
"""If given a longer ``since`` time it should not return resources that
were checked more recently."""
test_resource = factories.Resource()['id']
thirty_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=30)
results.upsert(test_resource, True, last_checked=thirty_hours_ago)
results_ = results.get_resources_to_check(
10, since=datetime.timedelta(hours=48))
assert results_ == []
def test_custom_longer_since(self):
"""If given a longer ``since`` time it should not return resources that
were checked more recently."""
test_resource = factories.Resource()['id']
thirty_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=30)
results.upsert(test_resource, True, last_checked=thirty_hours_ago)
results_ = results.get_resources_to_check(
10, since=datetime.timedelta(hours=48))
assert results_ == []
def test_custom_shorter_pending_since(self):
"""If given a shorter ``pending_since`` time it should return
resources that have more recent pending checks."""
test_resource = factories.Resource()['id']
one_hour_ago = datetime.datetime.utcnow() - datetime.timedelta(hours=1)
results._make_pending([test_resource], one_hour_ago)
results_ = results.get_resources_to_check(
10, pending_since=datetime.timedelta(hours=0.5))
assert len(results_) == 1
assert results_[0] == test_resource
def test_custom_longer_pending_since(self):
"""If given a longer ``pending_since`` time it should not return
resources that have more recent pending checks."""
test_resource = factories.Resource()['id']
three_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=3)
results._make_pending([test_resource], three_hours_ago)
results_ = results.get_resources_to_check(
10, pending_since=datetime.timedelta(hours=4))
assert results_ == []
def test_custom_shorter_since(self):
"""If given a shorter ``since`` time it should return resources that
have been checked more recently."""
test_resource = factories.Resource()['id']
ten_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=10)
results.upsert(test_resource, True, last_checked=ten_hours_ago)
results_ = results.get_resources_to_check(
10, since=datetime.timedelta(hours=5))
assert len(results_) == 1
assert results_[0] == test_resource
def test_custom_shorter_pending_since(self):
"""If given a shorter ``pending_since`` time it should return
resources that have more recent pending checks."""
test_resource = factories.Resource()['id']
one_hour_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=1)
results._make_pending([test_resource], one_hour_ago)
results_ = results.get_resources_to_check(
10, pending_since=datetime.timedelta(hours=0.5))
assert len(results_) == 1
assert results_[0] == test_resource
def test_custom_shorter_since(self):
"""If given a shorter ``since`` time it should return resources that
have been checked more recently."""
test_resource = factories.Resource()['id']
ten_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=10)
results.upsert(test_resource, True, last_checked=ten_hours_ago)
results_ = results.get_resources_to_check(
10, since=datetime.timedelta(hours=5))
assert len(results_) == 1
assert results_[0] == test_resource
def test_that_it_does_not_return_resources_with_pending_checks(self):
"""Resources with pending checks < 2 hours old should not be returned.
"""
now = datetime.datetime.utcnow()
# Create 5 resources that have been checked in the last 24 hours.
resource_1 = factories.Resource()['id']
resource_2 = factories.Resource()['id']
resource_3 = factories.Resource()['id']
resource_4 = factories.Resource()['id']
resource_5 = factories.Resource()['id']
twenty_hours_ago = now - datetime.timedelta(hours=20)
results.upsert(resource_1, True, last_checked=twenty_hours_ago)
results.upsert(resource_2, True, last_checked=twenty_hours_ago)
results.upsert(resource_3, True, last_checked=twenty_hours_ago)
results.upsert(resource_4, True, last_checked=twenty_hours_ago)
results.upsert(resource_5, True, last_checked=twenty_hours_ago)
# Create 5 resources with pending checks from < 2 hours ago.
resource_6 = factories.Resource()['id']
resource_7 = factories.Resource()['id']
resource_8 = factories.Resource()['id']
resource_9 = factories.Resource()['id']
resource_10 = factories.Resource()['id']
one_hour_ago = now - datetime.timedelta(hours=1)
results._make_pending(
[resource_6, resource_7, resource_8, resource_9, resource_10],
one_hour_ago)
# Create 5 resources that were last checked more than 24 hours ago.
resource_11 = factories.Resource()['id']
resource_12 = factories.Resource()['id']
resource_13 = factories.Resource()['id']
resource_14 = factories.Resource()['id']
resource_15 = factories.Resource()['id']
results.upsert(resource_11, True,
last_checked=now - datetime.timedelta(hours=35))
results.upsert(resource_12, True,
last_checked=now - datetime.timedelta(hours=34))
results.upsert(resource_13, True,
last_checked=now - datetime.timedelta(hours=33))
results.upsert(resource_14, True,
last_checked=now - datetime.timedelta(hours=32))
results.upsert(resource_15, True,
last_checked=now - datetime.timedelta(hours=31))
resources_to_check = results.get_resources_to_check(10)
assert resources_to_check == [resource_11, resource_12, resource_13,
resource_14, resource_15]
def test_that_it_creates_pending_checks(self):
"""get_resources_to_check() should create pending link checker results
for all the resources it returns."""
# A resource that has never been checked.
resource_1 = factories.Resource()['id']
# A resource that was checked > 24 hours ago.
resource_2 = factories.Resource()['id']
thirty_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=30)
results.upsert(resource_2, True, last_checked=thirty_hours_ago)
# A resource with a pending check from > 2 hours ago.
resource_3 = factories.Resource()['id']
three_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=3)
results._make_pending([resource_3], three_hours_ago)
results.get_resources_to_check(10)
for resource in (resource_1, resource_2, resource_3):
result = results.get(resource)
assert result["pending"] is True
def test_that_it_creates_pending_checks(self):
"""get_resources_to_check() should create pending link checker results
for all the resources it returns."""
# A resource that has never been checked.
resource_1 = factories.Resource()['id']
# A resource that was checked > 24 hours ago.
resource_2 = factories.Resource()['id']
thirty_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=30)
results.upsert(resource_2, True, last_checked=thirty_hours_ago)
# A resource with a pending check from > 2 hours ago.
resource_3 = factories.Resource()['id']
three_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=3)
results._make_pending([resource_3], three_hours_ago)
results.get_resources_to_check(10)
for resource in (resource_1, resource_2, resource_3):
result = results.get(resource)
assert result["pending"] is True
def test_with_some_resources_checked_recently_and_some_not_recently(self):
"""
If there are 5 resources that have been checked in last 24 hours and 5
that were last checked more than 24 hours ago and 10 resources are
requested, it should return the 5 that have not been checked recently,
sorted most-recently-checked last.
"""
now = datetime.datetime.utcnow()
resource_1 = factories.Resource()['id']
resource_2 = factories.Resource()['id']
resource_3 = factories.Resource()['id']
resource_4 = factories.Resource()['id']
resource_5 = factories.Resource()['id']
twenty_hours_ago = now - datetime.timedelta(hours=23)
results.upsert(resource_1, True, last_checked=twenty_hours_ago)
results.upsert(resource_2, True, last_checked=twenty_hours_ago)
results.upsert(resource_3, True, last_checked=twenty_hours_ago)
results.upsert(resource_4, True, last_checked=twenty_hours_ago)
results.upsert(resource_5, True, last_checked=twenty_hours_ago)
resource_6 = factories.Resource()['id']
resource_7 = factories.Resource()['id']
resource_8 = factories.Resource()['id']
resource_9 = factories.Resource()['id']
resource_10 = factories.Resource()['id']
# We mix up the order in which these resources were checked a bit.
results.upsert(resource_7,
True,
last_checked=now - datetime.timedelta(hours=34))
results.upsert(resource_6,
True,
last_checked=now - datetime.timedelta(hours=33))
results.upsert(resource_9,
True,
last_checked=now - datetime.timedelta(hours=32))
results.upsert(resource_10,
True,
last_checked=now - datetime.timedelta(hours=31))
results.upsert(resource_8,
True,
last_checked=now - datetime.timedelta(hours=30))
resources_to_check = results.get_resources_to_check(10)
assert resources_to_check == [
resource_7, resource_6, resource_9, resource_10, resource_8
]
def test_with_5_new_resources_and_request_10(self):
"""
If there are 5 new resources (that have never been checked before) and
10 resources to check are requested, the 5 should be returned in
oldest-first order.
"""
resource_1 = factories.Resource()['id']
resource_2 = factories.Resource()['id']
resource_3 = factories.Resource()['id']
resource_4 = factories.Resource()['id']
resource_5 = factories.Resource()['id']
resources_to_check = results.get_resources_to_check(10)
assert resources_to_check == [resource_1, resource_2, resource_3,
resource_4, resource_5]
def get_resources_to_check(context, data_dict):
"""Return a list of up to ``n`` resource IDs to be checked.
Returns up to ``n`` resource IDs to be checked for broken links.
Resources that have not been checked before will be returned first, oldest
resources first.
Resources that have not been checked in the last 24 hours (configurable:
``ckanext.deadoralive.recheck_resources_after``) will be returned next,
most-recently-checked resources last.
As soon as a resource's ID is returned by this function that resource is
considered to have a "pending" check (we are expecting to receive a link
check result for that resource soon). Resources with pending checks will
not be returned by this function again for at least 2 hours (configurable:
``ckanext.deadoralive.resend_pending_resources_after``).
:param n: the maximum number of resources to return at once
:type n: int
:rtype: list of strings
"""
toolkit.check_access("ckanext_deadoralive_get_resources_to_check", context,
data_dict)
# TODO: Validation.
recheck_resources_after = config.recheck_resources_after
since_delta = datetime.timedelta(hours=recheck_resources_after)
resend_pending_resources_after = (config.resend_pending_resources_after)
pending_since_delta = datetime.timedelta(
hours=resend_pending_resources_after)
n = data_dict.get("n", 50)
orgs = config.organization_to_filter #Orgs to filter from config.py
org_check = data_dict.get("org_check", "All")
return results.get_resources_to_check(n,
orgs,
org_check,
since=since_delta,
pending_since=pending_since_delta)
def test_with_5_new_resources_and_request_10(self):
"""
If there are 5 new resources (that have never been checked before) and
10 resources to check are requested, the 5 should be returned in
oldest-first order.
"""
resource_1 = factories.Resource()['id']
resource_2 = factories.Resource()['id']
resource_3 = factories.Resource()['id']
resource_4 = factories.Resource()['id']
resource_5 = factories.Resource()['id']
resources_to_check = results.get_resources_to_check(10)
assert resources_to_check == [
resource_1, resource_2, resource_3, resource_4, resource_5
]
def test_with_some_resources_checked_recently_and_some_not_recently(self):
"""
If there are 5 resources that have been checked in last 24 hours and 5
that were last checked more than 24 hours ago and 10 resources are
requested, it should return the 5 that have not been checked recently,
sorted most-recently-checked last.
"""
now = datetime.datetime.utcnow()
resource_1 = factories.Resource()['id']
resource_2 = factories.Resource()['id']
resource_3 = factories.Resource()['id']
resource_4 = factories.Resource()['id']
resource_5 = factories.Resource()['id']
twenty_hours_ago = now - datetime.timedelta(hours=23)
results.upsert(resource_1, True, last_checked=twenty_hours_ago)
results.upsert(resource_2, True, last_checked=twenty_hours_ago)
results.upsert(resource_3, True, last_checked=twenty_hours_ago)
results.upsert(resource_4, True, last_checked=twenty_hours_ago)
results.upsert(resource_5, True, last_checked=twenty_hours_ago)
resource_6 = factories.Resource()['id']
resource_7 = factories.Resource()['id']
resource_8 = factories.Resource()['id']
resource_9 = factories.Resource()['id']
resource_10 = factories.Resource()['id']
# We mix up the order in which these resources were checked a bit.
results.upsert(
resource_7, True, last_checked=now - datetime.timedelta(hours=34))
results.upsert(
resource_6, True, last_checked=now - datetime.timedelta(hours=33))
results.upsert(
resource_9, True, last_checked=now - datetime.timedelta(hours=32))
results.upsert(
resource_10, True, last_checked=now - datetime.timedelta(hours=31))
results.upsert(
resource_8, True, last_checked=now - datetime.timedelta(hours=30))
resources_to_check = results.get_resources_to_check(10)
assert resources_to_check == [resource_7, resource_6, resource_9,
resource_10, resource_8]
def get_resources_to_check(context, data_dict):
"""Return a list of up to ``n`` resource IDs to be checked.
Returns up to ``n`` resource IDs to be checked for broken links.
Resources that have not been checked before will be returned first, oldest
resources first.
Resources that have not been checked in the last 24 hours (configurable:
``ckanext.deadoralive.recheck_resources_after``) will be returned next,
most-recently-checked resources last.
As soon as a resource's ID is returned by this function that resource is
considered to have a "pending" check (we are expecting to receive a link
check result for that resource soon). Resources with pending checks will
not be returned by this function again for at least 2 hours (configurable:
``ckanext.deadoralive.resend_pending_resources_after``).
:param n: the maximum number of resources to return at once
:type n: int
:rtype: list of strings
"""
toolkit.check_access("ckanext_deadoralive_get_resources_to_check",
context, data_dict)
# TODO: Validation.
recheck_resources_after = config.recheck_resources_after
since_delta = datetime.timedelta(hours=recheck_resources_after)
resend_pending_resources_after = (
config.resend_pending_resources_after)
pending_since_delta = datetime.timedelta(
hours=resend_pending_resources_after)
n = data_dict.get("n", 50)
return results.get_resources_to_check(n, since=since_delta,
pending_since=pending_since_delta)
def test_that_it_does_return_resources_with_expired_pending_checks(self):
"""Resources with pending checks > 2 hours old should be returned.
And they should be sorted oldest-pending-check-first.
"""
# Create 5 resources with pending checks from > 2 hours ago.
resource_1 = factories.Resource()['id']
resource_2 = factories.Resource()['id']
resource_3 = factories.Resource()['id']
resource_4 = factories.Resource()['id']
resource_5 = factories.Resource()['id']
five_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=5)
results._make_pending(
[resource_1, resource_2, resource_3, resource_4, resource_5],
five_hours_ago)
resources_to_check = results.get_resources_to_check(10)
assert resources_to_check == [resource_1, resource_2, resource_3,
resource_4, resource_5]
def test_when_all_resources_have_been_checked_recently(self):
"""
If there are 5 resources and they have all been checked in last 24 hours
then it should return an empty list.
"""
resource_1 = factories.Resource()['id']
resource_2 = factories.Resource()['id']
resource_3 = factories.Resource()['id']
resource_4 = factories.Resource()['id']
resource_5 = factories.Resource()['id']
twenty_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=23)
results.upsert(resource_1, True, last_checked=twenty_hours_ago)
results.upsert(resource_2, True, last_checked=twenty_hours_ago)
results.upsert(resource_3, True, last_checked=twenty_hours_ago)
results.upsert(resource_4, True, last_checked=twenty_hours_ago)
results.upsert(resource_5, True, last_checked=twenty_hours_ago)
resources_to_check = results.get_resources_to_check(10)
assert resources_to_check == []
def test_that_it_does_return_resources_with_expired_pending_checks(self):
"""Resources with pending checks > 2 hours old should be returned.
And they should be sorted oldest-pending-check-first.
"""
# Create 5 resources with pending checks from > 2 hours ago.
resource_1 = factories.Resource()['id']
resource_2 = factories.Resource()['id']
resource_3 = factories.Resource()['id']
resource_4 = factories.Resource()['id']
resource_5 = factories.Resource()['id']
five_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=5)
results._make_pending(
[resource_1, resource_2, resource_3, resource_4, resource_5],
five_hours_ago)
resources_to_check = results.get_resources_to_check(10)
assert resources_to_check == [
resource_1, resource_2, resource_3, resource_4, resource_5
]
def test_with_10_new_resources_and_request_5(self):
"""
If there are 10 new resources (that have never been checked before) and
5 resources to check are requested, the oldest 5 should be returned in
oldest-first order.
"""
resource_1 = factories.Resource()['id']
resource_2 = factories.Resource()['id']
resource_3 = factories.Resource()['id']
resource_4 = factories.Resource()['id']
resource_5 = factories.Resource()['id']
factories.Resource()['id']
factories.Resource()['id']
factories.Resource()['id']
factories.Resource()['id']
factories.Resource()['id']
orgs = config.organization_to_filter
resources_to_check = results.get_resources_to_check(5, orgs)
assert resources_to_check == [
resource_1, resource_2, resource_3, resource_4, resource_5
]
def test_when_all_resources_have_been_checked_recently(self):
"""
If there are 5 resources and they have all been checked in last 24 hours
then it should return an empty list.
"""
resource_1 = factories.Resource()['id']
resource_2 = factories.Resource()['id']
resource_3 = factories.Resource()['id']
resource_4 = factories.Resource()['id']
resource_5 = factories.Resource()['id']
twenty_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=23)
results.upsert(resource_1, True, last_checked=twenty_hours_ago)
results.upsert(resource_2, True, last_checked=twenty_hours_ago)
results.upsert(resource_3, True, last_checked=twenty_hours_ago)
results.upsert(resource_4, True, last_checked=twenty_hours_ago)
results.upsert(resource_5, True, last_checked=twenty_hours_ago)
resources_to_check = results.get_resources_to_check(10)
assert resources_to_check == []
