def test_hours_one_accepted_one_rejected(self):
f = TimeFilter(rules={"hours": 1})
fse1 = FilterItem(modtime=time.time()-60*60*1.5)
fse2 = FilterItem(modtime=time.time()-60*60*1.6)
a, r = f.filter(objs=[fse1, fse2])
r = list(r)
# The younger one must be accepted.
assert a[0] == fse1
assert len(a) == 1
assert r[0] == fse2
assert len(r) == 1
def test_2_years_2_allowed(self):
# Request to keep more than available.
# Produce one 1 year old, one 2 year old, keep 10 years.
nowminus10years = time.time() - (60*60*24*365 * 2 + 1)
nowminus09years = time.time() - (60*60*24*365 * 1 + 1)
fse1 = FilterItem(modtime=nowminus10years)
fse2 = FilterItem(modtime=nowminus09years)
a, r = TimeFilter(rules={"years": 2}).filter(objs=[fse1, fse2])
r = list(r)
# All should be accepted.
assert len(a) == 2
assert len(r) == 0
def test_two_recent(self):
fse1 = FilterItem(modtime=time.time())
time.sleep(SHORTTIME)
fse2 = FilterItem(modtime=time.time())
# fse2 is a little younger than fse1.
time.sleep(SHORTTIME) # Make sure ref is newer than fse2.modtime.
a, r = TimeFilter(rules={"recent": 1}).filter(objs=[fse1, fse2])
r = list(r)
# The younger one must be accepted.
assert a[0] == fse2
assert len(a) == 1
assert r[0] == fse1
assert len(r) == 1
def test_2_recent_10_allowed(self):
# Request to keep more than available.
fse1 = FilterItem(modtime=time.time())
time.sleep(SHORTTIME)
fse2 = FilterItem(modtime=time.time())
time.sleep(SHORTTIME)
a, r = TimeFilter(rules={"recent": 10}).filter(objs=[fse1, fse2])
r = list(r)
# All should be accepted. Within `recent` category,
# items must be sorted by modtime, with the newest element being the
# last element.
assert a[0] == fse1
assert a[1] == fse2
assert len(a) == 2
assert len(r) == 0
def test_periodic_filtering_is_stable(self):
start = datetime(2016, 1, 1, 23, 0)
items = []
# create items every 57 minutes, old enough, so they don't fall in the
# 'recent' category. Choose the times so that they initially fall into
# separate buckets when the timefilter is run at start.
previous = start - timedelta(hours=3, minutes=20)
for _ in range(3):
items.append(FilterItem(moddate=previous))
previous = previous - timedelta(minutes=57)
rules = { "hours": 50 } # within the test period, everything should be
# retained.
# check precondition: when filtered at start, each item falls ito a
# separate bucket -- in other words, items is what could have been left
# by the 50-hours rule run at start time, even though the individual
# items are less than 60 minutes apart.
a, _ = TimeFilter(rules, start).filter(items)
assert len(a) == len(items)
# check stability: given that the oldest item is far from being dropped
# (way newer than 50 hours), every run within the following hour should
# leave what was left by the first run.
current = start
for _ in range(60):
a, _ = TimeFilter(rules, current).filter(items)
assert len(a) == len(items)
current = current + timedelta(minutes=1)
def test_10_days_2_weeks(self):
# Further define category 'overlap' behavior. {"days": 10, "weeks": 2}
# -> week 0 is included in the 10 days, week 1 is only partially
# included in the 10 days, and week 2 (14 days and older) is not
# included in the 10 days.
# Having 15 FSEs, 1 to 15 days in age, the first 10 of them must be
# accepted according to the 10-day-rule. According to the 2-weeks-rule,
# the 7th and 14th FSEs must be accepted. The 7th FSE is included in
# the first 10, so items 1-10 and 14 are the accepted ones.
now = datetime(2016, 1, 3)
nowminusXdays = (now - timedelta(days=i)
for i in range(1, 16))
fses = [FilterItem(moddate=d) for d in nowminusXdays]
rules = {"days": 10, "weeks": 2}
a, r = TimeFilter(rules, now).filter(fses)
r = list(r)
assert len(a) == 11
# Check if first 11 fses are in accepted list (order can be predicted
# according to current implementation, but should not be tested, as it
# is not guaranteed according to the current specification).
for fse in fses[:10]:
assert fse in a
# Check if 14th FSE is accepted.
assert fses[13] in a
# Check if FSEs 12, 13, 15 are rejected.
assert len(r) == 4
for i in (10, 11, 12, 14):
assert fses[i] in r
def test_10_days_2_weeks(self):
# Further define category 'overlap' behavior. {"days": 10, "weeks": 2}
# -> week 0 is included in the 10 days, week 1 is only partially
# included in the 10 days, and week 2 (14 days and older) is not
# included in the 10 days.
# Having 15 FSEs, 1 to 15 days in age, the first 10 of them must be
# accepted according to the 10-day-rule. The 11th, 12th, 13th FSE (11,
# 12, 13 days old) are categorized as 1 week old (their age A fulfills
# 7 days <= A < 14 days). According to the 2-weeks-rule, the most
# recent 1-week-old not affected by younger categories has to be
# accepted, which is the 11th FSE. Also according to the 2-weeks-rule,
# the most recent 2-week-old (not affected by a younger category, this
# is always condition) has to be accepted, which is the 14th FSE.
# In total FSEs 1-11,14 must be accepted, i.e. 12 FSEs. 15 FSEs are
# used as input (1-15 days old), i.e. 3 are to be rejected (FSEs 12,
# 13, 15).
now = time.time()
nowminusXdays = (now-(60*60*24*i+1) for i in range(1, 16))
fses = [FilterItem(modtime=t) for t in nowminusXdays]
rules = {"days": 10, "weeks": 2}
a, r = TimeFilter(rules, now).filter(fses)
r = list(r)
assert len(a) == 12
# Check if first 11 fses are in accepted list (order can be predicted
# according to current implementation, but should not be tested, as it
# is not guaranteed according to the current specification).
for fse in fses[:11]:
assert fse in a
# Check if 14th FSE is accepted.
assert fses[13] in a
# Check if FSEs 12, 13, 15 are rejected.
assert len(r) == 3
for i in (11, 12, 14):
assert fses[i] in r
def test_10_days_order(self):
# Having 15 FSEs, 1 to 15 days in age, the first 10 of them must be
# accepted according to the 10-day-rule. The last 5 must be rejected.
# This test is focused on the right internal ordering when making the
# decision to accept or reject an item. The newest ones are expected to
# be accepted, while the oldest ones are expected to be rejected.
# In order to test robustness against input order, the list of mock
# FSEs is shuffled before filtering. The filtering and checks are
# repeated a couple of times.
# It is tested whether all of the youngest 10 FSEs are accepted. It is
# not tested if these 10 FSEs have a certain order within the accepted-
# list, because we don't make any guarantees about the
# accepted-internal ordering.
now = time.time()
nowminusXdays = (now-(60*60*24*i+1) for i in range(1, 16))
fses = [FilterItem(modtime=t) for t in nowminusXdays]
rules = {"days": 10}
shuffledfses = fses[:]
for _ in range(100):
shuffle(shuffledfses)
a, r = TimeFilter(rules, now).filter(shuffledfses)
r = list(r)
assert len(a) == 10
assert len(r) == 5
for fse in fses[:10]:
assert fse in a
for fse in fses[10:]:
assert fse in r
def test_create_recent_allow_old(self):
now = time.time()
nowminusXseconds = (now - (i + 1) for i in range(1, 16))
fses = [FilterItem(modtime=t) for t in nowminusXseconds]
rules = {"years": 1}
a, r = TimeFilter(rules, now).filter(fses)
r = list(r)
assert len(a) == 0
assert len(r) == 15
def test_all_categories_1acc_1rej(self):
now = time.time()
nowminus1year = now - (60*60*24*365 * 1 + 1)
nowminus1month = now - (60*60*24*30 * 1 + 1)
nowminus1week = now - (60*60*24*7 * 1 + 1)
nowminus1day = now - (60*60*24 * 1 + 1)
nowminus1hour = now - (60*60 * 1 + 1)
nowminus1second = now - 1
nowminus2year = now - (60*60*24*365 * 2 + 1)
nowminus2month = now - (60*60*24*30 * 2 + 1)
nowminus2week = now - (60*60*24*7 * 2 + 1)
nowminus2day = now - (60*60*24 * 2 + 1)
nowminus2hour = now - (60*60 * 2 + 1)
nowminus2second = now - 2
atimes = (
nowminus1year,
nowminus1month,
nowminus1week,
nowminus1day,
nowminus1hour,
nowminus1second,
)
rtimes = (
nowminus2year,
nowminus2month,
nowminus2week,
nowminus2day,
nowminus2hour,
nowminus2second,
)
afses = [FilterItem(modtime=t) for t in atimes]
rfses = [FilterItem(modtime=t) for t in rtimes]
cats = ("days", "years", "months", "weeks", "hours", "recent")
rules = {c:1 for c in cats}
a, r = TimeFilter(rules, now).filter(chain(afses, rfses))
r = list(r)
# All nowminus1* must be accepted, all nowminus2* must be rejected.
assert len(a) == 6
for fse in afses:
assert fse in a
for fse in rfses:
assert fse in r
assert len(r) == 6
def test_create_recent_dont_request_recent(self):
# Create a few young items (recent ones). Then don't request any.
now = time.time()
nowminusXseconds = (now - (i + 1) for i in range(1, 16))
fses = [FilterItem(modtime=t) for t in nowminusXseconds]
rules = {"years": 1, "recent": 0}
a, r = TimeFilter(rules, now).filter(fses)
r = list(r)
assert len(a) == 0
assert len(r) == 15
def test_create_old_allow_recent(self):
# Create a few old items, between 1 and 15 years. Then only request one
# recent item. This discovered a mean bug, where items to be rejected
# ended up in the recent category.
now = time.time()
nowminusXyears = (now-(60*60*24*365 * i + 1) for i in range(1, 16))
fses = [FilterItem(modtime=t) for t in nowminusXyears]
rules = {"recent": 1}
a, r = TimeFilter(rules, now).filter(fses)
r = list(r)
assert len(a) == 0
assert len(r) == 15
def test_all_categories_1acc_1rej(self):
# test multiple categories -- for simplicity make sure that periods
# don't overlap
now = datetime(2015, 12, 31, 12, 30, 45)
nowminus1year = datetime(2014, 12, 31)
nowminus2year = datetime(2013, 12, 31)
nowminus1month = datetime(2015, 11, 30)
nowminus2month = datetime(2015, 10, 31)
nowminus1week = datetime(2015, 12, 24)
nowminus2week = datetime(2015, 12, 17)
nowminus1day = datetime(2015, 12, 30)
nowminus2day = datetime(2015, 12, 29)
nowminus1hour = datetime(2015, 12, 31, 11, 30)
nowminus2hour = datetime(2015, 12, 31, 10, 30)
nowminus1second = datetime(2015, 12, 31, 12, 30, 44)
nowminus2second = datetime(2015, 12, 31, 12, 30, 43)
adates = (
nowminus1year,
nowminus1month,
nowminus1week,
nowminus1day,
nowminus1hour,
nowminus1second,
)
rdates = (
nowminus2year,
nowminus2month,
nowminus2week,
nowminus2day,
nowminus2hour,
nowminus2second,
)
afses = [FilterItem(moddate=t) for t in adates]
rfses = [FilterItem(moddate=t) for t in rdates]
cats = ("days", "years", "months", "weeks", "hours", "recent")
rules = {c:1 for c in cats}
a, r = TimeFilter(rules, now).filter(chain(afses, rfses))
# All nowminus1* must be accepted, all nowminus2* must be rejected.
assert set(a) == set(afses)
assert set(r) == set(rfses)
def test_minimal_functionality_and_types(self):
# Create filter with reftime self.reftime
f = TimeFilter(rules={"hours": 1}, reftime=self.reftime)
# Create mock that is 1.5 hours old. Must end up in accepted list,
# since it's 1 hour old and one item should be kept from the 1-hour-
# old-category
fse = FilterItem(moddate=self.reftime-timedelta(hours=1.5))
a, r = f.filter([fse])
# http://stackoverflow.com/a/1952655/145400
assert isinstance(a, collections.Iterable)
assert isinstance(r, collections.Iterable)
assert a[0] == fse
assert len(r) == 0
def test_10_days_overlap(self):
# Category 'overlap' must be possible (10 days > 1 week).
# Having 15 FSEs, 1 to 15 days in age, the first 10 of them must be
# accepted according to the 10-day-rule. The last 5 must be rejected.
now = time.time()
nowminusXdays = (now-(60*60*24*i+1) for i in range(1, 16))
fses = [FilterItem(modtime=t) for t in nowminusXdays]
rules = {"days": 10}
a, r = TimeFilter(rules, now).filter(fses)
r = list(r)
assert len(a) == 10
assert len(r) == 5
for fse in fses[:10]:
assert fse in a
for fse in fses[10:]:
assert fse in r
def test_minimal_functionality_and_types(self):
# Create filter with reftime NOW (if not specified otherwise)
# and simple rules.
f = TimeFilter(rules={"hours": 1})
# Create mock that is 1.5 hours old. Must end up in accepted list,
# since it's 1 hour old and one item should be kept from the 1-hour-
# old-category
fse = FilterItem(modtime=time.time()-60*60*1.5)
a, r = f.filter(objs=[fse])
# http://stackoverflow.com/a/1952655/145400
assert isinstance(a, collections.Iterable)
assert isinstance(r, collections.Iterable)
assert a[0] == fse
# Rejected list `r` is expected to be an interator, so convert to
# list before evaluating length.
assert len(list(r)) == 0
def test_periodic_creation_and_filtering_propagates_items(self):
rules = {
"recent": 100, # ensure nothing is deleted before entering the
# 1-hour bucket
"hours": 2
}
current = datetime(2016, 1, 1, 0, 0, 0)
interval = timedelta(minutes=5)
items = []
for _ in range(50): # run for 4+ hrs
items.append(FilterItem(moddate=current))
items = self.filter_and_delete(items, rules, current)
current = current + interval
items = sorted(items, key=lambda f: f.moddate)
# the oldest item should be at least 2 hours old
assert timediff.hours(items[0].moddate, current) >= 2
def fsegen(ref, N_per_cat, max_timecount):
N = N_per_cat
c = max_timecount
nowminusXyears = (ref - 60 * 60 * 24 * 365 * i for i in nrndint(N, 1, c))
nowminusXmonths = (ref - 60 * 60 * 24 * 30 * i for i in nrndint(N, 1, c))
nowminusXweeks = (ref - 60 * 60 * 24 * 7 * i for i in nrndint(N, 1, c))
nowminusXdays = (ref - 60 * 60 * 24 * i for i in nrndint(N, 1, c))
nowminusXhours = (ref - 60 * 60 * i for i in nrndint(N, 1, c))
nowminusXseconds = (ref - 1 * i for i in nrndint(N, 1, c))
times = chain(
nowminusXyears,
nowminusXmonths,
nowminusXweeks,
nowminusXdays,
nowminusXhours,
nowminusXseconds,
)
return (FilterItem(modtime=t) for t in times)
def test_overlapping_rules_dont_accept_additional_items(self):
# check first rule: 24 hours, overlapping one day
rules = { "hours": 24 }
ref_time = datetime(2016, 1, 1)
moddates = (ref_time - timedelta(hours=i)
for i in range(1, 29))
items = [FilterItem(moddate=d) for d in moddates]
a, _ = TimeFilter(rules, ref_time).filter(items)
# expect the first 24 items to be accepted
assert len(a) == 24
assert set(a) == set(items[:24])
# combine with an overlapping "days1" rule
rules = { "hours": 24, "days": 1 }
a, _ = TimeFilter(rules, ref_time).filter(items)
# the result shouldn't change: the most recent 1-day old item is
# the same as the most recent 24-hour old item
assert len(a) == 24
assert set(a) == set(items[:24])
class TestTimeFilterBasic(object):
"""Test TimeFilter logic and arithmetics with small, well-defined mock
object lists.
"""
reftime = datetime(2016, 12, 31, 12, 35)
# lists of mock items, per category, partly overlapping (e.g. days/weeks,
# weeks/months)
fses10 = {
"recent": [FilterItem(moddate=reftime - timedelta(minutes=n))
for n in range(1, 11)],
"hours": make_fses(2016, 12, 31, range(2, 12), 0),
"days": make_fses(2016, 12, range(21, 31)),
"weeks": make_fses(2016,
[(12, 24), (12, 17), (12, 10), (12, 3), (11, 26),
(11, 19), (11, 12), (11, 5), (10, 29), (10, 22)]),
"months": make_fses(2016, range(2, 12), 28),
"years": make_fses(range(2006, 2016), 12, 31)
}
# lists of mock items per category, not overlapping. Useful for testing
# that items don't "spill" into other categories (e.g. a "days" rule
# shouldn't find any items from "hours" or "weeks")
fses4 = {
"recent": [FilterItem(moddate=reftime - timedelta(minutes=n))
for n in range(1, 5)],
"hours": make_fses(2016, 12, 31, range(8, 12), 0),
"days": make_fses(2016, 12, range(27, 31)),
"weeks": make_fses(2016, [(12, 24), (12, 17), (12, 10), (12, 3)]),
"months": make_fses(2016, range(8, 12), 20),
"years": make_fses(range(2012, 2016), 12, 31)
}
yearsago = make_fses(range(2016, 2005, -1), 12, 31)
def setup(self):
pass
def teardown(self):
pass
def test_minimal_functionality_and_types(self):
# Create filter with reftime self.reftime
f = TimeFilter(rules={"hours": 1}, reftime=self.reftime)
# Create mock that is 1.5 hours old. Must end up in accepted list,
# since it's 1 hour old and one item should be kept from the 1-hour-
# old-category
fse = FilterItem(moddate=self.reftime-timedelta(hours=1.5))
a, r = f.filter([fse])
# http://stackoverflow.com/a/1952655/145400
assert isinstance(a, collections.Iterable)
assert isinstance(r, collections.Iterable)
assert a[0] == fse
assert len(r) == 0
def test_requesting_one_retrieves_most_recent(self):
for category, fses in self.fses10.iteritems():
f = TimeFilter({category: 1}, self.reftime)
a, r = f.filter(fses)
assert len(a) == 1
def moddates(fses): return map(lambda fse: fse.moddate, fses)
assert a[0].moddate == max(moddates(fses))
assert len(r) == 9
assert a[0] not in r
def test_requesting_less_than_available_retrieves_most_recent(self):
for category, fses in self.fses10.iteritems():
f = TimeFilter({category: 5}, self.reftime)
a, r = f.filter(fses)
assert len(a) == 5
assert len(r) == 5
def moddates(fses): return map(lambda fse: fse.moddate, fses)
assert min(moddates(a)) > max(moddates(r))
def test_requesting_all_available_retrieves_all(self):
for category, fses in self.fses10.iteritems():
f = TimeFilter({category: 10}, self.reftime)
a, r = f.filter(fses)
assert set(a) == set(fses)
assert len(r) == 0
def test_requesting_more_than_available_retrieves_all(self):
for category, fses in self.fses10.iteritems():
f = TimeFilter({category: 15}, self.reftime)
a, r = f.filter(fses)
assert set(a) == set(fses)
assert len(r) == 0
def test_requesting_newer_than_available_retrieves_none(self):
# excluding the "recent" category which will always accept the newest N
# items.
categories = ("hours", "days", "weeks", "months", "years")
# generate items 6-10 per category, in reverse order to increase
# the chance of discovering order dependencies in the filter.
fses10to6 = {cat : sorted(self.fses10[cat],
key=lambda x: x.moddate,
reverse=True)[5:]
for cat in categories}
# now ask for the first 5 items of each category.
for category, fses in fses10to6.iteritems():
f = TimeFilter({category: 5}, self.reftime)
a, r = f.filter(fses)
assert len(a) == 0
assert set(r) == set(fses)
def test_request_less_than_available_distant(self):
# only distant items are present (at the beginning of the rule period)
fses = [self.yearsago[10], self.yearsago[9]]
rules = {"years": 10}
a, r = TimeFilter(rules, self.reftime).filter(fses)
assert set(a) == set(fses)
assert len(r) == 0
def test_request_less_than_available_close(self):
# only close items are present (at the end of the rule period)
fses = [self.yearsago[1], self.yearsago[2]]
rules = {"years": 10}
a, r = TimeFilter(rules, self.reftime).filter(fses)
assert set(a) == set(fses)
assert len(r) == 0
def test_all_categories_1acc_1rej(self):
# test multiple categories -- for simplicity make sure that periods
# don't overlap
now = datetime(2015, 12, 31, 12, 30, 45)
nowminus1year = datetime(2014, 12, 31)
nowminus2year = datetime(2013, 12, 31)
nowminus1month = datetime(2015, 11, 30)
nowminus2month = datetime(2015, 10, 31)
nowminus1week = datetime(2015, 12, 24)
nowminus2week = datetime(2015, 12, 17)
nowminus1day = datetime(2015, 12, 30)
nowminus2day = datetime(2015, 12, 29)
nowminus1hour = datetime(2015, 12, 31, 11, 30)
nowminus2hour = datetime(2015, 12, 31, 10, 30)
nowminus1second = datetime(2015, 12, 31, 12, 30, 44)
nowminus2second = datetime(2015, 12, 31, 12, 30, 43)
adates = (
nowminus1year,
nowminus1month,
nowminus1week,
nowminus1day,
nowminus1hour,
nowminus1second,
)
rdates = (
nowminus2year,
nowminus2month,
nowminus2week,
nowminus2day,
nowminus2hour,
nowminus2second,
)
afses = [FilterItem(moddate=t) for t in adates]
rfses = [FilterItem(moddate=t) for t in rdates]
cats = ("days", "years", "months", "weeks", "hours", "recent")
rules = {c:1 for c in cats}
a, r = TimeFilter(rules, now).filter(chain(afses, rfses))
# All nowminus1* must be accepted, all nowminus2* must be rejected.
assert set(a) == set(afses)
assert set(r) == set(rfses)
def test_requesting_older_categories_than_available_retrieves_none(self):
categories = ("recent", "hours", "days", "weeks", "months", "years")
fses = [self.fses4[cat] for cat in categories]
for n in range(1, len(categories)):
newer_fses = list(chain.from_iterable(islice(fses, n)))
rules = {categories[n]: 4}
a, r = TimeFilter(rules, self.reftime).filter(newer_fses)
assert len(a) == 0
assert set(r) == set(newer_fses)
def test_requesting_newer_categories_than_available_retrieves_none(self):
categories = ("years", "months", "weeks", "days", "hours", "recent")
fses = [self.fses4[cat] for cat in categories]
for n in range(1, len(categories)):
older_fses = list(chain.from_iterable(islice(fses, n)))
rules = {categories[n]: 4}
a, r = TimeFilter(rules, self.reftime).filter(older_fses)
assert len(a) == 0
assert set(r) == set(older_fses)
def make_fses(*args):
return [FilterItem(moddate=date) for date in make_moddates(*args)]
def add_fses(fses, ref, count, delta):
for _ in range(count):
ref = ref - delta;
fses.append(FilterItem(moddate=ref))
return ref