def test_search_with_before_and_filter(self):
event_dicts, events = self._add_events()
num = 1
for d in event_dicts:
d['title'] = 'changed to %d' % (num)
num += 1
events = [Event.from_dict(d) for d in event_dicts]
store.update_events(events)
to_filter = ['testfeed2']
unfiltered = [f for f in self._feeds if f not in to_filter]
num_filtered = len(
[e for e in events if e.feed['short_name'] in to_filter]
)
self.assertTrue(num_filtered > 0)
es = store.get_events_by_search("changed", to_filter=to_filter)
self.assertEqual(es.count, num_filtered)
self.assertEqual(len(es), num_filtered)
before = es.latest + datetime.timedelta(microseconds=1)
# add new event
new_time = datetime.datetime(2012, 4, 24, 0, 0, 0, 0)
event_dict = events_create_single(self._feeds[2], new_time)
event_dict['title'] = 'changed to %d' % (num)
e = Event.from_dict(event_dict)
store.add_events([e])
# the same search would now also return the new entry
es = store.get_events_by_search("changed", to_filter=to_filter)
self.assertEqual(es.count, num_filtered + 1)
# with time limit it shouldn't
es = store.get_events_by_search(
"changed",
before=before,
to_filter=to_filter
)
self.assertEqual(es.count, num_filtered)
self.assertEqual(len(es), num_filtered)
for _ in range(es.num_pages):
for e in es.page():
self.assertTrue(e.occurred != new_time)
self.assertTrue(e.occurred < before)
self.assertNotIn(e.feed['short_name'], unfiltered)
def test_search_with_after(self):
event_dicts, events = self._add_events()
num = 1
for d in event_dicts:
d['title'] = 'changed to %d' % (num)
num += 1
events = [Event.from_dict(d) for d in event_dicts]
store.update_events(events)
es = store.get_events_by_search("changed")
initial_count = len(events)
self.assertEqual(es.count, initial_count)
self.assertEqual(len(es), initial_count)
after = datetime.datetime(2012, 2, 1, 0, 0, 0, 0)
expected_num = 0
for e in events:
if e.occurred > after:
expected_num += 1
self.assertTrue(expected_num > 0)
es = store.get_events_by_search("changed", after=after)
# start using results to trigger reading of metadata, first page
p = es.page()
# add new event
new_time = datetime.datetime(2012, 4, 24, 0, 0, 0, 0)
event_dict = events_create_single(self._feeds[0], new_time)
event_dict['title'] = 'changed to %d' % (num)
e = Event.from_dict(event_dict)
store.add_events([e])
# original search should not contain the new event
self.assertEqual(len(es), expected_num)
for _ in range(es.num_pages):
for e in es.page():
self.assertTrue(e.occurred > after)
# new search should include new event
self.assertEqual(
len(store.get_events_by_search("changed", after=after)),
expected_num + 1
)
def test_search_with_mask(self):
event_dicts, events = self._add_events()
num = 1
for d in event_dicts:
d['title'] = 'changed to %d' % (num)
num += 1
events = [Event.from_dict(d) for d in event_dicts]
store.update_events(events)
to_mask = ['testfeed3', 'testfeed5']
num_masked = len(
[e for e in events if e.feed['short_name'] in to_mask]
)
num_un_masked = len(events) - num_masked
self.assertTrue(num_masked > 0)
self.assertTrue(num_un_masked > 0)
es = store.get_events_by_search("changed", to_mask=to_mask)
self.assertEqual(es.count, num_un_masked)
for _ in range(es.num_pages):
for e in es.page():
self.assertNotIn(e.feed['short_name'], to_mask)
def test_get_events_by_date_local(self):
d = datetime.datetime(2012, 11, 24, 0, 0, 0, 0)
tz = 'America/Toronto'
es = store.get_events_by_date(d, timezone=tz)
self.assertTrue(es.count > 0)
from_store = [e.dict() for e in es]
expected = []
for e in self._events:
new_time = utc_datetime_to_local(e.occurred, pytz.timezone(tz))
if new_time.date() == d.date():
new_event_dict = e.dict()
to_pg_datetime_str(new_event_dict, 'occurred')
if new_event_dict['related'] is not None:
for r in new_event_dict['related']:
to_pg_datetime_str(r, 'occurred')
new_event = Event.from_dict(new_event_dict) # need a copy
new_event.localize(tz)
expected.append(new_event)
expected.sort(key=lambda x: x.occurred)
expected.reverse()
expected = [e.dict() for e in expected]
for i in range(len(expected)):
self.assertDictEqual(from_store[i], expected[i])
def page(self, cursor=None):
# move cursor if provided
if cursor is not None:
self._cursor = cursor
# set query cursor
self._eventquery.set_cursor(self._cursor)
# set query limit
self._eventquery.set_limit(self.pagesize)
# perform query
with self._pool.connect() as cur:
cur.execute(self._eventquery.query, self._eventquery.params)
events = [Event.from_dict(r[0]) for r in cur]
# if there were at least pagesize events, set up next page
if len(events) == self.pagesize:
self._cursor = ByTimeRangeCursor(events[-1].occurred,
events[-1].id)
# otherwise, indicate there is no next page, and reset internal cursor
else:
self._cursor = None
return Page(events, self._cursor, timezone=self.timezone)
def _search_page(self):
if self._cursor is None:
# set initial cursor
self._cursor = BySearchCursor(1)
with self._index.searcher() as searcher:
# search!
hits = searcher.search_page(self._parsed_query,
self._cursor.page,
filter=self._filter_terms,
mask=self._mask_terms,
pagelen=self.pagesize)
if not hits and self._cursor.page == 1:
return []
elif not hits:
raise InvalidPage
event_ids = tuple([hit["id"] for hit in hits])
events = {}
# get events from db
with self._pool.connect() as cur:
cur.execute(self._eventquery.query, (event_ids, ))
# maintain ordering by score
for r in cur:
e = Event.from_dict(r[0])
events[e.id] = e
return [events[i] for i in event_ids]
def test_search_with_filter(self):
event_dicts, events = self._add_events()
num = 1
for d in event_dicts:
d['title'] = 'changed to %d' % (num)
num += 1
events = [Event.from_dict(d) for d in event_dicts]
store.update_events(events)
to_filter = ['testfeed2']
unfiltered = [f for f in self._feeds if f not in to_filter]
num_filtered = len(
[e for e in events if e.feed['short_name'] in to_filter]
)
self.assertTrue(num_filtered > 0)
es = store.get_events_by_search("changed", to_filter=to_filter)
self.assertEqual(es.count, num_filtered)
self.assertEqual(len(es), num_filtered)
for _ in range(es.num_pages):
for e in es.page():
self.assertNotIn(e.feed['short_name'], unfiltered)
def test_search_with_before(self):
event_dicts, events = self._add_events()
num = 1
for d in event_dicts:
d['title'] = 'changed to %d' % (num)
num += 1
events = [Event.from_dict(d) for d in event_dicts]
store.update_events(events)
es = store.get_events_by_search("changed")
initial_count = len(events)
self.assertEqual(es.count, initial_count)
self.assertEqual(len(es), initial_count)
before = es.latest + datetime.timedelta(microseconds=1)
# add new event
new_time = datetime.datetime(2012, 4, 24, 0, 0, 0, 0)
event_dict = events_create_single(self._feeds[0], new_time)
event_dict['title'] = 'changed to %d' % (num)
e = Event.from_dict(event_dict)
store.add_events([e])
# the same search would now also return the new entry
es = store.get_events_by_search("changed")
self.assertEqual(es.count, initial_count + 1)
# with time limit it shouldn't
es = store.get_events_by_search("changed", before=before)
self.assertEqual(es.count, initial_count)
self.assertEqual(len(es), initial_count)
for _ in range(es.num_pages):
for e in es.page():
self.assertTrue(e.occurred != new_time)
self.assertTrue(e.occurred < before)
def test_delete_non_existent_events(self):
e = Event.from_dict(
events_create_single(
self._feeds[0],
datetime.datetime(2012, 1, 12, 0, 0, 0, 0)
)
)
# should be a no-op
store.remove_events(events=[e])
def test_add_event_with_existing_related_event(self):
related = Event.from_dict(
events_create_single(
self._feeds[0],
datetime.datetime(2012, 1, 12, 0, 0, 0, 0)
)
)
store.add_events([related])
# remove feed data on related event to match what is retrieved from the
# store
related.feed = None
e = Event.from_dict(
events_create_single(
self._feeds[0],
datetime.datetime(2012, 1, 11, 0, 0, 0, 0)
)
)
e.add_related(related)
store.add_events([e])
es = store.get_events_by_ids([e.id])
self.assertEqual(es.count, 1)
from_store = es.page().events[0]
self.assertEqual(len(from_store.related), 1)
related_dict = related.dict()
related_dict['feed'] = None
self.assertEqual(from_store.related[0].dict(), related_dict)
es = store.get_events_by_ids([related.id])
related_from_store = es.page().events[0]
self.assertIsNotNone(related_from_store.feed)
def test_update_events_with_nonexistent(self):
e = Event.from_dict(
events_create_single(
self._feeds[0],
datetime.datetime(2012, 1, 12, 0, 0, 0, 0)
)
)
self.assertRaises(
MissingEventIDException,
store.update_events,
[e]
)
def test_search_with_timezone(self):
event_dicts, events = self._add_events()
num = 1
for d in event_dicts:
d['title'] = 'changed to %d' % (num)
num += 1
events = [Event.from_dict(d) for d in event_dicts]
store.update_events(events)
tz = 'America/Toronto'
es = store.get_events_by_search("changed", timezone=tz)
from_store = list(es)
from_store.sort(key=lambda x: x.occurred)
from_store = [e.dict() for e in from_store]
expected = []
for e in events:
new_event_dict = e.dict()
to_pg_datetime_str(new_event_dict, 'occurred')
new_event_dict['related'] = None
new_event = Event.from_dict(new_event_dict) # need a copy
new_event.localize(tz)
expected.append(new_event)
expected.sort(key=lambda x: x.occurred)
expected = [e.dict() for e in expected]
self.assertEqual(len(expected), len(from_store))
for i in range(len(expected)):
self.assertDictEqual(from_store[i], expected[i])
def test_add_event_bad_id_value(self):
e = Event.from_dict(
events_create_single(
self._feeds[0],
datetime.datetime(2012, 1, 12, 0, 0, 0, 0)
)
)
e.id = 7
self.assertRaises(
psycopg2.Error,
store.add_events,
[e]
)
def _add_events(self, dry=False):
distribution = [(json.dumps(feed), 3) for feed in self._feeds]
event_dicts = events_create_fake(
distribution,
datetime.datetime(2012, 1, 12, 0, 0, 0, 0),
datetime.datetime(2012, 3, 24, 0, 0, 0, 0)
)
event_dicts.reverse()
events = [Event.from_dict(d) for d in event_dicts]
store.add_events(events, dry=dry)
return event_dicts, events
def test_delete_event_with_bad_id(self):
e = Event.from_dict(
events_create_single(
self._feeds[0],
datetime.datetime(2012, 1, 12, 0, 0, 0, 0)
)
)
e.id = 7
self.assertRaises(
Exception,
store.remove_events,
[e]
)
def __iter__(self):
# reset query limit
self._eventquery.set_limit(None)
with self._pool.connect() as cur:
# executing as iterable, get all
cur.execute(self._eventquery.query, self._eventquery.params)
for r in cur:
e = Event.from_dict(r[0])
if self.timezone is not None:
e.localize(self.timezone)
yield e
def test_get_all_with_timezone(self):
event_dicts, events = self._add_events()
tz = 'America/Toronto'
es = store.get_events_by_timerange(timezone=tz)
from_store = []
# use pages to iterate
for i in range(1, es.num_pages + 1):
p = es.page()
data = list(p)
if i < es.num_pages:
self.assertEqual(data[-1].occurred, p.next.occurred)
from_store += [e.dict() for e in data]
expected = []
for e in events:
new_event_dict = e.dict()
to_pg_datetime_str(new_event_dict, 'occurred')
# need to convert related as well
if new_event_dict['related'] is not None:
for r in new_event_dict['related']:
r['feed'] = None
to_pg_datetime_str(r, 'occurred')
new_event = Event.from_dict(new_event_dict) # need a copy
new_event.localize(tz)
expected.append(new_event)
expected = [e.dict() for e in expected]
self.assertEqual(len(expected), len(from_store))
for i in range(len(expected)):
self.assertDictEqual(from_store[i], expected[i])
def test_update_events_change_title(self):
event_dicts, events = self._add_events()
num = 1
for d in event_dicts:
d['title'] = 'changed to %d' % (num)
num += 1
events = [Event.from_dict(d) for d in event_dicts]
store.update_events(events)
es = store.get_events_by_timerange()
from_store = list(es)
events_compare(self, event_dicts, from_store)
es = store.get_events_by_search("changed")
self.assertEqual(es.count, len(event_dicts))
def test_update_events_change_title_dry(self):
event_dicts, events = self._add_events()
for i, d in enumerate(event_dicts):
d['title'] = 'changed to %d' % (i)
events = [Event.from_dict(d) for d in event_dicts]
store.update_events(events, dry=True)
es = store.get_events_by_timerange()
from_store = list(es)
for e in from_store:
if e.title is not None:
self.assertNotIn('changed', e.title)
es = store.get_events_by_search("changed")
self.assertEqual(es.count, 0)
def setUpClass(cls):
TestStoreWithDBBase.setUpClass()
# add our events
distribution = [(json.dumps(feed), 30) for feed in cls._feeds]
event_dicts = events_create_fake(
distribution,
datetime.datetime(2012, 1, 12, 0, 0, 0, 0),
datetime.datetime(2013, 3, 24, 0, 0, 0, 0)
)
cls._events = [Event.from_dict(d) for d in event_dicts]
cls._events.sort(key=lambda x: x.occurred)
cls._events.reverse()
store.add_events(cls._events)
# need to stub out feeds in related events
for e in cls._events:
if e.related is not None:
for r in e.related:
r.feed = None