def test_search_with_before_and_filter(self):
event_dicts, events = self._add_events()
num = 1
for d in event_dicts:
d['title'] = 'changed to %d' % (num)
num += 1
events = [Event.from_dict(d) for d in event_dicts]
store.update_events(events)
to_filter = ['testfeed2']
unfiltered = [f for f in self._feeds if f not in to_filter]
num_filtered = len(
[e for e in events if e.feed['short_name'] in to_filter]
)
self.assertTrue(num_filtered > 0)
es = store.get_events_by_search("changed", to_filter=to_filter)
self.assertEqual(es.count, num_filtered)
self.assertEqual(len(es), num_filtered)
before = es.latest + datetime.timedelta(microseconds=1)
# add new event
new_time = datetime.datetime(2012, 4, 24, 0, 0, 0, 0)
event_dict = events_create_single(self._feeds[2], new_time)
event_dict['title'] = 'changed to %d' % (num)
e = Event.from_dict(event_dict)
store.add_events([e])
# the same search would now also return the new entry
es = store.get_events_by_search("changed", to_filter=to_filter)
self.assertEqual(es.count, num_filtered + 1)
# with time limit it shouldn't
es = store.get_events_by_search(
"changed",
before=before,
to_filter=to_filter
)
self.assertEqual(es.count, num_filtered)
self.assertEqual(len(es), num_filtered)
for _ in range(es.num_pages):
for e in es.page():
self.assertTrue(e.occurred != new_time)
self.assertTrue(e.occurred < before)
self.assertNotIn(e.feed['short_name'], unfiltered)
def test_search_with_after(self):
event_dicts, events = self._add_events()
num = 1
for d in event_dicts:
d['title'] = 'changed to %d' % (num)
num += 1
events = [Event.from_dict(d) for d in event_dicts]
store.update_events(events)
es = store.get_events_by_search("changed")
initial_count = len(events)
self.assertEqual(es.count, initial_count)
self.assertEqual(len(es), initial_count)
after = datetime.datetime(2012, 2, 1, 0, 0, 0, 0)
expected_num = 0
for e in events:
if e.occurred > after:
expected_num += 1
self.assertTrue(expected_num > 0)
es = store.get_events_by_search("changed", after=after)
# start using results to trigger reading of metadata, first page
p = es.page()
# add new event
new_time = datetime.datetime(2012, 4, 24, 0, 0, 0, 0)
event_dict = events_create_single(self._feeds[0], new_time)
event_dict['title'] = 'changed to %d' % (num)
e = Event.from_dict(event_dict)
store.add_events([e])
# original search should not contain the new event
self.assertEqual(len(es), expected_num)
for _ in range(es.num_pages):
for e in es.page():
self.assertTrue(e.occurred > after)
# new search should include new event
self.assertEqual(
len(store.get_events_by_search("changed", after=after)),
expected_num + 1
)
def to_event(self, raw):
e = Event()
e.feed = self.dict()
e.link = urllib.parse.urljoin('http://www.reddit.com',
raw['data']['permalink'])
e.title = raw['data']['title']
e.occurred = datetime.datetime.utcnow()
e.thumbnail_url = raw['data']['url']
e.archive_url = raw['data']['url']
e.raw = raw
return e
def test_search_with_mask(self):
event_dicts, events = self._add_events()
num = 1
for d in event_dicts:
d['title'] = 'changed to %d' % (num)
num += 1
events = [Event.from_dict(d) for d in event_dicts]
store.update_events(events)
to_mask = ['testfeed3', 'testfeed5']
num_masked = len(
[e for e in events if e.feed['short_name'] in to_mask]
)
num_un_masked = len(events) - num_masked
self.assertTrue(num_masked > 0)
self.assertTrue(num_un_masked > 0)
es = store.get_events_by_search("changed", to_mask=to_mask)
self.assertEqual(es.count, num_un_masked)
for _ in range(es.num_pages):
for e in es.page():
self.assertNotIn(e.feed['short_name'], to_mask)
def test_get_events_by_date_local(self):
d = datetime.datetime(2012, 11, 24, 0, 0, 0, 0)
tz = 'America/Toronto'
es = store.get_events_by_date(d, timezone=tz)
self.assertTrue(es.count > 0)
from_store = [e.dict() for e in es]
expected = []
for e in self._events:
new_time = utc_datetime_to_local(e.occurred, pytz.timezone(tz))
if new_time.date() == d.date():
new_event_dict = e.dict()
to_pg_datetime_str(new_event_dict, 'occurred')
if new_event_dict['related'] is not None:
for r in new_event_dict['related']:
to_pg_datetime_str(r, 'occurred')
new_event = Event.from_dict(new_event_dict) # need a copy
new_event.localize(tz)
expected.append(new_event)
expected.sort(key=lambda x: x.occurred)
expected.reverse()
expected = [e.dict() for e in expected]
for i in range(len(expected)):
self.assertDictEqual(from_store[i], expected[i])
def page(self, cursor=None):
# move cursor if provided
if cursor is not None:
self._cursor = cursor
# set query cursor
self._eventquery.set_cursor(self._cursor)
# set query limit
self._eventquery.set_limit(self.pagesize)
# perform query
with self._pool.connect() as cur:
cur.execute(self._eventquery.query, self._eventquery.params)
events = [Event.from_dict(r[0]) for r in cur]
# if there were at least pagesize events, set up next page
if len(events) == self.pagesize:
self._cursor = ByTimeRangeCursor(events[-1].occurred,
events[-1].id)
# otherwise, indicate there is no next page, and reset internal cursor
else:
self._cursor = None
return Page(events, self._cursor, timezone=self.timezone)
def _search_page(self):
if self._cursor is None:
# set initial cursor
self._cursor = BySearchCursor(1)
with self._index.searcher() as searcher:
# search!
hits = searcher.search_page(self._parsed_query,
self._cursor.page,
filter=self._filter_terms,
mask=self._mask_terms,
pagelen=self.pagesize)
if not hits and self._cursor.page == 1:
return []
elif not hits:
raise InvalidPage
event_ids = tuple([hit["id"] for hit in hits])
events = {}
# get events from db
with self._pool.connect() as cur:
cur.execute(self._eventquery.query, (event_ids, ))
# maintain ordering by score
for r in cur:
e = Event.from_dict(r[0])
events[e.id] = e
return [events[i] for i in event_ids]
def test_search_with_filter(self):
event_dicts, events = self._add_events()
num = 1
for d in event_dicts:
d['title'] = 'changed to %d' % (num)
num += 1
events = [Event.from_dict(d) for d in event_dicts]
store.update_events(events)
to_filter = ['testfeed2']
unfiltered = [f for f in self._feeds if f not in to_filter]
num_filtered = len(
[e for e in events if e.feed['short_name'] in to_filter]
)
self.assertTrue(num_filtered > 0)
es = store.get_events_by_search("changed", to_filter=to_filter)
self.assertEqual(es.count, num_filtered)
self.assertEqual(len(es), num_filtered)
for _ in range(es.num_pages):
for e in es.page():
self.assertNotIn(e.feed['short_name'], unfiltered)
def test_search_with_before(self):
event_dicts, events = self._add_events()
num = 1
for d in event_dicts:
d['title'] = 'changed to %d' % (num)
num += 1
events = [Event.from_dict(d) for d in event_dicts]
store.update_events(events)
es = store.get_events_by_search("changed")
initial_count = len(events)
self.assertEqual(es.count, initial_count)
self.assertEqual(len(es), initial_count)
before = es.latest + datetime.timedelta(microseconds=1)
# add new event
new_time = datetime.datetime(2012, 4, 24, 0, 0, 0, 0)
event_dict = events_create_single(self._feeds[0], new_time)
event_dict['title'] = 'changed to %d' % (num)
e = Event.from_dict(event_dict)
store.add_events([e])
# the same search would now also return the new entry
es = store.get_events_by_search("changed")
self.assertEqual(es.count, initial_count + 1)
# with time limit it shouldn't
es = store.get_events_by_search("changed", before=before)
self.assertEqual(es.count, initial_count)
self.assertEqual(len(es), initial_count)
for _ in range(es.num_pages):
for e in es.page():
self.assertTrue(e.occurred != new_time)
self.assertTrue(e.occurred < before)
def to_event(self, raw):
e = Event()
e.feed = self.dict()
e.title = raw['d']
e.link = raw['u']
e.occurred = datetime.datetime.strptime(
raw['dt'], '%Y-%m-%dT%H:%M:%SZ'
)
e.thumbnail_url = raw['u']
e.archive_url = raw['u']
e.raw = raw
return e
def to_event(self, raw):
e = Event()
e.feed = self.dict()
e.title = raw['shot']['title']
e.link = raw['shot']['html_url']
e.occurred = datetime.datetime.strptime(raw['created_at'],
'%Y-%m-%dT%H:%M:%SZ')
e.thumbnail_url = raw['shot']['html_url']
e.original_url = raw['shot']['images'].get('hidpi')
# if no hidpi image, use normal
if e.original_url is None:
e.original_url = raw['shot']['images'].get('normal')
e.raw = raw
return e
def to_event(self, raw):
e = Event()
e.feed = self.dict()
e.title = "[" + raw["name"] + "]"
e.text = "Obtained"
e.link = "http://www.wowhead.com/item=" + str(raw['id'])
e.occurred = WoW.parse_date(raw['timestamp'])
e.raw = raw
return e
def to_event(self, raw):
e = Event()
e.feed = self.dict()
e.text = raw['artist']['#text'] + " - " + raw['name']
e.link = raw.get('url')
e.occurred = datetime.datetime.utcfromtimestamp(
float(raw["date"]["uts"]))
e.thumbnail_url = Lastfm.get_image_url(raw)
e.raw = raw
return e
def to_event(self, raw):
e = Event()
e.feed = self.dict()
if raw['caption']:
e.title = raw['caption']['text']
e.link = raw["images"]["standard_resolution"]["url"]
e.occurred = datetime.datetime.utcfromtimestamp(
float(raw['created_time']))
e.thumbnail_url = e.link
e.original_url = e.link
e.raw = raw
return e
def test_delete_non_existent_events(self):
e = Event.from_dict(
events_create_single(
self._feeds[0],
datetime.datetime(2012, 1, 12, 0, 0, 0, 0)
)
)
# should be a no-op
store.remove_events(events=[e])
def to_event(self, raw):
e = Event()
e.feed = self.dict()
e.title = Foursquare.get_title(raw)
e.link = Foursquare.get_link(raw)
e.occurred = datetime.datetime.utcfromtimestamp(
float(raw['createdAt'])
)
e.thumbnail_url = Foursquare.get_image_url(raw)
e.raw = raw
return e
def test_add_event_with_existing_related_event(self):
related = Event.from_dict(
events_create_single(
self._feeds[0],
datetime.datetime(2012, 1, 12, 0, 0, 0, 0)
)
)
store.add_events([related])
# remove feed data on related event to match what is retrieved from the
# store
related.feed = None
e = Event.from_dict(
events_create_single(
self._feeds[0],
datetime.datetime(2012, 1, 11, 0, 0, 0, 0)
)
)
e.add_related(related)
store.add_events([e])
es = store.get_events_by_ids([e.id])
self.assertEqual(es.count, 1)
from_store = es.page().events[0]
self.assertEqual(len(from_store.related), 1)
related_dict = related.dict()
related_dict['feed'] = None
self.assertEqual(from_store.related[0].dict(), related_dict)
es = store.get_events_by_ids([related.id])
related_from_store = es.page().events[0]
self.assertIsNotNone(related_from_store.feed)
def to_event(self, raw):
e = Event()
e.feed = self.dict()
e.text = Runkeeper.get_text(raw)
e.link = raw['activity']
e.occurred = Runkeeper.get_date(raw, self.timezone)
e.raw = raw
return e
def to_event(self, raw):
e = Event()
e.feed = self.dict()
e.title = raw['t']
e.link = raw['l']
e.occurred = datetime.datetime.utcfromtimestamp(float(raw['time']))
e.raw = raw
return e
def test_update_events_with_nonexistent(self):
e = Event.from_dict(
events_create_single(
self._feeds[0],
datetime.datetime(2012, 1, 12, 0, 0, 0, 0)
)
)
self.assertRaises(
MissingEventIDException,
store.update_events,
[e]
)
def test_search_with_timezone(self):
event_dicts, events = self._add_events()
num = 1
for d in event_dicts:
d['title'] = 'changed to %d' % (num)
num += 1
events = [Event.from_dict(d) for d in event_dicts]
store.update_events(events)
tz = 'America/Toronto'
es = store.get_events_by_search("changed", timezone=tz)
from_store = list(es)
from_store.sort(key=lambda x: x.occurred)
from_store = [e.dict() for e in from_store]
expected = []
for e in events:
new_event_dict = e.dict()
to_pg_datetime_str(new_event_dict, 'occurred')
new_event_dict['related'] = None
new_event = Event.from_dict(new_event_dict) # need a copy
new_event.localize(tz)
expected.append(new_event)
expected.sort(key=lambda x: x.occurred)
expected = [e.dict() for e in expected]
self.assertEqual(len(expected), len(from_store))
for i in range(len(expected)):
self.assertDictEqual(from_store[i], expected[i])
def to_event(self, raw):
e = Event()
e.feed = self.dict()
e.title = raw['title']
e.link = "http://www.flickr.com/photos/%s/%s" % (raw['owner'],
raw['id'])
e.occurred = datetime.datetime.utcfromtimestamp(
float(raw['dateupload']))
e.thumbnail_url = e.link
e.original_url = raw['url_o']
e.raw = raw
# crude check for placeholder image
if (raw['height_o'], raw['width_o']) == (PLACEHOLDER_HEIGHT,
PLACEHOLDER_WIDTH):
raise FlickrImageNotReady(
'Upstream full size image is not ready for processing.')
return e
def test_add_event_bad_id_value(self):
e = Event.from_dict(
events_create_single(
self._feeds[0],
datetime.datetime(2012, 1, 12, 0, 0, 0, 0)
)
)
e.id = 7
self.assertRaises(
psycopg2.Error,
store.add_events,
[e]
)
def to_event(self, raw):
e = Event()
e.feed = self.dict()
e.text = raw['text']
e.occurred = datetime.datetime.strptime(raw['created_at'],
'%a %b %d %H:%M:%S +0000 %Y')
e.raw = raw
return e
def test_delete_event_with_bad_id(self):
e = Event.from_dict(
events_create_single(
self._feeds[0],
datetime.datetime(2012, 1, 12, 0, 0, 0, 0)
)
)
e.id = 7
self.assertRaises(
Exception,
store.remove_events,
[e]
)
def _add_events(self, dry=False):
distribution = [(json.dumps(feed), 3) for feed in self._feeds]
event_dicts = events_create_fake(
distribution,
datetime.datetime(2012, 1, 12, 0, 0, 0, 0),
datetime.datetime(2012, 3, 24, 0, 0, 0, 0)
)
event_dicts.reverse()
events = [Event.from_dict(d) for d in event_dicts]
store.add_events(events, dry=dry)
return event_dicts, events
def to_event(self, raw):
e = Event()
e.feed = self.dict()
e.title = raw['project']['name']
e.link = raw['project']['url']
e.occurred = datetime.datetime.utcfromtimestamp(raw['timestamp'])
# get largest cover
sizes = sorted(
[
int(size) for size in raw['project']['covers']
if size != 'original'
]
)
largest = sizes[-1]
e.thumbnail_url = raw['project']['covers'][str(largest)]
e.original_url = raw['project']['covers'][str(largest)]
e.raw = raw
return e
def to_event(self, raw):
e = Event()
e.feed = self.dict()
e.text = Fitbit.get_text(raw)
e.occurred = datetime.datetime.strptime(
raw['datetime'], '%a %b %d %H:%M:%S +0000 %Y'
)
e.raw = raw
return e
def __iter__(self):
# reset query limit
self._eventquery.set_limit(None)
with self._pool.connect() as cur:
# executing as iterable, get all
cur.execute(self._eventquery.query, self._eventquery.params)
for r in cur:
e = Event.from_dict(r[0])
if self.timezone is not None:
e.localize(self.timezone)
yield e
def test_get_all_with_timezone(self):
event_dicts, events = self._add_events()
tz = 'America/Toronto'
es = store.get_events_by_timerange(timezone=tz)
from_store = []
# use pages to iterate
for i in range(1, es.num_pages + 1):
p = es.page()
data = list(p)
if i < es.num_pages:
self.assertEqual(data[-1].occurred, p.next.occurred)
from_store += [e.dict() for e in data]
expected = []
for e in events:
new_event_dict = e.dict()
to_pg_datetime_str(new_event_dict, 'occurred')
# need to convert related as well
if new_event_dict['related'] is not None:
for r in new_event_dict['related']:
r['feed'] = None
to_pg_datetime_str(r, 'occurred')
new_event = Event.from_dict(new_event_dict) # need a copy
new_event.localize(tz)
expected.append(new_event)
expected = [e.dict() for e in expected]
self.assertEqual(len(expected), len(from_store))
for i in range(len(expected)):
self.assertDictEqual(from_store[i], expected[i])
