def by_time_span(self,
user,
channel,
start_time=None,
end_time=None,
level='hour',
**extra):
"Return generator of docs for time period"
start_time = start_time or TIMESLOT_EPOCH
end_time = end_time or get_now()
from_ts = datetime_to_timeslot(start_time, level)
to_ts = datetime_to_timeslot(end_time, level)
if from_ts == to_ts:
extra['time_slot'] = from_ts
else:
extra['time_slot__gte'] = from_ts
extra['time_slot__lte'] = to_ts
if hasattr(channel, '__iter__'):
extra['channel__in'] = channel
else:
extra['channel'] = channel
# from time import time
# start_t = time()
stats = self.find_by_user(user, **extra)
#LOGGER.debug("%s.by_time_span(): querty time %.3f sec", self.__class__.__name__, time() - start_t)
return stats
def test_stats_retrieving(self):
time_slot = datetime_to_timeslot(now(), 'day')
topics = ('laptop', 'laptop bag', 'good laptop bag', 'good laptop')
for topic in topics:
for term, is_leaf in gen_topic_tree(topic):
ChannelHotTopics.increment(self.channel,
time_slot,
term,
status=0,
intention_id=0,
is_leaf=is_leaf,
lang_id=Lang.EN,
agent=1)
stats = ChannelHotTopics.objects.by_time_span(
channel=self.channel,
from_ts=datetime_to_timeslot(None, 'day'),
languages=['en'])
expected_result = [{
u'term_count': 2,
u'topic': u'laptop',
u'topic_count': 1
}, {
u'term_count': 2,
u'topic': u'bag',
u'topic_count': 0
}]
self.assertListEqual(stats, expected_result)
def test_agent_search(self):
content = 'I need a bike. I like Honda.'
post = self._create_db_post(content)
posts = fetch_posts([self.channel],
datetime_to_timeslot(post.created_at, 'hour'),
datetime_to_timeslot(post.created_at, 'hour'),
[dict(topic='honda', topic_type='leaf')],
[SpeechActMap.ACTIONABLE],
intentions=[],
agents=[10],
min_conf=0.5)
self.assertEqual(len(posts), 0)
self.assertEqual(SpeechActMap.objects.count(), 2)
# Now reset with this agent, and into an ACTUAL stats
post.channel_assignments[str(self.channel.id)] = 'actual'
SpeechActMap.reset(post, [self.channel], agent=10)
# Should be the same number of speech act map entries
self.assertEqual(SpeechActMap.objects.count(), 2)
# And try again. We should get a hit
posts = fetch_posts([self.channel],
datetime_to_timeslot(post.created_at, 'hour'),
datetime_to_timeslot(post.created_at, 'hour'),
[dict(topic='honda', topic_type='leaf')],
[SpeechActMap.ACTUAL],
intentions=[],
agents=[10],
min_conf=0.5)
self.assertEqual(len(posts), 1)
def test_duplicate_handle_diff_channels(self):
channel3 = TwitterChannel.objects.create_by_user(
self.user,
title='TestChannel3',
type='twitter',
intention_types=SA_TYPES)
duplicate_post = self._create_db_post(
channels=[self.channel2, channel3],
content=self.duplicate_content,
url=self.url,
twitter={
"created_at": "Wed, 06 Aug 2014 18:38:47 +0000",
"id": "497089420017676290"
})
self.assertEqual(len(duplicate_post.channels), 3)
time_slot = datetime_to_timeslot(now(), 'day')
ht_stat = ChannelHotTopics.objects.by_time_span(
channel=self.channel2,
from_ts=datetime_to_timeslot(None, 'day'),
)
tt_stat = ChannelTopicTrends(channel=self.channel2,
time_slot=time_slot,
topic=self.topic,
status=0)
self.assertEqual(ht_stat, self.hot_topic_stat)
self.assertEqual(tt_stat, self.topic_trends_stat)
def test_select_by_time_point_3(self):
''' Test with different post creation dates'''
DAY_20131212 = utc(datetime(day=12, month=12, year=2013))
DAY_20131202 = utc(datetime(day=2, month=12, year=2013))
DAY_20131002 = utc(datetime(day=2, month=10, year=2013))
for d in [DAY_20131212, DAY_20131202, DAY_20131002]:
self._create_db_post(_created=d,
channel=self.channel,
content='i need some carrot')
# Test for 1 single day
results = ChannelHotTopics.objects.by_time_span(
channel=self.channel,
from_ts=datetime_to_timeslot(DAY_20131212, level='day'))
self.assertEqual(results[0]['term_count'], 1)
# For a month
results = ChannelHotTopics.objects.by_time_span(
channel=self.channel,
from_ts=datetime_to_timeslot(DAY_20131212, level='month'))
self.assertEqual(results[0]['term_count'], 2)
# For a different month
results = ChannelHotTopics.objects.by_time_span(
channel=self.channel,
from_ts=datetime_to_timeslot(DAY_20131002, level='month'))
self.assertEqual(results[0]['term_count'], 1)
def test_select_by_time_point(self):
content = "I need a mac laptop"
self._create_db_post(content)
'''
terms: mac laptop, laptop
'''
results = ChannelHotTopics.objects.by_time_span(
channel=self.channel,
from_ts=datetime_to_timeslot(None, level='hour'))
self.assertEqual(len(results),
0) # we don't store HOUR hot-topics stats
# The first test will be for root level topics.
results = ChannelHotTopics.objects.by_time_span(
channel=self.channel,
from_ts=datetime_to_timeslot(None, level='day'))
expected = set(['laptop'])
self.assertEqual(
set([r['topic'] for r in results]).difference(expected), set())
# Should be the same again - despite the change in month
results = ChannelHotTopics.objects.by_time_span(
channel=self.channel,
from_ts=datetime_to_timeslot(None, level='month'))
self.assertEqual(
set([r['topic'] for r in results]).difference(expected), set())
def postprocess_params(self, params):
r = params
from_dt = parse_datetime(r['from'])
to_dt = parse_datetime(r['to'])
r['from_ts'] = datetime_to_timeslot(from_dt, 'hour')
r['to_ts'] = datetime_to_timeslot(to_dt, 'hour')
del r['from']
del r['to']
r = super(PostsView, self).postprocess_params(r)
if r['sort_by'] == 'time':
r['sort_map'] = {'_created': -1}
else:
r['sort_map'] = {'intention_confidence': -1}
r['min_conf'] = r['thresholds']['intention']
if r['last_query_time'] is not None:
r['last_query_time'] = timestamp_ms_to_datetime(
r['last_query_time'])
del r['plot_type']
return params
def test_select_by_time_point_2(self):
'''
Create multiple posts and make sure the slots for
terms get aggregated.
'''
content = "I need a mac laptop"
for i in range(10):
self._create_db_post(content)
results = ChannelHotTopics.objects.by_time_span(
channel=self.channel,
from_ts=datetime_to_timeslot(None, level='day'))
# Should just be 1 at the top
self.assertEqual(len(results), 1)
# Make sure the aggregate count is correct
self.assertEqual(results[0]['term_count'], 10)
self.assertEqual(results[0]['topic_count'], 0)
# Should be 10 below for each. Only one item but counts of 10
results = ChannelHotTopics.objects.by_time_span(
channel=self.channel,
parent_topic='laptop',
from_ts=datetime_to_timeslot(None, level='day'))
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['topic_count'], 10)
def purge_outdated_trends_stats(coll, channel, level, delta):
initial_timedelta_arg_name = {"hour": "days", "day": "months"}[level]
timedelta_arg_name = {"hour": "hours", "day": "days"}[level]
start_dt = now() - relativedelta(**{initial_timedelta_arg_name: delta})
current_dt = start_dt
time_step = relativedelta(**{timedelta_arg_name: 1})
ts = datetime_to_timeslot(current_dt, level)
zero_counts = 0
total_records_removed = 0
EMPTY_SLOTS_NUMBER = 10
while zero_counts <= EMPTY_SLOTS_NUMBER:
t0 = datetime.now()
channel_ts_val = ChannelTopicTrends.make_channel_ts(channel, ts)
res = coll.objects.coll.remove(coll.objects.get_query(time_slot=ts))
if res['n'] == 0:
zero_counts += 1
current_dt = current_dt - time_step
total_records_removed += res['n']
ts = datetime_to_timeslot(current_dt, level)
LOGGER.info(
"purging Q:: collection: %s; func: %s; timedelta: %s; date: %s; level: %s; records removed: %s",
coll.__name__,
inspect.stack()[0][3],
datetime.now() - t0, current_dt, level, res['n'])
return total_records_removed
def purge_channel_stats(channel):
days = get_var('CHANNEL_STATS_KEEP_DAYS')
start_date = datetime(year=2012, month=1, day=1)
end_date = now() - timedelta(days=days)
# end_date = datetime(year=end_date.year, month=end_date.month, day=1)
timeslots = (
(datetime_to_timeslot(start_date, level), datetime_to_timeslot(end_date, level)) \
for level in TIMESLOT_LEVEL_NAMES
)
F = ChannelStats.F
removed_count = 0
for start_ts, end_ts in timeslots:
t0 = datetime.now()
res = ChannelStats.objects.coll.remove({
F('time_slot'): {
'$lte': end_ts,
'$gt': start_ts
},
F('channel'): channel.id
})
LOGGER.info(
"purging Q:: channel: %s; collection: ChannelStats; func: %s; timedelta: %s"
% (channel.title, inspect.stack()[0][3], datetime.now() - t0))
removed_count += res['n']
return removed_count
def test_select_by_time_span_3(self):
past_dt = now() - relativedelta(months=1) # big enough for all levels
post1 = self._create_db_post(_created=past_dt,
content='i need some carrot')
post2 = self._create_db_post(content='i need some carrot')
self.assertEqual(
Post.objects(channels__in=[self.channel.id]).count(), 2)
for level in ('hour', 'day'):
result = ChannelTopicTrends.objects.by_time_span(
channel=self.channel,
topic_pairs=[['carrot', True]],
from_ts=datetime_to_timeslot(past_dt, level),
to_ts=datetime_to_timeslot(None, level))
self.assertEqual(len(result), 2)
result = ChannelTopicTrends.objects.by_time_span(
channel=self.channel,
topic_pairs=[['carrot', True]],
from_ts=datetime_to_timeslot(
past_dt + relativedelta(**{level + 's': 1}), level),
to_ts=datetime_to_timeslot(None, level))
self.assertEqual(len(result), 1)
def test_timeslot_conversion(self):
past_dt = now() - timedelta(minutes=7 * 24 * 60)
ts_before = datetime_to_timeslot(past_dt, 'hour')
post = self._create_db_post(_created=past_dt,
content='i need some carrot')
ts_after = datetime_to_timeslot(post.created, 'hour')
self.assertEqual(ts_before, ts_after)
def setUp(self):
super(Purging, self).setUp()
# Start Fresh
ChannelHotTopics.objects.coll.remove()
self.assertEqual(ChannelHotTopics.objects.count(), 0)
self.this_month = datetime_to_timeslot(now(), 'month')
self.this_day = datetime_to_timeslot(now(), 'day')
def test_new_post_search(self):
content = 'I need a bike. I like Honda.'
post = self._create_db_post(content)
posts = fetch_posts([self.channel],
datetime_to_timeslot(post.created_at, 'hour'),
datetime_to_timeslot(
post.created_at + timedelta(hours=1), 'hour'),
[dict(topic='honda', topic_type='leaf')],
[SpeechActMap.ACTIONABLE],
intentions=[],
min_conf=0.5,
agents=[])
self.assertEqual(len(posts), 1)
def test_integer_id(self):
def make_id_ts_left(time_slot, dummy):
components = (
(time_slot, 22),
(dummy, 42),
)
id_ = pack_components(*components)
return id_
def make_id_ts_right(time_slot, dummy):
components = ((dummy, 42), (time_slot, 22))
id_ = pack_components(*components)
return id_
TimeSlotIntegerId.objects.coll.remove()
from solariat.utils.timeslot import datetime_to_timeslot, parse_date_interval, timedelta, timeslot_to_datetime
start_date, end_date = parse_date_interval('02/21/2013', '05/21/2013')
step = timedelta(hours=24)
dates = []
while start_date < end_date:
dates.append(start_date)
start_date += step
assert len(dates) == 90
data = enumerate(dates[::-1], start=100)
for dummy, date in data:
dummy %= 5
time_slot = datetime_to_timeslot(date)
id_ = make_id_ts_left(time_slot, dummy)
doc = TimeSlotIntegerId(id=id_, time_slot=time_slot, dummy=dummy)
doc.save()
#print list(TimeSlotIntegerId.objects.coll.find())
#fetch interval
start_date, end_date = parse_date_interval('03/21/2013', '04/21/2013')
#start_dummy = 0
#end_dummy = (1L << 41) - 1
start_id = make_id_ts_left(datetime_to_timeslot(start_date, 'hour'), 0)
end_id = make_id_ts_left(datetime_to_timeslot(end_date, 'hour'), 0)
# print start_id.bit_length()
# print end_id.bit_length()
for doc in TimeSlotIntegerId.objects(id__gte=start_id, id__lte=end_id):
print timeslot_to_datetime(doc.time_slot)
self.assertGreaterEqual(doc.time_slot,
datetime_to_timeslot(start_date, 'hour'))
self.assertLessEqual(doc.time_slot,
datetime_to_timeslot(end_date, 'hour'))
def test_speech_acts_creation(self):
content = 'I need a bike. I like Honda.'
post = self._create_db_post(content)
self.assertEqual(post.speech_acts[0]['content'], 'I need a bike.')
self.assertEqual(post.speech_acts[1]['content'], ' I like Honda.')
# Verify the id for the SpeechActMap, and make sure we can get it.
sam_id = pack_speech_act_map_id(
self.channel,
SpeechActMap.ACTIONABLE, # status
datetime_to_timeslot(post.created), # hour-timeslot of the post
post, # post
0)
sam = SpeechActMap.objects.get(id=sam_id)
self.assertTrue(sam)
self.assertEqual(str(sam.post), str(post.id))
self.assertEqual(sam.idx, 0)
# Verify we can retrieve the speech act objects
sam_ids = [
m.id for m in make_objects(self.channel, post, post.speech_acts,
'highlighted')
]
self.assertEqual(len(sam_ids), 2)
sams = SpeechActMap.objects(id__in=sam_ids)
self.assertEqual(len(sams), len(sam_ids))
sam = [m for m in sams
if m.id != sam_id][0] # we want the other one this time
self.assertEqual(str(sam.post), str(post.id))
self.assertEqual(sam.idx, 1)
def test_topics_in_speech_acts(self):
"""
This is related to issue #1623
https://github.com/solariat/tango/issues/1623
At some point if post had "DellCares" the keyword "Dell" still will be identified in it.
"""
content = '#solariat and @some_tag, How can I contact DellCares?'
# create keyword tracking channel with keyword 'dell'
keywords = ['dell', '#solariat', '@some_tag']
channel = KTC.objects.create_by_user(self.user,
title='Inbound Channel',
keywords=keywords)
post = self._create_db_post(content, channel=channel)
sam_id = pack_speech_act_map_id(
channel,
SpeechActMap.ACTIONABLE, # status
datetime_to_timeslot(post.created), # hour-timeslot of the post
post, # post
0)
sam = SpeechActMap.objects.get(id=sam_id)
self.assertTrue('#solariat' in sam.to_dict()['topics'])
self.assertTrue("en", '@some_tag' in sam.to_dict()['topics'])
self.assertTrue('dellcares' in sam.to_dict()['topics'])
self.assertFalse('dell' in sam.to_dict()['topics'])
def make_objects(channel, post, speech_acts, status, agent=0):
"""
Returns a list of speech act map objects to save or delete from
"""
channel_num = get_channel_num(channel)
time_slot = datetime_to_timeslot(post.created_at, 'hour')
for sa_idx, sa in enumerate(speech_acts):
topic_tuples = get_topic_tuples(sa, post, channel=channel)
sam_id = pack_speech_act_map_id(channel_num, status, time_slot, post,
sa_idx)
sam = SpeechActMap(id=sam_id,
channel=get_channel_id(channel),
post=post.id,
agent=agent,
language=get_lang_id(post.language),
idx=sa_idx,
intention_type_id=int(sa['intention_type_id']),
intention_type_conf=sa['intention_type_conf'],
time_slot=time_slot,
created_at=post.created_at,
topic_tuples=topic_tuples,
message_type=post._message_type)
yield sam
def _update_stats(response, channel_id, n=1):
#TODO: FIX
return
post = response.post
timeslots = dict((level, datetime_to_timeslot(post.created, level))
for level in ('day', 'month'))
def increment(topic, intention_id, is_leaf=True):
""" Recursively increments stats for topic and all its parents
"""
for level, timeslot in timeslots.items():
ResponseTermStats.increment(channel_id=channel_id,
topic=topic,
intention_id=intention_id,
timeslot=timeslot,
is_leaf=is_leaf,
response=response,
n=n)
# recursive calls to also update parent stats
for term in get_largest_subtopics(topic):
assert term, 'topic=%r, term=%r' % (topic, term)
increment(term, intention_id, is_leaf=False)
for sa in response.post.speech_acts:
if sa['intention_type'] == 'DISCARDED':
continue
intention_id = SATYPE_TITLE_TO_ID_MAP[sa['intention_type']]
for topic in sa['intention_topics']:
assert topic
increment(topic, intention_id, is_leaf=True)
def setUp(self):
super(TopicCountPlotDataCase, self).setUp()
self.login()
self.start_date = timeslot.parse_date('04/24/2013')
self.end_date = timeslot.parse_date('04/25/2013')
self.level = 'hour'
timeline = []
start_date = timeslot.parse_date('04/24/2013')
while start_date < self.end_date:
timeline.append(start_date)
start_date += timeslot.timedelta(seconds=60 * 60 *
2) #every 2 hours
self.time_slots = map(lambda d: datetime_to_timeslot(d, 'hour'),
timeline)
contents = cycle([
'I need a laptop', #laptop, need (intention id=2)
'My laptop is not working out for me:(', #laptop, problem (intention id=3)
'I need a display', #display, need
'My display is not working out for me:(', #display, problem
])
posts = []
for _created in timeline:
post = self._create_db_post(contents.next(), _created=_created)
#print post.speech_acts
posts.append(post)
def test_impressions(self):
"Test impressions stats"
pl1 = self._create_db_matchable('foo')
pl2 = self._create_db_matchable('bar')
pl3 = self._create_db_matchable('baz')
response = self.do_post('posts',
version='v1.2',
channel=str(self.channel.id),
content='i need a foo for bar but not baz')
post_dict = response['item']
#matchables = post_dict['matchables']
response = self.do_post('postmatches',
version='v1.2',
post=post_dict['id'],
impressions=[str(pl1.id),
str(pl2.id)],
rejects=[str(pl3.id)])
self.assertEqual(response['item']['rejects'][0], str(pl3.id))
time_slot = timeslot.datetime_to_timeslot(Post.objects()[0].created)
response = self.do_get('channelstats',
version='v1.2',
channel=str(self.channel.id),
time_slot=time_slot) # month stats object
stats = response['list'][0]
self.assertEqual(stats['number_of_impressions'], 2)
def test_post_creation(self):
content = 'I need a bike. I like Honda.'
post = self._create_db_post(content)
self.assertEqual(post.speech_acts[0]['content'], 'I need a bike.')
self.assertEqual(post.speech_acts[1]['content'], ' I like Honda.')
time_slot = datetime_to_timeslot(post.created, 'hour')
# single filters
posts, are_more_posts_available = Post.objects.by_time_point(
self.channel,
'bike',
time_slot,
status=SpeechActMap.POTENTIAL,
intention=sa_labels.NEEDS.oid,
)
# verify the result has the right content
self.assertEqual(len(posts), 1)
self.assertEqual(posts[0].content, post.content)
# multiple filters
posts, are_more_posts_available = Post.objects.by_time_point(
self.channel,
['bike', 'honda'],
time_slot,
status=SpeechActMap.POTENTIAL,
# no explicit intention id means ALL INTENTIONS
)
self.assertEqual(len(posts), 1)
def test_stat_update(self):
Leaf = Topic = True
Node = Term = False
HELP = 10
JUNK = 12
EN = Lang.EN
LALL = Lang.ALL
time_slot = datetime_to_timeslot(now(), 'day')
topic = 'laptop'
agent_id = 12345
hashed_parents = map(mhash, get_largest_subtopics(topic))
stat = ChannelHotTopics(channel_num=self.channel.counter,
time_slot=time_slot,
topic=topic,
status=0,
hashed_parents=hashed_parents)
stat.compute_increments(is_leaf=True,
intention_id=JUNK,
agent=None,
lang_id=Lang.EN,
n=1)
stat.compute_increments(is_leaf=False,
intention_id=HELP,
agent=None,
lang_id=Lang.EN,
n=1)
stat.upsert()
stat = ChannelHotTopics.objects.get(id=stat.id)
stat.compute_increments(is_leaf=True,
intention_id=JUNK,
agent=agent_id,
n=2)
stat.upsert()
stat.reload()
expected_stats = [
# agent | is_leaf | intent | language | topic_count
(ALL_AGENTS, Term, ALL_INTENTIONS_INT, LALL, 1),
(ALL_AGENTS, Term, ALL_INTENTIONS_INT, EN, 1),
(ALL_AGENTS, Term, HELP, LALL, 1),
(ALL_AGENTS, Term, HELP, EN, 1),
(ALL_AGENTS, Topic, ALL_INTENTIONS_INT, LALL,
1 + 2), # +2 from specific agent
(ALL_AGENTS, Topic, JUNK, LALL, 1 + 2),
(ALL_AGENTS, Topic, JUNK, EN, 1),
(ALL_AGENTS, Topic, ALL_INTENTIONS_INT, EN, 1),
(agent_id, Topic, ALL_INTENTIONS_INT, LALL, 2),
(agent_id, Topic, JUNK, LALL, 2)
]
self.assert_stats(stat, expected_stats)
self.assertFalse(stat.filter(agent=0, is_leaf=True,
intention=10)) #no such combination
def yield_channel_stats(channel, agent, date):
if date is None:
date = now()
for level in TIMESLOT_LEVEL_NAMES:
time_slot = datetime_to_timeslot(date, level)
yield ServiceChannelStats(channel=str(channel.id),
time_slot=time_slot,
agent=agent)
def _assert_topic_extraction(self, expect_topics):
past_dt = now() - relativedelta(months=1)
posts = [
"I need a riding lawnmower", "I need a lawnmower",
"I need a side ride push lawnmower"
]
for content in posts:
self._create_db_post(content)
for (topic, is_leaf, cnt) in expect_topics:
for level in ('hour', 'day'):
result = ChannelTopicTrends.objects.by_time_span(
channel=self.channel,
topic_pairs=[(topic, is_leaf)],
from_ts=datetime_to_timeslot(past_dt, level),
to_ts=datetime_to_timeslot(None, level))
self.assertEqual(len(result), cnt)
def test_stat_update(self):
time_slot = datetime_to_timeslot(now(), 'hour')
topic = 'laptop'
agent_id = 12345
stat = ChannelTopicTrends(channel=self.channel,
time_slot=time_slot,
topic=topic,
status=0)
stat.compute_increments(is_leaf=True,
intention_ids=JUNK,
agent=None,
inc_dict={'topic_count': 1},
n=1)
stat.compute_increments(is_leaf=False,
intention_ids=HELP,
agent=None,
inc_dict={'topic_count': 1},
n=1)
stat.upsert()
stat = ChannelTopicTrends.objects.get(id=stat.id)
stat.compute_increments(is_leaf=True,
intention_ids=JUNK,
agent=agent_id,
inc_dict={'topic_count': 2},
n=1)
stat.compute_increments(is_leaf=False,
intention_ids=HELP,
agent=None,
lang_id=EN,
inc_dict={'topic_count': 2},
n=1)
stat.upsert()
stat.reload()
expected_stats = [
(ALL_AGENTS, Term, ALL_INTENTIONS_INT, LALL, 1 + 2), # +2 for EN
(ALL_AGENTS, Term, HELP, LALL, 1 + 2),
(ALL_AGENTS, Term, ALL_INTENTIONS_INT, EN, 2),
(ALL_AGENTS, Term, HELP, EN, 2),
(ALL_AGENTS, Topic, ALL_INTENTIONS_INT, LALL,
1 + 2), # +2 from specific agent
(ALL_AGENTS, Topic, JUNK, LALL, 1 + 2),
(agent_id, Topic, ALL_INTENTIONS_INT, LALL, 2),
(agent_id, Topic, JUNK, LALL, 2)
]
self.assert_stats(stat, expected_stats)
self.assertFalse(stat.filter(agent=0, is_leaf=True,
intention=10)) # no such combination
def test_purge_none(self):
TWO_DAYS_AGO = now() - timedelta(days=2)
self._make_laptops_and_icecream(TWO_DAYS_AGO)
stats = purge_stats(self.channel)
last_purged = stats["last_purged"]
days = stats["purge_days"]
months = stats["purge_months"]
self.channel.reload()
self.assertEqual(
datetime_to_timeslot(self.channel.last_purged, 'hour'),
datetime_to_timeslot(last_purged, 'hour'))
# Should have purged over 15 days for time slots since we never urged before
self.assertEqual(len(days), 15)
# Months purged depends on how far in we are to the month when we run the test
self.assertTrue(len(months) in [2, 3])
import solariat_bottle.utils.purging
class MockLocaltime(object):
tm_mday = 6
solariat_bottle.utils.purging.localtime = MockLocaltime
stats = purge_stats(self.channel)
last_purged = stats["last_purged"]
days = stats["purge_days"]
months = stats["purge_months"]
self.assertEqual(len(days), 1)
self.assertEqual(days[0],
decode_timeslot(Timeslot(level='day').timeslot))
self.assertEqual(len(months), 0)
class MockLocaltime(object):
tm_mday = 8
solariat_bottle.utils.purging.localtime = MockLocaltime
stats = purge_stats(self.channel)
last_purged = stats["last_purged"]
days = stats["purge_days"]
months = stats["purge_months"]
self.assertEqual(len(days), 1)
self.assertEqual(len(months), 1)
self.assertEqual(months[0],
decode_timeslot(Timeslot(level='month').timeslot))
def discard_outdated_topics_for_day_level(channel):
# Remove all days 2 weeks old or more
channel_num = get_channel_num(channel)
until_day = datetime_to_timeslot(
now() - timedelta(days=get_var('HOT_TOPICS_DAY_STATS_KEEP_DAYS')),
'day')
from_day = datetime_to_timeslot(TIMESLOT_EPOCH, 'day')
t0 = datetime.now()
res = ChannelHotTopics.objects.coll.remove({
F('channel_num'): channel_num,
F('time_slot'): {
'$lt': until_day,
'$gte': from_day
}
})
LOGGER.info(
"purging Q:: channel: %s; collection: ChannelHotTopics; func: %s; timedelta: %s"
% (channel.title, inspect.stack()[0][3], datetime.now() - t0))
return res["n"]
def get_inbound_post(self):
statuses = [
SpeechActMap.POTENTIAL, SpeechActMap.ACTIONABLE,
SpeechActMap.ACTUAL, SpeechActMap.REJECTED
]
time_slot = datetime_to_timeslot(self.inbound_post_created_at, 'hour')
posts, are_more_posts_available = Post.objects.by_time_point(
self.inbound, ALL_TOPICS, time_slot, status=statuses)
self.assertEqual(len(posts), 1)
return posts[0]
def test_post_search(self):
content = 'I need a bike. I like Honda.'
post = self._create_db_post(content)
self.assertEqual(post.addressee, None)
posts, are_more_posts_available = Post.objects.by_time_point(
self.channel, 'honda',
datetime_to_timeslot(post.created_at, 'hour'))[:]
self.assertEqual(posts[0].id, post.id)
