def test_post_sort_order(self): """ Test the regression where user profile posts were not shown in most recent first order. Also make sure the ordering is preserved across multiple conversations. """ past_created = now() - timedelta(minutes=7*24*60) for idx in xrange(3): created = past_created + timedelta(minutes=idx) self._create_db_post(_created=created, content='I need a laptop bag' + str(idx), user_profile=self.up) conv = Conversation.objects()[:][0] conv.is_closed = True conv.save(is_safe=True) for idx in xrange(3): created = past_created + timedelta(minutes=3+idx) self._create_db_post(_created=created, content='I need a laptop bag' + str(idx), user_profile=self.up) data = dict(channel_id=str(self.inbound.id), user_name='joe') resp = self.client.get('/user_profile/json?%s' % urlencode(data)) self.assertEqual(resp.status_code, 200) u_p_data = json.loads(resp.data)['list'] last_post_date = datetime_to_timestamp_ms(now()) for conv_data in u_p_data: for post in conv_data: self.assertTrue(post['created_at'] <= last_post_date) last_post_date = post['created_at']
def main(manager=manager): """ Supposed to be invoked by cron periodically """ # 1. check sleeping jobs _now = now() for job_status in JobStatus.objects.find(status=JobStatus.SLEEPING): if utc(job_status.awake_at) > _now: continue job = manager.registry.get(job_status.name) manager.producer.send_message(job.topic, job_status) LOGGER.info('Job: %s awakened and sent to execution.', job_status.id) # 2. check timed out jobs for job_status in JobStatus.objects.find(status=JobStatus.RUNNING): job = manager.registry.get(job_status.name) last_activity = job_status.last_activity or job_status.started_date if _now - utc(last_activity) < timedelta(seconds=job.timeout): continue job_status.update(completion_date=now(), status=JobStatus.TERMINATED) LOGGER.info('Job: %s terminated. No activity last %s seconds.', job_status.id, job.timeout) if job.terminate_handler: try: job.terminate_handler(*job_status.args, **job_status.kwargs) LOGGER.info('terminate_handler complete for Job: %s.', job_status.id) except Exception as ex: LOGGER.error('Error executing terminate_handler: %s', ex, exc_info=True)
def test_channel_subscriptions(self): sc = TwitterServiceChannel.objects.create_by_user(self.user, title='TSC') token = self.get_token() data = { 'force': True, 'token': token, 'channel': str(sc.id), 'from_date': format_date(now() - timedelta(days=2)), #'07/03/2014 12:00:00', 'to_date': format_date(now()) # '07/13/2014 12:00:00' } self.do_post('historics', **data) resp = self.do_post('historics', **data) self.assertTrue(resp['subscription']['is_stoppable']) resp = self.do_put('historics/%(id)s' % resp['subscription'], action='stop') self.assertEqual(resp['message'], 'Subscription has been stopped') resp = self.do_get('historics', **data) self.assertTrue(resp['has_active']) for sub in resp['items']: self.do_put('historics/%(id)s' % sub, action='stop') resp = self.do_get('historics', **data) self.assertFalse(resp['has_active']) self.assertEqual(len(resp['items']), 2)
def purge_days(channel): ''' From now, purge days that we want to maintain in our history, that have not been purged yet. ''' # for all the days in the intersection between [last_purged, today], [3 days ago, today] if channel.last_purged: range_start = utc(channel.last_purged) else: range_start = now() - relativedelta(days=14) days_to_purge = list(gen_timeslots(range_start, now(), level='day')) trend_stats = [0, 0, 0] topic_stats = [0, 0, 0] for day in days_to_purge: topic_res = mark_and_sweep_topics(channel, day) topic_stats = [x + y for x, y in zip(topic_stats, topic_res)] #LOGGER.debug("TOPIC STATS: %s", topic_res) trend_res = purge_corresponding_trends(channel=channel, timeslot=day) trend_stats = [x + y for x, y in zip(trend_stats, trend_res)] return days_to_purge, topic_stats, trend_stats
def purge_months(channel): ''' From now, purge months that we want to maintain in our history, that have not been purged yet. ''' if channel.last_purged: range_start = utc(channel.last_purged) else: range_start = now() - relativedelta(months=2) mday = localtime().tm_mday if mday > 7: range_end = now() else: range_end = now() - relativedelta(months=1) months_to_purge = [] trend_stats = [0, 0, 0] topic_stats = [0, 0, 0] if range_start <= range_end: months_to_purge = list( gen_timeslots(range_start, range_end, level='month')) for month in months_to_purge: topic_res = mark_and_sweep_topics(channel, month) topic_stats = [x + y for x, y in zip(topic_stats, topic_res)] #LOGGER.debug("TOPIC STATS: %s", topic_res) trend_res = purge_corresponding_trends(channel=channel, timeslot=month) trend_stats = [x + y for x, y in zip(trend_stats, trend_res)] return months_to_purge, topic_stats, trend_stats
def test_range_query(self): start_dt = now() self._create_event_sequence(length=2, brand_involved=True, customer=self.customer2) self._create_event_sequence(length=3, brand_involved=True, customer=self.customer) self._create_event_sequence(length=2, brand_involved=True, customer=self.customer2) self._create_event_sequence(length=7, brand_involved=True, customer=self.customer) self._create_event_sequence(length=2, brand_involved=True, customer=self.customer2) end_dt = now() count = Event.objects.range_query_count(start=start_dt, end=end_dt, customer=self.customer) self.assertEqual(count, 10) count = Event.objects.range_query_count(start=start_dt, end=end_dt, customer=self.customer2) self.assertEqual(count, 6)
def _search(self, user, *args, **kwargs): error_desc = "Requests should contain a channel id and some query text: {'channel': <>, 'query': <>}" required_params = ['channel', 'query'] LOGGER.debug('Searching for FAQs with params: %s' % kwargs) for entry in required_params: if entry not in kwargs: error_msg = "Missing required parameter '%s'" % entry raise exc.InvalidParameterConfiguration(error_msg, description=error_desc) try: channel = Channel.objects.get(kwargs['channel']) except Channel.DoesNotExist: raise exc.ResourceDoesNotExist( "No channel with id=%s found in the system." % kwargs['channel']) result = FAQ.objects.text_search(channel, kwargs['query'], limit=100) LOGGER.debug('Search results for FAQs: %s' % result) from solariat.utils.timeslot import parse_datetime, now _created = parse_datetime(kwargs.get('_created', now()), default=now()) faq_event = FAQQueryEvent.objects.create_by_user(user, query=kwargs['query'], channels=[channel.id], _created=_created) if 'customer_id' in kwargs: CustomerProfile = user.account.get_customer_profile_class() customer = CustomerProfile.objects.get(kwargs['customer_id']) # import ipdb # ipdb.set_trace() # actions = NextBestActionEngine.upsert(account_id=customer.account_id).search(faq_event, customer) # TODO: THink how we can implement this properly with new advisors API return dict(list=result, event=faq_event.to_dict()) #, actions=actions) else: return dict(list=result, event=faq_event.to_dict())
def setUp(self): super(Purging, self).setUp() # Start Fresh ChannelHotTopics.objects.coll.remove() self.assertEqual(ChannelHotTopics.objects.count(), 0) self.this_month = datetime_to_timeslot(now(), 'month') self.this_day = datetime_to_timeslot(now(), 'day')
def __init__(self, *args, **kw): if '_created' not in kw: kw['_created'] = now() if '_native_id' not in kw: kw['_native_id'] = str(now()) super(Event, self).__init__(*args, **kw)
def _make_posts_and_stats(self): date_now = now() date_old = now() - timedelta(days=get_var('CHANNEL_STATS_KEEP_DAYS') + 1) content_list = [(date_old, "old post")] self._create_posts(content_list) content_list = [(date_now, "new post")] self._create_posts(content_list)
def test_policy_final_state(self): """ Test final policy state of a post """ policy_vec = csm.SimplePolicy() current_state = csm.StateVector(status=csm.ENGAGED, time_stamp=timeslot.now()) final_state = policy_vec.get_final_state(current_state, timeslot.now()) self.assertEqual(current_state, final_state)
def test_several_tags_posts(self): ''' Check `posts/json` endpoint when several tags are requested by `channel_id`. ''' customer1 = UserProfile.objects.upsert('Twitter', dict(screen_name='@customer1')) post1 = self._create_tweet('@test I need a laptop 1', channel=self.i, user_profile=customer1) customer2 = UserProfile.objects.upsert('Twitter', dict(screen_name='@customer2')) post2 = self._create_tweet('@test I need a laptop 2', channel=self.i, user_profile=customer2) customer3 = UserProfile.objects.upsert('Twitter', dict(screen_name='@customer3')) post3 = self._create_tweet('@test I need a laptop 3', channel=self.i, user_profile=customer3) foo_tag = self._create_smart_tag(self.i, 'Foo', status='Active') bar_tag = self._create_smart_tag(self.i, 'Bar', status='Active') post1.handle_add_tag(self.user, [foo_tag]) # remove tag from other two posts since they were highlighted post2.handle_remove_tag(self.user, [foo_tag]) post3.handle_remove_tag(self.user, [foo_tag]) post2.handle_add_tag(self.user, [bar_tag]) # remove tag from other two posts since they were highlighted post1.handle_remove_tag(self.user, [bar_tag]) post3.handle_remove_tag(self.user, [bar_tag]) # one post with foo_tag posts = self._get_posts(**{ 'channel_id': str(foo_tag.id), 'from': now(), 'to': now() + timedelta(minutes=1), 'level': 'hour', 'topics': [], 'intentions': ['needs'], }) self.assertEqual(len(posts), 1) # two posts with foo_tag and bar_tag posts = self._get_posts(**{ 'channel_id': [str(foo_tag.id), str(bar_tag.id)], 'from': now(), 'to': now() + timedelta(minutes=1), 'level': 'hour', 'topics': [], 'intentions': ['needs'], }) self.assertEqual(len(posts), 2)
def aggregate_state(self, worker, wstate): LOGGER.debug('[aggregate state invoked] worker: %s', worker) state = self.state if worker in state: name, _, _ = state[worker] state[worker] = [name, wstate, now()] else: idx = len([1 for w in state if type(w) == type(worker)]) name = '%s_%s' % (type(worker).__name__, idx) state[worker] = [name, wstate, now()] # res = {name: [wstate, dt] for name, wstate, dt in state.values()} return {name: wstate}
def setUp(self): UICase.setUp(self) self.login() self.created_at = now() self.created_at_str = self.created_at.strftime('%Y-%m-%d %H:%M:%S') for content in ('I need a bike. I like Honda.', 'Can somebody recommend a sturdy laptop?', 'I need an affordabl laptop. And a laptop bag', 'Whatever you buy, let it be an Apple laptop', 'I would like to have a thin and lightweight laptop.'): self._create_db_post(content=content, _created=now())
def query_responses(post_ids, sort_by, user): t_start = now() response_ids = [ "%s:r" % post_id for post_id in post_ids ] response_filter = {"id__in": response_ids} if response_type: queries = [] if 'review' in response_type: response_type.remove('review') response_type.extend(REVIEW_STATUSES) posted_matchable_filters = set(POSTED_MATCHABLE).intersection(set(response_type)) response_status_filters = set(RESPONSE_STATUSES).intersection(set(response_type)) if posted_matchable_filters: queries.append({ #"status": "posted", "posted_matchable__in": list(posted_matchable_filters)}) if 'clicks' in response_type: queries.append({"clicked_count__gt": 0}) if response_status_filters: queries.append({"status__in": response_type}) if len(queries) > 1: response_filter.update({"$or": queries}) elif queries: response_filter.update(queries[0]) responses = list(Response.objects.find(**response_filter).limit(paging_params['limit'])) total_count = len(responses) if sort_by == 'intention': responses.sort(key=lambda p: -p.intention_confidence) else: responses.sort(key=lambda p: p.created, reverse=True) app.logger.debug("Fetched %d responses in %s seconds" % ( total_count, now() - t_start)) res = [ {'match' : matchable_to_dict(resp.matchable, resp.relevance), 'post' : post_to_dict(resp.post, user), 'response_type' : resp.status, 'clicks' : resp.clicked_count, 'acted_date' : datetime_to_timestamp_ms(resp.created)} for resp in responses ] return res
def request( self, path, args=None, post_args=None, files=None, method=None): start = now() self.check_rate_limits(path) try: response = super(GraphAPI, self).request(path, args, post_args, files, method) except facebook.GraphAPIError as error: log_item = self.log_request(path, args, post_args, method, start, now(), error) self.handle_rate_limit_error(error, path, failed_request_time=start, log_item=log_item.id) raise error self.log_request(path, args, post_args, method, start, now(), error=None) return response
def _create_posts(self): past_created = now() - timedelta(minutes=7 * 24 * 60) post1 = self._create_db_post(_created=past_created, content='i need some carrot') past_created = now() - timedelta(minutes=7 * 24 * 60 + 10) post2 = self._create_db_post(_created=past_created, content='Where I can buy a carrot?') self._create_db_post(content='i need some carrot') self._create_db_post(content='Where I can buy a carrot?') self.assertEqual( Post.objects(channels__in=[self.channel.id]).count(), 4)
def create_by_user(self, user, **kw): # assert kw.get('case_number'), kw if not kw.get('_created'): kw['_created'] = now() if not kw.get('actor_id'): kw['actor_id'] = kw['case_number'] try: nps_profile = NPSProfile.objects.get(kw['actor_id']) except: nps_profile = NPSProfile.objects.create(id=kw['actor_id']) _id = NPSPost.gen_id(actor_id=nps_profile.id, _created=kw['_created'], in_reply_to_native_id=None) # from solariat_bottle.db.channel.voc import * # chan = VOCChannel.objects.get(kw['channels'][0]) # if isinstance(chan, VOCServiceChannel): # kw['channels'] = str(chan.inbound_channel.id) normalize_post_params(user, kw) post = super(NPSPostManager, self).create(_id=_id, force_create=True, **kw) for sc in post.service_channels: sc.post_received(post) _set_channel_and_tag_assignments(post) return post
def test_facebook_event(self): ts = str(now()) content = 'tweet content' status_id = "12345678901_123456789101" from solariat_bottle.db.channel.facebook import FacebookServiceChannel data = { 'content': content, 'channels': [ FacebookServiceChannel.objects.create_by_user(self.user, title='Fb') ], 'user_profile': self.get_user_profile('Facebook'), 'facebook': { 'facebook_post_id': status_id, 'created_at': ts, 'text': content, '_wrapped_data': { 'type': 'status', 'source_id': 'fake', 'source_type': 'event' } } } # posts with the same id and created date should not duplicate post1 = self._create_db_post(**data) post2 = self._create_db_post(**data) self.assertEqual(post1.id, post2.id) # posts with the same created date and different id should be separated. This is a test of the use # of fine-grain resolution data['facebook']['facebook_post_id'] = fake_status_id() post3 = self._create_db_post(**data) self.assertNotEqual(post2.id, post3.id)
def _create_tweet(self, content, channel=None, channels=None, demand_matchables=False, in_reply_to=None, **kw): from solariat.utils import timeslot status_id = fake_status_id() post_data = { 'twitter': { 'id': status_id, 'created_at': kw.get('_created', None) or timeslot.now(), 'text': content } } post_data.update(kw) if in_reply_to and hasattr(in_reply_to, 'native_id'): post_data['twitter']['in_reply_to_status_id'] = in_reply_to.native_id return self._create_db_post(content, channel=channel, channels=channels, demand_matchables=demand_matchables, **post_data)
def setUp(self): UICase.setUp(self) self.login() self.created_at = now() self.created_on_str = format_date(self.created_at) self.one_day_after_str = format_date(self.created_at + timedelta(days=1)) self.one_day_before_str = format_date(self.created_at - timedelta(days=1)) self.one_week_after_str = format_date(self.created_at + timedelta(days=7)) self.one_week_before_str = format_date(self.created_at - timedelta(days=7)) self.contents = ( 'I need a bike. I like Honda.', # needs, likes 'Can somebody recommend a sturdy laptop?', # consideration 'I need an affordabl laptop. And a laptop bag', # needs 'Whatever you buy, let it be an Apple laptop', # recommendation 'I would like to have a thin and lightweight laptop.', # needs 'Thank you very much!', # gratitude 'You\'re gonna end up with a broken laptop' # problem ) self.setup_posts()
def signal(self, sgn): self.logger.info("Received signal %s (%s)" % (sgn, trap_signals.get(sgn))) if sgn == signal.SIGHUP: self.server.sync_streams() elif sgn in {signal.SIGUSR1, signal.SIGUSR2}: try: status = self.get_status() print('---') print(json.dumps(status, indent=4)) print('---') sys.stdout.flush() if sgn == signal.SIGUSR2: status.update({"timestamp": now()}) self.db_events.add_bot_status(status) except: self.logger.exception("dm_bot.signal") elif sgn in {signal.SIGINT, signal.SIGQUIT, signal.SIGTERM}: self.stop() # gevent.get_hub().destroy() timeout = self.heartbeat for idx in xrange(timeout): sleep(1) if not self.isAlive(): break else: print("Bot never stopped after waiting %s seconds." % (timeout, )) sys.exit(2) sys.exit(0)
def _prepare_data(self, n_jobs=JobsNum(10, 5, 5), from_date=None, to_date=None): name = 'job_name' account = self.account self.jobs = [] if to_date is None: to_date = now() if from_date is None: from_date = to_date - datetime.timedelta(days=30) def gen_jobs_data(): for status in [JobModel.PENDING] * n_jobs.pending + [ JobModel.RUNNING ] * n_jobs.running + [JobModel.SUCCESSFUL] * n_jobs.completed: created_at, started_date, completion_date = random_status_dates( from_date, to_date) data = { 'created_at': created_at, 'status': status, 'account': account.id, } if status == JobModel.SUCCESSFUL: data['started_date'], data[ 'completion_date'] = started_date, completion_date elif status == JobModel.RUNNING: data['started_date'] = started_date yield data for job_data in gen_jobs_data(): job = self.create_job(name) job.update(**job_data) self.jobs.append(job)
def test_duplicate_handle_diff_channels(self): channel3 = TwitterChannel.objects.create_by_user( self.user, title='TestChannel3', type='twitter', intention_types=SA_TYPES) duplicate_post = self._create_db_post( channels=[self.channel2, channel3], content=self.duplicate_content, url=self.url, twitter={ "created_at": "Wed, 06 Aug 2014 18:38:47 +0000", "id": "497089420017676290" }) self.assertEqual(len(duplicate_post.channels), 3) time_slot = datetime_to_timeslot(now(), 'day') ht_stat = ChannelHotTopics.objects.by_time_span( channel=self.channel2, from_ts=datetime_to_timeslot(None, 'day'), ) tt_stat = ChannelTopicTrends(channel=self.channel2, time_slot=time_slot, topic=self.topic, status=0) self.assertEqual(ht_stat, self.hot_topic_stat) self.assertEqual(tt_stat, self.topic_trends_stat)
def test_select_by_time_span_3(self): past_dt = now() - relativedelta(months=1) # big enough for all levels post1 = self._create_db_post(_created=past_dt, content='i need some carrot') post2 = self._create_db_post(content='i need some carrot') self.assertEqual( Post.objects(channels__in=[self.channel.id]).count(), 2) for level in ('hour', 'day'): result = ChannelTopicTrends.objects.by_time_span( channel=self.channel, topic_pairs=[['carrot', True]], from_ts=datetime_to_timeslot(past_dt, level), to_ts=datetime_to_timeslot(None, level)) self.assertEqual(len(result), 2) result = ChannelTopicTrends.objects.by_time_span( channel=self.channel, topic_pairs=[['carrot', True]], from_ts=datetime_to_timeslot( past_dt + relativedelta(**{level + 's': 1}), level), to_ts=datetime_to_timeslot(None, level)) self.assertEqual(len(result), 1)
def test_purge_conversations_1(self): """ Deleting outdated conversation of two posts""" self._make_setup_for_conversations() self._create_db_post(channel=self.sc.inbound, content="I need a foo.", demand_matchables=True, user_profile={'screen_name': 'customer'}) self._create_db_post(channel=self.sc.inbound, content="Does anyone have a foo?", demand_matchables=True, user_profile={'screen_name': 'customer'}) self.assertEqual(Conversation.objects.count(), 1) self.assertEqual(Post.objects.count(), 2) self.assertEqual(SpeechActMap.objects.count(), 2) inbound_channel = Channel.objects.get(id=self.sc.inbound) purge_channel_entities( inbound_channel, run_in_prod_mod=True, now_date=now() + timedelta(days=get_var("CHANNEL_ENTITIES_KEEP_DAYS") + 1)) self.assertEqual(Conversation.objects.count(), 0) self.assertEqual(SpeechActMap.objects.count(), 0) self.assertEqual(Post.objects.count(), 0)
def test_facebook_post(self): updated_at = now().replace(microsecond=0) created_at = TIMESLOT_EPOCH - timedelta(days=300) content = 'post image caption' status_id = "12345678901_123456789101" from solariat_bottle.db.channel.facebook import FacebookServiceChannel data = { 'content': content, 'channels': [ FacebookServiceChannel.objects.create_by_user(self.user, title='Fb') ], 'user_profile': { 'user_name': 'fb_test_user' }, 'facebook': { 'facebook_post_id': status_id, 'text': content, 'created_at': str(created_at), '_wrapped_data': { 'type': 'status', 'source_id': 'fake', 'source_type': 'event', 'created_at': str(created_at), 'updated_time': str(updated_at) } } } post1 = self._create_db_post(**data) actor_id, creation_time = unpack_event_id(post1.id) self.assertEqual(post1.created_at.replace(microsecond=0), updated_at) self.assertEqual(creation_time.replace(microsecond=0), updated_at)
def on_data(self, raw_data): """Called when raw data is received from connection. Override this method if you wish to manually handle the stream data. Return False to stop stream and close connection. """ try: data = json.loads(raw_data) except ValueError: self.log_event("on_data() can't decode json: %s" % raw_data) return finally: self.last_keep_alive = now() if self.on_message(data): return # twitter warning events elif 'limit' in data: if self.on_limit(data['limit']) is False: return False elif 'disconnect' in data: if self.on_disconnect(data['disconnect']) is False: return False elif 'warning' in data: # warnings can be received only with stall_warnings=True if self.on_warning(data['warning']) is False: return False else: self.log_event("unknown message type: " + str(raw_data))
def get_or_create(self, track, follow, languages, account_ids, channel_ids): ref = StreamRef(track=track, follow=follow, languages=languages) is_new = False existing_ref = StreamRef.objects.find_one(id=ref.key) if existing_ref is None: is_new = True ref.status = StreamRef.QUEUED log = StreamLog(accounts=account_ids, channels=channel_ids, stream_ref_id=ref.id) log.save() ref.log = log ref.save() else: ref = existing_ref ref_log = ref.log if (not ref_log or set(ref_log.channels) != set(channel_ids) or set(ref_log.accounts) != set(account_ids) or ref.is_stopped()): # if tracked accounts or channels change # update stream log if ref_log: ref_log.update(stopped_at=now()) else: is_new = True if ref.is_stopped(): is_new = True log = StreamLog(accounts=account_ids, channels=channel_ids, stream_ref_id=existing_ref.id) log.save() ref.update(log=log, status=StreamRef.RUNNING) return ref, is_new
def test_stats_retrieving(self): time_slot = datetime_to_timeslot(now(), 'day') topics = ('laptop', 'laptop bag', 'good laptop bag', 'good laptop') for topic in topics: for term, is_leaf in gen_topic_tree(topic): ChannelHotTopics.increment(self.channel, time_slot, term, status=0, intention_id=0, is_leaf=is_leaf, lang_id=Lang.EN, agent=1) stats = ChannelHotTopics.objects.by_time_span( channel=self.channel, from_ts=datetime_to_timeslot(None, 'day'), languages=['en']) expected_result = [{ u'term_count': 2, u'topic': u'laptop', u'topic_count': 1 }, { u'term_count': 2, u'topic': u'bag', u'topic_count': 0 }] self.assertListEqual(stats, expected_result)