Example #1
0
    def test_post_sort_order(self):
        """ Test the regression where user profile posts were not shown
        in most recent first order. Also make sure the ordering is preserved
        across multiple conversations. """
        past_created = now() - timedelta(minutes=7*24*60)

        for idx in xrange(3):
            created = past_created + timedelta(minutes=idx)
            self._create_db_post(_created=created,
                                 content='I need a laptop bag' + str(idx),
                                 user_profile=self.up)

        conv = Conversation.objects()[:][0]
        conv.is_closed = True
        conv.save(is_safe=True)

        for idx in xrange(3):
            created = past_created + timedelta(minutes=3+idx)
            self._create_db_post(_created=created,
                                 content='I need a laptop bag' + str(idx),
                                 user_profile=self.up)

        data = dict(channel_id=str(self.inbound.id), user_name='joe')
        resp = self.client.get('/user_profile/json?%s' % urlencode(data))
        self.assertEqual(resp.status_code, 200)
        u_p_data = json.loads(resp.data)['list']
        last_post_date = datetime_to_timestamp_ms(now())
        for conv_data in u_p_data:
            for post in conv_data:
                self.assertTrue(post['created_at'] <= last_post_date)
                last_post_date = post['created_at']
Example #2
0
def main(manager=manager):
    """ Supposed to be invoked by cron periodically
    """

    # 1. check sleeping jobs
    _now = now()
    for job_status in JobStatus.objects.find(status=JobStatus.SLEEPING):
        if utc(job_status.awake_at) > _now:
            continue

        job = manager.registry.get(job_status.name)
        manager.producer.send_message(job.topic, job_status)
        LOGGER.info('Job: %s awakened and sent to execution.', job_status.id)

    # 2. check timed out jobs
    for job_status in JobStatus.objects.find(status=JobStatus.RUNNING):
        job = manager.registry.get(job_status.name)

        last_activity = job_status.last_activity or job_status.started_date
        if _now - utc(last_activity) < timedelta(seconds=job.timeout):
            continue

        job_status.update(completion_date=now(), status=JobStatus.TERMINATED)
        LOGGER.info('Job: %s terminated. No activity last %s seconds.',
                    job_status.id, job.timeout)
        if job.terminate_handler:
            try:
                job.terminate_handler(*job_status.args, **job_status.kwargs)
                LOGGER.info('terminate_handler complete for Job: %s.',
                            job_status.id)
            except Exception as ex:
                LOGGER.error('Error executing terminate_handler: %s',
                             ex,
                             exc_info=True)
Example #3
0
    def test_channel_subscriptions(self):
        sc = TwitterServiceChannel.objects.create_by_user(self.user,
                                                          title='TSC')
        token = self.get_token()
        data = {
            'force': True,
            'token': token,
            'channel': str(sc.id),
            'from_date':
            format_date(now() - timedelta(days=2)),  #'07/03/2014 12:00:00',
            'to_date': format_date(now())  # '07/13/2014 12:00:00'
        }
        self.do_post('historics', **data)
        resp = self.do_post('historics', **data)
        self.assertTrue(resp['subscription']['is_stoppable'])

        resp = self.do_put('historics/%(id)s' % resp['subscription'],
                           action='stop')
        self.assertEqual(resp['message'], 'Subscription has been stopped')

        resp = self.do_get('historics', **data)
        self.assertTrue(resp['has_active'])

        for sub in resp['items']:
            self.do_put('historics/%(id)s' % sub, action='stop')

        resp = self.do_get('historics', **data)
        self.assertFalse(resp['has_active'])
        self.assertEqual(len(resp['items']), 2)
Example #4
0
def purge_days(channel):
    '''
    From now, purge days that we want to maintain in our history, that have not been
    purged yet.
    '''

    # for all the days in the intersection between [last_purged, today], [3 days ago, today]
    if channel.last_purged:
        range_start = utc(channel.last_purged)
    else:
        range_start = now() - relativedelta(days=14)

    days_to_purge = list(gen_timeslots(range_start, now(), level='day'))

    trend_stats = [0, 0, 0]
    topic_stats = [0, 0, 0]

    for day in days_to_purge:
        topic_res = mark_and_sweep_topics(channel, day)
        topic_stats = [x + y for x, y in zip(topic_stats, topic_res)]
        #LOGGER.debug("TOPIC STATS: %s", topic_res)
        trend_res = purge_corresponding_trends(channel=channel, timeslot=day)
        trend_stats = [x + y for x, y in zip(trend_stats, trend_res)]

    return days_to_purge, topic_stats, trend_stats
Example #5
0
def purge_months(channel):
    '''
    From now, purge months that we want to maintain in our history, that have not been
    purged yet.
    '''
    if channel.last_purged:
        range_start = utc(channel.last_purged)
    else:
        range_start = now() - relativedelta(months=2)

    mday = localtime().tm_mday

    if mday > 7:
        range_end = now()
    else:
        range_end = now() - relativedelta(months=1)

    months_to_purge = []

    trend_stats = [0, 0, 0]
    topic_stats = [0, 0, 0]
    if range_start <= range_end:
        months_to_purge = list(
            gen_timeslots(range_start, range_end, level='month'))

        for month in months_to_purge:
            topic_res = mark_and_sweep_topics(channel, month)
            topic_stats = [x + y for x, y in zip(topic_stats, topic_res)]
            #LOGGER.debug("TOPIC STATS: %s", topic_res)
            trend_res = purge_corresponding_trends(channel=channel,
                                                   timeslot=month)
            trend_stats = [x + y for x, y in zip(trend_stats, trend_res)]

    return months_to_purge, topic_stats, trend_stats
Example #6
0
    def test_range_query(self):
        start_dt = now()
        self._create_event_sequence(length=2,
                                    brand_involved=True,
                                    customer=self.customer2)
        self._create_event_sequence(length=3,
                                    brand_involved=True,
                                    customer=self.customer)
        self._create_event_sequence(length=2,
                                    brand_involved=True,
                                    customer=self.customer2)
        self._create_event_sequence(length=7,
                                    brand_involved=True,
                                    customer=self.customer)
        self._create_event_sequence(length=2,
                                    brand_involved=True,
                                    customer=self.customer2)
        end_dt = now()

        count = Event.objects.range_query_count(start=start_dt,
                                                end=end_dt,
                                                customer=self.customer)
        self.assertEqual(count, 10)

        count = Event.objects.range_query_count(start=start_dt,
                                                end=end_dt,
                                                customer=self.customer2)
        self.assertEqual(count, 6)
Example #7
0
 def _search(self, user, *args, **kwargs):
     error_desc = "Requests should contain a channel id and some query text: {'channel': <>, 'query': <>}"
     required_params = ['channel', 'query']
     LOGGER.debug('Searching for FAQs with params: %s' % kwargs)
     for entry in required_params:
         if entry not in kwargs:
             error_msg = "Missing required parameter '%s'" % entry
             raise exc.InvalidParameterConfiguration(error_msg,
                                                     description=error_desc)
     try:
         channel = Channel.objects.get(kwargs['channel'])
     except Channel.DoesNotExist:
         raise exc.ResourceDoesNotExist(
             "No channel with id=%s found in the system." %
             kwargs['channel'])
     result = FAQ.objects.text_search(channel, kwargs['query'], limit=100)
     LOGGER.debug('Search results for FAQs: %s' % result)
     from solariat.utils.timeslot import parse_datetime, now
     _created = parse_datetime(kwargs.get('_created', now()), default=now())
     faq_event = FAQQueryEvent.objects.create_by_user(user,
                                                      query=kwargs['query'],
                                                      channels=[channel.id],
                                                      _created=_created)
     if 'customer_id' in kwargs:
         CustomerProfile = user.account.get_customer_profile_class()
         customer = CustomerProfile.objects.get(kwargs['customer_id'])
         # import ipdb
         # ipdb.set_trace()
         # actions = NextBestActionEngine.upsert(account_id=customer.account_id).search(faq_event, customer)
         # TODO: THink how we can implement this properly with new advisors API
         return dict(list=result,
                     event=faq_event.to_dict())  #, actions=actions)
     else:
         return dict(list=result, event=faq_event.to_dict())
Example #8
0
    def setUp(self):
        super(Purging, self).setUp()
        # Start Fresh
        ChannelHotTopics.objects.coll.remove()
        self.assertEqual(ChannelHotTopics.objects.count(), 0)

        self.this_month = datetime_to_timeslot(now(), 'month')
        self.this_day = datetime_to_timeslot(now(), 'day')
Example #9
0
    def __init__(self, *args, **kw):
        if '_created' not in kw:
            kw['_created'] = now()

        if '_native_id' not in kw:
            kw['_native_id'] = str(now())

        super(Event, self).__init__(*args, **kw)
Example #10
0
    def _make_posts_and_stats(self):
        date_now = now()
        date_old = now() - timedelta(days=get_var('CHANNEL_STATS_KEEP_DAYS') +
                                     1)

        content_list = [(date_old, "old post")]
        self._create_posts(content_list)
        content_list = [(date_now, "new post")]
        self._create_posts(content_list)
Example #11
0
    def test_policy_final_state(self):
        """
        Test final policy state of a post
        """
        policy_vec = csm.SimplePolicy()
        current_state = csm.StateVector(status=csm.ENGAGED,
                                        time_stamp=timeslot.now())

        final_state = policy_vec.get_final_state(current_state, timeslot.now())
        self.assertEqual(current_state, final_state)
Example #12
0
    def test_several_tags_posts(self):
        '''
        Check `posts/json` endpoint when several tags are requested by `channel_id`.
        '''

        customer1 = UserProfile.objects.upsert('Twitter', dict(screen_name='@customer1'))
        post1 = self._create_tweet('@test I need a laptop 1',
                                   channel=self.i,
                                   user_profile=customer1)

        customer2 = UserProfile.objects.upsert('Twitter', dict(screen_name='@customer2'))
        post2 = self._create_tweet('@test I need a laptop 2',
                                   channel=self.i,
                                   user_profile=customer2)

        customer3 = UserProfile.objects.upsert('Twitter', dict(screen_name='@customer3'))
        post3 = self._create_tweet('@test I need a laptop 3',
                                   channel=self.i,
                                   user_profile=customer3)

        foo_tag = self._create_smart_tag(self.i, 'Foo', status='Active')
        bar_tag = self._create_smart_tag(self.i, 'Bar', status='Active')

        post1.handle_add_tag(self.user, [foo_tag])
        # remove tag from other two posts since they were highlighted
        post2.handle_remove_tag(self.user, [foo_tag])
        post3.handle_remove_tag(self.user, [foo_tag])

        post2.handle_add_tag(self.user, [bar_tag])
        # remove tag from other two posts since they were highlighted
        post1.handle_remove_tag(self.user, [bar_tag])
        post3.handle_remove_tag(self.user, [bar_tag])

        # one post with foo_tag
        posts = self._get_posts(**{
            'channel_id': str(foo_tag.id),
            'from': now(),
            'to': now() + timedelta(minutes=1),
            'level': 'hour',
            'topics': [],
            'intentions': ['needs'],
        })
        self.assertEqual(len(posts), 1)

        # two posts with foo_tag and bar_tag
        posts = self._get_posts(**{
            'channel_id': [str(foo_tag.id), str(bar_tag.id)],
            'from': now(),
            'to': now() + timedelta(minutes=1),
            'level': 'hour',
            'topics': [],
            'intentions': ['needs'],
        })
        self.assertEqual(len(posts), 2)
Example #13
0
 def aggregate_state(self, worker, wstate):
     LOGGER.debug('[aggregate state invoked] worker: %s', worker)
     state = self.state
     if worker in state:
         name, _, _ = state[worker]
         state[worker] = [name, wstate, now()]
     else:
         idx = len([1 for w in state if type(w) == type(worker)])
         name = '%s_%s' % (type(worker).__name__, idx)
         state[worker] = [name, wstate, now()]
     # res = {name: [wstate, dt] for name, wstate, dt in state.values()}
     return {name: wstate}
Example #14
0
    def setUp(self):
        UICase.setUp(self)
        self.login()

        self.created_at = now()
        self.created_at_str = self.created_at.strftime('%Y-%m-%d %H:%M:%S')

        for content in ('I need a bike. I like Honda.',
                        'Can somebody recommend a sturdy laptop?',
                        'I need an affordabl laptop. And a laptop bag',
                        'Whatever you buy, let it be an Apple laptop',
                        'I would like to have a thin and lightweight laptop.'):
            self._create_db_post(content=content, _created=now())
Example #15
0
    def query_responses(post_ids, sort_by, user):
        t_start = now()
        response_ids = [ "%s:r" % post_id for post_id in post_ids ]
        response_filter = {"id__in": response_ids}
        if response_type:
            queries = []

            if 'review' in response_type:
                response_type.remove('review')
                response_type.extend(REVIEW_STATUSES)

            posted_matchable_filters = set(POSTED_MATCHABLE).intersection(set(response_type))
            response_status_filters  = set(RESPONSE_STATUSES).intersection(set(response_type))

            if posted_matchable_filters:
                queries.append({
                    #"status": "posted",
                    "posted_matchable__in": list(posted_matchable_filters)})

            if 'clicks' in response_type:
                queries.append({"clicked_count__gt": 0})

            if response_status_filters:
                queries.append({"status__in": response_type})

            if len(queries) > 1:
                response_filter.update({"$or": queries})
            elif queries:
                response_filter.update(queries[0])

        responses = list(Response.objects.find(**response_filter).limit(paging_params['limit']))

        total_count = len(responses)
        if sort_by == 'intention':
            responses.sort(key=lambda p: -p.intention_confidence)
        else:
            responses.sort(key=lambda p: p.created, reverse=True)

        app.logger.debug("Fetched %d responses in %s seconds" % (
                total_count, now() - t_start))

        res = [
            {'match'         : matchable_to_dict(resp.matchable, resp.relevance),
             'post'          : post_to_dict(resp.post, user),
             'response_type' : resp.status,
             'clicks'        : resp.clicked_count,
             'acted_date'    : datetime_to_timestamp_ms(resp.created)}
            for resp in responses
        ]

        return res
Example #16
0
    def request(
            self, path, args=None, post_args=None, files=None, method=None):

        start = now()
        self.check_rate_limits(path)
        try:
            response = super(GraphAPI, self).request(path, args, post_args, files, method)
        except facebook.GraphAPIError as error:
            log_item = self.log_request(path, args, post_args, method, start, now(), error)
            self.handle_rate_limit_error(error, path, failed_request_time=start, log_item=log_item.id)
            raise error

        self.log_request(path, args, post_args, method, start, now(), error=None)
        return response
Example #17
0
    def _create_posts(self):
        past_created = now() - timedelta(minutes=7 * 24 * 60)
        post1 = self._create_db_post(_created=past_created,
                                     content='i need some carrot')

        past_created = now() - timedelta(minutes=7 * 24 * 60 + 10)
        post2 = self._create_db_post(_created=past_created,
                                     content='Where I can buy a carrot?')

        self._create_db_post(content='i need some carrot')
        self._create_db_post(content='Where I can buy a carrot?')

        self.assertEqual(
            Post.objects(channels__in=[self.channel.id]).count(), 4)
Example #18
0
    def create_by_user(self, user, **kw):
        # assert kw.get('case_number'), kw

        if not kw.get('_created'):
            kw['_created'] = now()

        if not kw.get('actor_id'):
            kw['actor_id'] = kw['case_number']

        try:
            nps_profile = NPSProfile.objects.get(kw['actor_id'])
        except:
            nps_profile = NPSProfile.objects.create(id=kw['actor_id'])
        _id = NPSPost.gen_id(actor_id=nps_profile.id,
                             _created=kw['_created'],
                             in_reply_to_native_id=None)

        # from solariat_bottle.db.channel.voc import *
        # chan = VOCChannel.objects.get(kw['channels'][0])
        # if isinstance(chan, VOCServiceChannel):
        #     kw['channels'] = str(chan.inbound_channel.id)
        normalize_post_params(user, kw)
        post = super(NPSPostManager, self).create(_id=_id,
                                                  force_create=True,
                                                  **kw)

        for sc in post.service_channels:
            sc.post_received(post)
        _set_channel_and_tag_assignments(post)
        return post
Example #19
0
    def test_facebook_event(self):
        ts = str(now())
        content = 'tweet content'
        status_id = "12345678901_123456789101"
        from solariat_bottle.db.channel.facebook import FacebookServiceChannel
        data = {
            'content':
            content,
            'channels': [
                FacebookServiceChannel.objects.create_by_user(self.user,
                                                              title='Fb')
            ],
            'user_profile':
            self.get_user_profile('Facebook'),
            'facebook': {
                'facebook_post_id': status_id,
                'created_at': ts,
                'text': content,
                '_wrapped_data': {
                    'type': 'status',
                    'source_id': 'fake',
                    'source_type': 'event'
                }
            }
        }
        # posts with the same id and created date should not duplicate
        post1 = self._create_db_post(**data)
        post2 = self._create_db_post(**data)
        self.assertEqual(post1.id, post2.id)

        # posts with the same created date and different id should be separated. This is a test of the use
        # of fine-grain resolution
        data['facebook']['facebook_post_id'] = fake_status_id()
        post3 = self._create_db_post(**data)
        self.assertNotEqual(post2.id, post3.id)
Example #20
0
    def _create_tweet(self,
                      content,
                      channel=None,
                      channels=None,
                      demand_matchables=False,
                      in_reply_to=None,
                      **kw):
        from solariat.utils import timeslot

        status_id = fake_status_id()
        post_data = {
            'twitter': {
                'id': status_id,
                'created_at': kw.get('_created', None) or timeslot.now(),
                'text': content
            }
        }
        post_data.update(kw)
        if in_reply_to and hasattr(in_reply_to, 'native_id'):
            post_data['twitter']['in_reply_to_status_id'] = in_reply_to.native_id
        return self._create_db_post(content,
                                    channel=channel,
                                    channels=channels,
                                    demand_matchables=demand_matchables,
                                    **post_data)
Example #21
0
    def setUp(self):
        UICase.setUp(self)
        self.login()

        self.created_at = now()

        self.created_on_str = format_date(self.created_at)
        self.one_day_after_str = format_date(self.created_at +
                                             timedelta(days=1))
        self.one_day_before_str = format_date(self.created_at -
                                              timedelta(days=1))
        self.one_week_after_str = format_date(self.created_at +
                                              timedelta(days=7))
        self.one_week_before_str = format_date(self.created_at -
                                               timedelta(days=7))

        self.contents = (
            'I need a bike. I like Honda.',  # needs, likes
            'Can somebody recommend a sturdy laptop?',  # consideration
            'I need an affordabl laptop. And a laptop bag',  # needs
            'Whatever you buy, let it be an Apple laptop',  # recommendation
            'I would like to have a thin and lightweight laptop.',  # needs
            'Thank you very much!',  # gratitude
            'You\'re gonna end up with a broken laptop'  # problem
        )
        self.setup_posts()
Example #22
0
    def signal(self, sgn):
        self.logger.info("Received signal %s (%s)" %
                         (sgn, trap_signals.get(sgn)))

        if sgn == signal.SIGHUP:
            self.server.sync_streams()
        elif sgn in {signal.SIGUSR1, signal.SIGUSR2}:
            try:
                status = self.get_status()
                print('---')
                print(json.dumps(status, indent=4))
                print('---')
                sys.stdout.flush()

                if sgn == signal.SIGUSR2:
                    status.update({"timestamp": now()})
                    self.db_events.add_bot_status(status)
            except:
                self.logger.exception("dm_bot.signal")
        elif sgn in {signal.SIGINT, signal.SIGQUIT, signal.SIGTERM}:
            self.stop()
            # gevent.get_hub().destroy()
            timeout = self.heartbeat
            for idx in xrange(timeout):
                sleep(1)
                if not self.isAlive():
                    break
            else:
                print("Bot never stopped after waiting %s seconds." %
                      (timeout, ))
                sys.exit(2)
            sys.exit(0)
Example #23
0
    def _prepare_data(self,
                      n_jobs=JobsNum(10, 5, 5),
                      from_date=None,
                      to_date=None):
        name = 'job_name'
        account = self.account
        self.jobs = []
        if to_date is None:
            to_date = now()
        if from_date is None:
            from_date = to_date - datetime.timedelta(days=30)

        def gen_jobs_data():
            for status in [JobModel.PENDING] * n_jobs.pending + [
                    JobModel.RUNNING
            ] * n_jobs.running + [JobModel.SUCCESSFUL] * n_jobs.completed:
                created_at, started_date, completion_date = random_status_dates(
                    from_date, to_date)
                data = {
                    'created_at': created_at,
                    'status': status,
                    'account': account.id,
                }
                if status == JobModel.SUCCESSFUL:
                    data['started_date'], data[
                        'completion_date'] = started_date, completion_date
                elif status == JobModel.RUNNING:
                    data['started_date'] = started_date
                yield data

        for job_data in gen_jobs_data():
            job = self.create_job(name)
            job.update(**job_data)
            self.jobs.append(job)
Example #24
0
    def test_duplicate_handle_diff_channels(self):

        channel3 = TwitterChannel.objects.create_by_user(
            self.user,
            title='TestChannel3',
            type='twitter',
            intention_types=SA_TYPES)

        duplicate_post = self._create_db_post(
            channels=[self.channel2, channel3],
            content=self.duplicate_content,
            url=self.url,
            twitter={
                "created_at": "Wed, 06 Aug 2014 18:38:47 +0000",
                "id": "497089420017676290"
            })

        self.assertEqual(len(duplicate_post.channels), 3)

        time_slot = datetime_to_timeslot(now(), 'day')
        ht_stat = ChannelHotTopics.objects.by_time_span(
            channel=self.channel2,
            from_ts=datetime_to_timeslot(None, 'day'),
        )

        tt_stat = ChannelTopicTrends(channel=self.channel2,
                                     time_slot=time_slot,
                                     topic=self.topic,
                                     status=0)

        self.assertEqual(ht_stat, self.hot_topic_stat)
        self.assertEqual(tt_stat, self.topic_trends_stat)
Example #25
0
    def test_select_by_time_span_3(self):
        past_dt = now() - relativedelta(months=1)  # big enough for all levels

        post1 = self._create_db_post(_created=past_dt,
                                     content='i need some carrot')

        post2 = self._create_db_post(content='i need some carrot')

        self.assertEqual(
            Post.objects(channels__in=[self.channel.id]).count(), 2)

        for level in ('hour', 'day'):
            result = ChannelTopicTrends.objects.by_time_span(
                channel=self.channel,
                topic_pairs=[['carrot', True]],
                from_ts=datetime_to_timeslot(past_dt, level),
                to_ts=datetime_to_timeslot(None, level))
            self.assertEqual(len(result), 2)

            result = ChannelTopicTrends.objects.by_time_span(
                channel=self.channel,
                topic_pairs=[['carrot', True]],
                from_ts=datetime_to_timeslot(
                    past_dt + relativedelta(**{level + 's': 1}), level),
                to_ts=datetime_to_timeslot(None, level))
            self.assertEqual(len(result), 1)
Example #26
0
    def test_purge_conversations_1(self):
        """ Deleting outdated conversation of two posts"""
        self._make_setup_for_conversations()
        self._create_db_post(channel=self.sc.inbound,
                             content="I need a foo.",
                             demand_matchables=True,
                             user_profile={'screen_name': 'customer'})
        self._create_db_post(channel=self.sc.inbound,
                             content="Does anyone have a foo?",
                             demand_matchables=True,
                             user_profile={'screen_name': 'customer'})

        self.assertEqual(Conversation.objects.count(), 1)
        self.assertEqual(Post.objects.count(), 2)
        self.assertEqual(SpeechActMap.objects.count(), 2)

        inbound_channel = Channel.objects.get(id=self.sc.inbound)
        purge_channel_entities(
            inbound_channel,
            run_in_prod_mod=True,
            now_date=now() +
            timedelta(days=get_var("CHANNEL_ENTITIES_KEEP_DAYS") + 1))

        self.assertEqual(Conversation.objects.count(), 0)
        self.assertEqual(SpeechActMap.objects.count(), 0)
        self.assertEqual(Post.objects.count(), 0)
Example #27
0
    def test_facebook_post(self):
        updated_at = now().replace(microsecond=0)
        created_at = TIMESLOT_EPOCH - timedelta(days=300)

        content = 'post image caption'
        status_id = "12345678901_123456789101"
        from solariat_bottle.db.channel.facebook import FacebookServiceChannel
        data = {
            'content':
            content,
            'channels': [
                FacebookServiceChannel.objects.create_by_user(self.user,
                                                              title='Fb')
            ],
            'user_profile': {
                'user_name': 'fb_test_user'
            },
            'facebook': {
                'facebook_post_id': status_id,
                'text': content,
                'created_at': str(created_at),
                '_wrapped_data': {
                    'type': 'status',
                    'source_id': 'fake',
                    'source_type': 'event',
                    'created_at': str(created_at),
                    'updated_time': str(updated_at)
                }
            }
        }

        post1 = self._create_db_post(**data)
        actor_id, creation_time = unpack_event_id(post1.id)
        self.assertEqual(post1.created_at.replace(microsecond=0), updated_at)
        self.assertEqual(creation_time.replace(microsecond=0), updated_at)
Example #28
0
    def on_data(self, raw_data):
        """Called when raw data is received from connection.

        Override this method if you wish to manually handle
        the stream data. Return False to stop stream and close connection.
        """
        try:
            data = json.loads(raw_data)
        except ValueError:
            self.log_event("on_data() can't decode json: %s" % raw_data)
            return
        finally:
            self.last_keep_alive = now()

        if self.on_message(data):
            return
        # twitter warning events
        elif 'limit' in data:
            if self.on_limit(data['limit']) is False:
                return False
        elif 'disconnect' in data:
            if self.on_disconnect(data['disconnect']) is False:
                return False
        elif 'warning' in data:
            # warnings can be received only with stall_warnings=True
            if self.on_warning(data['warning']) is False:
                return False
        else:
            self.log_event("unknown message type: " + str(raw_data))
Example #29
0
 def get_or_create(self, track, follow, languages, account_ids,
                   channel_ids):
     ref = StreamRef(track=track, follow=follow, languages=languages)
     is_new = False
     existing_ref = StreamRef.objects.find_one(id=ref.key)
     if existing_ref is None:
         is_new = True
         ref.status = StreamRef.QUEUED
         log = StreamLog(accounts=account_ids,
                         channels=channel_ids,
                         stream_ref_id=ref.id)
         log.save()
         ref.log = log
         ref.save()
     else:
         ref = existing_ref
         ref_log = ref.log
         if (not ref_log or set(ref_log.channels) != set(channel_ids)
                 or set(ref_log.accounts) != set(account_ids)
                 or ref.is_stopped()):
             # if tracked accounts or channels change
             # update stream log
             if ref_log:
                 ref_log.update(stopped_at=now())
             else:
                 is_new = True
             if ref.is_stopped():
                 is_new = True
             log = StreamLog(accounts=account_ids,
                             channels=channel_ids,
                             stream_ref_id=existing_ref.id)
             log.save()
             ref.update(log=log, status=StreamRef.RUNNING)
     return ref, is_new
Example #30
0
    def test_stats_retrieving(self):
        time_slot = datetime_to_timeslot(now(), 'day')
        topics = ('laptop', 'laptop bag', 'good laptop bag', 'good laptop')
        for topic in topics:
            for term, is_leaf in gen_topic_tree(topic):
                ChannelHotTopics.increment(self.channel,
                                           time_slot,
                                           term,
                                           status=0,
                                           intention_id=0,
                                           is_leaf=is_leaf,
                                           lang_id=Lang.EN,
                                           agent=1)

        stats = ChannelHotTopics.objects.by_time_span(
            channel=self.channel,
            from_ts=datetime_to_timeslot(None, 'day'),
            languages=['en'])

        expected_result = [{
            u'term_count': 2,
            u'topic': u'laptop',
            u'topic_count': 1
        }, {
            u'term_count': 2,
            u'topic': u'bag',
            u'topic_count': 0
        }]

        self.assertListEqual(stats, expected_result)