Python RawStreamQueue Examples

Example #1

File: test_classifierWorker.py Project: BastinRobin/streamcrab

 def setUp(self):
     super(TestClassifierWorker, self).setUp()
     self.cls = nltk.NaiveBayesClassifier.train([({'a': 'a', 'b': 'b'}, 'positive')])
     self.data = []
     for i in range(5):
         d = RawStreamQueue()
         d.source = 'test'
         d.original = {}
         d.text = "this is a test"
         self.data.append(d)
     ClassifiedStream.drop_collection()

Example #2

    def _run(self):
        self.logger.info('Starting')

        while not self.stop.is_set():

            if not RawStreamQueue.objects().count():
                time.sleep(0.5)
                continue

            raw_data = RawStreamQueue.objects().order_by('-id')[0:config.classifier_pool_size]
            for task in raw_data:
                self.queue.put(task)
                task.delete()

Example #3

File: pool.py Project: BastinRobin/streamcrab

    def _run(self):
        self.logger.info('Starting')

        while not self.stop.is_set():

            if not RawStreamQueue.objects().count():
                time.sleep(0.5)
                continue

            raw_data = RawStreamQueue.objects().order_by('-id')[0:config.classifier_pool_size]
            for task in raw_data:
                self.queue.put(task)
                task.delete()

Example #4

 def setUp(self):
     super(TestClassifierWorker, self).setUp()
     self.cls = nltk.NaiveBayesClassifier.train([({
         'a': 'a',
         'b': 'b'
     }, 'positive')])
     self.data = []
     for i in range(5):
         d = RawStreamQueue()
         d.source = 'test'
         d.original = {}
         d.text = "this is a test"
         self.data.append(d)
     ClassifiedStream.drop_collection()

Example #5

File: test_twitterWorkerSleep.py Project: ishantanu/Tweetalytics

    def test_workerSleep(self):
        """
        twitter collector should sleep if not keywords found
        """
        RawStreamQueue.drop_collection()
        SocketSession.drop_collection()
        kill = threading.Event()

        self.assertEqual(len(SocketSession.get_keywords()), 0)
        w = TwitterWorker(kill)
        w.setDaemon(True)
        w.start()
        time.sleep(1)
        kill.set()
        self.assertEqual(RawStreamQueue.objects.count(), 0)

Example #6

File: test_twitterWorkerSleep.py Project: iamShantanu101/Tweetalytics

    def test_workerSleep(self):
        """
        twitter collector should sleep if not keywords found
        """
        RawStreamQueue.drop_collection()
        SocketSession.drop_collection()
        kill = threading.Event()

        self.assertEqual(len(SocketSession.get_keywords()),0)
        w = TwitterWorker(kill)
        w.setDaemon(True)
        w.start()
        time.sleep(1)
        kill.set()
        self.assertEqual(RawStreamQueue.objects.count(), 0)

Example #7

File: twitterworker.py Project: devjyotip/twitter-analytics-dashboard

 def save(self, tweet):
     if self.is_tweet_valid(tweet):
         o = RawStreamQueue()
         o.original = tweet
         o.source = StreamSource.TWITTER
         o.text = tweet['text']
         o.save()
         logger.debug("RawStreamQueue saved with id %s", o.id)

Example #8

File: twitterworker.py Project: jimmy0000/streamcrab

 def save(self, tweet):
     if self.is_tweet_valid(tweet):
         o = RawStreamQueue()
         o.original = tweet
         o.source = StreamSource.TWITTER
         o.text = tweet['text']
         o.save()
         logger.debug("RawStreamQueue saved with id %s", o.id)

Example #9

File: test_twitterWorker.py Project: BastinRobin/streamcrab

 def tearDown(self):
     RawStreamQueue.drop_collection()
     SocketSession.drop_collection()

Example #10

File: test_twitterWorker.py Project: ishantanu/Tweetalytics

 def tearDown(self):
     RawStreamQueue.drop_collection()
     SocketSession.drop_collection()

Example #11

File: test_classifierWorker.py Project: BastinRobin/streamcrab

 def tearDown(self):
     ClassifiedStream.drop_collection()
     RawStreamQueue.drop_collection()

Example #12

 def tearDown(self):
     ClassifiedStream.drop_collection()
     RawStreamQueue.drop_collection()