Exemplo n.º 1
0
 def test_addDocument(self):
     message1 = Message(3, 'sdf',
                        'A project to cluster high-dimensional streams.',
                        test_time)
     message1.vector = Vector({3: 4})
     stream1 = Stream(3, message1)
     message2 = Message(4, 'sdf',
                        'A project to cluster high-dimensional streams.',
                        test_time)
     message2.vector = Vector({2: 4})
     stream2 = Stream(4, message2)
     self.assertNotEqual(test_time, self.cluster1.lastStreamAddedTime)
     self.cluster1.addDocument(stream1)
     self.assertEqual(test_time, self.cluster1.lastStreamAddedTime)
     # Test if cluster id is set.
     self.assertEqual(self.cluster1.clusterId, stream1.clusterId)
     # Test that cluster mean is updated.
     self.assertEqual({1: 2 / 2., 2: 2., 3: 2.}, self.cluster1)
     # Test that cluster aggrefate is updated.
     self.assertEqual({1: 2, 2: 4, 3: 4}, self.cluster1.aggregateVector)
     # Test that document is added to cluster documents.
     self.assertEqual(stream1,
                      self.cluster1.documentsInCluster[stream1.docId])
     self.cluster1.addDocument(stream2)
     self.assertEqual(3, self.cluster1.vectorWeights)
     self.assertEqual({1: 2 / 3., 2: 8 / 3., 3: 4 / 3.}, self.cluster1)
     self.assertEqual({1: 2, 2: 8, 3: 4}, self.cluster1.aggregateVector)
Exemplo n.º 2
0
 def test_crowdSize(self):
     self.assertEqual(1, self.crowd.crowdSize)
     self.cluster.addDocument(Stream(2, self.m1))
     self.cluster.addDocument(Stream(3, self.m1))
     self.assertEqual(3, self.crowd.crowdSize)
     cluster = StreamCluster(Stream(3, self.m1))
     self.crowd.append(cluster, test_time + timedelta(days=2))
     self.assertNotEqual(4, self.crowd.crowdSize)
     self.assertEqual(3, self.crowd.crowdSize)
Exemplo n.º 3
0
 def setUp(self):
     self.m1 = Message(1, 'sdf',
                       'A project to cluster high-dimensional streams.',
                       test_time - timedelta(seconds=60))
     self.m1.vector = Vector({1: 1., 2: 3.})
     self.stream = Stream(1, self.m1)
     self.m2 = Message(1, 'sdf',
                       'A project to cluster high-dimensional streams.',
                       test_time)
     self.m2.vector = Vector({2: 3.})
Exemplo n.º 4
0
 def setUp(self):
     self.message = Message(
         1, 'sdf', 'A project to cluster high-dimensional streams.',
         datetime.now())
     self.message.vector = Vector({1: 2., 2: 3.})
     self.s1 = Stream(1, self.message)
     self.v1 = Vector({1: 2., 3: 3.})
Exemplo n.º 5
0
 def cluster(self, dataIterator):
     i = 1
     for data in dataIterator:
         message = self.convertDataToMessageMethod(data,
                                                   **self.stream_settings)
         #            message = data
         if DataStreamMethods.messageInOrder(message.timeStamp):
             UtilityMethods.updatePhraseTextToPhraseObject(
                 message.vector, message.timeStamp,
                 self.phraseTextToPhraseObjectMap, **self.stream_settings)
             if message.streamId not in self.streamIdToStreamObjectMap:
                 self.streamIdToStreamObjectMap[message.streamId] = Stream(
                     message.streamId, message)
             else:
                 self.streamIdToStreamObjectMap[
                     message.streamId].updateForMessage(
                         message, VectorUpdateMethods.exponentialDecay,
                         **self.stream_settings)
             streamObject = self.streamIdToStreamObjectMap[message.streamId]
             self.updateDimensionsMethod.call(
                 message.timeStamp,
                 hdStreamClusteringObject=self,
                 currentMessageTime=message.timeStamp)
             self.clusterFilteringMethod.call(
                 message.timeStamp,
                 hdStreamClusteringObject=self,
                 currentMessageTime=message.timeStamp)
             self.clusterAnalysisMethod.call(
                 message.timeStamp,
                 hdStreamClusteringObject=self,
                 currentMessageTime=message.timeStamp)
             self.getClusterAndUpdateExistingClusters(streamObject)
Exemplo n.º 6
0
 def test_maxClusterSize(self):
     self.assertEqual(1, self.crowd.maxClusterSize)
     message2 = Message(4, 'sdf',
                        'A project to cluster high-dimensional streams.',
                        test_time)
     message2.vector = Vector({2: 4})
     stream2 = Stream(4, message2)
     self.cluster.addDocument(stream2)
     self.assertEqual(2, self.crowd.maxClusterSize)
Exemplo n.º 7
0
 def setUp(self):
     self.m1 = Message(1, 'sdf',
                       'A project to cluster high-dimensional streams.',
                       test_time - timedelta(seconds=60))
     self.m1.vector = Vector({1: 2, 2: 4})
     self.stream1 = Stream(1, self.m1)
     self.m2 = Message(2, 'sdf',
                       'A project to cluster high-dimensional streams.',
                       test_time)
     self.m2.vector = Vector({2: 4})
     self.stream2 = Stream(2, self.m2)
     self.m3 = Message(3, 'sdf',
                       'A project to cluster high-dimensional streams.',
                       test_time + timedelta(seconds=60))
     self.m3.vector = Vector({2: 4})
     self.stream3 = Stream(3, self.m3)
     self.cluster1 = StreamCluster(self.stream1)
     self.cluster2 = StreamCluster(self.stream2)
     self.cluster3 = StreamCluster(self.stream3)
Exemplo n.º 8
0
class StreamTests(unittest.TestCase):
    def setUp(self):
        self.m1 = Message(1, 'sdf',
                          'A project to cluster high-dimensional streams.',
                          test_time - timedelta(seconds=60))
        self.m1.vector = Vector({1: 1., 2: 3.})
        self.stream = Stream(1, self.m1)
        self.m2 = Message(1, 'sdf',
                          'A project to cluster high-dimensional streams.',
                          test_time)
        self.m2.vector = Vector({2: 3.})

    def test_updateForMessage_addWithoutDecay(self):
        self.stream.updateForMessage(self.m2,
                                     VectorUpdateMethods.addWithoutDecay,
                                     **stream_settings)
        self.assertEqual(self.stream, Vector({1: 1., 2: 6.}))

    def test_updateForMessage_exponentialDecay(self):
        self.stream.updateForMessage(self.m2,
                                     VectorUpdateMethods.exponentialDecay,
                                     **stream_settings)
        self.assertEqual(self.stream, Vector({1: 0.5, 2: 4.5}))

    def test_check_lastMessageTime_is_updated(self):
        self.assertEqual(test_time - timedelta(seconds=60),
                         self.stream.lastMessageTime)
        self.stream.updateForMessage(self.m2,
                                     VectorUpdateMethods.addWithoutDecay,
                                     **stream_settings)
        self.assertNotEqual(test_time - timedelta(seconds=60),
                            self.stream.lastMessageTime)
        self.assertEqual(test_time, self.stream.lastMessageTime)
Exemplo n.º 9
0
 def getClusterFromMapFormat(clusterMap):
     dummyMessage = Message(1, '', '', datetime.now())
     dummyMessage.vector = Vector({})
     dummyStream = Stream(1, dummyMessage)
     cluster = StreamCluster(dummyStream)
     cluster.clusterId = clusterMap['clusterId']
     cluster.lastStreamAddedTime = getDateTimeObjectFromTweetTimestamp(
         clusterMap['lastStreamAddedTime'])
     cluster.mergedClustersList = clusterMap['mergedClustersList']
     cluster.documentsInCluster = clusterMap['streams']
     for k, v in clusterMap['dimensions'].iteritems():
         cluster[k] = v
     return cluster
Exemplo n.º 10
0
    def cluster(self, dataIterator):
        i = 1
        for data in dataIterator:
            message = self.convertDataToMessageMethod(data,
                                                      **self.stream_settings)
            #            message = data
            if DataStreamMethods.messageInOrder(message.timeStamp):
                UtilityMethods.updatePhraseTextToPhraseObject(
                    message.vector, message.timeStamp,
                    self.phraseTextToPhraseObjectMap, **self.stream_settings)
                if message.streamId not in self.streamIdToStreamObjectMap:
                    self.streamIdToStreamObjectMap[message.streamId] = Stream(
                        message.streamId, message)
                    self.getClusterAndUpdateExistingClusters(
                        self.streamIdToStreamObjectMap[message.streamId])
                else:
                    previousStreamObject = Vector(
                        vectorInitialValues=self.streamIdToStreamObjectMap[
                            message.streamId])
                    self.streamIdToStreamObjectMap[
                        message.streamId].updateForMessage(
                            message, VectorUpdateMethods.exponentialDecay,
                            **self.stream_settings)
                    streamObject = self.streamIdToStreamObjectMap[
                        message.streamId]
                    distance = Vector.euclideanDistance(
                        streamObject, previousStreamObject)
                    if distance > 10:
                        #                        print i, len(self.clusters), distance
                        self.getClusterAndUpdateExistingClusters(
                            self.streamIdToStreamObjectMap[message.streamId])

                        self.updateDimensionsMethod.call(
                            message.timeStamp,
                            hdStreamClusteringObject=self,
                            currentMessageTime=message.timeStamp)
                        self.clusterFilteringMethod.call(
                            message.timeStamp,
                            hdStreamClusteringObject=self,
                            currentMessageTime=message.timeStamp)

        #                self.clusterAnalysisMethod.call(message.timeStamp, hdStreamClusteringObject=self, currentMessageTime=message.timeStamp)

                    self.clusterAnalysisMethod.call(
                        time.time(),
                        hdStreamClusteringObject=self,
                        currentMessageTime=message.timeStamp,
                        numberOfMessages=i)

#                print i, len(self.clusters)
                i += 1
Exemplo n.º 11
0
def rtmp_stage1_streamkey_check(key, ipaddress):
    sysSettings = settings.settings.query.first()

    channelRequest = Channel.Channel.query.filter_by(streamKey=key).first()

    currentTime = datetime.datetime.utcnow()

    if channelRequest is not None:
        userQuery = Sec.User.query.filter_by(id=channelRequest.owningUser).first()
        if userQuery is not None:
            if userQuery.has_role('Streamer'):

                if not userQuery.active:
                    returnMessage = {'time': str(currentTime), 'request': 'Stage1', 'success': False, 'channelLoc': channelRequest.channelLoc, 'type': None, 'ipAddress': str(ipaddress), 'message': 'Unauthorized User - User has been disabled'}
                    return returnMessage

                existingStreamQuery = Stream.Stream.query.filter_by(linkedChannel=channelRequest.id).all()
                if existingStreamQuery:
                    for stream in existingStreamQuery:
                        db.session.delete(stream)
                    db.session.commit()

                defaultStreamName = templateFilters.normalize_date(str(currentTime))
                if channelRequest.defaultStreamName != "":
                    defaultStreamName = channelRequest.defaultStreamName

                newStream = Stream.Stream(key, defaultStreamName, int(channelRequest.id), channelRequest.topic)
                db.session.add(newStream)
                db.session.commit()

                if sysSettings.adaptiveStreaming:
                    returnMessage = {'time': str(currentTime), 'request': 'Stage1', 'success': True, 'channelLoc': channelRequest.channelLoc, 'type': 'adaptive', 'ipAddress': str(ipaddress), 'message': 'Success - Passing to Stage 2 - Adaptive'}
                    return returnMessage
                else:
                    returnMessage = {'time': str(currentTime), 'request': 'Stage1', 'success': True, 'channelLoc': channelRequest.channelLoc, 'type': 'standard', 'ipAddress': str(ipaddress), 'message': 'Success - Passing to Stage 2 - Standard'}
                    return returnMessage

            else:
                returnMessage = {'time': str(currentTime), 'request': 'Stage1', 'success': False, 'channelLoc': channelRequest.channelLoc, 'type': None, 'ipAddress': str(ipaddress), 'message': 'Unauthorized User - Missing Streamer Role'}
                db.session.close()
                return returnMessage
        else:
            returnMessage = {'time': str(currentTime), 'request': 'Stage1', 'success': False, 'channelLoc': channelRequest.channelLoc, 'type': None, 'ipAddress': str(ipaddress), 'message': 'Unauthorized User - No Such User'}
            db.session.close()
            return returnMessage
    else:
        returnMessage = {'time': str(currentTime), 'request': 'Stage1', 'success': False, 'channelLoc': None, 'type': None, 'ipAddress': str(ipaddress), 'message': 'Unauthorized Key'}
        db.session.close()
        return returnMessage
Exemplo n.º 12
0
class StreamTests(unittest.TestCase):
    def setUp(self):
        self.m1 = Message(1, 'sdf', 'A project to cluster high-dimensional streams.', test_time-timedelta(seconds=60))
        self.m1.vector=Vector({1:1.,2:3.})
        self.stream = Stream(1, self.m1)
        self.m2 = Message(1, 'sdf', 'A project to cluster high-dimensional streams.', test_time)
        self.m2.vector=Vector({2:3.})
    def test_updateForMessage_addWithoutDecay(self):
        self.stream.updateForMessage(self.m2, VectorUpdateMethods.addWithoutDecay, **stream_settings)
        self.assertEqual(self.stream, Vector({1:1.,2:6.}))
    def test_updateForMessage_exponentialDecay(self):
        self.stream.updateForMessage(self.m2, VectorUpdateMethods.exponentialDecay, **stream_settings)
        self.assertEqual(self.stream, Vector({1:0.5,2:4.5}))
    def test_check_lastMessageTime_is_updated(self):
        self.assertEqual(test_time-timedelta(seconds=60), self.stream.lastMessageTime)
        self.stream.updateForMessage(self.m2, VectorUpdateMethods.addWithoutDecay, **stream_settings)
        self.assertNotEqual(test_time-timedelta(seconds=60), self.stream.lastMessageTime)
        self.assertEqual(test_time, self.stream.lastMessageTime)
Exemplo n.º 13
0
 def setUp(self):
     self.m1 = Message(1, 'sdf', 'A project to cluster high-dimensional streams.', test_time-timedelta(seconds=60))
     self.m1.vector=Vector({1:1.,2:3.})
     self.stream = Stream(1, self.m1)
     self.m2 = Message(1, 'sdf', 'A project to cluster high-dimensional streams.', test_time)
     self.m2.vector=Vector({2:3.})
Exemplo n.º 14
0
def streamkey_check():
    sysSettings = settings.settings.query.first()

    key = request.form['name']
    ipaddress = request.form['addr']

    channelRequest = Channel.Channel.query.filter_by(streamKey=key).first()

    currentTime = datetime.datetime.now()

    if channelRequest is not None:
        userQuery = Sec.User.query.filter_by(
            id=channelRequest.owningUser).first()
        if userQuery is not None:
            if userQuery.has_role('Streamer'):

                if not userQuery.active:
                    returnMessage = {
                        'time': str(currentTime),
                        'status': 'Unauthorized User - User has been Disabled',
                        'key': str(key),
                        'ipAddress': str(ipaddress)
                    }
                    print(returnMessage)
                    return abort(400)

                returnMessage = {
                    'time': str(currentTime),
                    'status': 'Successful Key Auth',
                    'key': str(key),
                    'channelName': str(channelRequest.channelName),
                    'userName': str(channelRequest.owningUser),
                    'ipAddress': str(ipaddress)
                }
                print(returnMessage)

                validAddress = system.formatSiteAddress(
                    sysSettings.siteAddress)

                existingStreamQuery = Stream.Stream.query.filter_by(
                    linkedChannel=channelRequest.id).all()
                if existingStreamQuery:
                    for stream in existingStreamQuery:
                        db.session.delete(stream)
                    db.session.commit()

                defaultStreamName = templateFilters.normalize_date(
                    str(currentTime))
                if channelRequest.defaultStreamName != "":
                    defaultStreamName = channelRequest.defaultStreamName

                newStream = Stream.Stream(key, defaultStreamName,
                                          int(channelRequest.id),
                                          channelRequest.topic)
                db.session.add(newStream)
                db.session.commit()

                if sysSettings.adaptiveStreaming:
                    return redirect('rtmp://' + coreNginxRTMPAddress +
                                    '/stream-data-adapt/' +
                                    channelRequest.channelLoc,
                                    code=302)
                else:
                    return redirect('rtmp://' + coreNginxRTMPAddress +
                                    '/stream-data/' +
                                    channelRequest.channelLoc,
                                    code=302)

            else:
                returnMessage = {
                    'time': str(currentTime),
                    'status': 'Unauthorized User - Missing Streamer Role',
                    'key': str(key),
                    'ipAddress': str(ipaddress)
                }
                print(returnMessage)
                db.session.close()
                return abort(400)
        else:
            returnMessage = {
                'time': str(currentTime),
                'status': 'Unauthorized User - No Such User',
                'key': str(key),
                'ipAddress': str(ipaddress)
            }
            print(returnMessage)
            db.session.close()
            return abort(400)
    else:
        returnMessage = {
            'time': str(currentTime),
            'status': 'Failed Key Auth',
            'key': str(key),
            'ipAddress': str(ipaddress)
        }
        print(returnMessage)
        db.session.close()
        return abort(400)