def test_addDocument(self):
message1 = Message(3, 'sdf',
'A project to cluster high-dimensional streams.',
test_time)
message1.vector = Vector({3: 4})
stream1 = Stream(3, message1)
message2 = Message(4, 'sdf',
'A project to cluster high-dimensional streams.',
test_time)
message2.vector = Vector({2: 4})
stream2 = Stream(4, message2)
self.assertNotEqual(test_time, self.cluster1.lastStreamAddedTime)
self.cluster1.addDocument(stream1)
self.assertEqual(test_time, self.cluster1.lastStreamAddedTime)
# Test if cluster id is set.
self.assertEqual(self.cluster1.clusterId, stream1.clusterId)
# Test that cluster mean is updated.
self.assertEqual({1: 2 / 2., 2: 2., 3: 2.}, self.cluster1)
# Test that cluster aggrefate is updated.
self.assertEqual({1: 2, 2: 4, 3: 4}, self.cluster1.aggregateVector)
# Test that document is added to cluster documents.
self.assertEqual(stream1,
self.cluster1.documentsInCluster[stream1.docId])
self.cluster1.addDocument(stream2)
self.assertEqual(3, self.cluster1.vectorWeights)
self.assertEqual({1: 2 / 3., 2: 8 / 3., 3: 4 / 3.}, self.cluster1)
self.assertEqual({1: 2, 2: 8, 3: 4}, self.cluster1.aggregateVector)
def test_crowdSize(self):
self.assertEqual(1, self.crowd.crowdSize)
self.cluster.addDocument(Stream(2, self.m1))
self.cluster.addDocument(Stream(3, self.m1))
self.assertEqual(3, self.crowd.crowdSize)
cluster = StreamCluster(Stream(3, self.m1))
self.crowd.append(cluster, test_time + timedelta(days=2))
self.assertNotEqual(4, self.crowd.crowdSize)
self.assertEqual(3, self.crowd.crowdSize)
def setUp(self):
self.m1 = Message(1, 'sdf',
'A project to cluster high-dimensional streams.',
test_time - timedelta(seconds=60))
self.m1.vector = Vector({1: 1., 2: 3.})
self.stream = Stream(1, self.m1)
self.m2 = Message(1, 'sdf',
'A project to cluster high-dimensional streams.',
test_time)
self.m2.vector = Vector({2: 3.})
def setUp(self):
self.message = Message(
1, 'sdf', 'A project to cluster high-dimensional streams.',
datetime.now())
self.message.vector = Vector({1: 2., 2: 3.})
self.s1 = Stream(1, self.message)
self.v1 = Vector({1: 2., 3: 3.})
def cluster(self, dataIterator):
i = 1
for data in dataIterator:
message = self.convertDataToMessageMethod(data,
**self.stream_settings)
# message = data
if DataStreamMethods.messageInOrder(message.timeStamp):
UtilityMethods.updatePhraseTextToPhraseObject(
message.vector, message.timeStamp,
self.phraseTextToPhraseObjectMap, **self.stream_settings)
if message.streamId not in self.streamIdToStreamObjectMap:
self.streamIdToStreamObjectMap[message.streamId] = Stream(
message.streamId, message)
else:
self.streamIdToStreamObjectMap[
message.streamId].updateForMessage(
message, VectorUpdateMethods.exponentialDecay,
**self.stream_settings)
streamObject = self.streamIdToStreamObjectMap[message.streamId]
self.updateDimensionsMethod.call(
message.timeStamp,
hdStreamClusteringObject=self,
currentMessageTime=message.timeStamp)
self.clusterFilteringMethod.call(
message.timeStamp,
hdStreamClusteringObject=self,
currentMessageTime=message.timeStamp)
self.clusterAnalysisMethod.call(
message.timeStamp,
hdStreamClusteringObject=self,
currentMessageTime=message.timeStamp)
self.getClusterAndUpdateExistingClusters(streamObject)
def test_maxClusterSize(self):
self.assertEqual(1, self.crowd.maxClusterSize)
message2 = Message(4, 'sdf',
'A project to cluster high-dimensional streams.',
test_time)
message2.vector = Vector({2: 4})
stream2 = Stream(4, message2)
self.cluster.addDocument(stream2)
self.assertEqual(2, self.crowd.maxClusterSize)
def setUp(self):
self.m1 = Message(1, 'sdf',
'A project to cluster high-dimensional streams.',
test_time - timedelta(seconds=60))
self.m1.vector = Vector({1: 2, 2: 4})
self.stream1 = Stream(1, self.m1)
self.m2 = Message(2, 'sdf',
'A project to cluster high-dimensional streams.',
test_time)
self.m2.vector = Vector({2: 4})
self.stream2 = Stream(2, self.m2)
self.m3 = Message(3, 'sdf',
'A project to cluster high-dimensional streams.',
test_time + timedelta(seconds=60))
self.m3.vector = Vector({2: 4})
self.stream3 = Stream(3, self.m3)
self.cluster1 = StreamCluster(self.stream1)
self.cluster2 = StreamCluster(self.stream2)
self.cluster3 = StreamCluster(self.stream3)
class StreamTests(unittest.TestCase):
def setUp(self):
self.m1 = Message(1, 'sdf',
'A project to cluster high-dimensional streams.',
test_time - timedelta(seconds=60))
self.m1.vector = Vector({1: 1., 2: 3.})
self.stream = Stream(1, self.m1)
self.m2 = Message(1, 'sdf',
'A project to cluster high-dimensional streams.',
test_time)
self.m2.vector = Vector({2: 3.})
def test_updateForMessage_addWithoutDecay(self):
self.stream.updateForMessage(self.m2,
VectorUpdateMethods.addWithoutDecay,
**stream_settings)
self.assertEqual(self.stream, Vector({1: 1., 2: 6.}))
def test_updateForMessage_exponentialDecay(self):
self.stream.updateForMessage(self.m2,
VectorUpdateMethods.exponentialDecay,
**stream_settings)
self.assertEqual(self.stream, Vector({1: 0.5, 2: 4.5}))
def test_check_lastMessageTime_is_updated(self):
self.assertEqual(test_time - timedelta(seconds=60),
self.stream.lastMessageTime)
self.stream.updateForMessage(self.m2,
VectorUpdateMethods.addWithoutDecay,
**stream_settings)
self.assertNotEqual(test_time - timedelta(seconds=60),
self.stream.lastMessageTime)
self.assertEqual(test_time, self.stream.lastMessageTime)
def getClusterFromMapFormat(clusterMap):
dummyMessage = Message(1, '', '', datetime.now())
dummyMessage.vector = Vector({})
dummyStream = Stream(1, dummyMessage)
cluster = StreamCluster(dummyStream)
cluster.clusterId = clusterMap['clusterId']
cluster.lastStreamAddedTime = getDateTimeObjectFromTweetTimestamp(
clusterMap['lastStreamAddedTime'])
cluster.mergedClustersList = clusterMap['mergedClustersList']
cluster.documentsInCluster = clusterMap['streams']
for k, v in clusterMap['dimensions'].iteritems():
cluster[k] = v
return cluster
def cluster(self, dataIterator):
i = 1
for data in dataIterator:
message = self.convertDataToMessageMethod(data,
**self.stream_settings)
# message = data
if DataStreamMethods.messageInOrder(message.timeStamp):
UtilityMethods.updatePhraseTextToPhraseObject(
message.vector, message.timeStamp,
self.phraseTextToPhraseObjectMap, **self.stream_settings)
if message.streamId not in self.streamIdToStreamObjectMap:
self.streamIdToStreamObjectMap[message.streamId] = Stream(
message.streamId, message)
self.getClusterAndUpdateExistingClusters(
self.streamIdToStreamObjectMap[message.streamId])
else:
previousStreamObject = Vector(
vectorInitialValues=self.streamIdToStreamObjectMap[
message.streamId])
self.streamIdToStreamObjectMap[
message.streamId].updateForMessage(
message, VectorUpdateMethods.exponentialDecay,
**self.stream_settings)
streamObject = self.streamIdToStreamObjectMap[
message.streamId]
distance = Vector.euclideanDistance(
streamObject, previousStreamObject)
if distance > 10:
# print i, len(self.clusters), distance
self.getClusterAndUpdateExistingClusters(
self.streamIdToStreamObjectMap[message.streamId])
self.updateDimensionsMethod.call(
message.timeStamp,
hdStreamClusteringObject=self,
currentMessageTime=message.timeStamp)
self.clusterFilteringMethod.call(
message.timeStamp,
hdStreamClusteringObject=self,
currentMessageTime=message.timeStamp)
# self.clusterAnalysisMethod.call(message.timeStamp, hdStreamClusteringObject=self, currentMessageTime=message.timeStamp)
self.clusterAnalysisMethod.call(
time.time(),
hdStreamClusteringObject=self,
currentMessageTime=message.timeStamp,
numberOfMessages=i)
# print i, len(self.clusters)
i += 1
def rtmp_stage1_streamkey_check(key, ipaddress):
sysSettings = settings.settings.query.first()
channelRequest = Channel.Channel.query.filter_by(streamKey=key).first()
currentTime = datetime.datetime.utcnow()
if channelRequest is not None:
userQuery = Sec.User.query.filter_by(id=channelRequest.owningUser).first()
if userQuery is not None:
if userQuery.has_role('Streamer'):
if not userQuery.active:
returnMessage = {'time': str(currentTime), 'request': 'Stage1', 'success': False, 'channelLoc': channelRequest.channelLoc, 'type': None, 'ipAddress': str(ipaddress), 'message': 'Unauthorized User - User has been disabled'}
return returnMessage
existingStreamQuery = Stream.Stream.query.filter_by(linkedChannel=channelRequest.id).all()
if existingStreamQuery:
for stream in existingStreamQuery:
db.session.delete(stream)
db.session.commit()
defaultStreamName = templateFilters.normalize_date(str(currentTime))
if channelRequest.defaultStreamName != "":
defaultStreamName = channelRequest.defaultStreamName
newStream = Stream.Stream(key, defaultStreamName, int(channelRequest.id), channelRequest.topic)
db.session.add(newStream)
db.session.commit()
if sysSettings.adaptiveStreaming:
returnMessage = {'time': str(currentTime), 'request': 'Stage1', 'success': True, 'channelLoc': channelRequest.channelLoc, 'type': 'adaptive', 'ipAddress': str(ipaddress), 'message': 'Success - Passing to Stage 2 - Adaptive'}
return returnMessage
else:
returnMessage = {'time': str(currentTime), 'request': 'Stage1', 'success': True, 'channelLoc': channelRequest.channelLoc, 'type': 'standard', 'ipAddress': str(ipaddress), 'message': 'Success - Passing to Stage 2 - Standard'}
return returnMessage
else:
returnMessage = {'time': str(currentTime), 'request': 'Stage1', 'success': False, 'channelLoc': channelRequest.channelLoc, 'type': None, 'ipAddress': str(ipaddress), 'message': 'Unauthorized User - Missing Streamer Role'}
db.session.close()
return returnMessage
else:
returnMessage = {'time': str(currentTime), 'request': 'Stage1', 'success': False, 'channelLoc': channelRequest.channelLoc, 'type': None, 'ipAddress': str(ipaddress), 'message': 'Unauthorized User - No Such User'}
db.session.close()
return returnMessage
else:
returnMessage = {'time': str(currentTime), 'request': 'Stage1', 'success': False, 'channelLoc': None, 'type': None, 'ipAddress': str(ipaddress), 'message': 'Unauthorized Key'}
db.session.close()
return returnMessage
class StreamTests(unittest.TestCase):
def setUp(self):
self.m1 = Message(1, 'sdf', 'A project to cluster high-dimensional streams.', test_time-timedelta(seconds=60))
self.m1.vector=Vector({1:1.,2:3.})
self.stream = Stream(1, self.m1)
self.m2 = Message(1, 'sdf', 'A project to cluster high-dimensional streams.', test_time)
self.m2.vector=Vector({2:3.})
def test_updateForMessage_addWithoutDecay(self):
self.stream.updateForMessage(self.m2, VectorUpdateMethods.addWithoutDecay, **stream_settings)
self.assertEqual(self.stream, Vector({1:1.,2:6.}))
def test_updateForMessage_exponentialDecay(self):
self.stream.updateForMessage(self.m2, VectorUpdateMethods.exponentialDecay, **stream_settings)
self.assertEqual(self.stream, Vector({1:0.5,2:4.5}))
def test_check_lastMessageTime_is_updated(self):
self.assertEqual(test_time-timedelta(seconds=60), self.stream.lastMessageTime)
self.stream.updateForMessage(self.m2, VectorUpdateMethods.addWithoutDecay, **stream_settings)
self.assertNotEqual(test_time-timedelta(seconds=60), self.stream.lastMessageTime)
self.assertEqual(test_time, self.stream.lastMessageTime)
def setUp(self):
self.m1 = Message(1, 'sdf', 'A project to cluster high-dimensional streams.', test_time-timedelta(seconds=60))
self.m1.vector=Vector({1:1.,2:3.})
self.stream = Stream(1, self.m1)
self.m2 = Message(1, 'sdf', 'A project to cluster high-dimensional streams.', test_time)
self.m2.vector=Vector({2:3.})
def streamkey_check():
sysSettings = settings.settings.query.first()
key = request.form['name']
ipaddress = request.form['addr']
channelRequest = Channel.Channel.query.filter_by(streamKey=key).first()
currentTime = datetime.datetime.now()
if channelRequest is not None:
userQuery = Sec.User.query.filter_by(
id=channelRequest.owningUser).first()
if userQuery is not None:
if userQuery.has_role('Streamer'):
if not userQuery.active:
returnMessage = {
'time': str(currentTime),
'status': 'Unauthorized User - User has been Disabled',
'key': str(key),
'ipAddress': str(ipaddress)
}
print(returnMessage)
return abort(400)
returnMessage = {
'time': str(currentTime),
'status': 'Successful Key Auth',
'key': str(key),
'channelName': str(channelRequest.channelName),
'userName': str(channelRequest.owningUser),
'ipAddress': str(ipaddress)
}
print(returnMessage)
validAddress = system.formatSiteAddress(
sysSettings.siteAddress)
existingStreamQuery = Stream.Stream.query.filter_by(
linkedChannel=channelRequest.id).all()
if existingStreamQuery:
for stream in existingStreamQuery:
db.session.delete(stream)
db.session.commit()
defaultStreamName = templateFilters.normalize_date(
str(currentTime))
if channelRequest.defaultStreamName != "":
defaultStreamName = channelRequest.defaultStreamName
newStream = Stream.Stream(key, defaultStreamName,
int(channelRequest.id),
channelRequest.topic)
db.session.add(newStream)
db.session.commit()
if sysSettings.adaptiveStreaming:
return redirect('rtmp://' + coreNginxRTMPAddress +
'/stream-data-adapt/' +
channelRequest.channelLoc,
code=302)
else:
return redirect('rtmp://' + coreNginxRTMPAddress +
'/stream-data/' +
channelRequest.channelLoc,
code=302)
else:
returnMessage = {
'time': str(currentTime),
'status': 'Unauthorized User - Missing Streamer Role',
'key': str(key),
'ipAddress': str(ipaddress)
}
print(returnMessage)
db.session.close()
return abort(400)
else:
returnMessage = {
'time': str(currentTime),
'status': 'Unauthorized User - No Such User',
'key': str(key),
'ipAddress': str(ipaddress)
}
print(returnMessage)
db.session.close()
return abort(400)
else:
returnMessage = {
'time': str(currentTime),
'status': 'Failed Key Auth',
'key': str(key),
'ipAddress': str(ipaddress)
}
print(returnMessage)
db.session.close()
return abort(400)