def mse(modelDir, epoch, dataDirectories, seqCount=256, batchSize=16):
# Compute the mean squared error of all the test sequences.
model = util.loadModel(modelDir, epoch)
lstmmse = []
lstmrmse = []
msebatches = []
rmsebatches = []
for X, y in multiBatch(dataDirectories, seqCount=seqCount):
predictions = model.predict(X, batch_size=batchSize)
# MSE computations that the LSTM uses for it's loss function
lstmmse.append(np.mean((y - predictions) ** 2))
lstmrmse.append(np.sqrt(np.mean((y - predictions) ** 2)))
# Split data into estimations and activations
predActivations = predictions[:, :, predictions.shape[-1]/2:]
predictions = predictions[:, :, :predictions.shape[-1]/2]
yActivations = y[:, :, y.shape[-1]/2:]
y = y[:, :, :y.shape[-1]/2]
# Update data based on activations
predictions[predActivations < 0] = 0
y[yActivations < 0] = 0
msebatches.append(np.mean((y - predictions) ** 2))
rmsebatches.append(np.sqrt(msebatches[-1]))
print 'LSTM MSE:', lstmmse[-1], '| LSTM RMSE:', lstmrmse[-1], '| msebatches:', msebatches[-1], '| rmsebatches:', rmsebatches[-1]
# print np.mean(np.sum(np.mean((y - predictions) ** 2, axis=-1), axis=-1))
# print np.mean(np.mean((y - predictions) ** 2, axis=-1), axis=-1)
print 'Mean LSTM MSE', np.mean(lstmmse)
print 'Mean LSTM RMSE', np.mean(lstmrmse)
print 'Mean MSE', np.mean(msebatches)
print 'Mean RMSE', np.mean(rmsebatches)
def __init__(self, cowrieData, topics, iterations, filePath):
self.cowrieData = cowrieData
self.sessions = sessions = len(cowrieData)
self.CMDs = CMDs = sum(len(cmd) for cmd in cowrieData)
self.uniqueCMD = uniqueCMD = len(cowrieData.alphabet)
self.filePath = filePath
self.topics = topics # user input
self.iterations = iterations
self.alpha = .1 * ones(topics)
self.alphaSum = .1 * topics
self.beta = 0.01 * ones(uniqueCMD)
self.betaSum = 0.01 * uniqueCMD
self.numWordsTopic = zeros((uniqueCMD, topics), dtype=int)
self.numTopicDoc = zeros((topics, sessions), dtype=int)
self.numTopics = zeros((topics), dtype=int)
self.CMDsPerSess = CMDsPerSess = []
for session in cowrieData:
CMDsPerSess.append(zeros(len(session), dtype=int))
model = util.loadModel(self.filePath)
print "Loaded model from %s: " % self.filePath
self.validMeasure = similarityMeasure.withinTopicMeasure(
similarityMeasure.modelSimilar(model))
def __init__(self, armPositions, splines, saveDir, processId, modelDir, epoch):
# Process specific arm locations and spline trajectory
self.armPositions = armPositions
self.splines = splines
# Directory to save data
self.saveDir = saveDir
self.processId = processId
# Simulation specific
self.simulator = pysim.shortsSimulator()
# Initialize simulator
self.simulator.initialize(maxSteps=1199, enableWind=False)
# Initialize all simulation data variables
self.initialize()
# Preprocessing functions
# self.functions = [datapreprocess.gripperForce, datapreprocess.gripperTorque]
self.functions = [datapreprocess.gripperForce, datapreprocess.gripperTorque, datapreprocess.gripperVelocity]
self.labelFunction = datapreprocess.rawPressuremap
# Load learning algorithm architecture and model weights
self.model = util.loadModel(modelDir, epoch)
def __init__(self, modelDir, epoch, activations=True, featureDim=9, outputDim=300, hdVideo=False):
self.activations = activations
self.hdVideo = hdVideo
super(LearningGown, self).__init__(maxSteps=1199, enableWind=False, pressurePointCount=outputDim, showForcemap=True, dualArm=True)
# Preprocessing functions
self.functions = [datapreprocess.gripperForce, datapreprocess.gripperTorque, datapreprocess.gripperVelocity]
self.labelFunction = datapreprocess.rawPressuremap
# Load learning algorithm architecture and model weights
self.model = util.loadModel(modelDir, epoch)
# Make our LSTM stateful for fast predictions
weights = self.model.get_weights()
self.model = Sequential()
self.model.add(LSTM(50, batch_input_shape=(1, 1, featureDim), activation='tanh', return_sequences=True, stateful=True))
self.model.add(LSTM(50, activation='tanh', return_sequences=True, stateful=True))
self.model.add(LSTM(50, activation='tanh', return_sequences=True, stateful=True))
self.model.add(TimeDistributed(Dense(outputDim * 2, activation='linear')))
self.model.compile(loss='mse', optimizer='rmsprop', metrics=['accuracy'])
self.model.set_weights(weights)
# Compile and prepare model
self.y = None
self.predictions = None
self.predActivations = None
self.yActivations = None
def doctTermMatrix(self):
implementation = nonNegativeMatrix.nonNegMatrix(self.maxIterations, 'random', 0)
selectedTopics = []
for files in glob.glob('%s*_CORPUS.pkl' % self.filePath):
encode2Int = files + '_model.bin'
model = util.loadModel(encode2Int)
#print "Loaded model from %s: " % encode2Int
validMeasure = similarityMeasure.withinTopicMeasure(similarityMeasure.modelSimilar(model))
windowName = os.path.splitext(os.path.basename(files))[0]
(matrix, CMDs, sessions) = util.loadCorpusData(files)
numSessions = len(sessions)
topicMinAct = min(numSessions, self.topicMin)
topicMaxAct = min(numSessions, self.topicMax)
coherenceScore = {}
for topic in range(topicMinAct, topicMaxAct +1):
implementation.apply(matrix, self.topicMax)
partition = implementation.generatePartition()
topicLabels = []
for i in range(topic):
topicLabels.append("%s_Topic%02d" %(windowName, (i+1)))
termRankings = []
for topicIndex in range(topic):
rankedTermIndicies = implementation.rankTerms(topicIndex, self.numRankings)
termRank = [CMDs[i] for i in rankedTermIndicies]
termRankings.append(termRank)
truncatedTermRankings = rank.truncTermRankings(termRankings, self.numRankings)
coherenceScore[topic] = validMeasure.evalRankings(truncatedTermRankings)
print "Model coherence (Topic=%d) = %.4f" % (topic,coherenceScore[topic])
dirOut = self.filePath + "%s_" % (windowName)
resultsOutPath = "%s_DTM.pkl" % dirOut
print "saved to:", resultsOutPath
util.saveDtmResults(resultsOutPath, sessions, CMDs, termRankings, partition, implementation.W, implementation.H, topicLabels)
if len(coherenceScore) > 0:
sx = sorted(coherenceScore.items(), key = operator.itemgetter(1))
sx.reverse()
topTopics =[p[0] for p in sx] [0:min(3, len(sx))]
selectedTopics.append([files, topTopics[0]])
print "Top recommendations for number of dynamic topics: %s" % ",".join(map(str, topTopics) )
def dtmCollection(self):
self.randomSeed = random.randint(1,100000)
np.random.seed(self.randomSeed)
collection = topicCollect()
implementation = nonNegativeMatrix.nonNegMatrix(maxIterations = self.maxIterations, init_strategy = 'nndsvd', randomSeed = self.randomSeed)
coherenceScore = {}
for files in glob.glob('%s*_DTM.pkl' % self.filePath):
model = util.loadModel(self.modelPath)
print "Loaded model from %s: " % self.modelPath
validMeasure = similarityMeasure.withinTopicMeasure(similarityMeasure.modelSimilar(model))
windowName = os.path.splitext(os.path.basename(files))[0]
(docIDs, CMDs, termRankings, partition, W, H, windowTopicLabels) = util.loadDtmResults(files)
collection.addTopicModel(H,CMDs, windowTopicLabels)
matrix, allCMDs = collection.createMatrix()
for topic in range(self.topicMinAct, self.topicMaxAct +1):
implementation.apply(matrix, self.topicMaxAct)
partition = implementation.generatePartition()
topicLabels = []
for i in range(topic):
topicLabels.append("%s_Window%02d" %(windowName, (i+1)))
termRankings = []
for topicIndex in range(topic):
rankedTermsIndicies = implementation.rankTerms(topicIndex, self.numRankings)
termRanking = [allCMDs[i] for i in rankedTermsIndicies]
termRankings.append(termRanking)
truncatedTermRankings = rank.truncTermRankings(termRankings, self.numRankings)
coherenceScore[topic] = validMeasure.evalRankings(truncatedTermRankings)
print "Model coherence (Topic=%d) = %.4f" % (topic,coherenceScore[topic])
resultsOutPath = self.filePath + "%02d_dynamicTopic.pkl" % topic
util.saveDtmResults(resultsOutPath, collection.topicIDs, allCMDs, termRankings, partition, implementation.W, implementation.H, topicLabels)
#rank.formatTermRanks(termRankings, labels = None, numRankings = self.numRankings )
if len(coherenceScore) > 0:
sx = sorted(coherenceScore.items(), key=operator.itemgetter(1))
sx.reverse()
topTopic = [p[0] for p in sx] [0:min(3,len(sx))]
print "Top recommendations for number of dynamic topics: %s" % ",".join(map(str, topTopic))