def write(self, destPath):
"""Write the combined labels to a destination directory."""
createPath(destPath)
relaxedWindows = json.dumps(self.combinedRelaxedWindows,
sort_keys=True, indent=4, separators=(',', ': '))
with open(destPath, "w") as windowWriter:
windowWriter.write(relaxedWindows)
def detectDataSet(args):
"""
Function called in each detector process that run the detector that it is
given.
@param args (tuple) Arguments to run a detector on a file and then
"""
(i, detectorInstance, detectorName, labels, outputDir, relativePath) = args
relativeDir, fileName = os.path.split(relativePath)
fileName = detectorName + "_" + fileName
outputPath = os.path.join(outputDir, detectorName, relativeDir, fileName)
createPath(outputPath)
print "%s: Beginning detection with %s for %s" % \
(i, detectorName, relativePath)
detectorInstance.initialize()
results = detectorInstance.run()
# label=1 for relaxed windows, 0 otherwise
results["label"] = labels
results.to_csv(outputPath, index=False)
print "%s: Completed processing %s records at %s" % \
(i, len(results.index), datetime.now())
print "%s: Results have been written to %s" % (i, outputPath)
def detectDataSet(args):
"""
Function called in each detector process that run the detector that it is
given.
@param args (tuple) Arguments to run a detector on a file and then
"""
(i, detectorInstance, detectorName, labels, outputDir, relativePath) = args
relativeDir, fileName = os.path.split(relativePath)
fileName = detectorName + "_" + fileName
outputPath = os.path.join(outputDir, detectorName, relativeDir, fileName)
createPath(outputPath)
print "%s: Beginning detection with %s for %s" % \
(i, detectorName, relativePath)
detectorInstance.initialize()
results = detectorInstance.run()
results["label"] = labels
results.to_csv(outputPath, index=False, float_format="%.3f")
print "%s: Completed processing %s records at %s" % \
(i, len(results.index), datetime.now())
print "%s: Results have been written to %s" % (i, outputPath)
def write(self, labelsPath, windowsPath):
"""Write the combined labels and windows to destination directories."""
if not os.path.isdir(labelsPath):
createPath(labelsPath)
if not os.path.isdir(windowsPath):
createPath(windowsPath)
writeJSON(labelsPath, self.labelTimestamps)
writeJSON(windowsPath, self.combinedWindows)
def write(self, labelsPath, windowsPath):
"""Write the combined labels and windows to a destination directories."""
if not os.path.isdir(labelsPath):
createPath(labelsPath)
if not os.path.isdir(windowsPath):
createPath(windowsPath)
writeJSON(labelsPath, self.labelTimestamps)
writeJSON(windowsPath, self.combinedWindows)
def addDataSet(self, relativePath, dataSet):
"""Add datafile to corpus given its realtivePath within the corpus.
@param relativePath (string) Path of the new datafile relative to
the corpus directory.
@param datafile (datafile) Data set to be added to corpus.
"""
self.dataFiles[relativePath] = copy.deepcopy(dataSet)
newPath = self.srcRoot + relativePath
createPath(newPath)
self.dataFiles[relativePath].srcPath = newPath
self.dataFiles[relativePath].write()
self.numDataFiles = len(self.dataFiles)
def addDataSet(self, relativePath, dataSet):
"""Add datafile to corpus given its realtivePath within the corpus.
@param relativePath (string) Path of the new datafile relative to
the corpus directory.
@param datafile (datafile) Data set to be added to corpus.
"""
self.dataFiles[relativePath] = copy.deepcopy(dataSet)
newPath = self.srcRoot + relativePath
createPath(newPath)
self.dataFiles[relativePath].srcPath = newPath
self.dataFiles[relativePath].write()
self.numDataFiles = len(self.dataFiles)
def writeCorpus(corpusDir, corpusData):
"""
Create a corpus directory.
@param corpusDir (string) Directory to store the corpus data files.
@param corpusData (dict) Dictionary containing key value pairs of
a relative path and its corresponding data file
data (as a pandas.DataFrame).
"""
makeDirsExist(corpusDir)
for relativePath, data in corpusData.iteritems():
dataFilePath = os.path.join(corpusDir, relativePath)
createPath(dataFilePath)
data.to_csv(dataFilePath, index=False)
def writeCorpus(corpusDir, corpusData):
"""
Create a corpus directory.
@param corpusDir (string) Directory to store the corpus data files.
@param corpusData (dict) Dictionary containing key value pairs of
a relative path and its corresponding data file
data (as a pandas.DataFrame).
"""
makeDirsExist(corpusDir)
for relativePath, data in corpusData.iteritems():
dataFilePath = os.path.join(corpusDir, relativePath)
createPath(dataFilePath)
data.to_csv(dataFilePath, index=False)
def writeCorpusLabel(labelsPath, labelWindows):
"""
Create a CorpusLabel file.
@param labelsPath (string) Path to store the corpus label data.
@param labelWindows (dict) Dictionary containing key value pairs of
a relative path and its corresponding list of
windows.
"""
createPath(labelsPath)
windows = json.dumps(labelWindows,
sort_keys=True, indent=4, separators=(',', ': '))
with open(labelsPath, "w") as windowWriter:
windowWriter.write(windows)
def copy(self, newRoot=None):
"""Copy corpus to a newRoot which cannot already exist.
@param newRoot (string) Location of new directory to copy corpus
to.
"""
if newRoot[-1] != os.path.sep:
newRoot += os.path.sep
if os.path.isdir(newRoot):
print("directory already exists")
return None
else:
createPath(newRoot)
newCorpus = Corpus(newRoot)
for relativePath in self.dataFiles.keys():
newCorpus.addDataSet(relativePath, self.dataFiles[relativePath])
return newCorpus
def copy(self, newRoot=None):
"""Copy corpus to a newRoot which cannot already exist.
@param newRoot (string) Location of new directory to copy corpus
to.
"""
if newRoot[-1] != "/":
newRoot += "/"
if os.path.isdir(newRoot):
print "directory already exists"
return None
else:
createPath(newRoot)
newCorpus = Corpus(newRoot)
for relativePath in self.dataFiles.keys():
newCorpus.addDataSet(relativePath, self.dataFiles[relativePath])
return newCorpus
def write(self, destPath):
"""Write the combined labels to a destination directory."""
createPath(destPath)
relaxedWindows = json.dumps(self.combinedRelaxedWindows, indent=3)
with open(destPath, "w") as windowWriter:
windowWriter.write(relaxedWindows)
def detectDataSet(args):
"""
Function called in each detector process that run the detector that it is
given.
@param args (tuple) Arguments to run a detector on a file and then
"""
(i, detectorName, detectorInstance, detectorArgs, datasetName, plotFileName) = args
outputDir = "../results/"
fileName = datasetName[:-4] + '_' + str(datetime.now().strftime("%Y-%m-%d_%H-%M-%S")) + '.csv'
outputPath = os.path.join(outputDir, detectorName, fileName)
createPath(outputPath)
print ("%s: Beginning detection with %s on %s" % (i, detectorName, datasetName))
detectorInstance.initialize(**detectorArgs)
start = time.time()
results = detectorInstance.run()
end = time.time()
results.to_csv(outputPath, index=False)
print ("%s: Completed processing %s records at %s" % \
(i, len(results.index), datetime.now()))
print ("%s: Results have been written to %s" % (i, outputPath))
try:
if not os.path.exists('../results/plotFiles/'):
os.makedirs('../results/plotFiles/')
if not os.path.exists('../results/timings/'):
os.makedirs('../results/timings/')
except OSError:
print ('Error: Creating score directory.')
f=open('../results/plotFiles/'+plotFileName, "a+")
f.write('{},{},{},{}\n'.format(i,detectorName,datasetName,outputPath))
f.close()
f=open('../results/timings/'+plotFileName, "a+")
f.write('{},{},{},{},{}\n'.format(i,detectorName,datasetName,outputPath,end-start))
f.close()
print ('ResultUrl has been written to %s' % (plotFileName))
del detectorInstance