def run(limit=None):
outDir = "./DataCache/4_EvalSweeps/"
# get where the raw data and pre-processed data are
obj, Labels = Cacher.ReadProcessedFiles(baseDir, limit=limit)
evalObj = pCheckUtil.getCheckpoint(outDir + "svm.pkl", GetEvaluation, True,
obj, Labels, SVM_Learner)
MakeEvalutionPlot(evalObj, outName=outDir + "SVM_plot.png")
def GetOrCreatedPreProcessed(BaseDirIn,
SourceName,
BaseDirOut,
Opt,
Labels=None,
ForceUpdate=False,
UseLowOnly=False):
"""
Given a source name to read, reads from the pre-processed
cache, it if exists. Otherwise, creates the data object and pre-processes
Args:
BaseDirIn: Base directory of where to look for the SourceFiles
BaseDirOut: Base directory of where to look for the saved files
SourceName: the name of the source
Opt: Pre-processing options
Labels: the labels of the data object.
ForceUpdate: if true, forces an update of the pre-processed data
UseLowOnly: see GetPreProcessed
Returns:
the Pre Processed Object
"""
outPath = BaseDirOut + SourceName + ".pkl"
return pCheckUtil.getCheckpoint(outPath,
ReadAndProcess,
ForceUpdate,
BaseDirIn,
SourceName,
BaseDirOut,
Opt,
Labels=Labels,
UseLowOnly=UseLowOnly)
def run(limit=None):
outDir = "./DataCache/4_EvalSweeps/"
# get where the raw data and pre-processed data are
obj,Labels = Cacher.ReadProcessedFiles(baseDir,limit=limit)
evalObj = pCheckUtil.getCheckpoint(outDir + "forests.pkl",
GetEvaluation,True,
obj,Labels, RandomForest_Learner)
MakeEvalutionPlot(evalObj,outName=outDir + "forests.png")
def run(limit=5):
outDir = "./DataCache/4_EvalSweeps/"
# get where the raw data and pre-processed data are
obj,Labels = Cacher.ReadProcessedFiles(baseDir,limit=limit)
evalObj = pCheckUtil.getCheckpoint(outDir + "eval_GMM.pkl",
GetEvaluation,True,
obj,Labels,GaussianMixtureLearner)
MakeEvalutionPlot(evalObj,outName=outDir + "GMMPlot.png")
def run(limit=5):
outDir = "./DataCache/4_EvalSweeps/"
# get where the raw data and pre-processed data are
obj, Labels = Cacher.ReadProcessedFiles(baseDir, limit=limit)
evalObj = pCheckUtil.getCheckpoint(outDir + "eval_GMM.pkl", GetEvaluation,
True, obj, Labels,
GaussianMixtureLearner)
MakeEvalutionPlot(evalObj, outName=outDir + "GMMPlot.png")
def ReadProcessedFileFromDirectory(BaseDir, DirectoryPath):
"""
Given a single element from the output of GetListOfCacheFilesDirectory,
returns the pre-processed data
Args:
BaseDir: the base data directory
DirectoryPath: single element of output from GetListOfProcessedFiles
Returns:
PreProcessedObject corresponding to the file
"""
filePath = BaseDir + DirectoryPath + "/" + DirectoryPath + ".pkl"
return pCheckUtil.loadFile(filePath, useNpy=False)
def ReadProcessedFileFromDirectory(BaseDir,DirectoryPath):
"""
Given a single element from the output of GetListOfCacheFilesDirectory,
returns the pre-processed data
Args:
BaseDir: the base data directory
DirectoryPath: single element of output from GetListOfProcessedFiles
Returns:
PreProcessedObject corresponding to the file
"""
filePath = BaseDir + DirectoryPath + "/" + DirectoryPath + ".pkl"
return pCheckUtil.loadFile(filePath,useNpy=False)
def run(limit=1):
"""
Runs the main algorithm
Args:
limit: the number of files we limit ourselves to
"""
# get where the raw data and pre-processed data are
dataBase = "./DataCache/"
cacheSub = dataBase + "2_ProcessedData/"
# how many pre-processed objects to use
limit = 1
# where the (cached) feature maks should go
featureCache = dataBase + "3_FeatureMask/FeatureMask.pkl"
# get the feature mask, False means dont force regeneration
matr = pCheckUtil.getCheckpoint(featureCache,
Caching.GetFeatureMask,
True,
cacheSub,
limit=limit)
# create the learner
mLearner = NeuralLearner(matr)
# get the predictions (binary array for each point)
predictIdx = mLearner.FitAndPredict()
predEval = mLearner.Evaluate(predictIdx)
print(predEval.__dict__)
# get the *actual* 'gold standard' event labels.
eventIdx = mLearner.IdxWhereEvent
toPlot = mLearner.FeatureMask.SepStd
# find where we predict an event
eventPredicted = np.where(predictIdx == 1)[0]
plt.plot(toPlot, alpha=0.3, label="Feature")
plt.plot(eventPredicted,
toPlot[eventPredicted],
'b.',
label="Predicted (Neural Network)")
plt.plot(eventIdx,
toPlot[eventIdx],
'r.',
linewidth=3.0,
label="Labelled Events")
plt.legend()
plt.show()
def GetOrCreatedPreProcessed(BaseDirIn,SourceName,BaseDirOut,Opt,Labels=None,
ForceUpdate=False,UseLowOnly=False):
"""
Given a source name to read, reads from the pre-processed
cache, it if exists. Otherwise, creates the data object and pre-processes
Args:
BaseDirIn: Base directory of where to look for the SourceFiles
BaseDirOut: Base directory of where to look for the saved files
SourceName: the name of the source
Opt: Pre-processing options
Labels: the labels of the data object.
ForceUpdate: if true, forces an update of the pre-processed data
UseLowOnly: see GetPreProcessed
Returns:
the Pre Processed Object
"""
outPath = BaseDirOut + SourceName + ".pkl"
return pCheckUtil.getCheckpoint(outPath,ReadAndProcess,ForceUpdate,
BaseDirIn,SourceName,BaseDirOut,Opt,
Labels=Labels,UseLowOnly=UseLowOnly)
def run(limit=1): """ Runs the main algorithm Args: limit: the number of files we limit ourselves to """ # get where the raw data and pre-processed data are dataBase = "./DataCache/" cacheSub = dataBase + "2_ProcessedData/" # how many pre-processed objects to use limit=3 # where the (cached) feature maks should go featureCache = dataBase + "3_FeatureMask/FeatureMask.pkl" # get the feature mask, False means dont force regeneration matr = pCheckUtil.getCheckpoint(featureCache,Caching.GetFeatureMask,True, cacheSub,limit=limit) # create the learner mLearner = LogisticLearner(matr) # get the predictions (binary array for each point) # predictIdx = mLearner.FitAndPredict() predictIdx = mLearner.FitAndPredict() predEval = mLearner.Evaluate(predictIdx) print(predEval.__dict__) # get the *actual* 'gold standard' event labels. eventIdx = mLearner.IdxWhereEvent toPlot = mLearner.FeatureMask.SepStd # find where we predict an event eventPredicted = np.where(predictIdx==1)[0] plt.plot(toPlot,alpha=0.3,label="Feature") plt.plot(eventPredicted,toPlot[eventPredicted],'b.', label="Predicted (Regression)") plt.plot(eventIdx,toPlot[eventIdx],'r.', linewidth=3.0,label="Labelled Events") plt.legend() plt.show()
def CachedLowRes(base="../../../", **kwargs):
return pCheckUtil.getCheckpoint("./lowCache.pkl", GetLowResData, False,
base, **kwargs)
def CachedLowRes(base="../../../",**kwargs):
return pCheckUtil.getCheckpoint("./lowCache.pkl",GetLowResData,False,base,
**kwargs)