def make_KKfiles_Script_supercomp(hybdatadict, SDparams,prb, detectioncrit, KKparams,supercomparams):
'''Creates the files required to run KlustaKwik'''
argSD = [hybdatadict,SDparams,prb]
if ju.is_cached(rsd.run_spikedetekt,*argSD):
print 'Yes, SD has been run \n'
hash_hyb_SD = rsd.run_spikedetekt(hybdatadict,SDparams,prb)
else:
print 'You need to run Spikedetekt before attempting to analyse results '
argTD = [hybdatadict, SDparams,prb, detectioncrit]
if ju.is_cached(ds.test_detection_algorithm,*argTD):
print 'Yes, you have run detection_statistics.test_detection_algorithm() \n'
detcrit_groundtruth = ds.test_detection_algorithm(hybdatadict, SDparams,prb, detectioncrit)
else:
print 'You need to run detection_statistics.test_detection_algorithm() \n in order to obtain a groundtruth'
KKhash = hash_utils.hash_dictionary_md5(KKparams)
baselist = [hash_hyb_SD, detcrit_groundtruth['detection_hashname'], KKhash]
basefilename = hash_utils.make_concatenated_filename(baselist)
mainbasefilelist = [hash_hyb_SD, detcrit_groundtruth['detection_hashname']]
mainbasefilename = hash_utils.make_concatenated_filename(mainbasefilelist)
DIRPATH = hybdatadict['output_path']
os.chdir(DIRPATH)
KKscriptname = basefilename
make_KKscript_supercomp(KKparams,basefilename,KKscriptname,supercomparams)
return basefilename
def analysis_ind_confKK(hybdatadict, SDparams,prb, detectioncrit, defaultKKparams, paramtochange, listparamvalues, detcrit = None):
''' Analyse results of one parameter family of KK jobs
Not very different to the fucntion above only detcrit independent MKKbasefilenames'''
outlistKK = rkk.one_param_varyKK_ind(hybdatadict, SDparams,prb, defaultKKparams, paramtochange, listparamvalues)
#outlistKK = [listbasefiles, outputdicts]
argTD = [hybdatadict, SDparams,prb, detectioncrit]
if ju.is_cached(ds.test_detection_algorithm,*argTD):
print 'Yes, you have run detection_statistics.test_detection_algorithm() \n'
detcrit_groundtruth = ds.test_detection_algorithm(hybdatadict, SDparams,prb, detectioncrit)
else:
print 'You need to run detection_statistics.test_detection_algorithm() \n in order to obtain a groundtruth'
detcritclu = detcrit_groundtruth['detected_groundtruth']
NumSpikes = detcritclu.shape[0]
cluKK = np.zeros((len(outlistKK[0]),NumSpikes))
confusion = []
for k, basefilename in enumerate(outlistKK[0]):
clufile = hybdatadict['output_path'] + basefilename + '.clu.1'
print os.path.isfile(clufile)
if os.path.isfile(clufile):
cluKK[k,:] = np.loadtxt(clufile, dtype = np.int32, skiprows =1)
else:
print '%s does not exist '%(clufile)
conf = get_confusion_matrix(cluKK[k,:],detcritclu)
print conf
confusion.append(conf)
return confusion
def pre_learn_data_grid(hybdatadict, SDparams,prb,detectioncrit,supervised_params):
'''First this function will query whether the cached function:
detection_statistics.test_detection_algorithm(hybdatadict, SDparams, detectioncrit):,
has been called already with those arguments using `joblib_utils.is_cached`,
If it has, it calls it to obtain detcrit_groundtruth.
else if the hybrid dataset does not exist, it will raise an Error
and tell you to run SpikeDetekt on the dataset.
It scales the data using scale_data()
'''
argTD = [hybdatadict, SDparams,prb, detectioncrit]
if ju.is_cached(ds.test_detection_algorithm,*argTD):
print 'Yes, you have run detection_statistics.test_detection_algorithm() \n'
detcrit_groundtruth = ds.test_detection_algorithm(hybdatadict, SDparams,prb, detectioncrit)
else:
print 'You need to run detection_statistics.test_detection_algorithm() \n in order to obtain a groundtruth'
#'detection_hashname'
argSD = [hybdatadict,SDparams,prb]
if ju.is_cached(rsd.run_spikedetekt,*argSD):
print 'Yes, SD has been run \n'
hash_hyb_SD = rsd.run_spikedetekt(hybdatadict,SDparams,prb)
else:
print 'You need to run Spikedetekt before attempting to analyse results '
DIRPATH = hybdatadict['output_path']
with Experiment(hash_hyb_SD, dir= DIRPATH, mode='r') as expt:
#Load the detcrit groundtruth
#targetpathname = '/channel_groups/0/spikes/clusters' + '/' + detcrit_groundtruth['detection_hashname']
targetpathname = detcrit_groundtruth['detection_hashname']
targetsource = expt.channel_groups[0].spikes.clusters._get_child(targetpathname)
#take the first supervised_params['numfirstspikes'] spikes only
if supervised_params['numfirstspikes'] is not None:
fets = expt.channel_groups[0].spikes.features[0:supervised_params['numfirstspikes']]
target = targetsource[0:supervised_params['numfirstspikes']]
else:
fets = expt.channel_groups[0].spikes.features[:]
target = targetsource[:]
print expt
print 'fets.shape = ', fets.shape
print 'target.shape = ', target.shape
if supervised_params['subvector'] is not None:
subsetfets = fets[:,supervised_params['subvector']]
else:
subsetfets = fets
scaled_fets = scale_data(subsetfets)
classweights = compute_grid_weights(*supervised_params['grid_params'])
#print classweights
return hash_hyb_SD,classweights,scaled_fets, target
def create_confusion_matrix_fromclu_ind(hybdatadict, SDparams, prb, detectioncrit, KKparams):
''' will create the confusion matrix, using the equivalent to a clu file
and detcrit groundtruth res and clu files, which is now contained in the kwik file
which will either be from KK or SVM and of the form:
Hash(hybdatadict)_Hash(sdparams)_Hash(detectioncrit)_KK_Hash(kkparams).kwik
Hash(hybdatadict)_Hash(sdparams)_Hash(detectioncrit)_SVM_Hash(svmparams).kwik'''
argSD = [hybdatadict,SDparams,prb]
if ju.is_cached(rsd.run_spikedetekt,*argSD):
print 'Yes, SD has been run \n'
hash_hyb_SD = rsd.run_spikedetekt(hybdatadict,SDparams,prb)
else:
print 'You need to run Spikedetekt before attempting to analyse results '
argTD = [hybdatadict, SDparams,prb, detectioncrit]
if ju.is_cached(ds.test_detection_algorithm,*argTD):
print 'Yes, you have run detection_statistics.test_detection_algorithm() \n'
detcrit = ds.test_detection_algorithm(hybdatadict, SDparams,prb, detectioncrit)
else:
print 'You need to run detection_statistics.test_detection_algorithm() \n in order to obtain a groundtruth'
#argKK = [hybdatadict, SDparams, prb, detectioncrit, KKparams]
#print 'What the bloody hell is going on?'
#if ju.is_cached(rkk.make_KKfiles_Script,*argKK):
# print 'Yes, you have created the scripts for running KK, which you have hopefully run!'
# basefilename = rkk.make_KKfiles_Script(hybdatadict, SDparams, prb, detectioncrit, KKparams)
#else:
# print 'You need to run KK to generate a clu file '
#print 'Did you even get here?'
basefilename = rkk.make_KKfiles_Script_detindep_full(hybdatadict, SDparams, prb, KKparams)
DIRPATH = hybdatadict['output_path']
KKclufile = DIRPATH+ basefilename + '.clu.1'
KKclusters = np.loadtxt(KKclufile,dtype=np.int32,skiprows=1)
conf = get_confusion_matrix(KKclusters, detcrit['detected_groundtruth'])
return detcrit, KKclusters,conf
def make_KKfiles_viewer(hybdatadict, SDparams,prb, detectioncrit, KKparams):
argSD = [hybdatadict,SDparams,prb]
if ju.is_cached(rsd.run_spikedetekt,*argSD):
print 'Yes, SD has been run \n'
hash_hyb_SD = rsd.run_spikedetekt(hybdatadict,SDparams,prb)
else:
print 'You need to run Spikedetekt before attempting to analyse results '
argTD = [hybdatadict, SDparams,prb, detectioncrit]
if ju.is_cached(ds.test_detection_algorithm,*argTD):
print 'Yes, you have run detection_statistics.test_detection_algorithm() \n'
detcrit_groundtruth = ds.test_detection_algorithm(hybdatadict, SDparams,prb, detectioncrit)
else:
print 'You need to run detection_statistics.test_detection_algorithm() \n in order to obtain a groundtruth'
argKKfile = [hybdatadict, SDparams,prb, detectioncrit, KKparams]
if ju.is_cached(make_KKfiles_Script,*argKKfile):
print 'Yes, make_KKfiles_Script has been run \n'
else:
print 'Need to run make_KKfiles_Script first, running now '
basefilename = make_KKfiles_Script(hybdatadict, SDparams,prb, detectioncrit, KKparams)
mainbasefilelist = [hash_hyb_SD, detcrit_groundtruth['detection_hashname']]
mainbasefilename = hash_utils.make_concatenated_filename(mainbasefilelist)
DIRPATH = hybdatadict['output_path']
os.chdir(DIRPATH)
with Experiment(hash_hyb_SD, dir= DIRPATH, mode='r') as expt:
if KKparams['numspikesKK'] is not None:
#spk = expt.channel_groups[0].spikes.waveforms_filtered[0:KKparams['numspikesKK'],:,:]
res = expt.channel_groups[0].spikes.time_samples[0:KKparams['numspikesKK']]
#fets = expt.channel_groups[0].spikes.features[0:KKparams['numspikesKK']]
#fmasks = expt.channel_groups[0].spikes.features_masks[0:KKparams['numspikesKK'],:,1]
# masks = expt.channel_groups[0].spikes.masks[0:KKparams['numspikesKK']]
else:
#spk = expt.channel_groups[0].spikes.waveforms_filtered[:,:,:]
res = expt.channel_groups[0].spikes.time_samples[:]
#fets = expt.channel_groups[0].spikes.features[:]
#fmasks = expt.channel_groups[0].spikes.features_masks[:,:,1]
#print fmasks[3,:]
#masks = expt.channel_groups[0].spikes.masks[:]
mainresfile = DIRPATH + mainbasefilename + '.res.1'
mainspkfile = DIRPATH + mainbasefilename + '.spk.1'
detcritclufilename = DIRPATH + mainbasefilename + '.detcrit.clu.1'
trivialclufilename = DIRPATH + mainbasefilename + '.clu.1'
write_res(res,mainresfile)
write_trivial_clu(res,trivialclufilename)
# write_spk_buffered(exptable,filepath, indices,
# buffersize=512)
write_spk_buffered(expt.channel_groups[0].spikes.waveforms_filtered,
mainspkfile,
np.arange(len(res)))
write_clu(detcrit_groundtruth['detected_groundtruth'], detcritclufilename)
#s_total = SDparams['extract_s_before']+SDparams['extract_s_after']
#write_xml(prb,
# n_ch = SDparams['nchannels'],
# n_samp = SDparams['S_TOTAL'],
# n_feat = s_total,
# sample_rate = SDparams['sample_rate'],
# filepath = basename+'.xml')
mainxmlfile = hybdatadict['donor_path'] + hybdatadict['donor']+'_afterprocessing.xml'
#os.system('ln -s %s %s.clu.1 ' %(trivialclufilename,basefilename))
os.system('ln -s %s %s.spk.1 ' %(mainspkfile,basefilename))
os.system('ln -s %s %s.res.1 ' %(mainresfile,basefilename))
os.system('cp %s %s.xml ' %(mainxmlfile,basefilename))
return basefilename
def make_KKfiles_Script(hybdatadict, SDparams,prb, detectioncrit, KKparams):
'''Creates the files required to run KlustaKwik'''
argSD = [hybdatadict,SDparams,prb]
if ju.is_cached(rsd.run_spikedetekt,*argSD):
print 'Yes, SD has been run \n'
hash_hyb_SD = rsd.run_spikedetekt(hybdatadict,SDparams,prb)
else:
print 'You need to run Spikedetekt before attempting to analyse results '
argTD = [hybdatadict, SDparams,prb, detectioncrit]
if ju.is_cached(ds.test_detection_algorithm,*argTD):
print 'Yes, you have run detection_statistics.test_detection_algorithm() \n'
detcrit_groundtruth = ds.test_detection_algorithm(hybdatadict, SDparams,prb, detectioncrit)
else:
print 'You need to run detection_statistics.test_detection_algorithm() \n in order to obtain a groundtruth'
KKhash = hash_utils.hash_dictionary_md5(KKparams)
baselist = [hash_hyb_SD, detcrit_groundtruth['detection_hashname'], KKhash]
basefilename = hash_utils.make_concatenated_filename(baselist)
mainbasefilelist = [hash_hyb_SD, detcrit_groundtruth['detection_hashname']]
mainbasefilename = hash_utils.make_concatenated_filename(mainbasefilelist)
DIRPATH = hybdatadict['output_path']
os.chdir(DIRPATH)
with Experiment(hash_hyb_SD, dir= DIRPATH, mode='r') as expt:
if KKparams['numspikesKK'] is not None:
fets = expt.channel_groups[0].spikes.features[0:KKparams['numspikesKK']]
fmasks = expt.channel_groups[0].spikes.features_masks[0:KKparams['numspikesKK'],:,1]
masks = expt.channel_groups[0].spikes.masks[0:KKparams['numspikesKK']]
else:
fets = expt.channel_groups[0].spikes.features[:]
fmasks = expt.channel_groups[0].spikes.features_masks[:,:,1]
#print fmasks[3,:]
masks = expt.channel_groups[0].spikes.masks[:]
mainfetfile = DIRPATH + mainbasefilename+'.fet.1'
mainfmaskfile = DIRPATH + mainbasefilename+'.fmask.1'
mainmaskfile = DIRPATH + mainbasefilename+'.mask.1'
if not os.path.isfile(mainfetfile):
write_fet(fets,mainfetfile )
else:
print mainfetfile, ' already exists, moving on \n '
if not os.path.isfile(mainfmaskfile):
write_mask(fmasks,mainfmaskfile,fmt='%f')
else:
print mainfmaskfile, ' already exists, moving on \n '
if not os.path.isfile(mainmaskfile):
write_mask(masks,mainmaskfile,fmt='%f')
else:
print mainmaskfile, ' already exists, moving on \n '
os.system('ln -s %s %s.fet.1 ' %(mainfetfile,basefilename))
os.system('ln -s %s %s.fmask.1 ' %(mainfmaskfile,basefilename))
os.system('ln -s %s %s.mask.1 ' %(mainmaskfile,basefilename))
KKscriptname = basefilename
make_KKscript(KKparams,basefilename,KKscriptname)
return basefilename
def make_KKfiles_Script_full(hybdatadict, SDparams,prb, detectioncrit, KKparams):
'''Creates the files required to run KlustaKwik'''
argSD = [hybdatadict,SDparams,prb]
if ju.is_cached(rsd.run_spikedetekt,*argSD):
print 'Yes, SD has been run \n'
hash_hyb_SD = rsd.run_spikedetekt(hybdatadict,SDparams,prb)
else:
print 'You need to run Spikedetekt before attempting to analyse results '
argTD = [hybdatadict, SDparams,prb, detectioncrit]
if ju.is_cached(ds.test_detection_algorithm,*argTD):
print 'Yes, you have run detection_statistics.test_detection_algorithm() \n'
detcrit_groundtruth = ds.test_detection_algorithm(hybdatadict, SDparams,prb, detectioncrit)
else:
print 'You need to run detection_statistics.test_detection_algorithm() \n in order to obtain a groundtruth'
KKhash = hash_utils.hash_dictionary_md5(KKparams)
baselist = [hash_hyb_SD, detcrit_groundtruth['detection_hashname'], KKhash]
basefilename = hash_utils.make_concatenated_filename(baselist)
mainbasefilelist = [hash_hyb_SD, detcrit_groundtruth['detection_hashname']]
mainbasefilename = hash_utils.make_concatenated_filename(mainbasefilelist)
DIRPATH = hybdatadict['output_path']
os.chdir(DIRPATH)
with Experiment(hash_hyb_SD, dir= DIRPATH, mode='r') as expt:
if KKparams['numspikesKK'] is not None:
feats = expt.channel_groups[0].spikes.features[0:KKparams['numspikesKK']]
prefmasks = expt.channel_groups[0].spikes.features_masks[0:KKparams['numspikesKK'],:,1]
premasks = expt.channel_groups[0].spikes.masks[0:KKparams['numspikesKK']]
res = expt.channel_groups[0].spikes.time_samples[0:KKparams['numspikesKK']]
else:
feats = expt.channel_groups[0].spikes.features[:]
prefmasks = expt.channel_groups[0].spikes.features_masks[:,:,1]
#print fmasks[3,:]
premasks = expt.channel_groups[0].spikes.masks[:]
res = expt.channel_groups[0].spikes.time_samples[:]
mainresfile = DIRPATH + mainbasefilename + '.res.1'
mainspkfile = DIRPATH + mainbasefilename + '.spk.1'
detcritclufilename = DIRPATH + mainbasefilename + '.detcrit.clu.1'
trivialclufilename = DIRPATH + mainbasefilename + '.clu.1'
#arg_spkresdetclu = [expt,res,mainresfile, mainspkfile, detcritclufilename, trivialclufilename]
#if ju.is_cached(make_spkresdetclu_files,*arg_spkresdetclu):
if os.path.isfile(mainspkfile):
print 'miscellaneous files probably already exist, moving on, saving time'
else:
make_spkresdetclu_files(expt,res,mainresfile, mainspkfile, detcritclufilename, trivialclufilename)
#write_res(res,mainresfile)
#write_trivial_clu(res,trivialclufilename)
#write_spk_buffered(expt.channel_groups[0].spikes.waveforms_filtered,
# mainspkfile,
# np.arange(len(res)))
#write_clu(detcrit_groundtruth['detected_groundtruth'], detcritclufilename)
times = np.expand_dims(res, axis =1)
masktimezeros = np.zeros_like(times)
fets = np.concatenate((feats, times),axis = 1)
fmasks = np.concatenate((prefmasks, masktimezeros),axis = 1)
masks = np.concatenate((premasks, masktimezeros),axis = 1)
mainfetfile = DIRPATH + mainbasefilename+'.fet.1'
mainfmaskfile = DIRPATH + mainbasefilename+'.fmask.1'
mainmaskfile = DIRPATH + mainbasefilename+'.mask.1'
#print fets
#embed()
if not os.path.isfile(mainfetfile):
write_fet(fets,mainfetfile )
else:
print mainfetfile, ' already exists, moving on \n '
if not os.path.isfile(mainfmaskfile):
write_mask(fmasks,mainfmaskfile,fmt='%f')
else:
print mainfmaskfile, ' already exists, moving on \n '
if not os.path.isfile(mainmaskfile):
write_mask(masks,mainmaskfile,fmt='%f')
else:
print mainmaskfile, ' already exists, moving on \n '
mainxmlfile = hybdatadict['donor_path'] + hybdatadict['donor']+'_afterprocessing.xml'
os.system('ln -s %s %s.fet.1 ' %(mainfetfile,basefilename))
os.system('ln -s %s %s.fmask.1 ' %(mainfmaskfile,basefilename))
os.system('ln -s %s %s.mask.1 ' %(mainmaskfile,basefilename))
os.system('ln -s %s %s.trivial.clu.1 ' %(trivialclufilename,basefilename))
os.system('ln -s %s %s.spk.1 ' %(mainspkfile,basefilename))
os.system('ln -s %s %s.res.1 ' %(mainresfile,basefilename))
os.system('cp %s %s.xml ' %(mainxmlfile,mainbasefilename))
os.system('cp %s %s.xml ' %(mainxmlfile,basefilename))
KKscriptname = basefilename
make_KKscript(KKparams,basefilename,KKscriptname)
return basefilename