def make_KKfiles_Script_supercomp(hybdatadict, SDparams,prb, detectioncrit, KKparams,supercomparams): '''Creates the files required to run KlustaKwik''' argSD = [hybdatadict,SDparams,prb] if ju.is_cached(rsd.run_spikedetekt,*argSD): print 'Yes, SD has been run \n' hash_hyb_SD = rsd.run_spikedetekt(hybdatadict,SDparams,prb) else: print 'You need to run Spikedetekt before attempting to analyse results ' argTD = [hybdatadict, SDparams,prb, detectioncrit] if ju.is_cached(ds.test_detection_algorithm,*argTD): print 'Yes, you have run detection_statistics.test_detection_algorithm() \n' detcrit_groundtruth = ds.test_detection_algorithm(hybdatadict, SDparams,prb, detectioncrit) else: print 'You need to run detection_statistics.test_detection_algorithm() \n in order to obtain a groundtruth' KKhash = hash_utils.hash_dictionary_md5(KKparams) baselist = [hash_hyb_SD, detcrit_groundtruth['detection_hashname'], KKhash] basefilename = hash_utils.make_concatenated_filename(baselist) mainbasefilelist = [hash_hyb_SD, detcrit_groundtruth['detection_hashname']] mainbasefilename = hash_utils.make_concatenated_filename(mainbasefilelist) DIRPATH = hybdatadict['output_path'] os.chdir(DIRPATH) KKscriptname = basefilename make_KKscript_supercomp(KKparams,basefilename,KKscriptname,supercomparams) return basefilename
def pre_learn_data_grid(hybdatadict, SDparams,prb,detectioncrit,supervised_params): '''First this function will query whether the cached function: detection_statistics.test_detection_algorithm(hybdatadict, SDparams, detectioncrit):, has been called already with those arguments using `joblib_utils.is_cached`, If it has, it calls it to obtain detcrit_groundtruth. else if the hybrid dataset does not exist, it will raise an Error and tell you to run SpikeDetekt on the dataset. It scales the data using scale_data() ''' argTD = [hybdatadict, SDparams,prb, detectioncrit] if ju.is_cached(ds.test_detection_algorithm,*argTD): print 'Yes, you have run detection_statistics.test_detection_algorithm() \n' detcrit_groundtruth = ds.test_detection_algorithm(hybdatadict, SDparams,prb, detectioncrit) else: print 'You need to run detection_statistics.test_detection_algorithm() \n in order to obtain a groundtruth' #'detection_hashname' argSD = [hybdatadict,SDparams,prb] if ju.is_cached(rsd.run_spikedetekt,*argSD): print 'Yes, SD has been run \n' hash_hyb_SD = rsd.run_spikedetekt(hybdatadict,SDparams,prb) else: print 'You need to run Spikedetekt before attempting to analyse results ' DIRPATH = hybdatadict['output_path'] with Experiment(hash_hyb_SD, dir= DIRPATH, mode='r') as expt: #Load the detcrit groundtruth #targetpathname = '/channel_groups/0/spikes/clusters' + '/' + detcrit_groundtruth['detection_hashname'] targetpathname = detcrit_groundtruth['detection_hashname'] targetsource = expt.channel_groups[0].spikes.clusters._get_child(targetpathname) #take the first supervised_params['numfirstspikes'] spikes only if supervised_params['numfirstspikes'] is not None: fets = expt.channel_groups[0].spikes.features[0:supervised_params['numfirstspikes']] target = targetsource[0:supervised_params['numfirstspikes']] else: fets = expt.channel_groups[0].spikes.features[:] target = targetsource[:] print expt print 'fets.shape = ', fets.shape print 'target.shape = ', target.shape if supervised_params['subvector'] is not None: subsetfets = fets[:,supervised_params['subvector']] else: subsetfets = fets scaled_fets = scale_data(subsetfets) classweights = compute_grid_weights(*supervised_params['grid_params']) #print classweights return hash_hyb_SD,classweights,scaled_fets, target
def analysis_ind_confKK(hybdatadict, SDparams,prb, detectioncrit, defaultKKparams, paramtochange, listparamvalues, detcrit = None): ''' Analyse results of one parameter family of KK jobs Not very different to the fucntion above only detcrit independent MKKbasefilenames''' outlistKK = rkk.one_param_varyKK_ind(hybdatadict, SDparams,prb, defaultKKparams, paramtochange, listparamvalues) #outlistKK = [listbasefiles, outputdicts] argTD = [hybdatadict, SDparams,prb, detectioncrit] if ju.is_cached(ds.test_detection_algorithm,*argTD): print 'Yes, you have run detection_statistics.test_detection_algorithm() \n' detcrit_groundtruth = ds.test_detection_algorithm(hybdatadict, SDparams,prb, detectioncrit) else: print 'You need to run detection_statistics.test_detection_algorithm() \n in order to obtain a groundtruth' detcritclu = detcrit_groundtruth['detected_groundtruth'] NumSpikes = detcritclu.shape[0] cluKK = np.zeros((len(outlistKK[0]),NumSpikes)) confusion = [] for k, basefilename in enumerate(outlistKK[0]): clufile = hybdatadict['output_path'] + basefilename + '.clu.1' print os.path.isfile(clufile) if os.path.isfile(clufile): cluKK[k,:] = np.loadtxt(clufile, dtype = np.int32, skiprows =1) else: print '%s does not exist '%(clufile) conf = get_confusion_matrix(cluKK[k,:],detcritclu) print conf confusion.append(conf) return confusion
def create_confusion_matrix_fromclu_ind(hybdatadict, SDparams, prb, detectioncrit, KKparams): ''' will create the confusion matrix, using the equivalent to a clu file and detcrit groundtruth res and clu files, which is now contained in the kwik file which will either be from KK or SVM and of the form: Hash(hybdatadict)_Hash(sdparams)_Hash(detectioncrit)_KK_Hash(kkparams).kwik Hash(hybdatadict)_Hash(sdparams)_Hash(detectioncrit)_SVM_Hash(svmparams).kwik''' argSD = [hybdatadict,SDparams,prb] if ju.is_cached(rsd.run_spikedetekt,*argSD): print 'Yes, SD has been run \n' hash_hyb_SD = rsd.run_spikedetekt(hybdatadict,SDparams,prb) else: print 'You need to run Spikedetekt before attempting to analyse results ' argTD = [hybdatadict, SDparams,prb, detectioncrit] if ju.is_cached(ds.test_detection_algorithm,*argTD): print 'Yes, you have run detection_statistics.test_detection_algorithm() \n' detcrit = ds.test_detection_algorithm(hybdatadict, SDparams,prb, detectioncrit) else: print 'You need to run detection_statistics.test_detection_algorithm() \n in order to obtain a groundtruth' #argKK = [hybdatadict, SDparams, prb, detectioncrit, KKparams] #print 'What the bloody hell is going on?' #if ju.is_cached(rkk.make_KKfiles_Script,*argKK): # print 'Yes, you have created the scripts for running KK, which you have hopefully run!' # basefilename = rkk.make_KKfiles_Script(hybdatadict, SDparams, prb, detectioncrit, KKparams) #else: # print 'You need to run KK to generate a clu file ' #print 'Did you even get here?' basefilename = rkk.make_KKfiles_Script_detindep_full(hybdatadict, SDparams, prb, KKparams) DIRPATH = hybdatadict['output_path'] KKclufile = DIRPATH+ basefilename + '.clu.1' KKclusters = np.loadtxt(KKclufile,dtype=np.int32,skiprows=1) conf = get_confusion_matrix(KKclusters, detcrit['detected_groundtruth']) return detcrit, KKclusters,conf
def make_KKscript_supercomp(KKparams, filebase,scriptname,supercomparams): '''Create bash script on Legion required to run KlustaKwik supercomparams = {'time':'36:00:00','mem': '2G', 'tmpfs':'10G'} ''' argKKsc = [KKparams, filebase,scriptname] if ju.is_cached(make_KKscript,*argKKsc): print 'Yes, you have made the scripts for the local machine \n' #scriptstring = make_KKscript(KKparams, filebase,scriptname) else: print 'You need to run make_KKscript ' keylist = KKparams['keylist'] #keylist = ['MaskStarts','MaxPossibleClusters','FullStepEvery','MaxIter','RandomSeed', # 'Debug','SplitFirst','SplitEvery','PenaltyK','PenaltyKLogN','Subset', # 'PriorPoint','SaveSorted','SaveCovarianceMeans','UseMaskedInitialConditions', # 'AssignToFirstClosestMask','UseDistributional'] #KKlocation = '/martinottihome/skadir/GIT_masters/klustakwik/MaskedKlustaKwik' supercompstuff = '''#!/bin/bash -l #$ -S /bin/bash #$ -l h_rt=%s #$ -l mem=%s #$ -l tmpfs=%s #$ -N %s_supercomp #$ -P maskedklustakwik #$ -wd /home/smgxsk1/Scratch/ cd $TMPDIR '''%(supercomparams['time'],supercomparams['mem'],supercomparams['tmpfs'],scriptname) KKsupercomplocation = supercompstuff + '/home/smgxsk1/MKK_versions/klustakwik/MaskedKlustaKwik' scriptstring = KKsupercomplocation + ' /home/smgxsk1/Scratch/'+ filebase + ' 1 ' for KKey in keylist: #print '-'+KKey +' '+ str(KKparams[KKey]) scriptstring = scriptstring + ' -'+ KKey +' '+ str(KKparams[KKey]) print scriptstring scriptfile = open('%s_supercomp.sh' %(scriptname),'w') scriptfile.write(scriptstring) scriptfile.close() outputdir = ' /chandelierhome/skadir/hybrid_analysis/mariano/' #changeperms='chmod 777 %s.sh' %(scriptname) sendout = 'scp -r'+ outputdir + scriptname + '_supercomp.sh' + outputdir +scriptname + '.fet.1' + outputdir + scriptname + '.fmask.1 '+ '[email protected]:/home/smgxsk1/Scratch/' os.system(sendout) return scriptstring
def make_KKfiles_Script_detindep_supercomp(hybdatadict, SDparams,prb, KKparams,supercomparams): '''Creates the files required to run KlustaKwik''' argSD = [hybdatadict,SDparams,prb] if ju.is_cached(rsd.run_spikedetekt,*argSD): print 'Yes, SD has been run \n' hash_hyb_SD = rsd.run_spikedetekt(hybdatadict,SDparams,prb) else: print 'You need to run Spikedetekt before attempting to analyse results ' KKhash = hash_utils.hash_dictionary_md5(KKparams) baselist = [hash_hyb_SD, KKhash] KKbasefilename = hash_utils.make_concatenated_filename(baselist) mainbasefilename = hash_hyb_SD DIRPATH = hybdatadict['output_path'] os.chdir(DIRPATH) KKscriptname = KKbasefilename make_KKscript_supercomp(KKparams,KKbasefilename,KKscriptname,supercomparams) return KKbasefilename
def learn_data_grid_general(hybdatadict, SDparams,prb,detectioncrit,supervised_params,addtokwik): ''' If addtokwik == True, then the clusterings are also stored in the .kwik file calls learn_data() for various values of the grids and also the function compute_errors() Writes output as clusterings labelled by Hash(svmparams) of the grid in Hash(hybdatadict)_Hash(sdparams)_Hash(detectioncrit)_Hash(supervised_params).kwik using write_kwik(hybdatadict,sdparams,detectioncrit,svmparams,confusion_test,confusion_train) the new .kwik format can store multiple clusterings. supervised_params consists of the following quantities: supervised_params = {'numfirstspikes': 200000,'kernel': 'rbf','grid_C': [1,100000,0.00001], 'grid_weights': listofweights ,gammagrid : [1e-5, 0.001, 0.1, 1, 10, 1000, 100000], cross_param : 2, PCAS : 3, subvector: None} ''' #---------------------------------------------------------- argPLDG = [hybdatadict, SDparams,prb,detectioncrit,supervised_params] if ju.is_cached(pre_learn_data_grid,*argPLDG): print 'Yes, pre_learn_data_grid has been run \n' else: print 'Running pre_learn_data_grid(hybdatadict, SDparams,prb,detectioncrit,supervised_params), \n you have not run it yet' hash_hyb_SD,classweights,scaled_fets, target = pre_learn_data_grid(hybdatadict, SDparams,prb,detectioncrit,supervised_params) DIRPATH = hybdatadict['output_path'] number_of_weights = len(classweights) numspikes = scaled_fets.shape[0] cross_valid = do_cross_validation_shuffle(numspikes,supervised_params['cross_param']) #print cross_valid #do_supervised(supervised_params, #'grid_C': [1,100000,0.00001], number_cvalues = 3 number_cvalues = len(supervised_params['grid_C']) #number_support_vectors = {} weights_clu_test = np.zeros((number_cvalues,number_of_weights,numspikes,2),dtype=np.int32) weights_clu_train = np.zeros((number_cvalues,number_of_weights, numspikes,2),dtype=np.int32) cludict= {(0,0):1, (0,1):2, (1,0):3, (1,1):4} # (prediction, groundtruth) #(0,0) TN, (0,1) FN ,(1,0) FP ,(1,1) TP testclu = np.zeros((number_cvalues,number_of_weights,numspikes),dtype=np.int32) trainclu = np.zeros((number_cvalues,number_of_weights,numspikes),dtype=np.int32) for c, Cval in enumerate(supervised_params['grid_C']): preds = {} preds_train = {} ##Defined to avoid: TypeError: unhashable type: 'numpy.ndarray', something about dictionaries #testclu_pre = np.zeros((number_of_weights,numspikes),dtype=np.int32) #trainclu_pre = np.zeros((number_of_weights,numspikes),dtype=np.int32) for i, (weights) in enumerate(classweights): for j, (train, test) in enumerate(cross_valid): if supervised_params['kernel'] == 'poly': preds[i,j], preds_train[i,j]= do_supervised_learning(test, train,Cval, supervised_params['kernel'], scaled_fets, target,classweights[i]) else:#radial kernel, only allow a single gamma value at a time preds[i,j], preds_train[i,j]= do_supervised_learning_radial(test, train,Cval, supervised_params['kernel'],supervised_params['gamma'], scaled_fets, target,classweights[i]) print 'Computed ', classweights[i] #Used later to make equivalent to 4 seasons clu file weights_clu_test[c,i,test,0] = preds[i,j] weights_clu_test[c,i,test,1] = target[test] #Used later to make equivalent to 4 seasons clu file but for the training set weights_clu_train[c,i,train,0] = preds_train[i,j] weights_clu_train[c,i,train,1] = target[train] #Make 4 seasons clu file equivalent for k in np.arange(numspikes): testclu[c,i,k] = cludict[tuple(weights_clu_test[c,i,k,:])] trainclu[c,i,k] = cludict[tuple(weights_clu_train[c,i,k,:])] # supervisedinputdict = {'test':test, 'train':train, 'Cval': Cval, 'kernel': supervised_params['kernel'], 'scaled_fets':scaled_fets, 'target', target, 'classweights': classweigths #Add clusterings to .kwik file supervisedparamshash = None #if addtokwik = False otherwise crashes if addtokwik: kwikfiles = open_files(hash_hyb_SD,dir=DIRPATH, mode='a') supervisedparamshash = hash_utils.hash_dictionary_md5(supervised_params) supervisedhashname = supervisedparamshash + '_' + repr(c) + '_' + repr(i) add_clustering(kwikfiles,name = supervisedhashname + 'test', spike_clusters=testclu[c,i,:] ) add_clustering(kwikfiles,name = supervisedhashname + 'train', spike_clusters=trainclu[c,i,:] ) close_files(kwikfiles) #print 'testclu[',c,',',i,',',k,']=',testclu[c,i,k] # for c, Cval in enumerate(supervised_params['grid_C']): # kwikfilename = DIRPATH + hash_hyb_SD + '.kwik' # supervisedhashname = hash_utils.hash_dictionary_md5(detectioncrit) # add_clustering_kwik(kwikfilename, detectedgroundtruth, detectionhashname) ####Train and test look like this for 2-fold cross validation and 200 spikes # j = 0 train = [100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 # 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 # 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 # 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 # 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 # 190 191 192 193 194 195 196 197 198 199] # test = [ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 # 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 # 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 # 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99] # j = 1 train = [ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 # 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 # 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 # 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99] # test = [100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 # 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 # 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 # 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 # 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 # 190 191 192 193 194 195 196 197 198 199] return supervisedparamshash, classweights, testclu, trainclu
def test_detection_algorithm(hybdatadict, SDparams,prb, detectioncrit): ''' It will query whether the cached function: hybridata_creation_lib.create_hybrid_kwdfile(hybdatadict), has been called already with those arguments using `joblib_utils.is_cached`, If it has, it calls it to obtain creation_groundtruth. else if the hybrid dataset does not exist, it will raise an Error. creation_groundtruth consists of the equivalent to the old: GroundtruthResfile,GroundtruthClufile,... (i.e. the times and the cluster labels for the added hybrid spikes. detection criteria include: allowed_discrepancy,CSthreshold This function will call SpikeSimilarityMeasure(a,b) and output the file: Hash(hybdatadict)_Hash(sdparams)_Hash(detectioncrit).kwik It will return detcrit_groundtruth, the groundtruth relative to the criteria, detectioncrit. This will be an ordered dictionary so that hashing can work!''' if ju.is_cached(hcl.create_hybrid_kwdfile,hybdatadict): print 'Yes, this hybrid dataset exists, I shall now check if you have run SD \n' meanwaveform,kwdoutputname, creation_groundtruth, amplitude = hcl.create_hybrid_kwdfile(hybdatadict) #Take the means of the binary donor masks of the donor cluster binmeanmask = hcl.make_average_datamask_from_mean(hybdatadict, fmask= False) argSD = [hybdatadict,SDparams,prb] if ju.is_cached(rsd.run_spikedetekt,*argSD): print 'Yes, SD has been run \n' hash_hyb_SD = rsd.run_spikedetekt(hybdatadict,SDparams,prb) else: print 'You need to run Spikedetekt before attempting to analyse results ' DIRPATH = hybdatadict['output_path'] with Experiment(hash_hyb_SD, dir= DIRPATH, mode='a') as expt: res_int= expt.channel_groups[0].spikes.time_samples res_frac = expt.channel_groups[0].spikes.time_fractional res_int_arr = res_int[:] res_frac_arr = res_frac[:] detected_times = res_int_arr + res_frac_arr #Masks fmask = expt.channel_groups[0].spikes.features_masks #Spikes within time window existencewin = np.zeros_like(creation_groundtruth) #Mean binary mask for hybrid cluster if 'manual_meanmask' in detectioncrit.keys(): binmeanmask = detectioncrit['manual_meanmask'] else: binmeanmask = hcl.make_average_datamask_from_mean(hybdatadict, fmask= False) indices_in_timewindow = hash_utils.order_dictionary({}) #indices_in_timewindow = {0 (this is the 1st hybrid spike): (array([0, 1, 3]),), #1: (array([89, 90]),), #2: (array([154, 156, 157]),), #3: (array([191]),), #4: (array([259, 260, 261]),), num_spikes_in_timewindow = hash_utils.order_dictionary({}) CauchySchwarz = hash_utils.order_dictionary({}) detected = hash_utils.order_dictionary({}) NumHybSpikes = creation_groundtruth.shape[0] trivialmainclustering = np.zeros_like(detected_times) detectedgroundtruth = np.zeros_like(detected_times) print detectedgroundtruth.shape for k in np.arange(NumHybSpikes): list_of_differences = np.zeros((detected_times.shape[0])) list_of_differences[:]= detected_times[:] - creation_groundtruth[k] indices_in_timewindow[k] = np.nonzero(np.absolute(list_of_differences)<=detectioncrit['allowed_discrepancy']) num_spikes_in_timewindow[k] = indices_in_timewindow[k][0].shape[0] for j in np.arange(num_spikes_in_timewindow[k]): CauchySchwarz[k,j] = SpikeSimilarityMeasure(fmask[indices_in_timewindow[k][0][j],:,1],binmeanmask[0:3*hybdatadict['numchannels']]) if CauchySchwarz[k,j] > detectioncrit['CSthreshold']: detected[k,j] = 1 else: detected[k,j] = 0 detectedgroundtruth[indices_in_timewindow[k][0][j]]= detected[k,j] #Store detectedgroundtruth in a clustering detectionhashname = hash_utils.hash_dictionary_md5(detectioncrit) #kwikfilename = DIRPATH + hash_hyb_SD #+ '.kwik' #add_clustering_kwik(kwikfilename, detectedgroundtruth, detectionhashname) kwikfiles = open_files(hash_hyb_SD,dir=DIRPATH, mode='a') add_clustering(kwikfiles,name = detectionhashname, spike_clusters=detectedgroundtruth,overwrite = True ) print 'Added a clustering called ', detectionhashname add_clustering(kwikfiles,name = 'main', spike_clusters= trivialmainclustering, overwrite = True) close_files(kwikfiles) #clusters = '/channel_groups[0]/spikes/clusters' #detectionhash = hash_dictionary_md5(detectioncrit) #expt.createEArray(clusters, detectionhash, tb.UInt32Atom(), (0,), # expectedrows=1000000) #percentage_detected = float(sum(detected.values()))/NumHybSpikes detcrit_groundtruth_pre ={'detection_hashname': detectionhashname,'binmeanmask': binmeanmask,'indices_in_timewindow':indices_in_timewindow, 'numspikes_in_timeswindow': num_spikes_in_timewindow, 'Cauchy_Schwarz':CauchySchwarz,'detected': detected,'detected_groundtruth': detectedgroundtruth} detcrit_groundtruth = hash_utils.order_dictionary(detcrit_groundtruth_pre) return detcrit_groundtruth
def make_KKfiles_viewer(hybdatadict, SDparams,prb, detectioncrit, KKparams): argSD = [hybdatadict,SDparams,prb] if ju.is_cached(rsd.run_spikedetekt,*argSD): print 'Yes, SD has been run \n' hash_hyb_SD = rsd.run_spikedetekt(hybdatadict,SDparams,prb) else: print 'You need to run Spikedetekt before attempting to analyse results ' argTD = [hybdatadict, SDparams,prb, detectioncrit] if ju.is_cached(ds.test_detection_algorithm,*argTD): print 'Yes, you have run detection_statistics.test_detection_algorithm() \n' detcrit_groundtruth = ds.test_detection_algorithm(hybdatadict, SDparams,prb, detectioncrit) else: print 'You need to run detection_statistics.test_detection_algorithm() \n in order to obtain a groundtruth' argKKfile = [hybdatadict, SDparams,prb, detectioncrit, KKparams] if ju.is_cached(make_KKfiles_Script,*argKKfile): print 'Yes, make_KKfiles_Script has been run \n' else: print 'Need to run make_KKfiles_Script first, running now ' basefilename = make_KKfiles_Script(hybdatadict, SDparams,prb, detectioncrit, KKparams) mainbasefilelist = [hash_hyb_SD, detcrit_groundtruth['detection_hashname']] mainbasefilename = hash_utils.make_concatenated_filename(mainbasefilelist) DIRPATH = hybdatadict['output_path'] os.chdir(DIRPATH) with Experiment(hash_hyb_SD, dir= DIRPATH, mode='r') as expt: if KKparams['numspikesKK'] is not None: #spk = expt.channel_groups[0].spikes.waveforms_filtered[0:KKparams['numspikesKK'],:,:] res = expt.channel_groups[0].spikes.time_samples[0:KKparams['numspikesKK']] #fets = expt.channel_groups[0].spikes.features[0:KKparams['numspikesKK']] #fmasks = expt.channel_groups[0].spikes.features_masks[0:KKparams['numspikesKK'],:,1] # masks = expt.channel_groups[0].spikes.masks[0:KKparams['numspikesKK']] else: #spk = expt.channel_groups[0].spikes.waveforms_filtered[:,:,:] res = expt.channel_groups[0].spikes.time_samples[:] #fets = expt.channel_groups[0].spikes.features[:] #fmasks = expt.channel_groups[0].spikes.features_masks[:,:,1] #print fmasks[3,:] #masks = expt.channel_groups[0].spikes.masks[:] mainresfile = DIRPATH + mainbasefilename + '.res.1' mainspkfile = DIRPATH + mainbasefilename + '.spk.1' detcritclufilename = DIRPATH + mainbasefilename + '.detcrit.clu.1' trivialclufilename = DIRPATH + mainbasefilename + '.clu.1' write_res(res,mainresfile) write_trivial_clu(res,trivialclufilename) # write_spk_buffered(exptable,filepath, indices, # buffersize=512) write_spk_buffered(expt.channel_groups[0].spikes.waveforms_filtered, mainspkfile, np.arange(len(res))) write_clu(detcrit_groundtruth['detected_groundtruth'], detcritclufilename) #s_total = SDparams['extract_s_before']+SDparams['extract_s_after'] #write_xml(prb, # n_ch = SDparams['nchannels'], # n_samp = SDparams['S_TOTAL'], # n_feat = s_total, # sample_rate = SDparams['sample_rate'], # filepath = basename+'.xml') mainxmlfile = hybdatadict['donor_path'] + hybdatadict['donor']+'_afterprocessing.xml' #os.system('ln -s %s %s.clu.1 ' %(trivialclufilename,basefilename)) os.system('ln -s %s %s.spk.1 ' %(mainspkfile,basefilename)) os.system('ln -s %s %s.res.1 ' %(mainresfile,basefilename)) os.system('cp %s %s.xml ' %(mainxmlfile,basefilename)) return basefilename
def make_KKfiles_Script_detindep_full(hybdatadict, SDparams,prb, KKparams): '''Creates the files required to run KlustaKwik''' argSD = [hybdatadict,SDparams,prb] if ju.is_cached(rsd.run_spikedetekt,*argSD): print 'Yes, SD has been run \n' hash_hyb_SD = rsd.run_spikedetekt(hybdatadict,SDparams,prb) else: print 'You need to run Spikedetekt before attempting to analyse results ' KKhash = hash_utils.hash_dictionary_md5(KKparams) baselist = [hash_hyb_SD, KKhash] KKbasefilename = hash_utils.make_concatenated_filename(baselist) mainbasefilename = hash_hyb_SD DIRPATH = hybdatadict['output_path'] os.chdir(DIRPATH) mainresfile = DIRPATH + mainbasefilename + '.res.1' mainspkfile = DIRPATH + mainbasefilename + '.spk.1' trivialclufilename = DIRPATH + mainbasefilename + '.clu.1' mainfetfile = DIRPATH + mainbasefilename+'.fet.1' mainfmaskfile = DIRPATH + mainbasefilename+'.fmask.1' mainmaskfile = DIRPATH + mainbasefilename+'.mask.1' #arg_spkresdetclu = [expt,res,mainresfile, mainspkfile, detcritclufilename, trivialclufilename] #if ju.is_cached(make_spkresdetclu_files,*arg_spkresdetclu): if os.path.isfile(mainspkfile): print 'miscellaneous files probably already exist, moving on, saving time' else: with Experiment(hash_hyb_SD, dir= DIRPATH, mode='r') as expt: if KKparams['numspikesKK'] is not None: feats = expt.channel_groups[0].spikes.features[0:KKparams['numspikesKK']] prefmasks = expt.channel_groups[0].spikes.features_masks[0:KKparams['numspikesKK'],:,1] premasks = expt.channel_groups[0].spikes.masks[0:KKparams['numspikesKK']] res = expt.channel_groups[0].spikes.time_samples[0:KKparams['numspikesKK']] else: feats = expt.channel_groups[0].spikes.features[:] prefmasks = expt.channel_groups[0].spikes.features_masks[:,:,1] #print fmasks[3,:] premasks = expt.channel_groups[0].spikes.masks[:] res = expt.channel_groups[0].spikes.time_samples[:] #arg_spkresdetclu = [expt,res,mainresfile, mainspkfile, detcritclufilename, trivialclufilename] #if ju.is_cached(make_spkresdetclu_files,*arg_spkresdetclu): #if os.path.isfile(mainspkfile): # print 'miscellaneous files probably already exist, moving on, saving time' #else: make_spkresclu_files(expt,res,mainresfile, mainspkfile, trivialclufilename) #write_res(res,mainresfile) #write_trivial_clu(res,trivialclufilename) #write_spk_buffered(expt.channel_groups[0].spikes.waveforms_filtered, # mainspkfile, # np.arange(len(res))) #write_clu(detcrit_groundtruth['detected_groundtruth'], detcritclufilename) times = np.expand_dims(res, axis =1) masktimezeros = np.zeros_like(times) fets = np.concatenate((feats, times),axis = 1) fmasks = np.concatenate((prefmasks, masktimezeros),axis = 1) masks = np.concatenate((premasks, masktimezeros),axis = 1) #print fets #embed() if not os.path.isfile(mainfetfile): write_fet(fets,mainfetfile ) else: print mainfetfile, ' already exists, moving on \n ' if not os.path.isfile(mainfmaskfile): write_mask(fmasks,mainfmaskfile,fmt='%f') else: print mainfmaskfile, ' already exists, moving on \n ' if not os.path.isfile(mainmaskfile): write_mask(masks,mainmaskfile,fmt='%f') else: print mainmaskfile, ' already exists, moving on \n ' mainxmlfile = hybdatadict['donor_path'] + hybdatadict['donor']+'_afterprocessing.xml' os.system('ln -s %s %s.fet.1 ' %(mainfetfile,KKbasefilename)) os.system('ln -s %s %s.fmask.1 ' %(mainfmaskfile,KKbasefilename)) os.system('ln -s %s %s.mask.1 ' %(mainmaskfile,KKbasefilename)) os.system('ln -s %s %s.trivial.clu.1 ' %(trivialclufilename,KKbasefilename)) os.system('ln -s %s %s.spk.1 ' %(mainspkfile,KKbasefilename)) os.system('ln -s %s %s.res.1 ' %(mainresfile,KKbasefilename)) os.system('cp %s %s.xml ' %(mainxmlfile,mainbasefilename)) os.system('cp %s %s.xml ' %(mainxmlfile,KKbasefilename)) KKscriptname = KKbasefilename make_KKscript(KKparams,KKbasefilename,KKscriptname) return KKbasefilename
def make_KKfiles_Script(hybdatadict, SDparams,prb, detectioncrit, KKparams): '''Creates the files required to run KlustaKwik''' argSD = [hybdatadict,SDparams,prb] if ju.is_cached(rsd.run_spikedetekt,*argSD): print 'Yes, SD has been run \n' hash_hyb_SD = rsd.run_spikedetekt(hybdatadict,SDparams,prb) else: print 'You need to run Spikedetekt before attempting to analyse results ' argTD = [hybdatadict, SDparams,prb, detectioncrit] if ju.is_cached(ds.test_detection_algorithm,*argTD): print 'Yes, you have run detection_statistics.test_detection_algorithm() \n' detcrit_groundtruth = ds.test_detection_algorithm(hybdatadict, SDparams,prb, detectioncrit) else: print 'You need to run detection_statistics.test_detection_algorithm() \n in order to obtain a groundtruth' KKhash = hash_utils.hash_dictionary_md5(KKparams) baselist = [hash_hyb_SD, detcrit_groundtruth['detection_hashname'], KKhash] basefilename = hash_utils.make_concatenated_filename(baselist) mainbasefilelist = [hash_hyb_SD, detcrit_groundtruth['detection_hashname']] mainbasefilename = hash_utils.make_concatenated_filename(mainbasefilelist) DIRPATH = hybdatadict['output_path'] os.chdir(DIRPATH) with Experiment(hash_hyb_SD, dir= DIRPATH, mode='r') as expt: if KKparams['numspikesKK'] is not None: fets = expt.channel_groups[0].spikes.features[0:KKparams['numspikesKK']] fmasks = expt.channel_groups[0].spikes.features_masks[0:KKparams['numspikesKK'],:,1] masks = expt.channel_groups[0].spikes.masks[0:KKparams['numspikesKK']] else: fets = expt.channel_groups[0].spikes.features[:] fmasks = expt.channel_groups[0].spikes.features_masks[:,:,1] #print fmasks[3,:] masks = expt.channel_groups[0].spikes.masks[:] mainfetfile = DIRPATH + mainbasefilename+'.fet.1' mainfmaskfile = DIRPATH + mainbasefilename+'.fmask.1' mainmaskfile = DIRPATH + mainbasefilename+'.mask.1' if not os.path.isfile(mainfetfile): write_fet(fets,mainfetfile ) else: print mainfetfile, ' already exists, moving on \n ' if not os.path.isfile(mainfmaskfile): write_mask(fmasks,mainfmaskfile,fmt='%f') else: print mainfmaskfile, ' already exists, moving on \n ' if not os.path.isfile(mainmaskfile): write_mask(masks,mainmaskfile,fmt='%f') else: print mainmaskfile, ' already exists, moving on \n ' os.system('ln -s %s %s.fet.1 ' %(mainfetfile,basefilename)) os.system('ln -s %s %s.fmask.1 ' %(mainfmaskfile,basefilename)) os.system('ln -s %s %s.mask.1 ' %(mainmaskfile,basefilename)) KKscriptname = basefilename make_KKscript(KKparams,basefilename,KKscriptname) return basefilename