def classify_from_raw_data(JobType,
DatFileName,
ProbeFileName,
max_spikes=None,
output_dir=None,
clu_dir=None):
"""Top level function that starts a data processing job. JobType is "batch" or "generalize". """
print_params()
if not os.path.exists(DatFileName):
raise Exception("Dat file %s does not exist" % DatFileName)
DatFileName = os.path.abspath(DatFileName)
n_ch_dat, sample_rate = get_dat_pars(DatFileName)
set_globals_samples(sample_rate)
DatDir = os.path.abspath(os.path.dirname(DatFileName))
probe_stuff.load_probe(ProbeFileName)
global N_CH
N_CH = probe_stuff.N_SITES
basename = processed_basename(DatFileName, ProbeFileName, JobType)
OutDir = join(output_dir, basename) if output_dir else join(
DatDir, basename)
if clu_dir is not None: clu_dir = os.path.abspath(clu_dir)
with indir(OutDir):
Channels_dat = [site.dat for site in probe_stuff.PROBE_SITES]
if JobType == "batch":
cluster_from_raw_data(basename, DatFileName, n_ch_dat,
Channels_dat, probe_stuff.PROBE_GRAPH,
max_spikes)
def classify_from_raw_data(clusterdir,JobType,DatFileName,ProbeFileName,max_spikes=None,output_dir=None,clu_dir=None):
"""Top level function that starts a data processing job. JobType is "batch" or "generalize". """
print_params()
if not os.path.exists(DatFileName):
raise Exception("Dat file %s does not exist"%DatFileName)
DatFileName = os.path.abspath(DatFileName)
n_ch_dat,sample_rate = get_dat_pars(DatFileName)
set_globals_samples(sample_rate)
DatDir = os.path.abspath(os.path.dirname(DatFileName))
probe_stuff.load_probe(ProbeFileName)
global N_CH; N_CH = probe_stuff.N_SITES
basename = processed_basename(DatFileName,ProbeFileName,JobType)
OutDir = join(output_dir,basename) if output_dir else join(DatDir,basename)
if clu_dir is not None: clu_dir = os.path.abspath(clu_dir)
with indir(OutDir):
Channels_dat = [site.dat for site in probe_stuff.PROBE_SITES]
if JobType == "batch":
cluster_from_raw_data(clusterdir,basename,DatFileName,n_ch_dat,Channels_dat,probe_stuff.PROBE_GRAPH,max_spikes)
realOutDir = os.getcwd()
#elif JobType == "generalize":
#generalize_group_from_raw_data_splitprobe(basename,DatFileName,n_ch_dat,Channels_dat,probe_stuff.PROBE_GRAPH,max_spikes,clu_dir)
return realOutDir
def extract_intra_spikes(DatFileName,IntraChannel,output_dir=None,ExtraChannels=None):
"""extracts spikes times from intracellular data"""
THRESH_FRAC = .5
DatFileName = os.path.abspath(DatFileName)
DatDir = os.path.dirname(DatFileName)
basename = intra_basename(DatFileName)
OutDir = join(output_dir,basename) if output_dir else join(DatDir,basename)
with indir(OutDir):
SpkFileName = basename+'.spk.1'
n_ch_dat,sample_rate = get_dat_pars(DatFileName)
global N_CH
N_CH = 1 if ExtraChannels is None else 1 + len(ExtraChannels)
set_globals_samples(sample_rate)
print("extracting intracellular spikes from %s"%DatFileName)
n_samples = num_samples(DatFileName,n_ch_dat,n_bytes=np.nbytes[DTYPE])
AllDataArr = np.memmap(DatFileName,dtype=np.int16,shape=(n_samples,n_ch_dat),mode='r')
b,a = signal.butter(3,100./(SAMPLE_RATE/2),'high') #filter at 100 Hz
IntraArr = AllDataArr[:,IntraChannel].copy()
IntraArr = signal.filtfilt(b,a,IntraArr)
Thresh = IntraArr.max()*THRESH_FRAC
Segs = contig_segs(np.flatnonzero(IntraArr > Thresh),padding=2)
TmList = map(lambda Seg: Seg[IntraArr[Seg].argmax()],Segs)
CluList = np.ones(len(TmList),dtype=np.int)
FetList = np.zeros((len(TmList),1),dtype=np.int)
SpkList = [get_padded(IntraArr,PeakSample-S_BEFORE,PeakSample+S_AFTER) for PeakSample in TmList]
SpkArr = np.array(SpkList)[:,:,np.newaxis]
if ExtraChannels is not None:
ExtraArr = AllDataArr[:,ExtraChannels].copy()
#b,a = signal.butter(BUTTER_ORDER,(F_LOW/(SAMPLE_RATE/2),.95),'pass')
ExtraArr = filtfilt2d(b,a,ExtraArr)
ExtraSpkList = [get_padded(ExtraArr,PeakSample-S_BEFORE,PeakSample+S_AFTER) for PeakSample in TmList]
ExtraSpkArr = np.array(ExtraSpkList)
SpkArr *= ExtraSpkArr[0].max()/SpkArr[0].max()
SpkArr = np.concatenate((np.array(ExtraSpkList),SpkArr),axis=2)
output.write_spk(np.array(SpkArr),SpkFileName)
write_files(basename,CluList,TmList,FetList,[])
def combine_h5s(*dirs):
"combine the data from a bunch of h5 files. Also make klusters files"
outdirname = common_start(dirs) + "COMBINED"
print dirs
print os.path.abspath(os.curdir)
h5files = [
tables.openFile(find_file_with_ext(dir, ".h5"), mode="r")
for dir in dirs
]
spike_tables = [h5file.root.SpikeTable for h5file in h5files]
with indir(outdirname):
global N_CH, S_TOTAL, FPC, SAMPLE_RATE
num_ch = [table.cols.st.shape[1] for table in spike_tables]
N_CH = max(num_ch)
S_TOTAL = table.cols.wave.shape[1]
FPC = table.cols.fet.shape[2]
new_file = tables.openFile(outdirname + ".h5", mode="w")
new_table = new_file.createTable("/", "SpikeTable", spike_dtype())
clu_start = np.arange(0, 5000, 100)
SAMPLE_RATE = 25000. # Doesn't actually matter for this script, but write_xml wants it
# files in same order as tables
clu2DatChannels = {}
for (i_file, h5file) in enumerate(h5files):
for clu in xrange(clu_start[i_file], clu_start[i_file + 1]):
clu2DatChannels[clu] = list(h5file.root.DatChannels)
dump("clu2DatChannels.pickle", clu2DatChannels)
triples = [(row["time"], i_spike, i_table)
for (i_table, table) in enumerate(spike_tables)
for (i_spike, row) in enumerate(table)]
for time, i_spike, i_table in sorted(triples, key=lambda tup: tup[0]):
oldrow = spike_tables[i_table][i_spike]
new_table.row["time"] = time
new_table.row["fet"] = zero_pad(oldrow["fet"], N_CH)
new_table.row["st"] = zero_pad(oldrow["st"], N_CH)
new_table.row["wave"] = zero_pad(oldrow["wave"], N_CH)
new_table.row["clu"] = oldrow["clu"] + clu_start[i_table]
new_table.row.append()
new_table.flush()
klusters_files(new_table, outdirname)
new_file.close()
def combine_h5s(*dirs):
"combine the data from a bunch of h5 files. Also make klusters files"
outdirname = common_start(dirs) + "COMBINED"
print dirs
print os.path.abspath(os.curdir)
h5files = [tables.openFile(find_file_with_ext(dir,".h5"),mode="r") for dir in dirs]
spike_tables = [h5file.root.SpikeTable for h5file in h5files]
with indir(outdirname):
global N_CH,S_TOTAL,FPC,SAMPLE_RATE
num_ch =[table.cols.st.shape[1] for table in spike_tables]
N_CH = max(num_ch)
S_TOTAL = table.cols.wave.shape[1]
FPC = table.cols.fet.shape[2]
new_file = tables.openFile(outdirname+".h5",mode="w")
new_table = new_file.createTable("/","SpikeTable",spike_dtype())
clu_start = np.arange(0,5000,100)
SAMPLE_RATE = 25000. # Doesn't actually matter for this script, but write_xml wants it
# files in same order as tables
clu2DatChannels = {}
for (i_file,h5file) in enumerate(h5files):
for clu in xrange(clu_start[i_file],clu_start[i_file+1]):
clu2DatChannels[clu] = list(h5file.root.DatChannels)
dump("clu2DatChannels.pickle",clu2DatChannels)
triples = [(row["time"],i_spike,i_table) for (i_table,table) in enumerate(spike_tables) for (i_spike,row) in enumerate(table)]
for time,i_spike,i_table in sorted(triples, key = lambda tup: tup[0]):
oldrow = spike_tables[i_table][i_spike]
new_table.row["time"] = time
new_table.row["fet"]= zero_pad(oldrow["fet"],N_CH)
new_table.row["st"] = zero_pad(oldrow["st"],N_CH)
new_table.row["wave"] = zero_pad(oldrow["wave"],N_CH)
new_table.row["clu"] = oldrow["clu"]+clu_start[i_table]
new_table.row.append()
new_table.flush()
klusters_files(new_table,outdirname)
new_file.close()
def extract_intra_spikes(DatFileName,
IntraChannel,
output_dir=None,
ExtraChannels=None):
"""extracts spikes times from intracellular data"""
THRESH_FRAC = .5
DatFileName = os.path.abspath(DatFileName)
DatDir = os.path.dirname(DatFileName)
basename = intra_basename(DatFileName)
OutDir = join(output_dir, basename) if output_dir else join(
DatDir, basename)
with indir(OutDir):
SpkFileName = basename + '.spk.1'
n_ch_dat, sample_rate = get_dat_pars(DatFileName)
global N_CH
N_CH = 1 if ExtraChannels is None else 1 + len(ExtraChannels)
set_globals_samples(sample_rate)
print("extracting intracellular spikes from %s" % DatFileName)
n_samples = num_samples(DatFileName,
n_ch_dat,
n_bytes=np.nbytes[DTYPE])
AllDataArr = np.memmap(DatFileName,
dtype=np.int16,
shape=(n_samples, n_ch_dat),
mode='r')
b, a = signal.butter(3, 100. / (SAMPLE_RATE / 2),
'high') #filter at 100 Hz
IntraArr = AllDataArr[:, IntraChannel].copy()
IntraArr = signal.filtfilt(b, a, IntraArr)
Thresh = IntraArr.max() * THRESH_FRAC
Segs = contig_segs(np.flatnonzero(IntraArr > Thresh), padding=2)
TmList = map(lambda Seg: Seg[IntraArr[Seg].argmax()], Segs)
CluList = np.ones(len(TmList), dtype=np.int)
FetList = np.zeros((len(TmList), 1), dtype=np.int)
SpkList = [
get_padded(IntraArr, PeakSample - S_BEFORE, PeakSample + S_AFTER)
for PeakSample in TmList
]
SpkArr = np.array(SpkList)[:, :, np.newaxis]
if ExtraChannels is not None:
ExtraArr = AllDataArr[:, ExtraChannels].copy()
#b,a = signal.butter(BUTTER_ORDER,(F_LOW/(SAMPLE_RATE/2),.95),'pass')
ExtraArr = filtfilt2d(b, a, ExtraArr)
ExtraSpkList = [
get_padded(ExtraArr, PeakSample - S_BEFORE,
PeakSample + S_AFTER) for PeakSample in TmList
]
ExtraSpkArr = np.array(ExtraSpkList)
SpkArr *= ExtraSpkArr[0].max() / SpkArr[0].max()
SpkArr = np.concatenate((np.array(ExtraSpkList), SpkArr), axis=2)
output.write_spk(np.array(SpkArr), SpkFileName)
write_files(basename, CluList, TmList, FetList, [])
