def convert_continuous_to_mda(prm):
file_utility.create_folder_structure(prm)
# make_sorting_database.create_sorting_folder_structure(prm)
number_of_tetrodes = prm.get_num_tetrodes()
folder_path = prm.get_filepath()
spike_data_path = prm.get_spike_path() + '\\'
continuous_file_name = prm.get_continuous_file_name()
if os.path.isfile(spike_data_path + 't1_' + prm.get_date() +
'\\raw.mda') is False:
file_utility.create_ephys_folder_structure(prm)
for tetrode in range(number_of_tetrodes):
channel_data_all = []
for channel in range(4):
file_path = folder_path + continuous_file_name + str(
tetrode * 4 + channel + 1) + '.continuous'
channel_data = open_ephys_IO.get_data_continuous(prm, file_path)
channel_data_all.append(channel_data)
recording_length = len(channel_data_all[0])
channels_tetrode = np.zeros((4, recording_length))
for ch in range(4):
channels_tetrode[ch, :] = channel_data_all[ch]
mdaio.writemda16i(
channels_tetrode, spike_data_path + 't' + str(tetrode + 1) + '_' +
prm.get_date() + '_continuous\\data\\raw.mda')
def convert_spk_to_mda(prm):
file_utility.create_folder_structure(prm)
folder_path = prm.get_filepath()
spike_data_path = prm.get_spike_path() + '\\'
number_of_tetrodes = prm.get_num_tetrodes()
samples_per_spike = prm.get_waveform_size()
if os.path.isfile(spike_data_path + 't1_' + prm.get_date() +
'\\raw.nt1.mda') is False:
file_utility.create_ephys_folder_structure(prm)
for tetrode in range(number_of_tetrodes):
file_path = folder_path + 'TT' + str(tetrode) + '.spikes'
waveforms, timestamps = open_ephys_IO.get_data_spike(
folder_path, file_path, 'TT' + str(tetrode + 1))
np.save(
spike_data_path + 't' + str(tetrode + 1) + '_' +
prm.get_date() + '\\TT' + str(tetrode + 1) + '_timestamps',
timestamps
) # todo: this is shifted by 10 seconds relative to light and location!
padded_array = get_padded_array(waveforms, samples_per_spike)
mdaio.writemda16i(
padded_array, spike_data_path + 't' + str(tetrode + 1) + '_' +
prm.get_date() + '\\raw.nt' + str(tetrode + 1) + '.mda')
peak_indices = get_peak_indices(waveforms, samples_per_spike)
mdaio.writemda32i(
peak_indices, spike_data_path + 't' + str(tetrode + 1) + '_' +
prm.get_date() + '\\event_times.nt' + str(tetrode + 1) +
'.mda')
mdaio.writemda32(
timestamps, spike_data_path + 't' + str(tetrode + 1) + '_' +
prm.get_date() + '\\timestamps.nt' + str(tetrode + 1) + '.mda')
def main():
args = get_args()
tstart = datetime.datetime.now()
path = os.path.abspath(args.path)
dest = os.path.abspath(args.dest)
assert len(glob.glob(os.path.join(path, '*.raw.kwd'))) == 1, "Error finding .raw.kwd file in {}".format(path)
kwd = glob.glob(os.path.join(path, '*.raw.kwd'))[0]
out_mda = os.path.join(dest, 'raw.mda')
n_chans = -1
n_samples = 0
with h5.File(kwd, 'r') as kwd_f:
recordings = np.sort(np.array(kwd_f['recordings'].keys(), dtype=int)).astype('unicode')
for recording in recordings:
assert n_chans == -1 or n_chans == kwd_f['recordings'][recording]['data'].shape[1]
n_chans = kwd_f['recordings'][recording]['data'].shape[1]
n_samples += kwd_f['recordings'][recording]['data'].shape[0]
print "total number of samples %d" % (n_samples)
data = np.empty((n_samples, n_chans), dtype='int16')
idx = 0
for recording in recordings:
rec_len = kwd_f['recordings'][recording]['data'].shape[0]
print "loading recording %s with length of %d" % (recording, rec_len)
data[idx:idx + rec_len, :] = kwd_f['recordings'][recording]['data']
idx += rec_len
print "writing data"
writemda16i(data.T, out_mda)
print 'peak memory usage: %f GB' % (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024. / 1024.)
print 'time elapsed: %s' % (datetime.datetime.now() - tstart)
def kwd_to_mda(kwd_file, out_file_path, chan_list=None, chunk_size=None):
"""
:param kwd_file: kwd file or kwd file
:param out_file_path: path to the bin file that will be created
:param chan_list: list of channels (must be list or tuple). Default (None) will do the whole table
:param chunk_size: size in samples of the chunk
:return:
"""
# get the dataset of each recording and concatenateit to the out_file_path
logging.info('Writing kwd_file {} to binary'.format(kwd_file.filename))
logging.info('Channels to extract: {}'.format(chan_list))
logging.info('Creating mountainsort mda file {}'.format(out_file_path))
if chan_list is not None:
raise NotImplementedError
if (type(chan_list) is not list) and (type(chan_list) is not tuple):
assert (type(chan_list) is int)
chan_list = [chan_list]
chan_list = list(chan_list)
rec_list = get_rec_list(kwd_file)
logging.info('Will read through recs {}'.format(rec_list))
n_chans = -1
n_samples = 0
with h5.File(kwd, 'r') as kwd_f:
for recording in rec_list:
assert n_chans == -1 or n_chans == kwd_f['recordings'][recording]['data'].shape[1]
n_chans = kwd_f['recordings'][recording]['data'].shape[1]
n_samples += kwd_f['recordings'][recording]['data'].shape[0]
logging.debug("total number of samples %d" % (n_samples))
data = np.empty((n_samples, n_chans), dtype='int16')
idx = 0
for recording in recordings:
rec_len = kwd_f['recordings'][recording]['data'].shape[0]
logging.debug("loading recording %s with length of %d" % (recording, rec_len))
data[idx:idx + rec_len, :] = kwd_f['recordings'][recording]['data']
idx += rec_len
logging.info("writing data")
writemda16i(data.T, out_mda)
# read everything first
with open(out_file_path, 'wb') as out_file:
stored_elements = list(map(lambda rec_name: dset_to_binary_file(get_data_set(kwd_file, rec_name),
out_file,
chan_list=chan_list,
chunk_size=chunk_size
),
rec_list))
elements_in = np.array(list(stored_elements)).sum()
logging.info('{} elements written'.format(elements_in))
def convert_all_tetrodes_to_mda(prm):
file_utility.create_folder_structure(prm)
make_sorting_database.organize_files_for_ms(prm)
number_of_tetrodes = prm.get_num_tetrodes()
folder_path = prm.get_filepath()
spike_data_path = prm.get_spike_path() + '\\'
path = spike_data_path + 'all_tetrodes\\data\\raw.mda'
file_path = folder_path + '100_CH' + str(1) + '.continuous'
first_ch = open_ephys_IO.get_data_continuous(prm, file_path)
recording_length = len(first_ch)
channels_all = np.zeros((number_of_tetrodes * 4, recording_length))
channels_all[0, :] = first_ch
for channel in range(15):
file_path = folder_path + '100_CH' + str(channel + 2) + '.continuous'
channel_data = open_ephys_IO.get_data_continuous(prm, file_path)
channels_all[channel + 1, :] = channel_data
mdaio.writemda16i(channels_all, path)
def main():
args = get_args()
tstart = datetime.datetime.now()
path = os.path.abspath(args.path)
dest = os.path.abspath(args.dest)
assert len(glob.glob(os.path.join(
path,
'*.raw.kwd'))) == 1, "Error finding .raw.kwd file in {}".format(path)
kwd = glob.glob(os.path.join(path, '*.raw.kwd'))[0]
out_mda = os.path.join(dest, 'raw.mda')
n_chans = -1
n_samples = 0
with h5.File(kwd, 'r') as kwd_f:
recordings = np.sort(np.array(kwd_f['recordings'].keys(),
dtype=int)).astype('unicode')
for recording in recordings:
assert n_chans == -1 or n_chans == kwd_f['recordings'][recording][
'data'].shape[1]
n_chans = kwd_f['recordings'][recording]['data'].shape[1]
n_samples += kwd_f['recordings'][recording]['data'].shape[0]
print "total number of samples %d" % (n_samples)
data = np.empty((n_samples, n_chans), dtype='int16')
idx = 0
for recording in recordings:
rec_len = kwd_f['recordings'][recording]['data'].shape[0]
print "loading recording %s with length of %d" % (recording,
rec_len)
data[idx:idx + rec_len, :] = kwd_f['recordings'][recording]['data']
idx += rec_len
print "writing data"
writemda16i(data.T, out_mda)
print 'peak memory usage: %f GB' % (
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024. / 1024.)
print 'time elapsed: %s' % (datetime.datetime.now() - tstart)