def check_data_and_reformat(header, data, no_floats):
"""Performs some cleanup on data and builds a dictionary to return.
Args:
header (dict): metadata for the data
data (dict): different data fields are contained in numpy arrays
no_floats (bool): whether to expand 16-bit ints into 32-bit floats
Returns:
dict: combining data and some metadata
"""
extras = {} # dictionary for extra parameters
# Extract digital input channels to separate variables.
for i in range(header['num_board_dig_in_channels']):
data['board_dig_in_data'][i, :] = np.not_equal(
np.bitwise_and(
data['board_dig_in_raw'],
(1 << header['board_dig_in_channels'][i]['native_order'])), 0)
# Extract digital output channels to separate variables.
for i in range(header['num_board_dig_out_channels']):
data['board_dig_out_data'][i, :] = np.not_equal(
np.bitwise_and(
data['board_dig_out_raw'],
(1 << header['board_dig_out_channels'][i]['native_order'])), 0)
if no_floats:
# record the bit voltage scaling level but do not apply to the data
# converting to floats increases size 4x, which makes a big
# difference at the terabyte+ level.
extras['amplifier_bit_microvolts'] = AMPLIFIER_BIT_MICROVOLTS
# numpy doesn't do over/underflow checks, so this actually works as intended
np.subtract(data['amplifier_data'],
UINT16_BIT_OFFSET,
data['amplifier_data'],
casting='unsafe')
data['amplifier_data'] = data['amplifier_data'].astype(np.int16,
copy=False)
extras['aux_bit_volts'] = AUX_BIT_VOLTS
extras['supply_bit_volts'] = SUPPLY_BIT_VOLTS
extras['temp_bit_celcius'] = TEMP_BIT_CELCIUS
if header['eval_board_mode'] == 1:
extras['ADC_input_bit_volts'] = ADC_BIT_VOLTS_1
else:
extras['ADC_input_bit_volts'] = ADC_BIT_VOLTS_0
# underflow is intentional here as well
np.subtract(data['board_adc_data'],
UINT16_BIT_OFFSET,
data['board_adc_data'],
casting='unsafe')
data['board_adc_data'] = data['board_adc_data'].astype(np.int16,
copy=False)
else:
# Scale voltage levels appropriately.
data['amplifier_data'] = np.multiply(
AMPLIFIER_BIT_MICROVOLTS,
(data['amplifier_data'].astype(np.int32) -
UINT16_BIT_OFFSET)) # units = microvolts
data['aux_input_data'] = np.multiply(
AUX_BIT_VOLTS, data['aux_input_data']) # units = volts
data['supply_voltage_data'] = np.multiply(
SUPPLY_BIT_VOLTS, data['supply_voltage_data']) # units = volts
if header['eval_board_mode'] == 1:
data['board_adc_data'] = np.multiply(
ADC_BIT_VOLTS_1, (data['board_adc_data'].astype(np.int32) -
UINT16_BIT_OFFSET)) # units = volts
else:
data['board_adc_data'] = np.multiply(
ADC_BIT_VOLTS_0, data['board_adc_data']) # units = volts
data['temp_sensor_data'] = np.multiply(
TEMP_BIT_CELCIUS, data['temp_sensor_data']) # units = deg C
# Check for gaps in timestamps.
num_gaps = np.sum(
np.not_equal(data['t_amplifier'][1:] - data['t_amplifier'][:-1], 1))
if num_gaps != 0:
print(
'Warning: {0} gaps in timestamp data found. Time scale will not be uniform!'
.format(num_gaps))
# Scale time steps (units = seconds).
data['t_amplifier'] = data['t_amplifier'] / header['sample_rate']
data['t_aux_input'] = data['t_amplifier'][range(0,
len(data['t_amplifier']),
4)]
data['t_supply_voltage'] = data['t_amplifier'][range(
0, len(data['t_amplifier']), 60)]
data['t_board_adc'] = data['t_amplifier']
data['t_dig'] = data['t_amplifier']
data['t_temp_sensor'] = data['t_supply_voltage']
# Move variables to result struct.
result = data_to_result(header, data, data_present=True)
result.update(extras)
return result
def read_data(filename, no_floats=False, max_memory=0):
"""Reads Intan Technologies RHD2000 data file generated by evaluation board GUI.
Data are yielded in a dictionary, for future extensibility.
A file's contents are split into chunks governed by max_memory.
Args:
filename (str): .rhd file name
no_floats (bool): whether to avoid converting 16-bit integer values
to 32-bit floats (not converting saves disk space)
max_memory (int): size of chunks to split file's data into (in bytes)
Yields:
dict: containing data fields and some metadata
"""
fid = open(filename, 'rb')
filesize = os.path.getsize(filename)
header = read_header(fid)
if header['notch_filter_frequency'] > 0:
msg = 'Warning: a notch filter ({}Hz) was applied in the GUI, but has not been applied here.'
print(msg.format(header['notch_filter_frequency']))
print('Found {} amplifier channel{}.'.format(
header['num_amplifier_channels'],
plural(header['num_amplifier_channels'])))
print('Found {} auxiliary input channel{}.'.format(
header['num_aux_input_channels'],
plural(header['num_aux_input_channels'])))
print('Found {} supply voltage channel{}.'.format(
header['num_supply_voltage_channels'],
plural(header['num_supply_voltage_channels'])))
print('Found {} board ADC channel{}.'.format(
header['num_board_adc_channels'],
plural(header['num_board_adc_channels'])))
print('Found {} board digital input channel{}.'.format(
header['num_board_dig_in_channels'],
plural(header['num_board_dig_in_channels'])))
print('Found {} board digital output channel{}.'.format(
header['num_board_dig_out_channels'],
plural(header['num_board_dig_out_channels'])))
print('Found {} temperature sensors channel{}.'.format(
header['num_temp_sensor_channels'],
plural(header['num_temp_sensor_channels'])))
print('')
# Determine how many samples the data file contains.
bytes_per_block = get_bytes_per_data_block(header)
# How many data blocks remain in this file?
data_present = False
bytes_remaining = filesize - fid.tell()
if bytes_remaining > 0:
data_present = True
if bytes_remaining % bytes_per_block != 0:
raise Exception(
'Something is wrong with file size : should have a whole number of data blocks'
)
num_data_blocks = int(bytes_remaining / bytes_per_block)
record_time = AMP_SAMPLES * num_data_blocks / header['sample_rate']
if data_present:
print(
'File contains {:0.3f} seconds of data. Amplifiers were sampled at {:0.2f} kS/s.'
.format(record_time, header['sample_rate'] / 1000))
else:
print(
'Header file contains no data. Amplifiers were sampled at {:0.2f} kS/s.'
.format(header['sample_rate'] / 1000))
if data_present:
# chunk_size governs how many datablocks are read in and then written
# to file at once
# minimum is 1 datablock, maximum is every datablock in the file
# two copies of the chunk are in memory at once for some operations
# (reformatting, changing dtypes), so max_memory is divided by 2
chunk_size = min(max(int(0.5 * max_memory / bytes_per_block), 1),
num_data_blocks)
chunks, remainder = divmod(num_data_blocks, chunk_size)
data = preallocate_memory(header, chunk_size)
for i in range(chunks):
read_data_blocks(data,
header,
fid,
datablocks_per_chunk=chunk_size)
yield check_data_and_reformat(header, data, no_floats)
if remainder:
data = preallocate_memory(header, remainder)
read_data_blocks(data, header, fid, datablocks_per_chunk=remainder)
yield check_data_and_reformat(header, data, no_floats)
# Make sure we have read exactly the right amount of data.
bytes_remaining = filesize - fid.tell()
if bytes_remaining != 0:
raise Exception('Error: End of file not reached.')
else:
yield data_to_result(header, {}, data_present)
# Close data file.
fid.close()
def read_data(filename, no_floats=False):
"""Reads Intan Technologies RHD2000 data file generated by evaluation board GUI.
Data are returned in a dictionary, for future extensibility.
"""
tic = time.time()
fid = open(filename, 'rb')
filesize = os.path.getsize(filename)
header = read_header(fid)
print('Found {} amplifier channel{}.'.format(header[
'num_amplifier_channels'], plural(header['num_amplifier_channels'])))
print('Found {} auxiliary input channel{}.'.format(header[
'num_aux_input_channels'], plural(header['num_aux_input_channels'])))
print('Found {} supply voltage channel{}.'.format(header[
'num_supply_voltage_channels'], plural(header[
'num_supply_voltage_channels'])))
print('Found {} board ADC channel{}.'.format(header[
'num_board_adc_channels'], plural(header['num_board_adc_channels'])))
print('Found {} board digital input channel{}.'.format(header[
'num_board_dig_in_channels'], plural(header[
'num_board_dig_in_channels'])))
print('Found {} board digital output channel{}.'.format(header[
'num_board_dig_out_channels'], plural(header[
'num_board_dig_out_channels'])))
print('Found {} temperature sensors channel{}.'.format(header[
'num_temp_sensor_channels'], plural(header[
'num_temp_sensor_channels'])))
print('')
# Determine how many samples the data file contains.
bytes_per_block = get_bytes_per_data_block(header)
# How many data blocks remain in this file?
data_present = False
bytes_remaining = filesize - fid.tell()
if bytes_remaining > 0:
data_present = True
if bytes_remaining % bytes_per_block != 0:
raise Exception(
'Something is wrong with file size : should have a whole number of data blocks')
num_data_blocks = int(bytes_remaining / bytes_per_block)
num_amplifier_samples = 60 * num_data_blocks
num_aux_input_samples = 15 * num_data_blocks
num_supply_voltage_samples = 1 * num_data_blocks
num_board_adc_samples = 60 * num_data_blocks
num_board_dig_in_samples = 60 * num_data_blocks
num_board_dig_out_samples = 60 * num_data_blocks
record_time = num_amplifier_samples / header['sample_rate']
if data_present:
print(
'File contains {:0.3f} seconds of data. Amplifiers were sampled at {:0.2f} kS/s.'.format(
record_time, header['sample_rate'] / 1000))
else:
print(
'Header file contains no data. Amplifiers were sampled at {:0.2f} kS/s.'.format(
header['sample_rate'] / 1000))
if data_present:
# Pre-allocate memory for data.
data = {}
if (header['version']['major'] == 1 and
header['version']['minor'] >= 2) or (
header['version']['major'] > 1):
data['t_amplifier'] = np.zeros(num_amplifier_samples, dtype=np.int)
else:
data['t_amplifier'] = np.zeros(num_amplifier_samples,
dtype=np.uint)
data['amplifier_data'] = np.zeros(
[header['num_amplifier_channels'], num_amplifier_samples],
dtype=np.uint16)
data['aux_input_data'] = np.zeros(
[header['num_aux_input_channels'], num_aux_input_samples],
dtype=np.uint16)
data['supply_voltage_data'] = np.zeros(
[header['num_supply_voltage_channels'],
num_supply_voltage_samples],
dtype=np.uint16)
data['temp_sensor_data'] = np.zeros(
[header['num_temp_sensor_channels'], num_supply_voltage_samples],
dtype=np.uint16)
data['board_adc_data'] = np.zeros(
[header['num_board_adc_channels'], num_board_adc_samples],
dtype=np.uint16)
data['board_dig_in_data'] = np.zeros(
[header['num_board_dig_in_channels'], num_board_dig_in_samples],
dtype=np.uint)
data['board_dig_in_raw'] = np.zeros(num_board_dig_in_samples,
dtype=np.uint)
data['board_dig_out_data'] = np.zeros(
[header['num_board_dig_out_channels'], num_board_dig_out_samples],
dtype=np.uint)
data['board_dig_out_raw'] = np.zeros(num_board_dig_out_samples,
dtype=np.uint)
# Initialize indices used in looping
indices = {}
indices['amplifier'] = 0
indices['aux_input'] = 0
indices['supply_voltage'] = 0
indices['board_adc'] = 0
indices['board_dig_in'] = 0
indices['board_dig_out'] = 0
print_increment = 10
percent_done = print_increment
chunk_size = 1000 # data blocks to read at once; diminishing returns above 1000
chunks, remainder = divmod(num_data_blocks, chunk_size)
for i in range(chunks):
read_data_blocks(data, header, indices, fid, datablocks_per_chunk=chunk_size)
# Increment indices
indices['amplifier'] += 60 * chunk_size
indices['aux_input'] += 15 * chunk_size
indices['supply_voltage'] += 1 * chunk_size
indices['board_adc'] += 60 * chunk_size
indices['board_dig_in'] += 60 * chunk_size
indices['board_dig_out'] += 60 * chunk_size
fraction_done = 100 * (1.0 * i / num_data_blocks)
if fraction_done >= percent_done:
percent_done = percent_done + print_increment
if remainder:
read_data_blocks(data, header, indices, fid, datablocks_per_chunk=remainder)
# Make sure we have read exactly the right amount of data.
bytes_remaining = filesize - fid.tell()
if bytes_remaining != 0:
raise Exception('Error: End of file not reached.')
# Close data file.
fid.close()
extras = {} # dictionary for extra parameters
if (data_present):
# Extract digital input channels to separate variables.
for i in range(header['num_board_dig_in_channels']):
data['board_dig_in_data'][i, :] = np.not_equal(
np.bitwise_and(data['board_dig_in_raw'], (
1 << header['board_dig_in_channels'][i]['native_order'])),
0)
# Extract digital output channels to separate variables.
for i in range(header['num_board_dig_out_channels']):
data['board_dig_out_data'][i, :] = np.not_equal(
np.bitwise_and(data['board_dig_out_raw'], (
1 << header['board_dig_out_channels'][i]['native_order'])),
0)
if no_floats:
# record the bit voltage scaling level but do not apply to the data
# converting to floats increases size 4x, which makes a big difference at the terrabyte+ level.
extras['amplifier_bit_microvolts'] = AMPLIFIER_BIT_MICROVOLTS
data['amplifier_data'] = (data['amplifier_data'].astype(np.int32) -
UINT16_BIT_OFFSET).astype(np.int16)
extras['aux_bit_volts'] = AUX_BIT_VOLTS
extras['supply_bit_volts'] = SUPPLY_BIT_VOLTS
extras['temp_bit_celcius'] = TEMP_BIT_CELCIUS
if header['eval_board_mode'] == 1:
extras['ADC_input_bit_volts'] = ADC_BIT_VOLTS_1
else:
extras['ADC_input_bit_volts'] = ADC_BIT_VOLTS_0
data['board_adc_data'] = (data['board_adc_data'].astype(np.int32) -
UINT16_BIT_OFFSET).astype(np.int16)
else:
# Scale voltage levels appropriately.
data['amplifier_data'] = np.multiply(AMPLIFIER_BIT_MICROVOLTS, (
data['amplifier_data'].astype(np.int32) - UINT16_BIT_OFFSET)
) # units = microvolts
data['aux_input_data'] = np.multiply(
AUX_BIT_VOLTS, data['aux_input_data']) # units = volts
data['supply_voltage_data'] = np.multiply(
SUPPLY_BIT_VOLTS, data['supply_voltage_data']) # units = volts
if header['eval_board_mode'] == 1:
data['board_adc_data'] = np.multiply(
ADC_BIT_VOLTS_1, (data['board_adc_data'].astype(np.int32) -
UINT16_BIT_OFFSET)) # units = volts
else:
data['board_adc_data'] = np.multiply(
ADC_BIT_VOLTS_0, data['board_adc_data']) # units = volts
data['temp_sensor_data'] = np.multiply(
TEMP_BIT_CELCIUS, data['temp_sensor_data']) # units = deg C
# Check for gaps in timestamps.
num_gaps = np.sum(np.not_equal(data['t_amplifier'][1:] - data[
't_amplifier'][:-1], 1))
if num_gaps != 0:
print(
'Warning: {0} gaps in timestamp data found. Time scale will not be uniform!'.format(
num_gaps))
# Scale time steps (units = seconds).
data['t_amplifier'] = data['t_amplifier'] / header['sample_rate']
data['t_aux_input'] = data['t_amplifier'][range(0, len(data[
't_amplifier']), 4)]
data['t_supply_voltage'] = data['t_amplifier'][range(0, len(data[
't_amplifier']), 60)]
data['t_board_adc'] = data['t_amplifier']
data['t_dig'] = data['t_amplifier']
data['t_temp_sensor'] = data['t_supply_voltage']
# If the software notch filter was selected during the recording, apply the
# same notch filter to amplifier data here.
if header['notch_filter_frequency'] > 0:
print('Applying notch filter...')
print_increment = 10
percent_done = print_increment
for i in range(header['num_amplifier_channels']):
data['amplifier_data'][i, :] = notch_filter(
data['amplifier_data'][i, :], header['sample_rate'],
header['notch_filter_frequency'], 10)
fraction_done = 100 * (i / header['num_amplifier_channels'])
if fraction_done >= percent_done:
percent_done += print_increment
else:
data = []
# Move variables to result struct.
result = data_to_result(header, data, data_present)
result.update(extras)
return result