def read_activity_file(model_activity_fpath, openpv_path = '/home/mteti/OpenPV/mlab/util'):
'''
Read in the <model_layer_name>_A.pvp file containing the activations / feature maps of each neuron.
Args:
model_activity_fpath (str): Path to the <model_layer_name>_A.pvp where the
activations of each neuron are contained in.
openpv_path (str): Path to the openpv matlab utility directory (*/OpenPV/mlab/util).
Returns:
acts (np.ndarray): A B x H x W x # Neurons array of feature maps.
'''
# add OpenPV matlab utility dir to octave path
octave.addpath(openpv_path)
# Read in the activity file and get the number of batch samples
# Should be a list of length batch size, where each item in the list is an array
# of shape (input_width / stride x) x (input height / stride y) x # Neurons
# with the activations for that particular batch sample
act_data = octave.readpvpfile(model_activity_fpath)
n_batch = len(act_data)
# concatenate across batch samples to produce a single array of shape
# batch size x (input width / stride x) x (input height / stride y) x # Neurons
acts = np.concatenate([act_data[b]['values'][None, ...] for b in range(n_batch)], 0)
# transpose the 2nd and third dimensions to make it B x H x W x # Neurons
acts = acts.transpose([0, 2, 1, 3])
return acts
def read_simple_cell_weight_files(fpaths, n_features_x, n_features_y,
openpv_path = '/home/mteti/OpenPV/mlab/util'):
'''
Reads in .pvp shared weight files, where the features are the same size as the inputs,
and stride_x and stride_y equal the input width and height, respectively.
Args:
fpaths (list): List of file paths corresponding to the .pvp weight files to read.
n_features_x (int): Number of simple cell features to display along grid width.
n_features_y (int): Number of simple cell features to display down grid height.
openpv_path (str): Path to */OpenPV/mlab/util.
Returns:
weights_agg (list): List of len(fpaths) arrays of shape n_neurons x in_c x
n_features_y x n_features_x x kh x kw.
'''
octave.addpath(openpv_path)
weights_agg = []
for fpath in fpaths:
weights = octave.readpvpfile(fpath)[0]['values'][0]
w_x, w_y, w_in, w_out = weights.shape
n_neurons = w_out // n_features_x // n_features_y
# reshape to the original shape of the unshared weight tensor
# and reverse the x and y dims for image writing
weights = weights.reshape([w_x, w_y, w_in, n_neurons, n_features_x, n_features_y])
weights = weights.transpose([3, 2, 5, 4, 1, 0])
weights_agg.append(weights)
return weights_agg
def readpvpfile(filename):
# start a new octave session
octave.restart()
[data, header] = octave.readpvpfile(filename)
# close that octave session to save on memory
octave.close()
return data, header
def read_input_and_recon_files(fpaths, openpv_path = '/home/mteti/OpenPV/mlab/util'):
'''
Reads inputs and recons from .pvp files.
Args:
fpaths (list): List of filepaths to read data from.
openpv_path (str): Path to */OpenPV/mlab/util.
Returns:
data_agg (np.ndarray): Array of shape B x F x H x W, where F is len(fpaths).
'''
octave.addpath(openpv_path)
data_agg = []
for fpath_num, fpath in enumerate(fpaths):
batch = [sample['values'] for sample in octave.readpvpfile(fpath)]
data_agg.append(batch)
return np.asarray(data_agg).transpose([1, 0, 3, 2])
def read_complex_cell_weight_files(fpaths, openpv_path = '/home/mteti/OpenPV/mlab/util'):
'''
Reads in .pvp shared weights files with convolutional features.
Args:
fpaths (list): List of file paths corresponding to the .pvp weight files to read.
openpv_path (str): Path to */OpenPV/mlab/util.
Returns:
weights_agg (list): List of len(fpaths) arrays of shape n_neurons x in_c x kh x kw.
'''
octave.addpath(openpv_path)
weights_agg = []
for fpath in fpaths:
# original shape: patch width x patch height x in channels x out channels
# reshape to: out_channels x in channels x patch height x patch width
weights = octave.readpvpfile(fpath)[0]['values'][0]
weights = weights.transpose([3, 2, 1, 0])
weights_agg.append(weights)
return weights_agg
def readpvpfile(filename,
progressPeriod=0,
lastFrame=float('inf'),
startFrame=0,
skipFrames=1):
import os
assert startFrame >= 0
assert lastFrame >= 1
assert progressPeriod >= 0
assert skipFrames >= 1
assert startFrame < lastFrame
dataTypeSwitch = {1: np.uint8,
2: np.int32,
3: np.float32,
4: np.int32}
with open(filename,'rb') as stream:
header = readpvpheader(stream)
dataType = dataTypeSwitch[header['datatype']]
lastFrame = min(lastFrame, header['nbands'])
# Deprecated filetype not implemented in python (requires oct2py):
# PVP FILE (deprecated)
if header['filetype'] == 1:
from oct2py import octave
import re
octave.addpath(re.match('(.*)(python)',__file__).group(0) + '/mlab/util')
raw_data = octave.readpvpfile(filename)
return raw_data
# Supported older filetypes, not fully tested in Python 2
# SPIKING ACTIVITY FILE
if header['filetype'] == 2:
lastFrame = min(lastFrame, header['nbands'])
data = []
for frame in range(lastFrame):
if frame < startFrame or (frame % skipFrames):
numActive = np.fromfile(stream,np.uint32,3)[-1]
stream.seek(np.dtype(np.uint32).itemsize * numActive,
os.SEEK_CUR)
continue
else:
time = np.fromfile(stream,np.float64,1)[0]
numActive = np.fromfile(stream,np.uint32,1)
currentData = np.fromfile(stream,np.uint32,numActive)
data.append(DataFrame(time, currentData))
if progressPeriod:
if not frame % progressPeriod and frame:
print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame))
return PV_Object(data,header)
# NON-KERNEL WEIGHT FILE
elif header['filetype'] == 3:
fileSize = os.path.getsize(filename)
frameSize = header['recordsize'] * header['nbands'] + header['headersize']
lastFrame = min(lastFrame,fileSize/frameSize)
shape = (header['nxp'], header['nyp'], header['nfp'])
patchPattern = np.dtype([('nx', np.uint16),
('ny', np.uint16),
('offset', np.uint32),
('values', dataType, shape)])
data = []
stream.seek(0)
for frame in range(startFrame, lastFrame):
if not frame % skipFrames:
stream.seek(frame*frameSize)
time = np.fromfile(stream,extendedHeaderPattern,1)['time'][0]
data.append(DataFrame(time,[]))
for arbor in range(header['nbands']):
currentData = np.fromfile(stream,
patchPattern,
header['numpatches'])['values']
data[frame].values.append(np.squeeze(np.transpose(currentData,
[2,1,3,0])))
if progressPeriod:
if not frame % progressPeriod and frame:
print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame))
return PV_Object(data,header)
# Supported newer filetypes, fully implemented in python and tested as of 2/2/2016
# DENSE, NON-SPIKING ACTIVITY FILE
elif header['filetype'] == 4:
lastFrame = min(lastFrame, header['nbands'])
shape = (header['ny'], header['nx'], header['nf'])
pattern = np.dtype([('time', np.float64),
('values', dataType, shape)])
frameSize = pattern.itemsize
data = []
for frame in range(startFrame, lastFrame):
if not frame % skipFrames:
stream.seek(header['headersize'] + frame*frameSize)
currentData = np.fromfile(stream, pattern, 1)
data.append(DataFrame(currentData['time'][0],
currentData['values'][0]))
if progressPeriod:
if not frame % progressPeriod and frame:
print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame))
return PV_Object(data,header)
# KERNEL WEIGHT FILE
elif header['filetype'] == 5:
fileSize = os.path.getsize(filename)
frameSize = header['recordsize'] * header['nbands'] + header['headersize']
lastFrame = min(lastFrame,fileSize/frameSize)
shape = (header['nxp'], header['nyp'], header['nfp'])
patchPattern = np.dtype([('nx', np.uint16),
('ny', np.uint16),
('offset', np.uint32),
('values', dataType, shape)])
data = []
stream.seek(0)
for frame in range(startFrame, lastFrame):
if not frame % skipFrames:
stream.seek(frame*frameSize)
time = np.fromfile(stream,extendedHeaderPattern,1)['time'][0]
data.append(DataFrame(time,[]))
for arbor in range(header['nbands']):
currentData = np.fromfile(stream,
patchPattern,
header['numpatches'])['values']
data[frame].values.append(np.squeeze(np.transpose(currentData,
[2,1,3,0])))
if progressPeriod:
if not frame % progressPeriod and frame:
print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame))
return PV_Object(data,header)
# SPARSE ACTIVITY FILE
elif header['filetype'] == 6:
lastFrame = min(lastFrame, header['nbands'])
entryPattern = np.dtype([('index', np.int32),
('activation', np.float32)])
data = []
for frame in range(lastFrame):
if frame < startFrame or (frame % skipFrames):
numActive = np.fromfile(stream,np.uint32,3)[-1]
stream.seek(entryPattern.itemsize * numActive,
os.SEEK_CUR)
continue
else:
time = np.fromfile(stream,np.float64,1)[0]
numActive = np.fromfile(stream,np.uint32,1)
currentData = np.fromfile(stream,entryPattern,numActive)
data.append(DataFrame(time,zip(currentData['index'],
currentData['activation'])))
if progressPeriod:
if not frame % progressPeriod and frame:
print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame))
return PV_Object(data,header)
def readpvpfile(filename,
progressPeriod=0,
lastFrame=float('inf'),
startFrame=0,
skipFrames=1):
import os
assert startFrame >= 0
assert lastFrame >= 1
assert progressPeriod >= 0
assert skipFrames >= 1
assert startFrame < lastFrame
dataTypeSwitch = {1: np.uint8,
2: np.int32,
3: np.float32,
4: np.int32}
with open(filename,'rb') as stream:
header = readpvpheader(stream)
dataType = dataTypeSwitch[header['datatype']]
#lastFrame = min(lastFrame, header['nbands'])
#Initialize data structure
data = {}
data["header"] = header
# Deprecated filetype not implemented in python (requires oct2py):
# PVP FILE (deprecated)
if header['filetype'] == 1:
from oct2py import octave
import re
octave.addpath(re.match('(.*)(python)',__file__).group(0) + '/mlab/util')
raw_data = octave.readpvpfile(filename)
data["values"] = raw_data
return data
# Supported older filetypes, not fully tested in Python 2
# SPIKING ACTIVITY FILE
#TODO filetype 6 has fixed the frame indexing. It's probably broken here.
if header['filetype'] == 2:
lastFrame = min(lastFrame, header['nbands'])
framesList = []
idxList = []
timeList = []
frameNum = 0
for frame in range(lastFrame):
if frame < startFrame or (frame % skipFrames):
numActive = np.fromfile(stream,np.uint32,3)[-1]
stream.seek(np.dtype(np.uint32).itemsize * numActive,
os.SEEK_CUR)
continue
else:
time = np.fromfile(stream,np.float64,1)[0]
timeList.append(time)
numActive = np.fromfile(stream,np.uint32,1)
dataIdx = np.fromfile(stream,np.uint32,numActive)
idxList.extend(dataIdx)
framesList.extend(np.ones((len(dataIdx)))*frameNum)
frameNum += 1
if progressPeriod:
if not frame % progressPeriod and frame:
print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame))
#Make coosparsematrix
data["time"] = np.array(timeList)
#Values for sparse matrix are all 1's
data["values"] = sp.coo_matrix((np.ones((len(framesList))), (framesList, idxList)), shape=(frameNum, header["nx"]*header["ny"]*header["nf"]))
return data
# NON-KERNEL WEIGHT FILE
#TODO
elif header['filetype'] == 3:
# fileSize = os.path.getsize(filename)
# frameSize = header['recordsize'] * header['nbands'] + header['headersize']
# lastFrame = min(lastFrame,fileSize/frameSize)
# shape = (header['nxp'], header['nyp'], header['nfp'])
# patchPattern = np.dtype([('nx', np.uint16),
# ('ny', np.uint16),
# ('offset', np.uint32),
# ('values', dataType, shape)])
# stream.seek(0)
# #Initialize values and time arrays
# frameList = range(startFrame, lastFrame, skipFrames)
# data["values"] = np.zeros((len(frameList), header['nbands'], header['numpatches']
# for frame in frameList:
# stream.seek(frame*frameSize)
# time = np.fromfile(stream,extendedHeaderPattern,1)['time'][0]
# data.append(DataFrame(time,[]))
# for arbor in range(header['nbands']):
# currentData = np.fromfile(stream,
# patchPattern,
# header['numpatches'])['values']
# data[frame].values.append(np.squeeze(np.transpose(currentData,
# [2,1,3,0])))
# if progressPeriod:
# if not frame % progressPeriod and frame:
# print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame))
# return PV_Object(data,header)
assert(0)
# DENSE, NON-SPIKING ACTIVITY FILE
#Tested as of 2/15/16
elif header['filetype'] == 4:
lastFrame = min(lastFrame, header['nbands'])
shape = (header['ny'], header['nx'], header['nf'])
pattern = np.dtype([('time', np.float64),
('values', dataType, shape)])
frameSize = pattern.itemsize
frameRange = range(startFrame, lastFrame, skipFrames)
data["values"] = np.zeros((len(frameRange), header['ny'], header['nx'], header['nf']))
data["time"] = np.zeros((len(frameRange)))
for (i, frame) in enumerate(frameRange):
stream.seek(header['headersize'] + frame*frameSize)
currentData = np.fromfile(stream, pattern, 1)
time = currentData['time'][0]
values = currentData['values'][0]
data["time"][i] = time
data["values"][i, :, :, :] = values
if progressPeriod:
if not i % progressPeriod and frame:
print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame))
return data
# KERNEL WEIGHT FILE #TODO test
elif header['filetype'] == 5:
fileSize = os.path.getsize(filename)
frameSize = header['recordsize'] * header['nbands'] + header['headersize']
lastFrame = min(lastFrame,fileSize/frameSize)
shape = (header['nyp'], header['nxp'], header['nfp'])
patchPattern = np.dtype([('nx', np.uint16),
('ny', np.uint16),
('offset', np.uint32),
('values', dataType, shape)])
stream.seek(0)
frameRange = range(startFrame, lastFrame, skipFrames)
data["values"] = np.zeros((len(frameRange), header['nbands'], header['numpatches'], header['nyp'], header['nxp'], header['nfp']))
data["time"] = np.zeros((len(frameRange)));
for (i, frame) in enumerate(frameRange):
stream.seek(frame*frameSize)
time = np.fromfile(stream,extendedHeaderPattern,1)['time'][0]
data["time"][i] = time
for arbor in range(header['nbands']):
currentData = np.fromfile(stream,
patchPattern,
header['numpatches'])['values']
data["values"][i, arbor, :, :, :, :] = currentData
if progressPeriod:
if not i % progressPeriod and frame:
print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame))
return data
# SPARSE ACTIVITY FILE
#TODO
elif header['filetype'] == 6:
lastFrame = min(lastFrame, header['nbands'])
entryPattern = np.dtype([('index', np.int32),
('activation', np.float32)])
valuesList = []
framesList = []
idxList = []
timeList = []
frameNum = 0
for frame in range(lastFrame):
if frame in range(startFrame, lastFrame, skipFrames):
time = np.fromfile(stream,np.float64,1)[0]
timeList.append(time)
numActive = np.fromfile(stream,np.uint32,1)
currentData = np.fromfile(stream,entryPattern,numActive)
dataIdx = currentData['index']
dataValues = currentData['activation']
idxList.extend(dataIdx)
valuesList.extend(dataValues)
framesList.extend(np.ones((len(dataIdx)))*frameNum)
frameNum += 1
else:
numActive = np.fromfile(stream,np.uint32,3)[-1]
stream.seek(entryPattern.itemsize * numActive, os.SEEK_CUR)
continue
if progressPeriod:
if frame in range(startFrame, lastFrame, progressPeriod):
print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame))
#Make coosparsematrix
data["time"] = np.array(timeList)
data["values"] = sp.coo_matrix((valuesList, (framesList, idxList)), shape=(frameNum, header["nx"]*header["ny"]*header["nf"]))
return data
def readpvpfile(filename,
progressPeriod=0,
lastFrame=float('inf'),
startFrame=0,
skipFrames=1):
import os
assert startFrame >= 0
assert lastFrame >= 1
assert progressPeriod >= 0
assert skipFrames >= 1
assert startFrame < lastFrame
dataTypeSwitch = {1: np.uint8, 2: np.int32, 3: np.float32, 4: np.int32}
with open(filename, 'rb') as stream:
header = readpvpheader(stream)
dataType = dataTypeSwitch[header['datatype']]
lastFrame = min(lastFrame, header['nbands'])
# Deprecated filetype not implemented in python (requires oct2py):
# PVP FILE (deprecated)
if header['filetype'] == 1:
from oct2py import octave
import re
octave.addpath(
re.match('(.*)(python)', __file__).group(0) + '/mlab/util')
raw_data = octave.readpvpfile(filename)
return raw_data
# Supported older filetypes, not fully tested in Python 2
# SPIKING ACTIVITY FILE
if header['filetype'] == 2:
lastFrame = min(lastFrame, header['nbands'])
data = []
for frame in range(lastFrame):
if frame < startFrame or (frame % skipFrames):
numActive = np.fromfile(stream, np.uint32, 3)[-1]
stream.seek(
np.dtype(np.uint32).itemsize * numActive, os.SEEK_CUR)
continue
else:
time = np.fromfile(stream, np.float64, 1)[0]
numActive = np.fromfile(stream, np.uint32, 1)
currentData = np.fromfile(stream, np.uint32, numActive)
data.append(DataFrame(time, currentData))
if progressPeriod:
if not frame % progressPeriod and frame:
print("File " + filename + ": frame " +
str(frame) + " of " + str(lastFrame))
return PV_Object(data, header)
# NON-KERNEL WEIGHT FILE
elif header['filetype'] == 3:
fileSize = os.path.getsize(filename)
frameSize = header['recordsize'] * header['nbands'] + header[
'headersize']
lastFrame = min(lastFrame, fileSize / frameSize)
shape = (header['nxp'], header['nyp'], header['nfp'])
patchPattern = np.dtype([('nx', np.uint16), ('ny', np.uint16),
('offset', np.uint32),
('values', dataType, shape)])
data = []
stream.seek(0)
for frame in range(startFrame, lastFrame):
if not frame % skipFrames:
stream.seek(frame * frameSize)
time = np.fromfile(stream, extendedHeaderPattern,
1)['time'][0]
data.append(DataFrame(time, []))
for arbor in range(header['nbands']):
currentData = np.fromfile(
stream, patchPattern,
header['numpatches'])['values']
data[frame].values.append(
np.squeeze(np.transpose(currentData,
[2, 1, 3, 0])))
if progressPeriod:
if not frame % progressPeriod and frame:
print("File " + filename + ": frame " +
str(frame) + " of " + str(lastFrame))
return PV_Object(data, header)
# Supported newer filetypes, fully implemented in python and tested as of 2/2/2016
# DENSE, NON-SPIKING ACTIVITY FILE
elif header['filetype'] == 4:
lastFrame = min(lastFrame, header['nbands'])
shape = (header['ny'], header['nx'], header['nf'])
pattern = np.dtype([('time', np.float64),
('values', dataType, shape)])
frameSize = pattern.itemsize
data = []
for frame in range(startFrame, lastFrame):
if not frame % skipFrames:
stream.seek(header['headersize'] + frame * frameSize)
currentData = np.fromfile(stream, pattern, 1)
data.append(
DataFrame(currentData['time'][0],
currentData['values'][0]))
if progressPeriod:
if not frame % progressPeriod and frame:
print("File " + filename + ": frame " +
str(frame) + " of " + str(lastFrame))
return PV_Object(data, header)
# KERNEL WEIGHT FILE
elif header['filetype'] == 5:
fileSize = os.path.getsize(filename)
frameSize = header['recordsize'] * header['nbands'] + header[
'headersize']
lastFrame = min(lastFrame, fileSize / frameSize)
shape = (header['nxp'], header['nyp'], header['nfp'])
patchPattern = np.dtype([('nx', np.uint16), ('ny', np.uint16),
('offset', np.uint32),
('values', dataType, shape)])
data = []
stream.seek(0)
for frame in range(startFrame, lastFrame):
if not frame % skipFrames:
stream.seek(frame * frameSize)
time = np.fromfile(stream, extendedHeaderPattern,
1)['time'][0]
data.append(DataFrame(time, []))
for arbor in range(header['nbands']):
currentData = np.fromfile(
stream, patchPattern,
header['numpatches'])['values']
data[frame].values.append(
np.squeeze(np.transpose(currentData,
[2, 1, 3, 0])))
if progressPeriod:
if not frame % progressPeriod and frame:
print("File " + filename + ": frame " +
str(frame) + " of " + str(lastFrame))
return PV_Object(data, header)
# SPARSE ACTIVITY FILE
elif header['filetype'] == 6:
lastFrame = min(lastFrame, header['nbands'])
entryPattern = np.dtype([('index', np.int32),
('activation', np.float32)])
data = []
for frame in range(lastFrame):
if frame < startFrame or (frame % skipFrames):
numActive = np.fromfile(stream, np.uint32, 3)[-1]
stream.seek(entryPattern.itemsize * numActive, os.SEEK_CUR)
continue
else:
time = np.fromfile(stream, np.float64, 1)[0]
numActive = np.fromfile(stream, np.uint32, 1)
currentData = np.fromfile(stream, entryPattern, numActive)
data.append(
DataFrame(
time,
zip(currentData['index'],
currentData['activation'])))
if progressPeriod:
if not frame % progressPeriod and frame:
print("File " + filename + ": frame " +
str(frame) + " of " + str(lastFrame))
return PV_Object(data, header)
# add path to OpenPV octave utils
octave.addpath(args.openpv_path)
# get a list of the filepaths
weight_fpaths = glob(os.path.join(args.lca_ckpt_dir, args.weight_file_key))
weight_fpaths.sort()
# get top features if specified
if args.n_features_keep:
_, _, sorted_inds = get_mean_activations(args.act_fpath,
openpv_path=args.openpv_path)
sorted_inds_keep = sorted_inds[:args.n_features_keep]
for frame_num, fpath in ProgressBar()(enumerate(weight_fpaths)):
# read in the data from the weight file
feat_data = octave.readpvpfile(fpath)
weights = feat_data[0]['values'][0]
# pull out top features if specified
if args.n_features_keep:
weights = weights[..., sorted_inds_keep]
# get the shape of the weights and make sure they satisfy certain conditions
w_x, w_y, w_in, w_out = weights.shape
# compute the new number of features you will have and make placeholder to store them
nx = args.input_w - (w_x - 1)
ny = args.input_h - (w_y - 1)
w_out_new = (nx // args.stride_x) * (ny // args.stride_y) * w_out
nonshared = np.zeros([args.input_w, args.input_h, w_in, w_out_new],
dtype=np.float64)
for cell_num, cell_id in enumerate(cell_ids):
strf = h5file[cell_id][()]
cell_inds.append(cell_num)
if cell_num == 0:
h, w, n_frames = strf.shape
mouse_strfs = np.zeros([w, h, n_frames, n_cells], dtype=np.float64)
mouse_strfs[..., cell_num] = strf.transpose([1, 0, 2])
print('[INFO] FOUND {} MOUSE STRFS.'.format(len(cell_ids)))
# loop through each individual .pvp file
for frame_num, lca_fpath in enumerate(lca_fpaths):
# read in the pvp files and get the lca features
pvp_data = octave.readpvpfile(lca_fpath)
weights = pvp_data[0]['values'][0]
w_out_original = weights.shape[-1]
# take out the current frame from the strf and add to the weights
mouse_strfs_frame = mouse_strfs[:, :, frame_num, :]
mouse_strfs_frame = mouse_strfs_frame[:, :, None, :]
try: # if shapes don't line up
weights = np.concatenate((weights, mouse_strfs_frame), 3)
except ValueError:
raise
# add the new weights back to the pvp_data structure and push to octave session
pvp_data[0]['values'][0] = weights
write_fpath = os.path.join(args.save_dir, os.path.split(lca_fpath)[1])
def readpvpfile(filename,
progressPeriod=0,
lastFrame=float('inf'),
startFrame=0,
skipFrames=1):
import os
assert startFrame >= 0
assert lastFrame >= 1
assert progressPeriod >= 0
assert skipFrames >= 1
assert startFrame < lastFrame
dataTypeSwitch = {1: np.uint8, 2: np.int32, 3: np.float32, 4: np.int32}
with open(filename, 'rb') as stream:
header = readpvpheader(stream)
dataType = dataTypeSwitch[header['datatype']]
lastFrame = min(lastFrame, header['nbands'])
#Initialize data structure
data = {}
data["header"] = header
# Deprecated filetype not implemented in python (requires oct2py):
# PVP FILE (deprecated)
if header['filetype'] == 1:
from oct2py import octave
import re
octave.addpath(
re.match('(.*)(python)', __file__).group(0) + '/mlab/util')
raw_data = octave.readpvpfile(filename)
data["values"] = raw_data
return data
# Supported older filetypes, not fully tested in Python 2
# SPIKING ACTIVITY FILE
#TODO filetype 6 has fixed the frame indexing. It's probably broken here.
if header['filetype'] == 2:
lastFrame = min(lastFrame, header['nbands'])
framesList = []
idxList = []
timeList = []
frameNum = 0
for frame in range(lastFrame):
if frame < startFrame or (frame % skipFrames):
numActive = np.fromfile(stream, np.uint32, 3)[-1]
stream.seek(
np.dtype(np.uint32).itemsize * numActive, os.SEEK_CUR)
continue
else:
time = np.fromfile(stream, np.float64, 1)[0]
timeList.append(time)
numActive = np.fromfile(stream, np.uint32, 1)
dataIdx = np.fromfile(stream, np.uint32, numActive)
idxList.extend(dataIdx)
framesList.extend(np.ones((len(dataIdx))) * frameNum)
frameNum += 1
if progressPeriod:
if not frame % progressPeriod and frame:
print("File " + filename + ": frame " +
str(frame) + " of " + str(lastFrame))
#Make coosparsematrix
data["time"] = np.array(timeList)
#Values for sparse matrix are all 1's
data["values"] = sp.coo_matrix(
(np.ones((len(framesList))), (framesList, idxList)),
shape=(frameNum, header["nx"] * header["ny"] * header["nf"]))
return data
# NON-KERNEL WEIGHT FILE
#TODO
elif header['filetype'] == 3:
# fileSize = os.path.getsize(filename)
# frameSize = header['recordsize'] * header['nbands'] + header['headersize']
# lastFrame = min(lastFrame,fileSize/frameSize)
# shape = (header['nxp'], header['nyp'], header['nfp'])
# patchPattern = np.dtype([('nx', np.uint16),
# ('ny', np.uint16),
# ('offset', np.uint32),
# ('values', dataType, shape)])
# stream.seek(0)
# #Initialize values and time arrays
# frameList = range(startFrame, lastFrame, skipFrames)
# data["values"] = np.zeros((len(frameList), header['nbands'], header['numpatches']
# for frame in frameList:
# stream.seek(frame*frameSize)
# time = np.fromfile(stream,extendedHeaderPattern,1)['time'][0]
# data.append(DataFrame(time,[]))
# for arbor in range(header['nbands']):
# currentData = np.fromfile(stream,
# patchPattern,
# header['numpatches'])['values']
# data[frame].values.append(np.squeeze(np.transpose(currentData,
# [2,1,3,0])))
# if progressPeriod:
# if not frame % progressPeriod and frame:
# print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame))
# return PV_Object(data,header)
assert (0)
# DENSE, NON-SPIKING ACTIVITY FILE
#Tested as of 2/15/16
elif header['filetype'] == 4:
lastFrame = min(lastFrame, header['nbands'])
shape = (header['ny'], header['nx'], header['nf'])
pattern = np.dtype([('time', np.float64),
('values', dataType, shape)])
frameSize = pattern.itemsize
frameRange = range(startFrame, lastFrame, skipFrames)
data["values"] = np.zeros(
(len(frameRange), header['ny'], header['nx'], header['nf']))
data["time"] = np.zeros((len(frameRange)))
for (i, frame) in enumerate(frameRange):
stream.seek(header['headersize'] + frame * frameSize)
currentData = np.fromfile(stream, pattern, 1)
time = currentData['time'][0]
values = currentData['values'][0]
data["time"][i] = time
data["values"][i, :, :, :] = values
if progressPeriod:
if not i % progressPeriod and frame:
print("File " + filename + ": frame " + str(frame) +
" of " + str(lastFrame))
return data
# KERNEL WEIGHT FILE #TODO test
elif header['filetype'] == 5:
fileSize = os.path.getsize(filename)
frameSize = header['recordsize'] * header['nbands'] + header[
'headersize']
lastFrame = min(lastFrame, fileSize / frameSize)
shape = (header['nxp'], header['nyp'], header['nfp'])
patchPattern = np.dtype([('nx', np.uint16), ('ny', np.uint16),
('offset', np.uint32),
('values', dataType, shape)])
stream.seek(0)
frameRange = range(startFrame, lastFrame, skipFrames)
data["values"] = np.zeros(
(len(frameRange), header['nbands'], header['numpatches'],
header['nyp'], header['nxp'], header['nfp']))
data["time"] = np.zeros((len(frameRange)))
for (i, frame) in enumerate(frameRange):
stream.seek(frame * frameSize)
time = np.fromfile(stream, extendedHeaderPattern, 1)['time'][0]
data["time"][i] = time
for arbor in range(header['nbands']):
currentData = np.fromfile(stream, patchPattern,
header['numpatches'])['values']
data["values"][i, arbor, :, :, :, :] = currentData
if progressPeriod:
if not i % progressPeriod and frame:
print("File " + filename + ": frame " + str(frame) +
" of " + str(lastFrame))
return data
# SPARSE ACTIVITY FILE
#TODO
elif header['filetype'] == 6:
lastFrame = min(lastFrame, header['nbands'])
entryPattern = np.dtype([('index', np.int32),
('activation', np.float32)])
valuesList = []
framesList = []
idxList = []
timeList = []
validFrames = range(startFrame, lastFrame, skipFrames)
frameNum = 0
for frame in range(lastFrame):
if frame in validFrames:
time = np.fromfile(stream, np.float64, 1)[0]
timeList.append(time)
numActive = np.fromfile(stream, np.uint32, 1)
currentData = np.fromfile(stream, entryPattern, numActive)
dataIdx = currentData['index']
dataValues = currentData['activation']
idxList.extend(dataIdx)
valuesList.extend(dataValues)
framesList.extend(np.ones((len(dataIdx))) * frameNum)
frameNum += 1
else:
numActive = np.fromfile(stream, np.uint32, 3)[-1]
stream.seek(entryPattern.itemsize * numActive, os.SEEK_CUR)
continue
#data.append(DataFrame(time,zip(currentData['index'],
# currentData['activation'])))
if progressPeriod:
if not frame % progressPeriod and frame:
print("File " + filename + ": frame " + str(frame) +
" of " + str(lastFrame))
#Make coosparsematrix
data["time"] = np.array(timeList)
data["values"] = sp.coo_matrix(
(valuesList, (framesList, idxList)),
shape=(frameNum, header["nx"] * header["ny"] * header["nf"]))
return data
