def read_activity_file(model_activity_fpath, openpv_path = '/home/mteti/OpenPV/mlab/util'): ''' Read in the <model_layer_name>_A.pvp file containing the activations / feature maps of each neuron. Args: model_activity_fpath (str): Path to the <model_layer_name>_A.pvp where the activations of each neuron are contained in. openpv_path (str): Path to the openpv matlab utility directory (*/OpenPV/mlab/util). Returns: acts (np.ndarray): A B x H x W x # Neurons array of feature maps. ''' # add OpenPV matlab utility dir to octave path octave.addpath(openpv_path) # Read in the activity file and get the number of batch samples # Should be a list of length batch size, where each item in the list is an array # of shape (input_width / stride x) x (input height / stride y) x # Neurons # with the activations for that particular batch sample act_data = octave.readpvpfile(model_activity_fpath) n_batch = len(act_data) # concatenate across batch samples to produce a single array of shape # batch size x (input width / stride x) x (input height / stride y) x # Neurons acts = np.concatenate([act_data[b]['values'][None, ...] for b in range(n_batch)], 0) # transpose the 2nd and third dimensions to make it B x H x W x # Neurons acts = acts.transpose([0, 2, 1, 3]) return acts
def read_simple_cell_weight_files(fpaths, n_features_x, n_features_y, openpv_path = '/home/mteti/OpenPV/mlab/util'): ''' Reads in .pvp shared weight files, where the features are the same size as the inputs, and stride_x and stride_y equal the input width and height, respectively. Args: fpaths (list): List of file paths corresponding to the .pvp weight files to read. n_features_x (int): Number of simple cell features to display along grid width. n_features_y (int): Number of simple cell features to display down grid height. openpv_path (str): Path to */OpenPV/mlab/util. Returns: weights_agg (list): List of len(fpaths) arrays of shape n_neurons x in_c x n_features_y x n_features_x x kh x kw. ''' octave.addpath(openpv_path) weights_agg = [] for fpath in fpaths: weights = octave.readpvpfile(fpath)[0]['values'][0] w_x, w_y, w_in, w_out = weights.shape n_neurons = w_out // n_features_x // n_features_y # reshape to the original shape of the unshared weight tensor # and reverse the x and y dims for image writing weights = weights.reshape([w_x, w_y, w_in, n_neurons, n_features_x, n_features_y]) weights = weights.transpose([3, 2, 5, 4, 1, 0]) weights_agg.append(weights) return weights_agg
def readpvpfile(filename): # start a new octave session octave.restart() [data, header] = octave.readpvpfile(filename) # close that octave session to save on memory octave.close() return data, header
def read_input_and_recon_files(fpaths, openpv_path = '/home/mteti/OpenPV/mlab/util'): ''' Reads inputs and recons from .pvp files. Args: fpaths (list): List of filepaths to read data from. openpv_path (str): Path to */OpenPV/mlab/util. Returns: data_agg (np.ndarray): Array of shape B x F x H x W, where F is len(fpaths). ''' octave.addpath(openpv_path) data_agg = [] for fpath_num, fpath in enumerate(fpaths): batch = [sample['values'] for sample in octave.readpvpfile(fpath)] data_agg.append(batch) return np.asarray(data_agg).transpose([1, 0, 3, 2])
def read_complex_cell_weight_files(fpaths, openpv_path = '/home/mteti/OpenPV/mlab/util'): ''' Reads in .pvp shared weights files with convolutional features. Args: fpaths (list): List of file paths corresponding to the .pvp weight files to read. openpv_path (str): Path to */OpenPV/mlab/util. Returns: weights_agg (list): List of len(fpaths) arrays of shape n_neurons x in_c x kh x kw. ''' octave.addpath(openpv_path) weights_agg = [] for fpath in fpaths: # original shape: patch width x patch height x in channels x out channels # reshape to: out_channels x in channels x patch height x patch width weights = octave.readpvpfile(fpath)[0]['values'][0] weights = weights.transpose([3, 2, 1, 0]) weights_agg.append(weights) return weights_agg
def readpvpfile(filename, progressPeriod=0, lastFrame=float('inf'), startFrame=0, skipFrames=1): import os assert startFrame >= 0 assert lastFrame >= 1 assert progressPeriod >= 0 assert skipFrames >= 1 assert startFrame < lastFrame dataTypeSwitch = {1: np.uint8, 2: np.int32, 3: np.float32, 4: np.int32} with open(filename,'rb') as stream: header = readpvpheader(stream) dataType = dataTypeSwitch[header['datatype']] lastFrame = min(lastFrame, header['nbands']) # Deprecated filetype not implemented in python (requires oct2py): # PVP FILE (deprecated) if header['filetype'] == 1: from oct2py import octave import re octave.addpath(re.match('(.*)(python)',__file__).group(0) + '/mlab/util') raw_data = octave.readpvpfile(filename) return raw_data # Supported older filetypes, not fully tested in Python 2 # SPIKING ACTIVITY FILE if header['filetype'] == 2: lastFrame = min(lastFrame, header['nbands']) data = [] for frame in range(lastFrame): if frame < startFrame or (frame % skipFrames): numActive = np.fromfile(stream,np.uint32,3)[-1] stream.seek(np.dtype(np.uint32).itemsize * numActive, os.SEEK_CUR) continue else: time = np.fromfile(stream,np.float64,1)[0] numActive = np.fromfile(stream,np.uint32,1) currentData = np.fromfile(stream,np.uint32,numActive) data.append(DataFrame(time, currentData)) if progressPeriod: if not frame % progressPeriod and frame: print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame)) return PV_Object(data,header) # NON-KERNEL WEIGHT FILE elif header['filetype'] == 3: fileSize = os.path.getsize(filename) frameSize = header['recordsize'] * header['nbands'] + header['headersize'] lastFrame = min(lastFrame,fileSize/frameSize) shape = (header['nxp'], header['nyp'], header['nfp']) patchPattern = np.dtype([('nx', np.uint16), ('ny', np.uint16), ('offset', np.uint32), ('values', dataType, shape)]) data = [] stream.seek(0) for frame in range(startFrame, lastFrame): if not frame % skipFrames: stream.seek(frame*frameSize) time = np.fromfile(stream,extendedHeaderPattern,1)['time'][0] data.append(DataFrame(time,[])) for arbor in range(header['nbands']): currentData = np.fromfile(stream, patchPattern, header['numpatches'])['values'] data[frame].values.append(np.squeeze(np.transpose(currentData, [2,1,3,0]))) if progressPeriod: if not frame % progressPeriod and frame: print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame)) return PV_Object(data,header) # Supported newer filetypes, fully implemented in python and tested as of 2/2/2016 # DENSE, NON-SPIKING ACTIVITY FILE elif header['filetype'] == 4: lastFrame = min(lastFrame, header['nbands']) shape = (header['ny'], header['nx'], header['nf']) pattern = np.dtype([('time', np.float64), ('values', dataType, shape)]) frameSize = pattern.itemsize data = [] for frame in range(startFrame, lastFrame): if not frame % skipFrames: stream.seek(header['headersize'] + frame*frameSize) currentData = np.fromfile(stream, pattern, 1) data.append(DataFrame(currentData['time'][0], currentData['values'][0])) if progressPeriod: if not frame % progressPeriod and frame: print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame)) return PV_Object(data,header) # KERNEL WEIGHT FILE elif header['filetype'] == 5: fileSize = os.path.getsize(filename) frameSize = header['recordsize'] * header['nbands'] + header['headersize'] lastFrame = min(lastFrame,fileSize/frameSize) shape = (header['nxp'], header['nyp'], header['nfp']) patchPattern = np.dtype([('nx', np.uint16), ('ny', np.uint16), ('offset', np.uint32), ('values', dataType, shape)]) data = [] stream.seek(0) for frame in range(startFrame, lastFrame): if not frame % skipFrames: stream.seek(frame*frameSize) time = np.fromfile(stream,extendedHeaderPattern,1)['time'][0] data.append(DataFrame(time,[])) for arbor in range(header['nbands']): currentData = np.fromfile(stream, patchPattern, header['numpatches'])['values'] data[frame].values.append(np.squeeze(np.transpose(currentData, [2,1,3,0]))) if progressPeriod: if not frame % progressPeriod and frame: print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame)) return PV_Object(data,header) # SPARSE ACTIVITY FILE elif header['filetype'] == 6: lastFrame = min(lastFrame, header['nbands']) entryPattern = np.dtype([('index', np.int32), ('activation', np.float32)]) data = [] for frame in range(lastFrame): if frame < startFrame or (frame % skipFrames): numActive = np.fromfile(stream,np.uint32,3)[-1] stream.seek(entryPattern.itemsize * numActive, os.SEEK_CUR) continue else: time = np.fromfile(stream,np.float64,1)[0] numActive = np.fromfile(stream,np.uint32,1) currentData = np.fromfile(stream,entryPattern,numActive) data.append(DataFrame(time,zip(currentData['index'], currentData['activation']))) if progressPeriod: if not frame % progressPeriod and frame: print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame)) return PV_Object(data,header)
def readpvpfile(filename, progressPeriod=0, lastFrame=float('inf'), startFrame=0, skipFrames=1): import os assert startFrame >= 0 assert lastFrame >= 1 assert progressPeriod >= 0 assert skipFrames >= 1 assert startFrame < lastFrame dataTypeSwitch = {1: np.uint8, 2: np.int32, 3: np.float32, 4: np.int32} with open(filename,'rb') as stream: header = readpvpheader(stream) dataType = dataTypeSwitch[header['datatype']] #lastFrame = min(lastFrame, header['nbands']) #Initialize data structure data = {} data["header"] = header # Deprecated filetype not implemented in python (requires oct2py): # PVP FILE (deprecated) if header['filetype'] == 1: from oct2py import octave import re octave.addpath(re.match('(.*)(python)',__file__).group(0) + '/mlab/util') raw_data = octave.readpvpfile(filename) data["values"] = raw_data return data # Supported older filetypes, not fully tested in Python 2 # SPIKING ACTIVITY FILE #TODO filetype 6 has fixed the frame indexing. It's probably broken here. if header['filetype'] == 2: lastFrame = min(lastFrame, header['nbands']) framesList = [] idxList = [] timeList = [] frameNum = 0 for frame in range(lastFrame): if frame < startFrame or (frame % skipFrames): numActive = np.fromfile(stream,np.uint32,3)[-1] stream.seek(np.dtype(np.uint32).itemsize * numActive, os.SEEK_CUR) continue else: time = np.fromfile(stream,np.float64,1)[0] timeList.append(time) numActive = np.fromfile(stream,np.uint32,1) dataIdx = np.fromfile(stream,np.uint32,numActive) idxList.extend(dataIdx) framesList.extend(np.ones((len(dataIdx)))*frameNum) frameNum += 1 if progressPeriod: if not frame % progressPeriod and frame: print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame)) #Make coosparsematrix data["time"] = np.array(timeList) #Values for sparse matrix are all 1's data["values"] = sp.coo_matrix((np.ones((len(framesList))), (framesList, idxList)), shape=(frameNum, header["nx"]*header["ny"]*header["nf"])) return data # NON-KERNEL WEIGHT FILE #TODO elif header['filetype'] == 3: # fileSize = os.path.getsize(filename) # frameSize = header['recordsize'] * header['nbands'] + header['headersize'] # lastFrame = min(lastFrame,fileSize/frameSize) # shape = (header['nxp'], header['nyp'], header['nfp']) # patchPattern = np.dtype([('nx', np.uint16), # ('ny', np.uint16), # ('offset', np.uint32), # ('values', dataType, shape)]) # stream.seek(0) # #Initialize values and time arrays # frameList = range(startFrame, lastFrame, skipFrames) # data["values"] = np.zeros((len(frameList), header['nbands'], header['numpatches'] # for frame in frameList: # stream.seek(frame*frameSize) # time = np.fromfile(stream,extendedHeaderPattern,1)['time'][0] # data.append(DataFrame(time,[])) # for arbor in range(header['nbands']): # currentData = np.fromfile(stream, # patchPattern, # header['numpatches'])['values'] # data[frame].values.append(np.squeeze(np.transpose(currentData, # [2,1,3,0]))) # if progressPeriod: # if not frame % progressPeriod and frame: # print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame)) # return PV_Object(data,header) assert(0) # DENSE, NON-SPIKING ACTIVITY FILE #Tested as of 2/15/16 elif header['filetype'] == 4: lastFrame = min(lastFrame, header['nbands']) shape = (header['ny'], header['nx'], header['nf']) pattern = np.dtype([('time', np.float64), ('values', dataType, shape)]) frameSize = pattern.itemsize frameRange = range(startFrame, lastFrame, skipFrames) data["values"] = np.zeros((len(frameRange), header['ny'], header['nx'], header['nf'])) data["time"] = np.zeros((len(frameRange))) for (i, frame) in enumerate(frameRange): stream.seek(header['headersize'] + frame*frameSize) currentData = np.fromfile(stream, pattern, 1) time = currentData['time'][0] values = currentData['values'][0] data["time"][i] = time data["values"][i, :, :, :] = values if progressPeriod: if not i % progressPeriod and frame: print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame)) return data # KERNEL WEIGHT FILE #TODO test elif header['filetype'] == 5: fileSize = os.path.getsize(filename) frameSize = header['recordsize'] * header['nbands'] + header['headersize'] lastFrame = min(lastFrame,fileSize/frameSize) shape = (header['nyp'], header['nxp'], header['nfp']) patchPattern = np.dtype([('nx', np.uint16), ('ny', np.uint16), ('offset', np.uint32), ('values', dataType, shape)]) stream.seek(0) frameRange = range(startFrame, lastFrame, skipFrames) data["values"] = np.zeros((len(frameRange), header['nbands'], header['numpatches'], header['nyp'], header['nxp'], header['nfp'])) data["time"] = np.zeros((len(frameRange))); for (i, frame) in enumerate(frameRange): stream.seek(frame*frameSize) time = np.fromfile(stream,extendedHeaderPattern,1)['time'][0] data["time"][i] = time for arbor in range(header['nbands']): currentData = np.fromfile(stream, patchPattern, header['numpatches'])['values'] data["values"][i, arbor, :, :, :, :] = currentData if progressPeriod: if not i % progressPeriod and frame: print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame)) return data # SPARSE ACTIVITY FILE #TODO elif header['filetype'] == 6: lastFrame = min(lastFrame, header['nbands']) entryPattern = np.dtype([('index', np.int32), ('activation', np.float32)]) valuesList = [] framesList = [] idxList = [] timeList = [] frameNum = 0 for frame in range(lastFrame): if frame in range(startFrame, lastFrame, skipFrames): time = np.fromfile(stream,np.float64,1)[0] timeList.append(time) numActive = np.fromfile(stream,np.uint32,1) currentData = np.fromfile(stream,entryPattern,numActive) dataIdx = currentData['index'] dataValues = currentData['activation'] idxList.extend(dataIdx) valuesList.extend(dataValues) framesList.extend(np.ones((len(dataIdx)))*frameNum) frameNum += 1 else: numActive = np.fromfile(stream,np.uint32,3)[-1] stream.seek(entryPattern.itemsize * numActive, os.SEEK_CUR) continue if progressPeriod: if frame in range(startFrame, lastFrame, progressPeriod): print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame)) #Make coosparsematrix data["time"] = np.array(timeList) data["values"] = sp.coo_matrix((valuesList, (framesList, idxList)), shape=(frameNum, header["nx"]*header["ny"]*header["nf"])) return data
def readpvpfile(filename, progressPeriod=0, lastFrame=float('inf'), startFrame=0, skipFrames=1): import os assert startFrame >= 0 assert lastFrame >= 1 assert progressPeriod >= 0 assert skipFrames >= 1 assert startFrame < lastFrame dataTypeSwitch = {1: np.uint8, 2: np.int32, 3: np.float32, 4: np.int32} with open(filename, 'rb') as stream: header = readpvpheader(stream) dataType = dataTypeSwitch[header['datatype']] lastFrame = min(lastFrame, header['nbands']) # Deprecated filetype not implemented in python (requires oct2py): # PVP FILE (deprecated) if header['filetype'] == 1: from oct2py import octave import re octave.addpath( re.match('(.*)(python)', __file__).group(0) + '/mlab/util') raw_data = octave.readpvpfile(filename) return raw_data # Supported older filetypes, not fully tested in Python 2 # SPIKING ACTIVITY FILE if header['filetype'] == 2: lastFrame = min(lastFrame, header['nbands']) data = [] for frame in range(lastFrame): if frame < startFrame or (frame % skipFrames): numActive = np.fromfile(stream, np.uint32, 3)[-1] stream.seek( np.dtype(np.uint32).itemsize * numActive, os.SEEK_CUR) continue else: time = np.fromfile(stream, np.float64, 1)[0] numActive = np.fromfile(stream, np.uint32, 1) currentData = np.fromfile(stream, np.uint32, numActive) data.append(DataFrame(time, currentData)) if progressPeriod: if not frame % progressPeriod and frame: print("File " + filename + ": frame " + str(frame) + " of " + str(lastFrame)) return PV_Object(data, header) # NON-KERNEL WEIGHT FILE elif header['filetype'] == 3: fileSize = os.path.getsize(filename) frameSize = header['recordsize'] * header['nbands'] + header[ 'headersize'] lastFrame = min(lastFrame, fileSize / frameSize) shape = (header['nxp'], header['nyp'], header['nfp']) patchPattern = np.dtype([('nx', np.uint16), ('ny', np.uint16), ('offset', np.uint32), ('values', dataType, shape)]) data = [] stream.seek(0) for frame in range(startFrame, lastFrame): if not frame % skipFrames: stream.seek(frame * frameSize) time = np.fromfile(stream, extendedHeaderPattern, 1)['time'][0] data.append(DataFrame(time, [])) for arbor in range(header['nbands']): currentData = np.fromfile( stream, patchPattern, header['numpatches'])['values'] data[frame].values.append( np.squeeze(np.transpose(currentData, [2, 1, 3, 0]))) if progressPeriod: if not frame % progressPeriod and frame: print("File " + filename + ": frame " + str(frame) + " of " + str(lastFrame)) return PV_Object(data, header) # Supported newer filetypes, fully implemented in python and tested as of 2/2/2016 # DENSE, NON-SPIKING ACTIVITY FILE elif header['filetype'] == 4: lastFrame = min(lastFrame, header['nbands']) shape = (header['ny'], header['nx'], header['nf']) pattern = np.dtype([('time', np.float64), ('values', dataType, shape)]) frameSize = pattern.itemsize data = [] for frame in range(startFrame, lastFrame): if not frame % skipFrames: stream.seek(header['headersize'] + frame * frameSize) currentData = np.fromfile(stream, pattern, 1) data.append( DataFrame(currentData['time'][0], currentData['values'][0])) if progressPeriod: if not frame % progressPeriod and frame: print("File " + filename + ": frame " + str(frame) + " of " + str(lastFrame)) return PV_Object(data, header) # KERNEL WEIGHT FILE elif header['filetype'] == 5: fileSize = os.path.getsize(filename) frameSize = header['recordsize'] * header['nbands'] + header[ 'headersize'] lastFrame = min(lastFrame, fileSize / frameSize) shape = (header['nxp'], header['nyp'], header['nfp']) patchPattern = np.dtype([('nx', np.uint16), ('ny', np.uint16), ('offset', np.uint32), ('values', dataType, shape)]) data = [] stream.seek(0) for frame in range(startFrame, lastFrame): if not frame % skipFrames: stream.seek(frame * frameSize) time = np.fromfile(stream, extendedHeaderPattern, 1)['time'][0] data.append(DataFrame(time, [])) for arbor in range(header['nbands']): currentData = np.fromfile( stream, patchPattern, header['numpatches'])['values'] data[frame].values.append( np.squeeze(np.transpose(currentData, [2, 1, 3, 0]))) if progressPeriod: if not frame % progressPeriod and frame: print("File " + filename + ": frame " + str(frame) + " of " + str(lastFrame)) return PV_Object(data, header) # SPARSE ACTIVITY FILE elif header['filetype'] == 6: lastFrame = min(lastFrame, header['nbands']) entryPattern = np.dtype([('index', np.int32), ('activation', np.float32)]) data = [] for frame in range(lastFrame): if frame < startFrame or (frame % skipFrames): numActive = np.fromfile(stream, np.uint32, 3)[-1] stream.seek(entryPattern.itemsize * numActive, os.SEEK_CUR) continue else: time = np.fromfile(stream, np.float64, 1)[0] numActive = np.fromfile(stream, np.uint32, 1) currentData = np.fromfile(stream, entryPattern, numActive) data.append( DataFrame( time, zip(currentData['index'], currentData['activation']))) if progressPeriod: if not frame % progressPeriod and frame: print("File " + filename + ": frame " + str(frame) + " of " + str(lastFrame)) return PV_Object(data, header)
# add path to OpenPV octave utils octave.addpath(args.openpv_path) # get a list of the filepaths weight_fpaths = glob(os.path.join(args.lca_ckpt_dir, args.weight_file_key)) weight_fpaths.sort() # get top features if specified if args.n_features_keep: _, _, sorted_inds = get_mean_activations(args.act_fpath, openpv_path=args.openpv_path) sorted_inds_keep = sorted_inds[:args.n_features_keep] for frame_num, fpath in ProgressBar()(enumerate(weight_fpaths)): # read in the data from the weight file feat_data = octave.readpvpfile(fpath) weights = feat_data[0]['values'][0] # pull out top features if specified if args.n_features_keep: weights = weights[..., sorted_inds_keep] # get the shape of the weights and make sure they satisfy certain conditions w_x, w_y, w_in, w_out = weights.shape # compute the new number of features you will have and make placeholder to store them nx = args.input_w - (w_x - 1) ny = args.input_h - (w_y - 1) w_out_new = (nx // args.stride_x) * (ny // args.stride_y) * w_out nonshared = np.zeros([args.input_w, args.input_h, w_in, w_out_new], dtype=np.float64)
for cell_num, cell_id in enumerate(cell_ids): strf = h5file[cell_id][()] cell_inds.append(cell_num) if cell_num == 0: h, w, n_frames = strf.shape mouse_strfs = np.zeros([w, h, n_frames, n_cells], dtype=np.float64) mouse_strfs[..., cell_num] = strf.transpose([1, 0, 2]) print('[INFO] FOUND {} MOUSE STRFS.'.format(len(cell_ids))) # loop through each individual .pvp file for frame_num, lca_fpath in enumerate(lca_fpaths): # read in the pvp files and get the lca features pvp_data = octave.readpvpfile(lca_fpath) weights = pvp_data[0]['values'][0] w_out_original = weights.shape[-1] # take out the current frame from the strf and add to the weights mouse_strfs_frame = mouse_strfs[:, :, frame_num, :] mouse_strfs_frame = mouse_strfs_frame[:, :, None, :] try: # if shapes don't line up weights = np.concatenate((weights, mouse_strfs_frame), 3) except ValueError: raise # add the new weights back to the pvp_data structure and push to octave session pvp_data[0]['values'][0] = weights write_fpath = os.path.join(args.save_dir, os.path.split(lca_fpath)[1])
def readpvpfile(filename, progressPeriod=0, lastFrame=float('inf'), startFrame=0, skipFrames=1): import os assert startFrame >= 0 assert lastFrame >= 1 assert progressPeriod >= 0 assert skipFrames >= 1 assert startFrame < lastFrame dataTypeSwitch = {1: np.uint8, 2: np.int32, 3: np.float32, 4: np.int32} with open(filename, 'rb') as stream: header = readpvpheader(stream) dataType = dataTypeSwitch[header['datatype']] lastFrame = min(lastFrame, header['nbands']) #Initialize data structure data = {} data["header"] = header # Deprecated filetype not implemented in python (requires oct2py): # PVP FILE (deprecated) if header['filetype'] == 1: from oct2py import octave import re octave.addpath( re.match('(.*)(python)', __file__).group(0) + '/mlab/util') raw_data = octave.readpvpfile(filename) data["values"] = raw_data return data # Supported older filetypes, not fully tested in Python 2 # SPIKING ACTIVITY FILE #TODO filetype 6 has fixed the frame indexing. It's probably broken here. if header['filetype'] == 2: lastFrame = min(lastFrame, header['nbands']) framesList = [] idxList = [] timeList = [] frameNum = 0 for frame in range(lastFrame): if frame < startFrame or (frame % skipFrames): numActive = np.fromfile(stream, np.uint32, 3)[-1] stream.seek( np.dtype(np.uint32).itemsize * numActive, os.SEEK_CUR) continue else: time = np.fromfile(stream, np.float64, 1)[0] timeList.append(time) numActive = np.fromfile(stream, np.uint32, 1) dataIdx = np.fromfile(stream, np.uint32, numActive) idxList.extend(dataIdx) framesList.extend(np.ones((len(dataIdx))) * frameNum) frameNum += 1 if progressPeriod: if not frame % progressPeriod and frame: print("File " + filename + ": frame " + str(frame) + " of " + str(lastFrame)) #Make coosparsematrix data["time"] = np.array(timeList) #Values for sparse matrix are all 1's data["values"] = sp.coo_matrix( (np.ones((len(framesList))), (framesList, idxList)), shape=(frameNum, header["nx"] * header["ny"] * header["nf"])) return data # NON-KERNEL WEIGHT FILE #TODO elif header['filetype'] == 3: # fileSize = os.path.getsize(filename) # frameSize = header['recordsize'] * header['nbands'] + header['headersize'] # lastFrame = min(lastFrame,fileSize/frameSize) # shape = (header['nxp'], header['nyp'], header['nfp']) # patchPattern = np.dtype([('nx', np.uint16), # ('ny', np.uint16), # ('offset', np.uint32), # ('values', dataType, shape)]) # stream.seek(0) # #Initialize values and time arrays # frameList = range(startFrame, lastFrame, skipFrames) # data["values"] = np.zeros((len(frameList), header['nbands'], header['numpatches'] # for frame in frameList: # stream.seek(frame*frameSize) # time = np.fromfile(stream,extendedHeaderPattern,1)['time'][0] # data.append(DataFrame(time,[])) # for arbor in range(header['nbands']): # currentData = np.fromfile(stream, # patchPattern, # header['numpatches'])['values'] # data[frame].values.append(np.squeeze(np.transpose(currentData, # [2,1,3,0]))) # if progressPeriod: # if not frame % progressPeriod and frame: # print("File "+filename+": frame "+str(frame)+" of "+str(lastFrame)) # return PV_Object(data,header) assert (0) # DENSE, NON-SPIKING ACTIVITY FILE #Tested as of 2/15/16 elif header['filetype'] == 4: lastFrame = min(lastFrame, header['nbands']) shape = (header['ny'], header['nx'], header['nf']) pattern = np.dtype([('time', np.float64), ('values', dataType, shape)]) frameSize = pattern.itemsize frameRange = range(startFrame, lastFrame, skipFrames) data["values"] = np.zeros( (len(frameRange), header['ny'], header['nx'], header['nf'])) data["time"] = np.zeros((len(frameRange))) for (i, frame) in enumerate(frameRange): stream.seek(header['headersize'] + frame * frameSize) currentData = np.fromfile(stream, pattern, 1) time = currentData['time'][0] values = currentData['values'][0] data["time"][i] = time data["values"][i, :, :, :] = values if progressPeriod: if not i % progressPeriod and frame: print("File " + filename + ": frame " + str(frame) + " of " + str(lastFrame)) return data # KERNEL WEIGHT FILE #TODO test elif header['filetype'] == 5: fileSize = os.path.getsize(filename) frameSize = header['recordsize'] * header['nbands'] + header[ 'headersize'] lastFrame = min(lastFrame, fileSize / frameSize) shape = (header['nxp'], header['nyp'], header['nfp']) patchPattern = np.dtype([('nx', np.uint16), ('ny', np.uint16), ('offset', np.uint32), ('values', dataType, shape)]) stream.seek(0) frameRange = range(startFrame, lastFrame, skipFrames) data["values"] = np.zeros( (len(frameRange), header['nbands'], header['numpatches'], header['nyp'], header['nxp'], header['nfp'])) data["time"] = np.zeros((len(frameRange))) for (i, frame) in enumerate(frameRange): stream.seek(frame * frameSize) time = np.fromfile(stream, extendedHeaderPattern, 1)['time'][0] data["time"][i] = time for arbor in range(header['nbands']): currentData = np.fromfile(stream, patchPattern, header['numpatches'])['values'] data["values"][i, arbor, :, :, :, :] = currentData if progressPeriod: if not i % progressPeriod and frame: print("File " + filename + ": frame " + str(frame) + " of " + str(lastFrame)) return data # SPARSE ACTIVITY FILE #TODO elif header['filetype'] == 6: lastFrame = min(lastFrame, header['nbands']) entryPattern = np.dtype([('index', np.int32), ('activation', np.float32)]) valuesList = [] framesList = [] idxList = [] timeList = [] validFrames = range(startFrame, lastFrame, skipFrames) frameNum = 0 for frame in range(lastFrame): if frame in validFrames: time = np.fromfile(stream, np.float64, 1)[0] timeList.append(time) numActive = np.fromfile(stream, np.uint32, 1) currentData = np.fromfile(stream, entryPattern, numActive) dataIdx = currentData['index'] dataValues = currentData['activation'] idxList.extend(dataIdx) valuesList.extend(dataValues) framesList.extend(np.ones((len(dataIdx))) * frameNum) frameNum += 1 else: numActive = np.fromfile(stream, np.uint32, 3)[-1] stream.seek(entryPattern.itemsize * numActive, os.SEEK_CUR) continue #data.append(DataFrame(time,zip(currentData['index'], # currentData['activation']))) if progressPeriod: if not frame % progressPeriod and frame: print("File " + filename + ": frame " + str(frame) + " of " + str(lastFrame)) #Make coosparsematrix data["time"] = np.array(timeList) data["values"] = sp.coo_matrix( (valuesList, (framesList, idxList)), shape=(frameNum, header["nx"] * header["ny"] * header["nf"])) return data