def test_block_verify(self): # check that file verifies against itself for single block filename = self.get_filename_path( 'Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs') data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) hdr0 = NlxHeader(filename) nb0 = NcsSectionsFactory.build_for_ncs_file(data0, hdr0) self.assertTrue(NcsSectionsFactory._verifySectionsStructure( data0, nb0)) # check that fails against file with two blocks filename = self.get_filename_path( 'BML_unfilledsplit/original_data/unfilledSplitRecords.Ncs') data1 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) hdr1 = NlxHeader(filename) nb1 = NcsSectionsFactory.build_for_ncs_file(data1, hdr1) self.assertFalse( NcsSectionsFactory._verifySectionsStructure(data1, nb0)) # check that two blocks verify against self self.assertTrue(NcsSectionsFactory._verifySectionsStructure( data1, nb1))
def test_build_using_header_and_scanning(self): # Test early files where the frequency listed in the header is # floor(1e6/(actual number of microseconds between samples) filename = self.get_filename_path( 'Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs') hdr = NlxHeader(filename) data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) nb = NcsSectionsFactory.build_for_ncs_file(data0, hdr) self.assertEqual(nb.sampFreqUsed, 1 / 35e-6) self.assertEqual(nb.microsPerSampUsed, 35) self.assertEqual(len(nb.sects), 1) self.assertEqual(nb.sects[0].startRec, 0) self.assertEqual(nb.sects[0].endRec, 9) # test Cheetah 5.5.1, which is DigitalLynxSX and has two blocks of records # with a fairly large gap filename = self.get_filename_path( 'Cheetah_v5.5.1/original_data/Tet3a.ncs') hdr = NlxHeader(filename) data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) nb = NcsSectionsFactory.build_for_ncs_file(data0, hdr) self.assertEqual(nb.sampFreqUsed, 32000) self.assertEqual(nb.microsPerSampUsed, 31.25) self.assertEqual(len(nb.sects), 2) self.assertListEqual([blk.startRec for blk in nb.sects], [0, 2498]) self.assertListEqual([blk.endRec for blk in nb.sects], [2497, 3331])
def test_block_start_and_end_times(self): # digitallynxsx version to exercise the _parseForMaxGap function with multiple blocks filename = self.get_local_path( 'neuralynx/Cheetah_v6.3.2/incomplete_blocks/CSC1_reduced.ncs') data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) hdr = NlxHeader(filename) nb = NcsSectionsFactory.build_for_ncs_file(data0, hdr) self.assertListEqual([blk.startTime for blk in nb.sects], [8408806811, 8427832053, 8487768561]) self.assertListEqual([blk.endTime for blk in nb.sects], [8427831990, 8487768498, 8515816549]) # digitallynxsx with single block of records to exercise path in _buildForMaxGap filename = self.get_local_path( 'neuralynx/Cheetah_v1.1.0/original_data/CSC67_trunc.Ncs') data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) hdr = NlxHeader(filename) nb = NcsSectionsFactory.build_for_ncs_file(data0, hdr) self.assertEqual(len(nb.sects), 1) self.assertEqual(nb.sects[0].startTime, 253293161778) self.assertEqual(nb.sects[0].endTime, 253293349278) # PRE4 version with single block of records to exercise path in _buildGivenActualFrequency filename = self.get_local_path( 'neuralynx/Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs') data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) hdr = NlxHeader(filename) nb = NcsSectionsFactory.build_for_ncs_file(data0, hdr) self.assertEqual(len(nb.sects), 1) self.assertEqual(nb.sects[0].startTime, 266982936) self.assertEqual(nb.sects[0].endTime, 267162136) # BML style with two blocks of records and one partially filled record to exercise # _parseGivenActualFrequency filename = self.get_local_path( 'neuralynx/BML_unfilledsplit/original_data/unfilledSplitRecords.Ncs' ) data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) hdr = NlxHeader(filename) nb = NcsSectionsFactory.build_for_ncs_file(data0, hdr) self.assertEqual(len(nb.sects), 2) self.assertListEqual([blk.startTime for blk in nb.sects], [1837623129, 6132625241]) self.assertListEqual([blk.endTime for blk in nb.sects], [1837651009, 6132642649])
def test_build_given_actual_frequency(self): # Test early files where the frequency listed in the header is # floor(1e6/(actual number of microseconds between samples) filename = self.get_filename_path( 'Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs') data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) ncsBlocks = NcsSections() ncsBlocks.sampFreqUsed = 1 / (35e-6) ncsBlocks.microsPerSampUsed = 35 ncsBlocks = NcsSectionsFactory._buildGivenActualFrequency( data0, ncsBlocks.sampFreqUsed, 27789) self.assertEqual(len(ncsBlocks.sects), 1) self.assertEqual(ncsBlocks.sects[0].startRec, 0) self.assertEqual(ncsBlocks.sects[0].endRec, 9)
def test_ncsblocks_partial(self): filename = self.get_filename_path( 'Cheetah_v6.3.2/incomplete_blocks/CSC1_reduced.ncs') data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) self.assertEqual(data0.shape[0], 6690) self.assertEqual(data0['timestamp'][6689], 8515800549) # timestamp of last record hdr = NlxHeader(filename) nb = NcsSectionsFactory.build_for_ncs_file(data0, hdr) self.assertEqual(nb.sampFreqUsed, 32000.012813673042) self.assertEqual(nb.microsPerSampUsed, 31.249987486652431) self.assertListEqual([blk.startRec for blk in nb.sects], [0, 1190, 4937]) self.assertListEqual([blk.endRec for blk in nb.sects], [1189, 4936, 6689])
def scan_ncs_files(self, ncs_filenames): """ Given a list of ncs files, read their basic structure. PARAMETERS: ------ ncs_filenames - list of ncs filenames to scan. RETURNS: ------ memmaps [ {} for seg_index in range(self._nb_segment) ][chan_uid] seg_time_limits SegmentTimeLimits for sections in scanned Ncs files Files will be scanned to determine the sections of records. If file is a single section of records, this scan is brief, otherwise it will check each record which may take some time. """ # :TODO: Needs to account for gaps and start and end times potentially # being different in different groups of channels. These groups typically # correspond to the channels collected by a single ADC card. if len(ncs_filenames) == 0: return None, None # Build dictionary of chan_uid to associated NcsSections, memmap and NlxHeaders. Only # construct new NcsSections when it is different from that for the preceding file. chanSectMap = dict() for chan_uid, ncs_filename in self.ncs_filenames.items(): data = np.memmap(ncs_filename, dtype=self._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) nlxHeader = NlxHeader(ncs_filename) if not chanSectMap or ( chanSectMap and not NcsSectionsFactory._verifySectionsStructure( data, lastNcsSections)): lastNcsSections = NcsSectionsFactory.build_for_ncs_file( data, nlxHeader) chanSectMap[chan_uid] = [lastNcsSections, nlxHeader, data] # Construct an inverse dictionary from NcsSections to list of associated chan_uids revSectMap = dict() for k, v in chanSectMap.items(): revSectMap.setdefault(v[0], []).append(k) # If there is only one NcsSections structure in the set of ncs files, there should only # be one entry. Otherwise this is presently unsupported. if len(revSectMap) > 1: raise IOError( 'ncs files have {} different sections structures. Unsupported.' .format(len(revSectMap))) seg_time_limits = SegmentTimeLimits(nb_segment=len( lastNcsSections.sects), t_start=[], t_stop=[], length=[], timestamp_limits=[]) memmaps = [{} for seg_index in range(seg_time_limits.nb_segment)] # create segment with subdata block/t_start/t_stop/length for each channel for i, fileEntry in enumerate(self.ncs_filenames.items()): chan_uid = fileEntry[0] data = chanSectMap[chan_uid][2] # create a memmap for each record section of the current file curSects = chanSectMap[chan_uid][0] for seg_index in range(len(curSects.sects)): curSect = curSects.sects[seg_index] subdata = data[curSect.startRec:(curSect.endRec + 1)] memmaps[seg_index][chan_uid] = subdata # create segment timestamp limits based on only NcsSections structure in use if i == 0: numSampsLastSect = subdata[-1]['nb_valid'] ts0 = subdata[0]['timestamp'] ts1 = NcsSectionsFactory.calc_sample_time( curSects.sampFreqUsed, subdata[-1]['timestamp'], numSampsLastSect) seg_time_limits.timestamp_limits.append((ts0, ts1)) t_start = ts0 / 1e6 seg_time_limits.t_start.append(t_start) t_stop = ts1 / 1e6 seg_time_limits.t_stop.append(t_stop) # :NOTE: This should really be the total of nb_valid in records, but this # allows the last record of a section to be shorter, the most common case. # Have never seen a section of records with not full records before the last. length = (subdata.size - 1) * NcsSection._RECORD_SIZE + numSampsLastSect seg_time_limits.length.append(length) return memmaps, seg_time_limits