Ejemplo n.º 1
0
    def read3(self, fileName=None, info=None, multiProc=False, channelList=None, convertAfterRead=True, filterChannelNames=False):
        """ Reads mdf 3.x file data and stores it in dict

        Parameters
        ----------------
        fileName : str, optional
            file name

        info : mdfinfo3.info3 class
            info3 class containing all MDF Blocks

        multiProc : bool
            flag to activate multiprocessing of channel data conversion

        channelList : list of str, optional
            list of channel names to be read
            If you use channelList, reading might be much slower but it will save you memory. Can be used to read big files

        convertAfterRead : bool, optional
            flag to convert channel after read, True by default
            If you use convertAfterRead by setting it to false, all data from channels will be kept raw, no conversion applied.
            If many float are stored in file, you can gain from 3 to 4 times memory footprint
            To calculate value from channel, you can then use method .getChannelData()
        """
        self.multiProc = multiProc
        if platform == 'win32':
            self.multiProc = False  # no multiprocessing for windows platform

        if self.fileName is None and info is not None:
            self.fileName = info.fileName
        elif fileName is not None:
            self.fileName = fileName

        if channelList is None:
            allChannel = True
        else:
            allChannel = False

        # Read information block from file
        if info is None:
            info = info3(self.fileName, None, self.filterChannelNames)

        # reads metadata
        try:
            comment = info['HDBlock']['TXBlock']['Text']
        except:
            comment = ''
        # converts date to be compatible with ISO8601
        day, month, year = info['HDBlock']['Date'].split(':')
        ddate = year + '-' + month + '-' + day
        self.add_metadata(author=info['HDBlock']['Author'], \
                organisation=info['HDBlock']['Organization'], \
                project=info['HDBlock']['ProjectName'], \
                subject=info['HDBlock']['Subject'], comment=comment, \
                    date=ddate, time=info['HDBlock']['Time'])

        try:
            fid = open(self.fileName, 'rb')
        except IOError:
            raise Exception('Can not find file ' + self.fileName)


        # Read data from file
        for dataGroup in info['DGBlock'].keys():
            if info['DGBlock'][dataGroup]['numberOfChannelGroups'] > 0:  # data exists
                # Pointer to data block
                pointerToData = info['DGBlock'][dataGroup]['pointerToDataRecords']
                buf = DATA(fid, pointerToData)

                for channelGroup in range(info['DGBlock'][dataGroup]['numberOfChannelGroups']):
                    temp = record(dataGroup, channelGroup)  # create record class
                    temp.loadInfo(info)  # load all info related to record

                    if temp.numberOfRecords != 0:  # continue if there are at least some records
                        buf.addRecord(temp)

                buf.read(channelList) # reads datablock potentially containing several channel groups

                for recordID in buf.keys():
                    if 'record' in buf[recordID]:
                        master_channel = buf[recordID]['record'].master['name']
                        if master_channel in self.keys():
                            master_channel += '_' + str(dataGroup)

                        channels = (c for c in buf[recordID]['record']
                                    if allChannel or c.name in channelList)

                        for chan in channels: # for each recordchannel
                            recordName = buf[recordID]['record'].recordToChannelMatching[chan.name]  # in case record is used for several channels
                            temp = buf[recordID]['data'].__getattribute__(str(recordName) + '_title')

                            if len(temp) != 0:
                                # Process concatenated bits inside uint8
                                if not chan.bitCount // 8.0 == chan.bitCount / 8.0:  # if channel data do not use complete bytes
                                    mask = int(pow(2, chan.bitCount) - 1)  # masks isBitUint8
                                    if chan.signalDataType in (0, 1, 9, 10, 13, 14):  # integers
                                        temp = right_shift(temp, chan.bitOffset)
                                        temp = bitwise_and(temp, mask)
                                    else:  # should not happen
                                        print('bit count and offset not applied to correct data type', file=stderr)

                                self.add_channel(dataGroup, chan.name, temp, \
                                        master_channel, \
                                        master_type=1, \
                                        unit=chan.unit, \
                                        description=chan.desc, \
                                        conversion=chan.conversion, \
                                        info=None)
                del buf
        fid.close()  # close file
        if convertAfterRead:
            self._convertAllChannel3()
Ejemplo n.º 2
0
    def read3(self, fileName=None, info=None, multiProc=False, channelList=None, convertAfterRead=True, filterChannelNames=False):
        """ Reads mdf 3.x file data and stores it in dict

        Parameters
        ----------------
        fileName : str, optional
            file name

        info : mdfinfo3.info3 class
            info3 class containing all MDF Blocks

        multiProc : bool
            flag to activate multiprocessing of channel data conversion

        channelList : list of str, optional
            list of channel names to be read
            If you use channelList, reading might be much slower but it will save you memory. Can be used to read big files

        convertAfterRead : bool, optional
            flag to convert channel after read, True by default
            If you use convertAfterRead by setting it to false, all data from channels will be kept raw, no conversion applied.
            If many float are stored in file, you can gain from 3 to 4 times memory footprint
            To calculate value from channel, you can then use method .getChannelData()
        """
        self.multiProc = multiProc
        if platform == 'win32':
            self.multiProc = False  # no multiprocessing for windows platform

        if self.fileName is None and info is not None:
            self.fileName = info.fileName
        elif fileName is not None:
            self.fileName = fileName

        if channelList is None:
            allChannel = True
        else:
            allChannel = False

        # Read information block from file
        if info is None:
            info = info3(self.fileName, None, self.filterChannelNames)

        # reads metadata
        try:
            comment = info['HDBlock']['TXBlock']['Text']
        except:
            comment = ''
        # converts date to be compatible with ISO8601
        day, month, year = info['HDBlock']['Date'].split(':')
        ddate = year + '-' + month + '-' + day
        self.add_metadata(author=info['HDBlock']['Author'], \
                organisation=info['HDBlock']['Organization'], \
                project=info['HDBlock']['ProjectName'], \
                subject=info['HDBlock']['Subject'], comment=comment, \
                    date=ddate, time=info['HDBlock']['Time'])

        try:
            fid = open(self.fileName, 'rb')
        except IOError:
            raise Exception('Can not find file ' + self.fileName)


        # Read data from file
        for dataGroup in info['DGBlock'].keys():
            if info['DGBlock'][dataGroup]['numberOfChannelGroups'] > 0:  # data exists
                # Pointer to data block
                pointerToData = info['DGBlock'][dataGroup]['pointerToDataRecords']
                buf = DATA(fid, pointerToData)

                for channelGroup in range(info['DGBlock'][dataGroup]['numberOfChannelGroups']):
                    temp = record(dataGroup, channelGroup)  # create record class
                    temp.loadInfo(info)  # load all info related to record

                    if temp.numberOfRecords != 0:  # continue if there are at least some records
                        buf.addRecord(temp)

                buf.read(channelList) # reads datablock potentially containing several channel groups

                for recordID in buf.keys():
                    if 'record' in buf[recordID]:
                        master_channel = buf[recordID]['record'].master['name']
                        if master_channel in self.keys():
                            master_channel += '_' + str(dataGroup)

                        channels = (c for c in buf[recordID]['record']
                                    if allChannel or c.name in channelList)

                        for chan in channels: # for each recordchannel
                            recordName = buf[recordID]['record'].recordToChannelMatching[chan.name]  # in case record is used for several channels
                            temp = buf[recordID]['data'].__getattribute__(str(recordName) + '_title')

                            if len(temp) != 0:
                                # Process concatenated bits inside uint8
                                if not chan.bitCount // 8.0 == chan.bitCount / 8.0:  # if channel data do not use complete bytes
                                    mask = int(pow(2, chan.bitCount) - 1)  # masks isBitUint8
                                    if chan.signalDataType in (0, 1, 9, 10, 13, 14):  # integers
                                        temp = right_shift(temp, chan.bitOffset)
                                        temp = bitwise_and(temp, mask)
                                    else:  # should not happen
                                        print('bit count and offset not applied to correct data type', file=stderr)

                                self.add_channel(dataGroup, chan.name, temp, \
                                        master_channel, \
                                        master_type=1, \
                                        unit=chan.unit, \
                                        description=chan.desc, \
                                        conversion=chan.conversion)
                del buf
        fid.close()  # close file
        if convertAfterRead:
            self._convertAllChannel3()
Ejemplo n.º 3
0
    def read3(self, fileName=None, info=None, multiProc=False, channelList=None, convertAfterRead=True, filterChannelNames=False):
        """ Reads mdf 3.x file data and stores it in dict

        Parameters
        ----------------
        fileName : str, optional
            file name

        info : mdfinfo3.info3 class
            info3 class containing all MDF Blocks

        multiProc : bool
            flag to activate multiprocessing of channel data conversion

        channelList : list of str, optional
            list of channel names to be read
            If you use channelList, reading might be much slower but it will save you memory. Can be used to read big files

        convertAfterRead : bool, optional
            flag to convert channel after read, True by default
            If you use convertAfterRead by setting it to false, all data from channels will be kept raw, no conversion applied.
            If many float are stored in file, you can gain from 3 to 4 times memory footprint
            To calculate value from channel, you can then use method .getChannelData()
        """
        self.multiProc = multiProc
        if platform == 'win32':
            self.multiProc = False  # no multiprocessing for windows platform

        if self.fileName is None and info is not None:
            self.fileName = info.fileName
        elif fileName is not None:
            self.fileName = fileName
            
        if channelList is None:
            allChannel = True
        else:
            allChannel = False
        
        # Read information block from file
        if info is None:
            info = info3(self.fileName, None, self.filterChannelNames)

        # reads metadata
        self.file_metadata['author'] = info['HDBlock']['Author']
        self.file_metadata['organisation'] = info['HDBlock']['Organization']
        self.file_metadata['project'] = info['HDBlock']['ProjectName']
        self.file_metadata['subject'] = info['HDBlock']['Subject']
        try:
            self.comment = info['HDBlock']['TXBlock']['Text']
        except:
            pass
        self.file_metadata['time'] = info['HDBlock']['Time']
        self.file_metadata['date'] = info['HDBlock']['Date']
        # converts date to be compatible with ISO8601
        day, month, year = self.file_metadata['date'].split(':')
        self.file_metadata['date'] = year + '-' + month + '-' + day

        try:
            fid = open(self.fileName, 'rb')
        except IOError:
            raise Exception('Can not find file ' + self.fileName)
        
        D = {}
        # Read data from file
        for dataGroup in info['DGBlock'].keys():
            if info['DGBlock'][dataGroup]['numberOfChannelGroups'] > 0:  # data exists
                # Pointer to data block
                pointerToData = info['DGBlock'][dataGroup]['pointerToDataRecords']
                D[dataGroup] = DATA(fid, pointerToData)

                for channelGroup in range(info['DGBlock'][dataGroup]['numberOfChannelGroups']):
                    temp = record(dataGroup, channelGroup)  # create record class
                    temp.loadInfo(info)  # load all info related to record

                    if temp.numberOfRecords != 0:  # continue if there are at least some records
                        D[dataGroup].addRecord(temp)

                D[dataGroup].read(channelList)

        fid.close()  # close file

        # After all processing of channels,
        # prepare final class data with all its keys
        for dataGroup in range(info['HDBlock']['numberOfDataGroups']):
            for channelGroup in range(info['DGBlock'][dataGroup]['numberOfChannelGroups']):
                recordID = info['CGBlock'][dataGroup][channelGroup]['recordID']
                numberOfRecords = info['CGBlock'][dataGroup][channelGroup]['numberOfRecords']
                if numberOfRecords != 0:
                    for channel in range(info['CGBlock'][dataGroup][channelGroup]['numberOfChannels']):
                        channelName = info['CNBlock'][dataGroup][channelGroup][channel]['signalName']
                        recordName = D[dataGroup][recordID]['record'].recordToChannelMatching[channelName]  # in case record is used for several channels
                        temp = D[dataGroup][recordID]['data'].__getattribute__(str(recordName) + '_title')
                        master_channel = 'master' + str(dataGroup)
                        if info['CNBlock'][dataGroup][channelGroup][channel]['channelType'] == 1:  # time channel
                            channelName = master_channel
                        if (allChannel or channelName in channelList) and len(temp) != 0:
                            if 'physicalUnit' in info['CCBlock'][dataGroup][channelGroup][channel]:
                                unitStr = info['CCBlock'][dataGroup][channelGroup][channel]['physicalUnit']
                            else:
                                unitStr = ''
                            
                            # Process concatenated bits inside uint8
                            bitCount = info['CNBlock'][dataGroup][channelGroup][channel]['numberOfBits']
                            if not bitCount // 8.0 == bitCount / 8.0:  # if channel data do not use complete bytes
                                mask = int(pow(2, bitCount) - 1)  # masks isBitUint8
                                if info['CNBlock'][dataGroup][channelGroup][channel]['signalDataType'] in (0, 1, 9, 10, 13, 14):  # integers
                                    bitOffset = info['CNBlock'][dataGroup][channelGroup][channel]['numberOfTheFirstBits'] % 8
                                    temp = right_shift(temp, bitOffset)
                                    temp = bitwise_and(temp, mask)
                                    #self[channelName]['data'] = temp
                                else:  # should not happen
                                    print('bit count and offset not applied to correct data type')

                            self.add_channel(channelName, temp, master_channel, \
                                    master_type=1, \
                                    unit=unitStr, \
                                    description=info['CNBlock'][dataGroup][channelGroup][channel]['signalDescription'], \
                                    conversion=info['CCBlock'][dataGroup][channelGroup][channel])
            del D[dataGroup]

        if convertAfterRead:
            self._convertAllChannel3()