Пример #1
0
    def __init__(self, filename):
        # load entire into memory temporarly
        with open(filename, "rb") as fin:
            content = fin.read()
            #print ("Read raw data")

        self.length = len(content)
        #print (self.length)
        # extract first two bytes to determine file type version
        self.ftflg, self.fversn = struct.unpack('<cc'.encode('utf8'), content[:2])
        # --------------------------------------------
        # NEW FORMAT (LSB)
        # --------------------------------------------
        if self.fversn == b'\x4b':
            # format: new LSB 1st
            # -------------
            # unpack header
            # -------------
            # use little-endian format with standard sizes
            # use naming scheme in SPC.H header file
            self.ftflg, \
                self.fversn, \
                self.fexper, \
                self.fexp, \
                self.fnpts, \
                self.ffirst, \
                self.flast, \
                self.fnsub, \
                self.fxtype, \
                self.fytype, \
                self.fztype, \
                self.fpost, \
                self.fdate, \
                self.fres, \
                self.fsource, \
                self.fpeakpt, \
                self.fspare, \
                self.fcmnt, \
                self.fcatxt, \
                self.flogoff, \
                self.fmods, \
                self.fprocs, \
                self.flevel, \
                self.fsampin, \
                self.ffactor, \
                self.fmethod, \
                self.fzinc, \
                self.fwplanes, \
                self.fwinc, \
                self.fwtype, \
                self.freserv \
                = struct.unpack(self.head_str.encode('utf8'), content[:self.head_siz])

            # Flag bits
            self.tsprec, \
                self.tcgram, \
                self.tmulti, \
                self.trandm, \
                self.tordrd, \
                self.talabs, \
                self.txyxys, \
                self.txvals = flag_bits(self.ftflg)[::-1]

            # fix data types if necessary
            self.fnpts = int(self.fnpts)  # of points should be int
            self.fexp = ord(self.fexp)

            self.ffirst = float(self.ffirst)
            self.flast = float(self.flast)

            self.flogoff = int(self.flogoff)  # byte; should be int

            self.fxtype = ord(self.fxtype)
            self.fytype = ord(self.fytype)
            self.fztype = ord(self.fztype)

            self.fexper = ord(self.fexper)
            self.fcmnt = str(self.fcmnt)

            # Convert date time to appropriate format
            d = self.fdate
            self.year = d >> 20
            self.month = (d >> 16) % (2**4)
            self.day = (d >> 11) % (2**5)
            self.hour = (d >> 6) % (2**5)
            self.minute = d % (2**6)

            # null terminated string, replace null characters with spaces
            # split and join to remove multiple spaces
            try:
                self.cmnt = ' '.join((self.fcmnt.replace('\x00', ' ')).split())
            except:
                self.cmnt = self.fcmnt

            # figure out type of file
            if self.fnsub > 1:
                self.dat_multi = True

            if self.txyxys:
                # x values are given
                self.dat_fmt = '-xy'
            elif self.txvals:
                # only one subfile, which contains the x data
                self.dat_fmt = 'x-y'
            else:
                # no x values are given, but they can be generated
                self.dat_fmt = 'gx-y'

            #print('{}({})'.format(self.dat_fmt, self.fnsub))
            #print ("first of his kind")

            sub_pos = self.head_siz
            #print("sub pos:")
            #print (sub_pos)

            if not self.txyxys:
                print ("not txyxys")
                # txyxys don't have global x data
                if self.txvals:
                    # if global x data is given
                    x_dat_pos = self.head_siz
                    x_dat_end = self.head_siz + (4 * self.fnpts)
                    self.x = np.array(
                        [struct.unpack_from(
                            'f', content[x_dat_pos:x_dat_end], 4 * i)[0]
                            for i in range(0, self.fnpts)])
                    sub_pos = x_dat_end
                else:
                    # otherwise generate them
                    self.x = np.linspace(self.ffirst, self.flast, num=self.fnpts)

            # make a list of subfiles
            self.sub = []
            print("list of subfiles")
            

            # if subfile directory is given
            if self.dat_fmt == '-xy' and self.fnpts > 0:
                print ("subfile directory is given")
                self.directory = True
                # loop over entries in directory
                for i in range(0, self.fnsub):
                    ssfposn, ssfsize, ssftime = struct.unpack(
                        '<iif'.encode('utf8'), 
                        content[self.fnpts + (i * 12):self.fnpts + ((i + 1) * 12)])
                    # add sufile, load defaults for npts and exp
                    self.sub.append(subFile(content[ssfposn:ssfposn + ssfsize], 0, 0, True, self.tsprec))
                #print (self.sub)

            else:
                # don't have directory, for each subfile
                print("subfile directory is not given")
                for i in range(self.fnsub):
                    # figure out its size
                    if self.txyxys:
                        # use points in subfile
                        print("using points in subfile")
                        subhead_lst = read_subheader(content[sub_pos:(sub_pos + 32)])
                        print("subheader list:", subhead_lst)
                        pts = subhead_lst[6]
                        # 4 bytes each for x and y, and 32 for subheader
                        dat_siz = (8 * pts) + 32
                        print("data size:", dat_siz)

                    else:
                        # use global points
                        
                        pts = self.fnpts
                        dat_siz = (4 * pts) + 32
                        print("using global points:", pts)
                        print ("data size:", dat_siz)

                    sub_end = sub_pos + dat_siz
                    #print("sub position:", sub_pos)
                    #print ("content:", content[sub_pos:sub_end])
                    # read into object, add to list
                    self.sub.append(subFile(content[sub_pos:sub_end],
                                            self.fnpts, self.fexp, self.txyxys, self.tsprec))
                    # update positions
                    sub_pos = sub_end
                #print (self.sub)

            # if log data exists
            # flog offset to log data offset not zero (bytes)
            if self.flogoff:
                log_head_end = self.flogoff + self.log_siz
                self.logsizd, \
                    self.logsizm, \
                    self.logtxto, \
                    self.logbins, \
                    self.logdsks, \
                    self.logspar \
                    = struct.unpack(self.logstc_str.encode('utf8'),
                                    content[self.flogoff:log_head_end])
                log_pos = self.flogoff + self.logtxto

                log_end_pos = log_pos + self.logsizd

                # line endings: get rid of any '\r' and then split on '\n'
                self.log_content = content[log_pos:log_end_pos].replace(b'\r', b'').split(b'\n')

                # split log data into dictionary based on =
                self.log_dict = dict()
                self.log_other = []  # put the rest into a list
                for x in self.log_content:
                    if x.find(b'=') >= 0:
                        # stop it from breaking if there is more than 1 =
                        key, value = x.split(b'=')[:2]
                        self.log_dict[key] = value
                    else:
                        self.log_other.append(x)

            # spacing between data
            self.spacing = (self.flast - self.ffirst) / (self.fnpts - 1)

            # call functions
            self.set_labels()
            self.set_exp_type()

        # --------------------------------------------
        # NEW FORMAT (MSB)
        # --------------------------------------------
        elif self.fversn == b'\x4c':
            # new MSB 1st
            print("New MSB 1st, yet to be implemented")
            pass  # To be implemented

        # --------------------------------------------
        # OLD FORMAT
        # --------------------------------------------
        elif self.fversn == b'\x4d':
            # old format
            # oxtype -> fxtype
            # oytype -> fytype
            self.oftflgs, \
                self.oversn, \
                self.oexp, \
                self.onpts, \
                self.ofirst, \
                self.olast, \
                self.fxtype, \
                self.fytype, \
                self.oyear, \
                self.omonth, \
                self.oday, \
                self.ohour, \
                self.ominute, \
                self.ores, \
                self.opeakpt, \
                self.onscans, \
                self.ospare, \
                self.ocmnt, \
                self.ocatxt, \
                self.osubh1 = struct.unpack(self.old_head_str.encode('utf8'),
                                            content[:self.old_head_siz])

            # Flag bits (assuming same)
            self.tsprec, \
                self.tcgram, \
                self.tmulti, \
                self.trandm, \
                self.tordrd, \
                self.talabs, \
                self.txyxys, \
                self.txvals = flag_bits(self.oftflgs)[::-1]

            # fix data types
            self.oexp = int(self.oexp)
            self.onpts = int(self.onpts)  # can't have floating num of pts
            self.ofirst = float(self.ofirst)
            self.olast = float(self.olast)

            # Date information
            # !! to fix !!
            # Year collected (0=no date/time) - MSB 4 bits are Z type

            # extracted as characters, using ord
            self.omonth = ord(self.omonth)
            self.oday = ord(self.oday)
            self.ohour = ord(self.ohour)
            self.ominute = ord(self.ominute)

            # number of scans (? subfiles sometimes ?)
            self.onscans = int(self.onscans)

            # null terminated strings
            self.ores = self.ores.split(b'\x00')[0]
            self.ocmnt = self.ocmnt.split(b'\x00')[0]

            # can it have separate x values ?
            self.x = np.linspace(self.ofirst, self.olast, num=self.onpts)

            # make a list of subfiles
            self.sub = []

            # already have subheader from main header, retrace steps
            sub_pos = self.old_head_siz - self.subhead_siz

            # for each subfile
            # in the old format we don't know how many subfiles to expect,
            # just looping till we run out
            i = 0
            while True:
                try:
                    # read in subheader
                    subhead_lst = read_subheader(content[sub_pos:sub_pos + self.subhead_siz])
                    print ("subhead list:")
                    print (subhead_lst)
                    
                    if subhead_lst[6] > 0:
                        # default to subfile points, unless it is zero
                        pts = subhead_lst[6]
                    else:
                        pts = self.onpts

                    # figure out size of subheader
                    dat_siz = (4 * pts)
                    sub_end = sub_pos + self.subhead_siz + dat_siz

                    # read into object, add to list
                    # send it pts since we have already figured that out
                    self.sub.append(subFileOld(
                        content[sub_pos:sub_end], pts, self.oexp, self.txyxys))
                    # update next subfile postion, and index
                    sub_pos = sub_end

                    i += 1
                except:
                    # zero indexed, set the total number of subfile
                    self.fnsub = i + 1
                    break

            # assuming it can't have separate x values
            self.dat_fmt = 'gx-y'
            
            print('{}({})'.format(self.dat_fmt, self.fnsub))

            self.fxtype = ord(self.fxtype)
            self.fytype = ord(self.fytype)
            # need to find from year apparently
            self.fztype = 0
            self.set_labels()

        # --------------------------------------------
        # SHIMADZU
        # --------------------------------------------
        elif self.fversn == b'\xcf':
            #print("Highly experimental format, may not work ")
            raw_data = content[10240:]  # data starts here (maybe every time)
            # spacing between y and x data is atleast 0 bytes
            s_32 = chr(int('0', 2)) * 32
            s_8 = chr(int('0', 2)) * 8  # zero double
            dat_len = raw_data.find(s_32)
            for i in range(dat_len, len(raw_data), 8):
                # find first non zero double
                if raw_data[i:i + 8] != s_8:
                    break
            dat_siz = int(dat_len / 8)
            self.y = struct.unpack(('<' + dat_siz * 'd').encode('utf8'), raw_data[:dat_len])
            self.x = struct.unpack(('<' + dat_siz * 'd').encode('utf8'), raw_data[i:i + dat_len])

        else:
            print("File type %s not supported yet. Please add issue. "
                  % hex(ord(self.fversn)))
            self.content = content
Пример #2
0
Файл: spc.py Проект: Ahmsak/spc
    def __init__(self, filename):
        # load file
        with open(filename, "rb") as fin:
            content = fin.read()
            print "Read raw data"
            
            
        # extract first two bytes to determine file type version
        
        ftflg, fversn = struct.unpack('<cc',content[:2])
        
        if fversn == 'K': # new LSB 1st
            print "New LSB 1st"
            # unpack header  
            # -------------
            # use little-endian format with standard sizes 
            # use naming scheme in SPC.H header file
            self.ftflg, \
                self.fversn, \
                self.fexper, \
                self.fexp, \
                self.fnpts, \
                self.ffirst, \
                self.flast, \
                self.fnsub, \
                self.fxtype, \
                self.fytype, \
                self.fztype, \
                self.fpost, \
                self.fdate, \
                self.fres, \
                self.fsource, \
                self.fpeakpt, \
                self.fspare, \
                self.fcmnt, \
                self.fcatxt, \
                self.flogoff, \
                self.fmods, \
                self.fprocs, \
                self.flevel, \
                self.fsampin, \
                self.ffactor, \
                self.fmethod, \
                self.fzinc, \
                self.fwplanes, \
                self.fwinc, \
                self.fwtype, \
                self.freserv \
                = struct.unpack(self.head_str, content[:self.head_siz])

            # Flag bits
            self.tsprec, \
                self.tcgram, \
                self.tmulti, \
                self.trandm, \
                self.tordrd, \
                self.talabs, \
                self.txyxys, \
                self.txvals = flag_bits(self.ftflg)[::-1]

            # fix data types if necessary
            
            self.fnpts = int(self.fnpts) # #of points should be int
            self.fexp = ord(self.fexp)
            
            self.ffirst = float(self.ffirst)
            self.flast = float(self.flast)
            
            self.flogoff = int(self.flogoff) # byte; should be int
            
            self.fxtype = ord(self.fxtype)
            self.fytype = ord(self.fytype)
            self.fztype = ord(self.fztype)
            
            self.fexper = ord(self.fexper)
            
            # Convert date time to appropriate format
            d = self.fdate
            self.year = d >> 20
            self.month = (d >> 16) % (2**4)
            self.day = (d >> 11) % (2**5)
            self.hour = (d >> 6) % (2**5)
            self.minute = d % (2**6)
            
            # null terminated string
            self.fcmnt = str(self.fcmnt).split('\x00')[0]
            
            print "\nHEADER"

            # options
            # -------    

            sub_pos = self.head_siz
            
            # optional floating point x-values
            if self.txvals: 
                #print "Seperate x-values"
                
                if self.txyxys:
                    print "x-data in subfile"
                else:
                    x_dat_pos = self.head_siz
                    x_dat_end = self.head_siz + (4 * self.fnpts)
                    self.x = np.array([struct.unpack_from('f', content[x_dat_pos:x_dat_end], 4 * i)[0]
                                       for i in range(0, self.fnpts)])
                    sub_pos = x_dat_end
                    print "Read global x-data"
            else:
                print "Generated x-values"
                self.x = np.linspace(self.ffirst,self.flast,num=self.fnpts)
            
            # make a list of subfiles          
            self.sub = []
            
            # for each subfile
            for i in range(self.fnsub):
                print "\nSUBFILE", i, "\n----------"
                #print "start pos", sub_pos
                
                # figure out its size
                subhead_lst = read_subheader(content[sub_pos:(sub_pos+32)])
                #print subhead_lst
                if subhead_lst[6] > 0:
                    pts = subhead_lst[6]
                    print "Using subfile points"
                else:
                    pts = self.fnpts
                    print "Using global subpoints"
                    
                # if xvalues already set, should use that number of points
                # only necessary for f_xy.spc
                if self.fnpts > 0:
                    pts = self.fnpts
                    print "Using global subpoints"
                    
                #print "Points in subfile", pts
                    
                if self.txyxys:
                    dat_siz = (8*pts) + 32
                else:
                    dat_siz = (4*pts) + 32
                    
                #print "Data size", dat_siz
                    
                sub_end = sub_pos + dat_siz
                #print "sub_end", sub_end
                # read into object, add to list
                self.sub.append(subFile(content[sub_pos:sub_end], self.fnpts, self.fexp, self.txyxys))
                # print self.sub[i].y
                # update positions
                sub_pos = sub_end
                
            # flog offset to log data offset not zero (bytes)
            #print "log data position" , self.flogoff
            if self.flogoff: 
                print "Log data exists"
                log_head_end = self.flogoff + self.log_siz
                self.logsizd, \
                    self.logsizm, \
                    self.logtxto, \
                    self.logbins, \
                    self.logdsks, \
                    self.logspar \
                    = struct.unpack(self.logstc_str, content[self.flogoff:log_head_end])
                log_pos = self.flogoff + self.logtxto
                print "Offset to text", self.logtxto
                #print "log stuff", self.logsizd, self.logsizm
                log_end_pos = log_pos + self.logsizd
                self.log_content = content[log_pos:log_end_pos].split('\r\n')
                
                #print self.log_content
                # split log data into dictionary
                self.log_dict = dict()
                self.log_other = [] # put the rest into a list
                for x in self.log_content:
                    if x.find('=') >= 0:
                        key, value = x.split('=')
                        self.log_dict[key] = value
                    else:
                        self.log_other.append(x)

            # spacing between data
            self.pr_spacing = (self.flast-self.ffirst)/(self.fnpts-1)
            
            # call functions
            self.set_labels()
            self.set_exp_type()
            
            
        elif fversn == 'L': # new MSB 1st
            print "New MSB 1st, yet to be implemented"
            pass # To be implemented
            
            
        elif fversn == 'M': # old format
            print "Old Version"
            self.oftflgs, \
                self.oversn, \
                self.oexp, \
                self.onpts, \
                self.ofirst, \
                self.olast, \
                self.oxtype, \
                self.oytype, \
                self.oyear, \
                self.omonth, \
                self.oday, \
                self.ohour, \
                self.ominute, \
                self.ores, \
                self.opeakpt, \
                self.onscans, \
                self.ospare, \
                self.ocmnt, \
                self.ocatxt, \
                self.osubh1 = struct.unpack(self.old_head_str, content[:self.old_head_siz])
            
            # fix data types
            self.oexp = int(self.oexp)
            self.onpts = float(self.onpts)
            self.ofirst = float(self.ofirst)
            self.olast = float(self.olast)
            
            # Date information
            """self.oyear = int(self.oyear) # need to fix
            self.omonth = int(self.omonth)
            self.oday = int(self.oday)
            self.ohour = int(self.ohour)
            self.ominute = int(self.ominute)"""
            
            # number of scans (?subfiles?)
            self.onscans = int(self.onscans)
            
            # null terminated strings
            self.ores = str(self.ores).split('\x00')[0]
            self.ocmnt = str(self.ocmnt).split('\x00')[0]
            
            # !!! only works for single subfile as of now
            ## forcing 1 file 
            self.onsub = 1
            
            self.x = np.linspace(self.ofirst,self.olast,num=self.onpts)
            # make a list of subfiles          
            self.sub = []
           
            sub_pos = self.old_head_siz
            
            # make onpts integer
            self.onpts = int(self.onpts)
            
            # for each subfile
            for i in range(self.onsub):
                print "\nSUBFILE", i, "\n----------"
                print "start pos", sub_pos
                
                # figure out its size
                subhead_lst = read_subheader(self.osubh1)
                
                print subhead_lst
                if subhead_lst[6] > 0:
                    pts = subhead_lst[6]
                    print "Using subfile points"
                else:
                    pts = self.onpts
                    print "Using global subpoints"
                    
                # if xvalues already set, should use that number of points
                # only necessary for f_xy.spc
                if self.onpts > 0:
                    pts = self.onpts
                    print "Using global subpoints"
                    
                print "Points in subfile", pts
                dat_siz = (4*pts)
                    
                print "Data size", dat_siz
                    
                sub_end = sub_pos + dat_siz
                
                print "sub_end", sub_end
                
                # read into object, add to list
                
                print sub_pos, sub_end, self.onpts, self.oexp
                self.sub.append(subFileOld(content[sub_pos-32:sub_end], self.onpts, self.oexp, False))
                # print self.sub[i].y
                # update positions
                sub_pos = sub_end