def __init__(self, filename): # load entire into memory temporarly with open(filename, "rb") as fin: content = fin.read() #print ("Read raw data") self.length = len(content) #print (self.length) # extract first two bytes to determine file type version self.ftflg, self.fversn = struct.unpack('<cc'.encode('utf8'), content[:2]) # -------------------------------------------- # NEW FORMAT (LSB) # -------------------------------------------- if self.fversn == b'\x4b': # format: new LSB 1st # ------------- # unpack header # ------------- # use little-endian format with standard sizes # use naming scheme in SPC.H header file self.ftflg, \ self.fversn, \ self.fexper, \ self.fexp, \ self.fnpts, \ self.ffirst, \ self.flast, \ self.fnsub, \ self.fxtype, \ self.fytype, \ self.fztype, \ self.fpost, \ self.fdate, \ self.fres, \ self.fsource, \ self.fpeakpt, \ self.fspare, \ self.fcmnt, \ self.fcatxt, \ self.flogoff, \ self.fmods, \ self.fprocs, \ self.flevel, \ self.fsampin, \ self.ffactor, \ self.fmethod, \ self.fzinc, \ self.fwplanes, \ self.fwinc, \ self.fwtype, \ self.freserv \ = struct.unpack(self.head_str.encode('utf8'), content[:self.head_siz]) # Flag bits self.tsprec, \ self.tcgram, \ self.tmulti, \ self.trandm, \ self.tordrd, \ self.talabs, \ self.txyxys, \ self.txvals = flag_bits(self.ftflg)[::-1] # fix data types if necessary self.fnpts = int(self.fnpts) # of points should be int self.fexp = ord(self.fexp) self.ffirst = float(self.ffirst) self.flast = float(self.flast) self.flogoff = int(self.flogoff) # byte; should be int self.fxtype = ord(self.fxtype) self.fytype = ord(self.fytype) self.fztype = ord(self.fztype) self.fexper = ord(self.fexper) self.fcmnt = str(self.fcmnt) # Convert date time to appropriate format d = self.fdate self.year = d >> 20 self.month = (d >> 16) % (2**4) self.day = (d >> 11) % (2**5) self.hour = (d >> 6) % (2**5) self.minute = d % (2**6) # null terminated string, replace null characters with spaces # split and join to remove multiple spaces try: self.cmnt = ' '.join((self.fcmnt.replace('\x00', ' ')).split()) except: self.cmnt = self.fcmnt # figure out type of file if self.fnsub > 1: self.dat_multi = True if self.txyxys: # x values are given self.dat_fmt = '-xy' elif self.txvals: # only one subfile, which contains the x data self.dat_fmt = 'x-y' else: # no x values are given, but they can be generated self.dat_fmt = 'gx-y' #print('{}({})'.format(self.dat_fmt, self.fnsub)) #print ("first of his kind") sub_pos = self.head_siz #print("sub pos:") #print (sub_pos) if not self.txyxys: print ("not txyxys") # txyxys don't have global x data if self.txvals: # if global x data is given x_dat_pos = self.head_siz x_dat_end = self.head_siz + (4 * self.fnpts) self.x = np.array( [struct.unpack_from( 'f', content[x_dat_pos:x_dat_end], 4 * i)[0] for i in range(0, self.fnpts)]) sub_pos = x_dat_end else: # otherwise generate them self.x = np.linspace(self.ffirst, self.flast, num=self.fnpts) # make a list of subfiles self.sub = [] print("list of subfiles") # if subfile directory is given if self.dat_fmt == '-xy' and self.fnpts > 0: print ("subfile directory is given") self.directory = True # loop over entries in directory for i in range(0, self.fnsub): ssfposn, ssfsize, ssftime = struct.unpack( '<iif'.encode('utf8'), content[self.fnpts + (i * 12):self.fnpts + ((i + 1) * 12)]) # add sufile, load defaults for npts and exp self.sub.append(subFile(content[ssfposn:ssfposn + ssfsize], 0, 0, True, self.tsprec)) #print (self.sub) else: # don't have directory, for each subfile print("subfile directory is not given") for i in range(self.fnsub): # figure out its size if self.txyxys: # use points in subfile print("using points in subfile") subhead_lst = read_subheader(content[sub_pos:(sub_pos + 32)]) print("subheader list:", subhead_lst) pts = subhead_lst[6] # 4 bytes each for x and y, and 32 for subheader dat_siz = (8 * pts) + 32 print("data size:", dat_siz) else: # use global points pts = self.fnpts dat_siz = (4 * pts) + 32 print("using global points:", pts) print ("data size:", dat_siz) sub_end = sub_pos + dat_siz #print("sub position:", sub_pos) #print ("content:", content[sub_pos:sub_end]) # read into object, add to list self.sub.append(subFile(content[sub_pos:sub_end], self.fnpts, self.fexp, self.txyxys, self.tsprec)) # update positions sub_pos = sub_end #print (self.sub) # if log data exists # flog offset to log data offset not zero (bytes) if self.flogoff: log_head_end = self.flogoff + self.log_siz self.logsizd, \ self.logsizm, \ self.logtxto, \ self.logbins, \ self.logdsks, \ self.logspar \ = struct.unpack(self.logstc_str.encode('utf8'), content[self.flogoff:log_head_end]) log_pos = self.flogoff + self.logtxto log_end_pos = log_pos + self.logsizd # line endings: get rid of any '\r' and then split on '\n' self.log_content = content[log_pos:log_end_pos].replace(b'\r', b'').split(b'\n') # split log data into dictionary based on = self.log_dict = dict() self.log_other = [] # put the rest into a list for x in self.log_content: if x.find(b'=') >= 0: # stop it from breaking if there is more than 1 = key, value = x.split(b'=')[:2] self.log_dict[key] = value else: self.log_other.append(x) # spacing between data self.spacing = (self.flast - self.ffirst) / (self.fnpts - 1) # call functions self.set_labels() self.set_exp_type() # -------------------------------------------- # NEW FORMAT (MSB) # -------------------------------------------- elif self.fversn == b'\x4c': # new MSB 1st print("New MSB 1st, yet to be implemented") pass # To be implemented # -------------------------------------------- # OLD FORMAT # -------------------------------------------- elif self.fversn == b'\x4d': # old format # oxtype -> fxtype # oytype -> fytype self.oftflgs, \ self.oversn, \ self.oexp, \ self.onpts, \ self.ofirst, \ self.olast, \ self.fxtype, \ self.fytype, \ self.oyear, \ self.omonth, \ self.oday, \ self.ohour, \ self.ominute, \ self.ores, \ self.opeakpt, \ self.onscans, \ self.ospare, \ self.ocmnt, \ self.ocatxt, \ self.osubh1 = struct.unpack(self.old_head_str.encode('utf8'), content[:self.old_head_siz]) # Flag bits (assuming same) self.tsprec, \ self.tcgram, \ self.tmulti, \ self.trandm, \ self.tordrd, \ self.talabs, \ self.txyxys, \ self.txvals = flag_bits(self.oftflgs)[::-1] # fix data types self.oexp = int(self.oexp) self.onpts = int(self.onpts) # can't have floating num of pts self.ofirst = float(self.ofirst) self.olast = float(self.olast) # Date information # !! to fix !! # Year collected (0=no date/time) - MSB 4 bits are Z type # extracted as characters, using ord self.omonth = ord(self.omonth) self.oday = ord(self.oday) self.ohour = ord(self.ohour) self.ominute = ord(self.ominute) # number of scans (? subfiles sometimes ?) self.onscans = int(self.onscans) # null terminated strings self.ores = self.ores.split(b'\x00')[0] self.ocmnt = self.ocmnt.split(b'\x00')[0] # can it have separate x values ? self.x = np.linspace(self.ofirst, self.olast, num=self.onpts) # make a list of subfiles self.sub = [] # already have subheader from main header, retrace steps sub_pos = self.old_head_siz - self.subhead_siz # for each subfile # in the old format we don't know how many subfiles to expect, # just looping till we run out i = 0 while True: try: # read in subheader subhead_lst = read_subheader(content[sub_pos:sub_pos + self.subhead_siz]) print ("subhead list:") print (subhead_lst) if subhead_lst[6] > 0: # default to subfile points, unless it is zero pts = subhead_lst[6] else: pts = self.onpts # figure out size of subheader dat_siz = (4 * pts) sub_end = sub_pos + self.subhead_siz + dat_siz # read into object, add to list # send it pts since we have already figured that out self.sub.append(subFileOld( content[sub_pos:sub_end], pts, self.oexp, self.txyxys)) # update next subfile postion, and index sub_pos = sub_end i += 1 except: # zero indexed, set the total number of subfile self.fnsub = i + 1 break # assuming it can't have separate x values self.dat_fmt = 'gx-y' print('{}({})'.format(self.dat_fmt, self.fnsub)) self.fxtype = ord(self.fxtype) self.fytype = ord(self.fytype) # need to find from year apparently self.fztype = 0 self.set_labels() # -------------------------------------------- # SHIMADZU # -------------------------------------------- elif self.fversn == b'\xcf': #print("Highly experimental format, may not work ") raw_data = content[10240:] # data starts here (maybe every time) # spacing between y and x data is atleast 0 bytes s_32 = chr(int('0', 2)) * 32 s_8 = chr(int('0', 2)) * 8 # zero double dat_len = raw_data.find(s_32) for i in range(dat_len, len(raw_data), 8): # find first non zero double if raw_data[i:i + 8] != s_8: break dat_siz = int(dat_len / 8) self.y = struct.unpack(('<' + dat_siz * 'd').encode('utf8'), raw_data[:dat_len]) self.x = struct.unpack(('<' + dat_siz * 'd').encode('utf8'), raw_data[i:i + dat_len]) else: print("File type %s not supported yet. Please add issue. " % hex(ord(self.fversn))) self.content = content
def __init__(self, filename): # load file with open(filename, "rb") as fin: content = fin.read() print "Read raw data" # extract first two bytes to determine file type version ftflg, fversn = struct.unpack('<cc',content[:2]) if fversn == 'K': # new LSB 1st print "New LSB 1st" # unpack header # ------------- # use little-endian format with standard sizes # use naming scheme in SPC.H header file self.ftflg, \ self.fversn, \ self.fexper, \ self.fexp, \ self.fnpts, \ self.ffirst, \ self.flast, \ self.fnsub, \ self.fxtype, \ self.fytype, \ self.fztype, \ self.fpost, \ self.fdate, \ self.fres, \ self.fsource, \ self.fpeakpt, \ self.fspare, \ self.fcmnt, \ self.fcatxt, \ self.flogoff, \ self.fmods, \ self.fprocs, \ self.flevel, \ self.fsampin, \ self.ffactor, \ self.fmethod, \ self.fzinc, \ self.fwplanes, \ self.fwinc, \ self.fwtype, \ self.freserv \ = struct.unpack(self.head_str, content[:self.head_siz]) # Flag bits self.tsprec, \ self.tcgram, \ self.tmulti, \ self.trandm, \ self.tordrd, \ self.talabs, \ self.txyxys, \ self.txvals = flag_bits(self.ftflg)[::-1] # fix data types if necessary self.fnpts = int(self.fnpts) # #of points should be int self.fexp = ord(self.fexp) self.ffirst = float(self.ffirst) self.flast = float(self.flast) self.flogoff = int(self.flogoff) # byte; should be int self.fxtype = ord(self.fxtype) self.fytype = ord(self.fytype) self.fztype = ord(self.fztype) self.fexper = ord(self.fexper) # Convert date time to appropriate format d = self.fdate self.year = d >> 20 self.month = (d >> 16) % (2**4) self.day = (d >> 11) % (2**5) self.hour = (d >> 6) % (2**5) self.minute = d % (2**6) # null terminated string self.fcmnt = str(self.fcmnt).split('\x00')[0] print "\nHEADER" # options # ------- sub_pos = self.head_siz # optional floating point x-values if self.txvals: #print "Seperate x-values" if self.txyxys: print "x-data in subfile" else: x_dat_pos = self.head_siz x_dat_end = self.head_siz + (4 * self.fnpts) self.x = np.array([struct.unpack_from('f', content[x_dat_pos:x_dat_end], 4 * i)[0] for i in range(0, self.fnpts)]) sub_pos = x_dat_end print "Read global x-data" else: print "Generated x-values" self.x = np.linspace(self.ffirst,self.flast,num=self.fnpts) # make a list of subfiles self.sub = [] # for each subfile for i in range(self.fnsub): print "\nSUBFILE", i, "\n----------" #print "start pos", sub_pos # figure out its size subhead_lst = read_subheader(content[sub_pos:(sub_pos+32)]) #print subhead_lst if subhead_lst[6] > 0: pts = subhead_lst[6] print "Using subfile points" else: pts = self.fnpts print "Using global subpoints" # if xvalues already set, should use that number of points # only necessary for f_xy.spc if self.fnpts > 0: pts = self.fnpts print "Using global subpoints" #print "Points in subfile", pts if self.txyxys: dat_siz = (8*pts) + 32 else: dat_siz = (4*pts) + 32 #print "Data size", dat_siz sub_end = sub_pos + dat_siz #print "sub_end", sub_end # read into object, add to list self.sub.append(subFile(content[sub_pos:sub_end], self.fnpts, self.fexp, self.txyxys)) # print self.sub[i].y # update positions sub_pos = sub_end # flog offset to log data offset not zero (bytes) #print "log data position" , self.flogoff if self.flogoff: print "Log data exists" log_head_end = self.flogoff + self.log_siz self.logsizd, \ self.logsizm, \ self.logtxto, \ self.logbins, \ self.logdsks, \ self.logspar \ = struct.unpack(self.logstc_str, content[self.flogoff:log_head_end]) log_pos = self.flogoff + self.logtxto print "Offset to text", self.logtxto #print "log stuff", self.logsizd, self.logsizm log_end_pos = log_pos + self.logsizd self.log_content = content[log_pos:log_end_pos].split('\r\n') #print self.log_content # split log data into dictionary self.log_dict = dict() self.log_other = [] # put the rest into a list for x in self.log_content: if x.find('=') >= 0: key, value = x.split('=') self.log_dict[key] = value else: self.log_other.append(x) # spacing between data self.pr_spacing = (self.flast-self.ffirst)/(self.fnpts-1) # call functions self.set_labels() self.set_exp_type() elif fversn == 'L': # new MSB 1st print "New MSB 1st, yet to be implemented" pass # To be implemented elif fversn == 'M': # old format print "Old Version" self.oftflgs, \ self.oversn, \ self.oexp, \ self.onpts, \ self.ofirst, \ self.olast, \ self.oxtype, \ self.oytype, \ self.oyear, \ self.omonth, \ self.oday, \ self.ohour, \ self.ominute, \ self.ores, \ self.opeakpt, \ self.onscans, \ self.ospare, \ self.ocmnt, \ self.ocatxt, \ self.osubh1 = struct.unpack(self.old_head_str, content[:self.old_head_siz]) # fix data types self.oexp = int(self.oexp) self.onpts = float(self.onpts) self.ofirst = float(self.ofirst) self.olast = float(self.olast) # Date information """self.oyear = int(self.oyear) # need to fix self.omonth = int(self.omonth) self.oday = int(self.oday) self.ohour = int(self.ohour) self.ominute = int(self.ominute)""" # number of scans (?subfiles?) self.onscans = int(self.onscans) # null terminated strings self.ores = str(self.ores).split('\x00')[0] self.ocmnt = str(self.ocmnt).split('\x00')[0] # !!! only works for single subfile as of now ## forcing 1 file self.onsub = 1 self.x = np.linspace(self.ofirst,self.olast,num=self.onpts) # make a list of subfiles self.sub = [] sub_pos = self.old_head_siz # make onpts integer self.onpts = int(self.onpts) # for each subfile for i in range(self.onsub): print "\nSUBFILE", i, "\n----------" print "start pos", sub_pos # figure out its size subhead_lst = read_subheader(self.osubh1) print subhead_lst if subhead_lst[6] > 0: pts = subhead_lst[6] print "Using subfile points" else: pts = self.onpts print "Using global subpoints" # if xvalues already set, should use that number of points # only necessary for f_xy.spc if self.onpts > 0: pts = self.onpts print "Using global subpoints" print "Points in subfile", pts dat_siz = (4*pts) print "Data size", dat_siz sub_end = sub_pos + dat_siz print "sub_end", sub_end # read into object, add to list print sub_pos, sub_end, self.onpts, self.oexp self.sub.append(subFileOld(content[sub_pos-32:sub_end], self.onpts, self.oexp, False)) # print self.sub[i].y # update positions sub_pos = sub_end