def _load(self, filename, *args, **kargs): """Sheffield Focussed MOKE file loader routine. Args: filename (string or bool): File to load. If None then the existing filename is used, if False, then a file dialog will be used. Returns: A copy of the itself after loading the data. """ if filename is None or not filename: self.get_filename("r") else: self.filename = filename with io.open(self.filename, mode="rb") as f: try: value = [float(x.strip()) for x in bytes2str(f.readline()).split("\t")] except Exception: f.close() raise Core.StonerLoadError("Not an FMOKE file?") label = [x.strip() for x in bytes2str(f.readline()).split("\t")] if label[0] != "Header:": f.close() raise Core.StonerLoadError("Not a Focussed MOKE file !") del label[0] for k, v in zip(label, value): self.metadata[k] = v # Create metatdata from first 2 lines column_headers = [x.strip() for x in bytes2str(f.readline()).split("\t")] self.data = np.genfromtxt(f, dtype="float", delimiter="\t", invalid_raise=False) self.column_headers = column_headers return self
def _load(self, filename=None, *args, **kargs): """HyperSpy Loader file loader routine. Args: filename (string or bool): File to load. If None then the existing filename is used, if False, then a file dialog will be used. Returns: A copy of the itself after loading the data. """ if filename is None or not filename: self.get_filename("r") else: self.filename = filename # Open the file and read the main file header and unpack into a dict try: signal = hs.load(self.filename) if not isinstance(signal, hs.signals.Signal2D): raise Core.StonerLoadError( "Not a 2D signal object - aborting!") except Exception as e: # Pretty generic error catcher raise Core.StonerLoadError( "Not readable by HyperSpy error was {}".format(e)) self.data = signal.data self._unpack_meta("", signal.metadata.as_dictionary()) self._unpack_axes(signal.axes_manager) return self
def _load(self, filename, *args, **kargs): """File loader for PinkLib. Args: filename (string or bool): File to load. If None then the existing filename is used, if False, then a file dialog will be used. Returns: A copy of the itself after loading the data. """ if filename is None or not filename: self.get_filename("r") else: self.filename = filename ix = 0 with io.open(self.filename, "r", errors="ignore", encoding="utf-8") as f: # Read filename linewise if not re.compile(r"\d{1,2}/\d{1,2}/\d{4}").match(f.readline()): raise Core.StonerLoadError( "Not a BirgeIVFile as no date on first line") data = f.readlines() expected = ["Vo(-))", "Vo(+))", "Ic(+)", "Ic(-)"] for l, m in zip(data[-4:], expected): if not l.startswith(m): raise Core.StonerLoadError( "Not a BirgeIVFile as wrong footer line") key = l[:len(m)] val = l[len(m):] if "STDEV" in val: ix2 = val.index("STDEV") key2 = val[ix2:ix2 + 4 + len(key)] val2 = val[ix2 + 4 + len(key):] self.metadata[key2] = string_to_type(val2.strip()) val = val[:ix2] self.metadata[key] = string_to_type(val.strip()) for ix, line in enumerate( data): # Scan the ough lines to get metadata if ":" in line: parts = line.split(":") self.metadata[parts[0].strip()] = string_to_type( parts[1].strip()) elif "," in line: for part in line.split(","): parts = part.split(" ") self.metadata[parts[0].strip()] = string_to_type( parts[1].strip()) elif line.startswith("H "): self.metadata["H"] = string_to_type( line.split(" ")[1].strip()) else: headers = [x.strip() for x in line.split(" ")] break else: raise Core.StonerLoadError("Oops ran off the end of the file!") self.data = np.genfromtxt(filename, skip_header=ix + 2, skip_footer=4) self.column_headers = headers self.setas = "xy" return self
def __parse_VSM(self, header_line=3, data_line=3, header_delim=","): """An intrernal function for parsing deliminated data without a leading column of metadata.copy Keyword Arguments: header_line (int): The line in the file that contains the column headers. If None, then column headers are auotmatically generated. data_line (int): The line on which the data starts header_delim (strong): The delimiter used for separating header values Returns: Nothing, but modifies the current object. Note: The default values are configured fir read VSM data files """ try: with io.open(self.filename, errors="ignore", encoding="utf-8") as f: for i, line in enumerate(f): if i == 0: self["Timestamp"] = line.strip() check = datetime.strptime(self["Timestamp"], "%a %b %d %H:%M:%S %Y") if check is None: raise Core.StonerLoadError("Not a VSM file ?") elif i == 1: assertion(line.strip() == "") elif i == 2: header_string = line.strip() elif i == header_line: unit_string = line.strip() column_headers = [ "{} ({})".format(h.strip(), u.strip()) for h, u in zip(header_string.split(header_delim), unit_string.split(header_delim)) ] elif i > 3: break except (StonerAssertionError, ValueError, AssertionError, TypeError) as e: raise Core.StonerLoadError("Not a VSM File" + str(e.args)) self.data = np.genfromtxt( self.filename, dtype="float", usemask=True, skip_header=data_line - 1, missing_values=["6:0", "---"], invalid_raise=False, ) self.data = np.ma.mask_rows(self.data) cols = self.data.shape[1] self.data = np.reshape(self.data.compressed(), (-1, cols)) self.column_headers = column_headers self.setas(x="H_vsm (T)", y="m (emu)") # pylint: disable=not-callable
def _load(self, filename=None, *args, **kargs): """Load function. File format has space delimited columns from row 3 onwards.""" if filename is None or not filename: self.get_filename("r") else: self.filename = filename pattern = re.compile( r'# Dataset "([^\"]*)" exported from GenX on (.*)$') pattern2 = re.compile( r"#\sFile\sexported\sfrom\sGenX\'s\sReflectivity\splugin") i = 0 ix = 0 with io.open(self.filename, "r", errors="ignore", encoding="utf-8") as datafile: line = datafile.readline() match = pattern.match(line) match2 = pattern2.match(line) if match is not None: dataset = match.groups()[0] date = match.groups()[1] self["date"] = date i = 2 elif match2 is not None: line = datafile.readline() self["date"] = line.split(":")[1].strip() dataset = datafile.readline()[1:].strip() i = 3 else: raise _SC_.StonerLoadError("Not a GenXFile") for ix, line in enumerate(datafile): line = line.strip() if line in ["# Headers:", "# Column lables:"]: line = next(datafile)[1:].strip() break else: raise _SC_.StonerLoadError("Cannot find headers") skip = ix + i + 2 column_headers = [f.strip() for f in line.strip().split("\t")] self.data = _np_.real( _np_.genfromtxt(self.filename, skip_header=skip, dtype=complex)) self["dataset"] = dataset if "sld" in dataset.lower(): self["type"] = "SLD" elif "asymmetry" in dataset.lower(): self["type"] = "Asymmetry" elif "dd" in dataset.lower(): self["type"] = "Down" elif "uu" in dataset.lower(): self["type"] = "Up" self.column_headers = column_headers return self
def _read_xdata(self, f): """Read the xdata from the spc file.""" self._pts = self._header["fnpts"] if self._header["ftflgs"] & 128: # We need to read some X Data if 4 * self._pts > self._filesize - f.tell(): raise Core.StonerLoadError("Trying to read too much data!") xvals = f.read(4 * self._pts) # I think storing X vals directly implies that each one is 4 bytes.... xdata = np.array(struct.unpack(str2bytes(str(self._pts) + "f"), xvals)) else: # Generate the X Data ourselves first = self._header["ffirst"] last = self._header["flast"] if self._pts > 1e6: # Something not right here ! raise Core.StonerLoadError("More than 1 million points requested. Bugging out now!") xdata = np.linspace(first, last, self._pts) return xdata
def _load(self, filename=None, *args, **kargs): """File loader for PinkLib. Args: filename (string or bool): File to load. If None then the existing filename is used, if False, then a file dialog will be used. Returns: A copy of the itself after loading the data. """ if filename is None or not filename: self.get_filename("r") else: self.filename = filename with io.open(self.filename, "r", errors="ignore", encoding="utf-8") as f: # Read filename linewise if "PINKlibrary" not in f.readline(): raise Core.StonerLoadError("Not a PINK file") f = f.readlines() happened_before = False for i, line in enumerate(f): if line[0] != "#" and not happened_before: header_line = i - 2 # -2 because there's a commented out data line happened_before = True continue # want to get the metadata at the bottom of the file too elif any(s in line for s in ("Start time", "End time", "Title")): tmp = line.strip("#").split(":") self.metadata[tmp[0].strip()] = ":".join(tmp[1:]).strip() column_headers = f[header_line].strip("#\t ").split("\t") data = np.genfromtxt(self.filename, dtype="float", delimiter="\t", invalid_raise=False, comments="#") self.data = data[:, 0:-2] # Deal with an errant tab at the end of each line self.column_headers = column_headers if np.all([h in column_headers for h in ("T (C)", "R (Ohm)")]): self.setas(x="T (C)", y="R (Ohm)") # pylint: disable=not-callable return self
def __find_lines(self): """Returns an array of ints [header_line,data_line,scan_line,date_line,motor_line].""" with io.open(self.filename, "r", errors="ignore", encoding="utf-8") as fp: self.line_numbers = [0, 0, 0, 0, 0] counter = 0 for line in fp: counter += 1 if counter == 1 and line[0] != "#": raise Core.StonerLoadError("Not a BNL File ?") if len(line) < 2: continue # if there's nothing written on the line go to the next elif line[0:2] == "#L": self.line_numbers[0] = counter elif line[0:2] == "#S": self.line_numbers[2] = counter elif line[0:2] == "#D": self.line_numbers[3] = counter elif line[0:2] == "#P": self.line_numbers[4] = counter elif line[0] in [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9" ]: self.line_numbers[1] = counter break
def _read_loginfo(self, f): """Read the log info section of the spc file.""" logstc = struct.unpack(b"IIIII44s", f.read(64)) logstc_keys = ("logsizd", "logsizm", "logtxto", "logbins", "logdsks", "logrsvr") logheader = dict(zip(logstc_keys, logstc)) self._header = dict(self._header, **logheader) # Can't handle either binary log information or ion disk log information (wtf is this anyway !) if self._header["logbins"] + self._header[ "logdsks"] > self._filesize - f.tell(): raise Core.StonerLoadError("Too much logfile data to read") f.read(self._header["logbins"] + self._header["logdsks"]) # The renishaw seems to put a 16 character timestamp next - it's not in the spec but never mind that. self._header["Date-Time"] = f.read(16) # Now read the rest of the file as log text logtext = f.read() # We expect things to be single lines terminated with a CR-LF of the format key=value for line in re.split(b"[\r\n]+", logtext): if b"=" in line: parts = line.split(b"=") key = parts[0].decode() value = parts[1].decode() self._header[key] = value
def _load(self, filename=None, *args, **kargs): """Load an OpenGDA file. Args: filename (string or bool): File to load. If None then the existing filename is used, if False, then a file dialog will be used. Returns: A copy of the itself after loading the data. """ if filename is None or not filename: self.get_filename("r") else: self.filename = filename i = 0 with io.open(self.filename, "r", errors="ignore", encoding="utf-8") as f: for i, line in enumerate(f): line = line.strip() if i == 0 and line != "&SRS": raise Core.StonerLoadError("Not a GDA File from Rasor ?" + str(line)) if "&END" in line: break parts = line.split("=") if len(parts) != 2: continue key = parts[0] value = parts[1].strip() self.metadata[key] = string_to_type(value) column_headers = f.readline().strip().split("\t") self.data = np.genfromtxt([str2bytes(l) for l in f], dtype="float", invalid_raise=False) self.column_headers = column_headers return self
def _check_signature(self, filename): """Check that this is a PNG file and raie a Core.StonerLoadError if not.""" try: with io.open(filename, "rb") as test: sig = test.read(8) sig = [x for x in sig] if self.debug: print(sig) if sig != [137, 80, 78, 71, 13, 10, 26, 10]: raise Core.StonerLoadError("Signature mismatrch") except Exception: from traceback import format_exc raise Core.StonerLoadError("Not a PNG file!>\n{}".format( format_exc())) return True
def _load(self, filename, *args, **kargs): """Leeds MOKE file loader routine. Args: filename (string or bool): File to load. If None then the existing filename is used, if False, then a file dialog will be used. Returns: A copy of the itself after loading the data. """ if filename is None or not filename: self.get_filename("r") else: self.filename = filename with io.open(self.filename, mode="rb") as f: line = bytes2str(f.readline()).strip() if line != "#Leeds CM Physics MOKE": raise Core.StonerLoadError( "Not a Core.DataFile from the Leeds MOKE") while line.startswith("#") or line == "": parts = line.split(":") if len(parts) > 1: key = parts[0][1:] data = ":".join(parts[1:]).strip() self[key] = data line = bytes2str(f.readline()).strip() column_headers = [x.strip() for x in line.split(",")] self.data = np.genfromtxt(f, delimiter=",") self.setas = "xy.de" self.column_headers = column_headers return self
def _load(self, filename=None, *args, **kargs): """Reads an XRD Core.DataFile as produced by the Brucker diffractometer. Args: filename (string or bool): File to load. If None then the existing filename is used, if False, then a file dialog will be used. Returns: A copy of the itself after loading the data. Notes: Format is ini file like but not enough to do standard inifile processing - in particular one can have multiple sections with the same name (!) """ if filename is None or not filename: self.get_filename("r") else: self.filename = filename sh = re.compile(r"\[(.+)\]") # Regexp to grab section name with io.open(self.filename, errors="ignore", encoding="utf-8") as f: # Read filename linewise if f.readline().strip() != ";RAW4.00": # Check we have the corrrect fileformat raise Core.StonerLoadError("File Format Not Recognized !") drive = 0 for line in f: # for each line m = sh.search(line) if m: # This is a new section section = m.group(1) if section == "Drive": # If this is a Drive section we need to know which Drive Section it is section = section + str(drive) drive = drive + 1 elif section == "Data": # Data section contains the business but has a redundant first line f.readline() for line in f: # Now start reading lines in this section... if line.strip() == "": # A blank line marks the end of the section, so go back to the outer loop which will # handle a new section break if section == "Data": # In the Data section read lines of data value,vale parts = line.split(",") angle = parts[0].strip() counts = parts[1].strip() dataline = np.array([float(angle), float(counts)]) self.data = np.append(self.data, dataline) else: # Other sections contain metadata parts = line.split("=") key = parts[0].strip() data = parts[1].strip() # Keynames in main metadata are section:key - use theCore.DataFile magic to do type # determination self[section + ":" + key] = string_to_type(data) column_headers = ["Angle", "Counts"] # Assume the columns were Angles and Counts self.data = np.reshape(self.data, (-1, 2)) self.setas = "xy" self.four_bounce = self["HardwareConfiguration:Monochromator"] == 1 self.column_headers = column_headers if kargs.pop("Q", False): self.to_Q() return self
def _load(self, filename=None, *args, **kargs): """Load function. File format has space delimited columns from row 3 onwards.""" if filename is None or not filename: self.get_filename("r") else: self.filename = filename with io.open(self.filename, "r", errors="ignore", encoding="utf-8") as data: # Slightly ugly text handling line = data.readline() if not line.strip().startswith( "# Datafile created by QuickNXS 0.9.39" ): # bug out oif we don't like the header raise Core.StonerLoadError("Not a file from the SNS BL4A line") for line in data: if line.startswith("# "): # We're in the header line = line[2:].strip() # strip the header and whitespace if line.startswith("["): # Look for a section header section = line.strip().strip("[]") if section == "Data": # The Data section has one line of colum headers and then data header = next(data)[2:].split("\t") if not python_v3: column_headers = [h.strip().encode("ascii", errors="replace") for h in header] else: column_headers = [h.strip() for h in header] self.data = np.genfromtxt(data) # we end by reading the raw data elif section == "Global Options": # This section can go into metadata for line in data: line = line[2:].strip() if line.strip() == "": break else: self[line[2:10].strip()] = line[11:].strip() elif ( section == "Direct Beam Runs" or section == "Data Runs" ): # These are constructed into lists ofg dictionaries for each file sec = list() header = next(data) header = header[2:].strip() keys = [s.strip() for s in header.split(" ") if s.strip()] for line in data: line = line[2:].strip() if line == "": break else: values = [s.strip() for s in line.split(" ") if s.strip()] sec.append(dict(zip(keys, values))) self[section] = sec else: # We must still be in the opening un-labelled section of meta data if ":" in line: i = line.index(":") key = line[:i].strip() value = line[i + 1 :].strip() self[key.strip()] = value.strip() self.column_headers = column_headers return self
def _load(self, filename=None, *args, **kargs): """Data loader function for 340 files.""" if filename is None or not filename: self.get_filename("r") else: self.filename = filename with io.open(self.filename, "rb") as data: keys = [] vals = [] for line in data: line = bytes2str(line) if line.strip() == "": break parts = [p.strip() for p in line.split(":")] if len(parts) != 2: raise Core.StonerLoadError( "Header doesn't contain two parts at {}".format( line.strip())) else: keys.append(parts[0]) vals.append(parts[1]) else: raise Core.StonerLoadError("Overan the end of the file") if keys != [ "Sensor Model", "Serial Number", "Data Format", "SetPoint Limit", "Temperature coefficient", "Number of Breakpoints", ]: raise Core.StonerLoadError( "Header did not contain recognised keys.") for (k, v) in zip(keys, vals): v = v.split()[0] self.metadata[k] = string_to_type(v) headers = bytes2str(next(data)).strip().split() column_headers = headers[1:] dat = np.genfromtxt(data) self.data = dat[:, 1:] self.column_headers = column_headers return self
def _load(self, filename=None, header_line=0, data_line=1, data_delim=",", header_delim=",", **kargs): """Generic deliminated file loader routine. Args: filename (string or bool): File to load. If None then the existing filename is used, if False, then a file dialog will be used. Keyword Arguments: header_line (int): The line in the file that contains the column headers. If None, then column headers are auotmatically generated. data_line (int): The line on which the data starts data_delim (string): Thge delimiter used for separating data values header_delim (strong): The delimiter used for separating header values Returns: A copy of the current object after loading the data. """ if filename is None or not filename: self.get_filename("r") else: self.filename = filename if header_line is not None: try: header_string = linecache.getline(self.filename, header_line + 1) header_string = re.sub(r'["\n]', "", header_string) header_string.index(header_delim) except (ValueError, SyntaxError): linecache.clearcache() raise Core.StonerLoadError("No Delimiters in header line") column_headers = [x.strip() for x in header_string.split(header_delim)] else: column_headers = ["Column" + str(x) for x in range(np.shape(self.data)[1])] data_line = linecache.getline(self.filename, data_line) try: data_line.index(data_delim) except ValueError: linecache.clearcache() raise Core.StonerLoadError("No delimiters in data lines") self.data = np.genfromtxt(self.filename, dtype="float", delimiter=data_delim, skip_header=data_line) self.column_headers = column_headers linecache.clearcache() return self
def split_up_down(data, col=None, folder=None): """Splits the DataFile data into several files where the column *col* is either rising or falling Args: data (:py:class:`Stoner.Core.DataFile`): object containign the data to be sorted col (index): is something that :py:meth:`Stoner.Core.DataFile.find_col` can use folder (:py:class:`Stoner.Folders.DataFolder` or None): if this is an instance of :py:class:`Stoner.Folders.DataFolder` then add rising and falling files to groups of this DataFolder, otherwise create a new one Returns: A :py:class:`Sonter.Folder.DataFolder` object with two groups, rising and falling """ a = _SC_.Data(data) if col is None: _=a._col_args() col=_.xcol width = len(a) / 10 if width % 2 == 0: # Ensure the window for Satvisky Golay filter is odd width += 1 setas=a.setas.clone a.setas="" peaks = list(a.peaks(col, width,xcol=None, peaks=True, troughs=False,full_data=False)) troughs = list(a.peaks(col, width, xcol=None, peaks=False, troughs=True,full_data=False)) a.setas=setas if len(peaks) > 0 and len(troughs) > 0: #Ok more than up down here order = peaks[0] < troughs[0] elif len(peaks) > 0: #Rise then fall order = True elif len(troughs) > 0: # Fall then rise order = False else: #No peaks or troughs so just return a single rising ret=_SF_(readlist=False) ret+=data return ret splits = [0, len(a)] splits.extend(peaks) splits.extend(troughs) splits.sort() splits=[int(s) for s in splits] if not isinstance(folder, _SF_): # Create a new DataFolder object output = _SF_(readlist=False) else: output = folder output.add_group("rising") output.add_group("falling") if order: risefall=["rising","falling"] else: risefall=["falling","rising"] for i in range(len(splits)-1): working=data.clone working.data = data.data[splits[i]:splits[i+1],:] output.groups[risefall[i%2]].append(working) return output
def collate(grp,trail,**kargs): grp.sort() final=SC.DataFile() final.add_column(grp[0].column('Energy'),'Energy') for g in grp: final.add_column(g.column('Asym'),g.title) if "group_key" in kargs: final[kargs["group_key"]]=grp.key final["path"]=trail if "save" in kargs and kargs["save"]: final.save(kargs["filename"]) return final
def _load(self, filename, *args, **kargs): """Private loader method.""" if filename is None or not filename: self.get_filename("r") else: self.filename = filename datastart = -1 dataend = -1 i = 0 with io.open(self.filename, "r", errors="ignore", encoding="utf-8") as data: if "******** EasyPlot save file ********" not in data.read(1024): raise Core.StonerLoadError("Not an EasyPlot Save file?") data.seek(0) for i, line in enumerate(data): line = line.strip() if line == "": continue if line[0] not in "-0123456789" and datastart > 0 and dataend < 0: dataend = i if line.startswith('"') and ":" in line: parts = [x.strip() for x in line.strip('"').split(":")] self[parts[0]] = string_to_type(":".join(parts[1:])) elif line.startswith("/"): # command parts = [x.strip('"') for x in next(csv.reader([line], delimiter=" ")) if x != ""] cmd = parts[0].strip("/") if len(cmd) > 1: cmdname = "_{}_cmd".format(cmd) if cmdname in dir(self): # If this command is implemented as a function run it cmd = getattr(self, "_{}_cmd".format(cmd)) cmd(parts[1:]) else: if len(parts[1:]) > 1: cmd = cmd + "." + parts[1] value = ",".join(parts[2:]) elif len(parts[1:]) == 1: value = parts[1] else: value = True self[cmd] = value elif line[0] in "-0123456789" and datastart < 0: # start of data datastart = i if "," in line: delimiter = "," else: delimiter = None if dataend < 0: dataend = i self.data = np.genfromtxt(self.filename, skip_header=datastart, skip_footer=i - dataend, delimiter=delimiter) if self.data.shape[1] == 2: self.setas = "xy" return self
def split(data, col=None, folder=None, spliton=0, rising=True, falling=False, skip=0): """Splits the DataFile data into several files where the column *col* is either rising or falling Args: data (:py:class:`Stoner.Core.DataFile`): object containign the data to be sorted col (index): is something that :py:meth:`Stoner.Core.DataFile.find_col` can use folder (:py:class:`Stoner.Folders.DataFolder` or None): if this is an instance of :py:class:`Stoner.Folders.DataFolder` then add rising and falling files to groups of this DataFolder, otherwise create a new one spliton (str or float): Define where to split the data, 'peak' to split on peaks, 'trough' to split on troughs, 'both' to split on peaks and troughs or number to split at that number rising (bool): whether to split on threshold crossing when data is rising falling (bool): whether to split on threshold crossing when data is falling skip (int): skip this number of splitons each time. eg skip=1 picks out odd crossings Returns: A :py:class:`Sonter.Folder.DataFolder` object with two groups, rising and falling """ if col is None: col = data.setas["x"] d=_SC_.Data(data) if not isinstance(folder, _SF_): # Create a new DataFolder object output = _SF_() else: output = folder if isinstance(spliton, int_types+(float,)): spl=d.threshold(threshold=float(spliton),col=col,rising=rising,falling=falling,all_vals=True) elif spliton in ['peaks','troughs','both']: width = len(d) / 10 if width % 2 == 0: # Ensure the window for Satvisky Golay filter is odd width += 1 if spliton=='peaks': spl = list(d.peaks(col, width, xcol=False, peaks=True, troughs=False)) elif spliton=='troughs': spl = list(d.peaks(col, width, xcol=False, peaks=False, troughs=True)) else: spl = list(d.peaks(col, width, xcol=False, peaks=True, troughs=True)) else: raise ValueError('Did not recognise spliton') spl = [spl[i] for i in range(len(spl)) if i%(skip+1)==0] spl.extend([0,len(d)]) spl.sort() for i in range(len(spl)-1): tmp=d.clone tmp.data=tmp[spl[i]:spl[i+1]] output.files.append(tmp) return output
def _read_ydata(self, f, data, column_headers): """Read the y data and column headers from spc file.""" n = self._header["fnsub"] subhdr_keys = ( "subflgs", "subexp", "subindx", "subtime", "subnext", "subnois", "subnpts", "subscan", "subwlevel", "subresv", ) if self._header["ftflgs"] & 1: y_width = 2 y_fmt = "h" divisor = 2**16 else: y_width = 4 y_fmt = "i" divisor = 2**32 if n * (y_width * self._pts + 32) > self._filesize - f.tell(): raise Core.StonerLoadError("No good, going to read too much data!") for j in range(n): # We have n sub-scans # Read the subheader and import into the main metadata dictionary as scan#:<subheader item> subhdr = struct.unpack(b"BBHfffIIf4s", f.read(32)) subheader = dict( zip(["scan" + str(j) + ":" + x for x in subhdr_keys], subhdr)) # Now read the y-data exponent = subheader["scan" + str(j) + ":subexp"] if int(exponent) & -128: # Data is unscaled direct floats ydata = np.array( struct.unpack(str2bytes(str(self._pts) + "f"), f.read(self._pts * y_width))) else: # Data is scaled by exponent yvals = struct.unpack(str2bytes(str(self._pts) + y_fmt), f.read(self._pts * y_width)) ydata = np.array(yvals, dtype="float64") * (2** exponent) / divisor data[:, j + 1] = ydata self._header = dict(self._header, **subheader) column_headers.append("Scan" + str(j) + ":" + self._yvars[self._header["fytype"]]) return data
def _load(self, filename=None, *args, **kargs): """TDMS file loader routine. Args: filename (string or bool): File to load. If None then the existing filename is used, if False, then a file dialog will be used. Returns: A copy of the itself after loading the data. """ if filename is None or not filename: self.get_filename("r") else: self.filename = filename # Open the file and read the main file header and unpack into a dict try: f = TdmsFile(self.filename) column_headers = [] data = np.array([]) for grp in f.objects.keys(): if grp == "/": pass # skip the rooot group elif grp == "/'TDI Format 1.5'": metadata = f.object("TDI Format 1.5") for k, v in metadata.properties.items(): self.metadata[k] = string_to_type(str(v)) else: if f.objects[grp].has_data: chnl = grp.split("/")[-1] chnl.strip().strip("'") column_headers.append(chnl) if data.size == 0: data = f.objects[grp].data else: data = np.column_stack( [data, f.objects[grp].data]) self.data = data self.column_headers = column_headers except Exception: from traceback import format_exc raise Core.StonerLoadError("Not a TDMS File \n{}".format( format_exc())) return self
def _read_uvwdata(self, filename, fmt, lineno): """Read the numerical data taking account of the format.""" if fmt == "Text": uvwdata = _np_.genfromtxt(self.filename, skip_header=lineno + 2) elif fmt == "Binary 4": if self["version"] == 1: dt = _np_.dtype(">f4") else: dt = _np_.dtype("<f4") with io.open(filename, "rb") as bindata: bindata.seek(self._ptr) uvwdata = _np_.fromfile(bindata, dtype=dt, count=1 + self["xnodes"] * self["ynodes"] * self["znodes"] * self["valuedim"]) assertion( uvwdata[0] == 1234567.0, "Binary 4 format check value incorrect ! Actual Value was {}" .format(uvwdata[0]), ) uvwdata = uvwdata[1:] uvwdata = _np_.reshape(uvwdata, (-1, self["valuedim"])) elif fmt == "Binary 8": if self["version"] == 1: dt = _np_.dtype(">f8") else: dt = _np_.dtype("<f8") with io.open(filename, "rb") as bindata: bindata.seek(self._ptr) uvwdata = _np_.fromfile(bindata, dtype=dt, count=1 + self["xnodes"] * self["ynodes"] * self["znodes"] * self["valuedim"]) assertion( (uvwdata[0] == 123456789012345.0), "Binary 4 format check value incorrect ! Actual Value was {}" .format(uvwdata[0]), ) uvwdata = _np_.reshape(uvwdata, (-1, self["valuedim"])) else: raise _SC_.StonerLoadError("Unknow OVF Format {}".format(fmt)) return uvwdata
def _load(self, filename, *args, **kargs): """Just call the parent class but with the right parameters set Args: filename (string or bool): File to load. If None then the existing filename is used, if False, then a file dialog will be used. Returns: A copy of the itself after loading the data. """ if filename is None or not filename: self.get_filename("r") else: self.filename = filename super(BigBlueFile, self)._load( self.filename, *args, header_line=3, data_line=7, data_delim=" ", header_delim="," ) if np.all(np.isnan(self.data)): raise Core.StonerLoadError("All data was NaN in Big Blue format") return self
def _load(self, filename=None, *args, **kargs): """Load function. File format has space delimited columns from row 3 onwards.""" if filename is None or not filename: self.get_filename("r") else: self.filename = filename pattern = re.compile( r'# Dataset "([^\"]*)" exported from GenX on (.*)$') pattern2 = re.compile( r"#\sFile\sexported\sfrom\sGenX\'s\sReflectivity\splugin") with io.open(self.filename, "r", errors="ignore", encoding="utf-8") as datafile: line = datafile.readline() match = pattern.match(line) match2 = pattern2.match(line) if match is not None: dataset = match.groups()[0] date = match.groups()[1] line = datafile.readline() line = datafile.readline() line = line[1:] self["date"] = date elif match2 is not None: line = datafile.readline() self["date"] = line.split(":")[1].strip() datafile.readline() line = datafile.readline() line = line[1:] dataset = "asymmetry" else: raise _SC_.StonerLoadError("Not a GenXFile") column_headers = [f.strip() for f in line.strip().split("\t")] self.data = _np_.genfromtxt(self.filename, skip_header=4) self["dataset"] = dataset self.setas = "xye" self.column_headers = column_headers return self
def _load(self, filename=None, *args, **kargs): """PNG file loader routine. Args: filename (string or bool): File to load. If None then the existing filename is used, if False, then a file dialog will be used. Returns: A copy of the itself after loading the data. """ if filename is None or not filename: self.get_filename("r") else: self.filename = filename self._check_signature(filename) try: with PIL.Image.open(self.filename, "r") as img: for k in img.info: self.metadata[k] = img.info[k] self.data = np.asarray(img) except IOError: raise Core.StonerLoadError("Unable to read as a PNG file.") return self
def _load(self, filename=None, *args, **kargs): """Load function. File format has space delimited columns from row 3 onwards.""" if filename is None or not filename: self.get_filename("r") else: self.filename = filename i = [0, 0, 0, 0] with io.open(self.filename, "r", errors="ignore", encoding="utf-8") as data: # Slightly ugly text handling for i[0], line in enumerate(data): if (i[0] == 0 and line.strip() != "## mda2ascii 1.2 generated output" ): # bug out oif we don't like the header raise Core.StonerLoadError("Not a file mda2ascii") line.strip() if "=" in line: parts = line[2:].split("=") self[parts[0].strip()] = string_to_type("".join( parts[1:]).strip()) elif line.startswith("# Extra PV:"): # Onto the next metadata bit break pvpat = re.compile(r"^#\s+Extra\s+PV\s\d+\:(.*)") for i[1], line in enumerate(data): if line.strip() == "": continue elif line.startswith("# Extra PV"): res = pvpat.match(line) bits = [ b.strip().strip(r'"') for b in res.group(1).split(",") ] if bits[1] == "": key = bits[0] else: key = bits[1] if len(bits) > 3: key = key + " ({})".format(bits[3]) self[key] = string_to_type(bits[2]) else: break # End of Extra PV stuff else: raise Core.StonerLoadError("Overran Extra PV Block") for i[2], line in enumerate(data): line.strip() if line.strip() == "": continue elif line.startswith("# Column Descriptions:"): break # Start of column headers now elif "=" in line: parts = line[2:].split("=") self[parts[0].strip()] = string_to_type("".join( parts[1:]).strip()) else: raise Core.StonerLoadError( "Overran end of scan header before column descriptions") colpat = re.compile(r"#\s+\d+\s+\[([^\]]*)\](.*)") column_headers = [] for i[3], line in enumerate(data): res = colpat.match(line) line.strip() if line.strip() == "": continue elif line.startswith("# 1-D Scan Values"): break # Start of data elif res is not None: if "," in res.group(2): bits = [b.strip() for b in res.group(2).split(",")] if bits[-2] == "": colname = bits[0] else: colname = bits[-2] if bits[-1] != "": colname += " ({})".format(bits[-1]) if colname in column_headers: colname = "{}:{}".format(bits[0], colname) else: colname = res.group(1).strip() column_headers.append(colname) else: raise Core.StonerLoadError( "Overand the end of file without reading data") self.data = np.genfromtxt(self.filename, skip_header=sum(i)) # so that's ok then ! self.column_headers = column_headers return self
def _load(self, filename, *args, **kargs): """Generic deliminated file loader routine. Args: filename (string or bool): File to load. If None then the existing filename is used, if False, then a file dialog will be used. Keyword Arguments: header_line (int): The line in the file that contains the column headers. If None, then column headers are auotmatically generated. data_line (int): The line on which the data starts data_delim (string): Thge delimiter used for separating data values header_delim (strong): The delimiter used for separating header values Returns: A copy of the current object after loading the data. """ defaults = copy(self._defaults) defaults.update(kargs) keep = set(kargs.keys()) - (set(self._defaults.keys()) | set(["auto_load", "filetype"])) kargs = {k: kargs[k] for k in keep} header_line = defaults["header_line"] data_line = defaults["data_line"] data_delim = defaults["data_delim"] header_delim = defaults["header_delim"] if filename is None or not filename: self.get_filename("r") else: self.filename = filename if header_line is not None: try: header_string = linecache.getline(self.filename, header_line + 1) header_string = re.sub(r'["\n]', "", header_string) header_string.index(header_delim) except (ValueError, SyntaxError): linecache.clearcache() raise Core.StonerLoadError("No Delimiters in header line") column_headers = [ x.strip() for x in header_string.split(header_delim) ] else: column_headers = [ "Column" + str(x) for x in range(np.shape(self.data)[1]) ] data_test = linecache.getline(self.filename, data_line + 1) if data_delim is None: for data_delim in ["\t", ",", ";", " "]: if data_delim in data_test: break else: raise Core.StonerLoadError("No delimiters in data lines") elif data_delim not in data_test: linecache.clearcache() raise Core.StonerLoadError("No delimiters in data lines") self.data = np.genfromtxt(self.filename, dtype="float", delimiter=data_delim, skip_header=data_line) self.column_headers = column_headers linecache.clearcache() self._kargs = kargs return self
def _load(self, filename=None, *args, **kargs): """Reads a .scf file produced by the Renishaw Raman system (amongs others) Args: filename (string or bool): File to load. If None then the existing filename is used, if False, then a file dialog will be used. Returns: A copy of the itself after loading the data. Todo: Implement the second form of the file that stores multiple x-y curves in the one file. Notes: Metadata keys are pretty much as specified in the spc.h file that defines the filerformat. """ if filename is None or not filename: self.get_filename("r") else: self.filename = filename # Open the file and read the main file header and unpack into a dict self._filesize = os.stat(self.filename).st_size with io.open(filename, "rb") as f: spchdr = struct.unpack( b"BBBciddiBBBBi9s9sH8f30s130siiBBHf48sfifB187s", f.read(512)) keys = ( "ftflgs", "fversn", "fexper", "fexp", "fnpts", "ffirst", "flast", "fnsub", "fxtype", "fytype", "fztype", "fpost", "fres", "fsource", "fpeakpt", "fspare1", "fspare2", "fspare3", "fspare4", "fspare5", "fspare6", "fspare7", "fspare8", "fcm", "nt", "fcatx", "flogoff", "fmods", "fprocs", "flevel", "fsampin", "ffactor", "fmethod", "fzinc", "fwplanes", "fwinc", "fwtype", "fwtype", "fresv", ) self._xvars = [ "Arbitrary", "Wavenumber (cm-1)", "Micrometers (um)", "Nanometers (nm)", "Seconds", "Minutes", "Hertz (Hz)", "Kilohertz (KHz)", "Megahertz (MHz)", "Mass (M/z)", "Parts per million (PPM)", "Days", "Years", "Raman Shift (cm-1)", "Raman Shift (cm-1)", "eV", "XYZ text labels in fcatxt (old 0x4D version only)", "Diode Number", "Channel", "Degrees", "Temperature (F)", "Temperature (C)", "Temperature (K)", "Data Points", "Milliseconds (mSec)", "Microseconds (uSec)", "Nanoseconds (nSec)", "Gigahertz (GHz)", "Centimeters (cm)", "Meters (m)", "Millimeters (mm)", "Hours", "Hours", ] self._yvars = [ "Arbitrary Intensity", "Interferogram", "Absorbance", "Kubelka-Monk", "Counts", "Volts", "Degrees", "Milliamps", "Millimeters", "Millivolts", "Log(1/R)", "Percent", "Percent", "Intensity", "Relative Intensity", "Energy", "Decibel", "Temperature (F)", "Temperature (C)", "Temperature (K)", "Index of Refraction [N]", "Extinction Coeff. [K]", "Real", "Imaginary", "Complex", "Complex", "Transmission (ALL HIGHER MUST HAVE VALLEYS!)", "Reflectance", "Arbitrary or Single Beam with Valley Peaks", "Emission", "Emission", ] self._header = dict(zip(keys, spchdr)) n = self._header["fnsub"] if self._header["ftflgs"] & 64 == 64 or not ( 75 <= self._header["fversn"] <= 77): # This is the multiple XY curves in file flag. raise Core.StonerLoadError( "Filetype not implemented yet ! ftflgs={ftflgs}, fversn={fversn}" .format(**self._header)) else: # A single XY curve in the file. # Read the xdata and add it to the file. xdata = self._read_xdata(f) data = np.zeros( (self._pts, (n + 1))) # initialise the data soace data[:, 0] = xdata # Put in the X-Data column_headers = [self._xvars[self._header["fxtype"]] ] # And label the X column correctly # Now we're going to read the Y-data data = self._read_ydata(f, data, column_headers) if self._header[ "flogoff"] != 0: # Ok, we've got a log, so read the log header and merge into metadata self._read_loginfo(f) # Ok now build the Stoner.Core.DataFile instance to return self.data = data # The next bit generates the metadata. We don't just copy the metadata because we need to figure out the typehints first - hence the loop # here to call Core.DataFile.__setitem() for x in self._header: self[x] = self._header[x] self.column_headers = column_headers if len(self.column_headers) == 2: self.setas = "xy" return self
def _load(self, filename=None, *args, **kargs): """QD system file loader routine. Args: filename (string or bool): File to load. If None then the existing filename is used, if False, then a file dialog will be used. Returns: A copy of the itself after loading the data. """ if filename is None or not filename: self.get_filename("r") else: self.filename = filename setas = {} i = 0 with io.open(self.filename, "r", encoding="utf-8", errors="ignore") as f: # Read filename linewise for i, line in enumerate(f): line = line.strip() if i == 0 and line != "[Header]": raise Core.StonerLoadError("Not a Quantum Design File !") elif line == "[Header]" or line.startswith(";") or line == "": continue elif "[Data]" in line: break elif "," not in line: raise Core.StonerLoadError("No data in file!") parts = [x.strip() for x in line.split(",")] if parts[1].split(":")[0] == "SEQUENCE FILE": key = parts[1].split(":")[0].title() value = parts[1].split(":")[1] elif parts[0] == "INFO": if parts[1] == "APPNAME": parts[1], parts[2] = parts[2], parts[1] if len(parts) > 2: key = "{}.{}".format(parts[0], parts[2]) else: raise Core.StonerLoadError("No data in file!") key = key.title() value = parts[1] elif parts[0] in ["BYAPP", "FILEOPENTIME"]: key = parts[0].title() value = " ".join(parts[1:]) elif parts[0] == "FIELDGROUP": key = "{}.{}".format(parts[0], parts[1]).title() value = "[{}]".format(",".join(parts[2:])) elif parts[0] == "STARTUPAXIS": axis = parts[1][0].lower() setas[axis] = setas.get(axis, []) + [int(parts[2])] key = "Startupaxis-{}".format(parts[1].strip()) value = parts[2].strip() else: key = parts[0] + "," + parts[1] key = key.title() value = " ".join(parts[2:]) self.metadata[key] = string_to_type(value) else: raise Core.StonerLoadError("No data in file!") if "Byapp" not in self: raise Core.StonerLoadError("Not a Quantum Design File !") column_headers = f.readline().strip().split(",") data = np.genfromtxt([str2bytes(l) for l in f], dtype="float", delimiter=",", invalid_raise=False) if data.shape[0] == 0: raise Core.StonerLoadError("No data in file!") if data.shape[1] < len( column_headers ): # Trap for buggy QD software not giving ewnough columns of data data = np.append( data, np.ones( (data.shape[0], len(column_headers) - data.shape[1])) * np.NaN, axis=1) elif data.shape[1] > len(column_headers): # too much data data = data[:, :len(column_headers) - data.shape[1]] self.data = data self.column_headers = column_headers s = self.setas for k in setas: for ix in setas[k]: s[ix - 1] = k self.setas = s return self