def hdfgroup2dict(group, dictionary={}): for key, value in group.attrs.iteritems(): if isinstance(value, (np.string_, str)): if value == '_None_': value = None elif isinstance(value, np.bool_): value = bool(value) elif isinstance(value, np.ndarray) and \ value.dtype == np.dtype('|S1'): value = value.tolist() # skip signals - these are handled below. if key.startswith('_sig_'): pass elif key.startswith('_datetime_'): dictionary[key.replace("_datetime_", "")] = eval(value) else: dictionary[key] = value if not isinstance(group, h5py.Dataset): for key in group.keys(): if key.startswith('_sig_'): from hyperspy.io import dict2signal dictionary[key[len('_sig_'):]] = ( dict2signal(hdfgroup2signaldict(group[key]))) elif isinstance(group[key], h5py.Dataset): dictionary[key] = np.array(group[key]) elif key.startswith('_hspy_AxesManager_'): dictionary[key[len('_hspy_AxesManager_'):]] = \ AxesManager([i for k, i in sorted(iter( hdfgroup2dict(group[key]).iteritems()))]) else: dictionary[key] = {} hdfgroup2dict(group[key], dictionary[key]) return dictionary
def load( seq=None, gain=None, dark=None, metadata=None, xml_file=None, lazy=False, chunk_shape=None, nav_shape=None, ): """Loads a .seq file into hyperspy. Metadata taken from the .metadata file as well as from a paramters.txt file that can be passed as well. The parameters file is used calibrate using the 4-D STEM parameters for some signal. Parameters ----------- filename: str The name of the file to be loaded (.seq file) """ sig = dict2signal(file_reader(filename=seq, dark=dark, gain=gain, metadata=metadata, xml_file=xml_file, lazy=lazy, chunk_shape=chunk_shape, nav_shape=nav_shape), lazy=lazy) return sig
def hdfgroup2dict(group, dictionary={}): for key, value in group.attrs.iteritems(): if isinstance(value, (np.string_, str)): if value == '_None_': value = None elif isinstance(value, np.bool_): value = bool(value) elif isinstance(value, np.ndarray) and \ value.dtype == np.dtype('|S1'): value = value.tolist() # skip signals - these are handled below. if key.startswith('_sig_'): pass elif key.startswith('_datetime_'): dictionary[key.replace("_datetime_", "")] = eval(value) else: dictionary[key] = value if not isinstance(group, h5py.Dataset): for key in group.keys(): if key.startswith('_sig_'): from hyperspy.io import dict2signal dictionary[key[len('_sig_'):]] = (dict2signal( hdfgroup2signaldict(group[key]))) elif isinstance(group[key], h5py.Dataset): dictionary[key] = np.array(group[key]) elif key.startswith('_hspy_AxesManager_'): dictionary[key[len('_hspy_AxesManager_'):]] = \ AxesManager([i for k, i in sorted(iter( hdfgroup2dict(group[key]).iteritems()))]) else: dictionary[key] = {} hdfgroup2dict(group[key], dictionary[key]) return dictionary
def load_celeritas( top, bottom, dark=None, gain=None, metadata=None, xml_file=None, lazy=False, chunk_shape=None, nav_shape=None, ): """Loads a .seq file into hyperspy. Metadata taken from the .metadata file as well as from a paramters.txt file that can be passed as well. The parameters file is used calibrate using the 4-D STEM parameters for some signal. Parameters ----------- lazy : bool, default False Load the signal lazily. top : str The filename for the top part of the detector bottom: The filename for the bottom part of the detector dark: str The filename for the dark reference to be applied to the data gain: str The filename for the gain reference to be applied to the data metadata: str The filename for the metadata file xml_file: str The filename for the xml file to be applied. nav_shape: The navigation shape for the dataset to be divided into to chunks: If lazy=True this divides the dataset into this many chunks """ sig = dict2signal(c_reader(top=top, bottom=bottom, gain=gain, dark=dark, metadata=metadata, xml_file=xml_file, lazy=lazy, chunk_shape=chunk_shape, nav_shape=nav_shape), lazy=lazy) return sig
def hdfgroup2dict(group, dictionary = {}): for key, value in group.attrs.iteritems(): if type(value) is np.string_: if value == '_None_': value = None else: try: value = value.decode('utf8') except UnicodeError: # For old files value = value.decode('latin-1') elif type(value) is np.bool_: value = bool(value) elif type(value) is np.ndarray and \ value.dtype == np.dtype('|S1'): value = value.tolist() # skip signals - these are handled below. if key.startswith('_sig_'): pass else: dictionary[key] = value if not isinstance(group, h5py.Dataset): for key in group.keys(): if key.startswith('_sig_'): from hyperspy.io import dict2signal dictionary[key[len('_sig_'):]] = ( dict2signal(hdfgroup2signaldict(group[key]))) elif isinstance(group[key],h5py.Dataset): dictionary[key]=np.array(group[key]) elif key.startswith('_hspy_AxesManager_'): dictionary[key[len('_hspy_AxesManager_'):]] = \ AxesManager([i for k, i in sorted(iter( hdfgroup2dict(group[key]).iteritems()))]) else: dictionary[key] = {} hdfgroup2dict(group[key], dictionary[key]) return dictionary
def hdfgroup2dict(group, dictionary={}): for key, value in group.attrs.iteritems(): if type(value) is np.string_: if value == '_None_': value = None else: try: value = value.decode('utf8') except UnicodeError: # For old files value = value.decode('latin-1') elif type(value) is np.bool_: value = bool(value) elif type(value) is np.ndarray and \ value.dtype == np.dtype('|S1'): value = value.tolist() # skip signals - these are handled below. if key.startswith('_sig_'): pass else: dictionary[key] = value if not isinstance(group, h5py.Dataset): for key in group.keys(): if key.startswith('_sig_'): from hyperspy.io import dict2signal dictionary[key[len('_sig_'):]] = (dict2signal( hdfgroup2signaldict(group[key]))) elif isinstance(group[key], h5py.Dataset): dictionary[key] = np.array(group[key]) elif key.startswith('_hspy_AxesManager_'): dictionary[key[len('_hspy_AxesManager_'):]] = \ AxesManager([i for k, i in sorted(iter( hdfgroup2dict(group[key]).iteritems()))]) else: dictionary[key] = {} hdfgroup2dict(group[key], dictionary[key]) return dictionary
def hdfgroup2dict(group, dictionary=None, load_to_memory=True): if dictionary is None: dictionary = {} for key, value in group.attrs.items(): if isinstance(value, bytes): value = value.decode() if isinstance(value, (np.string_, str)): if value == '_None_': value = None elif isinstance(value, np.bool_): value = bool(value) elif isinstance(value, np.ndarray) and value.dtype.char == "S": # Convert strings to unicode value = value.astype("U") if value.dtype.str.endswith("U1"): value = value.tolist() # skip signals - these are handled below. if key.startswith('_sig_'): pass elif key.startswith('_list_empty_'): dictionary[key[len('_list_empty_'):]] = [] elif key.startswith('_tuple_empty_'): dictionary[key[len('_tuple_empty_'):]] = () elif key.startswith('_bs_'): dictionary[key[len('_bs_'):]] = value.tostring() # The following two elif stataments enable reading date and time from # v < 2 of HyperSpy's metadata specifications elif key.startswith('_datetime_date'): date_iso = datetime.date( *ast.literal_eval(value[value.index("("):])).isoformat() dictionary[key.replace("_datetime_", "")] = date_iso elif key.startswith('_datetime_time'): date_iso = datetime.time( *ast.literal_eval(value[value.index("("):])).isoformat() dictionary[key.replace("_datetime_", "")] = date_iso else: dictionary[key] = value if not isinstance(group, h5py.Dataset): for key in group.keys(): if key.startswith('_sig_'): from hyperspy.io import dict2signal dictionary[key[len('_sig_'):]] = (dict2signal( hdfgroup2signaldict(group[key], load_to_memory=load_to_memory))) elif isinstance(group[key], h5py.Dataset): ans = np.array(group[key]) if ans.dtype.char == "S": try: ans = ans.astype("U") except UnicodeDecodeError: # There are some strings that must stay in binary, # for example dill pickles. This will obviously also # let "wrong" binary string fail somewhere else... pass kn = key if key.startswith("_list_"): ans = ans.tolist() kn = key[6:] elif key.startswith("_tuple_"): ans = tuple(ans.tolist()) kn = key[7:] elif load_to_memory: kn = key else: # leave as h5py dataset ans = group[key] kn = key dictionary[kn] = ans elif key.startswith('_hspy_AxesManager_'): dictionary[key[len('_hspy_AxesManager_'):]] = AxesManager([ i for k, i in sorted( iter( hdfgroup2dict( group[key], load_to_memory=load_to_memory).items())) ]) elif key.startswith('_list_'): dictionary[key[7 + key[6:].find('_'):]] = \ [i for k, i in sorted(iter( hdfgroup2dict( group[key], load_to_memory=load_to_memory).items() ))] elif key.startswith('_tuple_'): dictionary[key[8 + key[7:].find('_'):]] = tuple([ i for k, i in sorted( iter( hdfgroup2dict( group[key], load_to_memory=load_to_memory).items())) ]) else: dictionary[key] = {} hdfgroup2dict(group[key], dictionary[key], load_to_memory=load_to_memory) return dictionary
def hdfgroup2dict(group, dictionary=None, load_to_memory=True): if dictionary is None: dictionary = {} for key, value in group.attrs.items(): if isinstance(value, bytes): value = value.decode() if isinstance(value, (np.string_, str)): if value == '_None_': value = None elif isinstance(value, np.bool_): value = bool(value) elif isinstance(value, np.ndarray) and value.dtype.char == "S": # Convert strings to unicode value = value.astype("U") if value.dtype.str.endswith("U1"): value = value.tolist() # skip signals - these are handled below. if key.startswith('_sig_'): pass elif key.startswith('_list_empty_'): dictionary[key[len('_list_empty_'):]] = [] elif key.startswith('_tuple_empty_'): dictionary[key[len('_tuple_empty_'):]] = () elif key.startswith('_bs_'): dictionary[key[len('_bs_'):]] = value.tostring() # The following two elif stataments enable reading date and time from # v < 2 of HyperSpy's metadata specifications elif key.startswith('_datetime_date'): date_iso = datetime.date( *ast.literal_eval(value[value.index("("):])).isoformat() dictionary[key.replace("_datetime_", "")] = date_iso elif key.startswith('_datetime_time'): date_iso = datetime.time( *ast.literal_eval(value[value.index("("):])).isoformat() dictionary[key.replace("_datetime_", "")] = date_iso else: dictionary[key] = value if not isinstance(group, h5py.Dataset): for key in group.keys(): if key.startswith('_sig_'): from hyperspy.io import dict2signal dictionary[key[len('_sig_'):]] = ( dict2signal(hdfgroup2signaldict( group[key], load_to_memory=load_to_memory))) elif isinstance(group[key], h5py.Dataset): ans = np.array(group[key]) if ans.dtype.char == "S": try: ans = ans.astype("U") except UnicodeDecodeError: # There are some strings that must stay in binary, # for example dill pickles. This will obviously also # let "wrong" binary string fail somewhere else... pass kn = key if key.startswith("_list_"): ans = ans.tolist() kn = key[6:] elif key.startswith("_tuple_"): ans = tuple(ans.tolist()) kn = key[7:] elif load_to_memory: kn = key else: # leave as h5py dataset ans = group[key] kn = key dictionary[kn] = ans elif key.startswith('_hspy_AxesManager_'): dictionary[key[len('_hspy_AxesManager_'):]] = AxesManager( [i for k, i in sorted(iter( hdfgroup2dict( group[key], load_to_memory=load_to_memory).items() ))]) elif key.startswith('_list_'): dictionary[key[7 + key[6:].find('_'):]] = \ [i for k, i in sorted(iter( hdfgroup2dict( group[key], load_to_memory=load_to_memory).items() ))] elif key.startswith('_tuple_'): dictionary[key[8 + key[7:].find('_'):]] = tuple( [i for k, i in sorted(iter( hdfgroup2dict( group[key], load_to_memory=load_to_memory).items() ))]) else: dictionary[key] = {} hdfgroup2dict( group[key], dictionary[key], load_to_memory=load_to_memory) return dictionary
def hdfgroup2dict(group, dictionary=None, load_to_memory=True): if dictionary is None: dictionary = {} for key, value in group.attrs.items(): if isinstance(value, bytes): value = value.decode() if isinstance(value, (np.string_, str)): if value == '_None_': value = None elif isinstance(value, np.bool_): value = bool(value) elif isinstance(value, np.ndarray) and value.dtype.char == "S": # Convert strings to unicode value = value.astype("U") if value.dtype.str.endswith("U1"): value = value.tolist() # skip signals - these are handled below. if key.startswith('_sig_'): pass elif key.startswith('_list_empty_'): dictionary[key[len('_list_empty_'):]] = [] elif key.startswith('_tuple_empty_'): dictionary[key[len('_tuple_empty_'):]] = () elif key.startswith('_bs_'): dictionary[key[len('_bs_'):]] = value.tostring() # The following is commented out as it could be used to evaluate # arbitrary code i.e. it was a security flaw. We should instead # use a standard string for date and time. # elif key.startswith('_datetime_'): # dictionary[key.replace("_datetime_", "")] = eval(value) else: dictionary[key] = value if not isinstance(group, h5py.Dataset): for key in group.keys(): if key.startswith('_sig_'): from hyperspy.io import dict2signal dictionary[key[len('_sig_'):]] = ( dict2signal(hdfgroup2signaldict( group[key], load_to_memory=load_to_memory))) elif isinstance(group[key], h5py.Dataset): ans = np.array(group[key]) if ans.dtype.char == "S": try: ans = ans.astype("U") except UnicodeDecodeError: # There are some strings that must stay in binary, # for example dill pickles. This will obviously also # let "wrong" binary string fail somewhere else... pass kn = key if key.startswith("_list_"): ans = ans.tolist() kn = key[6:] elif key.startswith("_tuple_"): ans = tuple(ans.tolist()) kn = key[7:] elif load_to_memory: kn = key else: # leave as h5py dataset ans = group[key] kn = key dictionary[kn] = ans elif key.startswith('_hspy_AxesManager_'): dictionary[key[len('_hspy_AxesManager_'):]] = AxesManager( [i for k, i in sorted(iter( hdfgroup2dict( group[key], load_to_memory=load_to_memory).items() ))]) elif key.startswith('_list_'): dictionary[key[7 + key[6:].find('_'):]] = \ [i for k, i in sorted(iter( hdfgroup2dict( group[key], load_to_memory=load_to_memory).items() ))] elif key.startswith('_tuple_'): dictionary[key[8 + key[7:].find('_'):]] = tuple( [i for k, i in sorted(iter( hdfgroup2dict( group[key], load_to_memory=load_to_memory).items() ))]) else: dictionary[key] = {} hdfgroup2dict( group[key], dictionary[key], load_to_memory=load_to_memory) return dictionary
def hdfgroup2dict(group, dictionary=None, load_to_memory=True): if dictionary is None: dictionary = {} for key, value in group.attrs.iteritems(): if isinstance(value, (np.string_, str)): if value == "_None_": value = None elif isinstance(value, np.bool_): value = bool(value) elif isinstance(value, np.ndarray) and value.dtype == np.dtype("|S1"): value = value.tolist() # skip signals - these are handled below. if key.startswith("_sig_"): pass elif key.startswith("_list_empty_"): dictionary[key[len("_list_empty_") :]] = [] elif key.startswith("_tuple_empty_"): dictionary[key[len("_tuple_empty_") :]] = () elif key.startswith("_bs_"): dictionary[key[len("_bs_") :]] = value.tostring() elif key.startswith("_datetime_"): dictionary[key.replace("_datetime_", "")] = eval(value) else: dictionary[key] = value if not isinstance(group, h5py.Dataset): for key in group.keys(): if key.startswith("_sig_"): from hyperspy.io import dict2signal dictionary[key[len("_sig_") :]] = dict2signal( hdfgroup2signaldict(group[key], load_to_memory=load_to_memory) ) elif isinstance(group[key], h5py.Dataset): if key.startswith("_list_"): ans = np.array(group[key]) ans = ans.tolist() kn = key[6:] elif key.startswith("_tuple_"): ans = np.array(group[key]) ans = tuple(ans.tolist()) kn = key[7:] elif load_to_memory: ans = np.array(group[key]) kn = key else: # leave as h5py dataset ans = group[key] kn = key dictionary[kn] = ans elif key.startswith("_hspy_AxesManager_"): dictionary[key[len("_hspy_AxesManager_") :]] = AxesManager( [i for k, i in sorted(iter(hdfgroup2dict(group[key], load_to_memory=load_to_memory).iteritems()))] ) elif key.startswith("_list_"): dictionary[key[7 + key[6:].find("_") :]] = [ i for k, i in sorted(iter(hdfgroup2dict(group[key], load_to_memory=load_to_memory).iteritems())) ] elif key.startswith("_tuple_"): dictionary[key[8 + key[7:].find("_") :]] = tuple( [i for k, i in sorted(iter(hdfgroup2dict(group[key], load_to_memory=load_to_memory).iteritems()))] ) else: dictionary[key] = {} hdfgroup2dict(group[key], dictionary[key], load_to_memory=load_to_memory) return dictionary
def eelsdb(spectrum_type=None, title=None, author=None, element=None, formula=None, edge=None, min_energy=None, max_energy=None, resolution=None, min_energy_compare="gt", max_energy_compare="lt", resolution_compare="lt", max_n=-1, monochromated=None, order=None, order_direction="ASC", verify_certificate=True): r"""Download spectra from the EELS Data Base. Parameters ---------- spectrum_type: {'coreloss', 'lowloss', 'zeroloss', 'xrayabs'}, optional title: string Search spectra titles for a text string. author: string, optional Search authors for a text string. element: string or list of strings, optional Filter for the presence of one or more element. Each string must correspond with a valid element symbol. formula: string Chemical formula of the sample. edge: {'K', 'L1', 'L2,3', 'M2,3', 'M4,5', 'N2,3', 'N4,5' 'O2,3', 'O4,5'}, optional Filter for spectra with a specific class of edge. min_energy, max_energy: float, optional Minimum and maximum energy in eV. resolution: float, optional Energy resolution in eV. resolution_compare: {"lt", "eq", "gt"}, optional, default "lt" "lt" to search for all spectra with resolution less than `resolution`. "eq" for equal, "gt" for greater than. min_energy_compare, max_energy_compare: {"lt", "eq", "gt"}, optional "lt" to search for all spectra with min/max energy less than `min_energy`\`max_energy`. "eq" for equal, "gt" for greater than. Deafault values are "gt"/"lt" for `min_energy`\`max_energy` respectively. monochromated: bool or None (default) max_n: int, default -1 Maximum number of spectra to return. -1 to return all. order: string Key to sort results by. Valid keys are: * "spectrumType", * "spectrumMin", * "spectrumMax", * "stepSize", * "spectrumFormula", * "spectrumElement", * "spectrumUpload", * "source_purity", * "spectrumEdges", * "microscope", * "guntype", * "beamenergy", * "resolution", * "monochromated", * "acquisition_mode", * "convergence", * "collection", * "probesize", * "beamcurrent", * "integratetime", * "readouts", * "detector", * "darkcurrent", * "gainvariation", * "calibration", * "zeroloss_deconv", * "thickness", * "deconv_fourier_log", * "deconv_fourier_ratio", * "deconv_stephens_deconvolution", * "deconv_richardson_lucy", * "deconv_maximum_entropy", * "deconv_other", * "assoc_spectra", * "ref_freetext", * "ref_doi", * "ref_url", * "ref_authors", * "ref_journal", * "ref_volume", * "ref_issue", * "ref_page", * "ref_year", * "ref_title", * "otherURLs" order_direction : {"ASC", "DESC"} Sorting `order` direction. verify_certificate: bool If True, verify the eelsdb website certificate and raise an error if it is invalid. If False, continue querying the database if the certificate is invalid. (This is a potential security risk.) Returns ------- spectra: list A list containing all the spectra matching the given criteria if any. """ # Verify arguments if spectrum_type is not None and spectrum_type not in { 'coreloss', 'lowloss', 'zeroloss', 'xrayabs'}: raise ValueError("spectrum_type must be one of \'coreloss\', \'lowloss\', " "\'zeroloss\', \'xrayabs\'.") valid_edges = [ 'K', 'L1', 'L2,3', 'M2,3', 'M4,5', 'N2,3', 'N4,5', 'O2,3', 'O4,5'] valid_order_keys = [ "spectrumType", "spectrumMin", "spectrumMax", "stepSize", "spectrumFormula", "spectrumElement", "spectrumUpload", "source_purity", "spectrumEdges", "microscope", "guntype", "beamenergy", "resolution", "monochromated", "acquisition_mode", "convergence", "collection", "probesize", "beamcurrent", "integratetime", "readouts", "detector", "darkcurrent", "gainvariation", "calibration", "zeroloss_deconv", "thickness", "deconv_fourier_log", "deconv_fourier_ratio", "deconv_stephens_deconvolution", "deconv_richardson_lucy", "deconv_maximum_entropy", "deconv_other", "assoc_spectra", "ref_freetext", "ref_doi", "ref_url", "ref_authors", "ref_journal", "ref_volume", "ref_issue", "ref_page", "ref_year", "ref_title", "otherURLs"] if edge is not None and edge not in valid_edges: raise ValueError("`edge` must be one of %s." % ", ".join(valid_edges)) if order is not None and order not in valid_order_keys: raise ValueError("`order` must be one of %s." % ", ".join( valid_order_keys)) if order_direction is not None and order_direction not in ["ASC", "DESC"]: raise ValueError("`order_direction` must be \"ASC\" or \"DESC\".") for kwarg, label in ( (resolution_compare, "resolution_compare"), (min_energy_compare, "min_energy_compare"), (max_energy_compare, "max_energy_compare")): if kwarg not in ("lt", "gt", "eq"): raise ValueError("`%s` must be \"lt\", \"eq\" or \"gt\"." % label) if monochromated is not None: monochromated = 1 if monochromated else 0 params = { "type": spectrum_type, "title": title, "author": author, "edge": edge, "min_energy": min_energy, "max_energy": max_energy, "resolution": resolution, "resolution_compare": resolution_compare, "monochromated": monochromated, "formula": formula, "min_energy_compare": min_energy_compare, "max_energy_compare": max_energy_compare, "per_page": max_n, "order": order, "order_direction": order_direction, } if isinstance(element, str): params["element"] = element else: params["element[]"] = element request = requests.get('http://api.eelsdb.eu/spectra', params=params, verify=verify_certificate) spectra = [] jsons = request.json() if "message" in jsons: # Invalid query, EELSdb raises error. raise IOError( "Please report the following error to the HyperSpy developers: " "%s" % jsons["message"]) for json_spectrum in jsons: download_link = json_spectrum['download_link'] msa_string = requests.get(download_link, verify=verify_certificate).text try: s = dict2signal(parse_msa_string(msa_string)[0]) emsa = s.original_metadata s.original_metadata = s.original_metadata.__class__( {'json': json_spectrum}) s.original_metadata.emsa = emsa spectra.append(s) except: # parse_msa_string or dict2signal may fail if the EMSA file is not # a valid one. _logger.exception( "Failed to load the spectrum.\n" "Title: %s id: %s.\n" "Please report this error to http://eelsdb.eu/about \n" % (json_spectrum["title"], json_spectrum["id"])) if not spectra: _logger.info( "The EELS database does not contain any spectra matching your query" ". If you have some, why not submitting them " "https://eelsdb.eu/submit-data/ ?\n") else: # Add some info from json to metadata # Values with units are not yet supported by HyperSpy (v0.8) so # we can't get map those fields. for s in spectra: if spectrum_type == "xrayabs": s.set_signal_type("XAS") json_md = s.original_metadata.json s.metadata.General.title = json_md.title if s.metadata.Signal.signal_type == "EELS": if json_md.elements: try: s.add_elements(json_md.elements) except ValueError: _logger.exception( "The following spectrum contains invalid chemical " "element information:\n" "Title: %s id: %s. Elements: %s.\n" "Please report this error in " "http://eelsdb.eu/about \n" % (json_md.title, json_md.id, json_md.elements)) if "collection" in json_md and " mrad" in json_md.collection: beta = float(json_md.collection.replace(" mrad", "")) s.metadata.set_item( "Acquisition_instrument.TEM.Detector.EELS.collection_angle", beta) if "convergence" in json_md and " mrad" in json_md.convergence: alpha = float(json_md.convergence.replace(" mrad", "")) s.metadata.set_item( "Acquisition_instrument.TEM.convergence_angle", alpha) if "beamenergy" in json_md and " kV" in json_md.beamenergy: beam_energy = float(json_md.beamenergy.replace(" kV", "")) s.metadata.set_item( "Acquisition_instrument.TEM.beam_energy", beam_energy) # We don't yet support units, so we cannot map the thickness # s.metadata.set_item("Sample.thickness", json_md.thickness) s.metadata.set_item("Sample.description", json_md.description) s.metadata.set_item("Sample.chemical_formula", json_md.formula) s.metadata.set_item("General.author", json_md.author.name) s.metadata.set_item("Acquisition_instrument.TEM.microscope", json_md.microscope) return spectra
def _group2dict(self, group, dictionary=None, lazy=False): if dictionary is None: dictionary = {} for key, value in group.attrs.items(): if isinstance(value, bytes): value = value.decode() if isinstance(value, (np.string_, str)): if value == '_None_': value = None elif isinstance(value, np.bool_): value = bool(value) elif isinstance(value, np.ndarray) and value.dtype.char == "S": # Convert strings to unicode value = value.astype("U") if value.dtype.str.endswith("U1"): value = value.tolist() # skip signals - these are handled below. if key.startswith('_sig_'): pass elif key.startswith('_list_empty_'): dictionary[key[len('_list_empty_'):]] = [] elif key.startswith('_tuple_empty_'): dictionary[key[len('_tuple_empty_'):]] = () elif key.startswith('_bs_'): dictionary[key[len('_bs_'):]] = value.tobytes() # The following two elif stataments enable reading date and time from # v < 2 of HyperSpy's metadata specifications elif key.startswith('_datetime_date'): date_iso = datetime.date( *ast.literal_eval(value[value.index("("):])).isoformat() dictionary[key.replace("_datetime_", "")] = date_iso elif key.startswith('_datetime_time'): date_iso = datetime.time( *ast.literal_eval(value[value.index("("):])).isoformat() dictionary[key.replace("_datetime_", "")] = date_iso else: dictionary[key] = value if not isinstance(group, self.Dataset): for key in group.keys(): if key.startswith('_sig_'): from hyperspy.io import dict2signal dictionary[key[len('_sig_'):]] = ( dict2signal(self.group2signaldict( group[key], lazy=lazy))) elif isinstance(group[key], self.Dataset): dat = group[key] kn = key if key.startswith("_list_"): if (h5py.check_string_dtype(dat.dtype) and hasattr(dat, 'asstr')): # h5py 3.0 and newer # https://docs.h5py.org/en/3.0.0/strings.html dat = dat.asstr()[:] ans = np.array(dat) ans = ans.tolist() kn = key[6:] elif key.startswith("_tuple_"): ans = np.array(dat) ans = tuple(ans.tolist()) kn = key[7:] elif dat.dtype.char == "S": ans = np.array(dat) try: ans = ans.astype("U") except UnicodeDecodeError: # There are some strings that must stay in binary, # for example dill pickles. This will obviously also # let "wrong" binary string fail somewhere else... pass elif lazy: ans = da.from_array(dat, chunks=dat.chunks) else: ans = np.array(dat) dictionary[kn] = ans elif key.startswith('_hspy_AxesManager_'): dictionary[key[len('_hspy_AxesManager_'):]] = AxesManager( [i for k, i in sorted(iter( self._group2dict( group[key], lazy=lazy).items() ))]) elif key.startswith('_list_'): dictionary[key[7 + key[6:].find('_'):]] = \ [i for k, i in sorted(iter( self._group2dict( group[key], lazy=lazy).items() ))] elif key.startswith('_tuple_'): dictionary[key[8 + key[7:].find('_'):]] = tuple( [i for k, i in sorted(iter( self._group2dict( group[key], lazy=lazy).items() ))]) else: dictionary[key] = {} self._group2dict( group[key], dictionary[key], lazy=lazy) return dictionary
def hdfgroup2dict(group, dictionary=None, load_to_memory=True): if dictionary is None: dictionary = {} for key, value in group.attrs.iteritems(): if isinstance(value, (np.string_, str)): if value == '_None_': value = None elif isinstance(value, np.bool_): value = bool(value) elif isinstance(value, np.ndarray) and \ value.dtype == np.dtype('|S1'): value = value.tolist() # skip signals - these are handled below. if key.startswith('_sig_'): pass elif key.startswith('_list_empty_'): dictionary[key[len('_list_empty_'):]] = [] elif key.startswith('_tuple_empty_'): dictionary[key[len('_tuple_empty_'):]] = () elif key.startswith('_bs_'): dictionary[key[len('_bs_'):]] = value.tostring() elif key.startswith('_datetime_'): dictionary[key.replace("_datetime_", "")] = eval(value) else: dictionary[key] = value if not isinstance(group, h5py.Dataset): for key in group.keys(): if key.startswith('_sig_'): from hyperspy.io import dict2signal dictionary[key[len('_sig_'):]] = ( dict2signal(hdfgroup2signaldict(group[key], load_to_memory=load_to_memory))) elif isinstance(group[key], h5py.Dataset): if key.startswith("_list_"): ans = np.array(group[key]) ans = ans.tolist() kn = key[6:] elif key.startswith("_tuple_"): ans = np.array(group[key]) ans = tuple(ans.tolist()) kn = key[7:] elif load_to_memory: ans = np.array(group[key]) kn = key else: # leave as h5py dataset ans = group[key] kn = key dictionary[kn] = ans elif key.startswith('_hspy_AxesManager_'): dictionary[key[len('_hspy_AxesManager_'):]] = \ AxesManager([i for k, i in sorted(iter( hdfgroup2dict(group[key], load_to_memory=load_to_memory).iteritems()))]) elif key.startswith('_list_'): dictionary[key[7 + key[6:].find('_'):]] = \ [i for k, i in sorted(iter( hdfgroup2dict(group[key], load_to_memory=load_to_memory).iteritems()))] elif key.startswith('_tuple_'): dictionary[key[8 + key[7:].find('_'):]] = tuple( [i for k, i in sorted(iter( hdfgroup2dict(group[key], load_to_memory=load_to_memory).iteritems()))]) else: dictionary[key] = {} hdfgroup2dict( group[key], dictionary[key], load_to_memory=load_to_memory) return dictionary
def eelsdb(spectrum_type=None, title=None, author=None, element=None, formula=None, edge=None, min_energy=None, max_energy=None, resolution=None, min_energy_compare="gt", max_energy_compare="lt", resolution_compare="lt", max_n=-1, monochromated=None, order=None, order_direction="ASC"): r"""Download spectra from the EELS Data Base. Parameters ---------- spectrum_type: {'coreloss', 'lowloss', 'zeroloss', 'xrayabs'}, optional title: string Search spectra titles for a text string. author: string, optional Search authors for a text string. element: string or list of strings, optional Filter for the presence of one or more element. Each string must correspond with a valid element symbol. formula: string Chemical formula of the sample. edge: {'K', 'L1', 'L2,3', 'M2,3', 'M4,5', 'N2,3', 'N4,5' 'O2,3', 'O4,5'}, optional Filter for spectra with a specific class of edge. min_energy, max_energy: float, optional Minimum and maximum energy in eV. resolution: float, optional Energy resolution in eV. resolution_compare: {"lt", "eq", "gt"}, optional, default "lt" "lt" to search for all spectra with resolution less than `resolution`. "eq" for equal, "gt" for greater than. min_energy_compare, max_energy_compare: {"lt", "eq", "gt"}, optional "lt" to search for all spectra with min/max energy less than `min_energy`\`max_energy`. "eq" for equal, "gt" for greater than. Deafault values are "gt"/"lt" for `min_energy`\`max_energy` respectively. monochromated: bool or None (default) max_n: int, default -1 Maximum number of spectra to return. -1 to return all. order: string Key to sort results by. Valid keys are: * "spectrumType", * "spectrumMin", * "spectrumMax", * "stepSize", * "spectrumFormula", * "spectrumElement", * "spectrumUpload", * "source_purity", * "spectrumEdges", * "microscope", * "guntype", * "beamenergy", * "resolution", * "monochromated", * "acquisition_mode", * "convergence", * "collection", * "probesize", * "beamcurrent", * "integratetime", * "readouts", * "detector", * "darkcurrent", * "gainvariation", * "calibration", * "zeroloss_deconv", * "thickness", * "deconv_fourier_log", * "deconv_fourier_ratio", * "deconv_stephens_deconvolution", * "deconv_richardson_lucy", * "deconv_maximum_entropy", * "deconv_other", * "assoc_spectra", * "ref_freetext", * "ref_doi", * "ref_url", * "ref_authors", * "ref_journal", * "ref_volume", * "ref_issue", * "ref_page", * "ref_year", * "ref_title", * "otherURLs" order_direction : {"ASC", "DESC"} Sorting `order` direction. Returns ------- spectra: list A list containing all the spectra matching the given criteria if any. """ # Verify arguments if spectrum_type is not None and spectrum_type not in { 'coreloss', 'lowloss', 'zeroloss', 'xrayabs'}: raise ValueError("spectrum_type must be one of \'coreloss\', \'lowloss\', " "\'zeroloss\', \'xrayabs\'.") valid_edges = [ 'K', 'L1', 'L2,3', 'M2,3', 'M4,5', 'N2,3', 'N4,5', 'O2,3', 'O4,5'] valid_order_keys = [ "spectrumType", "spectrumMin", "spectrumMax", "stepSize", "spectrumFormula", "spectrumElement", "spectrumUpload", "source_purity", "spectrumEdges", "microscope", "guntype", "beamenergy", "resolution", "monochromated", "acquisition_mode", "convergence", "collection", "probesize", "beamcurrent", "integratetime", "readouts", "detector", "darkcurrent", "gainvariation", "calibration", "zeroloss_deconv", "thickness", "deconv_fourier_log", "deconv_fourier_ratio", "deconv_stephens_deconvolution", "deconv_richardson_lucy", "deconv_maximum_entropy", "deconv_other", "assoc_spectra", "ref_freetext", "ref_doi", "ref_url", "ref_authors", "ref_journal", "ref_volume", "ref_issue", "ref_page", "ref_year", "ref_title", "otherURLs"] if edge is not None and edge not in valid_edges: raise ValueError("`edge` must be one of %s." % ", ".join(valid_edges)) if order is not None and order not in valid_order_keys: raise ValueError("`order` must be one of %s." % ", ".join( valid_order_keys)) if order_direction is not None and order_direction not in ["ASC", "DESC"]: raise ValueError("`order_direction` must be \"ASC\" or \"DESC\".") for kwarg, label in ( (resolution_compare, "resolution_compare"), (min_energy_compare, "min_energy_compare"), (max_energy_compare, "max_energy_compare")): if kwarg not in ("lt", "gt", "eq"): raise ValueError("`%s` must be \"lt\", \"eq\" or \"gt\"." % label) if monochromated is not None: monochromated = 1 if monochromated else 0 params = { "type": spectrum_type, "title": title, "author": author, "edge": edge, "min_energy": min_energy, "max_energy": max_energy, "resolution": resolution, "resolution_compare": resolution_compare, "monochromated": monochromated, "formula": formula, "min_energy_compare": min_energy_compare, "max_energy_compare": max_energy_compare, "per_page": max_n, "order": order, "order_direction": order_direction, } if isinstance(element, str): params["element"] = element else: params["element[]"] = element request = requests.get('http://api.eelsdb.eu/spectra', params=params, ) spectra = [] jsons = request.json() if "message" in jsons: # Invalid query, EELSdb raises error. raise IOError( "Please report the following error to the HyperSpy developers: " "%s" % jsons["message"]) for json_spectrum in jsons: download_link = json_spectrum['download_link'] msa_string = requests.get(download_link).text try: s = dict2signal(parse_msa_string(msa_string)[0]) emsa = s.original_metadata s.original_metadata = s.original_metadata.__class__( {'json': json_spectrum}) s.original_metadata.emsa = emsa spectra.append(s) except: # parse_msa_string or dict2signal may fail if the EMSA file is not # a valid one. _logger.exception( "Failed to load the spectrum.\n" "Title: %s id: %s.\n" "Please report this error to http://eelsdb.eu/about \n" % (json_spectrum["title"], json_spectrum["id"])) if not spectra: _logger.information( "The EELS database does not contain any spectra matching your query" ". If you have some, why not submitting them " "https://eelsdb.eu/submit-data/ ?\n") else: # Add some info from json to metadata # Values with units are not yet supported by HyperSpy (v0.8) so # we can't get map those fields. for s in spectra: if spectrum_type == "xrayabs": s.set_signal_type("XAS") json_md = s.original_metadata.json s.metadata.General.title = json_md.title if s.metadata.Signal.signal_type == "EELS": if json_md.elements: try: s.add_elements(json_md.elements) except ValueError: _logger.exception( "The following spectrum contains invalid chemical " "element information:\n" "Title: %s id: %s. Elements: %s.\n" "Please report this error in " "http://eelsdb.eu/about \n" % (json_md.title, json_md.id, json_md.elements)) if "collection" in json_md and " mrad" in json_md.collection: beta = float(json_md.collection.replace(" mrad", "")) s.metadata.set_item( "Acquisition_instrument.TEM.Detector.EELS.collection_angle", beta) if "convergence" in json_md and " mrad" in json_md.convergence: alpha = float(json_md.convergence.replace(" mrad", "")) s.metadata.set_item( "Acquisition_instrument.TEM.convergence_angle", alpha) if "beamenergy" in json_md and " kV" in json_md.beamenergy: beam_energy = float(json_md.beamenergy.replace(" kV", "")) s.metadata.set_item( "Acquisition_instrument.TEM.beam_energy", beam_energy) # We don't yet support units, so we cannot map the thickness # s.metadata.set_item("Sample.thickness", json_md.thickness) s.metadata.set_item("Sample.description", json_md.description) s.metadata.set_item("Sample.chemical_formula", json_md.formula) s.metadata.set_item("General.author", json_md.author.name) s.metadata.set_item("Acquisition_instrument.TEM.microscope", json_md.microscope) return spectra
def process(directory, threshold=6, integrate=False, counting=False, hdr=None, mean_e=256, nav_shape=None, chunk_shape=None, verbose=False): if verbose: _logger.setLevel(logging.DEBUG) handler = logging.StreamHandler(sys.stdout) handler.setLevel(logging.DEBUG) formatter = logging.Formatter('%(message)s \n') handler.setFormatter(formatter) _logger.addHandler(handler) _logger.info(msg="\n\n .SEQ Processor Application (and Counting)...\n" "Created by: Carter Francis ([email protected])\n" "Updated 2021-06-18\n" "------------------\n") _logger.info(msg="Version:" + __version__) tick = time.time() file_dict = get_files(folder=directory) for key in file_dict: if len(file_dict[key]) == 0: file_dict[key].pop() else: file_dict[key] = file_dict[key][0] if "top" in file_dict and "bottom" in file_dict: file_dict.pop("seq") data_dict = cel_file_reader(**file_dict, nav_shape=nav_shape, chunk_shape=chunk_shape, lazy=True) elif "seq" in file_dict: data_dict = file_reader(**file_dict, nav_shape=nav_shape, chunk_shape=chunk_shape, lazy=True) if hdr is not None: hdr = hs.load(hdr).data else: hdr = None if hdr is None and integrate is False: dtype = bool else: dtype = np.float32 if counting: data_dict["data"] = data_dict["data"].map_blocks( _counting_filter_cpu, threshold=threshold, integrate=integrate, hdr_mask=hdr, method="maximum", mean_electron_val=mean_e, dtype=dtype) _logger.info(data_dict) sig = dict2signal(data_dict, lazy=True) _logger.info("Data... :" + str(sig.data)) _logger.info("Dtype:" + str(sig.data.dtype)) _logger.info("Saving... ") da.to_zarr(sig.data, directory + "_zarr", overwrite=True) #sig.save(directory + ".hspy", # compression=False, # overwrite=True) tock = time.time() _logger.info("Total time elapsed : " + str(tock - tick) + " sec") return sig