def hdfgroup2dict(group, dictionary={}):
for key, value in group.attrs.iteritems():
if isinstance(value, (np.string_, str)):
if value == '_None_':
value = None
elif isinstance(value, np.bool_):
value = bool(value)
elif isinstance(value, np.ndarray) and \
value.dtype == np.dtype('|S1'):
value = value.tolist()
# skip signals - these are handled below.
if key.startswith('_sig_'):
pass
elif key.startswith('_datetime_'):
dictionary[key.replace("_datetime_", "")] = eval(value)
else:
dictionary[key] = value
if not isinstance(group, h5py.Dataset):
for key in group.keys():
if key.startswith('_sig_'):
from hyperspy.io import dict2signal
dictionary[key[len('_sig_'):]] = (
dict2signal(hdfgroup2signaldict(group[key])))
elif isinstance(group[key], h5py.Dataset):
dictionary[key] = np.array(group[key])
elif key.startswith('_hspy_AxesManager_'):
dictionary[key[len('_hspy_AxesManager_'):]] = \
AxesManager([i
for k, i in sorted(iter(
hdfgroup2dict(group[key]).iteritems()))])
else:
dictionary[key] = {}
hdfgroup2dict(group[key], dictionary[key])
return dictionary
def load(
seq=None,
gain=None,
dark=None,
metadata=None,
xml_file=None,
lazy=False,
chunk_shape=None,
nav_shape=None,
):
"""Loads a .seq file into hyperspy. Metadata taken from
the .metadata file as well as from a paramters.txt file that
can be passed as well. The parameters file is used calibrate using
the 4-D STEM parameters for some signal.
Parameters
-----------
filename: str
The name of the file to be loaded (.seq file)
"""
sig = dict2signal(file_reader(filename=seq,
dark=dark,
gain=gain,
metadata=metadata,
xml_file=xml_file,
lazy=lazy,
chunk_shape=chunk_shape,
nav_shape=nav_shape),
lazy=lazy)
return sig
def hdfgroup2dict(group, dictionary={}):
for key, value in group.attrs.iteritems():
if isinstance(value, (np.string_, str)):
if value == '_None_':
value = None
elif isinstance(value, np.bool_):
value = bool(value)
elif isinstance(value, np.ndarray) and \
value.dtype == np.dtype('|S1'):
value = value.tolist()
# skip signals - these are handled below.
if key.startswith('_sig_'):
pass
elif key.startswith('_datetime_'):
dictionary[key.replace("_datetime_", "")] = eval(value)
else:
dictionary[key] = value
if not isinstance(group, h5py.Dataset):
for key in group.keys():
if key.startswith('_sig_'):
from hyperspy.io import dict2signal
dictionary[key[len('_sig_'):]] = (dict2signal(
hdfgroup2signaldict(group[key])))
elif isinstance(group[key], h5py.Dataset):
dictionary[key] = np.array(group[key])
elif key.startswith('_hspy_AxesManager_'):
dictionary[key[len('_hspy_AxesManager_'):]] = \
AxesManager([i
for k, i in sorted(iter(
hdfgroup2dict(group[key]).iteritems()))])
else:
dictionary[key] = {}
hdfgroup2dict(group[key], dictionary[key])
return dictionary
def load_celeritas(
top,
bottom,
dark=None,
gain=None,
metadata=None,
xml_file=None,
lazy=False,
chunk_shape=None,
nav_shape=None,
):
"""Loads a .seq file into hyperspy. Metadata taken from
the .metadata file as well as from a paramters.txt file that
can be passed as well. The parameters file is used calibrate using
the 4-D STEM parameters for some signal.
Parameters
-----------
lazy : bool, default False
Load the signal lazily.
top : str
The filename for the top part of the detector
bottom:
The filename for the bottom part of the detector
dark: str
The filename for the dark reference to be applied to the data
gain: str
The filename for the gain reference to be applied to the data
metadata: str
The filename for the metadata file
xml_file: str
The filename for the xml file to be applied.
nav_shape:
The navigation shape for the dataset to be divided into to
chunks:
If lazy=True this divides the dataset into this many chunks
"""
sig = dict2signal(c_reader(top=top,
bottom=bottom,
gain=gain,
dark=dark,
metadata=metadata,
xml_file=xml_file,
lazy=lazy,
chunk_shape=chunk_shape,
nav_shape=nav_shape),
lazy=lazy)
return sig
def hdfgroup2dict(group, dictionary = {}):
for key, value in group.attrs.iteritems():
if type(value) is np.string_:
if value == '_None_':
value = None
else:
try:
value = value.decode('utf8')
except UnicodeError:
# For old files
value = value.decode('latin-1')
elif type(value) is np.bool_:
value = bool(value)
elif type(value) is np.ndarray and \
value.dtype == np.dtype('|S1'):
value = value.tolist()
# skip signals - these are handled below.
if key.startswith('_sig_'):
pass
else:
dictionary[key] = value
if not isinstance(group, h5py.Dataset):
for key in group.keys():
if key.startswith('_sig_'):
from hyperspy.io import dict2signal
dictionary[key[len('_sig_'):]] = (
dict2signal(hdfgroup2signaldict(group[key])))
elif isinstance(group[key],h5py.Dataset):
dictionary[key]=np.array(group[key])
elif key.startswith('_hspy_AxesManager_'):
dictionary[key[len('_hspy_AxesManager_'):]] = \
AxesManager([i
for k, i in sorted(iter(
hdfgroup2dict(group[key]).iteritems()))])
else:
dictionary[key] = {}
hdfgroup2dict(group[key], dictionary[key])
return dictionary
def hdfgroup2dict(group, dictionary={}):
for key, value in group.attrs.iteritems():
if type(value) is np.string_:
if value == '_None_':
value = None
else:
try:
value = value.decode('utf8')
except UnicodeError:
# For old files
value = value.decode('latin-1')
elif type(value) is np.bool_:
value = bool(value)
elif type(value) is np.ndarray and \
value.dtype == np.dtype('|S1'):
value = value.tolist()
# skip signals - these are handled below.
if key.startswith('_sig_'):
pass
else:
dictionary[key] = value
if not isinstance(group, h5py.Dataset):
for key in group.keys():
if key.startswith('_sig_'):
from hyperspy.io import dict2signal
dictionary[key[len('_sig_'):]] = (dict2signal(
hdfgroup2signaldict(group[key])))
elif isinstance(group[key], h5py.Dataset):
dictionary[key] = np.array(group[key])
elif key.startswith('_hspy_AxesManager_'):
dictionary[key[len('_hspy_AxesManager_'):]] = \
AxesManager([i
for k, i in sorted(iter(
hdfgroup2dict(group[key]).iteritems()))])
else:
dictionary[key] = {}
hdfgroup2dict(group[key], dictionary[key])
return dictionary
def hdfgroup2dict(group, dictionary=None, load_to_memory=True):
if dictionary is None:
dictionary = {}
for key, value in group.attrs.items():
if isinstance(value, bytes):
value = value.decode()
if isinstance(value, (np.string_, str)):
if value == '_None_':
value = None
elif isinstance(value, np.bool_):
value = bool(value)
elif isinstance(value, np.ndarray) and value.dtype.char == "S":
# Convert strings to unicode
value = value.astype("U")
if value.dtype.str.endswith("U1"):
value = value.tolist()
# skip signals - these are handled below.
if key.startswith('_sig_'):
pass
elif key.startswith('_list_empty_'):
dictionary[key[len('_list_empty_'):]] = []
elif key.startswith('_tuple_empty_'):
dictionary[key[len('_tuple_empty_'):]] = ()
elif key.startswith('_bs_'):
dictionary[key[len('_bs_'):]] = value.tostring()
# The following two elif stataments enable reading date and time from
# v < 2 of HyperSpy's metadata specifications
elif key.startswith('_datetime_date'):
date_iso = datetime.date(
*ast.literal_eval(value[value.index("("):])).isoformat()
dictionary[key.replace("_datetime_", "")] = date_iso
elif key.startswith('_datetime_time'):
date_iso = datetime.time(
*ast.literal_eval(value[value.index("("):])).isoformat()
dictionary[key.replace("_datetime_", "")] = date_iso
else:
dictionary[key] = value
if not isinstance(group, h5py.Dataset):
for key in group.keys():
if key.startswith('_sig_'):
from hyperspy.io import dict2signal
dictionary[key[len('_sig_'):]] = (dict2signal(
hdfgroup2signaldict(group[key],
load_to_memory=load_to_memory)))
elif isinstance(group[key], h5py.Dataset):
ans = np.array(group[key])
if ans.dtype.char == "S":
try:
ans = ans.astype("U")
except UnicodeDecodeError:
# There are some strings that must stay in binary,
# for example dill pickles. This will obviously also
# let "wrong" binary string fail somewhere else...
pass
kn = key
if key.startswith("_list_"):
ans = ans.tolist()
kn = key[6:]
elif key.startswith("_tuple_"):
ans = tuple(ans.tolist())
kn = key[7:]
elif load_to_memory:
kn = key
else:
# leave as h5py dataset
ans = group[key]
kn = key
dictionary[kn] = ans
elif key.startswith('_hspy_AxesManager_'):
dictionary[key[len('_hspy_AxesManager_'):]] = AxesManager([
i for k, i in sorted(
iter(
hdfgroup2dict(
group[key],
load_to_memory=load_to_memory).items()))
])
elif key.startswith('_list_'):
dictionary[key[7 + key[6:].find('_'):]] = \
[i for k, i in sorted(iter(
hdfgroup2dict(
group[key], load_to_memory=load_to_memory).items()
))]
elif key.startswith('_tuple_'):
dictionary[key[8 + key[7:].find('_'):]] = tuple([
i for k, i in sorted(
iter(
hdfgroup2dict(
group[key],
load_to_memory=load_to_memory).items()))
])
else:
dictionary[key] = {}
hdfgroup2dict(group[key],
dictionary[key],
load_to_memory=load_to_memory)
return dictionary
def hdfgroup2dict(group, dictionary=None, load_to_memory=True):
if dictionary is None:
dictionary = {}
for key, value in group.attrs.items():
if isinstance(value, bytes):
value = value.decode()
if isinstance(value, (np.string_, str)):
if value == '_None_':
value = None
elif isinstance(value, np.bool_):
value = bool(value)
elif isinstance(value, np.ndarray) and value.dtype.char == "S":
# Convert strings to unicode
value = value.astype("U")
if value.dtype.str.endswith("U1"):
value = value.tolist()
# skip signals - these are handled below.
if key.startswith('_sig_'):
pass
elif key.startswith('_list_empty_'):
dictionary[key[len('_list_empty_'):]] = []
elif key.startswith('_tuple_empty_'):
dictionary[key[len('_tuple_empty_'):]] = ()
elif key.startswith('_bs_'):
dictionary[key[len('_bs_'):]] = value.tostring()
# The following two elif stataments enable reading date and time from
# v < 2 of HyperSpy's metadata specifications
elif key.startswith('_datetime_date'):
date_iso = datetime.date(
*ast.literal_eval(value[value.index("("):])).isoformat()
dictionary[key.replace("_datetime_", "")] = date_iso
elif key.startswith('_datetime_time'):
date_iso = datetime.time(
*ast.literal_eval(value[value.index("("):])).isoformat()
dictionary[key.replace("_datetime_", "")] = date_iso
else:
dictionary[key] = value
if not isinstance(group, h5py.Dataset):
for key in group.keys():
if key.startswith('_sig_'):
from hyperspy.io import dict2signal
dictionary[key[len('_sig_'):]] = (
dict2signal(hdfgroup2signaldict(
group[key], load_to_memory=load_to_memory)))
elif isinstance(group[key], h5py.Dataset):
ans = np.array(group[key])
if ans.dtype.char == "S":
try:
ans = ans.astype("U")
except UnicodeDecodeError:
# There are some strings that must stay in binary,
# for example dill pickles. This will obviously also
# let "wrong" binary string fail somewhere else...
pass
kn = key
if key.startswith("_list_"):
ans = ans.tolist()
kn = key[6:]
elif key.startswith("_tuple_"):
ans = tuple(ans.tolist())
kn = key[7:]
elif load_to_memory:
kn = key
else:
# leave as h5py dataset
ans = group[key]
kn = key
dictionary[kn] = ans
elif key.startswith('_hspy_AxesManager_'):
dictionary[key[len('_hspy_AxesManager_'):]] = AxesManager(
[i for k, i in sorted(iter(
hdfgroup2dict(
group[key], load_to_memory=load_to_memory).items()
))])
elif key.startswith('_list_'):
dictionary[key[7 + key[6:].find('_'):]] = \
[i for k, i in sorted(iter(
hdfgroup2dict(
group[key], load_to_memory=load_to_memory).items()
))]
elif key.startswith('_tuple_'):
dictionary[key[8 + key[7:].find('_'):]] = tuple(
[i for k, i in sorted(iter(
hdfgroup2dict(
group[key], load_to_memory=load_to_memory).items()
))])
else:
dictionary[key] = {}
hdfgroup2dict(
group[key],
dictionary[key],
load_to_memory=load_to_memory)
return dictionary
def hdfgroup2dict(group, dictionary=None, load_to_memory=True):
if dictionary is None:
dictionary = {}
for key, value in group.attrs.items():
if isinstance(value, bytes):
value = value.decode()
if isinstance(value, (np.string_, str)):
if value == '_None_':
value = None
elif isinstance(value, np.bool_):
value = bool(value)
elif isinstance(value, np.ndarray) and value.dtype.char == "S":
# Convert strings to unicode
value = value.astype("U")
if value.dtype.str.endswith("U1"):
value = value.tolist()
# skip signals - these are handled below.
if key.startswith('_sig_'):
pass
elif key.startswith('_list_empty_'):
dictionary[key[len('_list_empty_'):]] = []
elif key.startswith('_tuple_empty_'):
dictionary[key[len('_tuple_empty_'):]] = ()
elif key.startswith('_bs_'):
dictionary[key[len('_bs_'):]] = value.tostring()
# The following is commented out as it could be used to evaluate
# arbitrary code i.e. it was a security flaw. We should instead
# use a standard string for date and time.
# elif key.startswith('_datetime_'):
# dictionary[key.replace("_datetime_", "")] = eval(value)
else:
dictionary[key] = value
if not isinstance(group, h5py.Dataset):
for key in group.keys():
if key.startswith('_sig_'):
from hyperspy.io import dict2signal
dictionary[key[len('_sig_'):]] = (
dict2signal(hdfgroup2signaldict(
group[key], load_to_memory=load_to_memory)))
elif isinstance(group[key], h5py.Dataset):
ans = np.array(group[key])
if ans.dtype.char == "S":
try:
ans = ans.astype("U")
except UnicodeDecodeError:
# There are some strings that must stay in binary,
# for example dill pickles. This will obviously also
# let "wrong" binary string fail somewhere else...
pass
kn = key
if key.startswith("_list_"):
ans = ans.tolist()
kn = key[6:]
elif key.startswith("_tuple_"):
ans = tuple(ans.tolist())
kn = key[7:]
elif load_to_memory:
kn = key
else:
# leave as h5py dataset
ans = group[key]
kn = key
dictionary[kn] = ans
elif key.startswith('_hspy_AxesManager_'):
dictionary[key[len('_hspy_AxesManager_'):]] = AxesManager(
[i for k, i in sorted(iter(
hdfgroup2dict(
group[key], load_to_memory=load_to_memory).items()
))])
elif key.startswith('_list_'):
dictionary[key[7 + key[6:].find('_'):]] = \
[i for k, i in sorted(iter(
hdfgroup2dict(
group[key], load_to_memory=load_to_memory).items()
))]
elif key.startswith('_tuple_'):
dictionary[key[8 + key[7:].find('_'):]] = tuple(
[i for k, i in sorted(iter(
hdfgroup2dict(
group[key], load_to_memory=load_to_memory).items()
))])
else:
dictionary[key] = {}
hdfgroup2dict(
group[key],
dictionary[key],
load_to_memory=load_to_memory)
return dictionary
def hdfgroup2dict(group, dictionary=None, load_to_memory=True):
if dictionary is None:
dictionary = {}
for key, value in group.attrs.iteritems():
if isinstance(value, (np.string_, str)):
if value == "_None_":
value = None
elif isinstance(value, np.bool_):
value = bool(value)
elif isinstance(value, np.ndarray) and value.dtype == np.dtype("|S1"):
value = value.tolist()
# skip signals - these are handled below.
if key.startswith("_sig_"):
pass
elif key.startswith("_list_empty_"):
dictionary[key[len("_list_empty_") :]] = []
elif key.startswith("_tuple_empty_"):
dictionary[key[len("_tuple_empty_") :]] = ()
elif key.startswith("_bs_"):
dictionary[key[len("_bs_") :]] = value.tostring()
elif key.startswith("_datetime_"):
dictionary[key.replace("_datetime_", "")] = eval(value)
else:
dictionary[key] = value
if not isinstance(group, h5py.Dataset):
for key in group.keys():
if key.startswith("_sig_"):
from hyperspy.io import dict2signal
dictionary[key[len("_sig_") :]] = dict2signal(
hdfgroup2signaldict(group[key], load_to_memory=load_to_memory)
)
elif isinstance(group[key], h5py.Dataset):
if key.startswith("_list_"):
ans = np.array(group[key])
ans = ans.tolist()
kn = key[6:]
elif key.startswith("_tuple_"):
ans = np.array(group[key])
ans = tuple(ans.tolist())
kn = key[7:]
elif load_to_memory:
ans = np.array(group[key])
kn = key
else:
# leave as h5py dataset
ans = group[key]
kn = key
dictionary[kn] = ans
elif key.startswith("_hspy_AxesManager_"):
dictionary[key[len("_hspy_AxesManager_") :]] = AxesManager(
[i for k, i in sorted(iter(hdfgroup2dict(group[key], load_to_memory=load_to_memory).iteritems()))]
)
elif key.startswith("_list_"):
dictionary[key[7 + key[6:].find("_") :]] = [
i for k, i in sorted(iter(hdfgroup2dict(group[key], load_to_memory=load_to_memory).iteritems()))
]
elif key.startswith("_tuple_"):
dictionary[key[8 + key[7:].find("_") :]] = tuple(
[i for k, i in sorted(iter(hdfgroup2dict(group[key], load_to_memory=load_to_memory).iteritems()))]
)
else:
dictionary[key] = {}
hdfgroup2dict(group[key], dictionary[key], load_to_memory=load_to_memory)
return dictionary
def eelsdb(spectrum_type=None, title=None, author=None, element=None, formula=None,
edge=None, min_energy=None, max_energy=None, resolution=None,
min_energy_compare="gt", max_energy_compare="lt",
resolution_compare="lt", max_n=-1, monochromated=None, order=None,
order_direction="ASC", verify_certificate=True):
r"""Download spectra from the EELS Data Base.
Parameters
----------
spectrum_type: {'coreloss', 'lowloss', 'zeroloss', 'xrayabs'}, optional
title: string
Search spectra titles for a text string.
author: string, optional
Search authors for a text string.
element: string or list of strings, optional
Filter for the presence of one or more element. Each string must
correspond with a valid element symbol.
formula: string
Chemical formula of the sample.
edge: {'K', 'L1', 'L2,3', 'M2,3', 'M4,5', 'N2,3', 'N4,5' 'O2,3', 'O4,5'}, optional
Filter for spectra with a specific class of edge.
min_energy, max_energy: float, optional
Minimum and maximum energy in eV.
resolution: float, optional
Energy resolution in eV.
resolution_compare: {"lt", "eq", "gt"}, optional, default "lt"
"lt" to search for all spectra with resolution less than `resolution`.
"eq" for equal, "gt" for greater than.
min_energy_compare, max_energy_compare: {"lt", "eq", "gt"}, optional
"lt" to search for all spectra with min/max energy less than
`min_energy`\`max_energy`. "eq" for equal, "gt" for greater than.
Deafault values are "gt"/"lt" for `min_energy`\`max_energy`
respectively.
monochromated: bool or None (default)
max_n: int, default -1
Maximum number of spectra to return. -1 to return all.
order: string
Key to sort results by. Valid keys are:
* "spectrumType",
* "spectrumMin",
* "spectrumMax",
* "stepSize",
* "spectrumFormula",
* "spectrumElement",
* "spectrumUpload",
* "source_purity",
* "spectrumEdges",
* "microscope",
* "guntype",
* "beamenergy",
* "resolution",
* "monochromated",
* "acquisition_mode",
* "convergence",
* "collection",
* "probesize",
* "beamcurrent",
* "integratetime",
* "readouts",
* "detector",
* "darkcurrent",
* "gainvariation",
* "calibration",
* "zeroloss_deconv",
* "thickness",
* "deconv_fourier_log",
* "deconv_fourier_ratio",
* "deconv_stephens_deconvolution",
* "deconv_richardson_lucy",
* "deconv_maximum_entropy",
* "deconv_other",
* "assoc_spectra",
* "ref_freetext",
* "ref_doi",
* "ref_url",
* "ref_authors",
* "ref_journal",
* "ref_volume",
* "ref_issue",
* "ref_page",
* "ref_year",
* "ref_title",
* "otherURLs"
order_direction : {"ASC", "DESC"}
Sorting `order` direction.
verify_certificate: bool
If True, verify the eelsdb website certificate and raise an error
if it is invalid. If False, continue querying the database if the certificate
is invalid. (This is a potential security risk.)
Returns
-------
spectra: list
A list containing all the spectra matching the given criteria if
any.
"""
# Verify arguments
if spectrum_type is not None and spectrum_type not in {
'coreloss', 'lowloss', 'zeroloss', 'xrayabs'}:
raise ValueError("spectrum_type must be one of \'coreloss\', \'lowloss\', "
"\'zeroloss\', \'xrayabs\'.")
valid_edges = [
'K', 'L1', 'L2,3', 'M2,3', 'M4,5', 'N2,3', 'N4,5', 'O2,3', 'O4,5']
valid_order_keys = [
"spectrumType",
"spectrumMin",
"spectrumMax",
"stepSize",
"spectrumFormula",
"spectrumElement",
"spectrumUpload",
"source_purity",
"spectrumEdges",
"microscope",
"guntype",
"beamenergy",
"resolution",
"monochromated",
"acquisition_mode",
"convergence",
"collection",
"probesize",
"beamcurrent",
"integratetime",
"readouts",
"detector",
"darkcurrent",
"gainvariation",
"calibration",
"zeroloss_deconv",
"thickness",
"deconv_fourier_log",
"deconv_fourier_ratio",
"deconv_stephens_deconvolution",
"deconv_richardson_lucy",
"deconv_maximum_entropy",
"deconv_other",
"assoc_spectra",
"ref_freetext",
"ref_doi",
"ref_url",
"ref_authors",
"ref_journal",
"ref_volume",
"ref_issue",
"ref_page",
"ref_year",
"ref_title",
"otherURLs"]
if edge is not None and edge not in valid_edges:
raise ValueError("`edge` must be one of %s." % ", ".join(valid_edges))
if order is not None and order not in valid_order_keys:
raise ValueError("`order` must be one of %s." % ", ".join(
valid_order_keys))
if order_direction is not None and order_direction not in ["ASC", "DESC"]:
raise ValueError("`order_direction` must be \"ASC\" or \"DESC\".")
for kwarg, label in (
(resolution_compare, "resolution_compare"),
(min_energy_compare, "min_energy_compare"),
(max_energy_compare, "max_energy_compare")):
if kwarg not in ("lt", "gt", "eq"):
raise ValueError("`%s` must be \"lt\", \"eq\" or \"gt\"." %
label)
if monochromated is not None:
monochromated = 1 if monochromated else 0
params = {
"type": spectrum_type,
"title": title,
"author": author,
"edge": edge,
"min_energy": min_energy,
"max_energy": max_energy,
"resolution": resolution,
"resolution_compare": resolution_compare,
"monochromated": monochromated,
"formula": formula,
"min_energy_compare": min_energy_compare,
"max_energy_compare": max_energy_compare,
"per_page": max_n,
"order": order,
"order_direction": order_direction,
}
if isinstance(element, str):
params["element"] = element
else:
params["element[]"] = element
request = requests.get('http://api.eelsdb.eu/spectra', params=params, verify=verify_certificate)
spectra = []
jsons = request.json()
if "message" in jsons:
# Invalid query, EELSdb raises error.
raise IOError(
"Please report the following error to the HyperSpy developers: "
"%s" % jsons["message"])
for json_spectrum in jsons:
download_link = json_spectrum['download_link']
msa_string = requests.get(download_link, verify=verify_certificate).text
try:
s = dict2signal(parse_msa_string(msa_string)[0])
emsa = s.original_metadata
s.original_metadata = s.original_metadata.__class__(
{'json': json_spectrum})
s.original_metadata.emsa = emsa
spectra.append(s)
except:
# parse_msa_string or dict2signal may fail if the EMSA file is not
# a valid one.
_logger.exception(
"Failed to load the spectrum.\n"
"Title: %s id: %s.\n"
"Please report this error to http://eelsdb.eu/about \n" %
(json_spectrum["title"], json_spectrum["id"]))
if not spectra:
_logger.info(
"The EELS database does not contain any spectra matching your query"
". If you have some, why not submitting them "
"https://eelsdb.eu/submit-data/ ?\n")
else:
# Add some info from json to metadata
# Values with units are not yet supported by HyperSpy (v0.8) so
# we can't get map those fields.
for s in spectra:
if spectrum_type == "xrayabs":
s.set_signal_type("XAS")
json_md = s.original_metadata.json
s.metadata.General.title = json_md.title
if s.metadata.Signal.signal_type == "EELS":
if json_md.elements:
try:
s.add_elements(json_md.elements)
except ValueError:
_logger.exception(
"The following spectrum contains invalid chemical "
"element information:\n"
"Title: %s id: %s. Elements: %s.\n"
"Please report this error in "
"http://eelsdb.eu/about \n" %
(json_md.title, json_md.id, json_md.elements))
if "collection" in json_md and " mrad" in json_md.collection:
beta = float(json_md.collection.replace(" mrad", ""))
s.metadata.set_item(
"Acquisition_instrument.TEM.Detector.EELS.collection_angle",
beta)
if "convergence" in json_md and " mrad" in json_md.convergence:
alpha = float(json_md.convergence.replace(" mrad", ""))
s.metadata.set_item(
"Acquisition_instrument.TEM.convergence_angle", alpha)
if "beamenergy" in json_md and " kV" in json_md.beamenergy:
beam_energy = float(json_md.beamenergy.replace(" kV", ""))
s.metadata.set_item(
"Acquisition_instrument.TEM.beam_energy", beam_energy)
# We don't yet support units, so we cannot map the thickness
# s.metadata.set_item("Sample.thickness", json_md.thickness)
s.metadata.set_item("Sample.description", json_md.description)
s.metadata.set_item("Sample.chemical_formula", json_md.formula)
s.metadata.set_item("General.author", json_md.author.name)
s.metadata.set_item("Acquisition_instrument.TEM.microscope",
json_md.microscope)
return spectra
def _group2dict(self, group, dictionary=None, lazy=False):
if dictionary is None:
dictionary = {}
for key, value in group.attrs.items():
if isinstance(value, bytes):
value = value.decode()
if isinstance(value, (np.string_, str)):
if value == '_None_':
value = None
elif isinstance(value, np.bool_):
value = bool(value)
elif isinstance(value, np.ndarray) and value.dtype.char == "S":
# Convert strings to unicode
value = value.astype("U")
if value.dtype.str.endswith("U1"):
value = value.tolist()
# skip signals - these are handled below.
if key.startswith('_sig_'):
pass
elif key.startswith('_list_empty_'):
dictionary[key[len('_list_empty_'):]] = []
elif key.startswith('_tuple_empty_'):
dictionary[key[len('_tuple_empty_'):]] = ()
elif key.startswith('_bs_'):
dictionary[key[len('_bs_'):]] = value.tobytes()
# The following two elif stataments enable reading date and time from
# v < 2 of HyperSpy's metadata specifications
elif key.startswith('_datetime_date'):
date_iso = datetime.date(
*ast.literal_eval(value[value.index("("):])).isoformat()
dictionary[key.replace("_datetime_", "")] = date_iso
elif key.startswith('_datetime_time'):
date_iso = datetime.time(
*ast.literal_eval(value[value.index("("):])).isoformat()
dictionary[key.replace("_datetime_", "")] = date_iso
else:
dictionary[key] = value
if not isinstance(group, self.Dataset):
for key in group.keys():
if key.startswith('_sig_'):
from hyperspy.io import dict2signal
dictionary[key[len('_sig_'):]] = (
dict2signal(self.group2signaldict(
group[key], lazy=lazy)))
elif isinstance(group[key], self.Dataset):
dat = group[key]
kn = key
if key.startswith("_list_"):
if (h5py.check_string_dtype(dat.dtype) and
hasattr(dat, 'asstr')):
# h5py 3.0 and newer
# https://docs.h5py.org/en/3.0.0/strings.html
dat = dat.asstr()[:]
ans = np.array(dat)
ans = ans.tolist()
kn = key[6:]
elif key.startswith("_tuple_"):
ans = np.array(dat)
ans = tuple(ans.tolist())
kn = key[7:]
elif dat.dtype.char == "S":
ans = np.array(dat)
try:
ans = ans.astype("U")
except UnicodeDecodeError:
# There are some strings that must stay in binary,
# for example dill pickles. This will obviously also
# let "wrong" binary string fail somewhere else...
pass
elif lazy:
ans = da.from_array(dat, chunks=dat.chunks)
else:
ans = np.array(dat)
dictionary[kn] = ans
elif key.startswith('_hspy_AxesManager_'):
dictionary[key[len('_hspy_AxesManager_'):]] = AxesManager(
[i for k, i in sorted(iter(
self._group2dict(
group[key], lazy=lazy).items()
))])
elif key.startswith('_list_'):
dictionary[key[7 + key[6:].find('_'):]] = \
[i for k, i in sorted(iter(
self._group2dict(
group[key], lazy=lazy).items()
))]
elif key.startswith('_tuple_'):
dictionary[key[8 + key[7:].find('_'):]] = tuple(
[i for k, i in sorted(iter(
self._group2dict(
group[key], lazy=lazy).items()
))])
else:
dictionary[key] = {}
self._group2dict(
group[key],
dictionary[key],
lazy=lazy)
return dictionary
def hdfgroup2dict(group, dictionary=None, load_to_memory=True):
if dictionary is None:
dictionary = {}
for key, value in group.attrs.iteritems():
if isinstance(value, (np.string_, str)):
if value == '_None_':
value = None
elif isinstance(value, np.bool_):
value = bool(value)
elif isinstance(value, np.ndarray) and \
value.dtype == np.dtype('|S1'):
value = value.tolist()
# skip signals - these are handled below.
if key.startswith('_sig_'):
pass
elif key.startswith('_list_empty_'):
dictionary[key[len('_list_empty_'):]] = []
elif key.startswith('_tuple_empty_'):
dictionary[key[len('_tuple_empty_'):]] = ()
elif key.startswith('_bs_'):
dictionary[key[len('_bs_'):]] = value.tostring()
elif key.startswith('_datetime_'):
dictionary[key.replace("_datetime_", "")] = eval(value)
else:
dictionary[key] = value
if not isinstance(group, h5py.Dataset):
for key in group.keys():
if key.startswith('_sig_'):
from hyperspy.io import dict2signal
dictionary[key[len('_sig_'):]] = (
dict2signal(hdfgroup2signaldict(group[key],
load_to_memory=load_to_memory)))
elif isinstance(group[key], h5py.Dataset):
if key.startswith("_list_"):
ans = np.array(group[key])
ans = ans.tolist()
kn = key[6:]
elif key.startswith("_tuple_"):
ans = np.array(group[key])
ans = tuple(ans.tolist())
kn = key[7:]
elif load_to_memory:
ans = np.array(group[key])
kn = key
else:
# leave as h5py dataset
ans = group[key]
kn = key
dictionary[kn] = ans
elif key.startswith('_hspy_AxesManager_'):
dictionary[key[len('_hspy_AxesManager_'):]] = \
AxesManager([i
for k, i in sorted(iter(
hdfgroup2dict(group[key], load_to_memory=load_to_memory).iteritems()))])
elif key.startswith('_list_'):
dictionary[key[7 + key[6:].find('_'):]] = \
[i for k, i in sorted(iter(
hdfgroup2dict(group[key], load_to_memory=load_to_memory).iteritems()))]
elif key.startswith('_tuple_'):
dictionary[key[8 + key[7:].find('_'):]] = tuple(
[i for k, i in sorted(iter(
hdfgroup2dict(group[key], load_to_memory=load_to_memory).iteritems()))])
else:
dictionary[key] = {}
hdfgroup2dict(
group[key],
dictionary[key],
load_to_memory=load_to_memory)
return dictionary
def eelsdb(spectrum_type=None, title=None, author=None, element=None, formula=None,
edge=None, min_energy=None, max_energy=None, resolution=None,
min_energy_compare="gt", max_energy_compare="lt",
resolution_compare="lt", max_n=-1, monochromated=None, order=None,
order_direction="ASC"):
r"""Download spectra from the EELS Data Base.
Parameters
----------
spectrum_type: {'coreloss', 'lowloss', 'zeroloss', 'xrayabs'}, optional
title: string
Search spectra titles for a text string.
author: string, optional
Search authors for a text string.
element: string or list of strings, optional
Filter for the presence of one or more element. Each string must
correspond with a valid element symbol.
formula: string
Chemical formula of the sample.
edge: {'K', 'L1', 'L2,3', 'M2,3', 'M4,5', 'N2,3', 'N4,5' 'O2,3', 'O4,5'}, optional
Filter for spectra with a specific class of edge.
min_energy, max_energy: float, optional
Minimum and maximum energy in eV.
resolution: float, optional
Energy resolution in eV.
resolution_compare: {"lt", "eq", "gt"}, optional, default "lt"
"lt" to search for all spectra with resolution less than `resolution`.
"eq" for equal, "gt" for greater than.
min_energy_compare, max_energy_compare: {"lt", "eq", "gt"}, optional
"lt" to search for all spectra with min/max energy less than
`min_energy`\`max_energy`. "eq" for equal, "gt" for greater than.
Deafault values are "gt"/"lt" for `min_energy`\`max_energy`
respectively.
monochromated: bool or None (default)
max_n: int, default -1
Maximum number of spectra to return. -1 to return all.
order: string
Key to sort results by. Valid keys are:
* "spectrumType",
* "spectrumMin",
* "spectrumMax",
* "stepSize",
* "spectrumFormula",
* "spectrumElement",
* "spectrumUpload",
* "source_purity",
* "spectrumEdges",
* "microscope",
* "guntype",
* "beamenergy",
* "resolution",
* "monochromated",
* "acquisition_mode",
* "convergence",
* "collection",
* "probesize",
* "beamcurrent",
* "integratetime",
* "readouts",
* "detector",
* "darkcurrent",
* "gainvariation",
* "calibration",
* "zeroloss_deconv",
* "thickness",
* "deconv_fourier_log",
* "deconv_fourier_ratio",
* "deconv_stephens_deconvolution",
* "deconv_richardson_lucy",
* "deconv_maximum_entropy",
* "deconv_other",
* "assoc_spectra",
* "ref_freetext",
* "ref_doi",
* "ref_url",
* "ref_authors",
* "ref_journal",
* "ref_volume",
* "ref_issue",
* "ref_page",
* "ref_year",
* "ref_title",
* "otherURLs"
order_direction : {"ASC", "DESC"}
Sorting `order` direction.
Returns
-------
spectra: list
A list containing all the spectra matching the given criteria if
any.
"""
# Verify arguments
if spectrum_type is not None and spectrum_type not in {
'coreloss', 'lowloss', 'zeroloss', 'xrayabs'}:
raise ValueError("spectrum_type must be one of \'coreloss\', \'lowloss\', "
"\'zeroloss\', \'xrayabs\'.")
valid_edges = [
'K', 'L1', 'L2,3', 'M2,3', 'M4,5', 'N2,3', 'N4,5', 'O2,3', 'O4,5']
valid_order_keys = [
"spectrumType",
"spectrumMin",
"spectrumMax",
"stepSize",
"spectrumFormula",
"spectrumElement",
"spectrumUpload",
"source_purity",
"spectrumEdges",
"microscope",
"guntype",
"beamenergy",
"resolution",
"monochromated",
"acquisition_mode",
"convergence",
"collection",
"probesize",
"beamcurrent",
"integratetime",
"readouts",
"detector",
"darkcurrent",
"gainvariation",
"calibration",
"zeroloss_deconv",
"thickness",
"deconv_fourier_log",
"deconv_fourier_ratio",
"deconv_stephens_deconvolution",
"deconv_richardson_lucy",
"deconv_maximum_entropy",
"deconv_other",
"assoc_spectra",
"ref_freetext",
"ref_doi",
"ref_url",
"ref_authors",
"ref_journal",
"ref_volume",
"ref_issue",
"ref_page",
"ref_year",
"ref_title",
"otherURLs"]
if edge is not None and edge not in valid_edges:
raise ValueError("`edge` must be one of %s." % ", ".join(valid_edges))
if order is not None and order not in valid_order_keys:
raise ValueError("`order` must be one of %s." % ", ".join(
valid_order_keys))
if order_direction is not None and order_direction not in ["ASC", "DESC"]:
raise ValueError("`order_direction` must be \"ASC\" or \"DESC\".")
for kwarg, label in (
(resolution_compare, "resolution_compare"),
(min_energy_compare, "min_energy_compare"),
(max_energy_compare, "max_energy_compare")):
if kwarg not in ("lt", "gt", "eq"):
raise ValueError("`%s` must be \"lt\", \"eq\" or \"gt\"." %
label)
if monochromated is not None:
monochromated = 1 if monochromated else 0
params = {
"type": spectrum_type,
"title": title,
"author": author,
"edge": edge,
"min_energy": min_energy,
"max_energy": max_energy,
"resolution": resolution,
"resolution_compare": resolution_compare,
"monochromated": monochromated,
"formula": formula,
"min_energy_compare": min_energy_compare,
"max_energy_compare": max_energy_compare,
"per_page": max_n,
"order": order,
"order_direction": order_direction,
}
if isinstance(element, str):
params["element"] = element
else:
params["element[]"] = element
request = requests.get('http://api.eelsdb.eu/spectra', params=params, )
spectra = []
jsons = request.json()
if "message" in jsons:
# Invalid query, EELSdb raises error.
raise IOError(
"Please report the following error to the HyperSpy developers: "
"%s" % jsons["message"])
for json_spectrum in jsons:
download_link = json_spectrum['download_link']
msa_string = requests.get(download_link).text
try:
s = dict2signal(parse_msa_string(msa_string)[0])
emsa = s.original_metadata
s.original_metadata = s.original_metadata.__class__(
{'json': json_spectrum})
s.original_metadata.emsa = emsa
spectra.append(s)
except:
# parse_msa_string or dict2signal may fail if the EMSA file is not
# a valid one.
_logger.exception(
"Failed to load the spectrum.\n"
"Title: %s id: %s.\n"
"Please report this error to http://eelsdb.eu/about \n" %
(json_spectrum["title"], json_spectrum["id"]))
if not spectra:
_logger.information(
"The EELS database does not contain any spectra matching your query"
". If you have some, why not submitting them "
"https://eelsdb.eu/submit-data/ ?\n")
else:
# Add some info from json to metadata
# Values with units are not yet supported by HyperSpy (v0.8) so
# we can't get map those fields.
for s in spectra:
if spectrum_type == "xrayabs":
s.set_signal_type("XAS")
json_md = s.original_metadata.json
s.metadata.General.title = json_md.title
if s.metadata.Signal.signal_type == "EELS":
if json_md.elements:
try:
s.add_elements(json_md.elements)
except ValueError:
_logger.exception(
"The following spectrum contains invalid chemical "
"element information:\n"
"Title: %s id: %s. Elements: %s.\n"
"Please report this error in "
"http://eelsdb.eu/about \n" %
(json_md.title, json_md.id, json_md.elements))
if "collection" in json_md and " mrad" in json_md.collection:
beta = float(json_md.collection.replace(" mrad", ""))
s.metadata.set_item(
"Acquisition_instrument.TEM.Detector.EELS.collection_angle",
beta)
if "convergence" in json_md and " mrad" in json_md.convergence:
alpha = float(json_md.convergence.replace(" mrad", ""))
s.metadata.set_item(
"Acquisition_instrument.TEM.convergence_angle", alpha)
if "beamenergy" in json_md and " kV" in json_md.beamenergy:
beam_energy = float(json_md.beamenergy.replace(" kV", ""))
s.metadata.set_item(
"Acquisition_instrument.TEM.beam_energy", beam_energy)
# We don't yet support units, so we cannot map the thickness
# s.metadata.set_item("Sample.thickness", json_md.thickness)
s.metadata.set_item("Sample.description", json_md.description)
s.metadata.set_item("Sample.chemical_formula", json_md.formula)
s.metadata.set_item("General.author", json_md.author.name)
s.metadata.set_item("Acquisition_instrument.TEM.microscope",
json_md.microscope)
return spectra
def process(directory,
threshold=6,
integrate=False,
counting=False,
hdr=None,
mean_e=256,
nav_shape=None,
chunk_shape=None,
verbose=False):
if verbose:
_logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s \n')
handler.setFormatter(formatter)
_logger.addHandler(handler)
_logger.info(msg="\n\n .SEQ Processor Application (and Counting)...\n"
"Created by: Carter Francis ([email protected])\n"
"Updated 2021-06-18\n"
"------------------\n")
_logger.info(msg="Version:" + __version__)
tick = time.time()
file_dict = get_files(folder=directory)
for key in file_dict:
if len(file_dict[key]) == 0:
file_dict[key].pop()
else:
file_dict[key] = file_dict[key][0]
if "top" in file_dict and "bottom" in file_dict:
file_dict.pop("seq")
data_dict = cel_file_reader(**file_dict,
nav_shape=nav_shape,
chunk_shape=chunk_shape,
lazy=True)
elif "seq" in file_dict:
data_dict = file_reader(**file_dict,
nav_shape=nav_shape,
chunk_shape=chunk_shape,
lazy=True)
if hdr is not None:
hdr = hs.load(hdr).data
else:
hdr = None
if hdr is None and integrate is False:
dtype = bool
else:
dtype = np.float32
if counting:
data_dict["data"] = data_dict["data"].map_blocks(
_counting_filter_cpu,
threshold=threshold,
integrate=integrate,
hdr_mask=hdr,
method="maximum",
mean_electron_val=mean_e,
dtype=dtype)
_logger.info(data_dict)
sig = dict2signal(data_dict, lazy=True)
_logger.info("Data... :" + str(sig.data))
_logger.info("Dtype:" + str(sig.data.dtype))
_logger.info("Saving... ")
da.to_zarr(sig.data, directory + "_zarr", overwrite=True)
#sig.save(directory + ".hspy",
# compression=False,
# overwrite=True)
tock = time.time()
_logger.info("Total time elapsed : " + str(tock - tick) + " sec")
return sig
