def print_custom_funs_info(self):
"""Print information about available curtom calib functions."""
logger.info("Available polyorders (with offset): %s"
"Available polyorders (without offset): %s"
% (list(self.polys[0].keys()), list(self.polys[1].keys())))
for k, v in six.iteritems(self._custom_funs_info):
logger.info("%s : %s" % (k, v))
def add_retrieval_point(self, pos_x_abs, pos_y_abs, dist=None):
"""Add a distinct pixel with known distance to image.
Parameters
----------
pos_x_abs : int
x-pixel position of point in image in absolute coordinate (i.e.
pyramid level 0 and not cropped)
pos_y_abs : int
y-pixel position of point in image in absolute coordinate (i.e.
pyramid level 0 and not cropped)
dist : :obj:`float`, optional
distance to feature in image in m. If None (default), the distance
will be estimated
"""
if not isnum(dist):
logger.info("Input distance for point unspecified, trying automatic "
"access")
(dist,
derr,
p) = self.meas_geometry.get_topo_distance_pix(pos_x_abs,
pos_y_abs)
self._geopoints["add_points"].append(p)
dist *= 1000.0
self._add_points.append((pos_x_abs, pos_y_abs, dist))
def _merge_tseries_nearest(self, time_series):
"""Find nearest in time image for each time stamp in input series.
Find indices (and time differences) in input time series of nearest
data point for each image in this stack. Then, get rid of all indices
showing double occurences using time delta information.
"""
stack, time_stamps, texps = self.get_data()
nearest_idxs, del_ts = self.get_nearest_indices(time_series.index)
img_idxs = []
spec_idxs_final = []
del_ts_abs = []
for idx in range(min(nearest_idxs), max(nearest_idxs) + 1):
logger.info("Current tseries index %s" % idx)
matches = where(nearest_idxs == idx)[0]
if len(matches) > 0:
del_ts_temp = del_ts[matches]
spec_idxs_final.append(idx)
del_ts_abs.append(min(del_ts_temp))
img_idxs.append(matches[argmin(del_ts_temp)])
series_new = time_series[spec_idxs_final]
try:
series_new.fit_errs = time_series.fit_errs[spec_idxs_final]
except BaseException:
pass
stack_new = self.stack[img_idxs]
texps_new = asarray(self.texps[img_idxs])
start_acq_new = asarray(self.start_acq[img_idxs])
stack_obj_new = ImgStack(stack_id=self.stack_id,
img_prep=self.img_prep, stack=stack_new,
start_acq=start_acq_new, texps=texps_new)
stack_obj_new.roi_abs = self.roi_abs
stack_obj_new.add_data = series_new
return (stack_obj_new, series_new)
def det_topo_dists_line(self, line_id, **settings):
"""Estimate distances to pixels on current lines.
Retrieves distances to all :class:`LineOnImage` objects in
``self.lines`` using ``self.meas_geometry`` (i.e. camera position
and viewing direction).
Parameters
----------
line_id : str
ID of line
**settings :
additional key word args used to update search settings (passed to
:func:`get_topo_distances_line` in :class:`MeasGeometry`)
Returns
-------
array
retrieved distances
"""
if line_id not in self.lines.keys():
raise KeyError("No line with ID %s available" % line_id)
logger.info("Searching topo distances for pixels on line %s" % line_id)
self.update_settings(**settings)
l = self.lines[line_id]
res = self.meas_geometry.get_topo_distances_line(l, **self.settings)
dists = res["dists"] * 1000. # convert to m
self._geopoints[line_id] = res["geo_points"]
self._dists_lines[line_id] = dists
self._masks_lines[line_id] = res["ok"]
self._skip_pix[line_id] = self.settings["skip_pix"]
return dists
def update_meas_geometry(self):
"""Update the meas geometry based on current settings."""
logger.info("Updating MeasGeometry in MeasSetup class")
self.meas_geometry.__init__(self.source.to_dict(),
self.camera.to_dict(),
self.wind_info,
auto_topo_access=self.auto_topo_access)
def calib_fun(self, val):
if not callable(val):
raise ValueError("Need a callable object (e.g. lambda function)")
args = getargspec(val).args
logger.info("Setting optimisation function in CalibData class. "
"Argspec: %s" % args)
self._calib_fun = val
def _import_from_dict(self, info_dict):
"""Try access default information of source.
Parameters
----------
info_dict : dict
dictonary containing source information (valid keys are all
keys ``self._type_dict``, e.g. ``lon``, ``lat``, ``altitude``)
Returns
-------
bool
success
"""
types = self._type_dict
if not isinstance(info_dict, dict):
raise TypeError(
"need dictionary like object for source info update")
err = []
for key, val in six.iteritems(info_dict):
if key in types:
func = types[key]
else:
func = str
try:
self[key] = func(val)
except BaseException:
err.append(key)
if bool(err) > 0:
logger.info("Failed to load the following source parameters\n%s" %
err)
return self.info_available
def set_test_data_path(save_path):
"""Set local path where test data is stored."""
if save_path.lower() in all_test_data_paths():
logger.info("Path is already in search tree")
return
dirs = data_search_dirs()
fp = join(dirs[0], "_paths.txt")
if not exists(fp):
fp = join(dirs[1], "_paths.txt")
save_path = abspath(save_path)
try:
if not exists(save_path):
raise IOError("Could not set test data path: specified location "
"does not exist: %s" % save_path)
with open(fp, "a") as f:
f.write("\n" + save_path + "\n")
print_log.info("Adding new path for test data location in "
"file _paths.txt: %s" % save_path)
f.close()
if "pyplis_etna_testdata" not in listdir(save_path):
logger.warning(
"WARNING: test data folder (name: pyplis_etna_testdata) "
"could not be found at specified location, please download "
"test data, unzip and save at: %s" % save_path)
except:
raise
def point_in_image(self, x, y, img_array):
"""Check if a given coordinate is within image.
Parameters
----------
x : int
x coordinate of point
y : int
y coordinate of point
img_array : array
image data
Returns
-------
bool
True if point is in image, False if not
"""
h, w = img_array.shape[:2]
if not 0 < x < w:
logger.info("x coordinate out of image")
return False
if not 0 < y < h:
logger.info("y coordinate out of image")
return False
return True
def __getitem__(self, name):
"""Get item.
Returns form with corresponding to input name (if it exists)
"""
try:
return self._forms[name]
except BaseException:
logger.info("No such form: " + str(name))
def check_geometry_info(self):
"""Check if all req. info for measurement geometry is available.
Relevant parameters are:
1. Lon, Lat of
i. source
#. camera
#. Meteorology info
i. Wind direction
#. Wind velocity (rough estimate)
#. Viewing direction of camera
i. Azimuth (N)
#. Elvation(from horizon)
#. Alitude of camera and source
#. Camera optics
i. Pixel size
#. Number of pixels detector
#. focal length
"""
if not isinstance(self.source, Source):
return 0
source = self.source
ok = 1
s = ("\n------------------------------\nChecking basic geometry info"
"\n------------------------------\n")
for key, val in six.iteritems(self.camera.geom_data):
if not self._check_if_number(val):
ok = 0
s += "Missing info in Camera setup\n"
s += self._dict_miss_info_str(key, val)
for key in self.camera.optics_keys:
val = self.camera[key]
if not self._check_if_number(val):
ok = 0
s += self._dict_miss_info_str(key, val)
for key, val in six.iteritems(source.geo_data):
if not self._check_if_number(val):
ok = 0
s += "Missing info in Source: %s\n" % source.name
s += self._dict_miss_info_str(key, val)
for key, val in six.iteritems(self.wind_info):
if not self._check_if_number(val):
ok = 0
s += "Missing Meteorology info\n"
s += self._dict_miss_info_str(key, val)
if ok:
s += "All necessary information available\n"
logger.info(s)
return ok, s
def update(self, x0, y0, x1, y1, id):
"""Update an existing form (or create new if it does not exist)."""
if id in self._forms.keys():
logger.info("Form with ID " + str(id) +
" could not be updated because "
"it does not exist, creating new form instead")
self.remove(id)
if self.add(x0, y0, x1, y1, id):
return 1
return 0
def get_source_info_online(source_id):
"""Try to load source info from online database (@ www.ngdc.noaa.gov).
:param str source_id: ID of source
"""
name = source_id
name = name.lower()
url = ("http://www.ngdc.noaa.gov/nndc/struts/results?type_0=Like&query_0="
"&op_8=eq&v_8=&type_10=EXACT&query_10=None+Selected&le_2=&ge_3="
"&le_3=&ge_2=&op_5=eq&v_5=&op_6=eq&v_6=&op_7=eq&v_7=&t=102557&s=5"
"&d=5")
logger.info("Trying to access volcano data from URL:")
logger.info(url)
try:
# it's a file like object and works just like a file
data = urlopen(url)
except BaseException:
raise
res = od()
in_row = 0
in_data = 0
lc = 0
col_num = 10
first_volcano_name = "Abu" # this needs to be identical
ids = [
"name", "country", "region", "lat", "lon", "altitude", "type",
"status", "last_eruption"
]
types = [str, str, str, float, float, float, str, str, str]
for line in data:
lc += 1
if first_volcano_name in line and line.split(">")[1].\
split("</td")[0].strip() == first_volcano_name:
in_data, c = 1, 0
if in_data:
if c % col_num == 0 and name in line.lower():
logger.info("FOUND candidate, line: ", lc)
spl = line.split(">")[1].split("</td")[0].strip().lower()
if name in spl:
logger.info("FOUND MATCH: ", spl)
in_row, cc = 1, 0
cid = spl
res[cid] = od()
if in_row:
spl = line.split(">")[1].split("</td")[0].strip()
res[cid][ids[cc]] = types[cc](spl)
cc += 1
if in_row and cc == 9:
logger.info("End of data row reached for %s" % cid)
cc, in_row = 0, 0
c += 1
return res
def merge_with_time_series(self, time_series, method="average",
**kwargs):
"""High level wrapper for data merging.
Choose from either of three methods to perform an index merging based
on time stamps of stack and of other time series data (provided on
input).
Parameters
----------
time_series : Series
time series data supposed to be merged with stack data
method : str
merge method, currently available methods are:
- average: determine new stack containing images averaged based
on start / stop time stamps of each datapoint in input
``time_series`` (requires corresponding data to be available
in input, i.e. ``time_series`` must be of type
:class:`DoasResults` of ``pydoas`` library).
- nearest: perform merging based on nearest datapoint per image
- interpolation: perform cross interpolation onto unified time
index array from stack and time series data
**kwargs
additional keyword args specifying additional merge settings (e.g.
``itp_type=quadratic`` in case ``method=interpolation`` is used)
Returns
-------
tuple
2-element tuple containing
- :obj:`ImgStack`: new stack containing merged data
- :obj:`Series`: merged time series data
"""
if not isinstance(time_series, Series):
raise TypeError("Could not merge stack data with input data: "
"wrong type: %s" % type(time_series))
if method == "average":
try:
return self._merge_tseries_average(time_series, **kwargs)
except BaseException:
logger.info("Failed to merge data using method average, trying "
"method nearest instead")
method = "nearest"
if method == "nearest":
return self._merge_tseries_nearest(time_series, **kwargs)
elif method == "interpolation":
return self._merge_tseries_cross_interpolation(time_series,
**kwargs)
else:
raise TypeError("Unkown merge type: %s. Choose from "
"[nearest, average, interpolation]")
def plot(self, add_label_str="", ax=None, **kwargs):
"""Plot calibration data and fit result.
Parameters
----------
add_label_str : str
additional string added to label of plots for legend
ax :
matplotlib axes object, if None, a new one is created
"""
if "color" not in kwargs:
kwargs["color"] = "b"
if ax is None:
fig, ax = subplots(1, 1, figsize=(10, 8))
taumin, taumax = self.tau_range
x = linspace(taumin, taumax, 100)
cds = self.cd_vec
ax.plot(self.tau_vec,
cds,
ls="",
marker=".",
label="Data %s" % add_label_str,
**kwargs)
try:
ax.errorbar(self.tau_vec,
cds,
yerr=self.cd_vec_err,
marker="None",
ls=" ",
c="#b3b3b3")
except BaseException:
logger.warning("No CD errors available")
try:
cds_calib = self.calib_fun(x, *self.calib_coeffs)
ax.plot(x,
cds_calib,
ls="-",
marker="",
label="Fit result",
**kwargs)
except TypeError:
logger.info("Calibration poly probably not fitted")
ax.set_title("Calibration data, ID: %s" % self.calib_id)
ax.set_ylabel(r"$S_{%s}$ [cm$^{-2}$]" % SPECIES_ID)
ax.set_xlabel(r"$\tau_{%s}$" % self.calib_id)
ax.grid()
ax.legend(loc='best', fancybox=True, framealpha=0.7)
return ax
def rect_roi_rot(self):
"""Rectangle specifying coordinates of ROI aligned with line normal."""
try:
if not self._rect_roi_rot.shape == (5, 2):
raise Exception
except BaseException:
logger.info("Rectangle for rotated ROI was not set and is not being "
"set to default depth of +/- 30 pix around line. Use "
"method set_rect_roi_rot to change the rectangle")
self.set_rect_roi_rot()
return self._rect_roi_rot
def remove(self, id):
"""Remove one form from collection.
:param str id: string id of the form to be deleted
"""
if id not in self._forms.keys():
logger.info("Error: could not delete form " + id + " from "
"collection, no such form in collection")
return 0
logger.info("Delete form " + id + " from collection")
del self._forms[id]
def __setitem__(self, key, val):
"""Set item method.
Adds one form to this collection
:param str key: string ID of new form
:param list val: list with start / stop coords ``[x0,y0, x1, y1]``
"""
try:
self.add(val[0], val[1], val[2], val[3], key)
except Exception as e:
logger.info("Adding form failed_ %s" % repr(e))
def print_list_info(self):
"""Print overview information about image lists."""
s = ("info about image lists in dataset\n-------------------------\n\n"
"Scene image lists:\n------------------------\n")
for lst in self.img_lists.values():
s += ("ID: %s, type: %s, %s images\n"
% (lst.list_id, lst.list_type, lst.nof))
s += "\nDark image lists:\n------------------------\n"
for lst in self.dark_lists.values():
s += ("ID: %s, type: %s, read_gain: %s, %s images\n"
% (lst.list_id, lst.list_type, lst.read_gain, lst.nof))
logger.info(s)
def plot_calib_fun(self,
add_label_str="",
shift_yoffset=False,
ax=None,
**kwargs):
"""Plot calibration fit result.
Parameters
----------
add_label_str : str
additional string added to label of plots for legend
shift_yoffset : bool
if True, the data is plotted without y-offset
ax :
matplotlib axes object, if None, a new one is created
"""
if "color" not in kwargs:
kwargs["color"] = "b"
if ax is None:
fig, ax = subplots(1, 1, figsize=(10, 8))
taumin, taumax = self.tau_range
x = linspace(taumin, taumax, 100)
if self.calib_coeffs is None:
logger.warning(
"Calibration function not yet fitted, applying default fit"
"(1. order polynomial)")
self.fit_calib_data()
cds_poly = self.calib_fun(x, *self.calib_coeffs)
if shift_yoffset:
try:
cds_poly -= self.y_offset
except BaseException:
logger.warning("Failed to subtract y offset")
try:
ax.plot(x,
cds_poly,
ls="-",
marker="",
label="Fit result %s" % add_label_str,
**kwargs)
except TypeError:
logger.info("Calibration poly probably not fitted")
ax.grid()
ax.legend(loc='best', fancybox=True, framealpha=0.7)
return ax
def update_filters_from_dict(self, filter_dict):
"""Add filter objects from a dictionary.
Parameters
----------
filter_dict : dict
dictionary, containing filter information
"""
for f in filter_dict.values():
if isinstance(f, Filter):
if f.id in self._filters:
logger.info("Filter %s was overwritten" % f.id)
self._filters[f.id] = f
def __init__(self, input=None, lst_type=ImgList, init=1):
self.setup = None
self.lst_type = lst_type
self._lists_intern = od()
self.lists_access_info = od()
ok = self.load_input(input)
if init and ok:
self.init_image_lists()
else:
self.create_lists_default()
logger.info("DATASET INITIALISED")
def set_setup(self, stp):
"""Set the current measurement setup.
Parameters
----------
stp : MeasSetup
Class containing information about measurement setup
"""
if isinstance(stp, MeasSetup):
logger.info("Updating measurement setup in Dataset")
self.setup = stp
return 1
return 0
def _read_binary_timestamp(timestamp):
u"""Read timestamp from pco camware binary format.
This converts an (1,14)-array of pixel as given by the pco camware software
to a valid datetime.
Parameters
----------
timestamp : array
array containg 14 pixel which code as the following
0 pixel 1 image counter (MSB) (00 … 99)
1 pixel 2 image counter (00 … 99)
2 pixel 3 image counter (00 … 99)
3 pixel 4 image counter (LSB) (00 … 99)
4 pixel 5 year (MSB) (20)
5 pixel 6 year (LSB) (03 … 99)
6 pixel 7 month (01 … 12)
7 pixel 8 day (01 ... 31)
8 pixel 9 h (00 … 23)
9 pixel 10 min (00 … 59)
10 pixel 11 s (00 … 59)
11 pixel 12 μs * 10000 (00 … 99)
12 pixel 13 μs * 100 (00 … 99)
13 pixel 14 μs (00 … 90)
Returns
-------
datetime.datetime
3-element tuple containing
"""
try:
values = [
10 * (timestamp[0, j] >> 4) + timestamp[0, j] -
((timestamp[0, j] >> 4) << 4) for j in range(14)
]
except:
try:
values = [
10 * (timestamp[j] >> 4) + timestamp[j] -
((timestamp[j] >> 4) << 4) for j in range(14)
]
except:
logger.info('Failed to convert the binary timestamp.')
year = int(values[4] * 100 + values[5])
microsecond = int(values[11] * 10000 + values[12] * 100 + values[13])
endtime = datetime(year, values[6], values[7], values[8], values[9],
values[10], microsecond)
return endtime
def __setitem__(self, key, value):
"""Update class item."""
if key in self.__dict__:
logger.info("Updating %s in background model" % key)
self.__dict__[key] = value
elif key == "mode":
"Updating %s in background model" % key
self.mode = value
elif key == "surface_fit_mask":
self.surface_fit_mask = value
elif key == "CORR_MODE":
logger.warning(
"Got input key CORR_MODE which is out-dated in versions 0.10+"
". Updated background modelling mode accordingly")
self.mode = value
def print_setup(self):
"""Print the current setup.
Returns
-------
str
print string representation
"""
s = ("pyplis FilterSetup\n------------------------------\n"
"All filters:\n\n")
for flt in self._filters.values():
s += ("%s" % flt)
s += "Default Filter: %s\n\n" % self.default_key_on
logger.info(s)
return s
def _check_bounds(self, fun, bounds):
"""Check fit boundaries and sets inits default if necessary."""
sd = False
nargs = self.num_optargs_fun(fun)
try:
if bounds is None:
sd = True
elif not len(bounds) == 2:
sd = True
elif not (len(x) == nargs for x in bounds):
sd = True
except BaseException:
sd = True
if sd:
logger.info("Input bounds invalid, initiating default")
bounds = (-ones(nargs) * inf, ones(nargs) * inf)
return bounds
def plot(self, include_tit=True, date_fmt=None, **kwargs):
"""Plot time series.
Parameters
----------
include_tit : bool
Include a title
date_fmt : str
Date / time formatting string for x labels, passed to
:class:`DateFormatter` instance (optional)
**kwargs
Additional keyword arguments passed to pandas Series plot method
Returns
-------
axes
matplotlib axes instance
"""
try:
self.index = self.index.to_pydatetime()
except BaseException:
pass
try:
if "style" not in kwargs:
kwargs["style"] = "--x"
ax = super(PixelMeanTimeSeries, self).plot(**kwargs)
try:
if date_fmt is not None:
ax.xaxis.set_major_formatter(DateFormatter(date_fmt))
except BaseException:
pass
if include_tit:
ax.set_title("Mean value (%s), roi_abs: %s"
% (self.name, self.roi_abs))
ax.grid()
return ax
except Exception as e:
logger.info(repr(e))
fig, ax = subplots(1, 1)
ax.text(.1, .1, "Plot of PixelMeanTimeSeries failed...")
fig.canvas.draw()
return ax
def get_line_profile(self, array, order=1, **kwargs):
"""Retrieve the line profile along pixels in input array.
Parameters
----------
array : array
2D data array (e.g. image data). Color images are converted into
gray scale using :func:`cv2.cvtColor`.
order : int
order of spline interpolation used to retrieve the values along
input coordinates (passed to :func:`map_coordinates`)
**kwargs
additional keword args passed to interpolation method
:func:`map_coordinates`
Returns
-------
array
profile
"""
try:
array = array.img # if input is Img object
except BaseException:
pass
if ndim(array) != 2:
if ndim(array) != 3:
logger.info("Error retrieving line profile, invalid dimension of "
"input array: %s" % (ndim(array)))
return
if array.shape[2] != 3:
logger.info("Error retrieving line profile, invalid dimension of "
"input array: %s" % (ndim(array)))
return
"Input in BGR, conversion into gray image"
array = cvtColor(array, COLOR_BGR2GRAY)
# Extract the values along the line, using interpolation
zi = map_coordinates(array, self.profile_coords, order=order, **kwargs)
if sum(isnan(zi)) != 0:
logger.warning("Retrieved NaN for one or more pixels along line on input "
"array")
return zi
def check_coordinates(self):
"""Check line coordinates.
Checks if coordinates are in the right order and exchanges start / stop
points if not
Raises
------
ValueError
if any of the current coordinates is smaller than zero
"""
if any([x < 0 for x in self.coords]):
raise ValueError("Invalid value encountered, all coordinates of "
"line must exceed zero, current coords: %s"
% self.coords)
if self.start[0] > self.stop[0]:
logger.info("x coordinate of start point is larger than of stop point: "
"start and stop will be exchanged")
self.start, self.stop = self.stop, self.start
