def Scipy_CircMean(angleDict, low=0, high=0): anglesFullList = np.array(ExpandToIndividualParticles(angleDict)) if (low==0 and high == 0): mean = stats.circmean(anglesFullList) # probably need convert to numpy arrray right ? Lets see else : mean = stats.circmean(anglesFullList, high=high, low=low) return mean
def circ_corr(alpha1, alpha2):
"""Helper to compute the circular correlation."""
alpha1_bar = stats.circmean(alpha1)
alpha2_bar = stats.circmean(alpha2)
num = np.sum(np.sin(alpha1 - alpha1_bar) * np.sin(alpha2 - alpha2_bar))
den = np.sqrt(np.sum(np.sin(alpha1 - alpha1_bar) ** 2) * np.sum(np.sin(alpha2 - alpha2_bar) ** 2))
rho = num / den
return rho
def test_circstd_nan(self):
""" Test custom circular mean with NaN."""
from scipy import stats
ref_mean = stats.circmean(self.test_angles, **self.circ_kwargs)
ref_nan = stats.circmean(self.test_nan, **self.circ_kwargs)
test_nan = pysat.utils.nan_circmean(self.test_nan, **self.circ_kwargs)
assert np.isnan(ref_nan)
assert ref_mean == test_nan
def get_average_objects(clusters, kind):
"""Create the average object out of a sequence of clusters.
Parameters
----------
clusters : sequence of pandas.DataFrames
table with rows of markings (fans or blotches) to be averaged
kind : {'fan', 'blotch}
Switch to control the circularity for the average angle calculation.
Returns
-------
Generator providing single row pandas.DataFrames with the average values
"""
logger.debug("Averaging clusters.")
for cluster_df in clusters:
# first filter for outliers more than 1 std away
# for
# reduced = df[df.apply(lambda x: np.abs(x - x.mean()) / x.std() < 1).all(axis=1)]
logger.debug("Averaging %i objects.", len(cluster_df))
logger.debug("x.mean: %f", cluster_df.x.mean())
logger.debug("y.mean: %f", cluster_df.y.mean())
meandata = cluster_df.mean()
# this determines the upper limit for circular mean
high = 180 if kind == 'blotch' else 360
meandata.angle = circmean(cluster_df.angle, high=high)
meandata['angle_std'] = circstd(cluster_df.angle, high=high)
meandata['n_votes'] = len(cluster_df)
yield meandata.to_frame().T
def test_circmean(self):
""" Test custom circular mean."""
from scipy import stats
ref_mean = stats.circmean(self.test_angles, **self.circ_kwargs)
test_mean = pysat.utils.nan_circmean(self.test_angles,
**self.circ_kwargs)
ans1 = ref_mean == test_mean
assert ans1
def get_average_object(df, kind):
"Create the average object out of a cluster of data."
# first filter for outliers more than 1 std away
# for
# reduced = df[df.apply(lambda x: np.abs(x - x.mean()) / x.std() < 1).all(axis=1)]
meandata = df.mean()
# this determines the upper limit for circular mean
high = 180 if kind == 'blotch' else 360
avg = circmean(df.angle, high=high)
meandata.angle = avg
return meandata
def test_circfuncs_small(self):
x = np.array([20,21,22,18,19,20.5,19.2])
M1 = x.mean()
M2 = stats.circmean(x, high=360)
assert_allclose(M2, M1, rtol=1e-5)
V1 = x.var()
V2 = stats.circvar(x, high=360)
assert_allclose(V2, V1, rtol=1e-4)
S1 = x.std()
S2 = stats.circstd(x, high=360)
assert_allclose(S2, S1, rtol=1e-4)
def test_circfuncs(self):
x = np.array([355,5,2,359,10,350])
M = stats.circmean(x, high=360)
Mval = 0.167690146
assert_allclose(M, Mval, rtol=1e-7)
V = stats.circvar(x, high=360)
Vval = 42.51955609
assert_allclose(V, Vval, rtol=1e-7)
S = stats.circstd(x, high=360)
Sval = 6.520702116
assert_allclose(S, Sval, rtol=1e-7)
def mean_longitude(longitudes):
"""
Compute sample mean longitude, assuming longitude in degrees from -180 to
180.
>>> lons = (-170.5, -178.3, 166)
>>> np.mean(lons) # doctest: +SKIP
-60.933
>>> mean_longitude(lons) # doctest: +ELLIPSIS
179.08509...
:type longitudes: :class:`~numpy.ndarray` (or list, ..)
:param longitudes: Geographical longitude values ranging from -180 to 180
in degrees.
"""
return circmean(np.array(longitudes), low=-180, high=180)
def test_circmean_axis():
x = np.array([[355, 5, 2, 359, 10, 350], [351, 7, 4, 352, 9, 349], [357, 9, 8, 358, 4, 356]])
M1 = stats.circmean(x, high=360)
M2 = stats.circmean(x.ravel(), high=360)
assert_allclose(M1, M2, rtol=1e-14)
M1 = stats.circmean(x, high=360, axis=1)
M2 = [stats.circmean(x[i], high=360) for i in range(x.shape[0])]
assert_allclose(M1, M2, rtol=1e-14)
M1 = stats.circmean(x, high=360, axis=0)
M2 = [stats.circmean(x[:, i], high=360) for i in range(x.shape[1])]
assert_allclose(M1, M2, rtol=1e-14)
def smoother_blockCircMean(dt,y,output_dates,date_incr,mode='l'):
means = []
if isinstance(y,list):
y = np.array(y)
# convert to radians
y = np.radians(y)
for i in range(len(output_dates)):
# check if more than 50% of values are valid
# if so compute mean
if mode == 'l':
idx = find_included_times(dt,\
sdate = output_dates[i] - \
timedelta(hours=date_incr),\
edate = output_dates[i], twin=0)
elif mode == 'c':
idx = find_included_times(dt,\
sdate = output_dates[i] - \
timedelta(hours=date_incr/2.),\
edate = output_dates[i] +
timedelta(hours=date_incr/2.),\
twin=0)
elif mode == 'r':
idx = find_included_times(dt,\
sdate = output_dates[i],
edate = output_dates[i] + \
timedelta(hours=date_incr),\
twin=0)
block = y[idx]
nominator = len(block[np.isnan(block)])
denominator = len(block)
if denominator == 0:
ratio = 1
else:
ratio = nominator/float(denominator)
if ratio < 0.5:
#means.append(circmean(block[~np.isnan(block)]))
means.append(circmean(block,nan_policy='omit'))
else:
means.append(np.nan)
means = np.array(means)
# convert to radians
means = np.degrees(means)
return means
def _balloon_color_data(tri, data, itype):
"""Return the data array that is to be mapped to the colormap of the
balloon.
Parameters
----------
tri : Triangulation
The matplotlib triangulation for the sphere
data : ndarray, double, complex double
The data array
itype : 'magnitude', 'phase', 'amplitude'
Whether to plot magnitude levels or the phase.
Returns
-------
color_data : ndarray, double
The data array for the colormap.
vmin : double
The minimum of the color data
vmax : double
The maximum of the color data
"""
if itype == 'phase':
cdata = np.mod(np.angle(data), 2 * np.pi)
vmin = 0
vmax = 2 * np.pi
colors = circmean(cdata[tri.triangles], axis=1)
elif itype == 'magnitude':
cdata = np.abs(data)
vmin = np.min(cdata)
vmax = np.max(cdata)
colors = np.mean(cdata[tri.triangles], axis=1)
elif itype == 'amplitude':
vmin = np.min(data)
vmax = np.max(data)
colors = np.mean(data[tri.triangles], axis=1)
else:
raise ValueError("Invalid type of data mapping.")
return colors, vmin, vmax
def plot_phase_information(ax, peak_pos, obs_phases, sim_phases):
""" Plots the phase information for the detected peaks
Parameters
----------
For a description of the parameters see 'plot_panel_row'.
"""
sim_mean_phases = circmean(sim_phases, axis=0)
sim_phase_stds = circstd(sim_phases, axis=0)
sim_phase_ranges = np.array(
(sim_mean_phases - sim_phase_stds, sim_mean_phases + sim_phase_stds)).T
lower_freq, upper_freq = ax.get_xlim()
for freq, obs_phase, sim_phase, sim_phase_range in zip(
peak_pos, obs_phases, sim_mean_phases, sim_phase_ranges):
if lower_freq < freq < upper_freq:
plot_phase_clock(get_theta(obs_phase), sim_phase, sim_phase_range,
freq, ax)
return ax
def serial_bias(prevcurr, error, window, step):
xxx = np.arange(-np.pi, np.pi, step)
m_err = []
std_err = []
for t in xxx:
idx = (prevcurr >= t - window / 2) & (prevcurr < t + window / 2)
if t - window / 2 < -np.pi:
idx = (prevcurr >= t - window / 2) & (
prevcurr < t + window / 2) | (prevcurr > np.pi -
(window / 2 -
(np.pi - np.abs(t))))
if t + window / 2 > np.pi:
idx = (prevcurr >= t - window / 2) & (
prevcurr < t + window / 2) | (prevcurr < -np.pi +
(window / 2 -
(np.pi - np.abs(t))))
m_err.append(sps.circmean(error[idx], low=-np.pi, high=np.pi))
std_err.append(sps.circstd(error[idx]) / np.sqrt(np.sum(idx)))
return np.array(m_err), np.array(std_err)
def mean_longitude(longitudes):
"""
Compute sample mean longitude, assuming longitude in degrees from -180 to
180.
>>> lons = (-170.5, -178.3, 166)
>>> np.mean(lons) # doctest: +SKIP
-60.933
>>> mean_longitude(lons) # doctest: +ELLIPSIS
179.08509...
:type longitudes: :class:`~numpy.ndarray` (or list, ..)
:param longitudes: Geographical longitude values ranging from -180 to 180
in degrees.
"""
from scipy.stats import circmean
mean_longitude = circmean(np.array(longitudes), low=-180, high=180)
mean_longitude = _normalize_longitude(mean_longitude)
return mean_longitude
def AngleAnalyzer_finegrid(sma,ang,loc,mass,snap):
'''
Function which gets the mean ie values over all four simulations for particular set of pertruber parameters
Parameters:
sma: semi-major axis of perturber
ang: inclination of perturber
loc: base directory of perturber simulations
mass: mass of perturber
snap: Snapshot out of the 500 orbits
Returns: The circular mean of ie for stars in the disk for the four simulations'''
redirs=directories(sma,ang,loc)[0]
os.chdir(redirs[mass])
bases=get_bases()
circmeans=[]
for i in range(len(bases)):
circmeans.append(stats.circmean(AngleAnalyzer_data_finegrid(sma,ang,loc,mass,snap,bases[i])))
circmeans=np.array(circmeans)*(180/pi)
return circmeans
def test_circmean_axis():
x = np.array([[355, 5, 2, 359, 10, 350], [351, 7, 4, 352, 9, 349],
[357, 9, 8, 358, 4, 356]])
M1 = stats.circmean(x, high=360)
M2 = stats.circmean(x.ravel(), high=360)
assert_allclose(M1, M2, rtol=1e-14)
M1 = stats.circmean(x, high=360, axis=1)
M2 = [stats.circmean(x[i], high=360) for i in range(x.shape[0])]
assert_allclose(M1, M2, rtol=1e-14)
M1 = stats.circmean(x, high=360, axis=0)
M2 = [stats.circmean(x[:, i], high=360) for i in range(x.shape[1])]
assert_allclose(M1, M2, rtol=1e-14)
def analyze(self, atype, periodic=True):
"""Analyzes the data held in numpy vector"""
if not periodic:
self.mean = self.values.mean()
self.stdev = self.values.std()
self.var = self.values.var()
else:
p_high = 180
if atype == "angle":
p_low = 0
else:
p_low = -180
self.mean = circmean(self.values, low=p_low, high=p_high)
self.stdev = circstd(self.values, low=p_low, high=p_high)
self.var = circvar(self.values, low=p_low, high=p_high)
# Analyze normality
# Note: this will be broken for distributions that go over a period
# To be fixed
self.anderson = stats.anderson(self.values, dist='norm')
def mean_longitude(longitudes):
"""
Compute sample mean longitude, assuming longitude in degrees from -180 to
180.
>>> lons = (-170.5, -178.3, 166)
>>> np.mean(lons) # doctest: +SKIP
-60.933
>>> mean_longitude(lons) # doctest: +ELLIPSIS
179.08509...
:type longitudes: :class:`~numpy.ndarray` (or list, ..)
:param longitudes: Geographical longitude values ranging from -180 to 180
in degrees.
"""
mean_longitude = circmean(np.array(longitudes), low=-180, high=180)
while mean_longitude < -180:
mean_longitude += 360
while mean_longitude > 180:
mean_longitude -= 360
return mean_longitude
def update_panel_heading(self, t, panel_jump, gain_yaw, open_loop,
open_loop_value):
if open_loop == 1:
self.panel_heading = open_loop_value
else: #if it is closed-loop
self.panel_heading = (self.panel_heading + self.velheading *
gain_yaw + panel_jump) % 360
#make the panel yaw be the previously stored panel yaw value + the change in heading of the animal by the yaw gain, plus the panel jump value
self.time_list.append(self.time)
self.panel_heading_list.append(self.panel_heading)
if self.count >= 2:
self.goal_heading = 360 * circmean(
[x * 2 * np.pi / 360
for x in self.panel_heading_list]) / 2 / np.pi
self.count += 1
# Cull old data points
while (t - self.time_list[0]) > self.time_window:
self.time_list.pop(0)
self.panel_heading_list.pop(0)
def folded_bias(prevcurr, error, window, step):
xxx = np.arange(-np.pi, np.pi, step)
t_err = []
err = []
for t in xxx:
idx = (prevcurr >= t - window / 2) & (prevcurr < t + window / 2)
if t - window / 2 < -np.pi:
idx = (prevcurr >= t - window / 2) & (
prevcurr < t + window / 2) | (prevcurr > np.pi -
(window / 2 -
(np.pi - np.abs(t))))
if t + window / 2 > np.pi:
idx = (prevcurr >= t - window / 2) & (
prevcurr < t + window / 2) | (prevcurr < -np.pi +
(window / 2 -
(np.pi - np.abs(t))))
t_err.append(list(error[idx]))
for t in reversed(range(int(len(xxx) / 2))):
err.append([x * -1 for x in t_err[t]] + t_err[-t - 1])
m_err = [sps.circmean(x, low=-np.pi, high=np.pi) for x in err]
se_err = [sps.circstd(x) / np.sqrt(len(x)) for x in err]
return np.array(m_err), np.array(se_err)
def mean_time(times, kind='time'):
"""mean_time [summary]
[extended_summary]
Args:
times ([type]): [description]
kind (str, optional): [description]. Defaults to 'time'.
Returns:
[type]: [description]
"""
day = 24 * 60**2
angles = times_to_angles(times, day)
mean_angle = circmean(angles, high=360)
if kind == 'time':
return dt.time(
*angles_to_time(mean_angle)) #np.timedelta64(sd_seconds,'s')
if kind == 'angles':
return mean_angle
if kind == 'datetime':
return dt.datetime(2021, 1, 1, *angles_to_time(mean_angle))
if kind == 'timedelta':
return dt.timedelta(seconds=(mean_angle * day) / 360)
def test_BasicSimpleMeanStartEdgeCrossing(self): # goes on the left side from origin, up to 349 degrees
mymean = round(StaCircStats.Sta_CircWMean({6:1,344:3}, low=0, high=360),4)
scimean = round(stats.circmean(samples=np.array([6,344,344, 344]), high=360, low=0),4)
self.assertEqual(scimean, mymean)
def circ_corrcc(x, y, tail='two-sided', correction_uniform=False):
"""Correlation coefficient between two circular variables.
Parameters
----------
x : np.array
First circular variable (expressed in radians)
y : np.array
Second circular variable (expressed in radians)
tail : string
Specify whether to return 'one-sided' or 'two-sided' p-value.
correction_uniform : bool
Use correction for uniform marginals
Returns
-------
r : float
Correlation coefficient
pval : float
Uncorrected p-value
Notes
-----
Adapted from the CircStats MATLAB toolbox (Berens 2009).
Use the :py:func:`numpy.deg2rad` function to convert angles from degrees
to radians.
Please note that NaN are automatically removed.
If the ``correction_uniform`` is True, an alternative equation from
Jammalamadaka & Sengupta (2001, p. 177) is used.
If the marginal distribution of ``x`` or ``y`` is uniform, the mean is
not well defined, which leads to wrong estimates of the circular
correlation. The alternative equation corrects for this by choosing the
means in a way that maximizes the postitive or negative correlation.
References
----------
.. [1] Berens, P. (2009). CircStat: A MATLAB Toolbox for Circular
Statistics. Journal of Statistical Software, Articles, 31(10), 1–21.
https://doi.org/10.18637/jss.v031.i10
.. [2] Jammalamadaka, S. R., & Sengupta, A. (2001). Topics in circular
statistics (Vol. 5). world scientific.
Examples
--------
Compute the r and p-value of two circular variables
>>> from pingouin import circ_corrcc
>>> x = [0.785, 1.570, 3.141, 3.839, 5.934]
>>> y = [0.593, 1.291, 2.879, 3.892, 6.108]
>>> r, pval = circ_corrcc(x, y)
>>> print(r, pval)
0.942 0.06579836070349088
With the correction for uniform marginals
>>> r, pval = circ_corrcc(x, y, correction_uniform=True)
>>> print(r, pval)
0.547 0.28585306869206784
"""
from scipy.stats import norm
x = np.asarray(x)
y = np.asarray(y)
# Check size
if x.size != y.size:
raise ValueError('x and y must have the same length.')
# Remove NA
x, y = remove_na(x, y, paired=True)
n = x.size
# Compute correlation coefficient
x_sin = np.sin(x - circmean(x))
y_sin = np.sin(y - circmean(y))
if not correction_uniform:
# Similar to np.corrcoef(x_sin, y_sin)[0][1]
r = np.sum(x_sin * y_sin) / np.sqrt(
np.sum(x_sin**2) * np.sum(y_sin**2))
else:
r_minus = np.abs(np.sum(np.exp((x - y) * 1j)))
r_plus = np.abs(np.sum(np.exp((x + y) * 1j)))
denom = 2 * np.sqrt(np.sum(x_sin**2) * np.sum(y_sin**2))
r = (r_minus - r_plus) / denom
# Compute T- and p-values
tval = np.sqrt((n * (x_sin**2).mean() *
(y_sin**2).mean()) / np.mean(x_sin**2 * y_sin**2)) * r
# Approximately distributed as a standard normal
pval = 2 * norm.sf(abs(tval))
pval = pval / 2 if tail == 'one-sided' else pval
return np.round(r, 3), pval
def analyze_color(rgb_img, mask, hist_plot_type=None):
"""Analyze the color properties of an image object
Inputs:
rgb_img = RGB image data
mask = Binary mask made from selected contours
hist_plot_type = 'None', 'all', 'rgb','lab' or 'hsv'
Returns:
analysis_image = histogram output
:param rgb_img: numpy.ndarray
:param mask: numpy.ndarray
:param hist_plot_type: str
:return analysis_images: list
"""
params.device += 1
if len(np.shape(rgb_img)) < 3:
fatal_error("rgb_img must be an RGB image")
# Mask the input image
masked = cv2.bitwise_and(rgb_img, rgb_img, mask=mask)
# Extract the blue, green, and red channels
b, g, r = cv2.split(masked)
# Convert the BGR image to LAB
lab = cv2.cvtColor(masked, cv2.COLOR_BGR2LAB)
# Extract the lightness, green-magenta, and blue-yellow channels
l, m, y = cv2.split(lab)
# Convert the BGR image to HSV
hsv = cv2.cvtColor(masked, cv2.COLOR_BGR2HSV)
# Extract the hue, saturation, and value channels
h, s, v = cv2.split(hsv)
# Color channel dictionary
channels = {"b": b, "g": g, "r": r, "l": l, "m": m, "y": y, "h": h, "s": s, "v": v}
# Histogram plot types
hist_types = {"ALL": ("b", "g", "r", "l", "m", "y", "h", "s", "v"),
"RGB": ("b", "g", "r"),
"LAB": ("l", "m", "y"),
"HSV": ("h", "s", "v")}
if hist_plot_type is not None and hist_plot_type.upper() not in hist_types:
fatal_error("The histogram plot type was " + str(hist_plot_type) +
', but can only be one of the following: None, "all", "rgb", "lab", or "hsv"!')
# Store histograms, plotting colors, and plotting labels
histograms = {
"b": {"label": "blue", "graph_color": "blue",
"hist": [float(l[0]) for l in cv2.calcHist([channels["b"]], [0], mask, [256], [0, 255])]},
"g": {"label": "green", "graph_color": "forestgreen",
"hist": [float(l[0]) for l in cv2.calcHist([channels["g"]], [0], mask, [256], [0, 255])]},
"r": {"label": "red", "graph_color": "red",
"hist": [float(l[0]) for l in cv2.calcHist([channels["r"]], [0], mask, [256], [0, 255])]},
"l": {"label": "lightness", "graph_color": "dimgray",
"hist": [float(l[0]) for l in cv2.calcHist([channels["l"]], [0], mask, [256], [0, 255])]},
"m": {"label": "green-magenta", "graph_color": "magenta",
"hist": [float(l[0]) for l in cv2.calcHist([channels["m"]], [0], mask, [256], [0, 255])]},
"y": {"label": "blue-yellow", "graph_color": "yellow",
"hist": [float(l[0]) for l in cv2.calcHist([channels["y"]], [0], mask, [256], [0, 255])]},
"h": {"label": "hue", "graph_color": "blueviolet",
"hist": [float(l[0]) for l in cv2.calcHist([channels["h"]], [0], mask, [256], [0, 255])]},
"s": {"label": "saturation", "graph_color": "cyan",
"hist": [float(l[0]) for l in cv2.calcHist([channels["s"]], [0], mask, [256], [0, 255])]},
"v": {"label": "value", "graph_color": "orange",
"hist": [float(l[0]) for l in cv2.calcHist([channels["v"]], [0], mask, [256], [0, 255])]}
}
# Create list of bin labels for 8-bit data
binval = np.arange(0, 256)
bin_values = [l for l in binval]
analysis_images = []
# Create a dataframe of bin labels and histogram data
dataset = pd.DataFrame({'bins': binval, 'blue': histograms["b"]["hist"],
'green': histograms["g"]["hist"], 'red': histograms["r"]["hist"],
'lightness': histograms["l"]["hist"], 'green-magenta': histograms["m"]["hist"],
'blue-yellow': histograms["y"]["hist"], 'hue': histograms["h"]["hist"],
'saturation': histograms["s"]["hist"], 'value': histograms["v"]["hist"]})
# Make the histogram figure using plotnine
if hist_plot_type is not None:
if hist_plot_type.upper() == 'RGB':
df_rgb = pd.melt(dataset, id_vars=['bins'], value_vars=['blue', 'green', 'red'],
var_name='Color Channel', value_name='Pixels')
hist_fig = (ggplot(df_rgb, aes(x='bins', y='Pixels', color='Color Channel'))
+ geom_line()
+ scale_x_continuous(breaks=list(range(0, 256, 25)))
+ scale_color_manual(['blue', 'green', 'red'])
)
analysis_images.append(hist_fig)
elif hist_plot_type.upper() == 'LAB':
df_lab = pd.melt(dataset, id_vars=['bins'],
value_vars=['lightness', 'green-magenta', 'blue-yellow'],
var_name='Color Channel', value_name='Pixels')
hist_fig = (ggplot(df_lab, aes(x='bins', y='Pixels', color='Color Channel'))
+ geom_line()
+ scale_x_continuous(breaks=list(range(0, 256, 25)))
+ scale_color_manual(['yellow', 'magenta', 'dimgray'])
)
analysis_images.append(hist_fig)
elif hist_plot_type.upper() == 'HSV':
df_hsv = pd.melt(dataset, id_vars=['bins'],
value_vars=['hue', 'saturation', 'value'],
var_name='Color Channel', value_name='Pixels')
hist_fig = (ggplot(df_hsv, aes(x='bins', y='Pixels', color='Color Channel'))
+ geom_line()
+ scale_x_continuous(breaks=list(range(0, 256, 25)))
+ scale_color_manual(['blueviolet', 'cyan', 'orange'])
)
analysis_images.append(hist_fig)
elif hist_plot_type.upper() == 'ALL':
s = pd.Series(['blue', 'green', 'red', 'lightness', 'green-magenta',
'blue-yellow', 'hue', 'saturation', 'value'], dtype="category")
color_channels = ['blue', 'yellow', 'green', 'magenta', 'blueviolet',
'dimgray', 'red', 'cyan', 'orange']
df_all = pd.melt(dataset, id_vars=['bins'], value_vars=s, var_name='Color Channel',
value_name='Pixels')
hist_fig = (ggplot(df_all, aes(x='bins', y='Pixels', color='Color Channel'))
+ geom_line()
+ scale_x_continuous(breaks=list(range(0, 256, 25)))
+ scale_color_manual(color_channels)
)
analysis_images.append(hist_fig)
# Hue values of zero are red but are also the value for pixels where hue is undefined
# The hue value of a pixel will be undefined when the color values are saturated
# Therefore, hue values of zero are excluded from the calculations below
# Calculate the median hue value
# The median is rescaled from the encoded 0-179 range to the 0-359 degree range
hue_median = np.median(h[np.where(h > 0)]) * 2
# Calculate the circular mean and standard deviation of the encoded hue values
# The mean and standard-deviation are rescaled from the encoded 0-179 range to the 0-359 degree range
hue_circular_mean = stats.circmean(h[np.where(h > 0)], high=179, low=0) * 2
hue_circular_std = stats.circstd(h[np.where(h > 0)], high=179, low=0) * 2
# Store into lists instead for pipeline and print_results
# stats_dict = {'mean': circular_mean, 'std' : circular_std, 'median': median}
# Plot or print the histogram
if hist_plot_type is not None:
if params.debug == 'print':
hist_fig.save(os.path.join(params.debug_outdir, str(params.device) + '_analyze_color_hist.png'))
elif params.debug == 'plot':
print(hist_fig)
# Store into global measurements
# RGB signal values are in an unsigned 8-bit scale of 0-255
rgb_values = [i for i in range(0, 256)]
# Hue values are in a 0-359 degree scale, every 2 degrees at the midpoint of the interval
hue_values = [i * 2 + 1 for i in range(0, 180)]
# Percentage values on a 0-100 scale (lightness, saturation, and value)
percent_values = [round((i / 255) * 100, 2) for i in range(0, 256)]
# Diverging values on a -128 to 127 scale (green-magenta and blue-yellow)
diverging_values = [i for i in range(-128, 128)]
# outputs.measurements['color_data'] = {
# 'histograms': {
# 'blue': {'signal_values': rgb_values, 'frequency': histograms["b"]["hist"]},
# 'green': {'signal_values': rgb_values, 'frequency': histograms["g"]["hist"]},
# 'red': {'signal_values': rgb_values, 'frequency': histograms["r"]["hist"]},
# 'lightness': {'signal_values': percent_values, 'frequency': histograms["l"]["hist"]},
# 'green-magenta': {'signal_values': diverging_values, 'frequency': histograms["m"]["hist"]},
# 'blue-yellow': {'signal_values': diverging_values, 'frequency': histograms["y"]["hist"]},
# 'hue': {'signal_values': hue_values, 'frequency': histograms["h"]["hist"]},
# 'saturation': {'signal_values': percent_values, 'frequency': histograms["s"]["hist"]},
# 'value': {'signal_values': percent_values, 'frequency': histograms["v"]["hist"]}
# },
# 'color_features': {
# 'hue_circular_mean': hue_circular_mean,
# 'hue_circular_std': hue_circular_std,
# 'hue_median': hue_median
# }
# }
outputs.add_observation(variable='blue_frequencies', trait='blue frequencies',
method='plantcv.plantcv.analyze_color', scale='frequency', datatype=list,
value=histograms["b"]["hist"], label=rgb_values)
outputs.add_observation(variable='green_frequencies', trait='green frequencies',
method='plantcv.plantcv.analyze_color', scale='frequency', datatype=list,
value=histograms["g"]["hist"], label=rgb_values)
outputs.add_observation(variable='red_frequencies', trait='red frequencies',
method='plantcv.plantcv.analyze_color', scale='frequency', datatype=list,
value=histograms["r"]["hist"], label=rgb_values)
outputs.add_observation(variable='lightness_frequencies', trait='lightness frequencies',
method='plantcv.plantcv.analyze_color', scale='frequency', datatype=list,
value=histograms["l"]["hist"], label=percent_values)
outputs.add_observation(variable='green-magenta_frequencies', trait='green-magenta frequencies',
method='plantcv.plantcv.analyze_color', scale='frequency', datatype=list,
value=histograms["m"]["hist"], label=diverging_values)
outputs.add_observation(variable='blue-yellow_frequencies', trait='blue-yellow frequencies',
method='plantcv.plantcv.analyze_color', scale='frequency', datatype=list,
value=histograms["y"]["hist"], label=diverging_values)
outputs.add_observation(variable='hue_frequencies', trait='hue frequencies',
method='plantcv.plantcv.analyze_color', scale='frequency', datatype=list,
value=histograms["h"]["hist"], label=hue_values)
outputs.add_observation(variable='saturation_frequencies', trait='saturation frequencies',
method='plantcv.plantcv.analyze_color', scale='frequency', datatype=list,
value=histograms["s"]["hist"], label=percent_values)
outputs.add_observation(variable='value_frequencies', trait='value frequencies',
method='plantcv.plantcv.analyze_color', scale='frequency', datatype=list,
value=histograms["v"]["hist"], label=percent_values)
outputs.add_observation(variable='hue_circular_mean', trait='hue circular mean',
method='plantcv.plantcv.analyze_color', scale='degrees', datatype=float,
value=hue_circular_mean, label='degrees')
outputs.add_observation(variable='hue_circular_std', trait='hue circular standard deviation',
method='plantcv.plantcv.analyze_color', scale='degrees', datatype=float,
value=hue_median, label='degrees')
outputs.add_observation(variable='hue_median', trait='hue median',
method='plantcv.plantcv.analyze_color', scale='degrees', datatype=float,
value=hue_median, label='degrees')
# Store images
outputs.images.append(analysis_images)
return analysis_images
edgecolors=['k' for i in hist]
bars = ax1.bar(edges[:-1],hist,width=width,color='#737373',edgecolor=edgecolors)
#ax1.yaxis.set_major_locator(pl.MaxNLocator(3))
#ax1.yaxis.set_major_locator(pl.LinearLocator(3))
max_hist=np.max(hist)
tick_locations=[int(max_hist/2.),max_hist]
print(tick_locations)
ax1.yaxis.set_major_locator(pl.FixedLocator(tick_locations))
xpos=-1.5*max_hist
#draw_straight_axis(xpos,ax1.yaxis.get_ticklocs(),ax1)
#ax1.yaxis.set_major_formatter(ticks)
ax1.yaxis.set_major_formatter(ticker.FixedFormatter(['',str(max_hist)]))
ax1.set_rlabel_position(90)
ax1.xaxis.set_major_formatter(ticks)
mean_angle=circmean(data['Corrected_Kino_Angle'])
std_angle=np.sqrt(circvar(data['Corrected_Kino_Angle']))
ymin,ymax=ax1.get_ylim()
ax1.plot([mean_angle,mean_angle],[0,ymax],'b')
std_angles=np.linspace(mean_angle-old_div(std_angle,2),mean_angle+old_div(std_angle,2),11)
std_radii=np.ones(11)*2.*max_hist/3.
#ax1.plot([mean_angle-std_angle/2,mean_angle+std_angle/2],[2.*max_hist/3.,2.*max_hist/3.],'r')
ax1.plot(std_angles,std_radii,'r')
ax1.set_ylim([0,max_hist])
pl.savefig('kinos_histogram_'+filename+'_'+sheet+extension)
pl.close(fig2)
#===========================================================================
# Plot histogram of bundle orientations
hist,edges=np.histogram(data['Corrected_Bundle_Angle'],density=False,bins=bins,range=[0,2.*np.pi])
def hpd(ary, credible_interval=0.94, circular=False):
"""
Calculate highest posterior density (HPD) of array for given credible_interval.
The HPD is the minimum width Bayesian credible interval (BCI). This implementation works only
for unimodal distributions.
Parameters
----------
x : Numpy array
An array containing posterior samples
credible_interval : float, optional
Credible interval to compute. Defaults to 0.94.
circular : bool, optional
Whether to compute the hpd taking into account `x` is a circular variable
(in the range [-np.pi, np.pi]) or not. Defaults to False (i.e non-circular variables).
Returns
-------
np.ndarray
lower and upper value of the interval.
Examples
--------
Calculate the hpd of a Normal random variable:
.. ipython::
In [1]: import arviz as az
...: import numpy as np
...: data = np.random.normal(size=2000)
...: az.hpd(data, credible_interval=.68)
"""
if ary.ndim > 1:
hpd_array = np.array(
[hpd(row, credible_interval=credible_interval, circular=circular) for row in ary.T]
)
return hpd_array
# Make a copy of trace
ary = ary.copy()
n = len(ary)
if circular:
mean = st.circmean(ary, high=np.pi, low=-np.pi)
ary = ary - mean
ary = np.arctan2(np.sin(ary), np.cos(ary))
ary = np.sort(ary)
interval_idx_inc = int(np.floor(credible_interval * n))
n_intervals = n - interval_idx_inc
interval_width = ary[interval_idx_inc:] - ary[:n_intervals]
if len(interval_width) == 0:
raise ValueError(
"Too few elements for interval calculation. "
"Check that credible_interval meets condition 0 =< credible_interval < 1"
)
min_idx = np.argmin(interval_width)
hdi_min = ary[min_idx]
hdi_max = ary[min_idx + interval_idx_inc]
if circular:
hdi_min = hdi_min + mean
hdi_max = hdi_max + mean
hdi_min = np.arctan2(np.sin(hdi_min), np.cos(hdi_min))
hdi_max = np.arctan2(np.sin(hdi_max), np.cos(hdi_max))
return np.array([hdi_min, hdi_max])
def test_empty(self):
assert_(np.isnan(stats.circmean([])))
assert_(np.isnan(stats.circstd([])))
assert_(np.isnan(stats.circvar([])))
def test_empty(self):
assert_(np.isnan(stats.circmean([])))
assert_(np.isnan(stats.circstd([])))
assert_(np.isnan(stats.circvar([])))
def hpd(x, credible_interval=0.94, transform=lambda x: x, circular=False):
"""
Calculate highest posterior density (HPD) of array for given credible_interval.
The HPD is the minimum width Bayesian credible interval (BCI). This implementation works only
for unimodal distributions.
Parameters
----------
x : Numpy array
An array containing posterior samples
credible_interval : float, optional
Credible interval to plot. Defaults to 0.94.
transform : callable
Function to transform data (defaults to identity)
circular : bool, optional
Whether to compute the error taking into account `x` is a circular variable
(in the range [-np.pi, np.pi]) or not. Defaults to False (i.e non-circular variables).
Returns
-------
np.ndarray
lower and upper value of the interval.
"""
if x.ndim > 1:
return np.array([
hpd(row,
credible_interval=credible_interval,
transform=transform,
circular=circular) for row in x.T
])
# Make a copy of trace
x = transform(x.copy())
len_x = len(x)
if circular:
mean = st.circmean(x, high=np.pi, low=-np.pi)
x = x - mean
x = np.arctan2(np.sin(x), np.cos(x))
x = np.sort(x)
interval_idx_inc = int(np.floor(credible_interval * len_x))
n_intervals = len_x - interval_idx_inc
interval_width = x[interval_idx_inc:] - x[:n_intervals]
if len(interval_width) == 0:
raise ValueError(
"Too few elements for interval calculation. "
"Check that credible_interval meets condition 0 =< credible_interval < 1"
)
min_idx = np.argmin(interval_width)
hdi_min = x[min_idx]
hdi_max = x[min_idx + interval_idx_inc]
if circular:
hdi_min = hdi_min + mean
hdi_max = hdi_max + mean
hdi_min = np.arctan2(np.sin(hdi_min), np.cos(hdi_min))
hdi_max = np.arctan2(np.sin(hdi_max), np.cos(hdi_max))
return np.array([hdi_min, hdi_max])
# for running on an interactive cluster node cellprofiler_path = "/g/almf/software/CP2C/CellProfiler11047/" import sys sys.path.append(cellprofiler_path) import cellprofiler.cpmath.cpmorphology as morph import Image import numpy as np from scipy import stats from scipy.ndimage import convolve x = np.array([0, 10, 20, 10, 20, 350]) / 360.0 * 2.0 * np.pi stats.circvar(x) stats.circmean(x) import numpy as np from scipy import stats from scipy.ndimage import convolve radius = 3 image = np.ones((10, 10)) kernel = np.ones((radius, radius)) / radius ** 2 image_sin_mean = convolve(np.sin(image), kernel, mode="constant") image_cos_mean = convolve(np.cos(image), kernel, mode="constant") image_var = 1 - (image_sin_mean ** 2 + image_cos_mean ** 2) import cellprofiler.cpmath.filter as filter import numpy as np
def mean_time(x): return pd.Series(circmean(x,high=360),name='mean') def std_time(x): return pd.Series(circstd(x,high=360),name='std')
def showModelDynamics(modeldata):
""" Model Dynamics for particular competition """
columns = 4
f1, ax_array = plt.subplots(1, columns, figsize=(8, 2), sharey=True)
f1.subplots_adjust(hspace=.3, wspace=.3, left=0.1, right=0.9)
''' colormap '''
cmap = plt.cm.jet
cNorm = colors.Normalize(vmin=np.min(modeldata.competition),
vmax=np.max(modeldata.competition))
scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=cmap)
low_col = scalarMap.to_rgba(np.min(modeldata.competition))
high_col = scalarMap.to_rgba(np.max(modeldata.competition))
for i, ax in enumerate(ax_array):
if i == 0:
ax.set_ylabel('value')
else:
ax.tick_params(labelleft='off')
run = int(i * (max(modeldata.Run) + 1) / (columns - 0.9))
competition = modeldata[modeldata.Run == run].competition.iloc[0]
competition_data = modeldata[modeldata.competition == competition]
xVals = competition_data.Step.unique()
proxy_mean = competition_data.groupby(['Step'
])['mean_proxy_value'].mean()
proxy_std = competition_data.groupby(['Step'
])['mean_proxy_value'].std()
# ax.scatter(competition_data.Step,
# competition_data.mean_proxy_value,
# alpha=0.2, c=high_col, marker=".", label='_nolegend_')
ax.plot(xVals, proxy_mean, c=high_col, label='proxy')
ax.fill_between(xVals,
proxy_mean - proxy_std,
proxy_mean + proxy_std,
alpha=0.2,
color=high_col)
goal_mean = competition_data.groupby(['Step'
])['mean_goal_value'].mean()
goal_std = competition_data.groupby(['Step'])['mean_goal_value'].std()
# ax.scatter(competition_data.Step,
# competition_data.mean_goal_value,
# alpha=0.2, c=low_col, marker=".", label='_nolegend_')
ax.plot(xVals, goal_mean, c=low_col, label='goal')
ax.fill_between(xVals,
goal_mean - goal_std,
goal_mean + goal_std,
alpha=0.2,
color=low_col)
ax.set_xlabel('step')
# ax.set_title('c{:.1f} r{:02d}'.format(competition, run))
ax.set_ylim([
np.min([
modeldata.mean_proxy_value.min(),
modeldata.mean_goal_value.min()
]),
np.max([
modeldata.mean_proxy_value.max(),
modeldata.mean_goal_value.max()
])
])
if i == 0:
ax.legend()
f1.savefig('Dynamics.pdf')
f2, ax_array = plt.subplots(1, columns, figsize=(8, 1.2), sharey=True)
f2.subplots_adjust(hspace=.3, wspace=.3, left=0.1, right=0.9)
for i, ax in enumerate(ax_array):
if i == 0:
ax.set_ylabel('mean practice (°)')
else:
ax.tick_params(labelleft='off')
run = int(i * (max(modeldata.Run) + 1) / (columns - 0.9))
competition = modeldata[modeldata.Run == run].competition.iloc[0]
# print(competition/np.pi * 180)
competition_data = modeldata[modeldata.competition == competition]
xVals = competition_data.Step.unique()
yVals_raw = competition_data.groupby(['Step'])['mean_practice']
practice_mean = np.empty(np.size(xVals))
practice_std = np.empty(np.size(xVals))
i = 0
for name, group in yVals_raw:
practice_mean[i] = stats.circmean(group, -np.pi, np.pi)
practice_std[i] = stats.circstd(group, -np.pi, np.pi)
i += 1
practice_mean = practice_mean / np.pi * 180
practice_std = practice_std / np.pi * 180
# ax.scatter(competition_data.Step,
# competition_data.mean_practice,
# alpha=0.2, c="k", marker=".")
ax.plot(xVals, practice_mean, c='k')
ax.fill_between(xVals,
practice_mean - practice_std,
practice_mean + practice_std,
alpha=0.2,
color="k")
ax.plot([0, np.max(xVals)], [0, 0], c='grey', ls='--', lw=0.5)
ax.set_xlabel('step')
# ax.set_ylim([0,1])
f2.savefig('PracticeDynamics.pdf')
def test_FloatValuesBasicVerificationSmaller(self): # goes on the left side from origin, up to 349 degrees
mymean = round(StaCircStats.Sta_CircWMean({6:1,343.5:3}, low=0, high=360),4)
scimean = round(stats.circmean(samples=np.array([6,344,344, 344]), high=360, low=0),4)
self.assertGreater(scimean,mymean)
start_index = anemometer_analyzer.find_nearest(
sec_since_release_sublist, start_time)
end_time = start_time + bin_duration
end_index = anemometer_analyzer.find_nearest(
sec_since_release_sublist, end_time)
if bin_count < total_bin_number:
direction_slice = direction_sublist[start_index:end_index]
speed_slice = speed_sublist[start_index:
end_index] #still in m/s
else:
print('this last bin is not a full %d seconds in length ' %
(bin_duration))
binned_wind_dir.append(
circmean(direction_sublist[start_index:-1],
high=360,
low=0))
binned_windspeeds.append(np.mean(
speed_sublist[start_index:-1]))
break
binned_wind_dir.append(circmean(direction_slice, high=360, low=0))
binned_windspeeds.append(np.mean(speed_slice)) # still in m/s
start_time = end_time
bin_count += 1
# now calculating the angle of the path from the release site to the trap in question
trap_number = d[experiment_date]['trap_num']
trap_lettering_list = [
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'
]
def test_BasicSimpleMean2(self):
mymean = StaCircStats.Sta_CircWMean({30:1,50:3}, low=0, high=360)
scimean = stats.circmean(samples=np.array([30,50,50, 50]), high=360, low=0)
self.assertEqual(round(scimean, 4) , round(mymean,4 ))
def test_circfuncs_array_like(self):
x = [355,5,2,359,10,350]
assert_allclose(stats.circmean(x, high=360), 0.167690146, rtol=1e-7)
assert_allclose(stats.circvar(x, high=360), 42.51955609, rtol=1e-7)
assert_allclose(stats.circstd(x, high=360), 6.520702116, rtol=1e-7)
def main():
# parse args --------------------------------------------------------------
args = _parse_args()
# set device and data format ----------------------------------------------
if args.cpu:
os.environ['CUDA_VISIBLE_DEVICES'] = ''
args.devices = ''
else:
os.environ['CUDA_VISIBLE_DEVICES'] = args.devices
if not args.devices or args.model == 'mobilenet_v2':
# note: tensorflow supports b01c pooling on cpu only
K.set_image_data_format('channels_last')
else:
K.set_image_data_format('channels_first')
# load model --------------------------------------------------------------
model = load_network(args.model,
args.weights_filepath,
args.input_type,
args.input_height,
args.input_width,
args.output_type,
sampling=args.n_samples > 1,
n_classes=args.n_classes,
mobilenet_v2_alpha=args.mobilenet_v2_alpha)
# parse for image files ---------------------------------------------------
# note: we do not search for mask files, but derive masks from either the
# depth or rgb image during preprocessing
DEPTH_SUFFIX = '_Depth.pgm'
RGB_SUFFIX = '_RGB.png'
MASK_SUFFIX = '_Mask.png'
# get filepaths
mask_filepaths = get_files_by_extension(args.image_or_image_basepath,
extension=MASK_SUFFIX.lower(),
flat_structure=True,
recursive=True,
follow_links=True)
if args.input_type in [INPUT_DEPTH, INPUT_DEPTH_AND_RGB]:
depth_filepaths = get_files_by_extension(
args.image_or_image_basepath,
extension=DEPTH_SUFFIX.lower(),
flat_structure=True,
recursive=True,
follow_links=True)
assert len(depth_filepaths) == len(mask_filepaths)
filepaths = list(zip(depth_filepaths, mask_filepaths))
assert all(
depth_fp.replace(DEPTH_SUFFIX, '') == mask_fp.replace(
MASK_SUFFIX, '') for depth_fp, mask_fp in filepaths)
if args.input_type in [INPUT_RGB, INPUT_DEPTH_AND_RGB]:
rgb_filepaths = get_files_by_extension(args.image_or_image_basepath,
extension=RGB_SUFFIX.lower(),
flat_structure=True,
recursive=True,
follow_links=True)
assert len(rgb_filepaths) == len(mask_filepaths)
filepaths = list(zip(rgb_filepaths, mask_filepaths))
assert all(
rgb_fp.replace(RGB_SUFFIX, '') == mask_fp.replace(MASK_SUFFIX, '')
for rgb_fp, mask_fp in filepaths)
if args.input_type == INPUT_DEPTH_AND_RGB:
filepaths = list(zip(depth_filepaths, rgb_filepaths, mask_filepaths))
# define preprocessing function -------------------------------------------
def load_and_preprocess(inputs):
# unpack inputs
if args.input_type == INPUT_DEPTH_AND_RGB:
depth_filepath, rgb_filepath, mask_filepath = inputs
elif args.input_type == INPUT_DEPTH:
depth_filepath, mask_filepath = inputs
else:
rgb_filepath, mask_filepath = inputs
# pack shape
shape = (args.input_height, args.input_width)
# load mask
mask = img_utils.load(mask_filepath)
mask_resized = pre.resize_mask(mask, shape)
mask_resized = mask_resized > 0
# prepare depth input
if args.input_type in [INPUT_DEPTH, INPUT_DEPTH_AND_RGB]:
# load
depth = img_utils.load(depth_filepath)
# create mask
# mask = depth > 0
# mask_resized = pre.resize_mask(mask.astype('uint8')*255, shape) > 0
# mask (redundant, since mask is derived from depth image)
# depth = pre.mask_img(depth, mask)
# resize
depth = pre.resize_depth_img(depth, shape)
# 01 -> 01c
depth = depth[..., None]
# preprocess
depth = pre.preprocess_img(
depth,
mask=mask_resized,
scale01=args.input_preprocessing == 'scale01',
standardize=args.input_preprocessing == 'standardize',
zero_mean=True,
unit_variance=True)
# convert to correct data format
if K.image_data_format() == 'channels_last':
axes = 'b01c'
else:
axes = 'bc01'
depth = img_utils.dimshuffle(depth, '01c', axes)
# repeat if sampling is enabled
if args.n_samples > 1:
depth = np.repeat(depth, args.n_samples, axis=0)
# prepare rgb input
if args.input_type in [INPUT_RGB, INPUT_DEPTH_AND_RGB]:
# load
rgb = img_utils.load(rgb_filepath)
# create mask
# if args.input_type == INPUT_RGB:
# # derive mask from rgb image
# mask = rgb > 0
# mask_resized = pre.resize_mask(mask.astype('uint8')*255,
# shape) > 0
# else:
# # mask rgb image using mask derived from depth image
# rgb = pre.mask_img(rgb, mask)
# resize
rgb = pre.resize_depth_img(rgb, shape)
# preprocess
rgb = pre.preprocess_img(
rgb,
mask=mask_resized,
scale01=args.input_preprocessing == 'scale01',
standardize=args.input_preprocessing == 'standardize',
zero_mean=True,
unit_variance=True)
# convert to correct data format
if K.image_data_format() == 'channels_last':
axes = 'b01c'
else:
axes = 'bc01'
rgb = img_utils.dimshuffle(rgb, '01c', axes)
# repeat if sampling is enabled
if args.n_samples > 1:
rgb = np.repeat(rgb, args.n_samples, axis=0)
# return preprocessed images
if args.input_type == INPUT_DEPTH_AND_RGB:
return depth, rgb
elif args.input_type == INPUT_DEPTH:
return depth,
else:
return rgb,
# define postprocessing function ------------------------------------------
def postprocess(output):
if args.output_type == OUTPUT_BITERNION:
return post.biternion2deg(output)
elif args.output_type == OUTPUT_REGRESSION:
return post.rad2deg(output)
else:
return post.class2deg(np.argmax(output, axis=-1), args.n_classes)
# process files -----------------------------------------------------------
len_cnt = len(str(len(filepaths)))
plt.ion()
fig = plt.figure(figsize=(8, 6))
for i, inputs in enumerate(filepaths):
print("[{:0{}d}/{:0{}d}]: {}".format(i + 1, len_cnt, len(filepaths),
len_cnt, inputs))
# load and preprocess inputs
nw_inputs = load_and_preprocess(inputs)
# predict
nw_output = model.predict(nw_inputs, batch_size=args.n_samples)
# postprocess output
output = postprocess(nw_output)
# visualize inputs and predicted angle
plt.clf()
# visualize inputs
for j, inp in enumerate(nw_inputs):
# first element of input batch
img = inp[0]
# convert to 01c
if K.image_data_format() == 'channels_last':
axes = '01c'
else:
axes = 'c01'
img = img_utils.dimshuffle(img, axes, '01c')
# inverse preprocessing
img = pre.preprocess_img_inverse(
img,
scale01=args.input_preprocessing == 'scale01',
standardize=args.input_preprocessing == 'standardize',
zero_mean=True,
unit_variance=True)
# show
ax = fig.add_subplot(1, len(nw_inputs) + 1, j + 1)
if img.shape[-1] == 1:
ax.imshow(img[:, :, 0],
cmap='gray',
vmin=img[img != 0].min(),
vmax=img.max())
else:
ax.imshow(img)
ax.axis('off')
# visualize output
ax = fig.add_subplot(1,
len(nw_inputs) + 1,
len(nw_inputs) + 1,
polar=True)
ax.set_theta_zero_location('S', offset=0)
ax.hist(np.deg2rad(output),
width=np.deg2rad(2),
density=True,
alpha=0.5 if args.n_samples > 1 else 1.0,
color='#1f77b4')
if args.n_samples > 1:
mean_rad = circmean(np.deg2rad(output))
std_rad = circstd(np.deg2rad(output))
x = np.deg2rad(np.linspace(0, 360, 360))
pdf_values = norm.pdf(x, mean_rad, std_rad)
ax.plot(x, pdf_values, color='#1f77b4', zorder=2, linewidth=2)
ax.fill(x, pdf_values, color='#1f77b4', zorder=2, alpha=0.3)
ax.set_yscale('symlog')
ax.set_ylim([0, 20])
plt.tight_layout()
# plt.savefig(f'./img{i}.png', bbox_inches='tight', dpi=75)
plt.pause(0.0005)
def test_circfuncs_array_like(self):
x = [355, 5, 2, 359, 10, 350]
assert_allclose(stats.circmean(x, high=360), 0.167690146, rtol=1e-7)
assert_allclose(stats.circvar(x, high=360), 42.51955609, rtol=1e-7)
assert_allclose(stats.circstd(x, high=360), 6.520702116, rtol=1e-7)
def test_circular_mean_scipy(mean):
"""Test our `_circular_mean()` function gives same result than Scipy version."""
rvs = st.vonmises.rvs(loc=mean, kappa=1, size=1000)
mean_az = _circular_mean(rvs)
mean_sp = st.circmean(rvs, low=-np.pi, high=np.pi)
np.testing.assert_almost_equal(mean_az, mean_sp)
def _init_(directory, start, end, levels):
print '#### Save merra to npy .... ####'
years = [start[0]]
months = range(start[1], end[1] + 1)
days = range(start[2], end[2] + 1)
# get all time stamps available
for year in years:
for month in months:
for day in days:
year_str = str(year)
month_str = str(month).zfill(2)
day_str = str(day).zfill(2)
suffix = year_str + month_str + day_str
filename_high = directory + "MERRA2_400.tavg3_3d_asm_Nv." + suffix + ".nc4"
if (os.path.exists(filename_high)):
fileList_high.append(filename_high)
dayStrings.append(suffix)
# We also require to have the corresponding file for lowest
filename_low = directory + "MERRA2_400.tavg1_2d_slv_Nx." + suffix + ".nc4"
if not os.path.exists(filename_low):
print(
"WARNING: Can not find corresponding lower level",
filename_low)
else:
fileList_low.append(filename_low)
ndays = len(fileList_high)
print("Found", len(fileList_high),
"matching files in date order for high altitudes")
print("Parsing data into time series")
print(dayStrings)
for dayString in dayStrings:
nc_f = directory + 'MERRA2_400.tavg3_3d_asm_Nv.' + dayString + '.nc4'
nc_f_low = directory + 'MERRA2_400.tavg1_2d_slv_Nx.' + dayString + '.nc4'
nc_fid = Dataset(nc_f, 'r')
nc_fid_low = Dataset(nc_f_low, 'r')
latlonPairs = []
for lat in nc_fid.variables['lat'][:]:
for lon in nc_fid.variables['lon'][:]:
latlonPairs.append([lat, lon])
latlonPairs_numpy = []
for lat in range(len(nc_fid.variables['lat'][:])):
for lon in range(len(nc_fid.variables['lon'][:])):
latlonPairs_numpy.append([lat, lon])
nlev = len(levels)
# prepare the arrays filled with zeros
for (index_lat, index_lon) in latlonPairs:
timeseries_heights = np.zeros((nlev + 3, 8 * ndays))
timeseries_heights[nlev] = 50 * np.ones(
(8 * ndays)) # 50 metres (fixed)
timeseries_heights[nlev + 1] = 10 * np.ones(
(8 * ndays)) # 10 metres (fixed)
timeseries_heights[nlev + 2] = 2 * np.ones(
(8 * ndays)) # 2 metres (fixed)
timeseries_wind = np.zeros((nlev + 3, 8 * ndays), dtype='2float32')
timeseries_features = np.zeros((nlev + 3, 8 * ndays), dtype='2float32')
dict_timeseries_heights[index_lat, index_lon] = timeseries_heights
dict_timeseries_wind[index_lat, index_lon] = timeseries_wind
dict_timeseries_features[index_lat, index_lon] = timeseries_features
# Running through days and save the data in the dicts
# Creating the features directly from the wind speed.
day = 0
for dayString in dayStrings:
nc_f = directory + 'MERRA2_400.tavg3_3d_asm_Nv.' + dayString + '.nc4'
nc_f_low = directory + 'MERRA2_400.tavg1_2d_slv_Nx.' + dayString + '.nc4'
nc_fid = Dataset(nc_f, 'r')
nc_fid_low = Dataset(nc_f_low, 'r')
print(nc_f, "day", day + 1)
for (index_lat, index_lon), (numpy_lat,
numpy_lon) in zip(latlonPairs,
latlonPairs_numpy):
for time in range(8): # 8 3-hour intervals
# get the low levels
timeindex = time * 3
level = len(levels) # At 50m: Call this "Level 73"
eastwinds = nc_fid_low.variables['U50M'][timeindex:timeindex +
3, numpy_lat,
numpy_lon]
northwinds = nc_fid_low.variables['V50M'][timeindex:timeindex +
3, numpy_lat,
numpy_lon]
eastwind_mean = np.mean(eastwinds)
northwind_mean = np.mean(northwinds)
timeseries_wind[level, day * 8 +
time] = [eastwind_mean, northwind_mean]
features = from_wind_to_features(eastwind_mean, northwind_mean)
timeseries_features[level, day * 8 +
time] = [features[0], features[1]]
level = len(levels) + 1 # At 10m: Call this "Level 74"
eastwinds = nc_fid_low.variables['U10M'][timeindex:timeindex +
3, numpy_lat,
numpy_lon]
northwinds = nc_fid_low.variables['V10M'][timeindex:timeindex +
3, numpy_lat,
numpy_lon]
eastwind_mean = np.mean(eastwinds)
northwind_mean = np.mean(northwinds)
timeseries_wind[level, day * 8 +
time] = [eastwind_mean, northwind_mean]
features = from_wind_to_features(eastwind_mean, northwind_mean)
timeseries_features[level, day * 8 +
time] = [features[0], features[1]]
level = len(levels) + 2 # At 2m: Call this "Level 75"
eastwinds = nc_fid_low.variables['U2M'][timeindex:timeindex +
3, numpy_lat,
numpy_lon]
northwinds = nc_fid_low.variables['V2M'][timeindex:timeindex +
3, numpy_lat,
numpy_lon]
eastwind_mean = np.mean(eastwinds)
northwind_mean = np.mean(northwinds)
timeseries_wind[level, day * 8 +
time] = [eastwind_mean, northwind_mean]
features = from_wind_to_features(eastwind_mean, northwind_mean)
timeseries_features[level, day * 8 +
time] = [features[0], features[1]]
for level in range(nlev): # 72 pressure levels
timeseries_heights[
level, day * 8 + time] = nc_fid.variables['H'][
time, level, numpy_lat,
numpy_lon] - nc_fid.variables['PHIS'][
time, numpy_lat,
numpy_lon] / 9.81 # height above sealevel - height of surface/9.81 = actual height (source: NASA email)
eastwind = nc_fid.variables['U'][time, level, numpy_lat,
numpy_lon] # eastwind
northwind = nc_fid.variables['V'][time, level, numpy_lat,
numpy_lon] # northwind
timeseries_wind[level,
day * 8 + time] = [eastwind, northwind]
features = from_wind_to_features(eastwind, northwind)
timeseries_features[level, day * 8 +
time] = [features[0], features[1]]
circular_mean = circmean(timeseries_features[:, day * 8 + time,
1])
for level in range(len(timeseries_features)):
timeseries_features[level, day * 8 + time,
1] = compute_angular_distance(
circular_mean,
timeseries_features[level,
day * 8 + time,
1])
dict_timeseries_heights[index_lat,
index_lon][:, day * 8:day * 8 +
8] = timeseries_heights[:, day *
8:day *
8 + 8]
dict_timeseries_wind[index_lat,
index_lon][:, day * 8:day * 8 +
8] = timeseries_wind[:, day *
8:day * 8 + 8]
dict_timeseries_features[
index_lat,
index_lon][:, day * 8:day * 8 +
8] = timeseries_features[:, day * 8:day * 8 + 8]
day = day + 1
# Save to npy:
start = dayStrings[0]
stop = dayStrings[-1]
for (index_lat, index_lon) in latlonPairs:
prefix = str(format(index_lat, '.2f')) + '_' + str(
format(index_lon, '.2f')) + '_' + start + '_' + stop
np.save(directory + prefix + '_winds',
dict_timeseries_wind[index_lat, index_lon][4:, :, :])
np.save(directory + prefix + '_heights',
dict_timeseries_heights[index_lat, index_lon][4:, :])
np.save(directory + prefix + '_features',
dict_timeseries_features[index_lat, index_lon][4:, :, :])
return prefix
def circ_corrcc(x, y, tail='two-sided'):
"""Correlation coefficient between two circular variables.
Parameters
----------
x : np.array
First circular variable (expressed in radians)
y : np.array
Second circular variable (expressed in radians)
tail : string
Specify whether to return 'one-sided' or 'two-sided' p-value.
Returns
-------
r : float
Correlation coefficient
pval : float
Uncorrected p-value
Notes
-----
Adapted from the CircStats MATLAB toolbox (Berens 2009).
Use the np.deg2rad function to convert angles from degrees to radians.
Please note that NaN are automatically removed.
Examples
--------
Compute the r and p-value of two circular variables
>>> from pingouin import circ_corrcc
>>> x = [0.785, 1.570, 3.141, 3.839, 5.934]
>>> y = [0.593, 1.291, 2.879, 3.892, 6.108]
>>> r, pval = circ_corrcc(x, y)
>>> print(r, pval)
0.942 0.06579836070349088
"""
from scipy.stats import norm
x = np.asarray(x)
y = np.asarray(y)
# Check size
if x.size != y.size:
raise ValueError('x and y must have the same length.')
# Remove NA
x, y = remove_na(x, y, paired=True)
n = x.size
# Compute correlation coefficient
x_sin = np.sin(x - circmean(x))
y_sin = np.sin(y - circmean(y))
# Similar to np.corrcoef(x_sin, y_sin)[0][1]
r = np.sum(x_sin * y_sin) / np.sqrt(np.sum(x_sin**2) * np.sum(y_sin**2))
# Compute T- and p-values
tval = np.sqrt((n * (x_sin**2).mean() *
(y_sin**2).mean()) / np.mean(x_sin**2 * y_sin**2)) * r
# Approximately distributed as a standard normal
pval = 2 * norm.sf(abs(tval))
pval = pval / 2 if tail == 'one-sided' else pval
return np.round(r, 3), pval
import glob
import numpy as np
from scipy.stats import circmean, circvar
fns = sorted(glob.glob("./*.txt"))
fp1 = open('bonds.dat', 'w')
fp2 = open('bend.dat', 'w')
fp3 = open('tor.dat', 'w')
for f in fns:
types = f.split('/')[1].split('.')[0]
d = np.loadtxt(f)
if (len(types.split('-')) == 4):
fp3.write("%s %f %f\n" % (types, circmean(d, low=-180., high=180.),
np.sqrt(circvar(d, low=-180., high=180.))))
elif (len(types.split('-')) == 3):
fp2.write("%s %f %f\n" % (types, np.mean(d), np.std(d)))
else:
fp1.write("%s %f %f\n" % (types, np.mean(d) * 0.1, np.std(d) * 0.1))
def pinhole(pixel_i, pixel_j, pixel_width=5, pixel_height=5,
source_aperture=50.8, sample_aperture=12.7,
source_distance=8500, detector_distance=4000,
beamstop=50.8,
wavelength=8, wavelength_resolution=0.12, aligned_wavelength=None,
N=5000, phi_mask=7.1,
Iq=None):
PI, PJ = np.meshgrid(pixel_i, pixel_j)
# ===== Generate a population of neutrons at the sample position =====
Rsource = source_aperture/2
Rsample = sample_aperture/2
Rbeamstop = beamstop/2
Dsource = source_distance
Ddetector = detector_distance
delta_el, delta_y, delta_p = aperture_alignment(
wavelength, aligned_wavelength, Dsource, Ddetector)
s_az, s_el, s_L, s_x, s_y = neutrons_on_sample(
Rsource, Rsample, Rbeamstop, Dsource, Ddetector,
wavelength, wavelength_resolution, aligned_wavelength,
N)
# ==== Compute image on detector without sample ====
#
#d_az, d_el, d_L, d_x, d_y = ballistics(s_az, s_el, s_L, s_x, s_y, Ddetector)
### filter through beam stop
##idx = (d_x**2 + (d_y-delta_y)**2 < Rbeamstop**2)
##s_az, s_el, s_L, s_x, s_y = [w[idx] for w in (s_az, s_el, s_L, s_x, s_y)]
##d_az, d_el, d_L, d_x, d_y = [w[idx] for w in (d_az, d_el, d_L, d_x, d_y)]
#plot(d_x/pixel_width, d_y/pixel_height, "G: neutron detector pixel"); return
# ==== Scatter off sample ====
mode = None
#mode = 'sum'
#mode = 'scatter'
if mode == 'sum' and Iq is not None:
# For each pixel, compute the scattering angle between the neutron
# on a direct path to the detector vs the pixel center, and compute
# I(q) based on that. Seems to underestimate the dQ/Q resolution
# for the pixels, so don't use this without figuring out what's wrong.
raise NotImplementedError("experimental code; see source")
# pixel centers relative to beam center
cx, cy = PI*pixel_width, PJ*pixel_height
pixel_r = sqrt(cx**2 + cy**2)
pixel_theta = arctan2(pixel_r, Ddetector)/2
#pixel_phi = arctan2(cy, cx)
pixel_nominal_q = 4*pi * sin(pixel_theta)/wavelength
# find neutron position on the detector without scattering
d_az, d_el, d_L, d_x, d_y = ballistics(s_az, s_el, s_L, s_x, s_y, Ddetector)
# find scattering angle from each neutron to each pixel
r = sqrt(((d_x-s_x)[:, None] - cx.flatten()[None, :])**2
+ ((d_y-s_y)[:, None] - (cy+delta_p).flatten()[None, :])**2)
theta = arctan2(r, Ddetector)/2
# find q value for each neutron at each pixel
q = 4*pi*sin(theta)/d_L[:, None]
# accumulate scattering patterns across all neutrons
I = Iq(q)
pixel_Iq = np.sum(I, axis=0).reshape(PI.shape)
pixel_dIq = pixel_Iq/sqrt(len(s_x))
pixel_q = np.mean(q, axis=0).reshape(PI.shape)
pixel_dq = np.std(q, axis=0, ddof=1).reshape(PI.shape)
#print("pixel_Iq", pixel_q.shape, pixel_Iq.shape)
if mode == 'scatter' and Iq is not None:
# For each neutron figure out the relative probability of the neutron
# arriving in each individual pixel, then choose one to add it to.
# The result is way off, probably because it doesn't include the
# probability that the neutron goes to none of the pixels.
raise NotImplementedError("experimental code; see source")
# pixel centers relative to beam center
cx, cy = PI*pixel_width, PJ*pixel_height
pixel_r = sqrt(cx**2 + cy**2)
pixel_theta = arctan2(pixel_r, Ddetector)/2
#pixel_phi = arctan2(cy, cx)
pixel_q = 4*pi * sin(pixel_theta)/wavelength
# find neutron position on the detector without scattering
d_az, d_el, d_L, d_x, d_y = ballistics(s_az, s_el, s_L, s_x, s_y, Ddetector)
# find scattering angle from each neutron to each pixel
# For each neutron generate the probability distribution corresponding
# to the various pixels that the neutron might land in and pick one.
counts = np.zeros(pixel_q.size, 'i')
counts_q = np.zeros(pixel_q.size, 'd')
for xk, yk, Lk in zip(d_x-s_x, d_y-s_y, d_L):
r = sqrt((xk - cx)**2 + (yk-delta_p - cy)**2)
theta = arctan2(r, Ddetector)/2
# find q value for each neutron at each pixel
q = (4*pi*sin(theta)/Lk).flatten()
# accumulate scattering patterns across all neutrons
invcdf = np.cumsum(Iq(q))
U = np.random.uniform(0, invcdf[-1])
index = np.searchsorted(invcdf, U)
counts[index] += 1
counts_q[index] += q[index]
counts_q /= counts + (counts==0)
counts.reshape(cx.shape)
counts_q.reshape(cx.shape)
stats = []
current_j = 1000001 # arbitrary unlikely number
for p_i, p_j in zip(PI.flat, PJ.flat):
if current_j != p_j:
print("pixel j=%d"%p_j)
current_j = p_j
## Generate a new set of points on the sample for each pixel
#s_az, s_el, s_L, s_x, s_y = neutrons_on_sample(
# Rsource, Rsample, Rbeamstop, Dsource, Ddetector,
# wavelength, wavelength_resolution, aligned_wavelength,
# N)
# ==== Compute scattering theta, phi for pixel ====
# find random point in pixel i, j to scatter to
xl, xu = (p_i-0.5)*pixel_width, (p_i+0.5)*pixel_width
yl, yu = delta_p+(p_j-0.5)*pixel_height, delta_p+(p_j+0.5)*pixel_height
p_x, p_y = rand(len(s_x))*(xu-xl)+xl, rand(len(s_x))*(yu-yl)+yl
#plot(px, py, "px,py pixel locations"); return
# find the scattering angle necessary to reach point P on the detector
q_az = arctan2(p_x-s_x, np.ones_like(s_x)*Ddetector)
q_el = throwing_angle(to_velocity(s_L), 0.001*Ddetector/cos(q_az),
0.001*(p_y-s_y))
#q_theta = arccos(sin(s_el)*sin(q_el) + cos(s_el)*cos(q_el)*cos(q_az-s_az))
#q_theta_2 = arctan2(sqrt((d_x-p_x)**2+(d_y-p_y)**2)), Ddetector)
#q_phi = arctan2(q_el, q_az)
# Note that q scattering calculations look at positions on the detector
# assuming neutrons travel in a straight line, and not the positions
# according to ballistics. The ballistics are taken into account by the
# choice of initial angle such that the neutron will hit the target
# position. The scattering function operates solely on incident and
# scattered angle with no hint of gravity, and so the resolution
# function which mixes the scattering theory must reflect this.
q_theta, q_phi = nominal_q(s_x, s_y, s_az, s_el, q_az, q_el, Ddetector)
q = 4*pi*sin(q_theta)/s_L
#return
# filter through beam stop, corrected for gravity alignment
#print(Rbeamstop**2, xu**2 + (yu-delta_p)**2, xl**2 + (yl-delta_p)**2)
idx = (p_x**2 + (p_y-delta_p)**2 > Rbeamstop**2)
q_theta, q_phi, q = [w[idx] for w in (q_theta, q_phi, q)]
# ==== calculate stats ====
cx, cy = p_i*pixel_width, p_j*pixel_height
theta_nominal = arctan2(sqrt(cx**2+cy**2), Ddetector)/2
phi_nominal = arctan2(cy, cx)
q_nominal = 4*pi*sin(theta_nominal)/wavelength
qperp_nominal = 0
# Approximate q_perp as arc length between nominal phi and actual phi
# at radius q.
qperp = q*(q_phi-phi_nominal)
if len(q) > 1:
theta_mean, theta_std = np.mean(q_theta), np.std(q_theta)
phi_mean, phi_std = circmean(q_phi, -pi, pi), circstd(q_phi, -pi, pi)
q_mean, q_std = np.mean(q), np.std(q)
qperp_mean, qperp_std = np.mean(qperp), np.std(qperp)
# weight each neutron by the sample scattering
I = np.sum(Iq(q))/len(q) if Iq is not None else 0
dI = I/sqrt(len(q))
else:
print("no q values for (%d, %d)"%(p_i, p_j))
theta_mean, theta_std = theta_nominal, 0.
phi_mean, phi_std = phi_nominal, 0.
q_mean, q_std = q_nominal, 0.
qperp_mean, qperp_std = qperp_nominal, 0.
I, dI = [], []
stats.append([
theta_nominal, theta_mean, theta_std,
phi_nominal, phi_mean, phi_std,
q_nominal, q_mean, q_std,
qperp_nominal, qperp_mean, qperp_std,
I, dI,
])
config = "src-ap:%.1fcm samp-ap:%.1fcm src-dist:%.1fm det-dist:%.1fm L:%.1fA" % (
source_aperture/10, sample_aperture/10,
Dsource/1000, Ddetector/1000, wavelength)
if len(stats) == 0:
pass # No samples fell in detector region
elif len(stats) == 1:
# print stats
pixel_config = "%s pixel:%d,%d (%dX%d mm^2)" %(
config, p_i, p_j, pixel_width, pixel_height)
print(pixel_config)
#print(" nominal lambda: %.4f actual lambda: %.4f +/- %.4f (Ang)"
# % (wavelength, np.mean(s_L), np.std(s_L)))
print(" nominal 1/lambda: %.4f actual 1/lambda: %.4f +/- %.4f (1/Ang)"
% (1./wavelength, np.mean(1./s_L), np.std(1./s_L)))
print(" nominal theta: %.4f actual theta: %.4f +/- %.4f (degrees)"
% (degrees(theta_nominal), degrees(theta_mean), degrees(theta_std)))
#print(" nominal phi: %.4f actual phi: %.4f +/- %.4f (degrees)"
# % (degrees(phi_nominal), degrees(phi_mean), degrees(phi_std)))
print(" nominal q: %.4f actual q: %.4f +/- %.4f (1/Ang)"
% (q_nominal, q_mean, q_std))
#plt.hist(degrees(q_az), bins=50); plt.title("G: scattered rotation"); plt.figure()
#plt.hist(degrees(q_el), bins=50); plt.title("G: scattered elevation"); plt.figure()
#plt.hist(degrees(q_theta), bins=50); plt.title("G: Q theta"); plt.figure()
#plt.hist(q, bins=50, density=True); plt.title("G: Q"); plt.figure()
# plot resolution
qual = "for pixel %d,%d"%(p_i, p_j)
#plot_angles(q_theta, q_phi); plt.figure()
plot_q(q, q_phi, "Q %s"%qual, plot_phi=True)
#plot_q(q, q_phi, "Q %s"%qual, plot_phi=False)
#plot_q(np.log10(q), q_phi, "Q %s"%qual, plot_phi=False)
#plot_qperp(q, qperp, "Q %s"%qual)
plt.suptitle(pixel_config)
elif len(pixel_i) == 1 or len(pixel_j) == 1:
stats = np.array(stats)
plt.suptitle(config)
plt.subplot(221)
plt.plot(stats[:, 6], degrees(stats[:, 2]), '.')
plt.grid(True)
plt.xlabel(r'$Q (1/A)$')
plt.ylabel(r'$\Delta\theta (\degree)$')
plt.subplot(222)
plt.plot(stats[:, 6], degrees(stats[:, 5]), '.')
plt.grid(True)
plt.xlabel(r'$Q (1/A)$')
plt.ylabel(r'$\Delta\phi (\degree)$')
if Iq is not None:
q, dq, I, dI = stats[:, 7], stats[:, 8], stats[:, 12], stats[:, 13]
plt.subplot(223)
plt.plot(q, 100*dq/q, '.')
plt.grid(True)
plt.xlabel(r'$Q (1/A)$')
plt.ylabel(r'$\Delta Q/Q (\%)$')
plt.subplot(224)
plt.errorbar(q, I, dI, fmt='.')
plt.xscale('log')
plt.yscale('log')
if mode == 'sum':
pixel_r, pixel_q, pixel_Iq, pixel_dIq, pixel_dq = (
v.flatten() for v in (pixel_r, pixel_q, pixel_Iq, pixel_dIq, pixel_dq)
)
mask = pixel_r >= Rbeamstop
#plt.loglog(pixel_q[mask], pixel_Iq[mask], '.')
plt.loglog(pixel_q, pixel_Iq, '.')
np.savetxt("res_sum.dat", np.array([pixel_q, pixel_Iq, pixel_dIq, pixel_dq]).T)
if mode == 'scatter':
qp, Ip = pixel_q.flatten(), counts.flatten()
qp = counts_q.flatten()
mask = (pixel_r.flatten() >= Rbeamstop) & (qp > 0)
qp, Ip = qp[mask], Ip[mask]
plt.loglog(qp, Ip, '.')
coeff = np.polyfit(log(qp), log(Ip), 1)
plt.loglog(qp, exp(np.polyval(coeff, log(qp))), '-')
print("fit to line", coeff)
if False: # add fit to line slope (for power law and fractal)
coeff = np.polyfit(log(q[1:-1]), log(I[1:-1]), 1)
plt.loglog(q, exp(np.polyval(coeff, log(q))), '-')
print("fit to line", coeff)
plt.grid(True)
plt.xlabel(r'$Q (1/A)$')
plt.ylabel(r'$I (1/cm)$')
np.savetxt("res_Iq.dat", np.array([q, I, dI, dq]).T)
if 1:
plt.figure()
plt.plot(stats[:, 6], stats[:, 7], '.')
plt.xlabel("Q nominal")
plt.ylabel("Q mean")
else:
plt.subplot(223)
plt.plot(stats[:, 6], stats[:, 8], '.')
plt.grid(True)
plt.xlabel(r'$Q (1/A)$')
plt.ylabel(r'$\Delta Q_\parallel (1/A)$')
plt.subplot(224)
plt.plot(stats[:, 6], stats[:, 11], '.')
plt.grid(True)
plt.xlabel(r'$Q (1/A)$')
plt.ylabel(r'$\Delta Q_\perp (1/A)$')
else:
stats = np.array(stats)
plt.suptitle(config)
plt.subplot(131)
data, title = degrees(stats[:, 2]), r"$\Delta\theta$"
data = np.ma.array(data, mask=(stats[:, 2] == 0))
data = data.reshape(len(pixel_i), len(pixel_j))
#mask = (PI**2 + PJ**2 < phi_mask**2)
#data = np.ma.array(data, mask=mask)
#data, title = stats[:, 1]-stats[:, 0], r"$\theta - \hat\theta$"
#data = np.clip(stats[:, 1]-stats[:, 0], 0, 0.02)
plt.pcolormesh(pixel_i, pixel_j, data)
plt.grid(True)
plt.axis('equal')
plt.title(title)
plt.colorbar()
plt.subplot(132)
data, title = degrees(stats[:, 5]), r"$\Delta\phi$"
#data, title = stats[:, 4]-stats[:, 3], r"$\phi - \hat\phi$"
data = np.ma.array(data, mask=(stats[:, 5] == 0))
data = data.reshape(len(pixel_i), len(pixel_j))
#mask = (PI < phi_mask) & (abs(PJ) < phi_mask)
plt.pcolormesh(pixel_i, pixel_j, data)
plt.grid(True)
plt.axis('equal')
plt.title(title)
plt.colorbar()
plt.subplot(133)
#data, title = stats[:, 8], r"$\Delta q$"
data, title = stats[:, 8]/stats[:, 6], r"$\Delta q/q$"
data = np.ma.array(data, mask=(stats[:, 8] == 0))
data = data.reshape(len(pixel_i), len(pixel_j))
#mask = (PI**2+PJ**2 < phi_mask**2)
#data = np.ma.array(data, mask=mask)
#data, title = stats[:, 7]-stats[:, 6], r"$q - \hat q$"
#data = np.clip(stats[:, 7]-stats[:, 6], 0, 0.0005)
plt.pcolormesh(pixel_i, pixel_j, data)
plt.grid(True)
plt.axis('equal')
plt.title(title)
plt.colorbar()
def _mc_error(ary, batches=5, circular=False):
"""Calculate the simulation standard error, accounting for non-independent samples.
The trace is divided into batches, and the standard deviation of the batch
means is calculated.
Parameters
----------
ary : Numpy array
An array containing MCMC samples
batches : integer
Number of batches
circular : bool
Whether to compute the error taking into account `ary` is a circular variable
(in the range [-np.pi, np.pi]) or not. Defaults to False (i.e non-circular variables).
Returns
-------
mc_error : float
Simulation standard error
"""
_numba_flag = Numba.numba_flag
if ary.ndim > 1:
dims = np.shape(ary)
trace = np.transpose([t.ravel() for t in ary])
return np.reshape([_mc_error(t, batches) for t in trace], dims[1:])
else:
if _not_valid(ary, check_shape=False):
return np.nan
if batches == 1:
if circular:
if _numba_flag:
std = _circular_standard_deviation(ary,
high=np.pi,
low=-np.pi)
else:
std = stats.circstd(ary, high=np.pi, low=-np.pi)
else:
if _numba_flag:
std = np.float(_sqrt(svar(ary), np.zeros(1)))
else:
std = np.std(ary)
return std / np.sqrt(len(ary))
batched_traces = np.resize(ary, (batches, int(len(ary) / batches)))
if circular:
means = stats.circmean(batched_traces,
high=np.pi,
low=-np.pi,
axis=1)
if _numba_flag:
std = _circular_standard_deviation(means,
high=np.pi,
low=-np.pi)
else:
std = stats.circstd(means, high=np.pi, low=-np.pi)
else:
means = np.mean(batched_traces, 1)
if _numba_flag:
std = _sqrt(svar(means), np.zeros(1))
else:
std = np.std(means)
return std / np.sqrt(batches)
def get_tilts_planarfit(self, config):
"""Compute tilt angles in all windows using planar fit method
Parameters
----------
config : OrderedDict
Dictionary with parsed configuration file.
Returns
-------
pandas.DataFrame
tilt angles for each window.
"""
wind_names = ["wind_speed_u", "wind_speed_v", "wind_speed_w"]
mot3d_names = ["acceleration_x", "acceleration_y",
"acceleration_z", "rate_x", "rate_y", "rate_z"]
# Iterate over the list of tuples (key=window time stamp, file-list)
msg_suffix = " tilt angles via planar fit for window %s"
for (k, l) in self.win_files:
logger.info("Begin" + msg_suffix, k)
ec_list = []
ec_ok_files = []
# Iterate over the file-list
for ec_file in l:
logger.info("Begin preparing %s", osp.basename(ec_file))
# Get a file name prefix to be shared by the output files
# from this period. Note iname is THE SAME AS THE INDEX IN
# prep_flags
iname = osp.basename(ec_file)
# Try to prepare file; set flags, and populate the list of
# OK files and summaries. Continue if preparation fails.
try:
ec_prep, prep_flags = prepare_period(ec_file, config)
except (NavigationError, SonicError) as err:
self.prep_flags.loc[iname, "failed_prep_flag"] = True
for flagname, flag in err.flags.items():
self.prep_flags.loc[iname, flagname] = flag
self.tilts.loc[k, "n" + flagname] += np.int(flag)
logger.info("Skip %s", ec_file)
continue
else:
for flagname, flag in prep_flags.iteritems():
self.prep_flags.loc[iname, flagname] = flag
self.tilts.loc[k, "n" + flagname] += np.int(flag)
ec_list.append(pd.DataFrame(ec_prep.mean()).transpose())
ec_ok_files.append(ec_file)
logger.info("End preparing %s", osp.basename(ec_file))
l = ec_ok_files
n_ok = len(ec_list)
self.tilts.loc[k, "nfiles_ok"] = n_ok
if n_ok < 3:
logger.info("Aborting window without enough adequate files %s",
k)
for ec_file in l:
self.tilts.loc[k, "failed_tilt_flag"] = True
continue
else:
ec_win = pd.concat(ec_list, keys=ec_ok_files)
logger.info("Combining %s periods in window %s", n_ok, k)
# Extract means from each period file for planar fitting
wnd = ec_win.loc[:, wind_names]
mot3d = ec_win.loc[:, mot3d_names[0:3]]
# Tilt angles for motion sensor
mot3d_pfit = planarfit(mot3d.values)
self.tilts.loc[k, "phi_motion"] = mot3d_pfit.phi
self.tilts.loc[k, "theta_motion"] = mot3d_pfit.theta
# Tilt angles for sonic anemometer
sonic_pfit = planarfit(wnd.values)
self.tilts.loc[k, "phi_sonic"] = sonic_pfit.phi
self.tilts.loc[k, "theta_sonic"] = sonic_pfit.theta
wind_direction_mean = circmean(ec_win["wind_direction"].values,
high=np.degrees(2 * np.pi))
self.tilts.loc[k, "wind_direction"] = wind_direction_mean
logger.info("End" + msg_suffix, k)
def Histogram(self, binsize=5.0, showfit='yes'):
# Method to create an histogram from a vector catalog
# Initialization
# binsize: Bin size for the histogram in degrees.
# showfit: Parameter to show the Gaussian fit on the final plot.
# Should be 'yes' or 'no'.
print()
print('========================================================')
print('Constructing a trusty histogram of polarization angles')
print('========================================================')
print()
print('Calculating mean and standard deviaton of the catalog')
print()
# Conversion from degrees to radians
conv_rad = math.pi / 180.0
conv_deg = conv_rad**-1.0
# Calculating the regular mean
norm_mean = np.mean(self.O)
norm_std = np.std(self.O)
print('Mean: ' + str(norm_mean))
print('Standard deviation: ' + str(norm_std))
print()
# Calculating the circular mean and circular standard deviation
# assuming boundaries of -90 to 90 degrees
circ_mean = conv_deg * stats.circmean(
conv_rad * self.O, low=-math.pi / 2.0, high=math.pi / 2.0)
circ_std = conv_deg * stats.circstd(
conv_rad * self.O, low=-math.pi / 2.0, high=math.pi / 2.0)
print('Circular mean: ' + str(circ_mean))
print('Circular standard deviation: ' + str(circ_std))
print()
# Calculating the number of bins by rounding to lowest integer
number_bins = math.floor(180.0 / (1.0 * binsize))
eff_binsize = 180.0 / number_bins
print('Number of bins: ' + str(number_bins))
print('Bin size used: ' + str(eff_binsize))
print()
# Creating the histogram arrays
histo_bins = np.zeros(number_bins)
histo_values = np.zeros(number_bins)
# Identifying the values of the bins
for i in range(0, number_bins):
histo_bins[i] = (i + 0.5) * eff_binsize - 90.0
# Counting the number of vectors per bin - BRUTE FORCE VERSION
# CONSIDER USING NUMPY HISTOGRAM FUNCTION INSTEAD
for i in range(0, number_bins):
# Limits of the bin
min_limit = histo_bins[i] - 0.5 * eff_binsize
max_limit = histo_bins[i] + 0.5 * eff_binsize
# Checking every vector if they should be in the bin
for j in range(0, self.size):
if (self.O[j] >= min_limit) and (self.O[j] < max_limit):
histo_values[i] = histo_values[i] + 1
# Find array value closest to circular mean
abs_array = np.abs(histo_bins - circ_mean)
smallest_diff = abs_array.argmin()
# Calculating shift needed to center the histogram on circular mean
shift = math.ceil(number_bins / 2.0) - smallest_diff
# Rolling the histogram
print('Using the circular mean to center the histogram')
print('Rolling histogram by ' + str(shift) + ' elements')
print()
histo_bins_rolled = np.roll(histo_bins, shift)
histo_values_rolled = np.roll(histo_values, shift)
# Fixing values of rolled bins so array stays sorted
if shift > 0:
for i in range(0, shift):
histo_bins_rolled[i] = histo_bins_rolled[i] - 180.0
if shift < 0:
for i in range(number_bins + shift, number_bins):
histo_bins_rolled[i] = histo_bins_rolled[i] + 180.0
# Attempting Gaussian fit to the data
# Defining the Gaussian function, stolen from the internet
# https://lmfit.github.io/lmfit-py/model.html
def gaussian(x, amp, cen, wid):
y = amp * np.exp(-(x - cen)**2 / (2 * wid**2))
return y
print('Fitting a Gaussian profile to the histogram')
print()
# Calling the lmfit package to model the 'gaussian' function
gaussian_model = Model(gaussian)
gaussian_fit = gaussian_model.fit(histo_values_rolled,
x=histo_bins_rolled,
amp=np.mean(histo_values_rolled),
cen=circ_mean,
wid=circ_std)
# Creating the array containing the parameters
fit_params = np.empty([3, 2])
# Amplitude
fit_params[0, 0] = gaussian_fit.params['amp'].value
fit_params[0, 1] = gaussian_fit.params['amp'].stderr
# Mean
fit_params[1, 0] = gaussian_fit.params['cen'].value
fit_params[1, 1] = gaussian_fit.params['cen'].stderr
# Standard deviation
fit_params[2, 0] = gaussian_fit.params['wid'].value
fit_params[2, 1] = gaussian_fit.params['wid'].stderr
print('Goodness of fit')
print('Number of iterations: ' + str(gaussian_fit.nfev))
print('Reduced chi-squared: ' + str(gaussian_fit.redchi))
print()
print('Fitted parameters')
print('Amplitude: ' + str(gaussian_fit.params['amp'].value))
print(' ± ' + str(gaussian_fit.params['amp'].stderr))
print()
print('Mean: ' + str(gaussian_fit.params['cen'].value))
print(' ± ' + str(gaussian_fit.params['cen'].stderr))
print()
print('Standard deviation: ' + str(gaussian_fit.params['wid'].value))
print(' ± ' + str(gaussian_fit.params['wid'].stderr))
print()
print('Plotting a dapper histogram')
print()
# Plotting the histogram
histo_plot = plt.figure(figsize=(6, 3)) # Creating the figure object
plt.bar(histo_bins_rolled,
histo_values_rolled,
width=eff_binsize,
fill=False) # Creating the bar plot
plt.xlabel('Polarization Angle (Degree)')
plt.ylabel('Number')
plt.tight_layout() # Using all available space in the plot window
# Setting range
xmin = histo_bins_rolled[0] - eff_binsize / 2.0
xmax = histo_bins_rolled[number_bins - 1] + eff_binsize / 2.0
plt.xlim(xmin, xmax)
# Modifying the axes directly
axes = plt.gca() # Calling the axes object of the figure
axes.xaxis.set_ticks_position('both') # Adding ticks to each side
axes.yaxis.set_ticks_position('both') # Adding ticks to each side
# Plotting the best fit Gaussian
# Creating x values
gaussian_bins = np.arange(xmin, xmax, 0.1)
# Calculating the y values
gaussian_values = gaussian(gaussian_bins, fit_params[0, 0],
fit_params[1, 0], fit_params[2, 0])
# Plotting the Gaussian fit
if showfit == 'yes':
plt.plot(gaussian_bins, gaussian_values, 'k')
print('Returning the histogram and the fit parameters')
print()
print('Stay classy!')
# Returning figure and fits parameters
return histo_plot, fit_params
def showModel(modeldata):
""" model vizualizations (particular timestep)"""
visualizeStep = max(modeldata.Step.unique())
f, ((ax1, ax2, ax3)) = plt.subplots(1, 3, figsize=(8, 2))
f.subplots_adjust(hspace=.3, wspace=.6, left=0.1, right=0.9)
plt.suptitle('Model average, Step ' + str(visualizeStep))
modeldata = modeldata[modeldata.Step == visualizeStep]
''' colormap '''
cmap = plt.cm.jet
cNorm = colors.Normalize(vmin=np.min(modeldata.competition),
vmax=np.max(modeldata.competition))
scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=cmap)
low_col = scalarMap.to_rgba(min(modeldata.competition))
high_col = scalarMap.to_rgba(max(modeldata.competition))
''' compute means & std '''
xVals = modeldata.competition.unique()
Gm = modeldata.groupby(['competition'])['mean_goal_value'].mean()
Gstd = modeldata.groupby(['competition'])['mean_goal_value'].std()
Pm = modeldata.groupby(['competition'])['mean_proxy_value'].mean()
Pstd = modeldata.groupby(['competition'])['mean_proxy_value'].std()
ax1.scatter(modeldata.competition,
modeldata.mean_goal_value,
alpha=0.5,
c=low_col,
marker=".",
label='_nolegend_')
ax1.scatter(modeldata.competition,
modeldata.mean_proxy_value,
alpha=0.5,
c=high_col,
marker=".",
label='_nolegend_')
ax1.plot(xVals, Pm, color=high_col, label='proxy')
ax1.fill_between(xVals, Pm - Pstd, Pm + Pstd, color=high_col, alpha=0.2)
ax1.plot(xVals, Gm, color=low_col, label='goal')
ax1.fill_between(xVals, Gm - Gstd, Gm + Gstd, color=low_col, alpha=0.2)
ax1.legend()
ax1.set_ylabel('mean value')
ax1.set_xlabel('competition')
utility_mean = modeldata.groupby(['competition'])['mean_utility'].mean()
utility_std = modeldata.groupby(['competition'])['mean_utility'].std()
ax2.scatter(modeldata.competition,
modeldata.mean_utility,
alpha=0.2,
c='g',
marker=".",
label='iterations')
ax2.plot(xVals, utility_mean, c='g')
ax2.fill_between(xVals,
utility_mean - utility_std,
utility_mean + utility_std,
alpha=0.2,
color='g')
ax2.set_ylabel('mean utility', color='g')
ax2.set_xlabel('competition')
groups_m = modeldata.groupby(['competition'])['mean_practice']
practice_mean = np.zeros(len(list(variable_parameters.values())[0]))
i = 0
for name, group in groups_m:
practice_mean[i] = stats.circmean(group, -np.pi, np.pi)
i += 1
groups_std = modeldata.groupby(['competition'])['mean_practice']
practice_std = np.zeros(len(list(variable_parameters.values())[0]))
i = 0
for name, group in groups_std:
practice_std[i] = stats.circstd(group, -np.pi, np.pi)
i += 1
practice_mean = practice_mean / np.pi * 180
practice_std = practice_std / np.pi * 180
ax2a = ax2.twinx()
ax2a.scatter(modeldata.competition,
modeldata.mean_practice / np.pi * 180,
alpha=0.2,
c='k',
marker=".",
label='iterations')
ax2a.plot(xVals, practice_mean, c='k')
ax2a.fill_between(xVals,
practice_mean - practice_std,
practice_mean + practice_std,
alpha=0.2,
color='k')
ax2a.set_ylim([0, max(modeldata.mean_practice / np.pi * 180)])
ax2a.set_ylabel('mean practice (°)')
''' vector plot '''
G_oc_m = modeldata.groupby(['competition'])['mean_goal_oc'].mean()
ax3.set_ylabel('goal (oc)')
ax3.set_xlabel('proxy')
for i in range(len(xVals)):
colorVal = scalarMap.to_rgba(xVals[i])
ax3.arrow(0, 0, Pm.values[i], G_oc_m.values[i], color=colorVal)
ax3.set_ylim([0, np.max(Pm) + 1])
ax3.set_xlim([0, np.max(Pm) + 1])
ax3.set_aspect('equal')
cbar_ax = f.add_axes([0.92, 0.15, 0.01, 0.7])
mpl.colorbar.ColorbarBase(cbar_ax,
cmap=cmap,
norm=cNorm,
orientation='vertical',
label='competition')
f.savefig('Model.pdf')
