def plot_sizes_by_track(tracks_df, pes=True):
prefix = 'pes' if pes else 'manus'
limb_ids = ['lp', 'rp'] if pes else ['lm', 'rm']
length_measurements = cd.shared.to_measurements(tracks_df, limb_ids, 'l')
length_dist = mstats.create_distribution(
[x for x in length_measurements if x.value > 0])
pop = mstats.distributions.population(length_dist)
length_median = mstats.ValueUncertainty(
mstats.distributions.percentile(pop),
mstats.distributions.weighted_median_average_deviation(pop))
width_measurements = cd.shared.to_measurements(tracks_df, limb_ids, 'w')
width_dist = mstats.create_distribution(
[x for x in width_measurements if x.value > 0])
pop = mstats.distributions.population(width_dist)
width_median = mstats.ValueUncertainty(
mstats.distributions.percentile(pop),
mstats.distributions.weighted_median_average_deviation(pop))
label = prefix.capitalize()
cd.display.plotly(
[
cd.shared.create_density_trace(length_dist, 0, 'Length'),
cd.shared.create_density_trace(width_dist, 1, 'Width'),
],
plotting.create_layout(
{},
title='{} Width & Length Distributions (by track)'.format(label),
x_label='Size (m)',
y_label='Probability Density'))
print('Median {} Width: {}m'.format(label, width_median.label))
print('Median {} Length: {}m'.format(label, length_median.label))
def calculate_persistence(df_row, trial):
"""
Calculates the persistence residual values for each trial
:param df_row:
:param trial:
:return:
"""
median = mstats.ValueUncertainty(**trial['couplings']['value'])
couplings = mstats.values.from_serialized(
[cl['value'] for cl in trial['couplings']['lengths']])
# Unnormalized
residuals = [abs(cl - median.value) for cl in couplings]
prss = mstats.ValueUncertainty(0, 0.0000001)
for index, residual in enumerate(residuals[:-1]):
prss += residual * residuals[index + 1]
prss.freeze()
unnormalized[trial['id']] = prss
# Normalized
residuals = [abs(cl / median.value - 1) for cl in couplings]
prss = mstats.ValueUncertainty(0, 0.0000001)
for index, residual in enumerate(residuals[:-1]):
prss += residual * residuals[index + 1]
prss /= len(residuals) - 1
prss.freeze()
persistence[trial['id']] = prss
def calculate_median_value(values):
if not values or not len(values):
return dict(values=[],
dist=None,
pop=None,
median=mstats.ValueUncertainty(0, 0.0001))
dist = mstats.create_distribution(values)
pop = mdist.population(dist)
median = mstats.ValueUncertainty(
mdist.percentile(pop), mdist.weighted_median_average_deviation(pop))
return dict(values=values, dist=dist, pop=pop, median=median)
def get_unweighted_median(quantities) -> mstats.ValueUncertainty:
values = [q.value for q in quantities]
median_value = np.median(values)
median_absolute_deviation = np.median(
[abs(value - median_value) for value in values])
return mstats.ValueUncertainty(median_value,
max(0.00001, median_absolute_deviation))
def get_median(
quantities: typing.List[mstats.ValueUncertainty]
) -> mstats.ValueUncertainty:
dist = mstats.create_distribution(quantities)
pop = mdist.population(dist)
return mstats.ValueUncertainty(
mdist.percentile(pop, 0.5),
max(0.00001, mdist.weighted_median_average_deviation(pop))
)
def calculate_gauge(track: dict,
tracks: typing.List[dict]) -> mstats.ValueUncertainty:
"""
:param track:
:param tracks:
:return:
"""
index = tracks.index(track)
last_index = len(tracks) - 1
before = tracks[index - 1] if index > 0 else tracks[index + 1]
after = tracks[index + 1] if index < last_index else tracks[index - 1]
y = mstats.ValueUncertainty(track['y'], track['dy'])
y_before = mstats.ValueUncertainty(before['y'], before['dy'])
y_after = mstats.ValueUncertainty(after['y'], after['dy'])
gauge = (abs(y) + abs(y_before + y_after) / 2
) # type: mstats.ValueUncertainty
return gauge
def test_perfect_overlap(self):
""" """
values = [1, 1]
uncertainties = [0.5, 0.5]
measurements = mstats.values.join(values, uncertainties)
dist = mstats.create_distribution(measurements)
comp = mstats.create_distribution(
[mstats.ValueUncertainty(values[0], uncertainties[0])])
result = mstats.distributions.overlap2(dist, comp)
self.assertEqual(result.value, 1.0)
def plot_sizes(trackway_df, pes=True):
prefix = 'pes' if pes else 'manus'
length_measurements = mstats.values.join(
trackway_df['{}_l'.format(prefix)].tolist(),
trackway_df['{}_dl'.format(prefix)].tolist())
length_dist = mstats.create_distribution(
[x for x in length_measurements if x.value > 0])
pop = mstats.distributions.population(length_dist)
length_median = mstats.ValueUncertainty(
mstats.distributions.percentile(pop),
mstats.distributions.weighted_median_average_deviation(pop))
width_measurements = mstats.values.join(
trackway_df['{}_w'.format(prefix)].tolist(),
trackway_df['{}_dw'.format(prefix)].tolist())
width_dist = mstats.create_distribution(
[x for x in width_measurements if x.value > 0])
pop = mstats.distributions.population(width_dist)
width_median = mstats.ValueUncertainty(
mstats.distributions.percentile(pop),
mstats.distributions.weighted_median_average_deviation(pop))
label = prefix.capitalize()
cd.display.plotly(
[
cd.shared.create_density_trace(length_dist, 0, 'Length'),
cd.shared.create_density_trace(width_dist, 1, 'Width'),
],
plotting.create_layout(
{},
title='{} Width & Length Distributions (by trackway)'.format(
label),
x_label='Size (m)',
y_label='Probability Density'))
print('Median {} Width: {}m'.format(label, width_median.label))
print('Median {} Length: {}m'.format(label, length_median.label))
def add_to_group(
segment: Segment,
groups: SegmentGroupList,
trackway_data: dict
) -> SegmentGroupList:
"""
:param segment:
:param groups:
:param trackway_data:
:return:
"""
deviations = [deviation_between(g, segment) for g in groups]
if not deviations or min(deviations) > 2:
return groups + [SegmentGroup(
index=len(groups),
segments=[segment],
median=segment.median,
normalized_median=segment.median / trackway_data['width_median']
)]
group_index = deviations.index(min(deviations))
group = groups[group_index]
cauldron.step.breathe()
dist = mstats.create_distribution([s.median for s in group.segments])
pop = mdist.population(dist)
median = mstats.ValueUncertainty(
mdist.percentile(pop, 0.5),
max(0.00001, mdist.weighted_median_average_deviation(pop))
)
normalized = median / trackway_data['width_median']
replacement_group = SegmentGroup(
index=group.index,
median=median,
segments=group.segments + [segment],
normalized_median=normalized
)
return (
groups[:group_index] +
[replacement_group] +
groups[(group_index + 1):]
)
def plot_couplings(df_row, trial):
"""
:param df_row:
:param trial:
:return:
"""
data = trial['couplings']['lengths']
median = mstats.ValueUncertainty(**trial['couplings']['value'])
couplings = mstats.values.from_serialized([v['value'] for v in data])
min_value = mstats.values.minimum(couplings)
max_value = mstats.values.maximum(couplings)
y, unc = mstats.values.unzip(couplings)
plot = plotting.make_line_data(
x=[v['time'] for v in data],
y=y,
y_unc=unc,
color=plotting.get_color(int(trial['id'][1]), 0.8),
fill_color=plotting.get_color(int(trial['id'][1]), 0.3))
cd.display.plotly(data=plot['data'],
layout=plotting.create_layout(
dict(showlegend=False),
title='{} Coupling Length'.format(trial['short_id']),
x_label='Cycle (#)',
y_label='Coupling Length (m)',
y_bounds=[median.value - 0.5, median.value + 0.5]))
delta = abs(min_value - max_value)
deviation = abs(min_value.value - max_value.value) / delta.uncertainty
cd.display.markdown("""
Reference Statistics:
* _Minimum:_ __{{ min }} m__
* _Median:_ __{{ median }} m__
* _Max:_ __{{ max }} m__
* _Deviations:_ __{{ deviation }}__
""",
min=min_value.html_label,
median=median.html_label,
max=max_value.html_label,
deviation=0.01 * round(100 * deviation))
def calculate_median(values, uncertainties, color_index=0, plot_name=None):
"""
:param values:
:param uncertainties:
:param color_index:
:param plot_name:
:return:
"""
try:
values = values.values
except:
pass
try:
uncertainties = uncertainties.values
except:
pass
measurements = [(val, unc) for val, unc in zip(values, uncertainties)
if not np.isnan(val) and not np.isnan(unc)]
values, uncertainties = zip(*measurements)
measurements = mstats.values.join(values, uncertainties)
dist = mstats.create_distribution(measurements)
pop = mdist.population(dist)
median = mstats.ValueUncertainty(
mdist.percentile(pop), mdist.weighted_median_average_deviation(pop))
x = mdist.adaptive_range(dist, 4)
y = dist.probabilities_at(x)
trace = go.Scatter(x=x,
y=y,
name=plot_name,
fill='tozeroy',
mode='lines',
line=dict(color=plotting.get_color(color_index)))
return dict(measurements=measurements, median=median, trace=trace)
def calculate_swing(df_row, trial):
"""
:param df_row:
:param trial:
:return:
"""
median = mstats.ValueUncertainty(**trial['couplings']['value'])
weighted, unweighted = average_coupling_lengths(df_row, trial)
max_cl = weighted[0]
min_cl = weighted[0]
for cl in weighted[1:]:
max_cl = mstats.value.maximum(max_cl, cl)
min_cl = mstats.value.minimum(min_cl, cl)
s = abs(max_cl - min_cl) / median.value
s.freeze()
swing[trial['id']] = s
def plot_pes_aspects(trackway_df):
measurements = mstats.values.join(trackway_df['pes_aspect'].tolist(),
trackway_df['pes_daspect'].tolist())
dist = mstats.create_distribution([x for x in measurements if x.value > 0])
pop = mstats.distributions.population(dist)
median = mstats.ValueUncertainty(
mstats.distributions.percentile(pop),
mstats.distributions.weighted_median_average_deviation(pop))
cd.display.plotly(
cd.shared.create_density_trace(dist),
plotting.create_layout(
{},
title='Pes Aspect Ratio Distribution (by trackway)',
x_label='Aspect Ratio (width / length)',
y_label='Probability Density'))
print('Median Pes Aspect Ratio:', median.label)
def plot_pes_aspects_by_tracks(tracks_df):
widths = cd.shared.to_measurements(tracks_df, ['lp', 'rp'], 'w')
lengths = cd.shared.to_measurements(tracks_df, ['lp', 'rp'], 'l')
measurements = [l / w for l, w in zip(lengths, widths)]
dist = mstats.create_distribution([x for x in measurements if x.value > 0])
pop = mstats.distributions.population(dist)
median = mstats.ValueUncertainty(
mstats.distributions.percentile(pop),
mstats.distributions.weighted_median_average_deviation(pop))
cd.display.plotly(
cd.shared.create_density_trace(dist),
plotting.create_layout(
{},
title='Pes Aspect Ratio Distribution (by track)',
x_label='Aspect Ratio (width / length)',
y_label='Probability Density'))
print('Median Pes Aspect Ratio:', median.label)
def calculate_distance(
pre: dict,
post: dict,
x: mstats.ValueUncertainty,
y: mstats.ValueUncertainty
) -> mstats.ValueUncertainty:
delta_y = post['y'] - pre['y']
delta_x = post['x'] - pre['x']
numerator_1 = delta_y * x
numerator_2 = delta_x * y
numerator_3 = post['x'] * pre['y']
numerator_4 = post['y'] * pre['x']
numerator = abs(numerator_1 - numerator_2 + numerator_3 - numerator_4)
denominator = math.sqrt(delta_x ** 2 + delta_y ** 2)
if denominator == 0:
return mstats.ValueUncertainty(0, 1)
result = numerator / denominator # type: mstats.ValueUncertainty
return result
def average_coupling_lengths(df_row, trial):
"""
Calculate the average coupling lengths at each data sample in the trial
using globally specified WINDOW_SIZE.
:param df_row:
:param trial:
:return:
A list of averaged coupling lengths for the specified trial
"""
couplings = mstats.values.from_serialized(
[cl['value'] for cl in trial['couplings']['lengths']]
)
weighted = []
unweighted = []
for index in range(len(couplings)):
segment = couplings[index:index + WINDOW_SIZE]
if len(segment) < WINDOW_SIZE:
break
wxs = 0.0
ws = 0.0
for v in segment:
w = 1.0 / (v.raw_uncertainty ** 2)
wxs += w * v.raw
ws += w
weighted.append(mstats.ValueUncertainty(
wxs / ws,
1.0 / math.sqrt(ws)
))
# weighted.append(mstats.mean.weighted(segment))
unweighted.append(sum(segment) / len(segment))
return weighted, unweighted
def calculate_deviations(df_row, trial):
"""
:param df_row:
:param trial:
:return:
"""
median = mstats.ValueUncertainty(**trial['couplings']['value'])
couplings = mstats.values.from_serialized(
[cl['value'] for cl in trial['couplings']['lengths']])
deviations[trial['id']] = [
abs(cl.value - median.value) / cl.uncertainty for cl in couplings
]
relative_uncertainty = df[df.id == trial['id']].relative_uncertainty
scale = float(relative_uncertainty / relative_uncertainty_min)
scaled_deviations[trial['id']] = [
scale * abs(cl.value - median.value) / cl.uncertainty
for cl in couplings
]
pes_tracks = tracks[tracks['pes'] == 1]
layout = go.Layout(
title='{} Pes Gauge Values Along Trackway'.format(trackway_name),
xaxis={'title': 'Trackway Position (m)'},
yaxis={'title': 'Gauge (m)'},
)
scatter_trace = plotting.create_scatter(pes_tracks['curvePosition'],
pes_tracks['simpleGauge'],
name='Gauges')
median_value = np.median(pes_tracks['simpleGauge'].values)
median_absolute_deviation = np.median(
[abs(gauge - median_value) for gauge in pes_tracks['simpleGauge']])
median = mstats.ValueUncertainty(median_value, median_absolute_deviation)
median_traces = plotting.make_ranged_quantity_traces(
x_start=pes_tracks['curvePosition'].min(),
x_end=pes_tracks['curvePosition'].max(),
quantity=median,
name='Median',
legend_group='median')
cd.display.plotly(data=[scatter_trace] + median_traces, layout=layout)
cd.display.markdown("""
We make no assumption that gauge in a trackway is either
single-valued or have a central tendency. Therefore, we use the median
and the median absolute deviation as the descriptive statistic instead of
mean and standard deviation. The results, which are plotted above in red
segment_trace = go.Scatter(
x=x_segment_values,
y=y_segment_values,
mode='lines',
name='Region',
line=go.Line(
color=plotting.get_color(1, 1.0),
width=4
)
)
population = mstats.distributions.population(distribution, 4096)
population = [x for x in population if SEGMENT_MIN < x < SEGMENT_MAX]
coupling_length = mstats.ValueUncertainty(
np.median(population),
mstats.distributions.weighted_median_average_deviation(population)
)
gaussian_fit = mstats.create_distribution([coupling_length])
y_gauss_values = gaussian_fit.probabilities_at(x_values)
y_gauss_values = [SCALING * y for y in y_gauss_values]
gaussian_trace = go.Scatter(
x=x_values,
y=y_gauss_values,
mode='lines',
name='Region Fit',
line=go.Line(
color='rgba(0, 0, 0, 0.75)',
dash='dash',
tracks = cd.shared.tracks # type: pd.DataFrame
trackway_name = cd.shared.TRACKWAY_NAME
unweighted = cd.shared.unweighted
pes_tracks = tracks[tracks['pes'] == 1]
layout = go.Layout(
title='{} Pes Gauge Values Along Trackway'.format(trackway_name),
xaxis={'title': 'Trackway Position (m)'},
yaxis={'title': 'Gauge (m)'})
dist = mstats.create_distribution(pes_tracks['simpleGauge'].tolist(),
pes_tracks['simpleGaugeUnc'].tolist())
pop = mstats.distributions.population(dist)
median_value = mstats.distributions.percentile(pop)
mad = mstats.distributions.weighted_median_average_deviation(pop)
median = mstats.ValueUncertainty(median_value, mad)
scatter_trace = plotting.create_scatter(pes_tracks['curvePosition'],
pes_tracks['simpleGauge'],
pes_tracks['simpleGaugeUnc'],
name='Gauges')
median_traces = plotting.make_ranged_quantity_traces(
x_start=pes_tracks['curvePosition'].min(),
x_end=pes_tracks['curvePosition'].max(),
quantity=median,
name='Median',
legend_group='median')
cd.display.plotly(data=[scatter_trace] + median_traces, layout=layout)
numerator_1 = delta_y * x
numerator_2 = delta_x * y
numerator_3 = post['x'] * pre['y']
numerator_4 = post['y'] * pre['x']
numerator = abs(numerator_1 - numerator_2 + numerator_3 - numerator_4)
denominator = math.sqrt(delta_x ** 2 + delta_y ** 2)
if denominator == 0:
return mstats.ValueUncertainty(0, 1)
result = numerator / denominator # type: mstats.ValueUncertainty
return result
pre_track = dict(
x=mstats.ValueUncertainty(1, 0.01),
y=mstats.ValueUncertainty(1, 0.01)
)
post_track = dict(
x=mstats.ValueUncertainty(2, 0.05),
y=mstats.ValueUncertainty(2, 0.05)
)
x_quantities = mstats.values.join(
list(np.linspace(
pre_track['x'].value - 0.25,
post_track['x'].value + 0.25,
20
)),
0.01
normalized = [100.0 * (v / median - 1.0) for v in d.measurements]
normalized_values += normalized
x_values = list(np.arange(1, len(normalized), 1))
y_values, y_uncertainties = mstats.values.unzip(normalized)
traces.append(
go.Bar(x=x_values,
y=y_values,
error_y=dict(type='data', array=y_uncertainties, visible=True),
name=key.split('-')[0],
marker=dict(color=plotting.get_color(len(traces), 0.5))))
d = mstats.create_distribution(normalized_values)
mad = mstats.distributions.weighted_median_average_deviation(d, count=2048)
result = mstats.ValueUncertainty(0, mad)
project.display.text("""
Finally, the overall width and height measurements were analyzed in the
same fashion as the above results. The result is that they exhibit the same
low and reasonable ±3% variance as the digit length measurements.
""")
project.display.plotly(data=traces,
layout=plotting.create_layout(
title='Size Measurements Deviations ({}%)'.format(
result.html_label.split(' ', 1)[-1]),
x_label='Drawing Index (#)',
y_label='Fractional Value (%)'))
project.shared.normalized_values += normalized_values
