def max_dataframe(self, dataframes: hint.dataframes) -> hint.dataframes:
"""
Generate a max dataframe for all of the data
Args:
dataframes: the dataframes we will use
Returns:
dataframes of all the max data
"""
print(' {} Getting max data'.format(datetime.datetime.now()))
maxs = {}
for table_name in self.table_names:
data_list = []
for data in dataframes.values():
data_list.append(data[table_name])
concat = pd.concat(data_list)
row_concat = concat.groupby(concat.index)
max_data = row_concat.max()
max_data['index'] = range(len(max_data.index))
maxs[table_name] = max_data
return maxs
def std_dataframe(self, dataframes: hint.dataframes) -> hint.dataframes:
"""
Generate a std dataframe for all of the data
Args:
dataframes: the dataframes we will use
Returns:
dataframes of all the std data
"""
print(' {} Getting standard deviation data'.
format(datetime.datetime.now()))
stds = {}
for table_name in self.table_names:
data_list = []
for data in dataframes.values():
data_list.append(data[table_name])
concat = pd.concat(data_list)
row_concat = concat.groupby(concat.index)
std_data = row_concat.std()
std_data['index'] = range(len(std_data.index))
stds[table_name] = std_data
return stds
def q_upper_dataframe(self, dataframes: hint.dataframes) -> hint.dataframes:
"""
Generate a q_upper dataframe for all of the data
Args:
dataframes: the dataframes we will use
Returns:
dataframes of all the q_upper data
"""
print(' {} Getting upper quantile data'.
format(datetime.datetime.now()))
q_uppers = {}
for table_name in self.table_names:
data_list = []
for data in dataframes.values():
data_list.append(data[table_name])
concat = pd.concat(data_list)
row_concat = concat.groupby(concat.index)
q_upper_data = row_concat.quantile(97.5)
q_upper_data['index'] = range(len(q_upper_data.index))
q_uppers[table_name] = q_upper_data
return q_uppers
def plot_seasonal(self, dataframes_low: hint.dataframes,
dataframes_high: hint.dataframes,
columns: list,
title: str) -> dict:
"""
Create all of the plots for a seasonal decomposition
Args:
dataframes_low: dataframes of the decomposition low
dataframes_high: dataframes of the decomposition high
columns: columns to plot
title: base title
Returns:
dictionary of plots
"""
decomp = {}
for decomp_title, dataframe_low in dataframes_low.items():
dataframe_high = dataframes_high[decomp_title]
plot_title = '{} {}'.format(title, decomp_title)
decomp[decomp_title] = self._plot_seasonal(dataframe_low,
dataframe_high,
columns,
plot_title)
return decomp
def create_prop(self, dataframes: hint.dataframes) -> None:
"""
Add the resistant prop to all the dataframes
Args:
dataframes: the dataframes to use
Effects:
adds proportion columns
"""
for dataframe in dataframes.values():
self.add_resistant_prop(dataframe)
def _percent_dataframes(self, dataframes: hint.dataframes) -> None:
"""
Add percent resist column to all dataframes in a run
Args:
dataframes: run's dataframes
Effects:
add percent columns
"""
for table_name, dataframe in dataframes.items():
print(' {} Calculating Percent for: {}'.format(
datetime.datetime.now(), table_name))
self._percent_dataframe(dataframe)
def cut_single_dataframes(self, dataframes: hint.dataframes)\
-> hint.dataframes:
"""
Cut the read dataframes to the correct start point
Args:
dataframes: the collection of dataframes we process
Returns:
the dataframes to the right start point
"""
new = {}
for table_name, dataframe in dataframes.items():
new[table_name] = self.cut_dataframe(dataframe)
return new
def _cut_dataframes(self, dataframes: hint.dataframes) -> hint.dataframes:
"""
Cut the dataframes in a given run
Args:
dataframes: all the dataframes for a given run
Returns:
the cut dataframes
"""
new = {}
for table_name, dataframe in dataframes.items():
print(' {} Cutting table: {}'.format(
datetime.datetime.now(), table_name))
new[table_name] = self._cut_dataframe(dataframe)
return new
def _integral(diffs: hint.dataframes, total: hint.dataframes) -> dict:
"""
Find the root of the column sums
Args:
diffs: difference data
total: the final data
Returns:
dict of data points
"""
values = {}
for table_name, diff in diffs.items():
actual = total[table_name].sum()
integral = diff.sum()
values[table_name] = integral.div(actual)
return values
def make_seasonal(self, dataframes: hint.dataframes) -> dict:
"""
Create seasonal decomposition for a collection of dataframes
Args:
dataframes: the dataframes
Returns:
all of the seasonal decomposes
"""
decomp = {}
for table_name, dataframe in dataframes.items():
print(' {} Creating {} table decomposition'.
format(datetime.datetime.now(), table_name))
decomp[table_name] = self._make_seasonal(dataframe)
return decomp
def cut_single_dataframes(self, dataframes: hint.dataframes)\
-> hint.dataframes:
"""
Cut the read dataframes to the correct start point
Args:
dataframes: the collection of dataframes we process
Returns:
the dataframes to the right start point
"""
new = {}
for table_name, dataframe in dataframes.items():
print(' {} Cutting table: {}'.
format(datetime.datetime.now(), table_name))
new[table_name] = self.cut_dataframe(dataframe)
return new
def find_periodogram(self, dataframes: hint.dataframes,
label: str) -> dict:
"""
Create the periodograms for the given dataframes
Args:
dataframes: the dataframes for a given combined type
label: label of data source
Returns:
periodograms
"""
periodograms = {}
for table_name, dataframe in dataframes.items():
print(' {} Creating periodograms for table: {}'.
format(datetime.datetime.now(), table_name))
periodograms[table_name] = self.make_periodogram(dataframe, label)
return periodograms
def _plot_periodograms(self, dataframes: hint.dataframes,
title: str) -> dict:
"""
Plot all of the periodograms for the data
Args:
dataframes: the periodogram data for a given title
title: the main title
Returns:
plots of all data from a given type
"""
periodograms = {}
for column_name, dataframe in dataframes.items():
print(' {} Creating plot for {}'.format(
datetime.datetime.now(), column_name))
periodograms[column_name] = self._plot_periodogram(
dataframe, title)
return periodograms
def plot_seasonal(self, dataframes: hint.dataframes,
columns: list,
title: str) -> dict:
"""
Create all of the plots for a seasonal decomposition
Args:
dataframes: dataframes of the decomposition
columns: columns to plot
title: base title
Returns:
dictionary of plots
"""
decomp = {}
for decomp_title, dataframe in dataframes.items():
plot_title = '{} {}'.format(title, decomp_title)
decomp[decomp_title] = self._plot_seasonal(dataframe,
columns,
plot_title)
return decomp
def _diffs(means_0: hint.dataframes, means_1: hint.dataframes) -> dict:
"""
Find the squared difference
Args:
means_0: data set 0
means_1: data set 1
Returns:
dictionary of square diff
"""
diffs = {}
for table_name, mean_0 in means_0.items():
mean_1 = means_1[table_name]
diff = mean_0 - mean_1
diff = diff.pow(2)
diff = diff.pow(0.5)
diffs[table_name] = diff
return diffs
def _plot_periodograms(self, dataframes_low: hint.dataframes,
dataframes_high: hint.dataframes,
title: str) -> dict:
"""
Plot all of the periodograms for the data
Args:
dataframes_low: the periodogram data for a given title low data
dataframes_high: the periodogram data for a given title high data
title: the main title
Returns:
plots of all data from a given type
"""
periodograms = {}
for column_name, dataframe_low in dataframes_low.items():
print(' {} Creating plot for {}'.format(
datetime.datetime.now(), column_name))
dataframe_high = dataframes_high[column_name]
periodograms[column_name] = self._plot_periodogram(
dataframe_low, dataframe_high, title)
return periodograms