def main(self, features: List[pd.DataFrame],
categories: List[pd.DataFrame], whiten: bool, id_filter: List[T],
subsets: List[List[T]]) -> dict:
# merge input data into single df
df = reduce(lambda a, b: a.append(b), features)
# apply id filter
df = utils.apply_id_filter(df=df, id_filter=id_filter)
if not subsets:
# empty subsets equals all samples in one subset
subsets = [df['id'].unique().tolist()]
# make matrix of data
df = df.pivot(index='feature', columns='id', values='value')
df = df.T
feature_labels = list(df)
# save ids so we can re-assign them after pca
ids = df.index.tolist()
# replace missing values with row medians
imp = Imputer(missing_values='NaN', strategy='median', axis=0)
imp.fit(df)
df = imp.transform(df)
# PCA
pca = PCA(whiten=whiten)
pca.fit(df)
reduced_df = pca.transform(df)
# get explained variance ratios of components
variance_ratios = pca.explained_variance_ratio_
# get loadings
loadings = -1 * pca.components_.T * np.sqrt(pca.explained_variance_)
loadings = pd.DataFrame(loadings)
loadings['feature'] = feature_labels
# re-assign ids
reduced_df = pd.DataFrame(reduced_df)
reduced_df['id'] = ids
# add category and subset column
reduced_df = utils.apply_subsets(df=reduced_df, subsets=subsets)
reduced_df = utils.apply_categories(df=reduced_df,
categories=categories)
return {
'data': reduced_df.to_dict(orient='list'),
'loadings': loadings.to_dict(orient='list'),
'variance_ratios': variance_ratios.tolist()
}
def main(self,
x: pd.DataFrame,
y: pd.DataFrame,
id_filter: List[str],
method: str,
subsets: List[List[str]],
categories: List[pd.DataFrame]) -> dict:
"""Compute correlation statistics for the given parameters.
:param x: DataFrame containing x axis values.
:param y: DataFrame containing y axis values.
:param id_filter: If specified use only given ids during the analysis.
:param method: pearson, spearman or kendall.
:param subsets: List of lists of subset ids.
:param categories: List of DataFrames that categorise the data points.
:return: corr. coef., p-value and other useful values.
"""
if len(x['feature'].unique().tolist()) != 1 \
or len(y['feature'].unique().tolist()) != 1:
error = "Input is invalid. Please make sure that the two " \
"variables to compare have exactly one dimension, each."
logger.error(error)
raise ValueError(error)
if method not in ['pearson', 'spearman', 'kendall']:
raise ValueError("Unknown method '{}'".format(method))
df = self.merge_x_y(x, y)
x_label = list(df['feature_x'])[0]
y_label = list(df['feature_y'])[0]
df = utils.apply_id_filter(df=df, id_filter=id_filter)
df = utils.apply_subsets(df=df, subsets=subsets)
df = utils.apply_categories(df=df, categories=categories)
global_stats = self.compute_stats(df, method)
output = global_stats
output['method'] = method
output['data'] = df.to_json(orient='records')
output['x_label'] = x_label
output['y_label'] = y_label
return output
def main(self,
bw_factor: float,
num_bins: int,
id_filter: List[str],
subsets: List[List[str]],
data: pd.DataFrame,
categories: List[pd.DataFrame]) -> dict:
"""Compute several basic statistics such as bin size and kde.
:param bw_factor: KDE resolution.
:param num_bins: Number of bins to use for histogram.
:param id_filter: If specified use only given ids during the analysis.
:param subsets: List of lists of subset ids.
:param data: Numerical values to create histogram of.
:param categories: The groups to split the values into.
"""
df = data
del data
df.dropna(inplace=True)
if df.shape[0] == 0:
error = 'The selected numerical variable must be non-empty.'
logger.exception(error)
raise ValueError(error)
df = utils.apply_id_filter(df=df, id_filter=id_filter)
df = utils.apply_subsets(df=df, subsets=subsets)
df = utils.apply_categories(df=df, categories=categories)
stats = {}
categories = df['category'].unique().tolist()
subsets = df['subset'].unique().tolist()
for category in categories:
for subset in subsets:
sub_df = df[(df['category'] == category) &
(df['subset'] == subset)]
values = sub_df['value']
if values.shape[0] < 2:
continue
hist, bin_edges = np.histogram(values, bins=num_bins)
hist = hist.tolist()
bin_edges = bin_edges.tolist()
mean = np.mean(values)
median = np.median(values)
std = np.std(values)
def bw(obj, fac):
return np.power(obj.n, -1.0 / (obj.d + 4)) * fac
kde = scipy.stats.gaussian_kde(
values, bw_method=partial(bw, fac=bw_factor))
xs = np.linspace(
start=np.min(values), stop=np.max(values), num=200)
dist = kde(xs).tolist()
if not stats.get(category):
stats[category] = {}
stats[category][subset] = {
'hist': hist,
'bin_edges': bin_edges,
'mean': mean,
'median': median,
'std': std,
'dist': dist
}
return {
'data': df.to_json(orient='records'),
'stats': stats,
'subsets': subsets,
'categories': categories,
'label': df['feature'].tolist()[0]
}
def main(self, durations: List[pd.DataFrame],
categories: List[pd.DataFrame],
event_observed: List[pd.DataFrame],
estimator: str,
id_filter: List[str],
subsets: List[List[str]]) -> dict:
# TODO: Docstring
if len(durations) != 1:
error = 'Analysis requires exactly one array that specifies the ' \
'duration length.'
logger.exception(error)
raise ValueError(error)
if len(event_observed) > 1:
error = 'Maximal one variable for "event_observed" allowed'
logger.exception(error)
raise ValueError(error)
df = durations[0]
df.dropna(inplace=True)
df = utils.apply_id_filter(df=df, id_filter=id_filter)
df = utils.apply_subsets(df=df, subsets=subsets)
df = utils.apply_categories(df=df, categories=categories)
stats = {}
categories = df['category'].unique().tolist()
subsets = df['subset'].unique().tolist()
# for every category and subset combination estimate the survival fun.
for category in categories:
for subset in subsets:
sub_df = df[(df['category'] == category) &
(df['subset'] == subset)]
T = sub_df['value']
E = None # default is nothing is censored
if len(T) <= 3:
continue
if event_observed:
# find observation boolean value for every duration
E = event_observed[0].merge(sub_df, how='right', on='id')
E = [not x for x in pd.isnull(E['value_x'])]
assert len(E) == len(T)
if estimator == 'NelsonAalen':
fitter = NelsonAalenFitter()
fitter.fit(durations=T, event_observed=E)
estimate = fitter.cumulative_hazard_[
'NA_estimate'].tolist()
ci_lower = fitter.confidence_interval_[
'NA_estimate_lower_0.95'].tolist()
ci_upper = fitter.confidence_interval_[
'NA_estimate_upper_0.95'].tolist()
elif estimator == 'KaplanMeier':
fitter = KaplanMeierFitter()
fitter.fit(durations=T, event_observed=E)
# noinspection PyUnresolvedReferences
estimate = fitter.survival_function_[
'KM_estimate'].tolist()
ci_lower = fitter.confidence_interval_[
'KM_estimate_lower_0.95'].tolist()
ci_upper = fitter.confidence_interval_[
'KM_estimate_upper_0.95'].tolist()
else:
error = 'Unknown estimator: {}'.format(estimator)
logger.exception(error)
raise ValueError(error)
timeline = fitter.timeline.tolist()
if not stats.get(category):
stats[category] = {}
stats[category][subset] = {
'timeline': timeline,
'estimate': estimate,
'ci_lower': ci_lower,
'ci_upper': ci_upper
}
return {
'label': df['feature'].tolist()[0],
'categories': categories,
'subsets': subsets,
'stats': stats
}
def main(self,
features: List[pd.DataFrame],
categories: List[pd.DataFrame],
id_filter: List[T],
transformation: str,
subsets: List[List[T]]) -> dict:
""" Compute boxplot statistics for the given parameters.
:param features: List of numerical features
:param categories: List of categorical features used to group numerical
features.
:param id_filter: List of ids that will be considered for analysis. If
empty all ids will be used.
:param transformation: Transformation that will be applied to the data.
:param subsets: List of subsets used as another way to group the
numerical features.
"""
if not len(features):
raise ValueError("Must at least specify one "
"non empty numerical feature.")
# merge dfs into single one
df = reduce(lambda l, r: l.append(r), features)
df = utils.apply_transformation(df=df, transformation=transformation)
df.dropna(inplace=True)
df = utils.apply_id_filter(df=df, id_filter=id_filter)
df = utils.apply_subsets(df=df, subsets=subsets)
df = utils.apply_categories(df=df, categories=categories)
df['outlier'] = None
results = {
'statistics': {},
'features': df['feature'].unique().tolist(),
'categories': df['category'].unique().tolist(),
'subsets': df['subset'].unique().tolist()
}
group_values = []
for feature in results['features']:
for subset in results['subsets']:
for category in results['categories']:
values = df[(df['subset'] == subset) &
(df['category'] == category) &
(df['feature'] == feature)]['value'].tolist()
if len(values) < 2:
continue
# FIXME: v This is ugly. Look at kaplan_meier_survival.py
label = '{}//{}//s{}'.format(feature, category, subset + 1)
group_values.append(values)
stats = self.boxplot_statistics(values)
u_outliers = np.array(values) > stats['u_wsk']
l_outliers = np.array(values) < stats['l_wsk']
outliers = np.bitwise_or(u_outliers, l_outliers)
df.loc[(df['subset'] == subset) &
(df['category'] == category) &
(df['feature'] == feature), 'outlier'] = outliers
kde = scipy.stats.gaussian_kde(values)
xs = np.linspace(start=stats['l_wsk'],
stop=stats['u_wsk'], num=100)
stats['kde'] = kde(xs).tolist()
results['statistics'][label] = stats
results['data'] = df.to_json(orient='records')
f_value, p_value = scipy.stats.f_oneway(*group_values)
results['anova'] = {
'p_value': p_value,
'f_value': f_value
}
return results
def main(self, numerical_arrays: List[pd.DataFrame],
numericals: List[pd.DataFrame], categoricals: List[pd.DataFrame],
ranking_method: str, params: dict, id_filter: List[T],
max_rows: int, subsets: List[List[T]]) -> dict:
# merge input data into single df
df = reduce(lambda a, b: a.append(b), numerical_arrays)
if not subsets:
# empty subsets equals all samples in one subset
subsets = [df['id'].unique().tolist()]
else:
# if subsets are defined we drop the rows that are not part of one
flattened_subsets = [x for subset in subsets for x in subset]
df = df[df['id'].isin(flattened_subsets)]
# apply id filter
df = utils.apply_id_filter(df=df, id_filter=id_filter)
# drop subset ids that are not in the df
subsets = utils.drop_unused_subset_ids(df=df, subsets=subsets)
# make sure the input data are still valid after the pre-processing
if df.shape[0] < 1:
error = "Either the input data set is too small or " \
"the subset sample ids do not match the data."
logger.error(error)
raise ValueError(error)
# make matrix of input data
df = df.pivot(index='feature', columns='id', values='value')
# create z-score matrix used for visualising the heatmap
z_df = [(df.iloc[i] - df.iloc[i].mean()) / df.iloc[i].std(ddof=0)
for i in range(df.shape[0])]
z_df = pd.DataFrame(z_df, columns=df.columns, index=df.index)
method = 'limma'
if ranking_method in ['mean', 'median', 'variance']:
method = ranking_method
# compute statistic for ranking
stats = array_stats.get_stats(df=df,
subsets=subsets,
params=params,
ranking_method=method)
# sort by ranking_value
self.sort(df, stats[ranking_method], ranking_method)
self.sort(z_df, stats[ranking_method], ranking_method)
self.sort(stats, stats[ranking_method], ranking_method)
# discard rows according to max_rows
df = df[:max_rows]
z_df = z_df[:max_rows]
stats = stats[:max_rows]
# prepare output for front-end
df['feature'] = df.index
z_df['feature'] = z_df.index
df = pd.melt(df, id_vars='feature', var_name='id')
z_df = pd.melt(z_df, id_vars='feature', var_name='id')
df = df.merge(z_df, on=['id', 'feature'])
df.rename(columns={
'value_x': 'value',
'value_y': 'zscore'
},
inplace=True)
df = utils.apply_subsets(df, subsets)
return {
'data': df.to_dict(orient='list'),
'stats': stats.to_dict(orient='list')
}
def test_apply_subsets(self):
df = pd.DataFrame([[101, 'foo', 1], [102, 'foo', 2], [103, 'foo', 3]],
columns=['id', 'feature', 'value'])
subsets = [[101, 102], [], [103, 102, 104]]
result = utils.apply_subsets(df=df, subsets=subsets)
assert result['subset'].tolist() == [0, 0, 2, 2]