def export_participants(point_model, path):
path += 'content/data_collection_study/import/'
utils.ensure_dir_exists(path)
participants = point_model.participants
stats = participants.describe(percentiles=[])\
.rename(columns=utils.participant_fields_to_readable())\
.drop(columns=['age'])\
.T\
.drop(columns=['count', '50%'])\
.rename(columns={
'mean': '\\thead{mean}',
'std': '\\thead{std}',
'min': '\\thead{min}',
'max': '\\thead{max}'
})\
.round(2)
path += 'table_participants.tex'
with open(path, 'w') as f:
table = stats.to_latex(escape=False)
latex = pack_table(
table,
'Overview of the participants [cm]',
'tab:data_collection:participants'
)
f.write(latex)
print('exported participant latex table.')
def export(point_model, path="./export/"):
utils.ensure_dir_exists(path)
# export_pca_components(point_model, path)
export_participants(point_model, path)
export_calibrations(point_model, path)
export_calibrations(point_model, path, True)
export_SelectKBest_chi2(point_model, path)
def sns_show_save(fig, **kwargs):
save = kwargs.get('save', False)
save_path = kwargs.get('save_path', 'figures/figure.png')
if save:
filename = save_path
utils.ensure_dir_exists(filename, is_file=True)
fig.savefig(filename)
else:
plt.show()
plt.clf()
def plt_show_save(fig=None, **kwargs):
save = kwargs.get('save', False)
save_path = kwargs.get('save_path', 'figures/figure.png')
tight_layout = kwargs.get('tight_layout', True)
grid = kwargs.get('grid', True)
plt.grid(grid)
if tight_layout:
plt.tight_layout()
if save:
filename = save_path
utils.ensure_dir_exists(filename, is_file=True)
plt.savefig(filename, dpi="figure")
else:
plt.show()
pass
if fig is not None:
plt.close(fig)
else:
plt.clf()
def train(self, X, y, **kwargs):
mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
model = self.model(X)
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
log_dir = ".\\logs\\fit\\" +\
datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
utils.ensure_dir_exists(log_dir, is_file=True)
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1)
model.fit(x=x_train,
y=y_train,
epochs=5,
validation_data=(x_test, y_test),
callbacks=[tensorboard_callback])
def analyze_feature(self,
participants,
featureX,
calibrations,
y,
key,
path='./feature_analysis',
**kwargs):
only_final = kwargs.get('only_final', True)
normalize = kwargs.get('normalize', True)
utils.ensure_dir_exists(path)
featureX = self.attach_feature(participants, featureX, calibrations, y,
key)
keys = [
'kde_%s' % key,
'boxplot_%s' % key,
'count_hist_plot_%s' % key, key
]
for k in keys:
plot_config = self.plotConfig.plotConfig.get(k, None)
if plot_config is None:
continue
plot_config = dict(
utils.merge(
{
'plt_type': 'feature_boxplot',
'feature_key': key,
'X': featureX,
'participants': participants,
'calibrations': calibrations,
'y': y,
}, plot_config))
self.plot(save=True,
save_path="%s/%s.png" % (path, k),
**plot_config)
def path_gen(folder):
path = figure_base_path + folder + path_end
utils.ensure_dir_exists(path)
return path
def analyze_all_features(self, path='./feature_analysis', **kwargs):
base_pairplot = kwargs.get('base_pairplot', False)
save = kwargs.get('save', True)
utils.ensure_dir_exists(path)
participants, X, calibrations, y = self.normalized
featureX = self.load_all_features(participants, X, calibrations, y)
featureX = self.attach_target(featureX, y)
features = utils.all_features()
describe_path = "%s/describe.csv" % path
featureX[features].describe().to_csv(describe_path)
for key in features:
self.analyze_feature(participants,
featureX,
calibrations,
y,
key,
path=path,
**kwargs)
self.plot_correlation_matrix(force=True,
save=True,
additional_fields=features,
save_path="%s/correlation_matrix.png" %
path)
self.plot_correlation_matrix(
force=True,
save=True,
additional_fields=features + utils.target_fields(),
save_path="%s/correlation_matrix_with_targets.png" % path)
self.plot_correlation_matrix(
force=True,
save=True,
base_fields=features + utils.target_fields(),
save_path="%s/correlation_matrix_features.png" % path)
self.plot_correlation_matrix(
force=True,
save=True,
include=list(self.feature_functions.keys()),
base_fields=(list(self.feature_functions.keys()) +
utils.target_fields()),
save_path=("%s/correlation_matrix_all_features_with_targets.png" %
path))
self.plot_correlation_matrix(
force=True,
save=True,
additional_fields=utils.target_fields(),
save_path="%s/correlation_matrix_base.png" % path)
self.plot_pca_coefficients(force=True,
save=True,
base_fields=utils.all_body_fields(),
save_path="%s/pca_coefficients_base.png" %
path)
self.plot_pca_coefficients(
force=True,
save=True,
base_fields=features,
save_path="%s/pca_coefficients_features.png" % path)
save_path = "%s/pca_base.png" % path
utils.ensure_dir_exists(save_path, is_file=True)
self.plot_pca(load_features=False,
force=True,
save=save,
save_path=save_path)
save_path = "%s/pca_all.png" % path
utils.ensure_dir_exists(save_path, is_file=True)
self.plot_pca(include_features=features,
force=True,
save=save,
save_path=save_path)
if base_pairplot:
print('plotting pairplots:')
print('\t start features')
self.plot_pairplot(force=True,
base_fields=features + utils.target_fields(),
save=save,
save_path="%s/pairplot_features.png" % path)
print('\t start base')
self.plot_pairplot(force=True,
save=save,
base_fields=utils.all_body_fields() +
utils.target_fields(),
save_path="%s/pairplot_base.png" % path)
print('\t start orientations')
self.plot_pairplot(
force=True,
save=save,
base_fields=(utils.all_body_orientation_fields() +
utils.target_fields()),
save_path="%s/pairplot_orientations.png" % path)
def export_calibrations(point_model, path, big=False):
section = 'appendix'
take = utils.all_body_fields()
note = ''
drop = ['count', '50%', 'min', 'max']
if not big:
section = 'pointing_movement'
take = utils.flatten([
utils.body_field('rightShoulder'),
utils.body_field('leftShoulder'),
utils.body_field('hmd')
])
note = '. See \\cref{} for all datapoints.'
# drop = ['count', '50%']
path += 'content/%s/import/' % section
utils.ensure_dir_exists(path)
participants = point_model.participants
calibrations = point_model.calibrations
def get_stats(take, only_total=False):
aggs = ['mean', 'std']
total = calibrations[take]\
.rename(columns=utils.body_fields_to_readable())\
.agg(aggs)
idx = pd.MultiIndex.from_product([take, aggs])
total = pd.DataFrame(np.ravel(total.values, order='F'), idx, [''])\
.T\
.rename(columns=utils.body_fields_to_readable())\
.rename(mapper=lambda s: '\\textbf{%s}' % s, axis=1)
if not only_total:
stats = calibrations[['pid'] + take]\
.rename(columns=utils.body_fields_to_readable())\
.groupby(['pid'])\
.describe(percentiles=[])\
.drop(drop, axis=1, level=1)\
.rename(mapper=lambda s: '\\textbf{%s}' % s, axis=1)
stats = stats.append(total, sort=False)
else:
stats = total.T
return stats * 100
def write_stats(path, stats):
with open(path, 'w') as f:
file_name = os.path.basename(f.name)
column_format = 'l' * (len(stats.columns)+1)
table = stats.to_latex(
escape=False, column_format=column_format,
float_format="{:0.2f}".format
)
latex = pack_table(
table,
'Overview of the calibration [cm]%s' % note,
'tab:%s:%s' % (section, file_name.replace('.tex', '')),
)
f.write(latex)
print('exported %s calibration latex table.' % section)
if not big:
stats = get_stats(take)
spath = path + 'table_calibration.tex'
write_stats(spath, stats)
stats = get_stats(take, True)
spath = path + 'table_calibration_total.tex'
write_stats(spath, stats)
else:
# num_chunks = 3
# chunk_len = round(len(take)/num_chunks)
chunk_len = 6
for i, cs in enumerate(utils.chunks(take, chunk_len)):
stats = get_stats(cs)
log_path = path + 'table_calibration_%s.tex' % i
write_stats(log_path, stats)