def update_config_records(filename, new_cfg): """ All records of config file are updated exept records_to_save. """ if os.path.isfile(filename): cfg = misc.obj_from_file(filename, filetype='yaml') cfg.update(new_cfg) print(cfg) misc.obj_to_file(new_cfg, "test_" + filename, filetype='yaml')
def delete_config_records(filename, records_to_save=[]): """ All records of config file are deleted except records_to_save. """ if os.path.isfile(filename): cfg = misc.obj_from_file(filename, filetype='yaml') new_cfg = subdict(cfg, records_to_save) print cfg misc.obj_to_file(new_cfg, "test_" + filename , filetype='yaml')
def delete_config_records(filename, records_to_save=[]): """ All records of config file are deleted except records_to_save. """ if os.path.isfile(filename): cfg = misc.obj_from_file(filename, filetype='yaml') new_cfg = subdict(cfg, records_to_save) print(cfg) misc.obj_to_file(new_cfg, "test_" + filename, filetype='yaml')
def update_config_records(filename, new_cfg): """ All records of config file are updated exept records_to_save. """ if os.path.isfile(filename): cfg = misc.obj_from_file(filename, filetype='yaml') cfg.update(new_cfg) print cfg misc.obj_to_file(new_cfg, "test_" + filename , filetype='yaml')
def check_config_version_and_remove_old_records(filename, version, records_to_save): """ Check if config file version is ok. If it is not all records except records_to_save are deleted and config_version in file is set to version. It is used to update user configuration. """ if os.path.isfile(filename): cfg = misc.obj_from_file(filename, filetype='yaml') if ('config_version' in cfg and (cfg['config_version'] == version)): # everything is ok, no need to panic return else: # older version of config file cfg = misc.obj_from_file(filename, filetype='yaml') misc.obj_to_file(cfg, filename + '.old', filetype='yaml') print 'cfg ', cfg new_cfg = subdict(cfg, records_to_save) new_cfg['config_version'] = version print 'ncfg ', new_cfg misc.obj_to_file(new_cfg, filename, filetype='yaml')
def test_obj_to_and_from_file_yaml_with_ndarray_to_yaml(self): testdata = np.random.random([4, 4, 3]) test_object = {'a': 1, 'data': testdata, "lst": [1, 2, 3]} filename = 'test_obj_to_and_from_file.yaml' misc.obj_to_file(test_object, filename, 'yaml', ndarray_to_list=True) saved_object = misc.obj_from_file(filename, 'yaml') self.assertTrue(saved_object['a'] == 1) self.assertTrue(saved_object['data'][1][1][1] == testdata[1, 1, 1]) os.remove(filename)
def check_config_version_and_remove_old_records(filename, version, records_to_save): """ Check if config file version is ok. If it is not all records except records_to_save are deleted and config_version in file is set to version. It is used to update user configuration. """ if os.path.isfile(filename): cfg = misc.obj_from_file(filename, filetype='yaml') if ('config_version' in cfg and (cfg['config_version'] == version)): # everything is ok, no need to panic return else: # older version of config file cfg = misc.obj_from_file(filename, filetype='yaml') misc.obj_to_file(cfg, filename + '.old', filetype='yaml') print('cfg ', cfg) new_cfg = subdict(cfg, records_to_save) new_cfg['config_version'] = version print('ncfg ', new_cfg) misc.obj_to_file(new_cfg, filename, filetype='yaml')
def test_obj_to_and_from_file_pklz_with_auto(self): testdata = np.random.random([4, 4, 3]) test_object = {'a': 1, 'data': testdata} filename = 'test_obj_to_and_from_file_with_auto.pklz' misc.obj_to_file(test_object, filename, 'auto') saved_object = misc.obj_from_file(filename, 'auto') self.assertTrue(saved_object['a'] == 1) self.assertTrue(saved_object['data'][1, 1, 1] == testdata[1, 1, 1]) os.remove(filename)
def test_obj_to_and_from_file_pickle(self): testdata = np.random.random([4, 4, 3]) test_object = {'a': 1, 'data': testdata} filename = 'test_obj_to_and_from_file.pkl' misc.obj_to_file(test_object, filename, 'pickle') saved_object = misc.obj_from_file(filename, 'pickle') self.assertTrue(saved_object['a'] == 1) self.assertTrue(saved_object['data'][1, 1, 1] == testdata[1, 1, 1]) os.remove(filename)
def test_obj_to_and_from_file_yaml_with_ndarray_to_yaml(self): testdata = np.random.random([4, 4, 3]) test_object = {'a': 1, 'data': testdata, "lst":[1, 2, 3]} filename = 'test_obj_to_and_from_file.yaml' misc.obj_to_file(test_object, filename, 'yaml', ndarray_to_list=True) saved_object = misc.obj_from_file(filename, 'yaml') self.assertTrue(saved_object['a'] == 1) self.assertTrue(saved_object['data'][1][1][1] == testdata[1, 1, 1]) os.remove(filename)
def test_obj_to_and_from_file_with_directories(self): import shutil testdata = np.random.random([4, 4, 3]) test_object = {'a': 1, 'data': testdata} dirname = '__test_write_and_read' filename = '__test_write_and_read/test_obj_to_and_from_file.pkl' misc.obj_to_file(test_object, filename, 'pickle') saved_object = misc.obj_from_file(filename, 'pickle') self.assertTrue(saved_object['a'] == 1) self.assertTrue(saved_object['data'][1, 1, 1] == testdata[1, 1, 1]) shutil.rmtree(dirname)
def get_config(filename, default_cfg): """ Looks config file and update default_cfg values. If file does not exist it is created. """ if os.path.isfile(filename): cfg = misc.obj_from_file(filename, filetype='yaml') default_cfg.update(cfg) cfg_out = default_cfg else: misc.obj_to_file(default_cfg, filename, filetype='yaml') cfg_out = default_cfg # default config return cfg_out
def report(eval_files, labels, markers, show=True, output_prefix='', use_plt=True, pklz_dirs=None, experiment_name='', filename_separator=''): """ based on :eval_files: list of filenames with input data without extension. If it is None, based on pklz_disr the default eval_list is generated. :use_plt: can supress using of matplotlib :pklz_dirs: is used for generating eval_files if eval_files is None. Obsolete """ # TODO image_basename generovat obrazky expn = np.array(range(0, len(markers))) expn_labels = labels dp_params = { 'markers': markers, 'labels': labels, 'loc': 0, 'show': show, 'filename': output_prefix, 'filename_separator': filename_separator } sp_params = { 'expn': expn, 'expn_labels': expn_labels, 'show': show, 'filename': output_prefix, 'use_plt': use_plt, 'filename_separator': filename_separator } # return # yaml_files = [os.path.normpath(path) + '.yaml' for path in pklz_dirs] # logger.debug(str(yaml_files)) if eval_files is None: if pklz_dirs is not None: eval_files = [ os.path.normpath(path) + '_eval' for path in pklz_dirs] else: logger.error('pklz_dirs and eval_files is None') raise Exception('pklz_dirs and eval_files is None') logger.debug(str(eval_files)) data = [misc.obj_from_file(fname + '.pkl', filetype='pkl') for fname in eval_files] df_all = create_data_frame(data, labels, eval_files, experiment_name) __df_to_csv_and_latex(df_all, output_prefix) if use_plt: logger.debug("first plot") dataplot(data, 'voe', 'Volume Difference Error [%]', **dp_params) dataplot(data, 'vd', 'Total Volume Difference [%]', **dp_params) dataplot(data, 'processing_time', 'Processing time [s]', **dp_params) dataplot(data, 'maxd', 'MaxD [mm]', **dp_params) dataplot(data, 'avgd', 'AvgD [mm]', **dp_params) dataplot(data, 'rmsd', 'RMSD [mm]', **dp_params) # dataplot(data, 'rmsd', 'RMSD [mm]', **dp_params) logger.debug("Souhrn měření") vd_mn, tmp = sumplot(data, 'vd', 'Total Volume Difference', **sp_params) voe_mn, tmp = sumplot(data, 'voe', 'Volume Difference Error', **sp_params) avgd_mn, tmp = sumplot(data, 'avgd', 'Average Distance', **sp_params) maxd_mn, tmp = sumplot(data, 'maxd', 'Maxiamal Distance', **sp_params) rmsd_mn, tmp = sumplot(data, 'rmsd', 'Square Distance', **sp_params) logger.info("\n") logger.info('vd ' + str(vd_mn)) logger.info("voe " + str(voe_mn)) logger.info('maxd ' + str(maxd_mn)) logger.info('avgd ' + str(avgd_mn)) logger.info('rmsd ' + str(rmsd_mn)) logger.info("Přepočteno na skóre") # print tables[0].shape # pandas.set_option('display.max_columns', None) scoreTotal, scoreMetrics, scoreAll = sliverScoreAll(data) tables, indexes, columns = scoreTableEvaluation(scoreMetrics) dataframes = __create_data_frames(tables, indexes, columns) __save_data_frames(dataframes, labels, output_prefix) # df = pandas.DataFrame(tables[0], index=indexes[0], columns=columns[0]) # print df.to_string() if use_plt: dataplot(scoreAll, 'voe', 'Volume Difference Error [points]', **dp_params) dataplot(scoreAll, 'vd', 'Total Volume Difference [points]', **dp_params) dataplot(scoreAll, 'maxd', 'MaxD [points]', **dp_params) dataplot(scoreAll, 'avgd', 'AvgD [points]', **dp_params) dataplot(scoreAll, 'rmsd', 'RMSD [points]', **dp_params) # tohle je zoufale propletani ruznych dat dataplot(data, 'score_pts', 'Score [points]', **dp_params) vd_mn, tmp = sumplot(scoreAll, 'vd', 'Total Volume Difference', **sp_params) voe_mn, tmp = sumplot(scoreAll, 'voe', 'Volume Difference Error', **sp_params) avgd_mn, tmp = sumplot(scoreAll, 'avgd', 'Average Distance', **sp_params) maxd_mn, tmp = sumplot(scoreAll, 'maxd', 'Maxiamal Distance', **sp_params) rmsd_mn, tmp = sumplot(scoreAll, 'rmsd', 'Square Distance', **sp_params) logger.info("Overall score") scoreTotal, scoreMetrics, scoreAll = sliverScoreAll(data) logger.info('Overall score: ' + str(scoreTotal)) if use_plt: plot_total(scoreMetrics, labels=labels, err_scale=0.05, show=show, filename=output_prefix) return df_all
def processIt(pklz_dirs, sliver_dir, yaml_files, eval_files, markers, labels): import misc """ Funkce vypíše report o celém experimentu. """ data = [misc.obj_from_file(fname + '.pkl', filetype='pkl') for fname in eval_files] print("Jednotlivá měření") dataplot(data, 'voe', 'Volume Difference Error [%]', markers=markers, labels=labels, loc=0) dataplot(data, 'vd', 'Total Volume Difference [%]', markers=markers, labels=labels, loc=0) dataplot(data, 'processing_time', 'Processing time [s]', markers=markers, labels=labels, loc=0) dataplot(data, 'maxd', 'MaxD [mm]', markers=markers, labels=labels, loc=0) dataplot(data, 'avgd', 'AvgD [mm]', markers=markers, labels=labels, loc=0) dataplot(data, 'rmsd', 'RMSD [mm]', markers=markers, labels=labels, loc=0) print("Souhrn měření") # import "experiment_support.ipynb" expn = np.array(range(0, len(labels))) expn_labels = labels print(expn_labels) print(expn) vd_mn, tmp = sumplot( data, 'vd', 'Total Volume Difference', expn, expn_labels) voe_mn, tmp = sumplot( data, 'voe', 'Volume Difference Error', expn, expn_labels) avgd_mn, tmp = sumplot(data, 'avgd', 'Average Distance', expn, expn_labels) maxd_mn, tmp = sumplot( data, 'maxd', 'Maxiamal Distance', expn, expn_labels) rmsd_mn, tmp = sumplot(data, 'rmsd', 'Square Distance', expn, expn_labels) print('vd ', vd_mn) print("voe ", voe_mn) print('maxd ', maxd_mn) print('avgd ', avgd_mn) print('rmsd ', rmsd_mn) print("Přepočteno na skóre") import pandas # print tables[0].shape # pandas.set_option('display.max_columns', None) scoreTotal, scoreMetrics, scoreAll = sliverScoreAll(data) tables, indexes, columns = scoreTableEvaluation(scoreMetrics) df = pandas.DataFrame(tables[0], index=indexes[0], columns=columns[0]) print(df.to_string()) dataplot(scoreAll, 'voe', 'Volume Difference Error [points]', markers=markers, labels=labels, loc=0) dataplot(scoreAll, 'vd', 'Total Volume Difference [points]', markers=markers, labels=labels, loc=0) dataplot(scoreAll, 'maxd', 'MaxD [mm]', markers=markers, labels=labels, loc=0) dataplot(scoreAll, 'avgd', 'AvgD [mm]', markers=markers, labels=labels, loc=0) dataplot(scoreAll, 'rmsd', 'RMSD [mm]', markers=markers, labels=labels, loc=0) vd_mn, tmp = sumplot( scoreAll, 'vd', 'Total Volume Difference', expn, expn_labels) voe_mn, tmp = sumplot( scoreAll, 'voe', 'Volume Difference Error', expn, expn_labels) avgd_mn, tmp = sumplot( scoreAll, 'avgd', 'Average Distance', expn, expn_labels) maxd_mn, tmp = sumplot( scoreAll, 'maxd', 'Maxiamal Distance', expn, expn_labels) rmsd_mn, tmp = sumplot( scoreAll, 'rmsd', 'Square Distance', expn, expn_labels) # scoreTotal, scoreMetrics, scoreAll = # volumetry_evaluation.sliverScoreAll(data) scoreTotal, scoreMetrics, scoreAll = sliverScoreAll(data) print('Score total: ', scoreTotal) plot_total(scoreMetrics, labels=labels, err_scale=0.05) return df
def report(eval_files, labels, markers, show=True, output_prefix='', use_plt=True, pklz_dirs=None, experiment_name='', filename_separator=''): """ based on :eval_files: list of filenames with input data without extension. If it is None, based on pklz_disr the default eval_list is generated. :use_plt: can supress using of matplotlib :pklz_dirs: is used for generating eval_files if eval_files is None. Obsolete """ # TODO image_basename generovat obrazky expn = np.array(range(0, len(markers))) expn_labels = labels dp_params = { 'markers': markers, 'labels': labels, 'loc': 0, 'show': show, 'filename': output_prefix, 'filename_separator': filename_separator } sp_params = { 'expn': expn, 'expn_labels': expn_labels, 'show': show, 'filename': output_prefix, 'use_plt': use_plt, 'filename_separator': filename_separator } # return # yaml_files = [os.path.normpath(path) + '.yaml' for path in pklz_dirs] # logger.debug(str(yaml_files)) if eval_files is None: if pklz_dirs is not None: eval_files = [ os.path.normpath(path) + '_eval' for path in pklz_dirs ] else: logger.error('pklz_dirs and eval_files is None') raise Exception('pklz_dirs and eval_files is None') logger.debug(str(eval_files)) data = [ misc.obj_from_file(fname + '.pkl', filetype='pkl') for fname in eval_files ] df_all = create_data_frame(data, labels, eval_files, experiment_name) __df_to_csv_and_latex(df_all, output_prefix) if use_plt: logger.debug("first plot") dataplot(data, 'voe', 'Volume Difference Error [%]', **dp_params) dataplot(data, 'vd', 'Total Volume Difference [%]', **dp_params) dataplot(data, 'processing_time', 'Processing time [s]', **dp_params) dataplot(data, 'maxd', 'MaxD [mm]', **dp_params) dataplot(data, 'avgd', 'AvgD [mm]', **dp_params) dataplot(data, 'rmsd', 'RMSD [mm]', **dp_params) # dataplot(data, 'rmsd', 'RMSD [mm]', **dp_params) logger.debug("Souhrn měření") vd_mn, tmp = sumplot(data, 'vd', 'Total Volume Difference', **sp_params) voe_mn, tmp = sumplot(data, 'voe', 'Volume Difference Error', **sp_params) avgd_mn, tmp = sumplot(data, 'avgd', 'Average Distance', **sp_params) maxd_mn, tmp = sumplot(data, 'maxd', 'Maxiamal Distance', **sp_params) rmsd_mn, tmp = sumplot(data, 'rmsd', 'Square Distance', **sp_params) logger.info("\n") logger.info('vd ' + str(vd_mn)) logger.info("voe " + str(voe_mn)) logger.info('maxd ' + str(maxd_mn)) logger.info('avgd ' + str(avgd_mn)) logger.info('rmsd ' + str(rmsd_mn)) logger.info("Přepočteno na skóre") # print tables[0].shape # pandas.set_option('display.max_columns', None) scoreTotal, scoreMetrics, scoreAll = sliverScoreAll(data) tables, indexes, columns = scoreTableEvaluation(scoreMetrics) dataframes = __create_data_frames(tables, indexes, columns) __save_data_frames(dataframes, labels, output_prefix) # df = pandas.DataFrame(tables[0], index=indexes[0], columns=columns[0]) # print df.to_string() if use_plt: dataplot(scoreAll, 'voe', 'Volume Difference Error [points]', **dp_params) dataplot(scoreAll, 'vd', 'Total Volume Difference [points]', **dp_params) dataplot(scoreAll, 'maxd', 'MaxD [points]', **dp_params) dataplot(scoreAll, 'avgd', 'AvgD [points]', **dp_params) dataplot(scoreAll, 'rmsd', 'RMSD [points]', **dp_params) # tohle je zoufale propletani ruznych dat dataplot(data, 'score_pts', 'Score [points]', **dp_params) vd_mn, tmp = sumplot(scoreAll, 'vd', 'Total Volume Difference', **sp_params) voe_mn, tmp = sumplot(scoreAll, 'voe', 'Volume Difference Error', **sp_params) avgd_mn, tmp = sumplot(scoreAll, 'avgd', 'Average Distance', **sp_params) maxd_mn, tmp = sumplot(scoreAll, 'maxd', 'Maxiamal Distance', **sp_params) rmsd_mn, tmp = sumplot(scoreAll, 'rmsd', 'Square Distance', **sp_params) logger.info("Overall score") scoreTotal, scoreMetrics, scoreAll = sliverScoreAll(data) logger.info('Overall score: ' + str(scoreTotal)) if use_plt: plot_total(scoreMetrics, labels=labels, err_scale=0.05, show=show, filename=output_prefix) return df_all
def processIt(pklz_dirs, sliver_dir, yaml_files, eval_files, markers, labels): import misc """ Funkce vypíše report o celém experimentu. """ data = [ misc.obj_from_file(fname + '.pkl', filetype='pkl') for fname in eval_files ] print("Jednotlivá měření") dataplot(data, 'voe', 'Volume Difference Error [%]', markers=markers, labels=labels, loc=0) dataplot(data, 'vd', 'Total Volume Difference [%]', markers=markers, labels=labels, loc=0) dataplot(data, 'processing_time', 'Processing time [s]', markers=markers, labels=labels, loc=0) dataplot(data, 'maxd', 'MaxD [mm]', markers=markers, labels=labels, loc=0) dataplot(data, 'avgd', 'AvgD [mm]', markers=markers, labels=labels, loc=0) dataplot(data, 'rmsd', 'RMSD [mm]', markers=markers, labels=labels, loc=0) print("Souhrn měření") # import "experiment_support.ipynb" expn = np.array(range(0, len(labels))) expn_labels = labels print(expn_labels) print(expn) vd_mn, tmp = sumplot(data, 'vd', 'Total Volume Difference', expn, expn_labels) voe_mn, tmp = sumplot(data, 'voe', 'Volume Difference Error', expn, expn_labels) avgd_mn, tmp = sumplot(data, 'avgd', 'Average Distance', expn, expn_labels) maxd_mn, tmp = sumplot(data, 'maxd', 'Maxiamal Distance', expn, expn_labels) rmsd_mn, tmp = sumplot(data, 'rmsd', 'Square Distance', expn, expn_labels) print('vd ', vd_mn) print("voe ", voe_mn) print('maxd ', maxd_mn) print('avgd ', avgd_mn) print('rmsd ', rmsd_mn) print("Přepočteno na skóre") import pandas # print tables[0].shape # pandas.set_option('display.max_columns', None) scoreTotal, scoreMetrics, scoreAll = sliverScoreAll(data) tables, indexes, columns = scoreTableEvaluation(scoreMetrics) df = pandas.DataFrame(tables[0], index=indexes[0], columns=columns[0]) print(df.to_string()) dataplot(scoreAll, 'voe', 'Volume Difference Error [points]', markers=markers, labels=labels, loc=0) dataplot(scoreAll, 'vd', 'Total Volume Difference [points]', markers=markers, labels=labels, loc=0) dataplot(scoreAll, 'maxd', 'MaxD [mm]', markers=markers, labels=labels, loc=0) dataplot(scoreAll, 'avgd', 'AvgD [mm]', markers=markers, labels=labels, loc=0) dataplot(scoreAll, 'rmsd', 'RMSD [mm]', markers=markers, labels=labels, loc=0) vd_mn, tmp = sumplot(scoreAll, 'vd', 'Total Volume Difference', expn, expn_labels) voe_mn, tmp = sumplot(scoreAll, 'voe', 'Volume Difference Error', expn, expn_labels) avgd_mn, tmp = sumplot(scoreAll, 'avgd', 'Average Distance', expn, expn_labels) maxd_mn, tmp = sumplot(scoreAll, 'maxd', 'Maxiamal Distance', expn, expn_labels) rmsd_mn, tmp = sumplot(scoreAll, 'rmsd', 'Square Distance', expn, expn_labels) # scoreTotal, scoreMetrics, scoreAll = # volumetry_evaluation.sliverScoreAll(data) scoreTotal, scoreMetrics, scoreAll = sliverScoreAll(data) print('Score total: ', scoreTotal) plot_total(scoreMetrics, labels=labels, err_scale=0.05) return df