def test_write_to_network_and_file_and_read_from_both(self):
if os.path.exists('sample.db'):
os.unlink('sample.db')
print("Writing to server")
database_manager = ExperimentDatabaseManager(
mysql_credentials=sql_credentials.credentials,
file='sample.db',
cache_size=40)
database_manager.delete_experiment('database_plots_test_case_1')
database_manager.set_experiment('database_plots_test_case_1')
self.write_to_database(database_manager)
database_manager.close()
download_experiment_to_file(
experiment_name='database_plots_test_case_1',
file_path='output/downloaded_database.db')
print("Reading back from downloaded file")
database_reading_manager = ExperimentDatabaseReadingManager(
file='output/downloaded_database.db')
self.read_from_database(database_reading_manager)
print("Reading back from cache file")
database_reading_manager = ExperimentDatabaseReadingManager(
file='sample.db')
self.read_from_database(database_reading_manager)
database_manager = ExperimentDatabaseManager(
mysql_credentials=sql_credentials.credentials, cache_size=40)
database_manager.delete_experiment('database_plots_test_case_1')
database_manager.close()
def download_experiment_to_file(experiment_name, file_path):
if os.path.exists(file_path):
os.unlink(file_path)
file_database = ExperimentDatabaseManager(file=file_path, cache_size=100000)
file_database.set_experiment(experiment_name)
reading_manager = ExperimentDatabaseReadingManager(mysql_credentials=credentials)
query = '''SELECT (table_name) FROM information_schema.columns WHERE column_name = 'experiment_name' AND table_schema = '%s';''' % credentials['database']
data = reading_manager.get_data_from_query(query)
tables = [x[0] for x in data]
tables.remove('experiments')
print("Gotten list of all tables:")
print('\n'.join(tables))
for table in tables:
print("Downloading data from ", table)
table_data = reading_manager.get_data(table, experiment_name)
if table_data is None:
print("No data found in table for this experiment, skipping")
continue
print("Gotten keys", table_data.keys())
for key in table_data.keys():
print('\t%s is %s' % (key, str(type(table_data[key][0]))))
table_data.pop('experiment_name')
file_database.insert_experiment_data(table, table_data)
print("Finishing up writing to file...")
file_database.close()
def test_read_write(self):
print("Writing to server")
database_manager = ExperimentDatabaseManager(
mysql_credentials=sql_credentials.credentials, cache_size=40)
self.write_to_database(database_manager)
database_manager.close()
print("Reading back from server")
database_reading_manager = ExperimentDatabaseReadingManager(
sql_credentials.credentials)
self.read_from_database(database_reading_manager)
database_manager = ExperimentDatabaseManager(
mysql_credentials=sql_credentials.credentials, cache_size=40)
database_manager.delete_experiment('database_plots_test_case_1')
database_manager.close()
def test_read_write_4(self):
if os.path.exists('sample.db'):
os.unlink('sample.db')
database_manager = ExperimentDatabaseManager(file='sample.db',
cache_size=40)
database_manager.delete_experiment(
'writing_numerical_data_test_case_4')
database_manager.set_experiment('writing_numerical_data_test_case_4')
inserted_data = dict()
inserted_data['var_1'] = 19
inserted_data['var_2'] = 109
inserted_data['var_3'] = np.nan
database_manager.insert_experiment_data(
'writing_numerical_data_test_case_4', inserted_data)
database_manager.flush()
database_manager_2 = ExperimentDatabaseReadingManager(file='sample.db')
read_back_data = database_manager_2.get_data(
'writing_numerical_data_test_case_4',
'writing_numerical_data_test_case_4')
assert read_back_data['var_1'][0] == inserted_data[
'var_1'] # Always returns list
assert read_back_data['var_2'][0] == inserted_data[
'var_2'] # Always returns list
assert read_back_data['var_3'][0] == 0 # Always returns list
# Doing it at the end to check if it flushing works properly
database_manager.delete_experiment(
'writing_numerical_data_test_case_4')
database_manager.close()
def test_read_write_4(self):
database_manager = ExperimentDatabaseManager(
mysql_credentials=sql_credentials.credentials, cache_size=40)
database_manager.delete_experiment(
'writing_numerical_data_test_case_4')
database_manager.set_experiment('writing_numerical_data_test_case_4')
inserted_data = dict()
inserted_data['var_1'] = 19
inserted_data['var_2'] = 109
inserted_data['var_3'] = np.nan
database_manager.insert_experiment_data(
'writing_numerical_data_test_case_4', inserted_data)
database_manager.flush()
database_manager_2 = ExperimentDatabaseReadingManager(
mysql_credentials=sql_credentials.credentials)
read_back_data = database_manager_2.get_data(
'writing_numerical_data_test_case_4',
'writing_numerical_data_test_case_4')
assert read_back_data['var_1'][0] == inserted_data[
'var_1'] # Always returns list
assert read_back_data['var_2'][0] == inserted_data[
'var_2'] # Always returns list
assert read_back_data['var_3'][0] == 0 # Always returns list
# Doing it at the end to check if it flushing works properly
database_manager.delete_experiment(
'writing_numerical_data_test_case_4')
database_manager.close()
def main(files, pdfpath, dumppath, soft, database_table_prefix, run_for=-1):
global dataset_analysis_dict, fake_max_iou_values
file_index = 0
t1 = time.time()
for file in files:
print("\nFILE\n", file_index)
with gzip.open(file, 'rb') as f:
data_loaded = pickle.load(f)
# print("XYZ", len(data_dict['features']), len(data_dict['predicted']), len(data_dict['truth']))
file_results = analyse_multiple_endcaps_multi_cpu(
data_loaded,
soft=soft,
beta_threshold=beta_threshold,
distance_threshold=distance_threshold,
iou_threshold=iou_threshold)
for r in file_results:
append_endcap_dict_to_dataset_dict(dataset_analysis_dict, r)
# analyse_one_file(data_loaded, soft=soft)
if file_index == run_for - 1:
break
file_index += 1
print("It took", time.time() - t1, "seconds")
if len(dumppath) > 0:
print("Dumping analysis to bin file", dumppath)
with mgzip.open(dumppath, 'wb', thread=8, blocksize=2 * 10**7) as f:
pickle.dump(dataset_analysis_dict, f)
else:
print(
"WARNING: No analysis output path specified. Skipped dumping of analysis."
)
# print("Number of total fakes is ", num_total_fakes)
# np.savetxt('max_fake_iou.txt', fake_max_iou_values, delimiter=',')
# 0/0
plotter = HGCalAnalysisPlotter()
plotter.add_data_from_analysis_dict(dataset_analysis_dict)
if len(pdfpath) != 0:
plotter.write_to_pdf(pdfpath)
if len(database_table_prefix) != 0:
print("Will write plots to database")
database_manager = ExperimentDatabaseManager(
mysql_credentials=sql_credentials.credentials, cache_size=40)
database_manager.set_experiment('analysis_plotting_experiments')
plotter.write_data_to_database(database_manager, database_table_prefix)
database_manager.close()
def test_read_write_file(self):
if os.path.exists('sample.db'):
os.unlink('sample.db')
database_manager = ExperimentDatabaseManager(file='sample.db',
cache_size=40)
print("Writing to file")
self.write_to_database(database_manager)
database_manager.close()
print("Reading back from file")
database_reading_manager = ExperimentDatabaseReadingManager(
file='sample.db')
self.read_from_database(database_reading_manager)
if os.path.exists('sample.db'):
os.unlink('sample.db')
def write_to_database(self):
database_manager = ExperimentDatabaseManager(mysql_credentials=sql_credentials.credentials, cache_size=40)
database_manager.delete_experiment('database_plots_test_case_1')
database_manager.set_experiment('database_plots_test_case_1')
efficiency_plot = General2dBinningPlot(bins=np.array([0, 1, 2, 3, 4]), histogram_log=False,
histogram_fraction=False)
x_values = np.array([0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3])
y_values = np.array([0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3])
#
# print(type(x_values))
efficiency_plot.add_raw_values(x_values=x_values, y_values=y_values,
tags={'beta_threshold': 0.1, 'dist_threshold': 0.9})
efficiency_plot.draw()
efficiency_plot.write_to_database(database_manager,table_name='database_plots_test_case_1')
plt.savefig('output/test-original-plot.png')
database_manager.close()
def test_read_write_3(self):
if os.path.exists('sample.db'):
os.unlink('sample.db')
database_manager = ExperimentDatabaseManager(file='sample.db',
cache_size=40)
database_manager.delete_experiment(
'writing_numerical_data_test_case_3')
database_manager.set_experiment('writing_numerical_data_test_case_3')
inserted_data = dict()
inserted_data['var_1'] = np.array([19, 20])
inserted_data['var_2'] = np.array([109, 110])
inserted_data['var_3'] = np.array([54.1, 43])
inserted_data['var_4'] = np.array(['hello', 'world'])
database_manager.insert_experiment_data(
'writing_numerical_data_test_case_3', inserted_data)
database_manager.flush()
database_manager_2 = ExperimentDatabaseReadingManager(file='sample.db')
read_back_data = database_manager_2.get_data(
'writing_numerical_data_test_case_3',
'writing_numerical_data_test_case_3')
assert read_back_data['var_1'][0] == inserted_data['var_1'][0]
assert read_back_data['var_1'][1] == inserted_data['var_1'][1]
assert read_back_data['var_2'][0] == inserted_data['var_2'][0]
assert read_back_data['var_2'][1] == inserted_data['var_2'][1]
assert read_back_data['var_3'][0] == inserted_data['var_3'][0]
assert read_back_data['var_3'][1] == inserted_data['var_3'][1]
assert read_back_data['var_4'][0] == inserted_data['var_4'][0]
assert read_back_data['var_4'][1] == inserted_data['var_4'][1]
# Doing it at the end to check if it flushing works properly
# database_manager.delete_experiment('writing_numerical_data_test_case_3')
database_manager.close()
def test_read_write_2(self):
database_manager = ExperimentDatabaseManager(
mysql_credentials=sql_credentials.credentials, cache_size=40)
database_manager.delete_experiment(
'writing_numerical_data_test_case_2')
database_manager.set_experiment('writing_numerical_data_test_case_2')
inserted_data = dict()
inserted_data['var_1'] = [19, 20]
inserted_data['var_2'] = [109, 110]
inserted_data['var_3'] = [54.1, 43]
database_manager.insert_experiment_data(
'writing_numerical_data_test_case_2', inserted_data)
database_manager.flush()
database_manager_2 = ExperimentDatabaseReadingManager(
mysql_credentials=sql_credentials.credentials)
read_back_data = database_manager_2.get_data(
'writing_numerical_data_test_case_2',
'writing_numerical_data_test_case_2')
assert read_back_data['var_1'][0] == inserted_data['var_1'][0]
assert read_back_data['var_1'][1] == inserted_data['var_1'][1]
assert read_back_data['var_2'][0] == inserted_data['var_2'][0]
assert read_back_data['var_2'][1] == inserted_data['var_2'][1]
assert read_back_data['var_3'][0] == inserted_data['var_3'][0]
assert read_back_data['var_3'][1] == inserted_data['var_3'][1]
# Doing it at the end to check if it flushing works properly
database_manager.delete_experiment(
'writing_numerical_data_test_case_2')
database_manager.close()
cb = []
os.system('mkdir -p %s' % (train.outputDir + "/summary/"))
tensorboard_manager = TensorBoardManager(train.outputDir + "/summary/")
unique_id_path = os.path.join(train.outputDir,'unique_id.txt')
if os.path.exists(unique_id_path):
with open(unique_id_path, 'r') as f:
unique_id = f.readlines()[0].strip()
else:
unique_id = str(uuid.uuid4())[:8]
with open(unique_id_path, 'w') as f:
f.write(unique_id+'\n')
database_manager = ExperimentDatabaseManager(mysql_credentials=sql_credentials.credentials, cache_size=40)
database_manager.set_experiment('alpha_experiment_june_pca_double_cords_' + unique_id)
cb += [RunningMetricsCallback(td, tensorboard_manager, dist_threshold=0.5, beta_threshold=0.5, database_manager=database_manager)]
cb += [plotClusteringDuringTraining(
use_backgather_idx=8 + i,
outputfile=train.outputDir + "/plts/sn" + str(i) + '_',
samplefile=samplepath,
after_n_batches=20,
on_epoch_end=False,
publish=None,
use_event=0)
for i in [0, 2]]
cb += [
plotEventDuringTraining(
tensorboard_manager = TensorBoardManager(train.outputDir + "/summary/")
unique_id_path = os.path.join(train.outputDir, 'unique_id.txt')
if os.path.exists(unique_id_path):
with open(unique_id_path, 'r') as f:
unique_id = f.readlines()[0].strip()
else:
unique_id = str(uuid.uuid4())[:8]
with open(unique_id_path, 'w') as f:
f.write(unique_id + '\n')
nbatch = 50000 #this is rather low, and can be set to a higher values e.g. when training on V100s
database_manager = ExperimentDatabaseManager(file=os.path.join(
train.outputDir, "training_metrics.db"),
cache_size=100)
database_reading_manager = ExperimentDatabaseReadingManager(
file=os.path.join(train.outputDir, "training_metrics.db"))
database_manager.set_experiment(unique_id)
metadata = matching_and_analysis.build_metadeta_dict(
beta_threshold=0.5,
distance_threshold=0.5,
iou_threshold=0.0001,
matching_type=matching_and_analysis.MATCHING_TYPE_MAX_FOUND)
analyzer = matching_and_analysis.OCAnlayzerWrapper(metadata)
cb += [
RunningMetricsDatabaseAdditionCallback(td,
tensorboard_manager,
database_manager=database_manager,
analyzer=analyzer)