def test_logging_stdout(self):
filename = 'teststdoutlog.hdf5'
filename = make_temp_dir(filename)
folder = make_temp_dir('logs')
env = Environment(trajectory=make_trajectory_name(self),
filename=filename, log_config=get_log_config(),
# log_levels=logging.CRITICAL, # needed for the test
log_stdout=('STDOUT', 50), #log_folder=folder
)
env.f_run(log_error)
traj = env.v_traj
path = get_log_path(traj)
mainstr = 'sTdOuTLoGGinG'
print(mainstr)
env.f_disable_logging()
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
self.assertTrue(mainstr in full_text)
self.assertTrue('4444444' not in full_text)
self.assertTrue('DEBUG' not in full_text)
def test_time_display_of_loading(self):
filename = make_temp_dir('sloooow.hdf5')
env = Environment(trajectory='traj', add_time=True, filename=filename,
log_stdout=False,
log_config=get_log_config(),
dynamic_imports=SlowResult,
display_time=0.1)
traj = env.v_traj
res=traj.f_add_result(SlowResult, 'iii', 42, 43, comment='llk')
traj.f_store()
service_logger = traj.v_storage_service._logger
root = logging.getLogger('pypet')
old_level = root.level
service_logger.setLevel(logging.INFO)
root.setLevel(logging.INFO)
traj.f_load(load_data=3)
service_logger.setLevel(old_level)
root.setLevel(old_level)
path = get_log_path(traj)
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
self.assertTrue('nodes/s)' in full_text)
env.f_disable_logging()
def test_logging_stdout(self):
filename = 'teststdoutlog.hdf5'
filename = make_temp_dir(filename)
folder = make_temp_dir('logs')
env = Environment(trajectory=make_trajectory_name(self),
filename=filename, log_config=get_log_config(),
# log_levels=logging.CRITICAL, # needed for the test
log_stdout=('STDOUT', 50), #log_folder=folder
)
env.f_run(log_error)
traj = env.v_traj
path = get_log_path(traj)
mainstr = 'sTdOuTLoGGinG'
print(mainstr)
env.f_disable_logging()
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
self.assertTrue(mainstr in full_text)
self.assertTrue('4444444' not in full_text)
self.assertTrue('DEBUG' not in full_text)
def test_time_display_of_loading(self):
filename = make_temp_dir('sloooow.hdf5')
env = Environment(trajectory='traj', add_time=True, filename=filename,
log_stdout=False,
log_config=get_log_config(),
dynamic_imports=SlowResult,
display_time=0.1)
traj = env.v_traj
res=traj.f_add_result(SlowResult, 'iii', 42, 43, comment='llk')
traj.f_store()
service_logger = traj.v_storage_service._logger
root = logging.getLogger('pypet')
old_level = root.level
service_logger.setLevel(logging.INFO)
root.setLevel(logging.INFO)
traj.f_load(load_data=3)
service_logger.setLevel(old_level)
root.setLevel(old_level)
path = get_log_path(traj)
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
self.assertTrue('nodes/s)' in full_text)
env.f_disable_logging()
def test_logfile_old_way_disabling_mp_log(self):
# if not self.multiproc:
# return
del self.mode.__dict__['log_config']
self.make_env(logger_names=('', 'pypet'),
log_level=logging.ERROR,
log_folder=make_temp_dir('logs'),
log_multiproc=False)
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_all_levels)
if self.mode.multiproc:
logging.getLogger('pypet.test').error('ttt')
self.env.f_disable_logging()
traj = self.env.v_traj
log_path = get_log_path(traj)
# if self.mode.multiproc:
length = 2
file_list = [file for file in os.listdir(log_path)]
self.assertEqual(len(file_list),
length) # assert that there are as many
def test_logging_show_progress_print(self):
self.make_env(log_config=get_log_config(), log_stdout=('prostdout', 50),
report_progress=(3, 'print'))
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_all_levels)
self.env.f_disable_logging()
path = get_log_path(self.env.v_traj)
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
progress = 'prostdout CRITICAL PROGRESS: Finished'
self.assertTrue(progress in full_text)
bar = '[=='
self.assertIn(bar, full_text)
def test_logging_show_progress_print(self):
self.make_env(log_config=get_log_config(), log_stdout=('prostdout', 50),
report_progress=(3, 'print'))
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_all_levels)
self.env.f_disable_logging()
path = get_log_path(self.env.v_traj)
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
progress = 'prostdout CRITICAL PROGRESS: Finished'
self.assertTrue(progress in full_text)
bar = '[=='
self.assertIn(bar, full_text)
def test_logging_show_progress(self):
self.make_env(log_config=get_log_config(),
# log_folder=make_temp_dir('logs'),
# log_levels=logging.ERROR,
report_progress=(3, 'progress', 40))
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_all_levels)
self.env.f_disable_logging()
traj = self.env.v_traj
path = get_log_path(traj)
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
progress = 'PROGRESS: Finished'
self.assertTrue(progress in full_text)
bar = '[=='
self.assertTrue(bar in full_text)
def test_logging_show_progress(self):
self.make_env(log_config=get_log_config(),
# log_folder=make_temp_dir('logs'),
# log_levels=logging.ERROR,
report_progress=(3, 'progress', 40))
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_all_levels)
self.env.f_disable_logging()
traj = self.env.v_traj
path = get_log_path(traj)
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
progress = 'PROGRESS: Finished'
self.assertTrue(progress in full_text)
bar = '[=='
self.assertTrue(bar in full_text)
def test_logfile_old_way_disabling_mp_log(self):
# if not self.multiproc:
# return
del self.mode.__dict__['log_config']
self.make_env(logger_names = ('','pypet'), log_level=logging.ERROR,
log_folder=make_temp_dir('logs'), log_multiproc=False)
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_all_levels)
if self.mode.multiproc:
logging.getLogger('pypet.test').error('ttt')
self.env.f_disable_logging()
traj = self.env.v_traj
log_path = get_log_path(traj)
# if self.mode.multiproc:
length = 2
file_list = [file for file in os.listdir(log_path)]
self.assertEqual(len(file_list), length) # assert that there are as many
def test_disable(self):
# if not self.multiproc:
# return
self.make_env(log_config=None)
traj = self.env.v_traj
log_path = get_log_path(traj)
self.assertFalse(os.path.isdir(log_path))
self.assertTrue(self.env._logging_manager._sp_config is None)
self.assertTrue(self.env._logging_manager._mp_config is None)
self.assertTrue(self.env._logging_manager.log_config is None)
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_all_levels)
self.assertFalse(os.path.isdir(log_path))
self.assertTrue(self.env._logging_manager._sp_config is None)
self.assertTrue(self.env._logging_manager._mp_config is None)
self.assertTrue(self.env._logging_manager.log_config is None)
self.env.f_disable_logging()
def test_disable(self):
# if not self.multiproc:
# return
self.make_env(log_config=None)
traj = self.env.v_traj
log_path = get_log_path(traj)
self.assertFalse(os.path.isdir(log_path))
self.assertTrue(self.env._logging_manager._sp_config is None)
self.assertTrue(self.env._logging_manager._mp_config is None)
self.assertTrue(self.env._logging_manager.log_config is None)
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_all_levels)
self.assertFalse(os.path.isdir(log_path))
self.assertTrue(self.env._logging_manager._sp_config is None)
self.assertTrue(self.env._logging_manager._mp_config is None)
self.assertTrue(self.env._logging_manager.log_config is None)
self.env.f_disable_logging()
def test_logfile_old_way_creation_with_errors(self):
# if not self.multiproc:
# return
del self.mode.__dict__['log_config']
self.make_env(logger_names = ('','pypet'), log_level=logging.ERROR,
log_folder=make_temp_dir('logs'))
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_all_levels)
if self.mode.multiproc:
logging.getLogger('pypet.test').error('ttt')
self.env.f_disable_logging()
traj = self.env.v_traj
log_path = get_log_path(traj)
if self.mode.multiproc:
if self.mode.use_pool:
length = self.mode.ncores * 2
else:
length = 2 * len(traj)
if self.mode.wrap_mode == 'LOCK':
length += 2
elif self.mode.wrap_mode == 'QUEUE':
length += 4
else:
raise RuntimeError('You shall not pass!')
else:
length = 2
file_list = [file for file in os.listdir(log_path)]
self.assertEqual(len(file_list), length) # assert that there are as many
# files as runs plus main.txt and errors and warnings
total_error_count = 0
total_store_count = 0
total_info_count = 0
total_retry_count = 0
for file in file_list:
with open(os.path.join(log_path, file), mode='r') as fh:
text = fh.read()
count = text.count('INFO_Test!')
total_info_count += count
error_count = text.count('ERROR_Test!')
total_error_count += error_count
store_count = text.count('STORE_Test!')
total_store_count += store_count
retry_count = text.count('Retry')
total_retry_count += retry_count
if 'LOG.txt' == file:
if self.mode.multiproc:
self.assertEqual(count,0)
self.assertEqual(store_count, 0)
else:
self.assertEqual(count, 0)
self.assertGreaterEqual(store_count, len(traj))
elif 'ERROR.txt' == file:
self.assertEqual(count, 0)
if self.mode.multiproc:
self.assertEqual(error_count,0)
self.assertEqual(store_count, 0)
else:
self.assertGreaterEqual(error_count, len(traj))
self.assertGreaterEqual(store_count, len(traj))
elif 'Queue' in file and 'ERROR' in file:
self.assertGreaterEqual(store_count, len(traj))
elif 'Queue' in file and 'LOG' in file:
self.assertGreaterEqual(store_count, len(traj))
elif 'LOG' in file:
if self.mode.multiproc and self.mode.use_pool:
self.assertEqual(count, 0)
self.assertGreaterEqual(error_count, 0)
else:
self.assertEqual(count, 0)
self.assertGreaterEqual(error_count, 1)
if self.mode.wrap_mode == 'QUEUE':
self.assertEqual(store_count, 0)
else:
self.assertGreaterEqual(store_count, 1)
elif 'ERROR' in file:
if self.mode.multiproc and self.mode.use_pool:
self.assertEqual(count, 0)
self.assertGreaterEqual(error_count, 0)
else:
self.assertEqual(count, 0)
self.assertGreaterEqual(error_count, 1)
if self.mode.wrap_mode == 'QUEUE':
self.assertEqual(store_count, 0)
else:
self.assertGreaterEqual(store_count, 1)
else:
self.assertTrue(False, 'There`s a file in the log folder that does not '
'belong there: %s' % str(file))
self.assertGreaterEqual(total_store_count, 2*len(traj))
self.assertGreaterEqual(total_error_count, 2*len(traj))
self.assertEqual(total_info_count, 0)
self.assertLess(total_retry_count, len(traj))
def test_logfile_creation_normal(self):
# if not self.multiproc:
# return
self.make_env()
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_wo_error_levels)
self.env.f_disable_logging()
traj = self.env.v_traj
log_path = get_log_path(traj)
if self.mode.multiproc:
if self.mode.use_pool:
length = self.mode.ncores * 2
else:
length = 2 * len(traj)
if self.mode.wrap_mode == 'LOCK':
length += 2
elif self.mode.wrap_mode == 'QUEUE':
length += 4
else:
raise RuntimeError('You shall not pass!')
else:
length = 2
file_list = [file for file in os.listdir(log_path)]
self.assertEqual(len(file_list), length) # assert that there are as many
# files as runs plus main.txt and errors and warnings
total_error_count = 0
total_store_count = 0
total_info_count = 0
total_retry_count = 0
for file in file_list:
with open(os.path.join(log_path, file), mode='r') as fh:
text = fh.read()
count = text.count('INFO_Test!')
total_info_count += count
error_count = text.count('ERROR_Test!')
total_error_count += error_count
store_count = text.count('STORE_Test!')
total_store_count += store_count
retry_count = text.count('Retry')
total_retry_count += retry_count
if 'LOG.txt' == file:
if self.mode.multiproc:
self.assertEqual(count,0)
self.assertEqual(store_count, 0)
else:
self.assertEqual(count, len(traj))
self.assertEqual(store_count, len(traj))
elif 'ERROR' in file:
full_path = os.path.join(log_path, file)
filesize = os.path.getsize(full_path)
with open(full_path) as fh:
text = fh.read()
if 'Retry' not in text:
self.assertEqual(filesize, 0)
elif 'Queue' in file:
self.assertEqual(store_count, len(traj))
elif 'LOG' in file:
if self.mode.multiproc and self.mode.use_pool:
self.assertGreaterEqual(count, 0, '%d < 1 for file %s' % (count, file))
else:
self.assertEqual(count, 1)
if self.mode.wrap_mode == 'QUEUE':
self.assertEqual(store_count, 0)
else:
self.assertEqual(store_count, 1)
else:
self.assertTrue(False, 'There`s a file in the log folder that does not '
'belong there: %s' % str(file))
self.assertEqual(total_store_count, len(traj))
self.assertEqual(total_error_count, 0)
self.assertEqual(total_info_count, len(traj))
self.assertLess(total_retry_count, len(traj))
def test_logfile_old_way_creation_with_errors(self):
# if not self.multiproc:
# return
del self.mode.__dict__['log_config']
self.make_env(logger_names = ('','pypet'), log_level=logging.ERROR,
log_folder=make_temp_dir('logs'))
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_all_levels)
if self.mode.multiproc:
logging.getLogger('pypet.test').error('ttt')
self.env.f_disable_logging()
traj = self.env.v_traj
log_path = get_log_path(traj)
if self.mode.multiproc:
if self.mode.use_pool:
length = self.mode.ncores * 2
else:
length = 2 * len(traj)
if self.mode.wrap_mode == 'LOCK':
length += 2
elif self.mode.wrap_mode == 'QUEUE':
length += 4
else:
raise RuntimeError('You shall not pass!')
else:
length = 2
file_list = [file for file in os.listdir(log_path)]
self.assertEqual(len(file_list), length) # assert that there are as many
# files as runs plus main.txt and errors and warnings
total_error_count = 0
total_store_count = 0
total_info_count = 0
total_retry_count = 0
for file in file_list:
with open(os.path.join(log_path, file), mode='r') as fh:
text = fh.read()
if len(text) == 0:
continue
count = text.count('INFO_Test!')
total_info_count += count
error_count = text.count('ERROR_Test!')
total_error_count += error_count
store_count = text.count('STORE_Test!')
total_store_count += store_count
retry_count = text.count('Retry')
total_retry_count += retry_count
if 'LOG.txt' == file:
if self.mode.multiproc:
self.assertEqual(count,0)
self.assertEqual(store_count, 0)
else:
self.assertEqual(count, 0)
self.assertGreaterEqual(store_count, len(traj))
elif 'ERROR.txt' == file:
self.assertEqual(count, 0)
if self.mode.multiproc:
self.assertEqual(error_count,0)
self.assertEqual(store_count, 0)
else:
self.assertGreaterEqual(error_count, len(traj))
self.assertGreaterEqual(store_count, len(traj))
elif 'Queue' in file and 'ERROR' in file:
self.assertGreaterEqual(store_count, len(traj))
elif 'Queue' in file and 'LOG' in file:
self.assertGreaterEqual(store_count, len(traj))
elif 'LOG' in file:
if self.mode.multiproc and self.mode.use_pool:
self.assertEqual(count, 0)
self.assertGreaterEqual(error_count, 0)
else:
self.assertEqual(count, 0)
self.assertGreaterEqual(error_count, 1)
if self.mode.wrap_mode == 'QUEUE':
self.assertEqual(store_count, 0)
else:
self.assertGreaterEqual(store_count, 1)
elif 'ERROR' in file:
if self.mode.multiproc and self.mode.use_pool:
self.assertEqual(count, 0)
self.assertGreaterEqual(error_count, 0)
else:
self.assertEqual(count, 0)
self.assertGreaterEqual(error_count, 1)
if self.mode.wrap_mode == 'QUEUE':
self.assertEqual(store_count, 0)
else:
self.assertGreaterEqual(store_count, 1)
else:
self.assertTrue(False, 'There`s a file in the log folder that does not '
'belong there: %s' % str(file))
self.assertGreaterEqual(total_store_count, 2*len(traj))
self.assertGreaterEqual(total_error_count, 2*len(traj))
self.assertEqual(total_info_count, 0)
self.assertLess(total_retry_count, len(traj))
def test_logfile_creation_normal(self):
# if not self.multiproc:
# return
self.make_env()
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_wo_error_levels)
self.env.f_disable_logging()
traj = self.env.v_traj
log_path = get_log_path(traj)
if self.mode.multiproc:
if self.mode.use_pool:
length = self.mode.ncores * 2
else:
length = 2 * len(traj)
if self.mode.wrap_mode == 'LOCK':
length += 2
elif self.mode.wrap_mode == 'QUEUE':
length += 4
else:
raise RuntimeError('You shall not pass!')
else:
length = 2
file_list = [file for file in os.listdir(log_path)]
self.assertEqual(len(file_list), length) # assert that there are as many
# files as runs plus main.txt and errors and warnings
total_error_count = 0
total_store_count = 0
total_info_count = 0
total_retry_count = 0
for file in file_list:
with open(os.path.join(log_path, file), mode='r') as fh:
text = fh.read()
if len(text) == 0:
continue
count = text.count('INFO_Test!')
total_info_count += count
error_count = text.count('ERROR_Test!')
total_error_count += error_count
store_count = text.count('STORE_Test!')
total_store_count += store_count
retry_count = text.count('Retry')
total_retry_count += retry_count
if 'LOG.txt' == file:
if self.mode.multiproc:
self.assertEqual(count,0)
self.assertEqual(store_count, 0)
else:
self.assertEqual(count, len(traj))
self.assertEqual(store_count, len(traj))
elif 'ERROR' in file:
full_path = os.path.join(log_path, file)
filesize = os.path.getsize(full_path)
with open(full_path) as fh:
text = fh.read()
if 'Retry' not in text:
self.assertEqual(filesize, 0)
elif 'Queue' in file:
self.assertEqual(store_count, len(traj))
elif 'LOG' in file:
if self.mode.multiproc and self.mode.use_pool:
self.assertGreaterEqual(count, 0, '%d < 1 for file %s' % (count, file))
else:
self.assertEqual(count, 1)
if self.mode.wrap_mode == 'QUEUE':
self.assertEqual(store_count, 0)
else:
self.assertEqual(store_count, 1)
else:
self.assertTrue(False, 'There`s a file in the log folder that does not '
'belong there: %s' % str(file))
self.assertEqual(total_store_count, len(traj))
self.assertEqual(total_error_count, 0)
self.assertEqual(total_info_count, len(traj))
self.assertLess(total_retry_count, len(traj))
