def test_without_pre_run(self): runner = NetworkRunner() components = [TheNeurons(), TheConnection(), TheSimulation()] analyser = [TheMonitors()] nm = NetworkManager(network_runner=runner, component_list=components, analyser_list=analyser) env = Environment(trajectory='Test_'+repr(time.time()).replace('.','_'), filename=make_temp_dir(os.path.join( 'experiments', 'tests', 'briantests', 'HDF5', 'briantest.hdf5')), file_title='test', log_config=get_log_config(), dynamic_imports=['pypet.brian2.parameter.BrianParameter', BrianMonitorResult, BrianResult], multiproc=True, ncores=2) traj = env.v_traj nm.add_parameters(traj) traj.f_explore({'v01': [11*mV, 13*mV]}) env.f_run(nm.run_network) self.check_data(traj)
def test_logging_stdout(self): filename = 'teststdoutlog.hdf5' filename = make_temp_dir(filename) folder = make_temp_dir('logs') env = Environment(trajectory=make_trajectory_name(self), filename=filename, log_config=get_log_config(), # log_levels=logging.CRITICAL, # needed for the test log_stdout=('STDOUT', 50), #log_folder=folder ) env.f_run(log_error) traj = env.v_traj path = get_log_path(traj) mainstr = 'sTdOuTLoGGinG' print(mainstr) env.f_disable_logging() mainfilename = os.path.join(path, 'LOG.txt') with open(mainfilename, mode='r') as mainf: full_text = mainf.read() self.assertTrue(mainstr in full_text) self.assertTrue('4444444' not in full_text) self.assertTrue('DEBUG' not in full_text)
def main(): filename = os.path.join('hdf5', 'FiringRate.hdf5') env = Environment(trajectory='FiringRate', comment='Experiment to measure the firing rate ' 'of a leaky integrate and fire neuron. ' 'Exploring different input currents, ' 'as well as refractory periods', add_time=False, # We don't want to add the current time to the name, log_stdout=True, log_config='DEFAULT', multiproc=True, ncores=2, #My laptop has 2 cores ;-) wrap_mode='QUEUE', filename=filename, overwrite_file=True) traj = env.v_trajectory # Add parameters add_parameters(traj) # Let's explore add_exploration(traj) # Ad the postprocessing function env.f_add_postprocessing(neuron_postproc) # Run the experiment env.f_run(run_neuron) # Finally disable logging and close all log-files env.f_disable_logging()
def test_without_pre_run(self): runner = NetworkRunner() components = [TheNeurons(), TheConnection(), TheSimulation()] analyser = [TheMonitors()] nm = NetworkManager(network_runner=runner, component_list=components, analyser_list=analyser) env = Environment( trajectory='Test_' + repr(time.time()).replace('.', '_'), filename=make_temp_dir( os.path.join('experiments', 'tests', 'briantests', 'HDF5', 'briantest.hdf5')), file_title='test', log_config=get_log_config(), dynamic_imports=[ 'pypet.brian2.parameter.BrianParameter', BrianMonitorResult, BrianResult ], multiproc=True, ncores=2) traj = env.v_traj nm.add_parameters(traj) traj.f_explore({'v01': [11 * mV, 13 * mV]}) env.f_run(nm.run_network) self.check_data(traj)
def main(): env = Environment(trajectory='FiringRate', comment='Experiment to measure the firing rate ' 'of a leaky integrate and fire neuron. ' 'Exploring different input currents, ' 'as well as refractory periods', add_time=False, # We don't want to add the current time to the name, log_folder='./logs/', log_level=logging.INFO, log_stdout=True, multiproc=True, ncores=2, #My laptop has 2 cores ;-) wrap_mode='QUEUE', filename='./hdf5/', # We only pass a folder here, so the name is chosen # automatically to be the same as the Trajectory ) traj = env.v_trajectory # Add parameters add_parameters(traj) # Let's explore add_exploration(traj) # Ad the postprocessing function env.f_add_postprocessing(neuron_postproc) # Run the experiment env.f_run(run_neuron)
def main(): """ Main *boilerplate* function to start simulation """ # Now let's make use of logging logger = logging.getLogger() # Create folders for data and plots folder = os.path.join(os.getcwd(), 'experiments', 'ca_patterns_pypet') if not os.path.isdir(folder): os.makedirs(folder) filename = os.path.join(folder, 'all_patterns.hdf5') # Create an environment env = Environment(trajectory='cellular_automata', multiproc=True, ncores=4, wrap_mode='QUEUE', filename=filename, overwrite_file=True) # extract the trajectory traj = env.v_traj traj.v_lazy_adding = True traj.par.ncells = 400, 'Number of cells' traj.par.steps = 250, 'Number of timesteps' traj.par.rule_number = 30, 'The ca rule' traj.par.initial_name = 'random', 'The type of initial state' traj.par.seed = 100042, 'RNG Seed' # Explore exp_dict = {'rule_number' : [10, 30, 90, 110, 184], 'initial_name' : ['single', 'random'],} # # You can uncomment the ``exp_dict`` below to see that changing the # # exploration scheme is now really easy: # exp_dict = {'rule_number' : [10, 30, 90, 110, 184], # 'ncells' : [100, 200, 300], # 'seed': [333444555, 123456]} exp_dict = cartesian_product(exp_dict) traj.f_explore(exp_dict) # Run the simulation logger.info('Starting Simulation') env.f_run(wrap_automaton) # Load all data traj.f_load(load_data=2) logger.info('Printing data') for idx, run_name in enumerate(traj.f_iter_runs()): # Plot all patterns filename = os.path.join(folder, make_filename(traj)) plot_pattern(traj.crun.pattern, traj.rule_number, filename) progressbar(idx, len(traj), logger=logger) # Finally disable logging and close all log-files env.f_disable_logging()
def test_run(): global filename np.random.seed() trajname = 'profiling' filename = make_temp_dir(os.path.join('hdf5', 'test%s.hdf5' % trajname)) env = Environment(trajectory=trajname, filename=filename, file_title=trajname, log_stdout=False, results_per_run=5, derived_parameters_per_run=5, multiproc=False, ncores=1, wrap_mode='LOCK', use_pool=False, overwrite_file=True) traj = env.v_trajectory traj.v_standard_parameter = Parameter ## Create some parameters param_dict = {} create_param_dict(param_dict) ### Add some parameter: add_params(traj, param_dict) #remember the trajectory and the environment traj = traj env = env traj.f_add_parameter('TEST', 'test_run') ###Explore explore(traj) ### Make a test run simple_arg = -13 simple_kwarg = 13.0 env.f_run(simple_calculations, simple_arg, simple_kwarg=simple_kwarg) size = os.path.getsize(filename) size_in_mb = size / 1000000. print('Size is %sMB' % str(size_in_mb))
def test_run(): global filename np.random.seed() trajname = 'profiling' filename = make_temp_dir(os.path.join('hdf5', 'test%s.hdf5' % trajname)) env = Environment(trajectory=trajname, filename=filename, file_title=trajname, log_stdout=False, results_per_run=5, derived_parameters_per_run=5, multiproc=False, ncores=1, wrap_mode='LOCK', use_pool=False, overwrite_file=True) traj = env.v_trajectory traj.v_standard_parameter=Parameter ## Create some parameters param_dict={} create_param_dict(param_dict) ### Add some parameter: add_params(traj,param_dict) #remember the trajectory and the environment traj = traj env = env traj.f_add_parameter('TEST', 'test_run') ###Explore explore(traj) ### Make a test run simple_arg = -13 simple_kwarg= 13.0 env.f_run(simple_calculations,simple_arg,simple_kwarg=simple_kwarg) size=os.path.getsize(filename) size_in_mb = size/1000000. print('Size is %sMB' % str(size_in_mb))
def test_net(self): env = Environment(trajectory='Test_'+repr(time.time()).replace('.','_'), filename=make_temp_dir(os.path.join( 'experiments', 'tests', 'briantests', 'HDF5', 'briantest.hdf5')), file_title='test', log_config=get_log_config(), dynamic_imports=['pypet.brian2.parameter.Brian2Parameter', Brian2MonitorResult], multiproc=False) traj = env.v_traj traj.f_add_parameter(Brian2Parameter, 'v0', 0.0*mV, comment='Input bias') traj.f_explore({'v0': [11*mV, 13*mV, 15*mV]}) env.f_run(run_network) self.get_data(traj)
def test_net(self): env = Environment( trajectory='Test_' + repr(time.time()).replace('.', '_'), filename=make_temp_dir( os.path.join('experiments', 'tests', 'briantests', 'HDF5', 'briantest.hdf5')), file_title='test', log_config=get_log_config(), dynamic_imports=[ 'pypet.brian2.parameter.Brian2Parameter', Brian2MonitorResult ], multiproc=False) traj = env.v_traj traj.f_add_parameter(Brian2Parameter, 'v0', 0.0 * mV, comment='Input bias') traj.f_explore({'v0': [11 * mV, 13 * mV, 15 * mV]}) env.f_run(run_network) self.get_data(traj)
def main(): """Main function to protect the *entry point* of the program. If you want to use multiprocessing under Windows you need to wrap your main code creating an environment into a function. Otherwise the newly started child processes will re-execute the code and throw errors (also see https://docs.python.org/2/library/multiprocessing.html#windows). """ # Create an environment that handles running. # Let's enable multiprocessing with 2 workers. filename = os.path.join('hdf5', 'example_04.hdf5') env = Environment(trajectory='Example_04_MP', filename=filename, file_title='Example_04_MP', log_stdout=True, comment='Multiprocessing example!', multiproc=True, ncores=4, use_pool=True, # Our runs are inexpensive we can get rid of overhead # by using a pool wrap_mode=pypetconstants.WRAP_MODE_QUEUE) # Get the trajectory from the environment traj = env.v_trajectory # Add both parameters traj.f_add_parameter('x', 1.0, comment='I am the first dimension!') traj.f_add_parameter('y', 1.0, comment='I am the second dimension!') # Explore the parameters with a cartesian product, but we want to explore a bit more traj.f_explore(cartesian_product({'x':[float(x) for x in range(20)], 'y':[float(y) for y in range(12)]})) # Run the simulation env.f_run(multiply) # Finally disable logging and close all log-files env.f_disable_logging()
def main(): # Create an environment that handles running filename = os.path.join('hdf5', 'example_12.hdf5') env = Environment(trajectory='Multiplication', filename=filename, file_title='Example_12_Sharing_Data', comment='The first example!', continuable=False, # We have shared data in terms of a multiprocessing list, # so we CANNOT use the continue feature. multiproc=True, ncores=2) # The environment has created a trajectory container for us traj = env.v_trajectory # Add both parameters traj.f_add_parameter('x', 1, comment='I am the first dimension!') traj.f_add_parameter('y', 1, comment='I am the second dimension!') # Explore the parameters with a cartesian product traj.f_explore(cartesian_product({'x':[1,2,3,4], 'y':[6,7,8]})) # We want a shared list where we can put all out results in. We use a manager for this: result_list = mp.Manager().list() # Let's make some space for potential results result_list[:] =[0 for _dummy in range(len(traj))] # Run the simulation env.f_run(multiply, result_list) # Now we want to store the final list as numpy array traj.f_add_result('z', np.array(result_list)) # Finally let's print the result to see that it worked print(traj.z) #Disable logging and close all log-files env.f_disable_logging()
traj2 = env2.v_trajectory # Add both parameters traj1.f_add_parameter("x", 1.0, comment="I am the first dimension!") traj1.f_add_parameter("y", 1.0, comment="I am the second dimension!") traj2.f_add_parameter("x", 1.0, comment="I am the first dimension!") traj2.f_add_parameter("y", 1.0, comment="I am the second dimension!") # Explore the parameters with a cartesian product for the first trajectory: traj1.f_explore(cartesian_product({"x": [1.0, 2.0, 3.0, 4.0], "y": [6.0, 7.0, 8.0]})) # Let's explore slightly differently for the second: traj2.f_explore(cartesian_product({"x": [3.0, 4.0, 5.0, 6.0], "y": [7.0, 8.0, 9.0]})) # Run the simulations with all parameter combinations env1.f_run(multiply) env2.f_run(multiply) # Now we merge them together into traj1 # We want to remove duplicate entries # like the parameter space point x=3.0, y=7.0. # Several points have been explored by both trajectories and we need them only once. # Therefore, we set remove_duplicates=True (Note this takes O(N1*N2)!). # We also want to backup both trajectories, but we let the system choose the filename. # Accordingly we choose backup_filename=True instead of providing a filename. # We want to move the hdf5 nodes from one trajectory to the other. # Thus we set move_nodes=True. # Finally,we want to delete the other trajectory afterwards since we already have a backup. traj1.f_merge(traj2, remove_duplicates=True, backup_filename=True, move_data=True, delete_other_trajectory=True) # And that's it, now we can take a look at the new trajectory and print all x,y,z triplets.
class LoggingTest(TrajectoryComparator): tags = 'integration', 'environment', 'logging' def setUp(self): root = logging.getLogger() for logger in itools.chain(root.manager.loggerDict.values(), [root]): if hasattr(logger, 'handlers'): for handler in logger.handlers: if hasattr(handler, 'flush'): handler.flush() if hasattr(handler, 'close'): handler.close() logger.handlers = [] if hasattr(logger, 'setLevel'): logger.setLevel(logging.NOTSET) self.set_mode() def tearDown(self): super(LoggingTest, self).tearDown() def set_mode(self): self.mode = Dummy() self.mode.wrap_mode = 'LOCK' self.mode.multiproc = False self.mode.ncores = 1 self.mode.use_pool=True self.mode.pandas_format='fixed' self.mode.pandas_append=False self.mode.complib = 'blosc' self.mode.complevel=9 self.mode.shuffle=True self.mode.fletcher32 = False self.mode.encoding = 'utf8' self.mode.log_stdout=False self.mode.log_config=get_log_config() def make_env(self, **kwargs): self.mode.__dict__.update(kwargs) filename = 'log_testing.hdf5' self.filename = make_temp_dir(filename) self.traj_name = make_trajectory_name(self) self.env = Environment(trajectory=self.traj_name, filename=self.filename, **self.mode.__dict__) self.traj = self.env.v_traj def add_params(self, traj): traj.v_lazy_adding = True traj.par.p1 = 42, 'Hey' traj.f_apar('g1.p2', 145, comment='Test') def explore(self, traj): traj.f_explore({'p1': range(7)}) @unittest.skipIf(platform.system() == 'Windows', 'Log file creation might fail under windows.') def test_logfile_creation_normal(self): # if not self.multiproc: # return self.make_env() self.add_params(self.traj) self.explore(self.traj) self.env.f_run(log_wo_error_levels) self.env.f_disable_logging() traj = self.env.v_traj log_path = get_log_path(traj) if self.mode.multiproc: if self.mode.use_pool: length = self.mode.ncores * 2 else: length = 2 * len(traj) if self.mode.wrap_mode == 'LOCK': length += 2 elif self.mode.wrap_mode == 'QUEUE': length += 4 else: raise RuntimeError('You shall not pass!') else: length = 2 file_list = [file for file in os.listdir(log_path)] self.assertEqual(len(file_list), length) # assert that there are as many # files as runs plus main.txt and errors and warnings total_error_count = 0 total_store_count = 0 total_info_count = 0 total_retry_count = 0 for file in file_list: with open(os.path.join(log_path, file), mode='r') as fh: text = fh.read() count = text.count('INFO_Test!') total_info_count += count error_count = text.count('ERROR_Test!') total_error_count += error_count store_count = text.count('STORE_Test!') total_store_count += store_count retry_count = text.count('Retry') total_retry_count += retry_count if 'LOG.txt' == file: if self.mode.multiproc: self.assertEqual(count,0) self.assertEqual(store_count, 0) else: self.assertEqual(count, len(traj)) self.assertEqual(store_count, len(traj)) elif 'ERROR' in file: full_path = os.path.join(log_path, file) filesize = os.path.getsize(full_path) with open(full_path) as fh: text = fh.read() if 'Retry' not in text: self.assertEqual(filesize, 0) elif 'Queue' in file: self.assertEqual(store_count, len(traj)) elif 'LOG' in file: if self.mode.multiproc and self.mode.use_pool: self.assertGreaterEqual(count, 0, '%d < 1 for file %s' % (count, file)) else: self.assertEqual(count, 1) if self.mode.wrap_mode == 'QUEUE': self.assertEqual(store_count, 0) else: self.assertEqual(store_count, 1) else: self.assertTrue(False, 'There`s a file in the log folder that does not ' 'belong there: %s' % str(file)) self.assertEqual(total_store_count, len(traj)) self.assertEqual(total_error_count, 0) self.assertEqual(total_info_count, len(traj)) self.assertLess(total_retry_count, len(traj)) def test_throw_error_when_specifying_config_and_old_method(self): with self.assertRaises(ValueError): self.make_env(log_config=None, logger_names='test') def test_disable(self): # if not self.multiproc: # return self.make_env(log_config=None) traj = self.env.v_traj log_path = get_log_path(traj) self.assertFalse(os.path.isdir(log_path)) self.assertTrue(self.env._logging_manager._sp_config is None) self.assertTrue(self.env._logging_manager._mp_config is None) self.assertTrue(self.env._logging_manager.log_config is None) self.add_params(self.traj) self.explore(self.traj) self.env.f_run(log_all_levels) self.assertFalse(os.path.isdir(log_path)) self.assertTrue(self.env._logging_manager._sp_config is None) self.assertTrue(self.env._logging_manager._mp_config is None) self.assertTrue(self.env._logging_manager.log_config is None) self.env.f_disable_logging() # pypet_path = os.path.abspath(os.path.dirname(pypet.pypetlogging)) # init_path = os.path.join(pypet_path, 'logging') # log_config = os.path.join(init_path, 'default.ini') @unittest.skipIf(platform.system() == 'Windows', 'Log file creation might fail under windows.') def test_logfile_creation_with_errors(self): # if not self.multiproc: # return self.make_env() self.add_params(self.traj) self.explore(self.traj) self.env.f_run(log_all_levels) if self.mode.multiproc: logging.getLogger('pypet.test').error('ttt') self.env.f_disable_logging() traj = self.env.v_traj log_path = get_log_path(traj) if self.mode.multiproc: if self.mode.use_pool: length = self.mode.ncores * 2 else: length = 2 * len(traj) if self.mode.wrap_mode == 'LOCK': length += 2 elif self.mode.wrap_mode == 'QUEUE': length += + 4 else: raise RuntimeError('You shall not pass!') else: length = 2 file_list = [file for file in os.listdir(log_path)] self.assertEqual(len(file_list), length) # assert that there are as many # files as runs plus main.txt and errors and warnings total_error_count = 0 total_store_count = 0 total_info_count = 0 total_retry_count = 0 for file in file_list: with open(os.path.join(log_path, file), mode='r') as fh: text = fh.read() count = text.count('INFO_Test!') total_info_count += count error_count = text.count('ERROR_Test!') total_error_count += error_count store_count = text.count('STORE_Test!') total_store_count += store_count retry_count = text.count('Retry') total_retry_count += retry_count if 'LOG.txt' == file: if self.mode.multiproc: self.assertEqual(count,0) self.assertEqual(store_count, 0) else: self.assertEqual(count, len(traj)) self.assertEqual(store_count, len(traj)) elif 'ERROR.txt' == file: self.assertEqual(count, 0) if self.mode.multiproc: self.assertEqual(error_count,0) self.assertEqual(store_count, 0) else: self.assertEqual(error_count, len(traj)) self.assertEqual(store_count, len(traj)) elif 'Queue' in file and 'ERROR' in file: self.assertEqual(store_count, len(traj)) elif 'Queue' in file and 'LOG' in file: self.assertEqual(store_count, len(traj)) elif 'LOG' in file: if self.mode.multiproc and self.mode.use_pool: self.assertGreaterEqual(count, 0) self.assertGreaterEqual(error_count, 0) else: self.assertEqual(count, 1) self.assertEqual(error_count, 1) if self.mode.wrap_mode == 'QUEUE': self.assertEqual(store_count, 0) else: self.assertEqual(store_count, 1) elif 'ERROR' in file: if self.mode.multiproc and self.mode.use_pool: self.assertEqual(count, 0) self.assertGreaterEqual(error_count, 1) else: self.assertEqual(count, 0) self.assertEqual(error_count, 1) if self.mode.wrap_mode == 'QUEUE': self.assertEqual(store_count, 0) else: self.assertEqual(store_count, 1) else: self.assertTrue(False, 'There`s a file in the log folder that does not ' 'belong there: %s' % str(file)) self.assertEqual(total_store_count, 2*len(traj)) self.assertEqual(total_error_count, 2*len(traj)) self.assertEqual(total_info_count, len(traj)) self.assertLess(total_retry_count, len(traj)) def test_file_renaming(self): traj_name = 'test' traj = Trajectory('test', add_time=False) traj.f_add_parameter('x', 42) traj.f_explore({'x': [1,2,3]}) rename_string = '$traj_$set_$run' solution_1 = 'test_run_set_ALL_run_ALL' solution_2 = 'test_run_set_00000_run_00000000' renaming_1 = rename_log_file(rename_string, traj) self.assertEqual(renaming_1, solution_1) traj.v_idx = 0 renaming_2 = rename_log_file(rename_string, traj) self.assertEqual(renaming_2, solution_2) @unittest.skipIf(platform.system() == 'Windows', 'Log file creation might fail under windows.') def test_logfile_old_way_creation_with_errors(self): # if not self.multiproc: # return del self.mode.__dict__['log_config'] self.make_env(logger_names = ('','pypet'), log_level=logging.ERROR, log_folder=make_temp_dir('logs')) self.add_params(self.traj) self.explore(self.traj) self.env.f_run(log_all_levels) if self.mode.multiproc: logging.getLogger('pypet.test').error('ttt') self.env.f_disable_logging() traj = self.env.v_traj log_path = get_log_path(traj) if self.mode.multiproc: if self.mode.use_pool: length = self.mode.ncores * 2 else: length = 2 * len(traj) if self.mode.wrap_mode == 'LOCK': length += 2 elif self.mode.wrap_mode == 'QUEUE': length += 4 else: raise RuntimeError('You shall not pass!') else: length = 2 file_list = [file for file in os.listdir(log_path)] self.assertEqual(len(file_list), length) # assert that there are as many # files as runs plus main.txt and errors and warnings total_error_count = 0 total_store_count = 0 total_info_count = 0 total_retry_count = 0 for file in file_list: with open(os.path.join(log_path, file), mode='r') as fh: text = fh.read() count = text.count('INFO_Test!') total_info_count += count error_count = text.count('ERROR_Test!') total_error_count += error_count store_count = text.count('STORE_Test!') total_store_count += store_count retry_count = text.count('Retry') total_retry_count += retry_count if 'LOG.txt' == file: if self.mode.multiproc: self.assertEqual(count,0) self.assertEqual(store_count, 0) else: self.assertEqual(count, 0) self.assertGreaterEqual(store_count, len(traj)) elif 'ERROR.txt' == file: self.assertEqual(count, 0) if self.mode.multiproc: self.assertEqual(error_count,0) self.assertEqual(store_count, 0) else: self.assertGreaterEqual(error_count, len(traj)) self.assertGreaterEqual(store_count, len(traj)) elif 'Queue' in file and 'ERROR' in file: self.assertGreaterEqual(store_count, len(traj)) elif 'Queue' in file and 'LOG' in file: self.assertGreaterEqual(store_count, len(traj)) elif 'LOG' in file: if self.mode.multiproc and self.mode.use_pool: self.assertEqual(count, 0) self.assertGreaterEqual(error_count, 0) else: self.assertEqual(count, 0) self.assertGreaterEqual(error_count, 1) if self.mode.wrap_mode == 'QUEUE': self.assertEqual(store_count, 0) else: self.assertGreaterEqual(store_count, 1) elif 'ERROR' in file: if self.mode.multiproc and self.mode.use_pool: self.assertEqual(count, 0) self.assertGreaterEqual(error_count, 0) else: self.assertEqual(count, 0) self.assertGreaterEqual(error_count, 1) if self.mode.wrap_mode == 'QUEUE': self.assertEqual(store_count, 0) else: self.assertGreaterEqual(store_count, 1) else: self.assertTrue(False, 'There`s a file in the log folder that does not ' 'belong there: %s' % str(file)) self.assertGreaterEqual(total_store_count, 2*len(traj)) self.assertGreaterEqual(total_error_count, 2*len(traj)) self.assertEqual(total_info_count, 0) self.assertLess(total_retry_count, len(traj)) @unittest.skipIf(platform.system() == 'Windows', 'Log file creation might fail under windows.') def test_logfile_old_way_disabling_mp_log(self): # if not self.multiproc: # return del self.mode.__dict__['log_config'] self.make_env(logger_names = ('','pypet'), log_level=logging.ERROR, log_folder=make_temp_dir('logs'), log_multiproc=False) self.add_params(self.traj) self.explore(self.traj) self.env.f_run(log_all_levels) if self.mode.multiproc: logging.getLogger('pypet.test').error('ttt') self.env.f_disable_logging() traj = self.env.v_traj log_path = get_log_path(traj) # if self.mode.multiproc: length = 2 file_list = [file for file in os.listdir(log_path)] self.assertEqual(len(file_list), length) # assert that there are as many # files as runs plus main.txt and errors and warnings # total_error_count = 0 # total_store_count = 0 # total_info_count = 0 # total_retry_count = 0 # # for file in file_list: # with open(os.path.join(log_path, file), mode='r') as fh: # text = fh.read() # count = text.count('INFO_Test!') # total_info_count += count # error_count = text.count('ERROR_Test!') # total_error_count += error_count # store_count = text.count('STORE_Test!') # total_store_count += store_count # retry_count = text.count('Retry') # total_retry_count += retry_count # if 'LOG.txt' == file: # if self.mode.multiproc: # self.assertEqual(count,0) # self.assertEqual(store_count, 0) # else: # self.assertEqual(count, 0) # self.assertGreaterEqual(store_count, len(traj)) # elif 'ERROR.txt' == file: # self.assertEqual(count, 0) # if self.mode.multiproc: # self.assertEqual(error_count,0) # self.assertEqual(store_count, 0) # else: # self.assertGreaterEqual(error_count, len(traj)) # self.assertGreaterEqual(store_count, len(traj)) # # elif 'Queue' in file and 'ERROR' in file: # self.assertGreaterEqual(store_count, len(traj)) # elif 'Queue' in file and 'LOG' in file: # self.assertGreaterEqual(store_count, len(traj)) # elif 'LOG' in file: # if self.mode.multiproc and self.mode.use_pool: # self.assertEqual(count, 0) # self.assertGreaterEqual(error_count, 0) # else: # self.assertEqual(count, 0) # self.assertGreaterEqual(error_count, 1) # if self.mode.wrap_mode == 'QUEUE': # self.assertEqual(store_count, 0) # else: # self.assertGreaterEqual(store_count, 1) # elif 'ERROR' in file: # if self.mode.multiproc and self.mode.use_pool: # self.assertEqual(count, 0) # self.assertGreaterEqual(error_count, 0) # else: # self.assertEqual(count, 0) # self.assertGreaterEqual(error_count, 1) # if self.mode.wrap_mode == 'QUEUE': # self.assertEqual(store_count, 0) # else: # self.assertGreaterEqual(store_count, 1) # else: # self.assertTrue(False, 'There`s a file in the log folder that does not ' # 'belong there: %s' % str(file)) # self.assertGreaterEqual(total_store_count, 2*len(traj)) # self.assertGreaterEqual(total_error_count, 2*len(traj)) # self.assertEqual(total_info_count, 0) # self.assertLess(total_retry_count, len(traj)) @unittest.skipIf(platform.system() == 'Windows', 'Log file creation might fail under windows.') def test_logging_stdout(self): filename = 'teststdoutlog.hdf5' filename = make_temp_dir(filename) folder = make_temp_dir('logs') env = Environment(trajectory=make_trajectory_name(self), filename=filename, log_config=get_log_config(), # log_levels=logging.CRITICAL, # needed for the test log_stdout=('STDOUT', 50), #log_folder=folder ) env.f_run(log_error) traj = env.v_traj path = get_log_path(traj) mainstr = 'sTdOuTLoGGinG' print(mainstr) env.f_disable_logging() mainfilename = os.path.join(path, 'LOG.txt') with open(mainfilename, mode='r') as mainf: full_text = mainf.read() self.assertTrue(mainstr in full_text) self.assertTrue('4444444' not in full_text) self.assertTrue('DEBUG' not in full_text) def test_logging_show_progress(self): self.make_env(log_config=get_log_config(), # log_folder=make_temp_dir('logs'), # log_levels=logging.ERROR, report_progress=(3, 'progress', 40)) self.add_params(self.traj) self.explore(self.traj) self.env.f_run(log_all_levels) self.env.f_disable_logging() traj = self.env.v_traj path = get_log_path(traj) mainfilename = os.path.join(path, 'LOG.txt') with open(mainfilename, mode='r') as mainf: full_text = mainf.read() progress = 'PROGRESS: Finished' self.assertTrue(progress in full_text) bar = '[==' self.assertTrue(bar in full_text) def test_logging_show_progress_print(self): self.make_env(log_config=get_log_config(), log_stdout=('prostdout', 50), report_progress=(3, 'print')) self.add_params(self.traj) self.explore(self.traj) self.env.f_run(log_all_levels) self.env.f_disable_logging() path = get_log_path(self.env.v_traj) mainfilename = os.path.join(path, 'LOG.txt') with open(mainfilename, mode='r') as mainf: full_text = mainf.read() progress = 'prostdout CRITICAL PROGRESS: Finished' self.assertTrue(progress in full_text) bar = '[==' self.assertIn(bar, full_text)
z=traj.x*traj.y traj.f_add_result('z',z=z, comment='I am the product of two reals!') # Create an environment that handles running. # Let's enable multiprocessing with 2 workers. env = Environment(trajectory='Example_04_MP', filename='experiments/example_04/HDF5/example_04.hdf5', file_title='Example_04_MP', log_folder='experiments/example_04/LOGS/', comment = 'Multiprocessing example!', multiproc=True, ncores=2, use_pool=True, wrap_mode=pypetconstants.WRAP_MODE_LOCK) # Get the trajectory from the environment traj = env.v_trajectory # Add both parameters traj.f_add_parameter('x', 1.0, comment='I am the first dimension!') traj.f_add_parameter('y', 1.0, comment='I am the second dimension!') # Explore the parameters with a cartesian product, but we want to explore a bit more traj.f_explore(cartesian_product({'x':[float(x) for x in range(15)], 'y':[float(y) for y in range(15)]})) # Run the simulation env.f_run(multiply)
def main(): filename = os.path.join('hdf5', 'example_06.hdf5') env = Environment(trajectory='Example_06_Euler_Integration', filename=filename, file_title='Example_06_Euler_Integration', comment = 'Go for Euler!') traj = env.v_trajectory # 1st a) phase parameter addition # Remember we have some control flow in the `add_parameters` function, the default parameter # set we choose is the `'diff_lorenz'` one, but we want to deviate from that and use the # `'diff_roessler'`. # In order to do that we can preset the corresponding name parameter to change the # control flow: traj.f_preset_parameter('diff_name', 'diff_roessler') # If you erase this line, you will get # again the lorenz attractor add_parameters(traj) # 1st b) phase preparation # Let's check which function we want to use if traj.diff_name=='diff_lorenz': diff_eq = diff_lorenz elif traj.diff_name=='diff_roessler': diff_eq = diff_roessler else: raise ValueError('I don\'t know what %s is.' % traj.diff_name) # And add the source code of the function as a derived parameter. traj.f_add_derived_parameter(FunctionParameter, 'diff_eq', diff_eq, comment='Source code of our equation!') # We want to explore some initial conditions traj.f_explore({'initial_conditions' : [ np.array([0.01,0.01,0.01]), np.array([2.02,0.02,0.02]), np.array([42.0,4.2,0.42]) ]}) # 3 different conditions are enough for now # 2nd phase let's run the experiment # We pass 'euler_scheme' as our top-level simulation function and # the Roessler function as an additional argument env.f_run(euler_scheme, diff_eq) # Again no post-processing # 4th phase analysis. # I would recommend to do the analysis completely independent from the simulation # but for simplicity let's do it here. # We won't reload the trajectory this time but simply update the skeleton traj.f_load_skeleton() #For the fun of it, let's print the source code print('\n ---------- The source code of your function ---------- \n %s' % traj.diff_eq) # Let's get the exploration array: initial_conditions_exploration_array = traj.f_get('initial_conditions').f_get_range() # Now let's plot our simulated equations for the different initial conditions. # We will iterate through the run names for idx, run_name in enumerate(traj.f_get_run_names()): # Get the result of run idx from the trajectory euler_result = traj.results.f_get(run_name).euler_evolution # Now we manually need to load the result. Actually the results are not so large and we # could load them all at once, but for demonstration we do as if they were huge: traj.f_load_item(euler_result) euler_data = euler_result.data # Plot fancy 3d plot fig = plt.figure(idx) ax = fig.gca(projection='3d') x = euler_data[:,0] y = euler_data[:,1] z = euler_data[:,2] ax.plot(x, y, z, label='Initial Conditions: %s' % str(initial_conditions_exploration_array[idx])) plt.legend() plt.show() # Now we free the data again (because we assume its huuuuuuge): del euler_data euler_result.f_empty() # Finally disable logging and close all log-files env.f_disable_logging()
def main(): filename = os.path.join('hdf5', 'example_05.hdf5') env = Environment(trajectory='Example_05_Euler_Integration', filename=filename, file_title='Example_05_Euler_Integration', comment='Go for Euler!') traj = env.v_trajectory trajectory_name = traj.v_name # 1st a) phase parameter addition add_parameters(traj) # 1st b) phase preparation # We will add the differential equation (well, its source code only) as a derived parameter traj.f_add_derived_parameter(FunctionParameter,'diff_eq', diff_lorenz, comment='Source code of our equation!') # We want to explore some initial conditions traj.f_explore({'initial_conditions' : [ np.array([0.01,0.01,0.01]), np.array([2.02,0.02,0.02]), np.array([42.0,4.2,0.42]) ]}) # 3 different conditions are enough for an illustrative example # 2nd phase let's run the experiment # We pass `euler_scheme` as our top-level simulation function and # the Lorenz equation 'diff_lorenz' as an additional argument env.f_run(euler_scheme, diff_lorenz) # We don't have a 3rd phase of post-processing here # 4th phase analysis. # I would recommend to do post-processing completely independent from the simulation, # but for simplicity let's do it here. # Let's assume that we start all over again and load the entire trajectory new. # Yet, there is an error within this approach, do you spot it? del traj traj = Trajectory(filename=filename) # We will only fully load parameters and derived parameters. # Results will be loaded manually later on. try: # However, this will fail because our trajectory does not know how to # build the FunctionParameter. You have seen this coming, right? traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2, load_results=1) except ImportError as e: print('That did\'nt work, I am sorry: %s ' % str(e)) # Ok, let's try again but this time with adding our parameter to the imports traj = Trajectory(filename=filename, dynamically_imported_classes=FunctionParameter) # Now it works: traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2, load_results=1) #For the fun of it, let's print the source code print('\n ---------- The source code of your function ---------- \n %s' % traj.diff_eq) # Let's get the exploration array: initial_conditions_exploration_array = traj.f_get('initial_conditions').f_get_range() # Now let's plot our simulated equations for the different initial conditions: # We will iterate through the run names for idx, run_name in enumerate(traj.f_get_run_names()): #Get the result of run idx from the trajectory euler_result = traj.results.f_get(run_name).euler_evolution # Now we manually need to load the result. Actually the results are not so large and we # could load them all at once. But for demonstration we do as if they were huge: traj.f_load_item(euler_result) euler_data = euler_result.data #Plot fancy 3d plot fig = plt.figure(idx) ax = fig.gca(projection='3d') x = euler_data[:,0] y = euler_data[:,1] z = euler_data[:,2] ax.plot(x, y, z, label='Initial Conditions: %s' % str(initial_conditions_exploration_array[idx])) plt.legend() plt.show() # Now we free the data again (because we assume its huuuuuuge): del euler_data euler_result.f_empty() # You have to click through the images to stop the example_05 module! # Finally disable logging and close all log-files env.f_disable_logging()
traj.f_add_result('positions', sim.positions, comment='End positions of particles') traj.f_add_result('t', sim.t, comment='duration of flight') env = Environment(trajectory='FanSimulation', filename='./pypet/', large_overview_tables=True, add_time=True, multiproc=False, ncores=6, log_config='DEFAULT') traj = env.v_trajectory add_parameters(traj, dt=1e-2) explore_dict = {'vent_radius':[0.1, 0.5, 1.0], 'vmax':[10, 50, 100], 'incline':[0.1, 1.0, 5.0]} to_explore = cartesian_product(explore_dict) traj.f_explore(to_explore) env.f_run(run_simulation) env.f_disable_logging() # In[ ]:
file_title='Example_01_First_Steps', log_folder='experiments/example_01/LOGS/', comment='The first example!', continuable=False, # We have shared data in terms of a multiprocessing list, # so we CANNOT use the continue feature. multiproc=True, ncores=2) # The environment has created a trajectory container for us traj = env.v_trajectory # Add both parameters traj.f_add_parameter('x', 1, comment='I am the first dimension!') traj.f_add_parameter('y', 1, comment='I am the second dimension!') # Explore the parameters with a cartesian product traj.f_explore(cartesian_product({'x':[1,2,3,4], 'y':[6,7,8]})) # We want a shared list where we can put all out results in. We use a manager for this: result_list = mp.Manager().list() # Let's make some space for potential results result_list[:] =[0 for _dummy in range(len(traj))] # Run the simulation env.f_run(multiply, result_list) # Now we want to store the final list as numpy array traj.f_add_result('z', np.array(result_list)) # Finally let's print the result to see that it worked print traj.z
class LoggingTest(TrajectoryComparator): tags = 'integration', 'environment', 'logging' def setUp(self): root = logging.getLogger() for logger in itools.chain(root.manager.loggerDict.values(), [root]): if hasattr(logger, 'handlers'): for handler in logger.handlers: if hasattr(handler, 'flush'): handler.flush() if hasattr(handler, 'close'): handler.close() logger.handlers = [] if hasattr(logger, 'setLevel'): logger.setLevel(logging.NOTSET) self.set_mode() def tearDown(self): super(LoggingTest, self).tearDown() def set_mode(self): self.mode = Dummy() self.mode.wrap_mode = 'LOCK' self.mode.multiproc = False self.mode.ncores = 1 self.mode.use_pool=True self.mode.pandas_format='fixed' self.mode.pandas_append=False self.mode.complib = 'blosc' self.mode.complevel=9 self.mode.shuffle=True self.mode.fletcher32 = False self.mode.encoding = 'utf8' self.mode.log_stdout=False self.mode.log_config=get_log_config() def make_env(self, **kwargs): self.mode.__dict__.update(kwargs) filename = 'log_testing.hdf5' self.filename = make_temp_dir(filename) self.traj_name = make_trajectory_name(self) self.env = Environment(trajectory=self.traj_name, filename=self.filename, **self.mode.__dict__) self.traj = self.env.v_traj def add_params(self, traj): traj.par.p1 = Parameter('', 42, 'Hey') traj.f_apar('g1.p2', 145, comment='Test') def explore(self, traj): traj.f_explore({'p1': range(7)}) # @unittest.skipIf(platform.system() == 'Windows', 'Log file creation might fail under windows.') def test_logfile_creation_normal(self): # if not self.multiproc: # return self.make_env() self.add_params(self.traj) self.explore(self.traj) self.env.f_run(log_wo_error_levels) self.env.f_disable_logging() traj = self.env.v_traj log_path = get_log_path(traj) if self.mode.multiproc: if self.mode.use_pool: length = self.mode.ncores * 2 else: length = 2 * len(traj) if self.mode.wrap_mode == 'LOCK': length += 2 elif self.mode.wrap_mode == 'QUEUE': length += 4 else: raise RuntimeError('You shall not pass!') else: length = 2 file_list = [file for file in os.listdir(log_path)] self.assertEqual(len(file_list), length) # assert that there are as many # files as runs plus main.txt and errors and warnings total_error_count = 0 total_store_count = 0 total_info_count = 0 total_retry_count = 0 for file in file_list: with open(os.path.join(log_path, file), mode='r') as fh: text = fh.read() if len(text) == 0: continue count = text.count('INFO_Test!') total_info_count += count error_count = text.count('ERROR_Test!') total_error_count += error_count store_count = text.count('STORE_Test!') total_store_count += store_count retry_count = text.count('Retry') total_retry_count += retry_count if 'LOG.txt' == file: if self.mode.multiproc: self.assertEqual(count,0) self.assertEqual(store_count, 0) else: self.assertEqual(count, len(traj)) self.assertEqual(store_count, len(traj)) elif 'ERROR' in file: full_path = os.path.join(log_path, file) filesize = os.path.getsize(full_path) with open(full_path) as fh: text = fh.read() if 'Retry' not in text: self.assertEqual(filesize, 0) elif 'Queue' in file: self.assertEqual(store_count, len(traj)) elif 'LOG' in file: if self.mode.multiproc and self.mode.use_pool: self.assertGreaterEqual(count, 0, '%d < 1 for file %s' % (count, file)) else: self.assertEqual(count, 1) if self.mode.wrap_mode == 'QUEUE': self.assertEqual(store_count, 0) else: self.assertEqual(store_count, 1) else: self.assertTrue(False, 'There`s a file in the log folder that does not ' 'belong there: %s' % str(file)) self.assertEqual(total_store_count, len(traj)) self.assertEqual(total_error_count, 0) self.assertEqual(total_info_count, len(traj)) self.assertLess(total_retry_count, len(traj)) def test_throw_error_when_specifying_config_and_old_method(self): with self.assertRaises(ValueError): self.make_env(log_config=None, logger_names='test') def test_disable(self): # if not self.multiproc: # return self.make_env(log_config=None) traj = self.env.v_traj log_path = get_log_path(traj) self.assertFalse(os.path.isdir(log_path)) self.assertTrue(self.env._logging_manager._sp_config is None) self.assertTrue(self.env._logging_manager._mp_config is None) self.assertTrue(self.env._logging_manager.log_config is None) self.add_params(self.traj) self.explore(self.traj) self.env.f_run(log_all_levels) self.assertFalse(os.path.isdir(log_path)) self.assertTrue(self.env._logging_manager._sp_config is None) self.assertTrue(self.env._logging_manager._mp_config is None) self.assertTrue(self.env._logging_manager.log_config is None) self.env.f_disable_logging() # pypet_path = os.path.abspath(os.path.dirname(pypet.pypetlogging)) # init_path = os.path.join(pypet_path, 'logging') # log_config = os.path.join(init_path, 'default.ini') # @unittest.skipIf(platform.system() == 'Windows', 'Log file creation might fail under windows.') def test_logfile_creation_with_errors(self): # if not self.multiproc: # return self.make_env() self.add_params(self.traj) self.explore(self.traj) self.env.f_run(log_all_levels) if self.mode.multiproc: logging.getLogger('pypet.test').error('ttt') self.env.f_disable_logging() traj = self.env.v_traj log_path = get_log_path(traj) if self.mode.multiproc: if self.mode.use_pool: length = self.mode.ncores * 2 else: length = 2 * len(traj) if self.mode.wrap_mode == 'LOCK': length += 2 elif self.mode.wrap_mode == 'QUEUE': length += + 4 else: raise RuntimeError('You shall not pass!') else: length = 2 file_list = [file for file in os.listdir(log_path)] self.assertEqual(len(file_list), length) # assert that there are as many # files as runs plus main.txt and errors and warnings total_error_count = 0 total_store_count = 0 total_info_count = 0 total_retry_count = 0 for file in file_list: with open(os.path.join(log_path, file), mode='r') as fh: text = fh.read() if len(text) == 0: continue count = text.count('INFO_Test!') total_info_count += count error_count = text.count('ERROR_Test!') total_error_count += error_count store_count = text.count('STORE_Test!') total_store_count += store_count retry_count = text.count('Retry') total_retry_count += retry_count if 'LOG.txt' == file: if self.mode.multiproc: self.assertEqual(count,0) self.assertEqual(store_count, 0) else: self.assertEqual(count, len(traj)) self.assertEqual(store_count, len(traj)) elif 'ERROR.txt' == file: self.assertEqual(count, 0) if self.mode.multiproc: self.assertEqual(error_count,0) self.assertEqual(store_count, 0) else: self.assertEqual(error_count, len(traj)) self.assertEqual(store_count, len(traj)) elif 'Queue' in file and 'ERROR' in file: self.assertEqual(store_count, len(traj)) elif 'Queue' in file and 'LOG' in file: self.assertEqual(store_count, len(traj)) elif 'LOG' in file: if self.mode.multiproc and self.mode.use_pool: self.assertGreaterEqual(count, 0) self.assertGreaterEqual(error_count, 0) else: self.assertEqual(count, 1) self.assertEqual(error_count, 1) if self.mode.wrap_mode == 'QUEUE': self.assertEqual(store_count, 0) else: self.assertEqual(store_count, 1) elif 'ERROR' in file: if self.mode.multiproc and self.mode.use_pool: self.assertEqual(count, 0) self.assertGreaterEqual(error_count, 1) else: self.assertEqual(count, 0) self.assertEqual(error_count, 1) if self.mode.wrap_mode == 'QUEUE': self.assertEqual(store_count, 0) else: self.assertEqual(store_count, 1) else: self.assertTrue(False, 'There`s a file in the log folder that does not ' 'belong there: %s' % str(file)) self.assertEqual(total_store_count, 2*len(traj)) self.assertEqual(total_error_count, 2*len(traj)) self.assertEqual(total_info_count, len(traj)) self.assertLess(total_retry_count, len(traj)) def test_file_renaming(self): traj_name = 'test' traj = Trajectory('test', add_time=False) traj.f_add_parameter('x', 42) traj.f_explore({'x': [1,2,3]}) rename_string = '$traj_$set_$run' solution_1 = 'test_run_set_ALL_run_ALL' solution_2 = 'test_run_set_00000_run_00000000' renaming_1 = rename_log_file(rename_string, traj) self.assertEqual(renaming_1, solution_1) traj.v_idx = 0 renaming_2 = rename_log_file(rename_string, traj) self.assertEqual(renaming_2, solution_2) # @unittest.skipIf(platform.system() == 'Windows', 'Log file creation might fail under windows.') def test_logfile_old_way_creation_with_errors(self): # if not self.multiproc: # return del self.mode.__dict__['log_config'] self.make_env(logger_names = ('','pypet'), log_level=logging.ERROR, log_folder=make_temp_dir('logs')) self.add_params(self.traj) self.explore(self.traj) self.env.f_run(log_all_levels) if self.mode.multiproc: logging.getLogger('pypet.test').error('ttt') self.env.f_disable_logging() traj = self.env.v_traj log_path = get_log_path(traj) if self.mode.multiproc: if self.mode.use_pool: length = self.mode.ncores * 2 else: length = 2 * len(traj) if self.mode.wrap_mode == 'LOCK': length += 2 elif self.mode.wrap_mode == 'QUEUE': length += 4 else: raise RuntimeError('You shall not pass!') else: length = 2 file_list = [file for file in os.listdir(log_path)] self.assertEqual(len(file_list), length) # assert that there are as many # files as runs plus main.txt and errors and warnings total_error_count = 0 total_store_count = 0 total_info_count = 0 total_retry_count = 0 for file in file_list: with open(os.path.join(log_path, file), mode='r') as fh: text = fh.read() if len(text) == 0: continue count = text.count('INFO_Test!') total_info_count += count error_count = text.count('ERROR_Test!') total_error_count += error_count store_count = text.count('STORE_Test!') total_store_count += store_count retry_count = text.count('Retry') total_retry_count += retry_count if 'LOG.txt' == file: if self.mode.multiproc: self.assertEqual(count,0) self.assertEqual(store_count, 0) else: self.assertEqual(count, 0) self.assertGreaterEqual(store_count, len(traj)) elif 'ERROR.txt' == file: self.assertEqual(count, 0) if self.mode.multiproc: self.assertEqual(error_count,0) self.assertEqual(store_count, 0) else: self.assertGreaterEqual(error_count, len(traj)) self.assertGreaterEqual(store_count, len(traj)) elif 'Queue' in file and 'ERROR' in file: self.assertGreaterEqual(store_count, len(traj)) elif 'Queue' in file and 'LOG' in file: self.assertGreaterEqual(store_count, len(traj)) elif 'LOG' in file: if self.mode.multiproc and self.mode.use_pool: self.assertEqual(count, 0) self.assertGreaterEqual(error_count, 0) else: self.assertEqual(count, 0) self.assertGreaterEqual(error_count, 1) if self.mode.wrap_mode == 'QUEUE': self.assertEqual(store_count, 0) else: self.assertGreaterEqual(store_count, 1) elif 'ERROR' in file: if self.mode.multiproc and self.mode.use_pool: self.assertEqual(count, 0) self.assertGreaterEqual(error_count, 0) else: self.assertEqual(count, 0) self.assertGreaterEqual(error_count, 1) if self.mode.wrap_mode == 'QUEUE': self.assertEqual(store_count, 0) else: self.assertGreaterEqual(store_count, 1) else: self.assertTrue(False, 'There`s a file in the log folder that does not ' 'belong there: %s' % str(file)) self.assertGreaterEqual(total_store_count, 2*len(traj)) self.assertGreaterEqual(total_error_count, 2*len(traj)) self.assertEqual(total_info_count, 0) self.assertLess(total_retry_count, len(traj)) # @unittest.skipIf(platform.system() == 'Windows', 'Log file creation might fail under windows.') def test_logfile_old_way_disabling_mp_log(self): # if not self.multiproc: # return del self.mode.__dict__['log_config'] self.make_env(logger_names = ('','pypet'), log_level=logging.ERROR, log_folder=make_temp_dir('logs'), log_multiproc=False) self.add_params(self.traj) self.explore(self.traj) self.env.f_run(log_all_levels) if self.mode.multiproc: logging.getLogger('pypet.test').error('ttt') self.env.f_disable_logging() traj = self.env.v_traj log_path = get_log_path(traj) # if self.mode.multiproc: length = 2 file_list = [file for file in os.listdir(log_path)] self.assertEqual(len(file_list), length) # assert that there are as many # files as runs plus main.txt and errors and warnings # total_error_count = 0 # total_store_count = 0 # total_info_count = 0 # total_retry_count = 0 # # for file in file_list: # with open(os.path.join(log_path, file), mode='r') as fh: # text = fh.read() # count = text.count('INFO_Test!') # total_info_count += count # error_count = text.count('ERROR_Test!') # total_error_count += error_count # store_count = text.count('STORE_Test!') # total_store_count += store_count # retry_count = text.count('Retry') # total_retry_count += retry_count # if 'LOG.txt' == file: # if self.mode.multiproc: # self.assertEqual(count,0) # self.assertEqual(store_count, 0) # else: # self.assertEqual(count, 0) # self.assertGreaterEqual(store_count, len(traj)) # elif 'ERROR.txt' == file: # self.assertEqual(count, 0) # if self.mode.multiproc: # self.assertEqual(error_count,0) # self.assertEqual(store_count, 0) # else: # self.assertGreaterEqual(error_count, len(traj)) # self.assertGreaterEqual(store_count, len(traj)) # # elif 'Queue' in file and 'ERROR' in file: # self.assertGreaterEqual(store_count, len(traj)) # elif 'Queue' in file and 'LOG' in file: # self.assertGreaterEqual(store_count, len(traj)) # elif 'LOG' in file: # if self.mode.multiproc and self.mode.use_pool: # self.assertEqual(count, 0) # self.assertGreaterEqual(error_count, 0) # else: # self.assertEqual(count, 0) # self.assertGreaterEqual(error_count, 1) # if self.mode.wrap_mode == 'QUEUE': # self.assertEqual(store_count, 0) # else: # self.assertGreaterEqual(store_count, 1) # elif 'ERROR' in file: # if self.mode.multiproc and self.mode.use_pool: # self.assertEqual(count, 0) # self.assertGreaterEqual(error_count, 0) # else: # self.assertEqual(count, 0) # self.assertGreaterEqual(error_count, 1) # if self.mode.wrap_mode == 'QUEUE': # self.assertEqual(store_count, 0) # else: # self.assertGreaterEqual(store_count, 1) # else: # self.assertTrue(False, 'There`s a file in the log folder that does not ' # 'belong there: %s' % str(file)) # self.assertGreaterEqual(total_store_count, 2*len(traj)) # self.assertGreaterEqual(total_error_count, 2*len(traj)) # self.assertEqual(total_info_count, 0) # self.assertLess(total_retry_count, len(traj)) # @unittest.skipIf(platform.system() == 'Windows', 'Log file creation might fail under windows.') def test_logging_stdout(self): filename = 'teststdoutlog.hdf5' filename = make_temp_dir(filename) folder = make_temp_dir('logs') env = Environment(trajectory=make_trajectory_name(self), filename=filename, log_config=get_log_config(), # log_levels=logging.CRITICAL, # needed for the test log_stdout=('STDOUT', 50), #log_folder=folder ) env.f_run(log_error) traj = env.v_traj path = get_log_path(traj) mainstr = 'sTdOuTLoGGinG' print(mainstr) env.f_disable_logging() mainfilename = os.path.join(path, 'LOG.txt') with open(mainfilename, mode='r') as mainf: full_text = mainf.read() self.assertTrue(mainstr in full_text) self.assertTrue('4444444' not in full_text) self.assertTrue('DEBUG' not in full_text) def test_logging_show_progress(self): self.make_env(log_config=get_log_config(), # log_folder=make_temp_dir('logs'), # log_levels=logging.ERROR, report_progress=(3, 'progress', 40)) self.add_params(self.traj) self.explore(self.traj) self.env.f_run(log_all_levels) self.env.f_disable_logging() traj = self.env.v_traj path = get_log_path(traj) mainfilename = os.path.join(path, 'LOG.txt') with open(mainfilename, mode='r') as mainf: full_text = mainf.read() progress = 'PROGRESS: Finished' self.assertTrue(progress in full_text) bar = '[==' self.assertTrue(bar in full_text) def test_logging_show_progress_print(self): self.make_env(log_config=get_log_config(), log_stdout=('prostdout', 50), report_progress=(3, 'print')) self.add_params(self.traj) self.explore(self.traj) self.env.f_run(log_all_levels) self.env.f_disable_logging() path = get_log_path(self.env.v_traj) mainfilename = os.path.join(path, 'LOG.txt') with open(mainfilename, mode='r') as mainf: full_text = mainf.read() progress = 'prostdout CRITICAL PROGRESS: Finished' self.assertTrue(progress in full_text) bar = '[==' self.assertIn(bar, full_text)
from pypet import Environment, cartesian_product def multiply(traj): """Example of a sophisticated numerical experiment that involves multiplying two integer values. :param traj: Trajectory containing the parameters in a particular combination, it also serves as a container for results. """ z = traj.x * traj.y traj.f_add_result('z', z, comment='Result of x*y') # Create an environment that handles running the experiment env = Environment(trajectory='Multiplication', filename='multiply.hdf5', comment='A simulation of multiplication') # The environment provides a trajectory container for us traj = env.v_trajectory # Add two parameters, both with default value 0 traj.f_add_parameter('x', 0, comment='First dimension') traj.f_add_parameter('y', 0, comment='Second dimension') # Explore the Cartesian product of x in {1,2,3,4} and y in {6,7,8} traj.f_explore(cartesian_product({'x': [1, 2, 3, 4], 'y': [6, 7, 8]})) # Run simulation function `multiply` with all parameter combinations env.f_run(multiply)