def test_partial_loading(self): traj = Trajectory(name='TestPartial', filename=make_temp_dir('testpartially.hdf5')) res = traj.f_add_result('mytest.test', a='b', c='d') traj.f_store() traj.f_remove_child('results', recursive=True) traj.f_load_skeleton() traj.f_load_item(traj.test, load_only=['a', 'x']) self.assertTrue('a' in traj.test) self.assertTrue('c' not in traj.test) traj.f_remove_child('results', recursive=True) traj.f_load_skeleton() load_except= ['c', 'd'] traj.f_load_item(traj.test, load_except=load_except) self.assertTrue(len(load_except)==2) self.assertTrue('a' in traj.test) self.assertTrue('c' not in traj.test) with self.assertRaises(ValueError): traj.f_load_item(traj.test, load_except=['x'], load_only=['y'])
def test_partial_loading(self): traj = Trajectory(name='TestPartial', filename=make_temp_dir('testpartially.hdf5'), add_time=True) res = traj.f_add_result('mytest.test', a='b', c='d') traj.f_store() traj.f_remove_child('results', recursive=True) traj.f_load_skeleton() traj.f_load_item(traj.test, load_only=['a', 'x']) self.assertTrue('a' in traj.test) self.assertTrue('c' not in traj.test) traj.f_remove_child('results', recursive=True) traj.f_load_skeleton() load_except= ['c', 'd'] traj.f_load_item(traj.test, load_except=load_except) self.assertTrue(len(load_except)==2) self.assertTrue('a' in traj.test) self.assertTrue('c' not in traj.test) with self.assertRaises(ValueError): traj.f_load_item(traj.test, load_except=['x'], load_only=['y'])
def test_conversions(self): filename = make_temp_dir("hdf5manipulation.hdf5") traj = Trajectory(name=make_trajectory_name(self), filename=filename) trajname = traj.v_name traj.v_standard_result = SharedResult traj.f_store(only_init=True) traj.f_add_result("shared_data") thedata = np.zeros((1000, 1000)) myarray = SharedArray("array", traj.shared_data, trajectory=traj) traj.shared_data["array"] = myarray mytable = SharedTable("t1", traj.shared_data, trajectory=traj) traj.shared_data["t1"] = mytable # mytable2 = SharedTableResult('h.t2', trajectory=traj) # mytable3 = SharedTableResult('jjj.t3', trajectory=traj) dadict = {"hi": [1, 2, 3, 4, 5], "shu": ["bi", "du", "da", "ha", "hui"]} dadict2 = {"answer": [42]} res = traj.f_add_result("shared.dfs") res["df"] = SharedPandasFrame() res["df"].create_shared_data(data=pd.DataFrame(dadict), trajectory=traj) frame = SharedPandasFrame("df1", traj.f_get("shared.dfs"), trajectory=traj) frame.create_shared_data(data=pd.DataFrame(dadict2)) res["df1"] = frame traj.f_add_result("mylist", [1, 2, 3]) traj.f_add_result("my.mytuple", k=(1, 2, 3), wa=42) traj.f_add_result("my.myarray", np.zeros((50, 50))) traj.f_add_result("my.myframe", data=pd.DataFrame(dadict2)) traj.f_add_result("my.mytable", ObjectTable(data=dadict2)) myarray.create_shared_data(data=thedata) mytable.create_shared_data(first_row={"hi": compat.tobytes("hi"), "huhu": np.ones(3)}) traj.f_store() data = myarray.read() arr = myarray.get_data_node() self.assertTrue(np.all(data == thedata)) with StorageContextManager(traj) as cm: myarray[2, 2] = 10 data = myarray.read() self.assertTrue(data[2, 2] == 10) self.assertTrue(data[2, 2] == 10) self.assertFalse(traj.v_storage_service.is_open) traj = load_trajectory(name=trajname, filename=filename, load_all=2, dynamic_imports=SharedResult) make_ordinary_result(traj.shared_data, "array", trajectory=traj) array = traj.shared_data.array self.assertTrue(isinstance(array, np.ndarray)) thedata[2, 2] = 10 self.assertTrue(np.all(array == thedata)) make_ordinary_result(traj.shared_data, "t1", trajectory=traj) t1 = traj.shared_data.t1 self.assertTrue(isinstance(t1, ObjectTable)) # self.assertTrue(np.all(t1["huhu"][0] == np.ones(3))) dfs = traj.shared.dfs make_ordinary_result(traj.shared.dfs, "df", trajectory=traj) theframe = dfs.f_get("df") self.assertTrue(isinstance(dfs, Result)) self.assertTrue(isinstance(theframe, pd.DataFrame)) self.assertTrue(theframe["hi"][0] == 1) listres = traj.f_get("mylist") listres = make_shared_result(listres, 0, trajectory=traj) with StorageContextManager(traj) as cm: self.assertTrue(listres[0][2] == 3) listres[0][0] = 4 self.assertTrue(listres[0][0] == 4) listres = make_ordinary_result(listres, 0, trajectory=traj) traj = load_trajectory(name=trajname, filename=filename, load_all=2, dynamic_imports=SharedResult) mylist = traj.mylist self.assertTrue(isinstance(listres, Result)) self.assertTrue(mylist[0] == 4) self.assertTrue(isinstance(mylist, list)) mytuple = traj.mytuple with self.assertRaises(AttributeError): mytuple = make_shared_result(mytuple, "mylist", traj, new_class=SharedArray) mytuple = make_shared_result(mytuple, "k", traj, new_class=SharedArray) self.assertTrue(mytuple.k[1] == 2) mytuple = make_ordinary_result(mytuple, "k", trajectory=traj) self.assertTrue(isinstance(mytuple.k, tuple)) self.assertTrue(mytuple.k[2] == 3) myframe = traj.myframe myframe = make_shared_result(myframe, "data", traj) theframe = myframe.data.read() self.assertTrue(theframe["answer"][0] == 42) myframe = make_ordinary_result(myframe, "data", trajectory=traj) traj.f_load_item(myframe) self.assertTrue(myframe.data["answer"][0] == 42) mytable = traj.f_get("mytable") mytable = make_shared_result(mytable, 0, traj) self.assertTrue(isinstance(mytable[0], SharedTable)) rows = mytable.mytable.read() self.assertTrue(rows[0][0] == 42) mytable = make_ordinary_result(mytable, 0, trajectory=traj) self.assertTrue(isinstance(mytable, Result)) self.assertTrue(mytable[0]["answer"][0] == 42)
def main(): filename = os.path.join('hdf5', 'example_05.hdf5') env = Environment(trajectory='Example_05_Euler_Integration', filename=filename, file_title='Example_05_Euler_Integration', comment='Go for Euler!') traj = env.v_trajectory trajectory_name = traj.v_name # 1st a) phase parameter addition add_parameters(traj) # 1st b) phase preparation # We will add the differential equation (well, its source code only) as a derived parameter traj.f_add_derived_parameter(FunctionParameter,'diff_eq', diff_lorenz, comment='Source code of our equation!') # We want to explore some initial conditions traj.f_explore({'initial_conditions' : [ np.array([0.01,0.01,0.01]), np.array([2.02,0.02,0.02]), np.array([42.0,4.2,0.42]) ]}) # 3 different conditions are enough for an illustrative example # 2nd phase let's run the experiment # We pass `euler_scheme` as our top-level simulation function and # the Lorenz equation 'diff_lorenz' as an additional argument env.f_run(euler_scheme, diff_lorenz) # We don't have a 3rd phase of post-processing here # 4th phase analysis. # I would recommend to do post-processing completely independent from the simulation, # but for simplicity let's do it here. # Let's assume that we start all over again and load the entire trajectory new. # Yet, there is an error within this approach, do you spot it? del traj traj = Trajectory(filename=filename) # We will only fully load parameters and derived parameters. # Results will be loaded manually later on. try: # However, this will fail because our trajectory does not know how to # build the FunctionParameter. You have seen this coming, right? traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2, load_results=1) except ImportError as e: print('That did\'nt work, I am sorry: %s ' % str(e)) # Ok, let's try again but this time with adding our parameter to the imports traj = Trajectory(filename=filename, dynamically_imported_classes=FunctionParameter) # Now it works: traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2, load_results=1) #For the fun of it, let's print the source code print('\n ---------- The source code of your function ---------- \n %s' % traj.diff_eq) # Let's get the exploration array: initial_conditions_exploration_array = traj.f_get('initial_conditions').f_get_range() # Now let's plot our simulated equations for the different initial conditions: # We will iterate through the run names for idx, run_name in enumerate(traj.f_get_run_names()): #Get the result of run idx from the trajectory euler_result = traj.results.f_get(run_name).euler_evolution # Now we manually need to load the result. Actually the results are not so large and we # could load them all at once. But for demonstration we do as if they were huge: traj.f_load_item(euler_result) euler_data = euler_result.data #Plot fancy 3d plot fig = plt.figure(idx) ax = fig.gca(projection='3d') x = euler_data[:,0] y = euler_data[:,1] z = euler_data[:,2] ax.plot(x, y, z, label='Initial Conditions: %s' % str(initial_conditions_exploration_array[idx])) plt.legend() plt.show() # Now we free the data again (because we assume its huuuuuuge): del euler_data euler_result.f_empty() # You have to click through the images to stop the example_05 module! # Finally disable logging and close all log-files env.f_disable_logging()
def test_conversions(self): filename = make_temp_dir('hdf5manipulation.hdf5') traj = Trajectory(name=make_trajectory_name(self), filename=filename) trajname = traj.v_name traj.v_standard_result = SharedResult traj.f_store(only_init=True) traj.f_add_result('shared_data') thedata = np.zeros((1000, 1000)) myarray = SharedArray('array', traj.shared_data, trajectory=traj) traj.shared_data['array'] = myarray mytable = SharedTable('t1', traj.shared_data, trajectory=traj) traj.shared_data['t1'] = mytable dadict = { 'hi': [1, 2, 3, 4, 5], 'shu': ['bi', 'du', 'da', 'ha', 'hui'] } dadict2 = {'answer': [42]} res = traj.f_add_result('shared.dfs') res['df'] = SharedPandasFrame() res['df'].create_shared_data(data=pd.DataFrame(dadict), trajectory=traj) frame = SharedPandasFrame('df1', traj.f_get('shared.dfs'), trajectory=traj, add_to_parent=True) frame.create_shared_data(data=pd.DataFrame(dadict2), ) res['df1'] = frame traj.f_add_result('mylist', [1, 2, 3]) traj.f_add_result('my.mytuple', k=(1, 2, 3), wa=42) traj.f_add_result('my.myarray', np.zeros((50, 50))) traj.f_add_result('my.myframe', data=pd.DataFrame(dadict2)) traj.f_add_result('my.mytable', ObjectTable(data=dadict2)) myarray.create_shared_data(data=thedata) mytable.create_shared_data(first_row={ 'hi': compat.tobytes('hi'), 'huhu': np.ones(3) }) traj.f_store() data = myarray.read() myarray.get_data_node() self.assertTrue(np.all(data == thedata)) with StorageContextManager(traj): myarray[2, 2] = 10 data = myarray.read() self.assertTrue(data[2, 2] == 10) self.assertTrue(data[2, 2] == 10) self.assertFalse(traj.v_storage_service.is_open) traj = load_trajectory(name=trajname, filename=filename, load_all=2, dynamic_imports=SharedResult) make_ordinary_result(traj.shared_data, 'array', trajectory=traj) array = traj.shared_data.array self.assertTrue(isinstance(array, np.ndarray)) thedata[2, 2] = 10 self.assertTrue(np.all(array == thedata)) make_ordinary_result( traj.shared_data, 't1', trajectory=traj, ) t1 = traj.shared_data.t1 self.assertTrue(isinstance(t1, ObjectTable)) self.assertTrue(np.all(t1['huhu'][0] == np.ones(3))) dfs = traj.shared.dfs make_ordinary_result(traj.shared.dfs, 'df', trajectory=traj) theframe = dfs.f_get('df') self.assertTrue(isinstance(dfs, Result)) self.assertTrue(isinstance(theframe, pd.DataFrame)) self.assertTrue(theframe['hi'][0] == 1) listres = traj.f_get('mylist') listres = make_shared_result(listres, 0, trajectory=traj) with StorageContextManager(traj): self.assertTrue(listres[0][2] == 3) listres[0][0] = 4 self.assertTrue(listres[0][0] == 4) listres = make_ordinary_result(listres, 0, trajectory=traj) traj = load_trajectory(name=trajname, filename=filename, load_all=2, dynamic_imports=SharedResult) mylist = traj.mylist self.assertTrue(isinstance(listres, Result)) self.assertTrue(mylist[0] == 4) self.assertTrue(isinstance(mylist, list)) mytuple = traj.mytuple with self.assertRaises(AttributeError): mytuple = make_shared_result(mytuple, 'mylist', traj, new_class=SharedArray) mytuple = make_shared_result(mytuple, 'k', traj, new_class=SharedArray) self.assertTrue(mytuple.k[1] == 2) mytuple = make_ordinary_result(mytuple, 'k', trajectory=traj) self.assertTrue(isinstance(mytuple.k, tuple)) self.assertTrue(mytuple.k[2] == 3) myframe = traj.myframe myframe = make_shared_result(myframe, 'data', traj) theframe = myframe.data.read() self.assertTrue(theframe['answer'][0] == 42) myframe = make_ordinary_result(myframe, 'data', trajectory=traj) traj.f_load_item(myframe) self.assertTrue(myframe.data['answer'][0] == 42) mytable = traj.f_get('mytable') mytable = make_shared_result(mytable, 0, traj) self.assertTrue(isinstance(mytable[0], SharedTable)) rows = mytable.mytable.read() self.assertTrue(rows[0][0] == 42) mytable = make_ordinary_result(mytable, 0, trajectory=traj) self.assertTrue(isinstance(mytable, Result)) self.assertTrue(mytable[0]['answer'][0] == 42)
traj.f_add_result('starwars.gross_income_of_film', amount=10.1 ** 11, currency='$$$', comment='George Lucas is rich, dude!') # This is a large number, we better store it and than free the memory: traj.f_store_item('gross_income_of_film') traj.gross_income_of_film.f_empty() # Now lets reload the trajectory del traj traj = Trajectory(filename=filename) # We want to load the last trajectory in the file, therefore index = -1 # We want to load the parameters, therefore load_parameters=2 # We only want to load the skeleton of the results, so load_results=1 traj.f_load(index=-1, load_parameters=2, load_results=1) # Let's check if our result is really empty if traj.gross_income_of_film.f_is_empty(): print('Nothing there!') else: print('I found something!') # Ok, let's manually reload the result traj.f_load_item('gross_income_of_film') if traj.gross_income_of_film.f_is_empty(): print('Still empty :-(') else: print('George Lucas earned %s%s!' %(str(traj.gross_income_of_film.amount), traj.gross_income_of_film.currency)) # And that's how it works! If you wish, you can inspect the # experiments/example_02/HDF5/example_02.hdf5 file to take a look at the tree structure
# Moreover, if you don't like prefixes `f_` and `v_` you can also use `func` and `vars`: traj.func.add_result('starwars.robots', c3p0='android', r2d2='beeep!', comment='Help me Obiwan!') print(traj.results.starwars.robots.vars.comment) # Now lets reload the trajectory del traj traj = Trajectory(filename=filename) # We want to load the last trajectory in the file, therefore index = -1 # We want to load the parameters, therefore load_parameters=2 # We only want to load the skeleton of the results, so load_results=1 traj.f_load(index=-1, load_parameters=2, load_results=1) # Let's check if our result is really empty if traj.gross_income_of_film.f_is_empty(): print('Nothing there!') else: print('I found something!') # Ok, let's manually reload the result traj.f_load_item('gross_income_of_film') if traj.gross_income_of_film.f_is_empty(): print('Still empty :-(') else: print('George Lucas earned %s%s!' % (str( traj.gross_income_of_film.amount), traj.gross_income_of_film.currency)) # And that's how it works! If you wish, you can inspect the # experiments/example_02/HDF5/example_02.hdf5 file to take a look at the tree structure
def main(): filename = os.path.join('hdf5', 'example_05.hdf5') env = Environment(trajectory='Example_05_Euler_Integration', filename=filename, file_title='Example_05_Euler_Integration', overwrite_file=True, comment='Go for Euler!') traj = env.trajectory trajectory_name = traj.v_name # 1st a) phase parameter addition add_parameters(traj) # 1st b) phase preparation # We will add the differential equation (well, its source code only) as a derived parameter traj.f_add_derived_parameter(FunctionParameter,'diff_eq', diff_lorenz, comment='Source code of our equation!') # We want to explore some initial conditions traj.f_explore({'initial_conditions' : [ np.array([0.01,0.01,0.01]), np.array([2.02,0.02,0.02]), np.array([42.0,4.2,0.42]) ]}) # 3 different conditions are enough for an illustrative example # 2nd phase let's run the experiment # We pass `euler_scheme` as our top-level simulation function and # the Lorenz equation 'diff_lorenz' as an additional argument env.run(euler_scheme, diff_lorenz) # We don't have a 3rd phase of post-processing here # 4th phase analysis. # I would recommend to do post-processing completely independent from the simulation, # but for simplicity let's do it here. # Let's assume that we start all over again and load the entire trajectory new. # Yet, there is an error within this approach, do you spot it? del traj traj = Trajectory(filename=filename) # We will only fully load parameters and derived parameters. # Results will be loaded manually later on. try: # However, this will fail because our trajectory does not know how to # build the FunctionParameter. You have seen this coming, right? traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2, load_results=1) except ImportError as e: print('That did\'nt work, I am sorry: %s ' % str(e)) # Ok, let's try again but this time with adding our parameter to the imports traj = Trajectory(filename=filename, dynamically_imported_classes=FunctionParameter) # Now it works: traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2, load_results=1) #For the fun of it, let's print the source code print('\n ---------- The source code of your function ---------- \n %s' % traj.diff_eq) # Let's get the exploration array: initial_conditions_exploration_array = traj.f_get('initial_conditions').f_get_range() # Now let's plot our simulated equations for the different initial conditions: # We will iterate through the run names for idx, run_name in enumerate(traj.f_get_run_names()): #Get the result of run idx from the trajectory euler_result = traj.results.f_get(run_name).euler_evolution # Now we manually need to load the result. Actually the results are not so large and we # could load them all at once. But for demonstration we do as if they were huge: traj.f_load_item(euler_result) euler_data = euler_result.data #Plot fancy 3d plot fig = plt.figure(idx) ax = fig.gca(projection='3d') x = euler_data[:,0] y = euler_data[:,1] z = euler_data[:,2] ax.plot(x, y, z, label='Initial Conditions: %s' % str(initial_conditions_exploration_array[idx])) plt.legend() plt.show() # Now we free the data again (because we assume its huuuuuuge): del euler_data euler_result.f_empty() # You have to click through the images to stop the example_05 module! # Finally disable logging and close all log-files env.disable_logging()
def test_conversions(self): filename = make_temp_dir('hdf5manipulation.hdf5') traj = Trajectory(name=make_trajectory_name(self), filename=filename) trajname = traj.v_name traj.v_standard_result = SharedResult traj.f_store(only_init=True) traj.f_add_result('shared_data') thedata = np.zeros((1000, 1000)) myarray = SharedArray('array', traj.shared_data, trajectory=traj) traj.shared_data['array'] = myarray mytable = SharedTable('t1', traj.shared_data, trajectory=traj) traj.shared_data['t1'] = mytable dadict = {'hi': [1, 2, 3, 4, 5], 'shu': ['bi', 'du', 'da', 'ha', 'hui']} dadict2 = {'answer': [42]} res = traj.f_add_result('shared.dfs') res['df'] = SharedPandasFrame() res['df'].create_shared_data(data=pd.DataFrame(dadict), trajectory=traj) frame = SharedPandasFrame('df1', traj.f_get('shared.dfs'), trajectory=traj, add_to_parent=True) frame.create_shared_data(data=pd.DataFrame(dadict2),) res['df1'] = frame traj.f_add_result('mylist', [1, 2, 3]) traj.f_add_result('my.mytuple', k=(1, 2, 3), wa=42) traj.f_add_result('my.myarray', np.zeros((50, 50))) traj.f_add_result('my.myframe', data=pd.DataFrame(dadict2)) traj.f_add_result('my.mytable', ObjectTable(data=dadict2)) myarray.create_shared_data(data=thedata) mytable.create_shared_data(first_row={'hi': compat.tobytes('hi'), 'huhu': np.ones(3)}) traj.f_store() data = myarray.read() myarray.get_data_node() self.assertTrue(np.all(data == thedata)) with StorageContextManager(traj): myarray[2, 2] = 10 data = myarray.read() self.assertTrue(data[2, 2] == 10) self.assertTrue(data[2, 2] == 10) self.assertFalse(traj.v_storage_service.is_open) traj = load_trajectory(name=trajname, filename=filename, load_all=2, dynamic_imports=SharedResult) make_ordinary_result(traj.shared_data, 'array', trajectory=traj) array = traj.shared_data.array self.assertTrue(isinstance(array, np.ndarray)) thedata[2, 2] = 10 self.assertTrue(np.all(array == thedata)) make_ordinary_result(traj.shared_data, 't1', trajectory=traj,) t1 = traj.shared_data.t1 self.assertTrue(isinstance(t1, ObjectTable)) self.assertTrue(np.all(t1['huhu'][0] == np.ones(3))) dfs = traj.shared.dfs make_ordinary_result(traj.shared.dfs, 'df', trajectory=traj) theframe = dfs.f_get('df') self.assertTrue(isinstance(dfs, Result)) self.assertTrue(isinstance(theframe, pd.DataFrame)) self.assertTrue(theframe['hi'][0] == 1) listres = traj.f_get('mylist') listres = make_shared_result(listres, 0, trajectory=traj) with StorageContextManager(traj): self.assertTrue(listres[0][2] == 3) listres[0][0] = 4 self.assertTrue(listres[0][0] == 4) listres = make_ordinary_result(listres, 0, trajectory=traj) traj = load_trajectory(name=trajname, filename=filename, load_all=2, dynamic_imports=SharedResult) mylist = traj.mylist self.assertTrue(isinstance(listres, Result)) self.assertTrue(mylist[0] == 4) self.assertTrue(isinstance(mylist, list)) mytuple = traj.mytuple with self.assertRaises(AttributeError): mytuple = make_shared_result(mytuple, 'mylist', traj, new_class=SharedArray) mytuple = make_shared_result(mytuple, 'k', traj, new_class=SharedArray) self.assertTrue(mytuple.k[1] == 2) mytuple = make_ordinary_result(mytuple, 'k', trajectory=traj) self.assertTrue(isinstance(mytuple.k, tuple)) self.assertTrue(mytuple.k[2] == 3) myframe = traj.myframe myframe = make_shared_result(myframe, 'data', traj) theframe = myframe.data.read() self.assertTrue(theframe['answer'][0] == 42) myframe = make_ordinary_result(myframe, 'data', trajectory=traj) traj.f_load_item(myframe) self.assertTrue(myframe.data['answer'][0] == 42) mytable = traj.f_get('mytable') mytable = make_shared_result(mytable, 0, traj) self.assertTrue(isinstance(mytable[0], SharedTable)) rows = mytable.mytable.read() self.assertTrue(rows[0][0] == 42) mytable = make_ordinary_result(mytable, 0, trajectory=traj) self.assertTrue(isinstance(mytable, Result)) self.assertTrue(mytable[0]['answer'][0] == 42)