Example #1
0
    def test_overwrite_stuff(self):
        traj = Trajectory(name='Test', filename=make_temp_file('testowrite.hdf5'))

        res = traj.f_add_result('mytest.test', a='b', c='d')

        traj.f_store()

        res['a'] = 333
        res['c'] = 123445

        traj.f_store_item(res, overwrite='a')

        # Should emit a warning
        traj.f_store_item(res, overwrite=['a', 'b'])

        traj.f_load(load_results=3)

        res = traj.test

        self.assertTrue(res['a']==333)
        self.assertTrue(res['c']=='d')

        res['c'] = 123445

        traj.f_store_item(res, overwrite=True)
        res.f_empty()

        traj.f_load(load_results=3)

        self.assertTrue(traj.test['c']==123445)
Example #2
0
    def test_store_and_load_large_dictionary(self):
        traj = Trajectory(name='Test', filename=make_temp_file('large_dict.hdf5'))

        large_dict = {}

        for irun in range(1025):
            large_dict['item_%d' % irun] = irun

        large_dict2 = {}

        for irun in range(33):
            large_dict2['item_%d' % irun] = irun

        traj.f_add_result('large_dict', large_dict, comment='Huge_dict!')
        traj.f_add_result('large_dict2', large_dict2, comment='Not so large dict!')

        traj.f_store()

        traj_name = traj.v_name

        traj2 = Trajectory(filename=make_temp_file('large_dict.hdf5'))

        traj2.f_load(name=traj_name, load_all=2)

        self.compare_trajectories(traj, traj2)
Example #3
0
 def load_trajectory(self,trajectory_index=None,trajectory_name=None,as_new=False):
     ### Load The Trajectory and check if the values are still the same
     newtraj = Trajectory()
     newtraj.v_storage_service=HDF5StorageService(filename=self.filename)
     newtraj.f_load(name=trajectory_name, load_derived_parameters=2,load_results=2,
                    index=trajectory_index, as_new=as_new)
     return newtraj
Example #4
0
    def test_a_large_run(self):
        get_root_logger().info('Testing large run')
        self.traj.f_add_parameter('TEST', 'test_run')
        ###Explore
        self.explore_large(self.traj)
        self.make_run_large_data()

        self.assertTrue(self.traj.f_is_completed())

        # Check if printing and repr work
        get_root_logger().info(str(self.env))
        get_root_logger().info(repr(self.env))

        newtraj = Trajectory()
        newtraj.f_load(name=self.traj.v_name, as_new=False, load_data=2, filename=self.filename)

        self.traj.f_load_skeleton()
        self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)

        self.compare_trajectories(self.traj,newtraj)

        size=os.path.getsize(self.filename)
        size_in_mb = size/1000000.
        get_root_logger().info('Size is %sMB' % str(size_in_mb))
        self.assertTrue(size_in_mb < 30.0, 'Size is %sMB > 30MB' % str(size_in_mb))
Example #5
0
 def load_trajectory(self,trajectory_index=None,trajectory_name=None,as_new=False, how=2):
     ### Load The Trajectory and check if the values are still the same
     newtraj = Trajectory()
     newtraj.v_storage_service=HDF5StorageService(filename=self.filename)
     newtraj.f_load(name=trajectory_name, index=trajectory_index, as_new=as_new,
                    load_derived_parameters=how, load_results=how)
     return newtraj
Example #6
0
    def test_store_items_and_groups(self):

        traj = Trajectory(name='testtraj', filename=make_temp_file('teststoreitems.hdf5'))

        traj.f_store()

        traj.f_add_parameter('group1.test',42, comment= 'TooLong' * pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH)

        traj.f_add_result('testres', 42)

        traj.group1.f_set_annotations(Test=44)

        traj.f_store_items(['test','testres','group1'])


        traj2 = Trajectory(name=traj.v_name, add_time=False,
                           filename=make_temp_file('teststoreitems.hdf5'))

        traj2.f_load(load_results=2,load_parameters=2)

        traj.f_add_result('Im.stored.along.a.path', 43)
        traj.Im.stored.along.v_annotations['wtf'] =4444
        traj.res.f_store_child('Im.stored.along.a.path')


        traj2.res.f_load_child('Im.stored.along.a.path', load_data=2)

        self.compare_trajectories(traj,traj2)
Example #7
0
    def test_partially_delete_stuff(self):
        traj = Trajectory(name='TestDelete',
                          filename=make_temp_file('testpartiallydel.hdf5'))

        res = traj.f_add_result('mytest.test', a='b', c='d')

        traj.f_store()

        self.assertTrue('a' in res)
        traj.f_delete_item(res, delete_only=['a'], remove_from_item=True)

        self.assertTrue('c' in res)
        self.assertTrue('a' not in res)

        res['a'] = 'offf'

        self.assertTrue('a' in res)

        traj.f_load(load_results=3)

        self.assertTrue('a' not in res)
        self.assertTrue('c' in res)

        traj.f_delete_item(res, remove_from_trajectory=True, remove_empty_groups=True)

        self.assertTrue('results' not in traj)
        self.assertTrue(res not in traj)
Example #8
0
def test_loading(filenames, traj_names):

    loading_times = np.zeros(len(traj_names))
    loading_times_wd = np.zeros(len(traj_names))

    n_groups= np.zeros(len(traj_names), dtype='int')

    for idx, traj_name in enumerate(traj_names):
        filename = filenames[idx]
        traj = Trajectory(name=traj_name, filename=filename, add_time=False)
        start = time.time()
        traj.f_load(load_parameters=2, load_results=1, load_derived_parameters=1)
        elapsed = (time.time() - start)
        loading_times[idx]=elapsed

        n_groups[idx] = len([x for x in traj.f_iter_nodes(recursive=True)])
        del traj

        traj = Trajectory(name=traj_name, filename=filename, add_time=False)
        start = time.time()
        traj.f_load(load_all=2)
        elapsed = (time.time() - start)
        loading_times_wd[idx]=elapsed

    for idx, loading_time in enumerate(loading_times):
        loading_time_wd = loading_times_wd[idx]
        groups = n_groups[idx]
        print('Groups: %d, Loading: %.3fs, with Data: %.3fs' % (groups, loading_time, loading_time_wd))
Example #9
0
    def test_a_large_run(self):
        get_root_logger().info('Testing large run')
        self.traj.f_add_parameter('TEST', 'test_run')
        ###Explore
        self.explore_large(self.traj)
        self.make_run_large_data()

        self.assertTrue(self.traj.f_is_completed())

        # Check if printing and repr work
        get_root_logger().info(str(self.env))
        get_root_logger().info(repr(self.env))

        newtraj = Trajectory()
        newtraj.f_load(name=self.traj.v_name, as_new=False, load_data=2, filename=self.filename)

        self.traj.f_load_skeleton()
        self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)

        self.compare_trajectories(self.traj,newtraj)

        size=os.path.getsize(self.filename)
        size_in_mb = size/1000000.
        get_root_logger().info('Size is %sMB' % str(size_in_mb))
        self.assertTrue(size_in_mb < 30.0, 'Size is %sMB > 30MB' % str(size_in_mb))
    def test_net(self):
        self.env.f_run(run_net)

        self.traj.f_load(load_derived_parameters=2, load_results=2)

        traj2 = Trajectory(name = self.traj.v_name, add_time=False,
                           filename=make_temp_file('experiments/tests/briantests/HDF5/briantest.hdf5'),
                           dynamically_imported_classes=['pypet.brian.parameter.BrianParameter',
                                                        BrianMonitorResult])

        traj2.f_load(load_parameters=2, load_derived_parameters=2, load_results=2)

        self.compare_trajectories(self.traj, traj2)
def load_trajectory(fname):

    tr = Trajectory(name='tr1',
                    add_time=False,
                    filename=fname,
                    dynamic_imports=[Brian2MonitorResult, Brian2Parameter])

    # pypet.pypetconstants.LOAD_NOTHING  --> 0
    # pypet.pypetconstants.LOAD_SKELETON --> 1
    # pypet.pypetconstants.LOAD_DATA     --> 2
    tr.f_load(load_parameters=2, load_derived_parameters=2, load_results=1)
    tr.v_auto_load = True

    return tr
Example #12
0
    def test_run(self):
        self.env.f_run(dostuff_and_add_links)

        self.traj.f_load(load_data=2)

        traj2 = Trajectory()

        traj2.f_load(name=self.traj.v_name, filename=self.filename)

        traj2.f_load(load_data=2)

        for run in self.traj.f_get_run_names():
            self.assertTrue(self.traj.res.runs[run].paraBL is self.traj.paramB)

        self.compare_trajectories(self.traj, traj2)
Example #13
0
    def test_run(self):
        self.env.f_run(dostuff_and_add_links)

        self.traj.f_load(load_data=2)

        traj2 = Trajectory()

        traj2.f_load(name=self.traj.v_name, filename=self.filename)

        traj2.f_load(load_data=2)

        for run in self.traj.f_get_run_names():
            self.assertTrue(self.traj.res.runs[run].paraBL is self.traj.paramB)

        self.compare_trajectories(self.traj, traj2)
Example #14
0
    def test_store_load_with_hdf5(self):
        traj_name = 'test_%s' % self.__class__.__name__
        filename = make_temp_dir(traj_name + '.hdf5')
        traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
                          filename = filename, overwrite_file=True)

        for res in self.results.values():
            traj.f_add_result(res)

        traj.f_store()

        new_traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
                              filename = filename)

        new_traj.f_load(load_data=2)
        self.compare_trajectories(traj, new_traj)
Example #15
0
    def test_store_load_with_hdf5(self):
        traj_name = 'test_%s' % self.__class__.__name__
        filename = make_temp_dir(traj_name + '.hdf5')
        traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
                          filename = filename, overwrite_file=True)

        for res in self.results.values():
            traj.f_add_result(res)

        traj.f_store()

        new_traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
                              filename = filename)

        new_traj.f_load(load_data=2)
        self.compare_trajectories(traj, new_traj)
Example #16
0
    def test_store_load_with_hdf5_no_data(self):
        traj_name = 'test_%s' % self.__class__.__name__
        filename = make_temp_dir(traj_name + 'nodata.hdf5')
        traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
                          filename = filename, overwrite_file=True)

        for param in self.param.values():
            param._data = None
            traj.f_add_parameter(param)

        traj.f_store()

        new_traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
                              filename = filename)

        new_traj.f_load(load_data=2)
        self.compare_trajectories(traj, new_traj)
Example #17
0
    def test_store_load_with_hdf5_no_data(self):
        traj_name = 'test_%s' % self.__class__.__name__
        filename = make_temp_dir(traj_name + 'nodata.hdf5')
        traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
                          filename = filename, overwrite_file=True)

        for param in self.param.values():
            param._data = None
            traj.f_add_parameter(param)

        traj.f_store()

        new_traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
                              filename = filename)

        new_traj.f_load(load_data=2)
        self.compare_trajectories(traj, new_traj)
    def test_net(self):
        self.env.f_run(run_net)

        self.traj.f_load(load_derived_parameters=2, load_results=2)

        traj2 = Trajectory(name = self.traj.v_name, add_time=False,
                           filename=make_temp_dir(os.path.join(
                               'experiments',
                               'tests',
                               'briantests',
                               'HDF5',
                               'briantest.hdf5')),
                           dynamic_imports=['pypet.brian.parameter.BrianParameter',
                                                        BrianMonitorResult])

        traj2.f_load(load_parameters=2, load_derived_parameters=2, load_results=2)

        self.compare_trajectories(self.traj, traj2)
Example #19
0
    def test_file_translation(self):
        filename = make_temp_dir('to_new_tree.hdf5')
        mytraj = Trajectory('SCRATCH', filename=filename)

        mytraj.f_add_parameter('Test.Group.Test', 42)

        mytraj.f_add_derived_parameter('trajectory.saaaa', 33)

        mytraj.f_add_derived_parameter('trajectory.intraj.dpar1', 33)

        mytraj.f_add_derived_parameter('run_00000008.inrun.dpar2', 33)
        mytraj.f_add_derived_parameter('run_00000001.inrun.dpar3', 35)

        mytraj.f_add_result('trajectory.intraj.res1', 33)

        mytraj.f_add_result('run_00000008.inrun.res1', 33)

        mytraj.f_store()

        mytraj.f_migrate(new_name=mytraj.v_name + 'PETER', in_store=True)

        mytraj.f_store()

        fu = FileUpdater(filename=filename, backup=True)

        fu.update_file()

        mytraj = Trajectory(name=mytraj.v_name,
                            add_time=False,
                            filename=filename)
        mytraj.f_load(load_parameters=2,
                      load_derived_parameters=2,
                      load_results=2)

        for node in mytraj.f_iter_nodes():
            self.assertTrue(node.v_name != 'trajectory')

            if 'run_' in node.v_full_name:
                self.assertTrue('.runs.' in node.v_full_name)

        remove_data()
Example #20
0
    def test_net(self):
        self.env.f_run(run_net)

        self.traj.f_load(load_derived_parameters=2, load_results=2)

        traj2 = Trajectory(name=self.traj.v_name,
                           add_time=False,
                           filename=make_temp_dir(
                               os.path.join('experiments', 'tests',
                                            'briantests', 'HDF5',
                                            'briantest.hdf5')),
                           dynamic_imports=[
                               'pypet.brian.parameter.BrianParameter',
                               BrianMonitorResult
                           ])

        traj2.f_load(load_parameters=2,
                     load_derived_parameters=2,
                     load_results=2)

        self.compare_trajectories(self.traj, traj2)
Example #21
0
    def test_storage_service_errors(self):

        traj = Trajectory(filename=make_temp_file('testnoservice.hdf5'))

        traj_name = traj.v_name

        # you cannot store stuff before the trajectory was stored once:
        with self.assertRaises(ValueError):
            traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = traj.v_name)


        traj.f_store()

        with self.assertRaises(ValueError):
            traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = 'test')

        with self.assertRaises(pex.NoSuchServiceError):
            traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = traj.v_name)

        with self.assertRaises(ValueError):
            traj.f_load(name = 'test', index=1)

        with self.assertRaises(RuntimeError):
            traj.v_storage_service.store('LIST', [('LEAF',None,None,None,None)], trajectory_name = traj.v_name)

        with self.assertRaises(ValueError):
            traj.f_load(index=9999)

        with self.assertRaises(ValueError):
            traj.f_load(name='Non-Existising-Traj')
Example #22
0
def main():

    folder = 'experiments/example_11/HDF5/'
    filename = 'Clustered_Network.hdf5'

    filename = os.path.join(folder, filename)
    # If we pass a filename to the trajectory a new HDF5StorageService will
    # be automatically created
    traj = Trajectory(filename=filename,
                    dynamically_imported_classes=[BrianDurationParameter,
                                                  BrianMonitorResult,
                                                  BrianParameter])

    # Let's create and fake environment to enable logging:
    Environment(traj, do_single_runs=False)


    # Load the trajectory, but onyl laod the skeleton of the results
    traj.f_load(index=0, # Change if you do not want to load the very first trajectory
                load_parameters=2,
                load_derived_parameters=2,
                load_results=1)

    # Find the result instances related to the fano factor
    fano_dict = traj.f_get_from_runs('mean_fano_factor', fast_access=False)

    # Load the data of the fano factor results
    ffs = fano_dict.values()
    traj.f_load_items(ffs)

    # Extract all values and R_ee values for each run
    ffs_values = [x.f_get() for x in ffs]
    Rees = traj.f_get('R_ee').f_get_range()

    # Plot average fano factor as a function of R_ee
    plt.plot(Rees, ffs_values)
    plt.xlabel('R_ee')
    plt.ylabel('Avg. Fano Factor')
    plt.show()
Example #23
0
    def test_pipeline(self):

        filename = 'testpostprocpipe.hdf5'
        env1, filename, _, _ = self.make_environment(filename, 'k1')
        env2 = self.make_environment(filename, 'k2', log=False)[0]

        traj1 = env1.v_trajectory
        traj2 = env2.v_trajectory

        trajs = [traj1, traj2]

        traj2.f_add_parameter('x', 1, comment='1st')
        traj2.f_add_parameter('y', 1, comment='1st')

        exp_dict2 = {
            'x': [1, 2, 3, 4, 1, 2, 2, 3],
            'y': [1, 2, 3, 4, 1, 2, 0, 1]
        }

        traj2.f_explore(exp_dict2)

        res1 = env1.pipeline(pipeline=mypipeline)

        self.are_results_in_order(res1)

        res2 = env2.f_run(Multiply(), 22)

        self.are_results_in_order(res2)

        traj_name = traj1.v_name
        traj1 = Trajectory(traj_name, add_time=False, filename=filename)

        traj1.f_load(load_data=2)
        traj2.f_load(load_data=2)

        self.compare_trajectories(traj1, traj2)

        env1.f_disable_logging()
        env2.f_disable_logging()
Example #24
0
    def test_pipeline(self):

        filename = 'testpostprocpipe.hdf5'
        env1, filename, _, _ = self.make_environment(filename, 'k1')
        env2 = self.make_environment(filename, 'k2', log=False)[0]

        traj1 = env1.v_trajectory
        traj2 = env2.v_trajectory

        trajs = [traj1, traj2]


        traj2.f_add_parameter('x', 1, comment='1st')
        traj2.f_add_parameter('y', 1, comment='1st')

        exp_dict2 = {'x':[1, 2, 3, 4, 1, 2, 2, 3],
                     'y':[1, 2, 3, 4, 1, 2, 0, 1]}

        traj2.f_explore(exp_dict2)


        res1 = env1.f_pipeline(pipeline=mypipeline)

        self.are_results_in_order(res1)

        res2 = env2.f_run(Multiply(), 22)

        self.are_results_in_order(res2)

        traj_name = traj1.v_name
        traj1 = Trajectory(traj_name, add_time=False, filename=filename)

        traj1.f_load(load_data=2)
        traj2.f_load(load_data=2)

        self.compare_trajectories(traj1, traj2)

        env1.f_disable_logging()
        env2.f_disable_logging()
Example #25
0
    def test_f_iter_runs_auto_load(self):

         ###Explore
        self.explore(self.traj)

        results = self.env.f_run(multiply)
        self.are_results_in_order(results)
        traj = self.traj
        self.assertTrue(len(traj) == len(compat.listvalues(self.explore_dict)[0]))

        self.traj.f_load_skeleton()
        self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
        self.check_if_z_is_correct(traj)

        newtraj = Trajectory()
        newtraj.v_storage_service=HDF5StorageService(filename=self.filename)
        newtraj.f_load(name=self.traj.v_name, index=None, as_new=False, load_data=0)
        newtraj.v_auto_load = True

        newtraj.par.f_load_child('y', load_data=1)

        for idx, run_name in enumerate(self.traj.f_iter_runs()):
            newtraj.v_crun=run_name
            self.traj.v_idx = idx
            newtraj.v_idx = idx
            nameset = set((x.v_name for x in traj.f_iter_nodes(predicate=(idx,))))
            self.assertTrue('run_%08d' % (idx+1) not in nameset)
            self.assertTrue('run_%08d' % idx in nameset)
            self.assertTrue(traj.v_crun == run_name)
            self.assertTrue(newtraj.res.runs.crun.z==newtraj.par.x*newtraj.par.y,' z != x*y: %s != %s * %s' %
                                                  (str(newtraj.crun.z),str(newtraj.x),str(newtraj.y)))


        traj = self.traj
        self.assertTrue(traj.v_idx == -1)
        self.assertTrue(traj.v_crun is None)
        self.assertTrue(traj.v_crun_ == pypetconstants.RUN_NAME_DUMMY)
        self.assertTrue(newtraj.v_idx == idx)
Example #26
0
    def test_file_translation(self):
        filename = make_temp_dir('to_new_tree.hdf5')
        mytraj = Trajectory('SCRATCH', filename=filename)

        mytraj.f_add_parameter('Test.Group.Test', 42)

        mytraj.f_add_derived_parameter('trajectory.saaaa',33)

        mytraj.f_add_derived_parameter('trajectory.intraj.dpar1',33)

        mytraj.f_add_derived_parameter('run_00000008.inrun.dpar2',33)
        mytraj.f_add_derived_parameter('run_00000001.inrun.dpar3',35)


        mytraj.f_add_result('trajectory.intraj.res1',33)

        mytraj.f_add_result('run_00000008.inrun.res1',33)

        mytraj.f_store()

        mytraj.f_migrate(new_name=mytraj.v_name + 'PETER', in_store=True)

        mytraj.f_store()

        fu=FileUpdater(filename=filename, backup=True)

        fu.update_file()

        mytraj = Trajectory(name=mytraj.v_name, add_time=False, filename=filename)
        mytraj.f_load(load_parameters=2, load_derived_parameters=2, load_results=2)

        for node in mytraj.f_iter_nodes():
            self.assertTrue(node.v_name != 'trajectory')

            if 'run_' in node.v_full_name:
                self.assertTrue('.runs.' in node.v_full_name)

        remove_data()
Example #27
0
    def test_f_iter_runs_auto_load(self):

         ###Explore
        self.explore(self.traj)

        results = self.env.f_run(multiply)
        self.are_results_in_order(results)
        traj = self.traj
        self.assertTrue(len(traj) == len(list(self.explore_dict.values())[0]))

        self.traj.f_load_skeleton()
        self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
        self.check_if_z_is_correct(traj)

        newtraj = Trajectory()
        newtraj.v_storage_service=HDF5StorageService(filename=self.filename)
        newtraj.f_load(name=self.traj.v_name, index=None, as_new=False, load_data=0)
        newtraj.v_auto_load = True

        newtraj.par.f_load_child('y', load_data=1)

        for idx, run_name in enumerate(self.traj.f_iter_runs()):
            newtraj.v_crun=run_name
            self.traj.v_idx = idx
            newtraj.v_idx = idx
            nameset = set((x.v_name for x in traj.f_iter_nodes(predicate=(idx,))))
            self.assertTrue('run_%08d' % (idx+1) not in nameset)
            self.assertTrue('run_%08d' % idx in nameset)
            self.assertTrue(traj.v_crun == run_name)
            self.assertTrue(newtraj.res.runs.crun.z==newtraj.par.x*newtraj.par.y,' z != x*y: %s != %s * %s' %
                                                  (str(newtraj.crun.z),str(newtraj.x),str(newtraj.y)))


        traj = self.traj
        self.assertTrue(traj.v_idx == -1)
        self.assertTrue(traj.v_crun is None)
        self.assertTrue(traj.v_crun_ == pypetconstants.RUN_NAME_DUMMY)
        self.assertTrue(newtraj.v_idx == idx)
Example #28
0
    def test_version_mismatch(self):
        traj = Trajectory(name='Test', filename=make_temp_file('testversionmismatch.hdf5'))

        traj.f_add_parameter('group1.test',42)

        traj.f_add_result('testres', 42)

        traj.group1.f_set_annotations(Test=44)

        traj._version='0.1a.1'

        traj.f_store()



        traj2 = Trajectory(name=traj.v_name, add_time=False,
                           filename=make_temp_file('testversionmismatch.hdf5'))

        with self.assertRaises(pex.VersionMismatchError):
            traj2.f_load(load_results=2,load_parameters=2)

        traj2.f_load(load_results=2,load_parameters=2, force=True)

        self.compare_trajectories(traj,traj2)
Example #29
0
def profile_single_storing(profile_stroing=False, profile_loading=True):

    logging.basicConfig(level = logging.INFO)

    logfolder = os.path.join(tempfile.gettempdir(), TEMPDIR, 'logs')
    pathfolder = os.path.join(tempfile.gettempdir(), TEMPDIR, 'hdf5')

    res_per_run = 100

    env = Environment(log_folder=logfolder, filename=pathfolder,
                      ncores=2, multiproc=False,
                      use_pool=True,
                      wrap_mode='QUEUE')

    traj = env.v_trajectory

    traj.f_add_parameter('res_per_run', res_per_run)
    traj.f_add_parameter('trial', 0)

    traj.f_explore({'trial':list(range(10))})

    runexp = lambda : env.f_run(add_data)


    if profile_stroing:
        cProfile.runctx('runexp()', {'runexp': runexp},globals(), sort=1,
                        filename='store_stats.profile')
    else:
        runexp()

    print('########################################################################')

    traj = Trajectory(name=traj.v_name, add_time=False, filename= traj.v_storage_service.filename)

    load = lambda : traj.f_load(load_parameters=2, load_results=1)

    if profile_loading:
        cProfile.runctx('load()', {'load': load},globals(), filename='load_stats.profile', sort=1)
traj.huge_matrices.f_set(monty='Always look on the bright side of life!')

# Next we can store our new string called monty to disk. Since our trajectory was already
# stored to disk once, we can make use of the functionality to store individual items:
traj.f_store_item('huge_matrices')

# Neat, hu? Ok now let's load some of it back, for educational purposes let's start with a fresh
# trajectory. Let's keep the old trajectory name in mind. The current time is added to the
# trajectory name on creation (if you do not want this, just say `add_time=False`).
# Thus, the name is not `example_09_huge_data`, but `example_09_huge_data_XXXX_XX_XX_XXhXXmXXs`:
old_traj_name = traj.v_name
del traj
traj = Trajectory(filename=filename)

# We only want to load the skeleton but not the data:
traj.f_load(name=old_traj_name, load_results=pypetconstants.LOAD_SKELETON)

# Check if we only loaded the skeleton, that means the `huge_matrices` result must be empty:
if traj.huge_matrices.f_is_empty():
    print('Told you!')
else:
    print('Unbelievable, this sucks!')

# Now let's only load `monty` and `mat1`.
# We can do this by passing the keyword argument `load_only` to the load item function:
traj.f_load_item('huge_matrices', load_only=['monty','mat1'])

# Check if this worked:
if ('monty' in traj.huge_matrices and
     'mat1' in traj.huge_matrices and
      not 'mat2' in traj.huge_matrices ):
def main():

    env = Environment(trajectory='Example_05_Euler_Integration',
                      filename='experiments/example_05/HDF5/example_05.hdf5',
                      file_title='Example_05_Euler_Integration',
                      log_folder='experiments/example_05/LOGS/',
                      comment = 'Go for Euler!')


    traj = env.v_trajectory
    trajectory_name = traj.v_name

    # 1st a) phase parameter addition
    add_parameters(traj)

    # 1st b) phase preparation
    # We will add the differential equation (well, its source code only) as a derived parameter
    traj.f_add_derived_parameter(FunctionParameter,'diff_eq', diff_lorenz,
                                 comment='Source code of our equation!')

    # We want to explore some initial conditions
    traj.f_explore({'initial_conditions' : [
        np.array([0.01,0.01,0.01]),
        np.array([2.02,0.02,0.02]),
        np.array([42.0,4.2,0.42])
    ]})
    # 3 different conditions are enough for an illustrative example

    # 2nd phase let's run the experiment
    # We pass `euler_scheme` as our top-level simulation function and
    # the Lorenz equation 'diff_lorenz' as an additional argument
    env.f_run(euler_scheme, diff_lorenz)

    # We don't have a 3rd phase of post-processing here

    # 4th phase analysis.
    # I would recommend to do post-processing completely independent from the simulation,
    # but for simplicity let's do it here.

    # Let's assume that we start all over again and load the entire trajectory new.
    # Yet, there is an error within this approach, do you spot it?
    del traj
    traj = Trajectory(filename='experiments/example_05/HDF5/example_05.hdf5')

    # We will only fully load parameters and derived parameters.
    # Results will be loaded manually later on.
    try:
        # However, this will fail because our trajectory does not know how to
        # build the FunctionParameter. You have seen this coming, right?
        traj.f_load(name=trajectory_name,load_parameters=2,
                    load_derived_parameters=2,load_results=1)
    except ImportError as e:

        print 'That did\'nt work, I am sorry. %s ' % e.message

        # Ok, let's try again but this time with adding our parameter to the imports
        traj = Trajectory(filename='experiments/example_05/HDF5/example_05.hdf5',
                           dynamically_imported_classes=FunctionParameter)

        # Now it works:
        traj.f_load(name=trajectory_name,load_parameters=2,
                    load_derived_parameters=2,load_results=1)


    #For the fun of it, let's print the source code
    print '\n ---------- The source code of your function ---------- \n %s' % traj.diff_eq

    # Let's get the exploration array:
    initial_conditions_exploration_array = traj.f_get('initial_conditions').f_get_range()
    # Now let's plot our simulated equations for the different initial conditions:
    # We will iterate through the run names
    for idx, run_name in enumerate(traj.f_get_run_names()):

        #Get the result of run idx from the trajectory
        euler_result = traj.results.f_get(run_name).euler_evolution
        # Now we manually need to load the result. Actually the results are not so large and we
        # could load them all at once. But for demonstration we do as if they were huge:
        traj.f_load_item(euler_result)
        euler_data = euler_result.data

        #Plot fancy 3d plot
        fig = plt.figure(idx)
        ax = fig.gca(projection='3d')
        x = euler_data[:,0]
        y = euler_data[:,1]
        z = euler_data[:,2]
        ax.plot(x, y, z, label='Initial Conditions: %s' % str(initial_conditions_exploration_array[idx]))
        plt.legend()
        plt.show()

        # Now we free the data again (because we assume its huuuuuuge):
        del euler_data
        euler_result.f_empty()
# Now let's see how we can reload the stored data from above.
# We do not need an environment for that, just a trajectory.
from pypet.trajectory import Trajectory

# So, first let's create a new trajectory and pass it the path and name of the HDF5 file.
# Yet, to be very clear let's delete all the old stuff.
del traj
# Before deleting the environment let's disable logging and close all log-files
env.disable_logging()
del env

traj = Trajectory(filename=filename)

# Now we want to load all stored data.
traj.f_load(index=-1, load_parameters=2, load_results=2)

# Above `index` specifies that we want to load the trajectory with that particular index
# within the HDF5 file. We could instead also specify a `name`.
# Counting works also backwards, so `-1` yields the last or newest trajectory in the file.
#
# Next we need to specify how the data is loaded.
# Therefore, we have to set the keyword arguments `load_parameters` and `load_results`,
# here we chose both to be `2`.
# `0` would mean we do not want to load anything at all.
# `1` would mean we only want to load the empty hulls or skeletons of our parameters
# or results. Accordingly, we would add parameters or results to our trajectory
# but they would not contain any data.
# Instead `2` means we want to load the parameters and results including the data they contain.

# Finally we want to print a result of a particular run.
traj.huge_matrices.f_set(monty='Always look on the bright side of life!')

# Next we can store our new string called monty to disk. Since our trajectory was already
# stored to disk once, we can make use of the functionality to store individual items:
traj.f_store_item('huge_matrices')

# Neat, hu? Ok now let's load some of it back, for educational purposes let's start with a fresh
# trajectory. Let's keep the old trajectory name in mind. The current time is added to the
# trajectory name on creation (if you do not want this, just say `add_time=False`).
# Thus, the name is not `example_09_huge_data`, but `example_09_huge_data_XXXX_XX_XX_XXhXXmXXs`:
old_traj_name = traj.v_name
del traj
traj = Trajectory(filename=filename)

# We only want to load the skeleton but not the data:
traj.f_load(name=old_traj_name, load_results=pypetconstants.LOAD_SKELETON)

# Check if we only loaded the skeleton, that means the `huge_matrices` result must be empty:
if traj.huge_matrices.f_is_empty():
    print('Told you!')
else:
    print('Unbelievable, this sucks!')

# Now let's only load `monty` and `mat1`.
# We can do this by passing the keyword argument `load_only` to the load item function:
traj.f_load_item('huge_matrices', load_only=['monty', 'mat1'])

# Check if this worked:
if ('monty' in traj.huge_matrices and 'mat1' in traj.huge_matrices
        and not 'mat2' in traj.huge_matrices):
Example #34
0
class AnnotationsTest(unittest.TestCase):

    def setUp(self):

        self.filename = make_temp_file('experiments/tests/HDF5/annotations.hdf5')

        self.traj = Trajectory(name='Annotations', filename = self.filename)

        self.traj.f_add_result('testres', 42)

        self.traj.f_add_parameter('testgroup.testparam', 42)

        self.make_annotations()

        self.add_annotations(self.traj)

        self.assertTrue(len([node for node in self.traj.f_iter_nodes(recursive=True)]) == 5)


    def make_annotations(self):

        self.annotations={}

        self.annotations['dict']={'33':12,'kkk':[1,2,'h'], 3:{'a':42.0}}
        self.annotations['list']= [self.annotations['dict'],33]
        self.annotations['string'] = 'string'
        self.annotations['integer'] = 42
        self.annotations['tuple']=(3,4,5)
        self.annotations['Numpy_Data'] = np.array(['fff','ddd'])
        self.annotations[0] = 7777


    def add_annotations(self, traj):
        funcs = 5

        for idx,node in enumerate(traj.f_iter_nodes(recursive=True)):
            for name in self.annotations:
                anno = self.annotations[name]
                if name == 0:
                    node.f_set_annotations(anno)
                    node.v_annotations.f_set(anno)
                elif idx % funcs == 0:
                    node.f_set_annotations(**{name:anno})
                elif idx % funcs == 1:
                    node.v_annotations.f_set(**{name:anno})
                elif idx % funcs == 2:
                    node.v_annotations.f_set_single(name,anno)
                elif idx % funcs == 3:
                    setattr(node.v_annotations,name, anno)
                elif idx % funcs == 4:
                    node.v_annotations[name]=anno


    def test_annotations_insert(self):

        for node in self.traj.f_iter_nodes(recursive=True):
            for name in self.annotations:
                anno = self.annotations[name]
                node_anno = node.v_annotations[name]
                self.assertTrue(comp.nested_equal(anno, node_anno),
                                                  '%s != %s' % (str(anno), str(node_anno)))

    def test_pickling(self):
        dump = pickle.dumps(self.traj)

        del self.traj

        self.traj = pickle.loads(dump)

        self.test_annotations_insert()


    def test_storage_and_loading(self):
        self.traj.f_store()

        traj_name = self.traj.v_name

        del self.traj

        self.traj = Trajectory(filename=self.filename)

        self.traj.f_load(name=traj_name, load_results=2, load_parameters=2, load_derived_parameters=2,
                         load_other_data=2)

        self.test_annotations_insert()


    def test_attribute_deletion(self):
        for node in self.traj.f_iter_nodes(recursive=True):
            name_list=[name for name in node.v_annotations]
            for name in name_list:
                delattr(node.v_annotations, name)

            self.assertTrue(node.v_annotations.f_is_empty())

    def test_get_item(self):
        for node in self.traj.f_iter_nodes(recursive=True):
            for key, val1 in node.v_annotations.f_to_dict().items():
                val2 = node.v_annotations[key]
                self.assertTrue(comp.nested_equal(val1,val2))

    def test_get_item_no_copy(self):
        for node in self.traj.f_iter_nodes(recursive=True):
            for key, val1 in node.v_annotations.f_to_dict(copy=False).items():
                val2 = node.v_annotations[key]
                self.assertTrue(comp.nested_equal(val1,val2))

    @staticmethod
    def dict_to_str(dictionary):
        resstr = ''
        new_dict={}
        for key, val in dictionary.items():
            if key == 0:
                key = 'annotation'
            new_dict[key]=val

        for key in sorted(new_dict.keys()):
            resstr+='%s=%s; ' % (key,str(new_dict[key]))
        return resstr[:-2]

    def test_to_str(self):
        dict_str = self.dict_to_str(self.annotations)
        for node in self.traj.f_iter_nodes(recursive=True):
            ann_str = node.f_ann_to_str()

            self.assertTrue(not ann_str.endswith(' ') or not ann_str.endswith(','))

            for name in self.annotations:
                if name==0:
                    name = 'annotation'
                self.assertTrue(name in ann_str)

            self.assertEqual(dict_str, ann_str, '%s!=%s' % (dict_str, ann_str))

            ann_str = str(node.v_annotations)
            self.assertEqual(dict_str, ann_str, '%s!=%s' % (dict_str, ann_str))

    def test_single_get_and_getattr_and_setattr(self):

        self.traj.f_add_parameter('test2', 42)

        self.traj.f_get('test2').v_annotations.test = 4

        self.assertTrue(self.traj.f_get('test2').v_annotations.test, 4)

        self.assertTrue(self.traj.f_get('test2').v_annotations.f_get(), 4)

    def test_get_annotations(self):
        key_list = sorted(self.annotations.keys())
        for node in self.traj.f_iter_nodes(recursive=True):
            for name in self.annotations:
                self.assertTrue(comp.nested_equal(self.annotations[name],
                                                  node.f_get_annotations(name)))

            val_list = node.f_get_annotations(*key_list)

            for idx, val in enumerate(val_list):
                self.assertTrue(comp.nested_equal(self.annotations[key_list[idx]], val))


    def test_f_get_errors(self):
        for node in self.traj.f_iter_nodes(recursive=True):
            with self.assertRaises(ValueError):
                node.v_annotations.f_get()

            with self.assertRaises(AttributeError):
                node.v_annotations.f_get('gdsdfd')

        testparam = self.traj.f_add_parameter('ggg',343)

        with self.assertRaises(AttributeError):
            testparam.v_annotations.f_get()




    def test_f_set_numbering(self):
        int_list = range(10)
        for node in self.traj.f_iter_nodes(recursive=True):
            node.v_annotations.f_set(*int_list)

            self.assertEqual(node.v_annotations.f_get(*int_list), tuple(int_list))

            for integer in int_list:
                if integer == 0:
                    name = 'annotation'
                else:
                    name = 'annotation_%d' % integer

                self.assertTrue(name in node.v_annotations)
Example #35
0
class AnnotationsTest(unittest.TestCase):

    tags = 'unittest', 'annotations'

    def setUp(self):

        self.filename = make_temp_dir(
            os.path.join('experiments', 'tests', 'HDF5', 'annotations.hdf5'))

        self.traj = Trajectory(name='Annotations', filename=self.filename)

        self.traj.f_add_result('testres', 42)

        self.traj.f_add_parameter('testgroup.testparam', 42)

        self.make_annotations()

        self.add_annotations(self.traj)

        pred = lambda x: 'config' not in x.v_full_name

        x = len([
            node
            for node in self.traj.f_iter_nodes(recursive=True, predicate=pred)
        ])
        self.assertTrue(x == 5, '%s != %s' % (str(x), str(5)))

    def tearDown(self):
        remove_data()

    def make_annotations(self):

        self.annotations = {}

        self.annotations['dict'] = {
            '33': 12,
            'kkk': [1, 2, 'h'],
            3: {
                'a': 42.0
            }
        }
        self.annotations['list'] = [self.annotations['dict'], 33]
        self.annotations['string'] = 'string'
        self.annotations['integer'] = 42
        self.annotations['tuple'] = (3, 4, 5)
        self.annotations['Numpy_Data'] = np.array(['fff', 'ddd'])
        self.annotations[0] = 7777

    def add_annotations(self, traj):
        funcs = 5

        for idx, node in enumerate(
            [traj] + [node for node in traj.f_iter_nodes(recursive=True)]):
            for name in self.annotations:
                anno = self.annotations[name]
                if name == 0:
                    node.f_set_annotations(anno)
                    node.v_annotations.f_set(anno)
                elif idx % funcs == 0:
                    node.f_set_annotations(**{name: anno})
                elif idx % funcs == 1:
                    node.v_annotations.f_set(**{name: anno})
                elif idx % funcs == 2:
                    node.v_annotations.f_set_single(name, anno)
                elif idx % funcs == 3:
                    setattr(node.v_annotations, name, anno)
                elif idx % funcs == 4:
                    node.v_annotations[name] = anno

    def test_annotations_insert(self):

        for idx,node in \
                enumerate([self.traj] + [node for node in self.traj.f_iter_nodes(recursive=True)]):
            for name in self.annotations:
                anno = self.annotations[name]
                node_anno = node.v_annotations[name]
                self.assertTrue(comp.nested_equal(anno, node_anno),
                                '%s != %s' % (str(anno), str(node_anno)))

    def test_pickling(self):
        dump = pickle.dumps(self.traj)

        del self.traj

        self.traj = pickle.loads(dump)

        self.test_annotations_insert()

    def test_storage_and_loading(self):
        self.traj.f_store()

        traj_name = self.traj.v_name

        del self.traj

        self.traj = Trajectory(filename=self.filename)

        self.traj.f_load(name=traj_name,
                         load_parameters=2,
                         load_derived_parameters=2,
                         load_results=2,
                         load_other_data=2)

        self.test_annotations_insert()

    def test_attribute_deletion(self):
        for node in self.traj.f_iter_nodes(recursive=True):
            name_list = [name for name in node.v_annotations]
            for name in name_list:
                delattr(node.v_annotations, name)

            self.assertTrue(node.v_annotations.f_is_empty())

    def test_item_deletion(self):
        for node in self.traj.f_iter_nodes(recursive=True):
            name_list = [name for name in node.v_annotations]
            for name in name_list:
                del node.v_annotations[name]

            self.assertTrue(node.v_annotations.f_is_empty())

    def test_get_item(self):
        for node in self.traj.f_iter_nodes(recursive=True):
            for key, val1 in node.v_annotations.f_to_dict().items():
                val2 = node.v_annotations[key]
                self.assertTrue(comp.nested_equal(val1, val2))

    def test_get_item_no_copy(self):
        for node in self.traj.f_iter_nodes(recursive=True):
            for key, val1 in node.v_annotations.f_to_dict(copy=False).items():
                val2 = node.v_annotations[key]
                self.assertTrue(comp.nested_equal(val1, val2))

    @staticmethod
    def dict_to_str(dictionary):
        resstr = ''
        new_dict = {}
        for key, val in dictionary.items():
            if key == 0:
                key = 'annotation'
            new_dict[key] = val

        for key in sorted(new_dict.keys()):
            resstr += '%s=%s; ' % (key, str(new_dict[key]))
        return resstr[:-2]

    def test_to_str(self):
        dict_str = self.dict_to_str(self.annotations)
        for node in self.traj.f_iter_nodes(recursive=True):
            ann_str = node.f_ann_to_str()

            self.assertTrue(not ann_str.endswith(' ')
                            or not ann_str.endswith(','))

            for name in self.annotations:
                if name == 0:
                    name = 'annotation'
                self.assertTrue(name in ann_str)

            self.assertEqual(dict_str, ann_str, '%s!=%s' % (dict_str, ann_str))

            ann_str = str(node.v_annotations)
            self.assertEqual(dict_str, ann_str, '%s!=%s' % (dict_str, ann_str))

    def test_single_get_and_getattr_and_setattr(self):

        self.traj.f_add_parameter('test2', 42)

        self.traj.f_get('test2').v_annotations.test = 4

        self.assertTrue(self.traj.f_get('test2').v_annotations.test, 4)

        self.assertTrue(self.traj.f_get('test2').v_annotations.f_get(), 4)

    def test_get_annotations(self):
        key_list = list(self.annotations.keys())
        for node in self.traj.f_iter_nodes(recursive=True):
            for name in self.annotations:
                self.assertTrue(
                    comp.nested_equal(self.annotations[name],
                                      node.f_get_annotations(name)))

            val_list = node.f_get_annotations(*key_list)

            for idx, val in enumerate(val_list):
                self.assertTrue(
                    comp.nested_equal(self.annotations[key_list[idx]], val))

    def test_f_get_errors(self):
        for node in self.traj.f_iter_nodes(recursive=True):
            with self.assertRaises(ValueError):
                node.v_annotations.f_get()

            with self.assertRaises(AttributeError):
                node.v_annotations.f_get('gdsdfd')

        testparam = self.traj.f_add_parameter('ggg', 343)

        with self.assertRaises(AttributeError):
            testparam.v_annotations.f_get()

    def test_f_set_numbering(self):
        int_list = list(range(10))
        for node in self.traj.f_iter_nodes(recursive=True):
            node.v_annotations.f_set(*int_list)

            self.assertEqual(node.v_annotations.f_get(*int_list),
                             tuple(int_list))

            for integer in int_list:
                if integer == 0:
                    name = 'annotation'
                else:
                    name = 'annotation_%d' % integer

                self.assertTrue(name in node.v_annotations)
# Now let's see how we can reload the stored data from above.
# We do not need an environment for that, just a trajectory.
from pypet.trajectory import Trajectory

# So, first let's create a new trajectory and pass it the path and name of the HDF5 file.
# Yet, to be very clear let's delete all the old stuff.
del traj
# Before deleting the environment let's disable logging and close all log-files
env.f_disable_logging()
del env

traj = Trajectory(filename=filename)

# Now we want to load all stored data.
traj.f_load(index=-1, load_parameters=2, load_results=2)

# Above `index` specifies that we want to load the trajectory with that particular index
# within the HDF5 file. We could instead also specify a `name`.
# Counting works also backwards, so `-1` yields the last or newest trajectory in the file.
#
# Next we need to specify how the data is loaded.
# Therefore, we have to set the keyword arguments `load_parameters` and `load_results`,
# here we chose both to be `2`.
# `0` would mean we do not want to load anything at all.
# `1` would mean we only want to load the empty hulls or skeletons of our parameters
# or results. Accordingly, we would add parameters or results to our trajectory
# but they would not contain any data.
# Instead `2` means we want to load the parameters and results including the data they contain.

# Finally we want to print a result of a particular run.
Example #37
0
def main():

    # This time we don't need an environment since we just going to look
    # at data in the trajectory
    traj = Trajectory('FiringRate', add_time=False)

    # Let's load the trajectory from the file
    # Only load the parameters, we will load the results on the fly as we need them
    traj.f_load(filename='./hdf5/FiringRate.hdf5', load_parameters=2,
                load_results=0, load_derived_parameters=0)

    # We'll simply use auto loading so all data will be loaded when needed.
    traj.v_auto_load = True

    rates_frame = traj.res.summary.firing_rates.rates_frame
    # Here we load the data automatically on the fly

    plt.figure()
    plt.subplot(2,1,1)
    #Let's iterate through the columns and plot the different firing rates :
    for tau_ref, I_col in rates_frame.iteritems():
        plt.plot(I_col.index, I_col, label='Avg. Rate for tau_ref=%s' % str(tau_ref))

    # Label the plot
    plt.xlabel('I')
    plt.ylabel('f[Hz]')
    plt.title('Firing as a function of input current `I`')
    plt.legend()

    # Also let's plot an example run, how about run 13 ?
    example_run = 13

    traj.v_idx = example_run # We make the trajectory behave as a single run container.
    # This short statement has two major effects:
    # a) all explored parameters are set to the value of run 13,
    # b) if there are tree nodes with names other than the current run aka `run_00000013`
    # they are simply ignored, if we use the `$` sign or the `crun` statement,
    # these are translated into `run_00000013`.

    # Get the example data
    example_I = traj.I
    example_tau_ref = traj.tau_ref
    example_V = traj.results.neuron.crun.V # Here crun stands for run_00000013

    # We need the time step...
    dt = traj.dt
    # ...to create an x-axis for the plot
    dt_array = [irun * dt for irun in range(len(example_V))]

    # And plot the development of V over time,
    # Since this is rather repetitive, we only
    # plot the first eighth of it.
    plt.subplot(2,1,2)
    plt.plot(dt_array, example_V)
    plt.xlim((0, dt*len(example_V)/8))

    # Label the axis
    plt.xlabel('t[ms]')
    plt.ylabel('V')
    plt.title('Example of development of V for I=%s, tau_ref=%s in run %d' %
              (str(example_I), str(example_tau_ref), traj.v_idx))

    # And let's take a look at it
    plt.show()

    # Finally revoke the `traj.v_idx=13` statement and set everything back to normal.
    # Since our analysis is done here, we could skip that, but it is always a good idea
    # to do that.
    traj.f_restore_default()