Example #1
0
    def test_version_mismatch(self):
        traj = Trajectory(name='TestVERSION', filename=make_temp_dir('testversionmismatch.hdf5'),
                          add_time=True)

        traj.f_add_parameter('group1.test',42)

        traj.f_add_result('testres', 42)

        traj.group1.f_set_annotations(Test=44)

        traj._version='0.1a.1'

        traj.f_store()

        traj2 = Trajectory(name=traj.v_name, add_time=False,
                           filename=make_temp_dir('testversionmismatch.hdf5'))

        with self.assertRaises(pex.VersionMismatchError):
            traj2.f_load(load_parameters=2, load_results=2)

        traj2.f_load(load_parameters=2, load_results=2, force=True)

        self.compare_trajectories(traj,traj2)

        get_root_logger().info('Mismatch testing done!')
Example #2
0
    def test_logging_stdout(self):
        filename = 'teststdoutlog.hdf5'
        filename = make_temp_dir(filename)
        folder = make_temp_dir('logs')
        env = Environment(trajectory=make_trajectory_name(self),
                          filename=filename, log_config=get_log_config(),
                          # log_levels=logging.CRITICAL, # needed for the test
                          log_stdout=('STDOUT', 50), #log_folder=folder
                          )

        env.f_run(log_error)
        traj = env.v_traj
        path = get_log_path(traj)

        mainstr = 'sTdOuTLoGGinG'
        print(mainstr)
        env.f_disable_logging()

        mainfilename = os.path.join(path, 'LOG.txt')
        with open(mainfilename, mode='r') as mainf:
            full_text = mainf.read()

        self.assertTrue(mainstr in full_text)
        self.assertTrue('4444444' not in full_text)
        self.assertTrue('DEBUG' not in full_text)
Example #3
0
    def test_store_and_load_large_dictionary(self):
        traj = Trajectory(name='Testlargedict', filename=make_temp_dir('large_dict.hdf5'))

        large_dict = {}

        for irun in range(1025):
            large_dict['item_%d' % irun] = irun

        large_dict2 = {}

        for irun in range(33):
            large_dict2['item_%d' % irun] = irun

        traj.f_add_result('large_dict', large_dict, comment='Huge_dict!')
        traj.f_add_result('large_dict2', large_dict2, comment='Not so large dict!')

        traj.f_store()

        traj_name = traj.v_name

        traj2 = Trajectory(filename=make_temp_dir('large_dict.hdf5'))

        traj2.f_load(name=traj_name, load_data=2)

        self.compare_trajectories(traj, traj2)
Example #4
0
    def test_version_mismatch(self):
        traj = Trajectory(name='TestVERSION', filename=make_temp_dir('testversionmismatch.hdf5'),
                          add_time=True)

        traj.f_add_parameter('group1.test',42)

        traj.f_add_result('testres', 42)

        traj.group1.f_set_annotations(Test=44)

        traj._version='0.1a.1'

        traj.f_store()

        traj2 = Trajectory(name=traj.v_name, add_time=False,
                           filename=make_temp_dir('testversionmismatch.hdf5'))

        with self.assertRaises(pex.VersionMismatchError):
            traj2.f_load(load_parameters=2, load_results=2)

        traj2.f_load(load_parameters=2, load_results=2, force=True)

        self.compare_trajectories(traj,traj2)

        get_root_logger().info('Mismatch testing done!')
    def make_environment(self, idx, filename):



        #self.filename = '../../experiments/tests/HDF5/test.hdf5'
        self.logfolder = make_temp_dir(os.path.join('experiments',
                                                     'tests',
                                                     'Log'))
        self.cnt_folder = make_temp_dir(os.path.join('experiments','tests','cnt'))
        trajname = 'Test%d' % idx + '_' + make_trajectory_name(self)

        env = Environment(trajectory=trajname,
                          filename=filename,
                          file_title=trajname,
                          log_stdout=False,

                          log_config=get_log_config(),
                          continuable=True,
                          continue_folder=self.cnt_folder,
                          delete_continue=False,
                          large_overview_tables=True)


        self.envs.append(env)
        self.trajs.append( env.v_trajectory)
Example #6
0
    def test_continueing_mp_custom(self):
        self.filenames = [
            make_temp_dir('test_continueing_mp_custom.hdf5'),
            make_temp_dir('test_continueing_mp_custom2.hdf5')
        ]

        self.envs = []
        self.trajs = []

        for irun, filename in enumerate(self.filenames):
            if isinstance(filename, int):
                filename = self.filenames[filename]

            self.make_environment(irun, filename, continuable=irun == 1)

        self.param_dict = {'x': 1.0, 'y': 2.0}

        for irun in range(len(self.filenames)):
            self.trajs[irun].f_add_parameter(CustomParameter, 'x', 1.0)
            self.trajs[irun].f_add_parameter(CustomParameter, 'y', 1.0)

        self.explore_mp(self.trajs[0])
        self.explore_mp(self.trajs[1])

        arg = 33
        for irun in range(len(self.filenames)):

            self.envs[irun].f_run(Multiply(), arg)

        traj_name = self.trajs[1].v_name
        continue_folder = os.path.join(self.cnt_folder, self.trajs[1].v_name)

        self.envs.pop()
        self.assertEqual(len(self.envs), 1)
        self.make_environment(1,
                              self.filenames[1],
                              continuable=True,
                              add_time=False,
                              trajectory=traj_name)
        self.trajs[1] = self.envs[1].v_traj
        self.trajs[1].f_load(load_data=pypetconstants.LOAD_NOTHING)
        self._remove_nresults(self.trajs[1], 3, continue_folder)

        results = self.envs[1].resume(trajectory_name=traj_name)
        results = sorted(results, key=lambda x: x[0])

        for irun in range(len(self.filenames)):
            self.trajs[irun].f_load(
                load_parameters=pypetconstants.OVERWRITE_DATA,
                load_derived_parameters=pypetconstants.OVERWRITE_DATA,
                load_results=pypetconstants.OVERWRITE_DATA,
                load_other_data=pypetconstants.OVERWRITE_DATA)

        self.compare_trajectories(self.trajs[1], self.trajs[0])

        for run_name in self.trajs[0].f_iter_runs():
            z = (self.trajs[0].v_idx, self.trajs[0].crun.z)
            self.assertTrue(z in results, '%s not in %s' % (z, results))

        self.assertTrue(len(self.trajs[-1]) == len(results))
Example #7
0
    def test_logging_stdout(self):
        filename = 'teststdoutlog.hdf5'
        filename = make_temp_dir(filename)
        folder = make_temp_dir('logs')
        env = Environment(trajectory=make_trajectory_name(self),
                          filename=filename, log_config=get_log_config(),
                          # log_levels=logging.CRITICAL, # needed for the test
                          log_stdout=('STDOUT', 50), #log_folder=folder
                          )

        env.f_run(log_error)
        traj = env.v_traj
        path = get_log_path(traj)

        mainstr = 'sTdOuTLoGGinG'
        print(mainstr)
        env.f_disable_logging()

        mainfilename = os.path.join(path, 'LOG.txt')
        with open(mainfilename, mode='r') as mainf:
            full_text = mainf.read()

        self.assertTrue(mainstr in full_text)
        self.assertTrue('4444444' not in full_text)
        self.assertTrue('DEBUG' not in full_text)
    def make_environment_mp(self, idx, filename):
        #self.filename = '../../experiments/tests/HDF5/test.hdf5'
        self.logfolder = make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'Log'))
        self.cnt_folder = make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'cnt'))
        trajname = 'Test%d' % idx + '_' + make_trajectory_name(self)

        env = Environment(trajectory=trajname,
                          dynamic_imports=[CustomParameter],
                          filename=filename,
                          file_title=trajname,
                          log_stdout=False,
                          purge_duplicate_comments=False,
                          log_config=get_log_config(),
                          continuable=True,
                          continue_folder=self.cnt_folder,
                          delete_continue=False,
                          multiproc=True,
                          use_pool=True,
                          ncores=4)

        self.envs.append(env)
        self.trajs.append( env.v_trajectory)
Example #9
0
    def test_store_and_load_large_dictionary(self):
        traj = Trajectory(name='Testlargedict', filename=make_temp_dir('large_dict.hdf5'),
                          add_time=True)

        large_dict = {}

        for irun in range(1025):
            large_dict['item_%d' % irun] = irun

        large_dict2 = {}

        for irun in range(33):
            large_dict2['item_%d' % irun] = irun

        traj.f_add_result('large_dict', large_dict, comment='Huge_dict!')
        traj.f_add_result('large_dict2', large_dict2, comment='Not so large dict!')

        traj.f_store()

        traj_name = traj.v_name

        traj2 = Trajectory(filename=make_temp_dir('large_dict.hdf5'),
                           add_time=True)

        traj2.f_load(name=traj_name, load_data=2)

        self.compare_trajectories(traj, traj2)
Example #10
0
    def test_store_items_and_groups(self):

        traj = Trajectory(name='testtraj', filename=make_temp_dir('teststoreitems.hdf5'),
                          add_time=True)

        traj.f_store()

        traj.f_add_parameter('group1.test',42, comment= 'TooLong' * pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH)

        traj.f_add_result('testres', 42)

        traj.group1.f_set_annotations(Test=44)

        traj.f_store_items(['test','testres','group1'])


        traj2 = Trajectory(name=traj.v_name, add_time=False,
                           filename=make_temp_dir('teststoreitems.hdf5'))

        traj2.f_load(load_parameters=2, load_results=2)

        traj.f_add_result('Im.stored.along.a.path', 43)
        traj.Im.stored.along.v_annotations['wtf'] =4444
        traj.res.f_store_child('Im.stored.along.a.path')


        traj2.res.f_load_child('Im.stored.along.a.path', load_data=2)

        self.compare_trajectories(traj,traj2)
Example #11
0
    def test_migrations(self):

        traj = Trajectory(name='Testmigrate',
                          filename=make_temp_dir('migrate.hdf5'),
                          add_time=True)

        traj.f_add_result('I.am.a.mean.resu', 42, comment='Test')
        traj.f_add_derived_parameter('ffa', 42)

        traj.f_store()

        new_file = make_temp_dir('migrate2.hdf5')
        traj.f_migrate(filename=new_file)

        traj.f_store()

        new_traj = Trajectory()

        new_traj.f_migrate(new_name=traj.v_name,
                           filename=new_file,
                           in_store=True)

        new_traj.v_auto_load = True

        self.assertTrue(new_traj.results.I.am.a.mean.resu == 42)
Example #12
0
    def make_environment(self,
                         idx,
                         filename,
                         continuable=True,
                         delete_continue=False):

        #self.filename = '../../experiments/tests/HDF5/test.hdf5'
        self.logfolder = make_temp_dir(
            os.path.join('experiments', 'tests', 'Log'))
        self.cnt_folder = make_temp_dir(
            os.path.join('experiments', 'tests', 'cnt'))
        trajname = 'Test%d' % idx + '_' + make_trajectory_name(self)

        env = Environment(trajectory=trajname,
                          filename=filename,
                          file_title=trajname,
                          log_stdout=False,
                          log_config=get_log_config(),
                          continuable=continuable,
                          continue_folder=self.cnt_folder,
                          delete_continue=delete_continue,
                          large_overview_tables=True)

        self.envs.append(env)
        self.trajs.append(env.v_trajectory)
Example #13
0
    def test_store_items_and_groups(self):

        traj = Trajectory(name='testtraj', filename=make_temp_dir('teststoreitems.hdf5'))

        traj.f_store()

        traj.f_add_parameter('group1.test',42, comment= 'TooLong' * pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH)

        traj.f_add_result('testres', 42)

        traj.group1.f_set_annotations(Test=44)

        traj.f_store_items(['test','testres','group1'])


        traj2 = Trajectory(name=traj.v_name, add_time=False,
                           filename=make_temp_dir('teststoreitems.hdf5'))

        traj2.f_load(load_parameters=2, load_results=2)

        traj.f_add_result('Im.stored.along.a.path', 43)
        traj.Im.stored.along.v_annotations['wtf'] =4444
        traj.res.f_store_child('Im.stored.along.a.path')


        traj2.res.f_load_child('Im.stored.along.a.path', load_data=2)

        self.compare_trajectories(traj,traj2)
    def make_environment_mp(self, idx, filename):
        #self.filename = '../../experiments/tests/HDF5/test.hdf5'
        self.logfolder = make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'Log'))
        self.cnt_folder = make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'cnt'))
        trajname = 'Test%d' % idx + '_' + make_trajectory_name(self)

        env = Environment(trajectory=trajname,
                          dynamic_imports=[CustomParameter],
                          filename=filename,
                          file_title=trajname,
                          log_stdout=False,
                          purge_duplicate_comments=False,
                          log_config=get_log_config(),
                          continuable=True,
                          continue_folder=self.cnt_folder,
                          delete_continue=False,
                          multiproc=True,
                          use_pool=True,
                          ncores=4)

        self.envs.append(env)
        self.trajs.append( env.v_trajectory)
    def test_continueing_mp_custom(self):
        self.filenames = [make_temp_dir('test_continueing_mp_custom.hdf5'),
                          make_temp_dir('test_continueing_mp_custom2.hdf5')]

        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment(irun, filename, continuable=irun == 1)

        self.param_dict={'x':1.0, 'y':2.0}

        for irun in range(len(self.filenames)):
            self.trajs[irun].f_add_parameter(CustomParameter,'x', 1.0)
            self.trajs[irun].f_add_parameter(CustomParameter, 'y', 1.0)


        self.explore_mp(self.trajs[0])
        self.explore_mp(self.trajs[1])

        arg=33
        for irun in range(len(self.filenames)):

            self.envs[irun].f_run(Multiply(), arg)

        traj_name = self.trajs[1].v_name
        continue_folder = os.path.join(self.cnt_folder, self.trajs[1].v_name)

        self.envs.pop()
        self.assertEqual(len(self.envs), 1)
        self.make_environment(1, self.filenames[1], continuable=True, add_time=False,
                              trajectory=traj_name)
        self.trajs[1] = self.envs[1].v_traj
        self.trajs[1].f_load(load_data=pypetconstants.LOAD_NOTHING)
        self._remove_nresults(self.trajs[1], 3, continue_folder)

        results = self.envs[1].f_continue(trajectory_name = traj_name)
        results = sorted(results, key = lambda x: x[0])

        for irun in range(len(self.filenames)):
            self.trajs[irun].f_load(load_parameters=pypetconstants.OVERWRITE_DATA,
                                    load_derived_parameters=pypetconstants.OVERWRITE_DATA,
                                    load_results=pypetconstants.OVERWRITE_DATA,
                                    load_other_data=pypetconstants.OVERWRITE_DATA)

        self.compare_trajectories(self.trajs[1],self.trajs[0])

        for run_name in self.trajs[0].f_iter_runs():
            z = (self.trajs[0].v_idx, self.trajs[0].crun.z)
            self.assertTrue(z in results, '%s not in %s' % (z, results))

        self.assertTrue(len(self.trajs[-1])== len(results))
    def setUp(self):

        self.multiproc = True
        self.mode = 'LOCK'

        self.trajname = make_trajectory_name(self)

        self.filename = make_temp_dir(
            os.path.join('experiments', 'tests', 'HDF5',
                         '%s.hdf5' % self.trajname))
        self.logfolder = make_temp_dir(
            os.path.join('experiments', 'tests', 'Log'))

        random.seed()

        cap_dicts = (
            dict(cpu_cap=0.000001),  # Ensure that these are triggered
            dict(memory_cap=(0.000001, 150.0)),
            dict(swap_cap=0.000001, ))

        cap_dict = cap_dicts[CapTest.cap_count]
        env = Environment(trajectory=self.trajname,
                          filename=self.filename,
                          file_title=self.trajname,
                          log_folder=self.logfolder,
                          logger_names=('pypet', 'test'),
                          log_levels='ERROR',
                          log_stdout=False,
                          results_per_run=5,
                          derived_parameters_per_run=5,
                          multiproc=True,
                          ncores=4,
                          use_pool=False,
                          niceness=check_nice(11),
                          **cap_dict)

        logging.getLogger('test').error('Using Cap: %s and file: %s' %
                                        (str(cap_dict), str(self.filename)))
        # Loop through all possible cap configurations
        # and test one at a time
        CapTest.cap_count += 1
        CapTest.cap_count = CapTest.cap_count % len(cap_dicts)

        traj = env.v_trajectory

        ## Create some parameters
        self.param_dict = {}
        create_param_dict(self.param_dict)
        ### Add some parameter:
        add_params(traj, self.param_dict)

        #remember the trajectory and the environment
        self.traj = traj
        self.env = env
Example #17
0
    def setUp(self):
        self.set_mode()
        self.logfolder = make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'Log'))

        random.seed()
        self.trajname = make_trajectory_name(self)
        self.filename = make_temp_dir(os.path.join('experiments',
                                                    'tests',
                                                    'HDF5',
                                                    'test%s.hdf5' % self.trajname))

        env = Environment(trajectory=self.trajname, filename=self.filename,
                          file_title=self.trajname,
                          log_stdout=self.log_stdout,
                          log_config=get_log_config(),
                          results_per_run=5,
                          wildcard_functions=self.wildcard_functions,
                          derived_parameters_per_run=5,
                          multiproc=self.multiproc,
                          ncores=self.ncores,
                          wrap_mode=self.mode,
                          use_pool=self.use_pool,
                          gc_interval=self.gc_interval,
                          freeze_input=self.freeze_input,
                          fletcher32=self.fletcher32,
                          complevel=self.complevel,
                          complib=self.complib,
                          shuffle=self.shuffle,
                          pandas_append=self.pandas_append,
                          pandas_format=self.pandas_format,
                          encoding=self.encoding,
                          niceness=self.niceness,
                          use_scoop=self.use_scoop,
                          port=self.port,
                          add_time=self.add_time,
                          timeout=self.timeout,
                          graceful_exit=self.graceful_exit)

        traj = env.v_trajectory

        traj.v_standard_parameter=Parameter

        ## Create some parameters
        self.param_dict={}
        create_param_dict(self.param_dict)
        ### Add some parameter:
        add_params(traj,self.param_dict)

        #remember the trajectory and the environment
        self.traj = traj
        self.env = env
Example #18
0
    def setUp(self):
        self.set_mode()
        self.logfolder = make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'Log'))

        random.seed()
        self.trajname = make_trajectory_name(self)
        self.filename = make_temp_dir(os.path.join('experiments',
                                                    'tests',
                                                    'HDF5',
                                                    'test%s.hdf5' % self.trajname))

        env = Environment(trajectory=self.trajname, filename=self.filename,
                          file_title=self.trajname,
                          log_stdout=self.log_stdout,
                          log_config=get_log_config(),
                          results_per_run=5,
                          wildcard_functions=self.wildcard_functions,
                          derived_parameters_per_run=5,
                          multiproc=self.multiproc,
                          ncores=self.ncores,
                          wrap_mode=self.mode,
                          use_pool=self.use_pool,
                          gc_interval=self.gc_interval,
                          freeze_input=self.freeze_input,
                          fletcher32=self.fletcher32,
                          complevel=self.complevel,
                          complib=self.complib,
                          shuffle=self.shuffle,
                          pandas_append=self.pandas_append,
                          pandas_format=self.pandas_format,
                          encoding=self.encoding,
                          niceness=self.niceness,
                          use_scoop=self.use_scoop,
                          port=self.port,
                          add_time=self.add_time,
                          timeout=self.timeout,
                          graceful_exit=self.graceful_exit)

        traj = env.v_trajectory

        traj.v_standard_parameter=Parameter

        ## Create some parameters
        self.param_dict={}
        create_param_dict(self.param_dict)
        ### Add some parameter:
        add_params(traj,self.param_dict)

        #remember the trajectory and the environment
        self.traj = traj
        self.env = env
Example #19
0
 def test_merge_basic_with_separate_files_only_adding_more_trials(self):
     self.filenames = [make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge2.hdf5')),
                       make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge3.hdf5')),
                       make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge4.hdf5'))]
     self.merge_basic_only_adding_more_trials(True)
Example #20
0
 def test_basic_within_separate_file_and_skipping_duplicates_which_leads_to_one_reamianing(self):
     self.filenames = [make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge2.hdf5')),
                       make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge3.hdf5')),
                       make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge4.hdf5'))]
     self. basic_and_skipping_duplicates_which_leads_to_one_remaining()
Example #21
0
 def test_merge_basic_with_separate_files_only_adding_more_trials_slow_merge(self):
     self.filenames = [make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'slow_merge2.hdf5')),
                       make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'slow_merge3.hdf5')),
                       make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'slow_merge4.hdf5'))]
     self.merge_basic_only_adding_more_trials(True, slow_merge=True)
    def setUp(self):

        self.multiproc = True
        self.mode = 'LOCK'

        self.trajname = make_trajectory_name(self)

        self.filename = make_temp_dir(os.path.join('experiments',
                                                    'tests',
                                                    'HDF5',
                                                    '%s.hdf5' % self.trajname))
        self.logfolder = make_temp_dir(os.path.join('experiments', 'tests', 'Log'))

        random.seed()

        cap_dicts = (dict(cpu_cap=0.000001), # Ensure that these are triggered
                      dict(memory_cap=(0.000001, 150.0)),
                      dict(swap_cap=0.000001,))

        cap_dict = cap_dicts[CapTest.cap_count]
        env = Environment(trajectory=self.trajname,filename=self.filename,
                          file_title=self.trajname, log_folder=self.logfolder,
                          logger_names=('pypet', 'test'), log_levels='ERROR',
                          log_stdout=False,
                          results_per_run=5,
                          derived_parameters_per_run=5,
                          multiproc=True,
                          ncores=4,
                          use_pool=False,
                          niceness = check_nice(11),
                          **cap_dict)

        logging.getLogger('test').error('Using Cap: %s and file: %s' % (str(cap_dict),
                                                                        str(self.filename)))
        # Loop through all possible cap configurations
        # and test one at a time
        CapTest.cap_count += 1
        CapTest.cap_count = CapTest.cap_count % len(cap_dicts)

        traj = env.v_trajectory

        ## Create some parameters
        self.param_dict={}
        create_param_dict(self.param_dict)
        ### Add some parameter:
        add_params(traj,self.param_dict)

        #remember the trajectory and the environment
        self.traj = traj
        self.env = env
Example #23
0
 def test_basic_within_separate_file_and_skipping_duplicates_which_leads_to_one_reamianing(self):
     self.filenames = [make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge2_one_remaining.hdf5')),
                       make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge3_one_remaining.hdf5')),
                       make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge4_one_remaining.hdf5'))]
     self. basic_and_skipping_duplicates_which_leads_to_one_remaining()
Example #24
0
    def setUp(self):
        self.set_mode()

        logging.basicConfig(level=logging.ERROR)


        self.logfolder = make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'Log'))

        random.seed()
        self.trajname = make_trajectory_name(self)
        self.filename = make_temp_dir(os.path.join('experiments',
                                                    'tests',
                                                    'HDF5',
                                                    'test%s.hdf5' % self.trajname))

        env = Environment(trajectory=self.trajname, filename=self.filename,
                          file_title=self.trajname,
                          log_stdout=self.log_stdout,

                          log_config=get_log_config(),
                          results_per_run=5,
                          derived_parameters_per_run=5,
                          multiproc=self.multiproc,
                          ncores=self.ncores,
                          wrap_mode=self.mode,
                          use_pool=self.use_pool,
                          fletcher32=self.fletcher32,
                          complevel=self.complevel,
                          complib=self.complib,
                          shuffle=self.shuffle,
                          pandas_append=self.pandas_append,
                          pandas_format=self.pandas_format,
                          encoding=self.encoding)

        traj = env.v_trajectory

        traj.v_standard_parameter=Parameter

        ## Create some parameters
        create_link_params(traj)
        ### Add some parameter:
        explore_params(traj)

        #remember the trajectory and the environment
        self.traj = traj
        self.env = env
    def test_hdf5_store_load_monitorresult(self):
        traj_name = make_trajectory_name(self)
        file_name = make_temp_dir(os.path.join('brian2', 'tests', 'hdf5', 'test_%s.hdf5' % traj_name))
        env = Environment(trajectory=traj_name, filename=file_name, log_config=get_log_config(),
                            dynamic_imports=[Brian2MonitorResult], add_time=False, storage_service=HDF5StorageService)
        traj = env.v_trajectory
        traj.v_standard_result = Brian2MonitorResult
        traj.f_add_result('brian2.single.millivolts_single_a', 10*mvolt, comment='single value a')

        traj.f_add_result('brian2.single.millivolts_single_c', 11*mvolt, comment='single value b')

        traj.f_add_result('brian2.array.millivolts_array_a', [11, 12]*mvolt, comment='array')
        traj.f_add_result('mV1', 42.0*mV)
        # results can hold much more than a single data item:
        traj.f_add_result('ampere1', 1*mA, 44, test=300*mV, test2=[1,2,3],
                          test3=np.array([1,2,3])*mA, comment='Result keeping track of many things')
        traj.f_add_result('integer', 16)
        traj.f_add_result('kHz05', 0.5*kHz)
        traj.f_add_result('nested_array', np.array([[6.,7.,8.],[9.,10.,11.]]) * ms)
        traj.f_add_result('b2a', np.array([1., 2.]) * mV)


        traj.f_store()

        traj2 = load_trajectory(filename=file_name, name=traj_name, dynamic_imports=[Brian2MonitorResult], load_data=2)

        #traj._logger.error('traj :'+str(traj))
        #traj._logger.error('traj2:'+str(traj2))
        self.compare_trajectories(traj, traj2)
Example #26
0
 def test_errors(self):
     tmp = make_temp_dir('cont')
     if dill is not None:
         env1 = Environment(continuable=True, continue_folder=tmp,
                        log_config=None, filename=self.filename)
         with self.assertRaises(ValueError):
             env1.f_run_map(multiply_args, [1], [2], [3])
     with self.assertRaises(ValueError):
         Environment(multiproc=True, use_pool=False, freeze_input=True,
                        filename=self.filename, log_config=None)
     env3 = Environment(log_config=None, filename=self.filename)
     with self.assertRaises(ValueError):
         env3.f_run_map(multiply_args)
     with self.assertRaises(ValueError):
         Environment(use_scoop=True, immediate_postproc=True)
     with self.assertRaises(ValueError):
         Environment(use_pool=True, immediate_postproc=True)
     with self.assertRaises(ValueError):
         Environment(continuable=True, wrap_mode='QUEUE', continue_folder=tmp)
     with self.assertRaises(ValueError):
         Environment(use_scoop=True, wrap_mode='QUEUE')
     with self.assertRaises(ValueError):
         Environment(automatic_storing=False,
                     continuable=True, continue_folder=tmp)
     with self.assertRaises(ValueError):
         Environment(port='www.nosi.de', wrap_mode='LOCK')
Example #27
0
    def test_storage_and_loading(self):
        filename = make_temp_dir('linktest.hdf5')
        traj = Trajectory(filename=filename)

        traj.f_add_parameter_group('test')
        traj.f_add_parameter_group('test2')
        res = traj.f_add_result('kk', 42)

        traj.par.f_add_link('gg', res)

        traj.f_add_link('hh', res)
        traj.f_add_link('jj', traj.par)
        traj.f_add_link('ii', res)

        traj.test.f_add_link('circle1', traj.test2)
        traj.test2.f_add_link('circle2', traj.test)
        traj.test.f_add_link('circle2', traj.test.circle1.circle2)

        traj.f_add_parameter_group('test.ab.bc.cd')
        traj.cd.f_add_link(traj.test)
        traj.test.f_add_link(traj.cd)

        traj.f_store()

        traj2 = Trajectory(filename=filename)
        traj2.f_load(name=traj.v_name, load_data=2)

        self.assertTrue(traj.kk == traj2.gg, '%s != %s' % (traj.kk, traj2.gg))
        self.assertTrue(traj.cd.test is traj.test)

        self.assertTrue(len(traj._linked_by), len(traj2._linked_by))
        self.compare_trajectories(traj, traj2)

        self.assertTrue('jj' in traj2._nn_interface._links_count)
        traj2.f_remove_child('jj')
        self.assertTrue('jj' not in traj2._nn_interface._links_count)
        traj2.f_remove_child('hh')
        traj2.f_remove_child('ii')

        traj2.f_remove_child('parameters', recursive=True)

        traj2.v_auto_load = True

        group = traj2.par.test2.circle2

        self.assertTrue(group is traj2.test)

        retest = traj2.test.circle1

        self.assertTrue(retest is traj2.test2)

        self.assertTrue(traj2.test.circle2 is traj2.test)

        self.assertTrue(traj2.hh == traj2.res.kk)

        traj2.v_auto_load = False
        traj2.f_load_child('jj')
        self.assertTrue(traj2.jj is traj2.par)
        traj2.f_load(load_data=2)
        self.assertTrue(traj2.ii == traj2.res.kk)
    def setUp(self):
        env = Environment(trajectory='Test_'+repr(time.time()).replace('.','_'),
                          filename=make_temp_dir(os.path.join(
                              'experiments',
                              'tests',
                              'briantests',
                              'HDF5',
                               'briantest.hdf5')),
                          file_title='test',
                          log_config=get_log_config(),
                          dynamic_imports=['pypet.brian.parameter.BrianParameter',
                                                        BrianMonitorResult],
                          multiproc=False)

        traj = env.v_trajectory

        #env._set_standard_storage()
        #env._hdf5_queue_writer._hdf5storageservice = LazyStorageService()
        traj = env.v_trajectory
        #traj.set_storage_service(LazyStorageService())

        add_params(traj)
        #traj.mode='Parallel'


        traj.f_explore(cartesian_product({traj.f_get('N').v_full_name:[50,60],
                               traj.f_get('tauw').v_full_name:[30*ms,40*ms]}))

        self.traj = traj

        self.env = env
        self.traj = traj
Example #29
0
    def test_get_default(self):


        traj = Trajectory(name='Testgetdefault', filename=make_temp_dir('autoload.hdf5'),
                          add_time=True)

        traj.v_auto_load = True

        traj.f_add_result('I.am.$.a.mean.resu', 42, comment='Test')

        val = traj.f_get_default('jjjjjjjjjj', 555)
        self.assertTrue(val==555)

        traj.f_store()

        traj.f_remove_child('results', recursive=True)




        val = traj.f_get_default('res.I.am.crun.a.mean.answ', 444, auto_load=True)

        self.assertTrue(val==444)

        val = traj.f_get_default('res.I.am.crun.a.mean.resu', auto_load=True, fast_access=True)

        self.assertTrue(val==42)

        with self.assertRaises(Exception):
            traj.kdsfdsf
    def test_continueing_remove_completed(self):
        self.filenames = [make_temp_dir('test_continueing_remove_completed.hdf5')]

        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename, continuable=True, delete_continue=True)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in range(len(self.filenames)):
            add_params(self.trajs[irun], self.param_dict)


        self.explore(self.trajs[0])

        for irun in range(len(self.filenames)):
            self.make_run(self.envs[irun])

        traj_name = self.trajs[0].v_name
        continue_folder = os.path.join(self.cnt_folder, self.trajs[0].v_name)
        self.assertFalse(os.path.isdir(continue_folder))
Example #31
0
    def test_partially_delete_stuff(self):
        traj = Trajectory(name='TestDelete',
                          filename=make_temp_dir('testpartiallydel.hdf5'),
                          add_time=True)

        res = traj.f_add_result('mytest.test', a='b', c='d')

        traj.f_store()

        self.assertTrue('a' in res)
        traj.f_delete_item(res, delete_only=['a'], remove_from_item=True)

        self.assertTrue('c' in res)
        self.assertTrue('a' not in res)

        res['a'] = 'offf'

        self.assertTrue('a' in res)

        traj.f_load(load_results=3)

        self.assertTrue('a' not in res)
        self.assertTrue('c' in res)

        traj.f_delete_item(res, remove_from_trajectory=True)

        self.assertTrue('results' in traj)
        self.assertTrue(res not in traj)
Example #32
0
    def test_loading_as_new(self):
        filename = make_temp_dir('asnew.h5')
        traj = Trajectory(name='TestPartial', filename=filename, add_time=True)

        traj.f_add_parameter('x', 3)
        traj.f_add_parameter('y', 2)

        traj.f_explore({'x': [12,3,44], 'y':[1,23,4]})

        traj.f_store()

        traj = load_trajectory(name=traj.v_name, filename=filename)

        with self.assertRaises(TypeError):
            traj.f_shrink()

        traj = load_trajectory(name=traj.v_name, filename=filename, as_new=True,
                               new_name='TestTraj', add_time=False)

        self.assertTrue(traj.v_name == 'TestTraj')

        self.assertEqual(len(traj), 3)

        self.assertEqual(len(traj._explored_parameters), 2)

        traj.f_shrink()

        self.assertTrue(len(traj) == 1)
Example #33
0
    def test_auto_load(self):


        traj = Trajectory(name='Testautoload', filename=make_temp_dir('autoload.hdf5'))

        traj.v_auto_load = True

        traj.f_add_result('I.am.$.a.mean.resu', 42, comment='Test')

        traj.f_add_derived_parameter('ffa', 42)

        traj.f_store()

        ffa=traj.f_get('ffa')
        ffa.f_unlock()
        ffa.f_empty()

        self.assertTrue(ffa.f_is_empty())

        traj.f_remove_child('results', recursive=True)

        # check auto load
        val = traj.res.I.am.crun.a.mean.resu

        self.assertTrue(val==42)

        val = traj.ffa

        self.assertTrue(val==42)

        with self.assertRaises(pex.DataNotInStorageError):
            traj.kdsfdsf
Example #34
0
    def test_removal_of_error_parameter(self):

        filename = make_temp_dir('remove_errored.hdf5')
        traj = Trajectory(name='traj', add_time=True, filename=filename)
        traj.f_add_result('iii', 42)
        traj.f_add_result(FakeResult, 'j.j.josie', 43)

        file = traj.v_storage_service.filename
        traj.f_store(only_init=True)
        with self.assertRaises(RuntimeError):
            traj.f_store()

        with ptcompat.open_file(file, mode='r') as fh:
            jj = ptcompat.get_node(fh, where='/%s/results/j/j' % traj.v_name)
            self.assertTrue('josie' not in jj)

        traj.j.j.f_remove_child('josie')
        traj.j.j.f_add_result(FakeResult2, 'josie2', 444)

        traj.f_store()
        with self.assertRaises(pex.NoSuchServiceError):
            traj.f_store_child('results', recursive=True)

        with ptcompat.open_file(file, mode='r') as fh:
            jj = ptcompat.get_node(fh, where='/%s/results/j/j' % traj.v_name)
            self.assertTrue('josie2' in jj)
            josie2 = ptcompat.get_child(jj, 'josie2')
            self.assertTrue('hey' in josie2)
            self.assertTrue('fail' not in josie2)
Example #35
0
    def test_partial_loading(self):
        traj = Trajectory(name='TestPartial', filename=make_temp_dir('testpartially.hdf5'),
                          add_time=True)

        res = traj.f_add_result('mytest.test', a='b', c='d')

        traj.f_store()

        traj.f_remove_child('results', recursive=True)

        traj.f_load_skeleton()

        traj.f_load_item(traj.test, load_only=['a', 'x'])

        self.assertTrue('a' in traj.test)
        self.assertTrue('c' not in traj.test)

        traj.f_remove_child('results', recursive=True)

        traj.f_load_skeleton()

        load_except= ['c', 'd']
        traj.f_load_item(traj.test, load_except=load_except)

        self.assertTrue(len(load_except)==2)

        self.assertTrue('a' in traj.test)
        self.assertTrue('c' not in traj.test)

        with self.assertRaises(ValueError):
            traj.f_load_item(traj.test, load_except=['x'], load_only=['y'])
Example #36
0
    def test_storage_service_errors(self):

        traj = Trajectory(filename=make_temp_dir('testnoservice.hdf5'))

        traj_name = traj.v_name

        # you cannot store stuff before the trajectory was stored once:
        with self.assertRaises(ValueError):
            traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = traj.v_name)

        traj.f_store()

        with self.assertRaises(ValueError):
            traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = 'test')

        with self.assertRaises(pex.NoSuchServiceError):
            traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = traj.v_name)

        with self.assertRaises(ValueError):
            traj.f_load(name='test', index=1)

        with self.assertRaises(RuntimeError):
            traj.v_storage_service.store('LIST', [('LEAF',None,None,None,None)],
                                         trajectory_name = traj.v_name)

        with self.assertRaises(ValueError):
            traj.f_load(index=9999)

        with self.assertRaises(ValueError):
            traj.f_load(name='Non-Existising-Traj')
Example #37
0
 def test_basic_within_same_file_and_skipping_duplicates_which_will_be_all(self):
     self.filenames = [make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge1.hdf5')), 0]
     with self.assertRaises(ValueError):
         self.basic_and_skipping_duplicates_which_will_be_all()
Example #38
0
    def test_merge_all_in_folder(self):

        self.filename = make_temp_dir(os.path.join('experiments','tests','HDF5', 'subfolder',
                                                    'test.hdf5'))

        path, _ = os.path.split(self.filename)

        ntrajs = 4
        total_len = 0
        for irun in range(ntrajs):
            new_filename = os.path.join(path, 'test%d.hdf5' % irun)
            self.envs.append(self._make_env(irun, filename=new_filename))
            self.trajs.append(self.envs[-1].v_traj)
            self.trajs[-1].f_add_parameter('x',0)
            self.trajs[-1].f_add_parameter('y',0)
            self.explore(self.trajs[-1])
            total_len += len(self.trajs[-1])

        for irun in range(ntrajs):
            self.envs[irun].f_run(multiply)

        merge_traj = merge_all_in_folder(path, delete_other_files=True)
        merge_traj.f_load(load_data=2)

        self.assertEqual(len(merge_traj), total_len)
        self.check_if_z_is_correct(merge_traj)
Example #39
0
    def test_df(self):
        filename = make_temp_dir('hdf5errors.hdf5')
        traj = Trajectory(name=make_trajectory_name(self), filename=filename)
        traj.f_store()
        dadict = {
            'hi': [1, 2, 3, 4, 5],
            'shu': ['bi', 'du', 'da', 'ha', 'hui']
        }
        dadict2 = {'answer': [42]}
        traj.f_add_result(
            SharedResult, 'dfs.df',
            SharedPandasFrame()).create_shared_data(data=pd.DataFrame(dadict))
        traj.f_add_result(
            SharedResult, 'dfs.df1',
            SharedPandasFrame()).create_shared_data(data=pd.DataFrame(dadict2))
        traj.f_add_result(SharedResult, 'dfs.df3', SharedPandasFrame())

        for irun in range(10):
            traj.df3.append(traj.df1.read())

        dframe = traj.df3.read()

        self.assertTrue(len(dframe) == 10)

        what = traj.df.select(where='index == 2')
        self.assertTrue(len(what) == 1)
Example #40
0
    def test_get_default(self):


        traj = Trajectory(name='Testgetdefault', filename=make_temp_dir('autoload.hdf5'))

        traj.v_auto_load = True

        traj.f_add_result('I.am.$.a.mean.resu', 42, comment='Test')

        val = traj.f_get_default('jjjjjjjjjj', 555)
        self.assertTrue(val==555)

        traj.f_store()

        traj.f_remove_child('results', recursive=True)




        val = traj.f_get_default('res.I.am.crun.a.mean.answ', 444, auto_load=True)

        self.assertTrue(val==444)

        val = traj.f_get_default('res.I.am.crun.a.mean.resu', auto_load=True, fast_access=True)

        self.assertTrue(val==42)

        with self.assertRaises(Exception):
            traj.kdsfdsf
Example #41
0
    def test_maximum_overview_size(self):

        filename = make_temp_dir('maxisze.hdf5')

        env = Environment(trajectory='Testmigrate', filename=filename,

                          log_config=get_log_config(), add_time=True)

        traj = env.v_trajectory
        for irun in range(pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH):
            traj.f_add_parameter('f%d.x' % irun, 5)

        traj.f_store()


        store = pt.open_file(filename, mode='r+')
        table = store.root._f_get_child(traj.v_name).overview.parameters_overview
        self.assertEquals(table.nrows, pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH)
        store.close()

        for irun in range(pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH,
                  2*pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH):
            traj.f_add_parameter('f%d.x' % irun, 5)

        traj.f_store()

        store = pt.open_file(filename, mode='r+')
        table = store.root._f_get_child(traj.v_name).overview.parameters_overview
        self.assertEquals(table.nrows, pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH)
        store.close()

        env.f_disable_logging()
Example #42
0
    def test_partially_delete_stuff(self):
        traj = Trajectory(name='TestDelete',
                          filename=make_temp_dir('testpartiallydel.hdf5'))

        res = traj.f_add_result('mytest.test', a='b', c='d')

        traj.f_store()

        self.assertTrue('a' in res)
        traj.f_delete_item(res, delete_only=['a'], remove_from_item=True)

        self.assertTrue('c' in res)
        self.assertTrue('a' not in res)

        res['a'] = 'offf'

        self.assertTrue('a' in res)

        traj.f_load(load_results=3)

        self.assertTrue('a' not in res)
        self.assertTrue('c' in res)

        traj.f_delete_item(res, remove_from_trajectory=True)

        self.assertTrue('results' in traj)
        self.assertTrue(res not in traj)
    def test_continueing_remove_completed(self):
        self.filenames = [make_temp_dir('test_continueing_remove_completed.hdf5')]

        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename, continuable=True, delete_continue=True)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in range(len(self.filenames)):
            add_params(self.trajs[irun], self.param_dict)


        self.explore(self.trajs[0])

        for irun in range(len(self.filenames)):
            self.make_run(self.envs[irun])

        traj_name = self.trajs[0].v_name
        continue_folder = os.path.join(self.cnt_folder, self.trajs[0].v_name)
        self.assertFalse(os.path.isdir(continue_folder))
Example #44
0
    def test_overwrite_stuff(self):
        traj = Trajectory(name='TestOverwrite', filename=make_temp_dir('testowrite.hdf5'))

        res = traj.f_add_result('mytest.test', a='b', c='d')

        traj.f_store()

        res['a'] = np.array([1,2,3])
        res['c'] = 123445

        traj.f_store_item(res, overwrite='a', complevel=4)

        # Should emit a warning
        traj.f_store_item(res, overwrite=['a', 'b'])

        traj.f_load(load_results=3)

        res = traj.test

        self.assertTrue((res['a']==np.array([1,2,3])).all())
        self.assertTrue(res['c']=='d')

        res['c'] = 123445

        traj.f_store_item(res, store_data=3)
        res.f_empty()

        traj.f_load(load_results=3)

        self.assertTrue(traj.test['c']==123445)
Example #45
0
    def test_delete_links(self):
        traj = Trajectory(name='TestDelete',
                          filename=make_temp_dir('testpartiallydel.hdf5'),
                          add_time=True)

        res = traj.f_add_result('mytest.test', a='b', c='d')

        traj.f_add_link('x.y', res)
        traj.f_add_link('x.g.h', res)

        traj.f_store()

        traj.f_remove_child('x', recursive=True)
        traj.f_load()

        self.assertEqual(traj.x.y.a, traj.test.a)
        self.assertEqual(traj.x.g.h.a, traj.test.a)

        traj.f_delete_link('x.y', remove_from_trajectory=True)
        traj.f_delete_link((traj.x.g, 'h'), remove_from_trajectory=True)

        traj.f_load()

        with self.assertRaises(AttributeError):
            traj.x.g.h
Example #46
0
    def test_loading_as_new(self):
        filename = make_temp_dir('asnew.h5')
        traj = Trajectory(name='TestPartial', filename=filename)

        traj.f_add_parameter('x', 3)
        traj.f_add_parameter('y', 2)

        traj.f_explore({'x': [12,3,44], 'y':[1,23,4]})

        traj.f_store()

        traj = load_trajectory(name=traj.v_name, filename=filename)

        with self.assertRaises(TypeError):
            traj.f_shrink()

        traj = load_trajectory(name=traj.v_name, filename=filename, as_new=True,
                               new_name='TestTraj', add_time=False)

        self.assertTrue(traj.v_name == 'TestTraj')

        self.assertTrue(len(traj) == 3)

        traj.f_shrink()

        self.assertTrue(len(traj) == 1)
Example #47
0
 def test_make_default_file_when_giving_directory_without_slash(self):
     filename = make_temp_dir('test.hdf5')
     head, tail = os.path.split(filename)
     env = Environment(filename=head)
     the_file_name = env.v_traj.v_name + '.hdf5'
     head, tail = os.path.split(env.v_traj.v_storage_service.filename)
     self.assertEqual(tail, the_file_name)
Example #48
0
    def test_partial_loading(self):
        traj = Trajectory(name='TestPartial', filename=make_temp_dir('testpartially.hdf5'))

        res = traj.f_add_result('mytest.test', a='b', c='d')

        traj.f_store()

        traj.f_remove_child('results', recursive=True)

        traj.f_load_skeleton()

        traj.f_load_item(traj.test, load_only=['a', 'x'])

        self.assertTrue('a' in traj.test)
        self.assertTrue('c' not in traj.test)

        traj.f_remove_child('results', recursive=True)

        traj.f_load_skeleton()

        load_except= ['c', 'd']
        traj.f_load_item(traj.test, load_except=load_except)

        self.assertTrue(len(load_except)==2)

        self.assertTrue('a' in traj.test)
        self.assertTrue('c' not in traj.test)

        with self.assertRaises(ValueError):
            traj.f_load_item(traj.test, load_except=['x'], load_only=['y'])
Example #49
0
    def test_time_display_of_loading(self):
        filename = make_temp_dir('sloooow.hdf5')
        env = Environment(trajectory='traj', add_time=True, filename=filename,
                          log_stdout=False,
                          log_config=get_log_config(),
                          dynamic_imports=SlowResult,
                          display_time=0.1)
        traj = env.v_traj
        res=traj.f_add_result(SlowResult, 'iii', 42, 43, comment='llk')
        traj.f_store()
        service_logger = traj.v_storage_service._logger
        root = logging.getLogger('pypet')
        old_level = root.level
        service_logger.setLevel(logging.INFO)
        root.setLevel(logging.INFO)

        traj.f_load(load_data=3)
        service_logger.setLevel(old_level)
        root.setLevel(old_level)

        path = get_log_path(traj)
        mainfilename = os.path.join(path, 'LOG.txt')
        with open(mainfilename, mode='r') as mainf:
            full_text = mainf.read()
            self.assertTrue('nodes/s)' in full_text)

        env.f_disable_logging()
Example #50
0
    def test_overwrite_stuff(self):
        traj = Trajectory(name='TestOverwrite', filename=make_temp_dir('testowrite.hdf5'),
                          add_time=True)

        res = traj.f_add_result('mytest.test', a='b', c='d')

        traj.f_store()

        res['a'] = np.array([1,2,3])
        res['c'] = 123445

        traj.f_store_item(res, overwrite='a', complevel=4)

        # Should emit a warning
        traj.f_store_item(res, overwrite=['a', 'b'])

        traj.f_load(load_results=3)

        res = traj.test

        self.assertTrue((res['a']==np.array([1,2,3])).all())
        self.assertTrue(res['c']=='d')

        res['c'] = 123445

        traj.f_store_item(res, store_data=3)
        res.f_empty()

        traj.f_load(load_results=3)

        self.assertTrue(traj.test['c']==123445)
Example #51
0
    def setUp(self):
        self.set_mode()

        self.filename = make_temp_dir(os.path.join('experiments','tests','HDF5','sort_tests.hdf5'))

        self.trajname = make_trajectory_name(self)

        env = Environment(trajectory=self.trajname,filename=self.filename,
                          file_title=self.trajname,
                          log_stdout=self.log_stdout,
                          log_config=get_log_config() if self.log_config else None,
                          multiproc=self.multiproc,
                          wrap_mode=self.mode,
                          ncores=self.ncores,
                          use_pool=self.use_pool,
                          use_scoop=self.use_scoop,
                          port=self.port,
                          freeze_input=self.freeze_input,
                          graceful_exit=self.graceful_exit)

        traj = env.v_trajectory


        traj.v_standard_parameter=Parameter

        traj.f_add_parameter('x',99)
        traj.f_add_parameter('y',99)

        self.env=env
        self.traj=traj
Example #52
0
 def test_make_default_file_when_giving_directory_without_slash(self):
     filename = make_temp_dir('test.hdf5')
     head, tail = os.path.split(filename)
     env = Environment(filename=head)
     the_file_name = env.v_traj.v_name + '.hdf5'
     head, tail = os.path.split(env.v_traj.v_storage_service.filename)
     self.assertEqual(tail, the_file_name)
    def test_hdf5_store_load_parameter(self):
        traj_name = make_trajectory_name(self)
        file_name = make_temp_dir(os.path.join('brian2', 'tests', 'hdf5', 'test_%s.hdf5' % traj_name))
        env = Environment(trajectory=traj_name, filename=file_name, log_config=get_log_config(),
                            dynamic_imports=[Brian2Parameter], add_time=False, storage_service=HDF5StorageService)
        traj = env.v_trajectory
        traj.v_standard_parameter = Brian2Parameter
        traj.f_add_parameter('brian2.single.millivolts', 10*mvolt, comment='single value')

        #traj.f_add_parameter('brian2.array.millivolts', [11, 12]*mvolt, comment='array')
        #traj.f_add_parameter('mV1', 42.0*mV)
        #traj.f_add_parameter('ampere1', 1*mA)
        #traj.f_add_parameter('integer', 16)
        #traj.f_add_parameter('kHz05', 0.5*kHz)
        #traj.f_add_parameter('nested_array', np.array([[6.,7.,8.],[9.,10.,11.]]) * ms)
        #traj.f_add_parameter('b2a', np.array([1., 2.]) * mV)

        # We also need to check if explorations work with hdf5 store!
        #explore_dict = {'ampere1': [1*mA, 2*mA, 3*mA],
        #                'integer': [42,43,44],
        #                'b2a': [np.array([1., 2.]) * mV, np.array([1., 4.]) * mV,
        #                       np.array([1., 2.]) * mV]}
        #traj.f_explore(explore_dict)

        traj.f_store()

        traj2 = load_trajectory(filename=file_name, name=traj_name, dynamic_imports=[Brian2Parameter],
                                load_data=2)
        self.compare_trajectories(traj, traj2)
Example #54
0
    def test_time_display_of_loading(self):
        filename = make_temp_dir('sloooow.hdf5')
        env = Environment(trajectory='traj', add_time=True, filename=filename,
                          log_stdout=False,
                          log_config=get_log_config(),
                          dynamic_imports=SlowResult,
                          display_time=0.1)
        traj = env.v_traj
        res=traj.f_add_result(SlowResult, 'iii', 42, 43, comment='llk')
        traj.f_store()
        service_logger = traj.v_storage_service._logger
        root = logging.getLogger('pypet')
        old_level = root.level
        service_logger.setLevel(logging.INFO)
        root.setLevel(logging.INFO)

        traj.f_load(load_data=3)
        service_logger.setLevel(old_level)
        root.setLevel(old_level)

        path = get_log_path(traj)
        mainfilename = os.path.join(path, 'LOG.txt')
        with open(mainfilename, mode='r') as mainf:
            full_text = mainf.read()
            self.assertTrue('nodes/s)' in full_text)

        env.f_disable_logging()
Example #55
0
    def test_auto_load(self):


        traj = Trajectory(name='Testautoload', filename=make_temp_dir('autoload.hdf5'),
                          add_time=True)

        traj.v_auto_load = True

        traj.f_add_result('I.am.$.a.mean.resu', 42, comment='Test')

        traj.f_add_derived_parameter('ffa', 42)

        traj.f_store()

        ffa=traj.f_get('ffa')
        ffa.f_unlock()
        ffa.f_empty()

        self.assertTrue(ffa.f_is_empty())

        traj.f_remove_child('results', recursive=True)

        # check auto load
        val = traj.res.I.am.crun.a.mean.resu

        self.assertTrue(val==42)

        val = traj.ffa

        self.assertTrue(val==42)

        with self.assertRaises(pex.DataNotInStorageError):
            traj.kdsfdsf
Example #56
0
 def test_errors(self):
     tmp = make_temp_dir('cont')
     if dill is not None:
         env1 = Environment(continuable=True, continue_folder=tmp,
                        log_config=None, filename=self.filename)
         with self.assertRaises(ValueError):
             env1.f_run_map(multiply_args, [1], [2], [3])
     with self.assertRaises(ValueError):
         Environment(multiproc=True, use_pool=False, freeze_input=True,
                        filename=self.filename, log_config=None)
     env3 = Environment(log_config=None, filename=self.filename)
     with self.assertRaises(ValueError):
         env3.f_run_map(multiply_args)
     with self.assertRaises(ValueError):
         Environment(use_scoop=True, immediate_postproc=True)
     with self.assertRaises(ValueError):
         Environment(use_pool=True, immediate_postproc=True)
     with self.assertRaises(ValueError):
         Environment(continuable=True, wrap_mode='QUEUE', continue_folder=tmp)
     with self.assertRaises(ValueError):
         Environment(use_scoop=True, wrap_mode='QUEUE')
     with self.assertRaises(ValueError):
         Environment(automatic_storing=False,
                     continuable=True, continue_folder=tmp)
     with self.assertRaises(ValueError):
         Environment(port='www.nosi.de', wrap_mode='LOCK')
Example #57
0
    def test_errors(self):
        filename = make_temp_dir("hdf5errors.hdf5")
        traj = Trajectory(name=make_trajectory_name(self), filename=filename)
        trajname = traj.v_name

        npearray = np.ones((2, 10, 3), dtype=np.float)
        thevlarray = np.array([compat.tobytes("j"), 22.2, compat.tobytes("gutter")])

        with self.assertRaises(TypeError):
            traj.f_add_result(SharedResult, "arrays.vlarray", SharedVLArray()).create_shared_data(obj=thevlarray)
        traj.f_store()
        traj.arrays.vlarray.create_shared_data(obj=thevlarray)
        traj.f_add_result(SharedResult, "arrays.array", SharedArray()).create_shared_data(data=npearray)
        traj.arrays.f_add_result(SharedResult, "super.carray", SharedCArray(), comment="carray").create_shared_data(
            shape=(10, 10), atom=pt.atom.FloatAtom()
        )
        traj.arrays.f_add_result(SharedResult, "earray", SharedEArray()).create_shared_data("earray", obj=npearray)

        traj.f_store()

        with self.assertRaises(TypeError):
            traj.arrays.array.iter_rows()

        with StorageContextManager(traj) as cm:
            with self.assertRaises(RuntimeError):
                with StorageContextManager(traj) as cm2:
                    pass
            self.assertTrue(traj.v_storage_service.is_open)
            with self.assertRaises(RuntimeError):
                StorageContextManager(traj).f_open_store()

        self.assertFalse(traj.v_storage_service.is_open)
Example #58
0
    def setUp(self):
        self.set_mode()

        self.filename = make_temp_dir(os.path.join('experiments','tests','HDF5','sort_tests.hdf5'))

        self.trajname = make_trajectory_name(self)

        env = Environment(trajectory=self.trajname,filename=self.filename,
                          file_title=self.trajname,
                          log_stdout=self.log_stdout,
                          log_config=get_log_config() if self.log_config else None,
                          multiproc=self.multiproc,
                          wrap_mode=self.mode,
                          ncores=self.ncores,
                          use_pool=self.use_pool,
                          use_scoop=self.use_scoop,
                          port=self.port,
                          freeze_input=self.freeze_input,)

        traj = env.v_trajectory


        traj.v_standard_parameter=Parameter

        traj.f_add_parameter('x',0)
        traj.f_add_parameter('y',0)

        self.env=env
        self.traj=traj
Example #59
0
    def test_maximum_overview_size(self):

        filename = make_temp_dir('maxisze.hdf5')

        env = Environment(trajectory='Testmigrate', filename=filename,

                          log_config=get_log_config())

        traj = env.v_trajectory
        for irun in range(pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH):
            traj.f_add_parameter('f%d.x' % irun, 5)

        traj.f_store()


        store = ptcompat.open_file(filename, mode='r+')
        table = ptcompat.get_child(store.root,traj.v_name).overview.parameters_overview
        self.assertEquals(table.nrows, pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH)
        store.close()

        for irun in range(pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH,
                  2*pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH):
            traj.f_add_parameter('f%d.x' % irun, 5)

        traj.f_store()

        store = ptcompat.open_file(filename, mode='r+')
        table = ptcompat.get_child(store.root,traj.v_name).overview.parameters_overview
        self.assertEquals(table.nrows, pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH)
        store.close()

        env.f_disable_logging()
Example #60
0
    def setUp(self):
        env = Environment(
            trajectory='Test_' + repr(time.time()).replace('.', '_'),
            filename=make_temp_dir(
                os.path.join('experiments', 'tests', 'briantests', 'HDF5',
                             'briantest.hdf5')),
            file_title='test',
            log_config=get_log_config(),
            dynamic_imports=[
                'pypet.brian.parameter.BrianParameter', BrianMonitorResult
            ],
            multiproc=False)

        traj = env.v_trajectory

        #env._set_standard_storage()
        #env._hdf5_queue_writer._hdf5storageservice = LazyStorageService()
        traj = env.v_trajectory
        #traj.set_storage_service(LazyStorageService())

        add_params(traj)
        #traj.mode='Parallel'

        traj.f_explore(
            cartesian_product({
                traj.f_get('N').v_full_name: [50, 60],
                traj.f_get('tauw').v_full_name: [30 * ms, 40 * ms]
            }))

        self.traj = traj

        self.env = env
        self.traj = traj