Пример #1
0
    def test_continueing_remove_completed(self):
        self.filenames = [make_temp_dir('test_continueing_remove_completed.hdf5')]

        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename, continuable=True, delete_continue=True)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in range(len(self.filenames)):
            add_params(self.trajs[irun], self.param_dict)


        self.explore(self.trajs[0])

        for irun in range(len(self.filenames)):
            self.make_run(self.envs[irun])

        traj_name = self.trajs[0].v_name
        continue_folder = os.path.join(self.cnt_folder, self.trajs[0].v_name)
        self.assertFalse(os.path.isdir(continue_folder))
    def test_continueing_remove_completed(self):
        self.filenames = [make_temp_dir('test_continueing_remove_completed.hdf5')]

        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename, continuable=True, delete_continue=True)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in range(len(self.filenames)):
            add_params(self.trajs[irun], self.param_dict)


        self.explore(self.trajs[0])

        for irun in range(len(self.filenames)):
            self.make_run(self.envs[irun])

        traj_name = self.trajs[0].v_name
        continue_folder = os.path.join(self.cnt_folder, self.trajs[0].v_name)
        self.assertFalse(os.path.isdir(continue_folder))
Пример #3
0
    def merge_basic_only_adding_more_trials(self, copy_nodes=False, delete_traj=False,
                                            slow_merge=False):

        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in [0,1,2]:
            add_params(self.trajs[irun], self.param_dict)


        self.explore(self.trajs[0])
        self.explore(self.trajs[1])
        self.compare_explore_more_trials(self.trajs[2])

        for irun in [0,1,2]:
            self.make_run(self.envs[irun])

        for irun in [0,1,2]:
            self.trajs[irun].f_load_skeleton()
            self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA,
                                    load_derived_parameters=pypetconstants.UPDATE_DATA,
                                    load_results=pypetconstants.UPDATE_DATA,
                                    load_other_data=pypetconstants.UPDATE_DATA)


        self.trajs[1].f_add_result('gg.rrororo33o333o3o3oo3',1234567890)
        self.trajs[1].f_store_item('rrororo33o333o3o3oo3')
        self.trajs[1].res.gg.v_annotations['lala'] = 'Sonnenschein'
        self.trajs[1].f_store_item('gg')
        self.trajs[2].f_add_result('gg.rrororo33o333o3o3oo3',1234567890)
        self.trajs[2].f_store_item('rrororo33o333o3o3oo3')
        self.trajs[2].res.gg.v_annotations['lala'] = 'Sonnenschein'
        self.trajs[2].f_store_item('gg')

        ##f_merge without destroying the original trajectory
        merged_traj = self.trajs[0]

        self.trajs[1].f_remove_child('results', recursive=True)

        merged_traj.f_merge(self.trajs[1], move_data=not copy_nodes,
                            delete_other_trajectory=delete_traj,
                            trial_parameter='trial',
                            slow_merge=slow_merge)

        merged_traj.f_load(load_parameters=pypetconstants.UPDATE_DATA,
                           load_derived_parameters=pypetconstants.UPDATE_DATA,
                           load_results=pypetconstants.UPDATE_DATA,
                           load_other_data=pypetconstants.UPDATE_DATA)

        self.compare_trajectories(merged_traj,self.trajs[2])
Пример #4
0
    def test_removal(self):
        self.filenames = [
            make_temp_dir(
                os.path.join('experiments', 'tests', 'HDF5', 'removal.hdf5')),
            0
        ]

        self.envs = []
        self.trajs = []

        for irun, filename in enumerate(self.filenames):
            if isinstance(filename, int):
                filename = self.filenames[filename]

            self.make_environment(irun, filename)

        self.param_dict = {}
        create_param_dict(self.param_dict)

        for irun in range(len(self.filenames)):
            add_params(self.trajs[irun], self.param_dict)

        self.explore(self.trajs[0])
        self.explore(self.trajs[1])

        for irun in range(len(self.filenames)):
            self.make_run(self.envs[irun])

        self.trajs[0].f_add_parameter('Delete.Me', 'I will be deleted!')
        self.trajs[0].f_store_item('Delete.Me')

        self.trajs[0].f_remove_item(self.trajs[0].f_get('Delete.Me'))

        self.assertTrue('Delete.Me' not in self.trajs[0],
                        'Delete.Me is still in traj')

        self.trajs[0].f_load_skeleton()
        self.trajs[0].f_load_item('Delete.Me')
        self.trajs[0].f_delete_item(self.trajs[0].f_get('Delete.Me'),
                                    remove_from_trajectory=True)

        self.trajs[0].f_load_skeleton()
        self.assertTrue(not 'Delete.Me' in self.trajs[0],
                        'Delete.Me is still in traj')

        for irun in range(len(self.filenames)):
            self.trajs[irun].f_load_child(
                'results',
                recursive=True,
                load_data=pypetconstants.OVERWRITE_DATA)
            self.trajs[irun].f_load_child(
                'derived_parameters',
                recursive=True,
                load_data=pypetconstants.OVERWRITE_DATA)

        self.compare_trajectories(self.trajs[0], self.trajs[1])
Пример #5
0
    def test_merging_errors_if_trajs_do_not_match(self):

        self.filenames = [make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'HDF5',
                                                      'merge_errors.hdf5')), 0]

        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in [0,1]:
            add_params(self.trajs[irun], self.param_dict)


        self.explore(self.trajs[0])
        self.explore(self.trajs[1])

        for irun in [0,1]:
            self.make_run(self.envs[irun])

        for irun in [0,1]:
            self.trajs[irun].f_load_skeleton()
            self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA,
                                    load_derived_parameters=pypetconstants.UPDATE_DATA,
                                    load_results=pypetconstants.UPDATE_DATA,
                                    load_other_data=pypetconstants.UPDATE_DATA)


        self.trajs[0].f_add_parameter('Merging.Denied', 13)

        ##f_merge without destroying the original trajectory
        merged_traj = self.trajs[0]

        # We cannot merge trajectories which parameters differ
        with self.assertRaises(TypeError):
            merged_traj.f_merge(self.trajs[1])

        self.trajs[1].f_add_parameter('Merging.Denied', 13.13)
        # We cannot merge trajectories where parameters differ in type
        with self.assertRaises(TypeError):
            merged_traj.f_merge(self.trajs[1])

        self.trajs[1].f_get('Denied').f_unlock()
        self.trajs[1].f_get('Denied').f_set(15)

        merged_traj.f_merge(self.trajs[1])
Пример #6
0
    def test_merging_errors_if_trajs_do_not_match(self):

        self.filenames = [make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'HDF5',
                                                      'merge_errors.hdf5')), 0]

        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in [0,1]:
            add_params(self.trajs[irun], self.param_dict)


        self.explore(self.trajs[0])
        self.explore(self.trajs[1])

        for irun in [0,1]:
            self.make_run(self.envs[irun])

        for irun in [0,1]:
            self.trajs[irun].f_load_skeleton()
            self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA,
                                    load_derived_parameters=pypetconstants.UPDATE_DATA,
                                    load_results=pypetconstants.UPDATE_DATA,
                                    load_other_data=pypetconstants.UPDATE_DATA)


        self.trajs[0].f_add_parameter('Merging.Denied', 13)

        ##f_merge without destroying the original trajectory
        merged_traj = self.trajs[0]

        # We cannot merge trajectories which parameters differ
        with self.assertRaises(TypeError):
            merged_traj.f_merge(self.trajs[1])

        self.trajs[1].f_add_parameter('Merging.Denied', 13.13)
        # We cannot merge trajectories where parameters differ in type
        with self.assertRaises(TypeError):
            merged_traj.f_merge(self.trajs[1])

        self.trajs[1].f_get('Denied').f_unlock()
        self.trajs[1].f_get('Denied').f_set(15)

        merged_traj.f_merge(self.trajs[1])
Пример #7
0
    def test_merging_trajectories_in_different_subspace(self):
        self.filenames = [make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'HDF5',
                                                      'merge_diff_subspace.hdf5')), 0, 0]
        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename, wildcard_functions =
            {('$', 'crun') : my_run_func, ('$set', 'crunset'): my_set_func})

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in [0,1,2]:
            add_params(self.trajs[irun], self.param_dict)


        self.explore(self.trajs[0])
        self.explore2(self.trajs[1])
        self.compare_explore_diff_subspace(self.trajs[2])

        for irun in [0,1,2]:
            self.make_run(self.envs[irun])

        for irun in [0,1,2]:
            self.trajs[irun].f_load_skeleton()
            self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA,
                                    load_derived_parameters=pypetconstants.UPDATE_DATA,
                                    load_results=pypetconstants.UPDATE_DATA,
                                    load_other_data=pypetconstants.UPDATE_DATA)


        self.trajs[1].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[1].f_store_item('rrororo33o333o3o3oo3')
        self.trajs[2].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[2].f_store_item('rrororo33o333o3o3oo3')

        ##f_merge without destroying the original trajectory
        merged_traj = self.trajs[0]

        merged_traj.f_merge(self.trajs[1], move_data=True,
                            delete_other_trajectory=True)

        merged_traj.f_load(load_parameters=pypetconstants.UPDATE_DATA,
                           load_derived_parameters=pypetconstants.UPDATE_DATA,
                           load_results=pypetconstants.UPDATE_DATA,
                           load_other_data=pypetconstants.UPDATE_DATA)

        self.compare_trajectories(merged_traj,self.trajs[2])
    def setUp(self):

        self.multiproc = True
        self.mode = 'LOCK'

        self.trajname = make_trajectory_name(self)

        self.filename = make_temp_dir(
            os.path.join('experiments', 'tests', 'HDF5',
                         '%s.hdf5' % self.trajname))
        self.logfolder = make_temp_dir(
            os.path.join('experiments', 'tests', 'Log'))

        random.seed()

        cap_dicts = (
            dict(cpu_cap=0.000001),  # Ensure that these are triggered
            dict(memory_cap=(0.000001, 150.0)),
            dict(swap_cap=0.000001, ))

        cap_dict = cap_dicts[CapTest.cap_count]
        env = Environment(trajectory=self.trajname,
                          filename=self.filename,
                          file_title=self.trajname,
                          log_folder=self.logfolder,
                          logger_names=('pypet', 'test'),
                          log_levels='ERROR',
                          log_stdout=False,
                          results_per_run=5,
                          derived_parameters_per_run=5,
                          multiproc=True,
                          ncores=4,
                          use_pool=False,
                          niceness=check_nice(11),
                          **cap_dict)

        logging.getLogger('test').error('Using Cap: %s and file: %s' %
                                        (str(cap_dict), str(self.filename)))
        # Loop through all possible cap configurations
        # and test one at a time
        CapTest.cap_count += 1
        CapTest.cap_count = CapTest.cap_count % len(cap_dicts)

        traj = env.v_trajectory

        ## Create some parameters
        self.param_dict = {}
        create_param_dict(self.param_dict)
        ### Add some parameter:
        add_params(traj, self.param_dict)

        #remember the trajectory and the environment
        self.traj = traj
        self.env = env
Пример #9
0
    def test_merging_trajectories_in_different_subspace(self):
        self.filenames = [make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'HDF5',
                                                      'merge_diff_subspace.hdf5')), 0, 0]
        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename, wildcard_functions =
            {('$', 'crun') : my_run_func, ('$set', 'crunset'): my_set_func})

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in [0,1,2]:
            add_params(self.trajs[irun], self.param_dict)


        self.explore(self.trajs[0])
        self.explore2(self.trajs[1])
        self.compare_explore_diff_subspace(self.trajs[2])

        for irun in [0,1,2]:
            self.make_run(self.envs[irun])

        for irun in [0,1,2]:
            self.trajs[irun].f_load_skeleton()
            self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA,
                                    load_derived_parameters=pypetconstants.UPDATE_DATA,
                                    load_results=pypetconstants.UPDATE_DATA,
                                    load_other_data=pypetconstants.UPDATE_DATA)


        self.trajs[1].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[1].f_store_item('rrororo33o333o3o3oo3')
        self.trajs[2].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[2].f_store_item('rrororo33o333o3o3oo3')

        ##f_merge without destroying the original trajectory
        merged_traj = self.trajs[0]

        merged_traj.f_merge(self.trajs[1], move_data=True,
                            delete_other_trajectory=True)

        merged_traj.f_load(load_parameters=pypetconstants.UPDATE_DATA,
                           load_derived_parameters=pypetconstants.UPDATE_DATA,
                           load_results=pypetconstants.UPDATE_DATA,
                           load_other_data=pypetconstants.UPDATE_DATA)

        self.compare_trajectories(merged_traj,self.trajs[2])
Пример #10
0
    def setUp(self):
        self.set_mode()
        self.logfolder = make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'Log'))

        random.seed()
        self.trajname = make_trajectory_name(self)
        self.filename = make_temp_dir(os.path.join('experiments',
                                                    'tests',
                                                    'HDF5',
                                                    'test%s.hdf5' % self.trajname))

        env = Environment(trajectory=self.trajname, filename=self.filename,
                          file_title=self.trajname,
                          log_stdout=self.log_stdout,
                          log_config=get_log_config(),
                          results_per_run=5,
                          wildcard_functions=self.wildcard_functions,
                          derived_parameters_per_run=5,
                          multiproc=self.multiproc,
                          ncores=self.ncores,
                          wrap_mode=self.mode,
                          use_pool=self.use_pool,
                          gc_interval=self.gc_interval,
                          freeze_input=self.freeze_input,
                          fletcher32=self.fletcher32,
                          complevel=self.complevel,
                          complib=self.complib,
                          shuffle=self.shuffle,
                          pandas_append=self.pandas_append,
                          pandas_format=self.pandas_format,
                          encoding=self.encoding,
                          niceness=self.niceness,
                          use_scoop=self.use_scoop,
                          port=self.port,
                          add_time=self.add_time,
                          timeout=self.timeout,
                          graceful_exit=self.graceful_exit)

        traj = env.v_trajectory

        traj.v_standard_parameter=Parameter

        ## Create some parameters
        self.param_dict={}
        create_param_dict(self.param_dict)
        ### Add some parameter:
        add_params(traj,self.param_dict)

        #remember the trajectory and the environment
        self.traj = traj
        self.env = env
Пример #11
0
    def setUp(self):
        self.set_mode()
        self.logfolder = make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'Log'))

        random.seed()
        self.trajname = make_trajectory_name(self)
        self.filename = make_temp_dir(os.path.join('experiments',
                                                    'tests',
                                                    'HDF5',
                                                    'test%s.hdf5' % self.trajname))

        env = Environment(trajectory=self.trajname, filename=self.filename,
                          file_title=self.trajname,
                          log_stdout=self.log_stdout,
                          log_config=get_log_config(),
                          results_per_run=5,
                          wildcard_functions=self.wildcard_functions,
                          derived_parameters_per_run=5,
                          multiproc=self.multiproc,
                          ncores=self.ncores,
                          wrap_mode=self.mode,
                          use_pool=self.use_pool,
                          gc_interval=self.gc_interval,
                          freeze_input=self.freeze_input,
                          fletcher32=self.fletcher32,
                          complevel=self.complevel,
                          complib=self.complib,
                          shuffle=self.shuffle,
                          pandas_append=self.pandas_append,
                          pandas_format=self.pandas_format,
                          encoding=self.encoding,
                          niceness=self.niceness,
                          use_scoop=self.use_scoop,
                          port=self.port,
                          add_time=self.add_time,
                          timeout=self.timeout,
                          graceful_exit=self.graceful_exit)

        traj = env.v_trajectory

        traj.v_standard_parameter=Parameter

        ## Create some parameters
        self.param_dict={}
        create_param_dict(self.param_dict)
        ### Add some parameter:
        add_params(traj,self.param_dict)

        #remember the trajectory and the environment
        self.traj = traj
        self.env = env
    def test_removal(self):
        self.filenames = [make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'HDF5',
                                                      'removal.hdf5')), 0]

        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in range(len(self.filenames)):
            add_params(self.trajs[irun], self.param_dict)


        self.explore(self.trajs[0])
        self.explore(self.trajs[1])


        for irun in range(len(self.filenames)):
            self.make_run(self.envs[irun])

        self.trajs[0].f_add_parameter('Delete.Me', 'I will be deleted!')
        self.trajs[0].f_store_item('Delete.Me')

        self.trajs[0].f_remove_item(self.trajs[0].f_get('Delete.Me'))

        self.assertTrue('Delete.Me' not in self.trajs[0],'Delete.Me is still in traj')

        self.trajs[0].f_load_skeleton()
        self.trajs[0].f_load_item('Delete.Me')
        self.trajs[0].f_delete_item(self.trajs[0].f_get('Delete.Me'),
                                        remove_from_trajectory=True)

        self.trajs[0].f_load_skeleton()
        self.assertTrue(not 'Delete.Me' in self.trajs[0],'Delete.Me is still in traj')


        for irun in range(len(self.filenames)):
            self.trajs[irun].f_load_skeleton()
            self.trajs[irun].f_load_child('results',recursive=True,load_data=pypetconstants.UPDATE_DATA)
            self.trajs[irun].f_load_child('derived_parameters',recursive=True,load_data=pypetconstants.UPDATE_DATA)

        self.compare_trajectories(self.trajs[0],self.trajs[1])
    def setUp(self):

        self.multiproc = True
        self.mode = 'LOCK'

        self.trajname = make_trajectory_name(self)

        self.filename = make_temp_dir(os.path.join('experiments',
                                                    'tests',
                                                    'HDF5',
                                                    '%s.hdf5' % self.trajname))
        self.logfolder = make_temp_dir(os.path.join('experiments', 'tests', 'Log'))

        random.seed()

        cap_dicts = (dict(cpu_cap=0.000001), # Ensure that these are triggered
                      dict(memory_cap=(0.000001, 150.0)),
                      dict(swap_cap=0.000001,))

        cap_dict = cap_dicts[CapTest.cap_count]
        env = Environment(trajectory=self.trajname,filename=self.filename,
                          file_title=self.trajname, log_folder=self.logfolder,
                          logger_names=('pypet', 'test'), log_levels='ERROR',
                          log_stdout=False,
                          results_per_run=5,
                          derived_parameters_per_run=5,
                          multiproc=True,
                          ncores=4,
                          use_pool=False,
                          niceness = check_nice(11),
                          **cap_dict)

        logging.getLogger('test').error('Using Cap: %s and file: %s' % (str(cap_dict),
                                                                        str(self.filename)))
        # Loop through all possible cap configurations
        # and test one at a time
        CapTest.cap_count += 1
        CapTest.cap_count = CapTest.cap_count % len(cap_dicts)

        traj = env.v_trajectory

        ## Create some parameters
        self.param_dict={}
        create_param_dict(self.param_dict)
        ### Add some parameter:
        add_params(traj,self.param_dict)

        #remember the trajectory and the environment
        self.traj = traj
        self.env = env
Пример #14
0
    def basic_and_skipping_duplicates_which_will_be_all(self):


        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in [0,1]:
            add_params(self.trajs[irun],self.param_dict)


        self.explore(self.trajs[0])
        self.explore(self.trajs[1])


        for irun in [0,1]:
            self.make_run(self.envs[irun])

        for irun in [0,1]:
            self.trajs[irun].f_load_skeleton()
            self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA,
                                    load_derived_parameters=pypetconstants.UPDATE_DATA,
                                    load_results=pypetconstants.UPDATE_DATA,
                                    load_other_data=pypetconstants.UPDATE_DATA)


        self.trajs[0].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[0].f_store_item('rrororo33o333o3o3oo3')
        self.trajs[1].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[1].f_store_item('rrororo33o333o3o3oo3')

        ##f_merge without destroying the original trajectory
        merged_traj = self.trajs[0]
        merged_traj.f_merge(self.trajs[1], move_data=False, delete_other_trajectory=False, remove_duplicates=True)
        merged_traj.f_load_skeleton()
        merged_traj.f_load(load_parameters=pypetconstants.UPDATE_DATA,
                           load_derived_parameters=pypetconstants.UPDATE_DATA,
                           load_results=pypetconstants.UPDATE_DATA,
                           load_other_data=pypetconstants.UPDATE_DATA)

        self.compare_trajectories(merged_traj,self.trajs[1])
Пример #15
0
    def merge_basic_only_adding_more_trials_with_backup(self,copy_nodes):


        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in [0,1,2]:
            add_params(self.trajs[irun],self.param_dict)


        self.explore(self.trajs[0])
        self.explore(self.trajs[1])
        self.compare_explore_more_trials(self.trajs[2])

        for irun in [0,1,2]:
            self.make_run(self.envs[irun])

        for irun in [0,1,2]:
            self.trajs[irun].f_load_skeleton()
            self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA,
                                    load_derived_parameters=pypetconstants.UPDATE_DATA,
                                    load_results=pypetconstants.UPDATE_DATA,
                                    load_other_data=pypetconstants.UPDATE_DATA)

        self.trajs[1].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[1].f_store_item('rrororo33o333o3o3oo3')
        self.trajs[2].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[2].f_store_item('rrororo33o333o3o3oo3')

        ##f_merge without destroying the original trajectory
        merged_traj = self.trajs[0]
        merged_traj.f_merge(self.trajs[1], move_data=not copy_nodes, delete_other_trajectory=False, trial_parameter='trial',
                            backup_filename=1)
        merged_traj.f_load_skeleton()
        merged_traj.f_load(load_parameters=pypetconstants.UPDATE_DATA,
                           load_derived_parameters=pypetconstants.UPDATE_DATA,
                           load_results=pypetconstants.UPDATE_DATA,
                           load_other_data=pypetconstants.UPDATE_DATA)

        self.compare_trajectories(merged_traj,self.trajs[2])
Пример #16
0
    def merge_basic_only_adding_more_trials_with_backup(self,copy_nodes):


        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in [0,1,2]:
            add_params(self.trajs[irun],self.param_dict)


        self.explore(self.trajs[0])
        self.explore(self.trajs[1])
        self.compare_explore_more_trials(self.trajs[2])

        for irun in [0,1,2]:
            self.make_run(self.envs[irun])

        for irun in [0,1,2]:
            self.trajs[irun].f_load_skeleton()
            self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA,
                                    load_derived_parameters=pypetconstants.UPDATE_DATA,
                                    load_results=pypetconstants.UPDATE_DATA,
                                    load_other_data=pypetconstants.UPDATE_DATA)

        self.trajs[1].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[1].f_store_item('rrororo33o333o3o3oo3')
        self.trajs[2].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[2].f_store_item('rrororo33o333o3o3oo3')

        ##f_merge without destroying the original trajectory
        merged_traj = self.trajs[0]
        merged_traj.f_merge(self.trajs[1], move_data=not copy_nodes, delete_other_trajectory=False, trial_parameter='trial',
                            backup_filename=1)
        merged_traj.f_load_skeleton()
        merged_traj.f_load(load_parameters=pypetconstants.UPDATE_DATA,
                           load_derived_parameters=pypetconstants.UPDATE_DATA,
                           load_results=pypetconstants.UPDATE_DATA,
                           load_other_data=pypetconstants.UPDATE_DATA)

        self.compare_trajectories(merged_traj,self.trajs[2])
Пример #17
0
    def basic_and_skipping_duplicates_which_will_be_all(self):


        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in [0,1]:
            add_params(self.trajs[irun],self.param_dict)


        self.explore(self.trajs[0])
        self.explore(self.trajs[1])


        for irun in [0,1]:
            self.make_run(self.envs[irun])

        for irun in [0,1]:
            self.trajs[irun].f_load_skeleton()
            self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA,
                                    load_derived_parameters=pypetconstants.UPDATE_DATA,
                                    load_results=pypetconstants.UPDATE_DATA,
                                    load_other_data=pypetconstants.UPDATE_DATA)


        self.trajs[0].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[0].f_store_item('rrororo33o333o3o3oo3')
        self.trajs[1].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[1].f_store_item('rrororo33o333o3o3oo3')

        ##f_merge without destroying the original trajectory
        merged_traj = self.trajs[0]
        merged_traj.f_merge(self.trajs[1], move_data=False, delete_other_trajectory=False, remove_duplicates=True)
        merged_traj.f_load_skeleton()
        merged_traj.f_load(load_parameters=pypetconstants.UPDATE_DATA,
                           load_derived_parameters=pypetconstants.UPDATE_DATA,
                           load_results=pypetconstants.UPDATE_DATA,
                           load_other_data=pypetconstants.UPDATE_DATA)

        self.compare_trajectories(merged_traj,self.trajs[1])
    def test_multiple_storage_and_loading(self):
        self.filenames = [make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'HDF5',
                                                      'merge1.hdf5')), 0]



        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in range(len(self.filenames)):
            add_params(self.trajs[irun],self.param_dict)


        self.explore(self.trajs[0])
        self.explore(self.trajs[1])





        for irun in range(len(self.filenames)):
            self.make_run(self.envs[irun])

        #self.trajs[0].f_store()

        temp_sservice = self.trajs[0].v_storage_service
        temp_name = self.trajs[0].v_name

        self.trajs[0] = Trajectory()
        self.trajs[0].v_storage_service=temp_sservice
        self.trajs[0].f_load(name=temp_name, as_new=False, load_parameters=2,
                             load_derived_parameters=2, load_results=2, load_other_data=2)
        #self.trajs[0].f_load(trajectory_name=temp_name,as_new=False, load_params=2, load_derived_params=2, load_results=2)

        self.trajs[1].f_load_skeleton()
        self.trajs[1].f_load_items(self.trajs[1].f_to_dict().values(),only_empties=True)
        self.compare_trajectories(self.trajs[0],self.trajs[1])
Пример #19
0
    def test_continueing(self):
        self.filenames = [make_temp_dir('test_continuing.hdf5'), 0]

        self.envs = []
        self.trajs = []

        for irun, filename in enumerate(self.filenames):
            if isinstance(filename, int):
                filename = self.filenames[filename]

            self.make_environment(irun,
                                  filename,
                                  continuable=irun == 1,
                                  delete_continue=False)

        self.param_dict = {}
        create_param_dict(self.param_dict)

        for irun in range(len(self.filenames)):
            add_params(self.trajs[irun], self.param_dict)

        self.explore(self.trajs[0])
        self.explore(self.trajs[1])

        for irun in range(len(self.filenames)):
            self.make_run(self.envs[irun])

        traj_name = self.trajs[1].v_name
        continue_folder = os.path.join(self.cnt_folder, self.trajs[1].v_name)

        self.trajs[1] = self.envs[1].v_trajectory
        self._remove_nresults(self.trajs[1], 3, continue_folder)

        self.envs[1].v_current_idx = 0
        results = self.envs[1].resume(trajectory_name=traj_name)

        for irun in range(len(self.filenames)):
            self.trajs[irun].f_load(
                load_parameters=pypetconstants.OVERWRITE_DATA,
                load_derived_parameters=pypetconstants.OVERWRITE_DATA,
                load_results=pypetconstants.OVERWRITE_DATA,
                load_other_data=pypetconstants.OVERWRITE_DATA)

        self.compare_trajectories(self.trajs[0], self.trajs[1])
        #shutil.rmtree(self.cnt_folder)

        self.assertEqual(len(self.trajs[1]), len(results))
Пример #20
0
    def test_multiple_storage_and_loading(self):
        self.filenames = [
            make_temp_dir(
                os.path.join('experiments', 'tests', 'HDF5',
                             'multiple_storage_and_loading.hdf5')), 0
        ]

        self.envs = []
        self.trajs = []

        for irun, filename in enumerate(self.filenames):
            if isinstance(filename, int):
                filename = self.filenames[filename]

            self.make_environment(irun, filename)

        self.param_dict = {}
        create_param_dict(self.param_dict)

        for irun in range(len(self.filenames)):
            add_params(self.trajs[irun], self.param_dict)

        self.explore(self.trajs[0])
        self.explore(self.trajs[1])

        for irun in range(len(self.filenames)):
            self.make_run(self.envs[irun])

        #self.trajs[0].f_store()

        temp_sservice = self.trajs[0].v_storage_service
        temp_name = self.trajs[0].v_name

        self.trajs[0] = Trajectory()
        self.trajs[0].v_storage_service = temp_sservice
        self.trajs[0].f_load(name=temp_name,
                             as_new=False,
                             load_parameters=2,
                             load_derived_parameters=2,
                             load_results=2,
                             load_other_data=2)
        #self.trajs[0].f_load(trajectory_name=temp_name,as_new=False, load_params=2, load_derived_params=2, load_results=2)

        self.trajs[1].f_load_skeleton()
        self.trajs[1].f_load_items(self.trajs[1].f_to_dict().values(),
                                   only_empties=True)
        self.compare_trajectories(self.trajs[0], self.trajs[1])
Пример #21
0
    def test_continueing(self):
        self.filenames = [make_temp_dir('test_continuing.hdf5'), 0]

        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename, continuable=irun==1, delete_continue=False)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in range(len(self.filenames)):
            add_params(self.trajs[irun], self.param_dict)

        self.explore(self.trajs[0])
        self.explore(self.trajs[1])

        for irun in range(len(self.filenames)):
            self.make_run(self.envs[irun])

        traj_name = self.trajs[1].v_name
        continue_folder = os.path.join(self.cnt_folder, self.trajs[1].v_name)


        self.trajs[1]=self.envs[1].v_trajectory
        self._remove_nresults(self.trajs[1], 3, continue_folder)

        self.envs[1].v_current_idx = 0
        results = self.envs[1].f_continue(trajectory_name = traj_name)


        for irun in range(len(self.filenames)):
            self.trajs[irun].f_load(load_parameters=pypetconstants.OVERWRITE_DATA,
                                    load_derived_parameters=pypetconstants.OVERWRITE_DATA,
                                    load_results=pypetconstants.OVERWRITE_DATA,
                                    load_other_data=pypetconstants.OVERWRITE_DATA)


        self.compare_trajectories(self.trajs[0],self.trajs[1])
        #shutil.rmtree(self.cnt_folder)

        self.assertEqual(len(self.trajs[1]), len(results))
Пример #22
0
def test_run():

    global filename

    np.random.seed()
    trajname = 'profiling'
    filename = make_temp_dir(os.path.join('hdf5', 'test%s.hdf5' % trajname))

    env = Environment(trajectory=trajname,
                      filename=filename,
                      file_title=trajname,
                      log_stdout=False,
                      results_per_run=5,
                      derived_parameters_per_run=5,
                      multiproc=False,
                      ncores=1,
                      wrap_mode='LOCK',
                      use_pool=False,
                      overwrite_file=True)

    traj = env.v_trajectory

    traj.v_standard_parameter = Parameter

    ## Create some parameters
    param_dict = {}
    create_param_dict(param_dict)
    ### Add some parameter:
    add_params(traj, param_dict)

    #remember the trajectory and the environment
    traj = traj
    env = env

    traj.f_add_parameter('TEST', 'test_run')
    ###Explore
    explore(traj)

    ### Make a test run
    simple_arg = -13
    simple_kwarg = 13.0
    env.f_run(simple_calculations, simple_arg, simple_kwarg=simple_kwarg)

    size = os.path.getsize(filename)
    size_in_mb = size / 1000000.
    print('Size is %sMB' % str(size_in_mb))
Пример #23
0
def test_run():

    global filename


    np.random.seed()
    trajname = 'profiling'
    filename = make_temp_dir(os.path.join('hdf5', 'test%s.hdf5' % trajname))

    env = Environment(trajectory=trajname, filename=filename,
                      file_title=trajname,
                      log_stdout=False,
                      results_per_run=5,
                      derived_parameters_per_run=5,
                      multiproc=False,
                      ncores=1,
                      wrap_mode='LOCK',
                      use_pool=False,
                      overwrite_file=True)

    traj = env.v_trajectory

    traj.v_standard_parameter=Parameter

    ## Create some parameters
    param_dict={}
    create_param_dict(param_dict)
    ### Add some parameter:
    add_params(traj,param_dict)

    #remember the trajectory and the environment
    traj = traj
    env = env

    traj.f_add_parameter('TEST', 'test_run')
    ###Explore
    explore(traj)

    ### Make a test run
    simple_arg = -13
    simple_kwarg= 13.0
    env.f_run(simple_calculations,simple_arg,simple_kwarg=simple_kwarg)

    size=os.path.getsize(filename)
    size_in_mb = size/1000000.
    print('Size is %sMB' % str(size_in_mb))
    def test_continueing_remove_completed(self):
        self.filenames = [make_temp_dir('test_removal_comp.hdf5'), 0]

        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in range(len(self.filenames)):
            add_params(self.trajs[irun], self.param_dict)


        self.explore(self.trajs[0])
        self.explore(self.trajs[1])

        for irun in range(len(self.filenames)):
            self.make_run(self.envs[irun])

        traj_name = self.trajs[0].v_name
        continue_folder = os.path.join(self.cnt_folder, self.trajs[0].v_name)
        self._remove_nresults_from_traj(2)
        self.make_environment(0, self.filenames[0])
        self.envs[-1].f_continue(trajectory_name = traj_name)

        self.trajs[-1]=self.envs[-1].v_trajectory

        for irun in range(len(self.filenames)+1):
            self.trajs[irun].f_load_skeleton()
            self.trajs[irun].f_load(load_parameters=pypetconstants.OVERWRITE_DATA,
                                    load_derived_parameters=pypetconstants.OVERWRITE_DATA,
                                    load_results=pypetconstants.OVERWRITE_DATA,
                                    load_other_data=pypetconstants.OVERWRITE_DATA)

        self.compare_trajectories(self.trajs[-1],self.trajs[1])
Пример #25
0
    def setUp(self):
        self.set_mode()
        self.logfolder = make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'Log'))

        random.seed()
        self.trajname = make_trajectory_name(self)
        self.filename = make_temp_dir(os.path.join('experiments',
                                                    'tests',
                                                    'HDF5',
                                                    'test%s.hdf5' % self.trajname))

        env = Environment(trajectory=self.trajname, filename=self.filename,
                          file_title=self.trajname,
                          log_stdout=False,

                          log_config=get_log_config(),
                          results_per_run=5,
                          derived_parameters_per_run=5,
                          multiproc=self.multiproc,
                          ncores=self.ncores,
                          wrap_mode=self.mode,
                          use_pool=self.use_pool,
                          fletcher32=self.fletcher32,
                          complevel=self.complevel,
                          complib=self.complib,
                          shuffle=self.shuffle,
                          pandas_append=self.pandas_append,
                          pandas_format=self.pandas_format,
                          encoding=self.encoding)

        traj = env.v_trajectory

        self.param_dict={}
        create_param_dict(self.param_dict)
        add_params(traj,self.param_dict)

        self.traj = traj
        self.env = env
Пример #26
0
    def setUp(self):
        self.set_mode()
        self.logfolder = make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'Log'))

        random.seed()
        self.trajname = make_trajectory_name(self)
        self.filename = make_temp_dir(os.path.join('experiments',
                                                    'tests',
                                                    'HDF5',
                                                    'test%s.hdf5' % self.trajname))

        env = Environment(trajectory=self.trajname, filename=self.filename,
                          file_title=self.trajname,
                          log_stdout=False,
                          port=self.url,
                          log_config=get_log_config(),
                          results_per_run=5,
                          derived_parameters_per_run=5,
                          multiproc=self.multiproc,
                          ncores=self.ncores,
                          wrap_mode=self.mode,
                          use_pool=self.use_pool,
                          fletcher32=self.fletcher32,
                          complevel=self.complevel,
                          complib=self.complib,
                          shuffle=self.shuffle,
                          pandas_append=self.pandas_append,
                          pandas_format=self.pandas_format,
                          encoding=self.encoding)

        traj = env.v_trajectory

        self.param_dict={}
        create_param_dict(self.param_dict)
        add_params(traj,self.param_dict)

        self.traj = traj
        self.env = env
Пример #27
0
    def basic_and_skipping_duplicates_which_leads_to_one_remaining(self, slow_merge=False):

        self.envs=[]
        self.trajs = []

        ntrajs = len(self.filenames)

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in range(ntrajs):
            add_params(self.trajs[irun],self.param_dict)


        self.explore(self.trajs[0])
        self.explore_trials_differently(self.trajs[1])
        self.compare_explore_more_trials_with_removing_duplicates(self.trajs[2])

        for irun in range(ntrajs):
            self.make_run(self.envs[irun])

        for irun in range(ntrajs):
            self.trajs[irun].f_load_skeleton()
            self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA,
                                    load_derived_parameters=pypetconstants.UPDATE_DATA,
                                    load_results=pypetconstants.UPDATE_DATA,
                                    load_other_data=pypetconstants.UPDATE_DATA)


        self.trajs[1].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[1].f_store_item('rrororo33o333o3o3oo3')
        self.trajs[2].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[2].f_store_item('rrororo33o333o3o3oo3')

        run_name = pypetconstants.FORMATTED_RUN_NAME % 1
        run_name2 = pypetconstants.FORMATTED_RUN_NAME % 5
        self.trajs[1].f_add_result('%s.rrr' % run_name, 123)
        self.trajs[1].f_store_item('%s.rrr' % run_name)
        self.trajs[2].f_add_result('%s.rrr' % run_name2, 123)
        self.trajs[2].f_store_item('%s.rrr' % run_name2)

        self.trajs[1].f_add_result('Ignore.Me', 42)
        self.trajs[1].f_apar('Ignore.Me', 42)
        self.trajs[1].f_adpar('Ignore.Me', 42)

        ##f_merge without destroying the original trajectory
        merged_traj = self.trajs[0]
        merged_traj.f_merge(self.trajs[1], move_data=False,
                            delete_other_trajectory=False,
                            remove_duplicates=True,
                            ignore_data=('results.Ignore.Me', 'parameters.Ignore.Me',
                            'derived_parameters.Ignore.Me'),
                            slow_merge=slow_merge)
        merged_traj.f_load_skeleton()
        merged_traj.f_load(load_parameters=pypetconstants.UPDATE_DATA,
                           load_derived_parameters=pypetconstants.UPDATE_DATA,
                           load_results=pypetconstants.UPDATE_DATA,
                           load_other_data=pypetconstants.UPDATE_DATA)

        self.compare_trajectories(merged_traj,self.trajs[2])
Пример #28
0
    def basic_and_skipping_duplicates_which_leads_to_one_remaining(self):

        self.envs=[]
        self.trajs = []

        ntrajs = len(self.filenames)

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment( irun, filename)

        self.param_dict={}
        create_param_dict(self.param_dict)

        for irun in range(ntrajs):
            add_params(self.trajs[irun],self.param_dict)


        self.explore(self.trajs[0])
        self.explore_trials_differently(self.trajs[1])
        self.compare_explore_more_trials_with_removing_duplicates(self.trajs[2])

        for irun in range(ntrajs):
            self.make_run(self.envs[irun])

        for irun in range(ntrajs):
            self.trajs[irun].f_load_skeleton()
            self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA,
                                    load_derived_parameters=pypetconstants.UPDATE_DATA,
                                    load_results=pypetconstants.UPDATE_DATA,
                                    load_other_data=pypetconstants.UPDATE_DATA)


        self.trajs[1].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[1].f_store_item('rrororo33o333o3o3oo3')
        self.trajs[2].f_add_result('rrororo33o333o3o3oo3',1234567890)
        self.trajs[2].f_store_item('rrororo33o333o3o3oo3')

        run_name = pypetconstants.FORMATTED_RUN_NAME % 1
        run_name2 = pypetconstants.FORMATTED_RUN_NAME % 5
        self.trajs[1].f_add_result('%s.rrr' % run_name, 123)
        self.trajs[1].f_store_item('%s.rrr' % run_name)
        self.trajs[2].f_add_result('%s.rrr' % run_name2, 123)
        self.trajs[2].f_store_item('%s.rrr' % run_name2)

        self.trajs[1].f_add_result('Ignore.Me', 42)
        self.trajs[1].f_apar('Ignore.Me', 42)
        self.trajs[1].f_adpar('Ignore.Me', 42)

        ##f_merge without destroying the original trajectory
        merged_traj = self.trajs[0]
        merged_traj.f_merge(self.trajs[1], move_data=False,
                            delete_other_trajectory=False,
                            remove_duplicates=True,
                            ignore_data=('results.Ignore.Me', 'parameters.Ignore.Me',
                            'derived_parameters.Ignore.Me'))
        merged_traj.f_load_skeleton()
        merged_traj.f_load(load_parameters=pypetconstants.UPDATE_DATA,
                           load_derived_parameters=pypetconstants.UPDATE_DATA,
                           load_results=pypetconstants.UPDATE_DATA,
                           load_other_data=pypetconstants.UPDATE_DATA)

        self.compare_trajectories(merged_traj,self.trajs[2])