def test_irsync_r_nested_coll_to_coll_large_files(self):
        # test settings
        depth = 4
        files_per_level = 4
        file_size = 1024*1024*40

        # make local nested dirs
        source_base_name = "test_irsync_r_nested_coll_to_coll_source"
        dest_base_name = "test_irsync_r_nested_coll_to_coll_dest"
        local_dir = os.path.join(self.testing_tmp_dir, source_base_name)
        local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size)

        # iput dir
        self.user0.assert_icommand("iput -r {local_dir}".format(**locals()), "EMPTY")

        # sync collections
        self.user0.assert_icommand("irsync -r i:{source_base_name} i:{dest_base_name}".format(**locals()), "EMPTY")

        # compare files at each level
        for dir, files in local_dirs.iteritems():
            source_partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1)
            dest_partial_path = dir.replace(self.testing_tmp_dir+'/'+source_base_name, dest_base_name, 1)

            # run ils on source subcollection
            self.user0.assert_icommand(['ils', source_partial_path], 'STDOUT_SINGLELINE')
            ils_out = lib.ils_output_to_entries(self.user0.run_icommand(['ils', source_partial_path])[1])

            # compare local files with irods objects
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertTrue(local_files == rods_files,
                            msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
                            "Extra files:\n" + str(rods_files - local_files))

            # compare local files with files in vault
            files_in_vault = set(lib.files_in_dir(os.path.join(lib.get_vault_session_path(self.user0),
                                                              source_partial_path)))
            self.assertTrue(local_files == files_in_vault,
                        msg="Files missing from vault:\n" + str(local_files - files_in_vault) + "\n\n" +
                            "Extra files in vault:\n" + str(files_in_vault - local_files))

            # now the same thing with dest subcollection
            self.user0.assert_icommand(['ils', dest_partial_path], 'STDOUT_SINGLELINE')
            ils_out = lib.ils_output_to_entries(self.user0.run_icommand(['ils', dest_partial_path])[1])

            # compare local files with irods objects
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertTrue(local_files == rods_files,
                            msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
                            "Extra files:\n" + str(rods_files - local_files))

            # compare local files with files in vault
            files_in_vault = set(lib.files_in_dir(os.path.join(lib.get_vault_session_path(self.user0),
                                                              dest_partial_path)))
            self.assertTrue(local_files == files_in_vault,
                        msg="Files missing from vault:\n" + str(local_files - files_in_vault) + "\n\n" +
                            "Extra files in vault:\n" + str(files_in_vault - local_files))
Exemplo n.º 2
0
    def test_irsync_r_nested_dir_to_coll(self):
        # test settings
        depth = 10
        files_per_level = 100
        file_size = 100
        
        # make local nested dirs
        base_name = "test_irsync_r_nested_dir_to_coll"
        local_dir = os.path.join(self.testing_tmp_dir, base_name)        
        local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size)
        
        # sync dir to coll
        self.user0.assert_icommand("irsync -r {local_dir} i:{base_name}".format(**locals()), "EMPTY")

        # compare files at each level
        for dir, files in local_dirs.iteritems():
            partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1)

            # run ils on subcollection
            self.user0.assert_icommand(['ils', partial_path], 'STDOUT_SINGLELINE')
            ils_out = lib.ils_output_to_entries(self.user0.run_icommand(['ils', partial_path])[1])
            
            # compare local files with irods objects
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertTrue(local_files == rods_files,
                            msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
                            "Extra files:\n" + str(rods_files - local_files))
Exemplo n.º 3
0
    def test_irsync_r_nested_coll_to_coll_large_files(self):
        # test settings
        depth = 4
        files_per_level = 4
        file_size = 1024*1024*40
        
        # make local nested dirs
        source_base_name = "test_irsync_r_nested_coll_to_coll_source"
        dest_base_name = "test_irsync_r_nested_coll_to_coll_dest"
        local_dir = os.path.join(self.testing_tmp_dir, source_base_name)        
        local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size)

        # iput dir
        self.user0.assert_icommand("iput -r {local_dir}".format(**locals()), "EMPTY")
        
        # sync collections
        self.user0.assert_icommand("irsync -r i:{source_base_name} i:{dest_base_name}".format(**locals()), "EMPTY")

        # compare files at each level
        for dir, files in local_dirs.iteritems():
            source_partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1)
            dest_partial_path = dir.replace(self.testing_tmp_dir+'/'+source_base_name, dest_base_name, 1)
 
            # run ils on source subcollection
            self.user0.assert_icommand(['ils', source_partial_path], 'STDOUT_SINGLELINE')
            ils_out = lib.ils_output_to_entries(self.user0.run_icommand(['ils', source_partial_path])[1])
             
            # compare local files with irods objects
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertTrue(local_files == rods_files,
                            msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
                            "Extra files:\n" + str(rods_files - local_files))

            # now the same thing with dest subcollection
            self.user0.assert_icommand(['ils', dest_partial_path], 'STDOUT_SINGLELINE')
            ils_out = lib.ils_output_to_entries(self.user0.run_icommand(['ils', dest_partial_path])[1])
             
            # compare local files with irods objects
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertTrue(local_files == rods_files,
                            msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
                            "Extra files:\n" + str(rods_files - local_files))
Exemplo n.º 4
0
    def test_ireg_dir_exclude_from(self):
        # test settings
        depth = 10
        files_per_level = 10
        file_size = 100

        # make a local test dir under /tmp/
        test_dir_name = 'test_ireg_dir_exclude_from'
        local_dir = os.path.join(self.testing_tmp_dir, test_dir_name)
        local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth,
                                                 files_per_level, file_size)

        # arbitrary list of files to exclude
        excluded = set(['junk0001', 'junk0002', 'junk0003'])

        # make exclusion file
        exclude_file_path = os.path.join(self.testing_tmp_dir, 'exclude.txt')
        with open(exclude_file_path, 'w') as exclude_file:
            for filename in excluded:
                print >> exclude_file, filename

        # register local dir
        target_collection = self.admin.home_collection + '/' + test_dir_name
        self.admin.assert_icommand(
            "ireg --exclude-from {exclude_file_path} -C {local_dir} {target_collection}"
            .format(**locals()), "EMPTY")

        # compare files at each level
        for dir, files in local_dirs.iteritems():
            subcollection = dir.replace(local_dir, target_collection, 1)

            # run ils on subcollection
            self.admin.assert_icommand(['ils', subcollection],
                                       'STDOUT_SINGLELINE')
            ils_out = lib.ils_output_to_entries(
                self.admin.run_icommand(['ils', subcollection])[1])

            # compare irods objects with local files, minus the excluded ones
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertSetEqual(
                rods_files,
                local_files - excluded,
                msg="Files missing:\n" + str(local_files - rods_files) +
                "\n\n" + "Extra files:\n" + str(rods_files - local_files))

        # unregister collection
        self.admin.assert_icommand(
            "irm -rfU {target_collection}".format(**locals()))

        # remove local test dir and exclusion file
        os.remove(exclude_file_path)
        shutil.rmtree(local_dir)
Exemplo n.º 5
0
    def test_ireg_dir_exclude_from(self):
        # test settings
        depth = 10
        files_per_level = 10
        file_size = 100

        # make a local test dir under /tmp/
        test_dir_name = 'test_ireg_dir_exclude_from'
        local_dir = os.path.join(self.testing_tmp_dir, test_dir_name)
        local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size)

        # arbitrary list of files to exclude
        excluded = set(['junk0001', 'junk0002', 'junk0003'])

        # make exclusion file
        exclude_file_path = os.path.join(self.testing_tmp_dir, 'exclude.txt')
        with open(exclude_file_path, 'w') as exclude_file:
            for filename in excluded:
                print >>exclude_file, filename

        # register local dir
        target_collection = self.admin.home_collection + '/' + test_dir_name
        self.admin.assert_icommand("ireg --exclude-from {exclude_file_path} -C {local_dir} {target_collection}".format(**locals()), "EMPTY")

        # compare files at each level
        for dir, files in local_dirs.iteritems():
            subcollection = dir.replace(local_dir, target_collection, 1)

            # run ils on subcollection
            self.admin.assert_icommand(['ils', subcollection], 'STDOUT_SINGLELINE')
            ils_out = lib.ils_output_to_entries(self.admin.run_icommand(['ils', subcollection])[1])

            # compare irods objects with local files, minus the excluded ones
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertSetEqual(rods_files, local_files - excluded,
                            msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
                            "Extra files:\n" + str(rods_files - local_files))

        # unregister collection
        self.admin.assert_icommand("irm -rfU {target_collection}".format(**locals()))

        # remove local test dir and exclusion file
        os.remove(exclude_file_path)
        shutil.rmtree(local_dir)
    def test_irsync_r_nested_dir_to_coll(self):
        # test settings
        depth = 10
        files_per_level = 100
        file_size = 100

        # make local nested dirs
        base_name = "test_irsync_r_nested_dir_to_coll"
        local_dir = os.path.join(self.testing_tmp_dir, base_name)
        local_dirs = lib.make_deep_local_tmp_dir(
            local_dir, depth, files_per_level, file_size)

        # set client env
        env = os.environ.copy()
        env['X509_USER_PROXY'] = self.config['client_user_proxy']

        # sync dir to coll
        self.gsi_user.assert_icommand(
            "irsync -r {local_dir} i:{base_name}".format(**locals()), "EMPTY", env=env)

        # compare files at each level
        for directory, files in local_dirs.iteritems():
            partial_path = directory.replace(self.testing_tmp_dir + '/', '', 1)

            # run ils on subcollection
            self.gsi_user.assert_icommand(
                ['ils', partial_path], 'STDOUT_SINGLELINE', env=env)
            ils_out = lib.ils_output_to_entries(
                self.gsi_user.run_icommand(['ils', partial_path], env=env)[1])

            # compare local files with irods objects
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertTrue(local_files == rods_files,
                            msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
                            "Extra files:\n" + str(rods_files - local_files))

        # cleanup
        self.gsi_user.assert_icommand(
            "irm -rf {base_name}".format(**locals()), "EMPTY", env=env)