コード例 #1
0
    def test_irsync_r_coll_to_coll(self):
        base_name_source = "test_irsync_r_coll_to_coll_source"
        file_names = set(self.iput_r_large_collection(
            self.user0, base_name_source, file_count=1000, file_size=100)[1])
        base_name_target = "test_irsync_r_coll_to_coll_target"
        self.user0.assert_icommand("irsync -r i:" + base_name_source + " i:" + base_name_target, "EMPTY")
        self.user0.assert_icommand("ils", 'STDOUT_SINGLELINE', base_name_source)
        self.user0.assert_icommand("ils", 'STDOUT_SINGLELINE', base_name_target)
        rods_files_source = set(lib.ils_output_to_entries(self.user0.run_icommand(['ils', base_name_source])[1]))
        self.assertTrue(file_names == rods_files_source,
                        msg="Files missing:\n" + str(file_names - rods_files_source) + "\n\n" +
                            "Extra files:\n" + str(rods_files_source - file_names))

        rods_files_target = set(lib.ils_output_to_entries(self.user0.run_icommand(['ils', base_name_target])[1]))
        self.assertTrue(file_names == rods_files_target,
                        msg="Files missing:\n" + str(file_names - rods_files_target) + "\n\n" +
                            "Extra files :\n" + str(rods_files_target - file_names))

        vault_files_post_irsync_source = set(os.listdir(os.path.join(lib.get_vault_session_path(self.user0),
                                                                     base_name_source)))

        self.assertTrue(file_names == vault_files_post_irsync_source,
                        msg="Files missing from vault:\n" + str(file_names - vault_files_post_irsync_source) + "\n\n" +
                            "Extra files in vault:\n" + str(vault_files_post_irsync_source - file_names))

        vault_files_post_irsync_target = set(os.listdir(os.path.join(lib.get_vault_session_path(self.user0),
                                                                     base_name_target)))

        self.assertTrue(file_names == vault_files_post_irsync_target,
                        msg="Files missing from vault:\n" + str(file_names - vault_files_post_irsync_target) + "\n\n" +
                            "Extra files in vault:\n" + str(vault_files_post_irsync_target - file_names))
コード例 #2
0
    def test_irsync_r_nested_coll_to_coll_large_files(self):
        # test settings
        depth = 4
        files_per_level = 4
        file_size = 1024*1024*40

        # make local nested dirs
        source_base_name = "test_irsync_r_nested_coll_to_coll_source"
        dest_base_name = "test_irsync_r_nested_coll_to_coll_dest"
        local_dir = os.path.join(self.testing_tmp_dir, source_base_name)
        local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size)

        # iput dir
        self.user0.assert_icommand("iput -r {local_dir}".format(**locals()), "EMPTY")

        # sync collections
        self.user0.assert_icommand("irsync -r i:{source_base_name} i:{dest_base_name}".format(**locals()), "EMPTY")

        # compare files at each level
        for dir, files in local_dirs.iteritems():
            source_partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1)
            dest_partial_path = dir.replace(self.testing_tmp_dir+'/'+source_base_name, dest_base_name, 1)

            # run ils on source subcollection
            self.user0.assert_icommand(['ils', source_partial_path], 'STDOUT_SINGLELINE')
            ils_out = lib.ils_output_to_entries(self.user0.run_icommand(['ils', source_partial_path])[1])

            # compare local files with irods objects
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertTrue(local_files == rods_files,
                            msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
                            "Extra files:\n" + str(rods_files - local_files))

            # compare local files with files in vault
            files_in_vault = set(lib.files_in_dir(os.path.join(lib.get_vault_session_path(self.user0),
                                                              source_partial_path)))
            self.assertTrue(local_files == files_in_vault,
                        msg="Files missing from vault:\n" + str(local_files - files_in_vault) + "\n\n" +
                            "Extra files in vault:\n" + str(files_in_vault - local_files))

            # now the same thing with dest subcollection
            self.user0.assert_icommand(['ils', dest_partial_path], 'STDOUT_SINGLELINE')
            ils_out = lib.ils_output_to_entries(self.user0.run_icommand(['ils', dest_partial_path])[1])

            # compare local files with irods objects
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertTrue(local_files == rods_files,
                            msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
                            "Extra files:\n" + str(rods_files - local_files))

            # compare local files with files in vault
            files_in_vault = set(lib.files_in_dir(os.path.join(lib.get_vault_session_path(self.user0),
                                                              dest_partial_path)))
            self.assertTrue(local_files == files_in_vault,
                        msg="Files missing from vault:\n" + str(local_files - files_in_vault) + "\n\n" +
                            "Extra files in vault:\n" + str(files_in_vault - local_files))
コード例 #3
0
    def test_irsync_r_coll_to_coll(self):
        base_name_source = "test_irsync_r_coll_to_coll_source"
        file_names = set(
            self.iput_r_large_collection(self.user0,
                                         base_name_source,
                                         file_count=1000,
                                         file_size=100)[1])
        base_name_target = "test_irsync_r_coll_to_coll_target"
        self.user0.assert_icommand(
            "irsync -r i:" + base_name_source + " i:" + base_name_target,
            "EMPTY")
        self.user0.assert_icommand("ils", 'STDOUT_SINGLELINE',
                                   base_name_source)
        self.user0.assert_icommand("ils", 'STDOUT_SINGLELINE',
                                   base_name_target)
        rods_files_source = set(
            lib.ils_output_to_entries(
                self.user0.run_icommand(['ils', base_name_source])[1]))
        self.assertTrue(file_names == rods_files_source,
                        msg="Files missing:\n" +
                        str(file_names - rods_files_source) + "\n\n" +
                        "Extra files:\n" + str(rods_files_source - file_names))

        rods_files_target = set(
            lib.ils_output_to_entries(
                self.user0.run_icommand(['ils', base_name_target])[1]))
        self.assertTrue(
            file_names == rods_files_target,
            msg="Files missing:\n" + str(file_names - rods_files_target) +
            "\n\n" + "Extra files :\n" + str(rods_files_target - file_names))

        vault_files_post_irsync_source = set(
            os.listdir(
                os.path.join(lib.get_vault_session_path(self.user0),
                             base_name_source)))

        self.assertTrue(file_names == vault_files_post_irsync_source,
                        msg="Files missing from vault:\n" +
                        str(file_names - vault_files_post_irsync_source) +
                        "\n\n" + "Extra files in vault:\n" +
                        str(vault_files_post_irsync_source - file_names))

        vault_files_post_irsync_target = set(
            os.listdir(
                os.path.join(lib.get_vault_session_path(self.user0),
                             base_name_target)))

        self.assertTrue(file_names == vault_files_post_irsync_target,
                        msg="Files missing from vault:\n" +
                        str(file_names - vault_files_post_irsync_target) +
                        "\n\n" + "Extra files in vault:\n" +
                        str(vault_files_post_irsync_target - file_names))
コード例 #4
0
    def test_irsync_r_dir_to_coll(self):
        base_name = "test_irsync_r_dir_to_coll"
        local_dir = os.path.join(self.testing_tmp_dir, base_name)
        file_names = set(
            lib.make_large_local_tmp_dir(local_dir,
                                         file_count=1000,
                                         file_size=100))

        self.user0.assert_icommand(
            "irsync -r " + local_dir + " i:" + base_name, "EMPTY")
        self.user0.assert_icommand("ils", 'STDOUT_SINGLELINE', base_name)
        rods_files = set(
            lib.ils_output_to_entries(
                self.user0.run_icommand(['ils', base_name])[1]))
        self.assertTrue(file_names == rods_files,
                        msg="Files missing:\n" + str(file_names - rods_files) +
                        "\n\n" + "Extra files:\n" +
                        str(rods_files - file_names))

        vault_files_post_irsync = set(
            os.listdir(
                os.path.join(lib.get_vault_session_path(self.user0),
                             base_name)))

        self.assertTrue(file_names == vault_files_post_irsync,
                        msg="Files missing from vault:\n" +
                        str(file_names - vault_files_post_irsync) + "\n\n" +
                        "Extra files in vault:\n" +
                        str(vault_files_post_irsync - file_names))
コード例 #5
0
    def test_irsync_r_nested_dir_to_coll(self):
        # test settings
        depth = 10
        files_per_level = 100
        file_size = 100
        
        # make local nested dirs
        base_name = "test_irsync_r_nested_dir_to_coll"
        local_dir = os.path.join(self.testing_tmp_dir, base_name)        
        local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size)
        
        # sync dir to coll
        self.user0.assert_icommand("irsync -r {local_dir} i:{base_name}".format(**locals()), "EMPTY")

        # compare files at each level
        for dir, files in local_dirs.iteritems():
            partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1)

            # run ils on subcollection
            self.user0.assert_icommand(['ils', partial_path], 'STDOUT_SINGLELINE')
            ils_out = lib.ils_output_to_entries(self.user0.run_icommand(['ils', partial_path])[1])
            
            # compare local files with irods objects
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertTrue(local_files == rods_files,
                            msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
                            "Extra files:\n" + str(rods_files - local_files))
コード例 #6
0
ファイル: test_federation.py プロジェクト: pombreda/irods
    def test_irsync_r_dir_to_coll(self):
        # make test dir
        dir_name = 'irsync_test_dir'
        dir_path = os.path.abspath(dir_name)
        local_files = lib.make_large_local_tmp_dir(
            dir_path, self.config['test_file_count'], self.config['test_file_size'])

        kwargs = {'dir_name': dir_name,
                  'collection': self.config['remote_home_coll']}

        # sync dir with remote collection
        self.admin_sessions[0].assert_icommand(
            "irsync -r {dir_name} i:{collection}/{dir_name}".format(**kwargs))

        # new collection should be there
        self.admin_sessions[0].assert_icommand(
            "ils -L {collection}/{dir_name}".format(**kwargs), 'STDOUT_SINGLELINE', dir_name)

        # files should be there
        rods_files = set(lib.ils_output_to_entries(
            self.admin_sessions[0].run_icommand(['ils', "{collection}/{dir_name}".format(**kwargs)])[1]))
        self.assertTrue(set(local_files) == rods_files)

        # cleanup
        self.admin_sessions[0].assert_icommand(
            "irm -rf {collection}/{dir_name}".format(**kwargs))
        shutil.rmtree(dir_path)
コード例 #7
0
    def test_irsync_r_nested_coll_to_coll_large_files(self):
        # test settings
        depth = 4
        files_per_level = 4
        file_size = 1024*1024*40
        
        # make local nested dirs
        source_base_name = "test_irsync_r_nested_coll_to_coll_source"
        dest_base_name = "test_irsync_r_nested_coll_to_coll_dest"
        local_dir = os.path.join(self.testing_tmp_dir, source_base_name)        
        local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size)

        # iput dir
        self.user0.assert_icommand("iput -r {local_dir}".format(**locals()), "EMPTY")
        
        # sync collections
        self.user0.assert_icommand("irsync -r i:{source_base_name} i:{dest_base_name}".format(**locals()), "EMPTY")

        # compare files at each level
        for dir, files in local_dirs.iteritems():
            source_partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1)
            dest_partial_path = dir.replace(self.testing_tmp_dir+'/'+source_base_name, dest_base_name, 1)
 
            # run ils on source subcollection
            self.user0.assert_icommand(['ils', source_partial_path], 'STDOUT_SINGLELINE')
            ils_out = lib.ils_output_to_entries(self.user0.run_icommand(['ils', source_partial_path])[1])
             
            # compare local files with irods objects
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertTrue(local_files == rods_files,
                            msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
                            "Extra files:\n" + str(rods_files - local_files))

            # now the same thing with dest subcollection
            self.user0.assert_icommand(['ils', dest_partial_path], 'STDOUT_SINGLELINE')
            ils_out = lib.ils_output_to_entries(self.user0.run_icommand(['ils', dest_partial_path])[1])
             
            # compare local files with irods objects
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertTrue(local_files == rods_files,
                            msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
                            "Extra files:\n" + str(rods_files - local_files))
コード例 #8
0
ファイル: test_federation.py プロジェクト: kript/irods
    def test_irsync_r_coll_to_coll(self):
        # pick session(s) for the test
        test_session = self.user_sessions[0]

        # make test dir
        dir_name = "irsync_test_dir"
        dir_path = os.path.join(self.local_test_dir_path, dir_name)
        local_files = lib.make_large_local_tmp_dir(
            dir_path, self.config["test_file_count"], self.config["test_file_size"]
        )

        # test specific parameters
        parameters = self.config.copy()
        parameters["dir_path"] = dir_path
        parameters["dir_name"] = dir_name
        parameters["user_name"] = test_session.username
        parameters["local_home_collection"] = test_session.home_collection
        parameters["remote_home_collection"] = "/{remote_zone}/home/{user_name}#{local_zone}".format(**parameters)

        # put dir in local collection
        test_session.assert_icommand("iput -r {dir_path} {local_home_collection}/".format(**parameters))

        # remove local test dir
        shutil.rmtree(dir_path)

        # sync local collection with remote collection
        test_session.assert_icommand(
            "irsync -r i:{local_home_collection}/{dir_name} i:{remote_home_collection}/{dir_name}".format(**parameters)
        )

        # collection should be there
        test_session.assert_icommand(
            "ils -L {remote_home_collection}/{dir_name}".format(**parameters), "STDOUT_SINGLELINE", dir_name
        )

        # files should be there
        rods_files = set(
            lib.ils_output_to_entries(
                test_session.run_icommand(["ils", "{remote_home_collection}/{dir_name}".format(**parameters)])[1]
            )
        )
        self.assertTrue(set(local_files) == rods_files)

        # get collection back
        test_session.assert_icommand("iget -r {remote_home_collection}/{dir_name} {dir_path}".format(**parameters))

        # compare list of files
        received_files = os.listdir(dir_path)
        self.assertTrue(set(local_files) == set(received_files))

        # cleanup
        test_session.assert_icommand("irm -rf {local_home_collection}/{dir_name}".format(**parameters))
        test_session.assert_icommand("irm -rf {remote_home_collection}/{dir_name}".format(**parameters))
        shutil.rmtree(dir_path)
コード例 #9
0
ファイル: test_ireg.py プロジェクト: cyverse/irods
    def test_ireg_dir_exclude_from(self):
        # test settings
        depth = 10
        files_per_level = 10
        file_size = 100

        # make a local test dir under /tmp/
        test_dir_name = 'test_ireg_dir_exclude_from'
        local_dir = os.path.join(self.testing_tmp_dir, test_dir_name)
        local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth,
                                                 files_per_level, file_size)

        # arbitrary list of files to exclude
        excluded = set(['junk0001', 'junk0002', 'junk0003'])

        # make exclusion file
        exclude_file_path = os.path.join(self.testing_tmp_dir, 'exclude.txt')
        with open(exclude_file_path, 'w') as exclude_file:
            for filename in excluded:
                print >> exclude_file, filename

        # register local dir
        target_collection = self.admin.home_collection + '/' + test_dir_name
        self.admin.assert_icommand(
            "ireg --exclude-from {exclude_file_path} -C {local_dir} {target_collection}"
            .format(**locals()), "EMPTY")

        # compare files at each level
        for dir, files in local_dirs.iteritems():
            subcollection = dir.replace(local_dir, target_collection, 1)

            # run ils on subcollection
            self.admin.assert_icommand(['ils', subcollection],
                                       'STDOUT_SINGLELINE')
            ils_out = lib.ils_output_to_entries(
                self.admin.run_icommand(['ils', subcollection])[1])

            # compare irods objects with local files, minus the excluded ones
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertSetEqual(
                rods_files,
                local_files - excluded,
                msg="Files missing:\n" + str(local_files - rods_files) +
                "\n\n" + "Extra files:\n" + str(rods_files - local_files))

        # unregister collection
        self.admin.assert_icommand(
            "irm -rfU {target_collection}".format(**locals()))

        # remove local test dir and exclusion file
        os.remove(exclude_file_path)
        shutil.rmtree(local_dir)
コード例 #10
0
 def iput_r_large_collection(self, session, base_name, file_count, file_size):
     local_dir = os.path.join(self.testing_tmp_dir, base_name)
     local_files = lib.make_large_local_tmp_dir(local_dir, file_count, file_size)
     session.assert_icommand(['iput', '-r', local_dir])
     rods_files = set(lib.ils_output_to_entries(session.run_icommand(['ils', base_name])[1]))
     self.assertTrue(set(local_files) == rods_files,
                     msg="Files missing:\n" + str(set(local_files) - rods_files) + "\n\n" +
                         "Extra files:\n" + str(rods_files - set(local_files)))
     vault_files = set(os.listdir(os.path.join(lib.get_vault_session_path(session), base_name)))
     self.assertTrue(set(local_files) == vault_files,
                     msg="Files missing from vault:\n" + str(set(local_files) - vault_files) + "\n\n" +
                         "Extra files in vault:\n" + str(vault_files - set(local_files)))
     return (local_dir, local_files)
コード例 #11
0
ファイル: test_federation.py プロジェクト: kript/irods
    def test_msirmcoll(self):
        # pick session(s) for the test
        test_session = self.user_sessions[0]

        # make test dir
        dir_name = "msiRmColl_test_dir"
        dir_path = os.path.join(self.local_test_dir_path, dir_name)
        local_files = lib.make_large_local_tmp_dir(
            dir_path, self.config["test_file_count"], self.config["test_file_size"]
        )

        # test specific parameters
        parameters = self.config.copy()
        parameters["dir_path"] = dir_path
        parameters["dir_name"] = dir_name
        parameters["user_name"] = test_session.username
        parameters["remote_home_collection"] = "/{remote_zone}/home/{user_name}#{local_zone}".format(**parameters)

        # put dir in remote collection
        test_session.assert_icommand("iput -r {dir_path} {remote_home_collection}/".format(**parameters))

        # new collection should be there
        test_session.assert_icommand(
            "ils -L {remote_home_collection}/{dir_name}".format(**parameters), "STDOUT_SINGLELINE", dir_name
        )

        # files should be there
        rods_files = set(
            lib.ils_output_to_entries(
                test_session.run_icommand(["ils", "{remote_home_collection}/{dir_name}".format(**parameters)])[1]
            )
        )
        self.assertTrue(set(local_files) == rods_files)

        # prepare irule sequence
        # the rule is simple enough not to need a rule file
        irule_str = '''irule "msiRmColl(*coll, 'forceFlag=', *status); writeLine('stdout', *status)" "*coll={remote_home_collection}/{dir_name}" "ruleExecOut"'''.format(
            **parameters
        )

        # invoke msiRmColl() and checks that it returns 0
        test_session.assert_icommand(irule_str, "STDOUT", "^0$", use_regex=True)

        # collection should be gone
        test_session.assert_icommand(
            "ils -L {remote_home_collection}/{dir_name}".format(**parameters), "STDERR_SINGLELINE", "does not exist"
        )

        # cleanup
        shutil.rmtree(dir_path)
コード例 #12
0
    def test_get_null_perms__2833(self):
        base_name = 'test_dir_for_perms'
        local_dir = os.path.join(self.testing_tmp_dir, base_name)
        local_files = lib.make_large_local_tmp_dir(local_dir, 30, 10)
        self.admin.assert_icommand(['iput', '-r', local_dir])
        rods_files = lib.ils_output_to_entries(self.admin.run_icommand(['ils', base_name])[1])

        test_dir = 'test_get_null_perms__2833_dir'
        self.admin.assert_icommand(['ichmod','-r','null',self.admin.username,base_name])
        self.admin.assert_icommand(['ichmod','read',self.admin.username,base_name+'/'+rods_files[-1]])
        self.admin.assert_icommand(['iget','-r',base_name,test_dir],'STDERR_SINGLELINE','CAT_NO_ACCESS_PERMISSION')

        assert os.path.isfile(os.path.join(test_dir,'junk0029'))

        self.admin.assert_icommand(['ichmod','-r','own',self.admin.username,base_name])
        lib.run_command(['rm','-rf',test_dir])
コード例 #13
0
ファイル: test_federation.py プロジェクト: pombreda/irods
    def test_irsync_r_coll_to_coll(self):
        # make test dir
        dir_name = 'irsync_test_dir'
        dir_path = os.path.abspath(dir_name)
        local_files = lib.make_large_local_tmp_dir(
            dir_path, self.config['test_file_count'], self.config['test_file_size'])

        kwargs = {'dir_name': dir_name,
                  'remote_collection': self.config['remote_home_coll'],
                  'local_collection': self.config['local_home_coll']}

        # put dir in local collection
        self.admin_sessions[0].assert_icommand(
            "iput -r {dir_name} {local_collection}/".format(**kwargs))

        # remove local test dir
        shutil.rmtree(dir_path)

        # sync local collection with remote collection
        self.admin_sessions[0].assert_icommand(
            "irsync -r i:{local_collection}/{dir_name} i:{remote_collection}/{dir_name}".format(**kwargs))

        # collection should be there
        self.admin_sessions[0].assert_icommand(
            "ils -L {remote_collection}/{dir_name}".format(**kwargs), 'STDOUT_SINGLELINE', dir_name)

        # files should be there
        rods_files = set(lib.ils_output_to_entries(
            self.admin_sessions[0].run_icommand(['ils', "{remote_collection}/{dir_name}".format(**kwargs)])[1]))
        self.assertTrue(set(local_files) == rods_files)

        # get collection back
        self.admin_sessions[0].assert_icommand(
            "iget -r {remote_collection}/{dir_name}".format(**kwargs))

        # compare list of files
        received_files = os.listdir(dir_path)
        self.assertTrue(set(local_files) == set(received_files))

        # cleanup
        self.admin_sessions[0].assert_icommand(
            "irm -rf {local_collection}/{dir_name}".format(**kwargs))
        self.admin_sessions[0].assert_icommand(
            "irm -rf {remote_collection}/{dir_name}".format(**kwargs))
        shutil.rmtree(dir_path)
コード例 #14
0
ファイル: test_ireg.py プロジェクト: javenwu/irods
    def test_ireg_dir_exclude_from(self):
        # test settings
        depth = 10
        files_per_level = 10
        file_size = 100

        # make a local test dir under /tmp/
        test_dir_name = 'test_ireg_dir_exclude_from'
        local_dir = os.path.join(self.testing_tmp_dir, test_dir_name)
        local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size)

        # arbitrary list of files to exclude
        excluded = set(['junk0001', 'junk0002', 'junk0003'])

        # make exclusion file
        exclude_file_path = os.path.join(self.testing_tmp_dir, 'exclude.txt')
        with open(exclude_file_path, 'w') as exclude_file:
            for filename in excluded:
                print >>exclude_file, filename

        # register local dir
        target_collection = self.admin.home_collection + '/' + test_dir_name
        self.admin.assert_icommand("ireg --exclude-from {exclude_file_path} -C {local_dir} {target_collection}".format(**locals()), "EMPTY")

        # compare files at each level
        for dir, files in local_dirs.iteritems():
            subcollection = dir.replace(local_dir, target_collection, 1)

            # run ils on subcollection
            self.admin.assert_icommand(['ils', subcollection], 'STDOUT_SINGLELINE')
            ils_out = lib.ils_output_to_entries(self.admin.run_icommand(['ils', subcollection])[1])

            # compare irods objects with local files, minus the excluded ones
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertSetEqual(rods_files, local_files - excluded,
                            msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
                            "Extra files:\n" + str(rods_files - local_files))

        # unregister collection
        self.admin.assert_icommand("irm -rfU {target_collection}".format(**locals()))

        # remove local test dir and exclusion file
        os.remove(exclude_file_path)
        shutil.rmtree(local_dir)
コード例 #15
0
    def test_irsync_r_dir_to_coll(self):
        base_name = "test_irsync_r_dir_to_coll"
        local_dir = os.path.join(self.testing_tmp_dir, base_name)
        file_names = set(lib.make_large_local_tmp_dir(local_dir, file_count=1000, file_size=100))

        self.user0.assert_icommand("irsync -r " + local_dir + " i:" + base_name, "EMPTY")
        self.user0.assert_icommand("ils", 'STDOUT_SINGLELINE', base_name)
        rods_files = set(lib.ils_output_to_entries(self.user0.run_icommand(['ils', base_name])[1]))
        self.assertTrue(file_names == rods_files,
                        msg="Files missing:\n" + str(file_names - rods_files) + "\n\n" +
                            "Extra files:\n" + str(rods_files - file_names))

        vault_files_post_irsync = set(os.listdir(os.path.join(lib.get_vault_session_path(self.user0),
                                                              base_name)))

        self.assertTrue(file_names == vault_files_post_irsync,
                        msg="Files missing from vault:\n" + str(file_names - vault_files_post_irsync) + "\n\n" +
                            "Extra files in vault:\n" + str(vault_files_post_irsync - file_names))
コード例 #16
0
ファイル: test_federation.py プロジェクト: kript/irods
    def test_irm_rf(self):
        # pick session(s) for the test
        test_session = self.user_sessions[0]

        # make test dir
        dir_name = "irm_test_dir"
        dir_path = os.path.join(self.local_test_dir_path, dir_name)
        local_files = lib.make_large_local_tmp_dir(
            dir_path, self.config["test_file_count"], self.config["test_file_size"]
        )

        # test specific parameters
        parameters = self.config.copy()
        parameters["dir_path"] = dir_path
        parameters["dir_name"] = dir_name
        parameters["user_name"] = test_session.username
        parameters["remote_home_collection"] = "/{remote_zone}/home/{user_name}#{local_zone}".format(**parameters)

        # put dir in remote collection
        test_session.assert_icommand("iput -r {dir_path} {remote_home_collection}/".format(**parameters))

        # new collection should be there
        test_session.assert_icommand(
            "ils -L {remote_home_collection}/{dir_name}".format(**parameters), "STDOUT_SINGLELINE", dir_name
        )

        # files should be there
        rods_files = set(
            lib.ils_output_to_entries(
                test_session.run_icommand(["ils", "{remote_home_collection}/{dir_name}".format(**parameters)])[1]
            )
        )
        self.assertTrue(set(local_files) == rods_files)

        # remove remote coll
        test_session.assert_icommand("irm -rf {remote_home_collection}/{dir_name}".format(**parameters))

        # coll should be gone
        test_session.assert_icommand(
            "ils -L {remote_home_collection}/{dir_name}".format(**parameters), "STDERR_SINGLELINE", "does not exist"
        )

        # cleanup
        shutil.rmtree(dir_path)
コード例 #17
0
    def test_irsync_r_nested_dir_to_coll(self):
        # test settings
        depth = 10
        files_per_level = 100
        file_size = 100

        # make local nested dirs
        base_name = "test_irsync_r_nested_dir_to_coll"
        local_dir = os.path.join(self.testing_tmp_dir, base_name)
        local_dirs = lib.make_deep_local_tmp_dir(
            local_dir, depth, files_per_level, file_size)

        # set client env
        env = os.environ.copy()
        env['X509_USER_PROXY'] = self.config['client_user_proxy']

        # sync dir to coll
        self.gsi_user.assert_icommand(
            "irsync -r {local_dir} i:{base_name}".format(**locals()), "EMPTY", env=env)

        # compare files at each level
        for directory, files in local_dirs.iteritems():
            partial_path = directory.replace(self.testing_tmp_dir + '/', '', 1)

            # run ils on subcollection
            self.gsi_user.assert_icommand(
                ['ils', partial_path], 'STDOUT_SINGLELINE', env=env)
            ils_out = lib.ils_output_to_entries(
                self.gsi_user.run_icommand(['ils', partial_path], env=env)[1])

            # compare local files with irods objects
            local_files = set(files)
            rods_files = set(lib.files_in_ils_output(ils_out))
            self.assertTrue(local_files == rods_files,
                            msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
                            "Extra files:\n" + str(rods_files - local_files))

        # cleanup
        self.gsi_user.assert_icommand(
            "irm -rf {base_name}".format(**locals()), "EMPTY", env=env)
コード例 #18
0
 def iput_r_large_collection(self, session, base_name, file_count,
                             file_size):
     local_dir = os.path.join(self.testing_tmp_dir, base_name)
     local_files = lib.make_large_local_tmp_dir(local_dir, file_count,
                                                file_size)
     session.assert_icommand(['iput', '-r', local_dir])
     rods_files = set(
         lib.ils_output_to_entries(
             session.run_icommand(['ils', base_name])[1]))
     self.assertTrue(set(local_files) == rods_files,
                     msg="Files missing:\n" +
                     str(set(local_files) - rods_files) + "\n\n" +
                     "Extra files:\n" + str(rods_files - set(local_files)))
     vault_files = set(
         os.listdir(
             os.path.join(lib.get_vault_session_path(session), base_name)))
     self.assertTrue(set(local_files) == vault_files,
                     msg="Files missing from vault:\n" +
                     str(set(local_files) - vault_files) + "\n\n" +
                     "Extra files in vault:\n" +
                     str(vault_files - set(local_files)))
     return (local_dir, local_files)
コード例 #19
0
    def test_imv_r(self):
        base_name_source = "test_imv_r_dir_source"
        file_names = set(
            self.iput_r_large_collection(self.user0,
                                         base_name_source,
                                         file_count=1000,
                                         file_size=100)[1])

        base_name_target = "test_imv_r_dir_target"
        self.user0.assert_icommand(
            "imv " + base_name_source + " " + base_name_target, "EMPTY")
        self.user0.assert_icommand("ils " + base_name_source,
                                   'STDERR_SINGLELINE', "does not exist")
        self.user0.assert_icommand("ils", 'STDOUT_SINGLELINE',
                                   base_name_target)
        rods_files_post_imv = set(
            lib.ils_output_to_entries(
                self.user0.run_icommand(['ils', base_name_target])[1]))
        self.assertTrue(
            file_names == rods_files_post_imv,
            msg="Files missing:\n" + str(file_names - rods_files_post_imv) +
            "\n\n" + "Extra files:\n" + str(rods_files_post_imv - file_names))

        vault_files_post_irm_source = set(
            os.listdir(
                os.path.join(lib.get_vault_session_path(self.user0),
                             base_name_source)))
        self.assertTrue(len(vault_files_post_irm_source) == 0)

        vault_files_post_irm_target = set(
            os.listdir(
                os.path.join(lib.get_vault_session_path(self.user0),
                             base_name_target)))
        self.assertTrue(file_names == vault_files_post_irm_target,
                        msg="Files missing from vault:\n" +
                        str(file_names - vault_files_post_irm_target) +
                        "\n\n" + "Extra files in vault:\n" +
                        str(vault_files_post_irm_target - file_names))
コード例 #20
0
    def test_irsync_r_coll_to_dir(self):
        base_name_source = "test_irsync_r_coll_to_dir_source"
        file_names = set(self.iput_r_large_collection(
            self.user0, base_name_source, file_count=1000, file_size=100)[1])
        local_dir = os.path.join(self.testing_tmp_dir, "test_irsync_r_coll_to_dir_target")
        self.user0.assert_icommand("irsync -r i:" + base_name_source + " " + local_dir, "EMPTY")
        self.user0.assert_icommand("ils", 'STDOUT', base_name_source)
        rods_files_source = set(lib.ils_output_to_entries(self.user0.run_icommand(['ils', base_name_source])[1]))
        self.assertTrue(file_names == rods_files_source,
                        msg="Files missing:\n" + str(file_names - rods_files_source) + "\n\n" +
                            "Extra files:\n" + str(rods_files_source - file_names))

        local_files = set(os.listdir(local_dir))
        self.assertTrue(file_names == local_files,
                        msg="Files missing from local dir:\n" + str(file_names - local_files) + "\n\n" +
                            "Extra files in local dir:\n" + str(local_files - file_names))

        vault_files_post_irsync_source = set(os.listdir(os.path.join(lib.get_vault_session_path(self.user0),
                                                                     base_name_source)))

        self.assertTrue(file_names == vault_files_post_irsync_source,
                        msg="Files missing from vault:\n" + str(file_names - vault_files_post_irsync_source) + "\n\n" +
                            "Extra files in vault:\n" + str(vault_files_post_irsync_source - file_names))
コード例 #21
0
    def test_imv_r(self):
        base_name_source = "test_imv_r_dir_source"
        file_names = set(self.iput_r_large_collection(
            self.user0, base_name_source, file_count=1000, file_size=100)[1])

        base_name_target = "test_imv_r_dir_target"
        self.user0.assert_icommand("imv " + base_name_source + " " + base_name_target, "EMPTY")
        self.user0.assert_icommand("ils " + base_name_source, 'STDERR_SINGLELINE', "does not exist")
        self.user0.assert_icommand("ils", 'STDOUT_SINGLELINE', base_name_target)
        rods_files_post_imv = set(lib.ils_output_to_entries(self.user0.run_icommand(['ils', base_name_target])[1]))
        self.assertTrue(file_names == rods_files_post_imv,
                        msg="Files missing:\n" + str(file_names - rods_files_post_imv) + "\n\n" +
                            "Extra files:\n" + str(rods_files_post_imv - file_names))

        vault_files_post_irm_source = set(os.listdir(os.path.join(lib.get_vault_session_path(self.user0),
                                                                  base_name_source)))
        self.assertTrue(len(vault_files_post_irm_source) == 0)

        vault_files_post_irm_target = set(os.listdir(os.path.join(lib.get_vault_session_path(self.user0),
                                                                  base_name_target)))
        self.assertTrue(file_names == vault_files_post_irm_target,
                        msg="Files missing from vault:\n" + str(file_names - vault_files_post_irm_target) + "\n\n" +
                            "Extra files in vault:\n" + str(vault_files_post_irm_target - file_names))