コード例 #1
0
ファイル: srm.py プロジェクト: pombredanne/rucio
    def put(self, source, target, source_dir):
        """
        Allows to store files inside the referred RSE.

        :param source: path to the source file on the client file system
        :param target: path to the destination file on the storage
        :param source_dir: Path where the to be transferred files are stored in the local file system
        :raises DestinationNotAccessible: if the destination storage was not accessible.
        :raises ServiceUnavailable: if some generic error occured in the library.
        :raises SourceNotFound: if the source file was not found on the referred storage.
        """

        source_url = '%s/%s' % (source_dir, source) if source_dir else source

        if not os.path.exists(source_url):
            raise exception.SourceNotFound()

        space_token = ''
        if self.attributes['extended_attributes'] is not None and 'space_token' in self.attributes['extended_attributes'].keys():
            space_token = '--dst %s' % self.attributes['extended_attributes']['space_token']

        try:
            cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 -D srmv2 %s file:%s %s' % (space_token, source_url, target)
            status, out, err = execute(cmd)
            if status:
                raise exception.RucioException(err)
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #2
0
ファイル: srm.py プロジェクト: pombredanne/rucio
    def get(self, path, dest):
        """
        Provides access to files stored inside connected the RSE.

        :param path: Physical file name of requested file
        :param dest: Name and path of the files when stored at the client
        :raises DestinationNotAccessible: if the destination storage was not accessible.
        :raises ServiceUnavailable: if some generic error occured in the library.
        :raises SourceNotFound: if the source file was not found on the referred storage.
        """

        space_token = ''
        if self.attributes['extended_attributes'] is not None and 'space_token' in self.attributes['extended_attributes'].keys():
            space_token = '--sst %s' % self.attributes['extended_attributes']['space_token']

        try:
            cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 -D srmv2 %s %s file:%s' % (space_token, path, dest)
            status, out, err = execute(cmd)
            if status:
                if self.__parse_srm_error__("SRM_INVALID_PATH", out, err):
                    raise exception.SourceNotFound(err)
                raise exception.RucioException(err)
        except exception.SourceNotFound as e:
            raise exception.SourceNotFound(str(e))
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #3
0
ファイル: srm.py プロジェクト: pombredanne/rucio
    def delete(self, path):
        """
        Deletes a file from the connected RSE.

        :param path: path to the to be deleted file
        :raises ServiceUnavailable: if some generic error occured in the library.
        :raises SourceNotFound: if the source file was not found on the referred storage.
        """

        pfns = [path] if ((type(path) == str) or (type(path) == unicode)) else path

        try:
            pfn_chunks = [pfns[i:i + 20] for i in range(0, len(pfns), 20)]
            for pfn_chunk in pfn_chunks:
                cmd = 'lcg-del $LCGVO -v -b -l --srm-timeout 600 -D srmv2'
                for pfn in pfn_chunk:
                    cmd += ' ' + pfn
                status, out, err = execute(cmd)
                if status:
                    if self.__parse_srm_error__("SRM_INVALID_PATH", out, err):
                        raise exception.SourceNotFound(err)
                    raise exception.RucioException(err)
        except exception.SourceNotFound as e:
            raise exception.SourceNotFound(str(e))
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #4
0
ファイル: rfio.py プロジェクト: pombredanne/rucio
    def exists(self, path):
        """
            Checks if the requested file is known by the referred RSE.

            :param path: Physical file name

            :returns: True if the file exists, False if it doesn't

            :raises SourceNotFound: if the source file was not found on the referred storage.
        """
        cmd = 'rfstat %(path)s' % locals()
        status, out, err = execute(cmd)
        return status == 0
コード例 #5
0
    def setupClass(cls):
        """SRM (RSE/PROTOCOLS): Creating necessary directories and files """
        # Creating local files
        cls.tmpdir = tempfile.mkdtemp()
        cls.user = uuid()

        with open("%s/data.raw" % cls.tmpdir, "wb") as out:
            out.seek((1024 * 1024) - 1)  # 1 MB
            out.write('\0')
        for f in MgrTestCases.files_local:
            shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))

        with open('etc/rse_repository.json') as f:
            data = json.load(f)
        prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['prefix']
        hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['hostname']
        if hostname.count("://"):
            hostname = hostname.split("://")[1]
        if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys():
            port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['port'])
        else:
            port = 0
        if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys() and 'web_service_path' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes'].keys():
            web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes']['web_service_path']
        else:
            web_service_path = ''

        os.system('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' % cls.tmpdir)
        if port > 0:
            cls.static_file = 'srm://%s:%s%s%s/data.raw' % (hostname, port, web_service_path, prefix)
        else:
            cls.static_file = 'srm://%s%s%s/data.raw' % (hostname, web_service_path, prefix)
        cmd = 'srmcp --debug=false -retry_num=0 file:///%s/data.raw %s' % (cls.tmpdir, cls.static_file)
        execute(cmd)

        for f in MgrTestCases.files_remote:
            tmp = mgr.lfns2pfns(mgr.get_rse_info('FZK-LCG2_SCRATCHDISK'), {'name': f, 'scope': 'user.%s' % cls.user}, scheme='srm').values()[0]
            cmd = 'srmcp --debug=false -retry_num=0  file:///%s/data.raw %s' % (cls.tmpdir, tmp)
            execute(cmd)
コード例 #6
0
ファイル: xrootd.py プロジェクト: pombredanne/rucio
    def rename(self, pfn, new_pfn):
        """ Allows to rename a file stored inside the connected RSE.

            :param pfn      Current physical file name
            :param new_pfn  New physical file name
            :raises DestinationNotAccessible: if the destination storage was not accessible.
            :raises ServiceUnavailable: if some generic error occured in the library.
            :raises SourceNotFound: if the source file was not found on the referred storage.
        """
        if not self.exists(pfn):
            raise exception.SourceNotFound()
        try:
            path = self.pfn2path(pfn)
            new_path = self.pfn2path(new_pfn)
            new_dir = new_path[:new_path.rindex('/') + 1]
            cmd = 'xrdfs %s:%s mkdir -p %s' % (self.hostname, self.port, new_dir)
            status, out, err = execute(cmd)
            cmd = 'xrdfs %s:%s mv %s %s' % (self.hostname, self.port, path, new_path)
            status, out, err = execute(cmd)
            if not status == 0:
                raise exception.RucioException(err)
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #7
0
ファイル: srm.py プロジェクト: pombredanne/rucio
    def rename(self, path, new_path):
        """
        Allows to rename a file stored inside the connected RSE.

        :param path: path to the current file on the storage
        :param new_path: path to the new file on the storage
        :raises DestinationNotAccessible: if the destination storage was not accessible.
        :raises ServiceUnavailable: if some generic error occured in the library.
        :raises SourceNotFound: if the source file was not found on the referred storage.
        """

        try:
            cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 -D srmv2 %s %s' % (path, new_path)
            status, out, err = execute(cmd)
            if status:
                raise exception.RucioException(err)

            cmd = 'lcg-del $LCGVO -v -b -l --srm-timeout 600 -D srmv2 %s' % (path)
            status, out, err = execute(cmd)
            if status:
                raise exception.RucioException(err)
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #8
0
ファイル: srm.py プロジェクト: ijjorama/rucio
    def exists(self, path):
        """
        Checks if the requested file is known by the referred RSE.

        :param path: Physical file name
        :returns: True if the file exists, False if it doesn't
        :raises SourceNotFound: if the source file was not found on the referred storage.
        """

        try:
            cmd = 'lcg-ls $LCGVO -v -b --srm-timeout 60 -D srmv2  %s' % (path)
            status, out, err = execute(cmd)
            if status:
                return False
            return True
        except Exception as error:
            raise exception.ServiceUnavailable(error)
コード例 #9
0
ファイル: xrootd.py プロジェクト: pombredanne/rucio
    def connect(self):
        """ Establishes the actual connection to the referred RSE.

            :param credentials Provides information to establish a connection
                to the referred storage system. For S3 connections these are
                access_key, secretkey, host_base, host_bucket, progress_meter
                and skip_existing.

            :raises RSEAccessDenied
        """
        try:
            cmd = 'xrdfs %s:%s query stats %s:%s' % (self.hostname, self.port, self.hostname, self.port)
            status, out, err = execute(cmd)
            if not status == 0:
                raise exception.RSEAccessDenied(err)
        except Exception as e:
            raise exception.RSEAccessDenied(e)
コード例 #10
0
ファイル: uploadclient.py プロジェクト: poush/rucio
 def get_file_guid(self, file):
     guid = file.get('guid')
     if not guid and 'pool.root' in file['basename'].lower() and not file.get('no_register'):
         status, output, err = execute('pool_extractFileIdentifier %s' % file['path'])
         if status != 0:
             msg = 'Trying to upload ROOT files but pool_extractFileIdentifier tool can not be found.\n'
             msg += 'Setup your ATHENA environment and try again.'
             raise RucioException(msg)
         try:
             guid = output.splitlines()[-1].split()[0].replace('-', '').lower()
         except Exception:
             raise RucioException('Error extracting GUID from ouput of pool_extractFileIdentifier')
     elif guid:
         guid = guid.replace('-', '')
     else:
         guid = generate_uuid()
     return guid
コード例 #11
0
ファイル: srm.py プロジェクト: pombredanne/rucio
    def exists(self, path):
        """
        Checks if the requested file is known by the referred RSE.

        :param path: Physical file name
        :returns: True if the file exists, False if it doesn't
        :raises SourceNotFound: if the source file was not found on the referred storage.
        """

        try:
            cmd = 'lcg-ls $LCGVO -v -b --srm-timeout 60 -D srmv2  %s' % (path)
            status, out, err = execute(cmd)
            if status:
                return False
            return True
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #12
0
ファイル: xrootd.py プロジェクト: pombredanne/rucio
    def get(self, pfn, dest):
        """ Provides access to files stored inside connected the RSE.

            :param pfn Physical file name of requested file
            :param dest Name and path of the files when stored at the client

            :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
        """
        if not self.exists(pfn):
            raise exception.SourceNotFound()
        try:
            cmd = 'xrdcp -f %s %s' % (pfn, dest)
            status, out, err = execute(cmd)
            if not status == 0:
                raise exception.RucioException(err)
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #13
0
    def tearDownClass(cls):
        """GFAL2 (RSE/PROTOCOLS): Removing created directorie s and files"""
        data = load_test_conf_file('rse_repository.json')
        prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][
            'prefix']
        hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][
            'srm']['hostname']
        if hostname.count("://"):
            hostname = hostname.split("://")[1]
        if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][
                'srm'].keys():
            port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']
                       ['srm']['port'])
        else:
            port = 0
        if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols'][
                'supported']['srm'].keys() and 'web_service_path' in data[
                    'FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][
                        'extended_attributes'].keys():
            web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols'][
                'supported']['srm']['extended_attributes']['web_service_path']
        else:
            web_service_path = ''

        shutil.rmtree(cls.tmpdir)

        clean_raw = '%s/data.raw' % prefix
        if int(port) > 0:
            srm_path = ''.join(
                ["srm://", hostname, ":", port, web_service_path])
        else:
            srm_path = ''.join(["srm://", hostname, web_service_path])

        list_files_cmd_user = '******' % (
            srm_path, prefix, cls.user)
        clean_files = str(execute(list_files_cmd_user)[1]).split('\n')
        list_files_cmd_user = '******' % (
            srm_path, prefix, cls.user)
        clean_files += str(execute(list_files_cmd_user)[1]).split('\n')
        clean_files.append("1024  " + clean_raw)
        for files in clean_files:
            if len(files.strip()) > 0:
                file = files.split()[1]
                if not file.endswith("/"):
                    clean_cmd = 'srmrm -2 --debug=false -retry_num=0 %s/%s' % (
                        srm_path, file)
                    execute(clean_cmd)

        clean_directory = ['user', 'group']
        for directory in clean_directory:
            clean_cmd = 'srmrmdir -2 --debug=false -retry_num=0 -recursive %s%s/%s/%s' % (
                srm_path, prefix, directory, cls.user)
            execute(clean_cmd)
コード例 #14
0
ファイル: xrootd.py プロジェクト: tbeerman/rucio
    def get(self, pfn, dest):
        """ Provides access to files stored inside connected the RSE.

            :param pfn Physical file name of requested file
            :param dest Name and path of the files when stored at the client

            :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
        """

        try:
            cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (pfn, dest)
            status, out, err = execute(cmd)
            if status == 54:
                raise exception.SourceNotFound()
            elif status != 0:
                raise exception.RucioException(err)
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #15
0
ファイル: rfio.py プロジェクト: pombredanne/rucio
    def put(self, source, target, source_dir):
        """
            Allows to store files inside the referred RSE.

            :param source: path to the source file on the client file system
            :param target: path to the destination file on the storage
            :param source_dir: Path where the to be transferred files are stored in the local file system

            :raises DestinationNotAccessible: if the destination storage was not accessible.
            :raises ServiceUnavailable: if some generic error occured in the library.
            :raises SourceNotFound: if the source file was not found on the referred storage.
        """
        if not self.exists(dirname(target)):
            self.mkdir(dirname(target))

        cmd = 'rfcp %(source)s %(path)s' % locals()
        status, out, err = execute(cmd)
        return status == 0
コード例 #16
0
ファイル: ssh.py プロジェクト: pradeepjasal/rucio
    def connect(self):
        """ Establishes the actual connection to the referred RSE.

            :raises RSEAccessDenied
        """
        self.logger(
            logging.DEBUG,
            'ssh.connect: port: {}, hostname {}, ssh-user {}'.format(
                self.port, self.hostname, self.sshuser))
        try:
            cmd = 'ssh -p %s %s%s echo ok 2>&1' % (self.port, self.sshuser,
                                                   self.hostname)
            status, out, err = execute(cmd)
            checker = re.search(r'ok', out)
            if not checker:
                raise exception.RSEAccessDenied(err)
        except Exception as e:
            raise exception.RSEAccessDenied(e)
コード例 #17
0
    def put(self, source, target, source_dir):
        """
            Allows to store files inside the referred RSE.

            :param source: path to the source file on the client file system
            :param target: path to the destination file on the storage
            :param source_dir: Path where the to be transferred files are stored in the local file system

            :raises DestinationNotAccessible: if the destination storage was not accessible.
            :raises ServiceUnavailable: if some generic error occured in the library.
            :raises SourceNotFound: if the source file was not found on the referred storage.
        """
        if not self.exists(dirname(target)):
            self.mkdir(dirname(target))

        cmd = 'rfcp %(source)s %(path)s' % locals()
        status, out, err = execute(cmd)
        return status == 0
コード例 #18
0
ファイル: xrootd.py プロジェクト: pombredanne/rucio
    def exists(self, pfn):
        """ Checks if the requested file is known by the referred RSE.

            :param pfn Physical file name

            :returns: True if the file exists, False if it doesn't

            :raise  ServiceUnavailable
        """
        try:
            path = self.pfn2path(pfn)
            cmd = 'xrdfs %s:%s stat %s' % (self.hostname, self.port, path)
            status, out, err = execute(cmd)
            if not status == 0:
                return False
        except Exception as e:
            raise exception.ServiceUnavailable(e)

        return True
コード例 #19
0
ファイル: xrootd.py プロジェクト: pombredanne/rucio
    def delete(self, pfn):
        """
            Deletes a file from the connected RSE.

            :param pfn Physical file name

            :raises ServiceUnavailable: if some generic error occured in the library.
            :raises SourceNotFound: if the source file was not found on the referred storage.
        """
        if not self.exists(pfn):
            raise exception.SourceNotFound()
        try:
            path = self.pfn2path(pfn)
            cmd = 'xrdfs %s:%s rm %s' % (self.hostname, self.port, path)
            status, out, err = execute(cmd)
            if not status == 0:
                raise exception.RucioException(err)
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #20
0
    def get(self, pfn, dest, transfer_timeout=None):
        """ Provides access to files stored inside connected the RSE.

            :param pfn Physical file name of requested file
            :param dest Name and path of the files when stored at the client
            :param transfer_timeout: Transfer timeout (in seconds) - dummy

            :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
        """
        self.logger(logging.DEBUG, 'rclone.get: pfn: {}'.format(pfn))
        try:
            path = self.pfn2path(pfn)
            cmd = 'rclone copyto %s:%s %s' % (self.hostname, path, dest)
            self.logger(logging.DEBUG, 'rclone.get: cmd: {}'.format(cmd))
            status, out, err = execute(cmd)
            if status:
                raise exception.RucioException(err)
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #21
0
    def exists(self, pfn):
        """ Checks if the requested file is known by the referred RSE.

            :param pfn Physical file name

            :returns: True if the file exists, False if it doesn't

            :raise  ServiceUnavailable
        """
        try:
            path = self.pfn2path(pfn)
            cmd = 'xrdfs %s:%s stat %s' % (self.hostname, self.port, path)
            status, out, err = execute(cmd)
            if not status == 0:
                return False
        except Exception as e:
            raise exception.ServiceUnavailable(e)

        return True
コード例 #22
0
    def delete(self, pfn):
        """
            Deletes a file from the connected RSE.

            :param pfn Physical file name

            :raises ServiceUnavailable: if some generic error occured in the library.
            :raises SourceNotFound: if the source file was not found on the referred storage.
        """
        if not self.exists(pfn):
            raise exception.SourceNotFound()
        try:
            path = self.pfn2path(pfn)
            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s rm %s' % (self.hostname, self.port, path)
            status, out, err = execute(cmd)
            if not status == 0:
                raise exception.RucioException(err)
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #23
0
    def test_upload_file_guid_with_impl(self):
        """CLIENT(USER): Rucio upload file with guid"""
        tmp_file1 = file_generator()
        tmp_guid = uuid()
        impl = 'xrootd'
        cmd = 'rucio upload --rse {0} --scope {1} --impl {2} --guid {3} {4}'.format(
            self.rse, self.scope, impl, tmp_guid, tmp_file1)
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out)
        print(err)
        remove(tmp_file1)
        self.assertEqual(exitcode, 0)
        upload_string_1 = ('Successfully uploaded file %s' %
                           path.basename(tmp_file1))
        self.assertIsNotNone(re.search(upload_string_1, err))

        added_dids = ['{0}:{1}'.format(self.scope, did) for did in tmp_file1]
        self.generated_dids += added_dids
コード例 #24
0
    def get(self, pfn, dest, transfer_timeout=None):
        """ Provides access to files stored inside connected the RSE.

            :param pfn Physical file name of requested file
            :param dest Name and path of the files when stored at the client
            :param transfer_timeout: Transfer timeout (in seconds) - dummy

            :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
        """
        self.logger.debug('xrootd.get: pfn: {}'.format(pfn))
        try:
            cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (pfn, dest)
            self.logger.info('xrootd.get: cmd: {}'.format(cmd))
            status, out, err = execute(cmd)
            if status == 54:
                raise exception.SourceNotFound()
            elif status != 0:
                raise exception.RucioException(err)
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #25
0
    def connect(self):
        """ Establishes the actual connection to the referred RSE.

            :param credentials Provides information to establish a connection
                to the referred storage system. For S3 connections these are
                access_key, secretkey, host_base, host_bucket, progress_meter
                and skip_existing.

            :raises RSEAccessDenied
        """

        try:
            # The query stats call is not implemented on some xroot doors.
            # Workaround: fail, if server does not reply within 10 seconds for static config query
            cmd = 'XrdSecPROTOCOL=gsi XRD_REQUESTTIMEOUT=10 xrdfs %s:%s query config %s:%s' % (self.hostname, self.port, self.hostname, self.port)
            status, out, err = execute(cmd)
            if not status == 0:
                raise exception.RSEAccessDenied(err)
        except Exception as e:
            raise exception.RSEAccessDenied(e)
コード例 #26
0
def test_reaper():
    """ REAPER (DAEMON): Test the reaper daemon."""
    if config_get_bool('common',
                       'multi_vo',
                       raise_exception=False,
                       default=False):
        vo = {'vo': 'tst'}
    else:
        vo = {}

    nb_files = 30
    file_size = 2147483648  # 2G
    rse_id = rse_core.get_rse_id(rse='MOCK', **vo)

    for i in range(nb_files):
        replica_core.add_replica(rse_id=rse_id,
                                 scope=InternalScope('data13_hip', **vo),
                                 name='lfn' + generate_uuid(),
                                 bytes=file_size,
                                 account=InternalAccount('root', **vo),
                                 adler32=None,
                                 md5=None)

    rse_core.set_rse_usage(rse_id=rse_id,
                           source='srm',
                           used=nb_files * file_size,
                           free=800)
    rse_core.set_rse_limits(rse_id=rse_id,
                            name='MinFreeSpace',
                            value=10737418240)
    rse_core.set_rse_limits(rse_id=rse_id,
                            name='MaxBeingDeletedFiles',
                            value=10)

    argv = ['--run-once', '--rses', 'MOCK']
    main(argv)
    # Test the rucio-reaper console script
    cmd = 'rucio-reaper ' + ' '.join(argv)
    exitcode, out, err = execute(cmd)
    print(cmd, out, err)
    nose.tools.assert_equal(exitcode, 0)
コード例 #27
0
    def delete(self, pfn):
        """
            Deletes a file from the connected RSE.

            :param pfn Physical file name

            :raises ServiceUnavailable: if some generic error occured in the library.
            :raises SourceNotFound: if the source file was not found on the referred storage.
        """
        self.logger(logging.DEBUG, 'rclone.delete: pfn: {}'.format(pfn))
        if not self.exists(pfn):
            raise exception.SourceNotFound()
        try:
            path = self.pfn2path(pfn)
            cmd = 'rclone delete %s:%s' % (self.hostname, path)
            self.logger(logging.DEBUG, 'rclone.delete: cmd: {}'.format(cmd))
            status, out, err = execute(cmd)
            if status != 0:
                raise exception.RucioException(err)
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #28
0
    def exists(self, pfn):
        """ Checks if the requested file is known by the referred RSE.

            :param pfn Physical file name

            :returns: True if the file exists, False if it doesn't

            :raise  ServiceUnavailable
        """
        self.logger(logging.DEBUG, 'rclone.exists: pfn: {}'.format(pfn))
        try:
            path = self.pfn2path(pfn)
            cmd = 'rclone lsf %s:%s' % (self.hostname, path)
            self.logger(logging.DEBUG, 'rclone.exists: cmd: {}'.format(cmd))
            status, out, err = execute(cmd)
            if status:
                return False
        except Exception as e:
            raise exception.ServiceUnavailable(e)

        return True
コード例 #29
0
ファイル: ssh.py プロジェクト: pradeepjasal/rucio
    def get(self, pfn, dest, transfer_timeout=None):
        """ Provides access to files stored inside connected the RSE.

            :param pfn: Physical file name of requested file
            :param dest: Name and path of the files when stored at the client
            :param transfer_timeout: Transfer timeout (in seconds) - dummy

            :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
        """
        self.logger(logging.DEBUG, 'rsync.get: pfn: {}'.format(pfn))
        try:
            path = self.pfn2path(pfn)
            destdir = os.path.dirname(dest)
            cmd = 'mkdir -p %s && rsync -az -e "ssh -p %s" --append-verify %s%s:%s %s' % (
                destdir, self.port, self.sshuser, self.hostname, path, dest)
            self.logger(logging.DEBUG, 'rsync.get: cmd: {}'.format(cmd))
            status, out, err = execute(cmd)
            if status:
                raise exception.RucioException(err)
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #30
0
    def exists(self, pfn):
        """ Checks if the requested file is known by the referred RSE.

            :param pfn Physical file name

            :returns: True if the file exists, False if it doesn't

            :raise  ServiceUnavailable
        """
        self.logger.debug('xrootd.exists: pfn: {}'.format(pfn))
        try:
            path = self.pfn2path(pfn)
            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s stat %s' % (self.hostname,
                                                              self.port, path)
            self.logger.info('xrootd.exists: cmd: {}'.format(cmd))
            status, out, err = execute(cmd)
            if status != 0:
                return False
        except Exception as e:
            raise exception.ServiceUnavailable(e)

        return True
コード例 #31
0
ファイル: srm.py プロジェクト: ijjorama/rucio
    def put(self, source, target, source_dir, transfer_timeout=None):
        """
        Allows to store files inside the referred RSE.

        :param source: path to the source file on the client file system
        :param target: path to the destination file on the storage
        :param source_dir: Path where the to be transferred files are stored in the local file system
        :param transfer_timeout: Transfer timeout (in seconds)

        :raises DestinationNotAccessible: if the destination storage was not accessible.
        :raises ServiceUnavailable: if some generic error occured in the library.
        :raises SourceNotFound: if the source file was not found on the referred storage.
        """

        source_url = '%s/%s' % (source_dir, source) if source_dir else source

        if not os.path.exists(source_url):
            raise exception.SourceNotFound()

        space_token = ''
        if self.attributes[
                'extended_attributes'] is not None and 'space_token' in list(
                    self.attributes['extended_attributes'].keys()):
            space_token = '--dst %s' % self.attributes['extended_attributes'][
                'space_token']

        timeout_option = ''
        if transfer_timeout:
            timeout_option = '--sendreceive-timeout %s' % transfer_timeout

        try:
            cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 %s -D srmv2 %s file:%s %s' % (
                timeout_option, space_token, source_url, target)
            status, out, err = execute(cmd)
            if status:
                raise exception.RucioException(err)
        except Exception as error:
            raise exception.ServiceUnavailable(error)
コード例 #32
0
 def _get_file_guid(self, file):
     """
     Get the guid of a file, trying different strategies
     (This function is meant to be used as class internal only)
     :param file: dictionary describing the file
     :returns: the guid
     """
     guid = file.get('guid')
     if not guid and 'pool.root' in file['basename'].lower() and not file.get('no_register'):
         status, output, err = execute('pool_extractFileIdentifier %s' % file['path'])
         if status != 0:
             msg = 'Trying to upload ROOT files but pool_extractFileIdentifier tool can not be found.\n'
             msg += 'Setup your ATHENA environment and try again.'
             raise RucioException(msg)
         try:
             guid = output.splitlines()[-1].split()[0].replace('-', '').lower()
         except Exception:
             raise RucioException('Error extracting GUID from ouput of pool_extractFileIdentifier')
     elif guid:
         guid = guid.replace('-', '')
     else:
         guid = generate_uuid()
     return guid
コード例 #33
0
ファイル: srm.py プロジェクト: poush/rucio
    def get(self, path, dest):
        """
        Provides access to files stored inside connected the RSE.

        :param path: Physical file name of requested file
        :param dest: Name and path of the files when stored at the client
        :raises DestinationNotAccessible: if the destination storage was not accessible.
        :raises ServiceUnavailable: if some generic error occured in the library.
        :raises SourceNotFound: if the source file was not found on the referred storage.
        """

        try:
            cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 -D srmv2 %s file:%s' % (
                path, dest)
            status, out, err = execute(cmd)
            if status:
                if self.__parse_srm_error__("SRM_INVALID_PATH", out, err):
                    raise exception.SourceNotFound(err)
                raise exception.RucioException(err)
        except exception.SourceNotFound as error:
            raise exception.SourceNotFound(str(error))
        except Exception as error:
            raise exception.ServiceUnavailable(error)
コード例 #34
0
    def setUpClass(cls):
        """XROOTD (RSE/PROTOCOLS): Creating necessary directories and files """

        # Getting info for the test environment
        rse_id, prefix, hostname, port = cls.get_rse_info()

        try:
            os.mkdir(prefix)
        except Exception as e:
            print(e)

        # Creating local files
        cls.tmpdir = tempfile.mkdtemp()
        cls.user = uuid()

        with open("%s/data.raw" % cls.tmpdir, "wb") as out:
            out.seek((1024 * 1024) - 1)  # 1 MB
            out.write(b'\0')
        for f in MgrTestCases.files_local:
            shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))

        protocol = rsemanager.create_protocol(rsemanager.get_rse_info(rse_id),
                                              'write')
        protocol.connect()

        os.system('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' %
                  prefix)
        cls.static_file = 'xroot://%s:%d/%s/data.raw' % (hostname, port,
                                                         prefix)
        cmd = 'xrdcp %s/data.raw %s' % (prefix, cls.static_file)
        execute(cmd)

        for f in MgrTestCases.files_remote:
            path = protocol.path2pfn(
                prefix + protocol._get_path('user.%s' % cls.user, f))
            cmd = 'xrdcp %s/data.raw %s' % (prefix, path)
            execute(cmd)

        for f in MgrTestCases.files_local_and_remote:
            shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
            path = protocol.path2pfn(
                prefix + protocol._get_path('user.%s' % cls.user, f))
            cmd = 'xrdcp %s/%s %s' % (cls.tmpdir, f, path)
            execute(cmd)
コード例 #35
0
ファイル: xrootd.py プロジェクト: tbeerman/rucio
    def put(self, filename, target, source_dir):
        """
            Allows to store files inside the referred RSE.

            :param source: path to the source file on the client file system
            :param target: path to the destination file on the storage
            :param source_dir: Path where the to be transferred files are stored in the local file system

            :raises DestinationNotAccessible: if the destination storage was not accessible.
            :raises ServiceUnavailable: if some generic error occured in the library.
            :raises SourceNotFound: if the source file was not found on the referred storage.
        """
        source_url = '%s/%s' % (source_dir, filename)
        path = self.path2pfn(target)

        if not os.path.exists(source_url):
            raise exception.SourceNotFound()
        try:
            cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (source_url, path)
            status, out, err = execute(cmd)
            if not status == 0:
                raise exception.RucioException(err)
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #36
0
ファイル: xrootd.py プロジェクト: pombredanne/rucio
    def put(self, filename, target, source_dir):
        """
            Allows to store files inside the referred RSE.

            :param source: path to the source file on the client file system
            :param target: path to the destination file on the storage
            :param source_dir: Path where the to be transferred files are stored in the local file system

            :raises DestinationNotAccessible: if the destination storage was not accessible.
            :raises ServiceUnavailable: if some generic error occured in the library.
            :raises SourceNotFound: if the source file was not found on the referred storage.
        """
        source_url = '%s/%s' % (source_dir, filename)
        path = self.path2pfn(target)

        if not os.path.exists(source_url):
            raise exception.SourceNotFound()
        try:
            cmd = 'xrdcp -f %s %s' % (source_url, path)
            status, out, err = execute(cmd)
            if not status == 0:
                raise exception.RucioException(err)
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #37
0
    def rename(self, pfn, new_pfn):
        """ Allows to rename a file stored inside the connected RSE.

            :param pfn      Current physical file name
            :param new_pfn  New physical file name
            :raises DestinationNotAccessible: if the destination storage was not accessible.
            :raises ServiceUnavailable: if some generic error occured in the library.
            :raises SourceNotFound: if the source file was not found on the referred storage.
        """
        self.logger(logging.DEBUG, 'rclone.rename: pfn: {}'.format(pfn))
        if not self.exists(pfn):
            raise exception.SourceNotFound()
        try:
            path = self.pfn2path(pfn)
            new_path = self.pfn2path(new_pfn)
            cmd = 'rclone moveto %s:%s %s:%s' % (self.hostname, path,
                                                 self.hostname, new_path)
            self.logger(logging.DEBUG,
                        'rclone.stat: rename cmd: {}'.format(cmd))
            status, out, err = execute(cmd)
            if status != 0:
                raise exception.RucioException(err)
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #38
0
    def get_rse_info(cls):
        """
        Detects if containerized rses for xrootd are available in the testing environment.
        :return: A tuple (rse, prefix, hostname, port).
        """
        cmd = "rucio list-rses --expression 'test_container_xrd=True'"
        print(cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        rses = out.split()

        data = load_test_conf_file('rse_repository.json')
        prefix = data['WJ-XROOTD']['protocols']['supported']['xroot']['prefix']
        port = data['WJ-XROOTD']['protocols']['supported']['xroot']['port']

        if len(rses) == 0:
            rse_id = 'WJ-XROOTD'
            hostname = data['WJ-XROOTD']['protocols']['supported']['xroot']['hostname']
        else:
            rse_id = 'XRD1'
            hostname = 'xrd1'
            prefix = '/rucio/'
        # TODO should read container info from a config file
        return rse_id, prefix, hostname, port
コード例 #39
0
ファイル: ssh.py プロジェクト: pradeepjasal/rucio
    def put(self, filename, target, source_dir, transfer_timeout=None):
        """
            Allows to store files inside the referred RSE.

            :param source: path to the source file on the client file system
            :param target: path to the destination file on the storage
            :param source_dir: Path where the to be transferred files are stored in the local file system
            :param transfer_timeout: Transfer timeout (in seconds) - dummy

            :raises DestinationNotAccessible: if the destination storage was not accessible.
            :raises ServiceUnavailable: if some generic error occured in the library.
            :raises SourceNotFound: if the source file was not found on the referred storage.
        """
        self.logger(
            logging.DEBUG,
            'rsync.put: filename: {} target: {}'.format(filename, target))
        source_dir = source_dir or '.'
        source_url = '%s/%s' % (source_dir, filename)
        self.logger(logging.DEBUG,
                    'rsync.put: source url: {}'.format(source_url))

        path = self.pfn2path(target)
        pathdir = os.path.dirname(path)
        if not os.path.exists(source_url):
            raise exception.SourceNotFound()

        try:
            cmd = 'ssh -p %s %s%s "mkdir -p %s" && rsync -az -e "ssh -p %s" --append-verify %s %s%s:%s' % (
                self.port, self.sshuser, self.hostname, pathdir, self.port,
                source_url, self.sshuser, self.hostname, path)
            self.logger(logging.DEBUG, 'rsync.put: cmd: {}'.format(cmd))
            status, out, err = execute(cmd)
            if status:
                raise exception.RucioException(err)
        except Exception as e:
            raise exception.ServiceUnavailable(e)
コード例 #40
0
    def teardownClass(cls):
        """SRM (RSE/PROTOCOLS): Removing created directorie s and files"""
        with open('etc/rse_repository.json') as f:
            data = json.load(f)
        prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['prefix']
        hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['hostname']
        if hostname.count("://"):
            hostname = hostname.split("://")[1]
        if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys():
            port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['port'])
        else:
            port = 0
        if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys() and 'web_service_path' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes'].keys():
            web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes']['web_service_path']
        else:
            web_service_path = ''

        shutil.rmtree(cls.tmpdir)

        clean_raw = '%s/data.raw' % prefix
        if int(port) > 0:
            srm_path = ''.join(["srm://", hostname, ":", port, web_service_path])
        else:
            srm_path = ''.join(["srm://", hostname, web_service_path])

        list_files_cmd_user = '******' % (srm_path, prefix, cls.user)
        clean_files = str(execute(list_files_cmd_user)[1]).split('\n')
        list_files_cmd_user = '******' % (srm_path, prefix, cls.user)
        clean_files += str(execute(list_files_cmd_user)[1]).split('\n')
        clean_files.append("1024  " + clean_raw)
        for files in clean_files:
            if len(files.strip()) > 0:
                file = files.split()[1]
                if not file.endswith("/"):
                    clean_cmd = 'srmrm --debug=false -retry_num=0 %s/%s' % (srm_path, file)
                    execute(clean_cmd)

        clean_directory = ['user', 'group']
        for directory in clean_directory:
            clean_cmd = 'srmrmdir --debug=false -retry_num=0 -recursive %s%s/%s/%s' % (srm_path, prefix, directory, cls.user)
            execute(clean_cmd)
コード例 #41
0
    def tearDownClass(cls):
        """XROOTD (RSE/PROTOCOLS): Removing created directorie s and files"""
        rse_id, prefix, hostname, port = cls.get_rse_info()

        shutil.rmtree(prefix)
        shutil.rmtree(cls.tmpdir)

        clean_raw = '%s/data.raw' % prefix
        list_files_cmd_user = '******' % (hostname, prefix, cls.user)
        clean_files = str(execute(list_files_cmd_user)[1]).split('\n')
        list_files_cmd_group = 'xrdfs %s ls %s/group.%s' % (hostname, prefix, cls.user)
        clean_files += str(execute(list_files_cmd_group)[1]).split('\n')
        clean_files.append(clean_raw)
        for files in clean_files:
            clean_cmd = 'xrdfs %s rm %s' % (hostname, files)
            execute(clean_cmd)

        clean_prefix = '%s' % prefix
        list_directory = 'xrdfs %s ls %s' % (hostname, prefix)
        clean_directory = str(execute(list_directory)[1]).split('\n')
        clean_directory.append(clean_prefix)
        for directory in clean_directory:
            clean_cmd = 'xrdfs %s rmdir %s' % (hostname, directory)
            execute(clean_cmd)
コード例 #42
0
ファイル: automatix.py プロジェクト: wdconinc/rucio
def generate_file(fname, size, logger=logging.log):
    cmd = '/bin/dd if=/dev/urandom of=%s bs=%s count=1' % (fname, size)
    exitcode, out, err = execute(cmd)
    logger(logging.DEBUG, out)
    logger(logging.DEBUG, err)
    return exitcode
コード例 #43
0
ファイル: rfio.py プロジェクト: pombredanne/rucio
 def mkdir(self, directory):
     cmd = 'rfmkdir -p %(path)s' % locals()
     status, out, err = execute(cmd)
     return status == 0
コード例 #44
0
ファイル: Automatix.py プロジェクト: pombredanne/rucio
def generate_file(fname, size):
    cmd = '/bin/dd if=/dev/urandom of=%(fname)s bs=%(size)s count=1' % locals()
    exitcode, out, err = execute(cmd)
    logging.debug(out)
    logging.debug(err)
    return exitcode
コード例 #45
0
ファイル: automatix.py プロジェクト: andrea-manzi/rucio
def generate_file(fname, size):
    cmd = '/bin/dd if=/dev/urandom of=%(fname)s bs=%(size)s count=1' % locals()
    exitcode, out, err = execute(cmd)
    logging.debug(out)
    logging.debug(err)
    return exitcode
コード例 #46
0
    def setUpClass(cls):
        """GFAL2 (RSE/PROTOCOLS): Creating necessary directories and files """
        # Creating local files
        cls.tmpdir = tempfile.mkdtemp()
        cls.user = uuid()

        with open("%s/data.raw" % cls.tmpdir, "wb") as out:
            out.seek((1024 * 1024) - 1)  # 1 MB
            out.write('\0')
        for f in MgrTestCases.files_local:
            shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))

        with open('etc/rse_repository.json') as f:
            data = json.load(f)
        prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][
            'prefix']
        hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][
            'srm']['hostname']
        if hostname.count("://"):
            hostname = hostname.split("://")[1]
        if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][
                'srm'].keys():
            port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']
                       ['srm']['port'])
        else:
            port = 0
        if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols'][
                'supported']['srm'].keys() and 'web_service_path' in data[
                    'FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][
                        'extended_attributes'].keys():
            web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols'][
                'supported']['srm']['extended_attributes']['web_service_path']
        else:
            web_service_path = ''

        os.system('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' %
                  cls.tmpdir)
        if port > 0:
            cls.static_file = 'srm://%s:%s%s%s/data.raw' % (
                hostname, port, web_service_path, prefix)
        else:
            cls.static_file = 'srm://%s%s%s/data.raw' % (
                hostname, web_service_path, prefix)
        cmd = 'srmcp -2 --debug=false -retry_num=0 file:///%s/data.raw %s' % (
            cls.tmpdir, cls.static_file)
        execute(cmd)

        rse_settings = mgr.get_rse_info('FZK-LCG2_SCRATCHDISK')
        for protocol in rse_settings['protocols']:
            if protocol['scheme'] != "srm":
                rse_settings['protocols'].remove(protocol)
        if len(rse_settings['protocols']) > 0:
            rse_settings['protocols'][0][
                'impl'] = 'rucio.rse.protocols.gfal.Default'

        for f in MgrTestCases.files_remote:
            tmp = mgr.lfns2pfns(rse_settings, {
                'name': f,
                'scope': 'user.%s' % cls.user
            },
                                scheme='srm').values()[0]
            cmd = 'srmcp -2 --debug=false -retry_num=0  file:///%s/data.raw %s' % (
                cls.tmpdir, tmp)
            execute(cmd)
コード例 #47
0
 def mkdir(self, directory):
     """ Create new directory. """
     cmd = 'rfmkdir -p %(path)s' % locals()
     status, out, err = execute(cmd)
     return status == 0