def put(self, source, target, source_dir):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
source_url = '%s/%s' % (source_dir, source) if source_dir else source
if not os.path.exists(source_url):
raise exception.SourceNotFound()
space_token = ''
if self.attributes['extended_attributes'] is not None and 'space_token' in self.attributes['extended_attributes'].keys():
space_token = '--dst %s' % self.attributes['extended_attributes']['space_token']
try:
cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 -D srmv2 %s file:%s %s' % (space_token, source_url, target)
status, out, err = execute(cmd)
if status:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def get(self, path, dest):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
space_token = ''
if self.attributes['extended_attributes'] is not None and 'space_token' in self.attributes['extended_attributes'].keys():
space_token = '--sst %s' % self.attributes['extended_attributes']['space_token']
try:
cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 -D srmv2 %s %s file:%s' % (space_token, path, dest)
status, out, err = execute(cmd)
if status:
if self.__parse_srm_error__("SRM_INVALID_PATH", out, err):
raise exception.SourceNotFound(err)
raise exception.RucioException(err)
except exception.SourceNotFound as e:
raise exception.SourceNotFound(str(e))
except Exception as e:
raise exception.ServiceUnavailable(e)
def delete(self, path):
"""
Deletes a file from the connected RSE.
:param path: path to the to be deleted file
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
pfns = [path] if ((type(path) == str) or (type(path) == unicode)) else path
try:
pfn_chunks = [pfns[i:i + 20] for i in range(0, len(pfns), 20)]
for pfn_chunk in pfn_chunks:
cmd = 'lcg-del $LCGVO -v -b -l --srm-timeout 600 -D srmv2'
for pfn in pfn_chunk:
cmd += ' ' + pfn
status, out, err = execute(cmd)
if status:
if self.__parse_srm_error__("SRM_INVALID_PATH", out, err):
raise exception.SourceNotFound(err)
raise exception.RucioException(err)
except exception.SourceNotFound as e:
raise exception.SourceNotFound(str(e))
except Exception as e:
raise exception.ServiceUnavailable(e)
def exists(self, path):
"""
Checks if the requested file is known by the referred RSE.
:param path: Physical file name
:returns: True if the file exists, False if it doesn't
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
cmd = 'rfstat %(path)s' % locals()
status, out, err = execute(cmd)
return status == 0
def setupClass(cls):
"""SRM (RSE/PROTOCOLS): Creating necessary directories and files """
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = uuid()
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024 * 1024) - 1) # 1 MB
out.write('\0')
for f in MgrTestCases.files_local:
shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
with open('etc/rse_repository.json') as f:
data = json.load(f)
prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['prefix']
hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['hostname']
if hostname.count("://"):
hostname = hostname.split("://")[1]
if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys():
port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['port'])
else:
port = 0
if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys() and 'web_service_path' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes'].keys():
web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes']['web_service_path']
else:
web_service_path = ''
os.system('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' % cls.tmpdir)
if port > 0:
cls.static_file = 'srm://%s:%s%s%s/data.raw' % (hostname, port, web_service_path, prefix)
else:
cls.static_file = 'srm://%s%s%s/data.raw' % (hostname, web_service_path, prefix)
cmd = 'srmcp --debug=false -retry_num=0 file:///%s/data.raw %s' % (cls.tmpdir, cls.static_file)
execute(cmd)
for f in MgrTestCases.files_remote:
tmp = mgr.lfns2pfns(mgr.get_rse_info('FZK-LCG2_SCRATCHDISK'), {'name': f, 'scope': 'user.%s' % cls.user}, scheme='srm').values()[0]
cmd = 'srmcp --debug=false -retry_num=0 file:///%s/data.raw %s' % (cls.tmpdir, tmp)
execute(cmd)
def rename(self, pfn, new_pfn):
""" Allows to rename a file stored inside the connected RSE.
:param pfn Current physical file name
:param new_pfn New physical file name
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
if not self.exists(pfn):
raise exception.SourceNotFound()
try:
path = self.pfn2path(pfn)
new_path = self.pfn2path(new_pfn)
new_dir = new_path[:new_path.rindex('/') + 1]
cmd = 'xrdfs %s:%s mkdir -p %s' % (self.hostname, self.port, new_dir)
status, out, err = execute(cmd)
cmd = 'xrdfs %s:%s mv %s %s' % (self.hostname, self.port, path, new_path)
status, out, err = execute(cmd)
if not status == 0:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def rename(self, path, new_path):
"""
Allows to rename a file stored inside the connected RSE.
:param path: path to the current file on the storage
:param new_path: path to the new file on the storage
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
try:
cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 -D srmv2 %s %s' % (path, new_path)
status, out, err = execute(cmd)
if status:
raise exception.RucioException(err)
cmd = 'lcg-del $LCGVO -v -b -l --srm-timeout 600 -D srmv2 %s' % (path)
status, out, err = execute(cmd)
if status:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def exists(self, path):
"""
Checks if the requested file is known by the referred RSE.
:param path: Physical file name
:returns: True if the file exists, False if it doesn't
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
try:
cmd = 'lcg-ls $LCGVO -v -b --srm-timeout 60 -D srmv2 %s' % (path)
status, out, err = execute(cmd)
if status:
return False
return True
except Exception as error:
raise exception.ServiceUnavailable(error)
def connect(self):
""" Establishes the actual connection to the referred RSE.
:param credentials Provides information to establish a connection
to the referred storage system. For S3 connections these are
access_key, secretkey, host_base, host_bucket, progress_meter
and skip_existing.
:raises RSEAccessDenied
"""
try:
cmd = 'xrdfs %s:%s query stats %s:%s' % (self.hostname, self.port, self.hostname, self.port)
status, out, err = execute(cmd)
if not status == 0:
raise exception.RSEAccessDenied(err)
except Exception as e:
raise exception.RSEAccessDenied(e)
def get_file_guid(self, file):
guid = file.get('guid')
if not guid and 'pool.root' in file['basename'].lower() and not file.get('no_register'):
status, output, err = execute('pool_extractFileIdentifier %s' % file['path'])
if status != 0:
msg = 'Trying to upload ROOT files but pool_extractFileIdentifier tool can not be found.\n'
msg += 'Setup your ATHENA environment and try again.'
raise RucioException(msg)
try:
guid = output.splitlines()[-1].split()[0].replace('-', '').lower()
except Exception:
raise RucioException('Error extracting GUID from ouput of pool_extractFileIdentifier')
elif guid:
guid = guid.replace('-', '')
else:
guid = generate_uuid()
return guid
def exists(self, path):
"""
Checks if the requested file is known by the referred RSE.
:param path: Physical file name
:returns: True if the file exists, False if it doesn't
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
try:
cmd = 'lcg-ls $LCGVO -v -b --srm-timeout 60 -D srmv2 %s' % (path)
status, out, err = execute(cmd)
if status:
return False
return True
except Exception as e:
raise exception.ServiceUnavailable(e)
def get(self, pfn, dest):
""" Provides access to files stored inside connected the RSE.
:param pfn Physical file name of requested file
:param dest Name and path of the files when stored at the client
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
"""
if not self.exists(pfn):
raise exception.SourceNotFound()
try:
cmd = 'xrdcp -f %s %s' % (pfn, dest)
status, out, err = execute(cmd)
if not status == 0:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def tearDownClass(cls):
"""GFAL2 (RSE/PROTOCOLS): Removing created directorie s and files"""
data = load_test_conf_file('rse_repository.json')
prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][
'prefix']
hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][
'srm']['hostname']
if hostname.count("://"):
hostname = hostname.split("://")[1]
if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][
'srm'].keys():
port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']
['srm']['port'])
else:
port = 0
if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols'][
'supported']['srm'].keys() and 'web_service_path' in data[
'FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][
'extended_attributes'].keys():
web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols'][
'supported']['srm']['extended_attributes']['web_service_path']
else:
web_service_path = ''
shutil.rmtree(cls.tmpdir)
clean_raw = '%s/data.raw' % prefix
if int(port) > 0:
srm_path = ''.join(
["srm://", hostname, ":", port, web_service_path])
else:
srm_path = ''.join(["srm://", hostname, web_service_path])
list_files_cmd_user = '******' % (
srm_path, prefix, cls.user)
clean_files = str(execute(list_files_cmd_user)[1]).split('\n')
list_files_cmd_user = '******' % (
srm_path, prefix, cls.user)
clean_files += str(execute(list_files_cmd_user)[1]).split('\n')
clean_files.append("1024 " + clean_raw)
for files in clean_files:
if len(files.strip()) > 0:
file = files.split()[1]
if not file.endswith("/"):
clean_cmd = 'srmrm -2 --debug=false -retry_num=0 %s/%s' % (
srm_path, file)
execute(clean_cmd)
clean_directory = ['user', 'group']
for directory in clean_directory:
clean_cmd = 'srmrmdir -2 --debug=false -retry_num=0 -recursive %s%s/%s/%s' % (
srm_path, prefix, directory, cls.user)
execute(clean_cmd)
def get(self, pfn, dest):
""" Provides access to files stored inside connected the RSE.
:param pfn Physical file name of requested file
:param dest Name and path of the files when stored at the client
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
"""
try:
cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (pfn, dest)
status, out, err = execute(cmd)
if status == 54:
raise exception.SourceNotFound()
elif status != 0:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def put(self, source, target, source_dir):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
if not self.exists(dirname(target)):
self.mkdir(dirname(target))
cmd = 'rfcp %(source)s %(path)s' % locals()
status, out, err = execute(cmd)
return status == 0
def connect(self):
""" Establishes the actual connection to the referred RSE.
:raises RSEAccessDenied
"""
self.logger(
logging.DEBUG,
'ssh.connect: port: {}, hostname {}, ssh-user {}'.format(
self.port, self.hostname, self.sshuser))
try:
cmd = 'ssh -p %s %s%s echo ok 2>&1' % (self.port, self.sshuser,
self.hostname)
status, out, err = execute(cmd)
checker = re.search(r'ok', out)
if not checker:
raise exception.RSEAccessDenied(err)
except Exception as e:
raise exception.RSEAccessDenied(e)
def put(self, source, target, source_dir):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
if not self.exists(dirname(target)):
self.mkdir(dirname(target))
cmd = 'rfcp %(source)s %(path)s' % locals()
status, out, err = execute(cmd)
return status == 0
def exists(self, pfn):
""" Checks if the requested file is known by the referred RSE.
:param pfn Physical file name
:returns: True if the file exists, False if it doesn't
:raise ServiceUnavailable
"""
try:
path = self.pfn2path(pfn)
cmd = 'xrdfs %s:%s stat %s' % (self.hostname, self.port, path)
status, out, err = execute(cmd)
if not status == 0:
return False
except Exception as e:
raise exception.ServiceUnavailable(e)
return True
def delete(self, pfn):
"""
Deletes a file from the connected RSE.
:param pfn Physical file name
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
if not self.exists(pfn):
raise exception.SourceNotFound()
try:
path = self.pfn2path(pfn)
cmd = 'xrdfs %s:%s rm %s' % (self.hostname, self.port, path)
status, out, err = execute(cmd)
if not status == 0:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def get(self, pfn, dest, transfer_timeout=None):
""" Provides access to files stored inside connected the RSE.
:param pfn Physical file name of requested file
:param dest Name and path of the files when stored at the client
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
"""
self.logger(logging.DEBUG, 'rclone.get: pfn: {}'.format(pfn))
try:
path = self.pfn2path(pfn)
cmd = 'rclone copyto %s:%s %s' % (self.hostname, path, dest)
self.logger(logging.DEBUG, 'rclone.get: cmd: {}'.format(cmd))
status, out, err = execute(cmd)
if status:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def exists(self, pfn):
""" Checks if the requested file is known by the referred RSE.
:param pfn Physical file name
:returns: True if the file exists, False if it doesn't
:raise ServiceUnavailable
"""
try:
path = self.pfn2path(pfn)
cmd = 'xrdfs %s:%s stat %s' % (self.hostname, self.port, path)
status, out, err = execute(cmd)
if not status == 0:
return False
except Exception as e:
raise exception.ServiceUnavailable(e)
return True
def delete(self, pfn):
"""
Deletes a file from the connected RSE.
:param pfn Physical file name
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
if not self.exists(pfn):
raise exception.SourceNotFound()
try:
path = self.pfn2path(pfn)
cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s rm %s' % (self.hostname, self.port, path)
status, out, err = execute(cmd)
if not status == 0:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def test_upload_file_guid_with_impl(self):
"""CLIENT(USER): Rucio upload file with guid"""
tmp_file1 = file_generator()
tmp_guid = uuid()
impl = 'xrootd'
cmd = 'rucio upload --rse {0} --scope {1} --impl {2} --guid {3} {4}'.format(
self.rse, self.scope, impl, tmp_guid, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
self.assertEqual(exitcode, 0)
upload_string_1 = ('Successfully uploaded file %s' %
path.basename(tmp_file1))
self.assertIsNotNone(re.search(upload_string_1, err))
added_dids = ['{0}:{1}'.format(self.scope, did) for did in tmp_file1]
self.generated_dids += added_dids
def get(self, pfn, dest, transfer_timeout=None):
""" Provides access to files stored inside connected the RSE.
:param pfn Physical file name of requested file
:param dest Name and path of the files when stored at the client
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
"""
self.logger.debug('xrootd.get: pfn: {}'.format(pfn))
try:
cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (pfn, dest)
self.logger.info('xrootd.get: cmd: {}'.format(cmd))
status, out, err = execute(cmd)
if status == 54:
raise exception.SourceNotFound()
elif status != 0:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def connect(self):
""" Establishes the actual connection to the referred RSE.
:param credentials Provides information to establish a connection
to the referred storage system. For S3 connections these are
access_key, secretkey, host_base, host_bucket, progress_meter
and skip_existing.
:raises RSEAccessDenied
"""
try:
# The query stats call is not implemented on some xroot doors.
# Workaround: fail, if server does not reply within 10 seconds for static config query
cmd = 'XrdSecPROTOCOL=gsi XRD_REQUESTTIMEOUT=10 xrdfs %s:%s query config %s:%s' % (self.hostname, self.port, self.hostname, self.port)
status, out, err = execute(cmd)
if not status == 0:
raise exception.RSEAccessDenied(err)
except Exception as e:
raise exception.RSEAccessDenied(e)
def test_reaper():
""" REAPER (DAEMON): Test the reaper daemon."""
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
vo = {'vo': 'tst'}
else:
vo = {}
nb_files = 30
file_size = 2147483648 # 2G
rse_id = rse_core.get_rse_id(rse='MOCK', **vo)
for i in range(nb_files):
replica_core.add_replica(rse_id=rse_id,
scope=InternalScope('data13_hip', **vo),
name='lfn' + generate_uuid(),
bytes=file_size,
account=InternalAccount('root', **vo),
adler32=None,
md5=None)
rse_core.set_rse_usage(rse_id=rse_id,
source='srm',
used=nb_files * file_size,
free=800)
rse_core.set_rse_limits(rse_id=rse_id,
name='MinFreeSpace',
value=10737418240)
rse_core.set_rse_limits(rse_id=rse_id,
name='MaxBeingDeletedFiles',
value=10)
argv = ['--run-once', '--rses', 'MOCK']
main(argv)
# Test the rucio-reaper console script
cmd = 'rucio-reaper ' + ' '.join(argv)
exitcode, out, err = execute(cmd)
print(cmd, out, err)
nose.tools.assert_equal(exitcode, 0)
def delete(self, pfn):
"""
Deletes a file from the connected RSE.
:param pfn Physical file name
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
self.logger(logging.DEBUG, 'rclone.delete: pfn: {}'.format(pfn))
if not self.exists(pfn):
raise exception.SourceNotFound()
try:
path = self.pfn2path(pfn)
cmd = 'rclone delete %s:%s' % (self.hostname, path)
self.logger(logging.DEBUG, 'rclone.delete: cmd: {}'.format(cmd))
status, out, err = execute(cmd)
if status != 0:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def exists(self, pfn):
""" Checks if the requested file is known by the referred RSE.
:param pfn Physical file name
:returns: True if the file exists, False if it doesn't
:raise ServiceUnavailable
"""
self.logger(logging.DEBUG, 'rclone.exists: pfn: {}'.format(pfn))
try:
path = self.pfn2path(pfn)
cmd = 'rclone lsf %s:%s' % (self.hostname, path)
self.logger(logging.DEBUG, 'rclone.exists: cmd: {}'.format(cmd))
status, out, err = execute(cmd)
if status:
return False
except Exception as e:
raise exception.ServiceUnavailable(e)
return True
def get(self, pfn, dest, transfer_timeout=None):
""" Provides access to files stored inside connected the RSE.
:param pfn: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
"""
self.logger(logging.DEBUG, 'rsync.get: pfn: {}'.format(pfn))
try:
path = self.pfn2path(pfn)
destdir = os.path.dirname(dest)
cmd = 'mkdir -p %s && rsync -az -e "ssh -p %s" --append-verify %s%s:%s %s' % (
destdir, self.port, self.sshuser, self.hostname, path, dest)
self.logger(logging.DEBUG, 'rsync.get: cmd: {}'.format(cmd))
status, out, err = execute(cmd)
if status:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def exists(self, pfn):
""" Checks if the requested file is known by the referred RSE.
:param pfn Physical file name
:returns: True if the file exists, False if it doesn't
:raise ServiceUnavailable
"""
self.logger.debug('xrootd.exists: pfn: {}'.format(pfn))
try:
path = self.pfn2path(pfn)
cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s stat %s' % (self.hostname,
self.port, path)
self.logger.info('xrootd.exists: cmd: {}'.format(cmd))
status, out, err = execute(cmd)
if status != 0:
return False
except Exception as e:
raise exception.ServiceUnavailable(e)
return True
def put(self, source, target, source_dir, transfer_timeout=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:param transfer_timeout: Transfer timeout (in seconds)
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
source_url = '%s/%s' % (source_dir, source) if source_dir else source
if not os.path.exists(source_url):
raise exception.SourceNotFound()
space_token = ''
if self.attributes[
'extended_attributes'] is not None and 'space_token' in list(
self.attributes['extended_attributes'].keys()):
space_token = '--dst %s' % self.attributes['extended_attributes'][
'space_token']
timeout_option = ''
if transfer_timeout:
timeout_option = '--sendreceive-timeout %s' % transfer_timeout
try:
cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 %s -D srmv2 %s file:%s %s' % (
timeout_option, space_token, source_url, target)
status, out, err = execute(cmd)
if status:
raise exception.RucioException(err)
except Exception as error:
raise exception.ServiceUnavailable(error)
def _get_file_guid(self, file):
"""
Get the guid of a file, trying different strategies
(This function is meant to be used as class internal only)
:param file: dictionary describing the file
:returns: the guid
"""
guid = file.get('guid')
if not guid and 'pool.root' in file['basename'].lower() and not file.get('no_register'):
status, output, err = execute('pool_extractFileIdentifier %s' % file['path'])
if status != 0:
msg = 'Trying to upload ROOT files but pool_extractFileIdentifier tool can not be found.\n'
msg += 'Setup your ATHENA environment and try again.'
raise RucioException(msg)
try:
guid = output.splitlines()[-1].split()[0].replace('-', '').lower()
except Exception:
raise RucioException('Error extracting GUID from ouput of pool_extractFileIdentifier')
elif guid:
guid = guid.replace('-', '')
else:
guid = generate_uuid()
return guid
def get(self, path, dest):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
try:
cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 -D srmv2 %s file:%s' % (
path, dest)
status, out, err = execute(cmd)
if status:
if self.__parse_srm_error__("SRM_INVALID_PATH", out, err):
raise exception.SourceNotFound(err)
raise exception.RucioException(err)
except exception.SourceNotFound as error:
raise exception.SourceNotFound(str(error))
except Exception as error:
raise exception.ServiceUnavailable(error)
def setUpClass(cls):
"""XROOTD (RSE/PROTOCOLS): Creating necessary directories and files """
# Getting info for the test environment
rse_id, prefix, hostname, port = cls.get_rse_info()
try:
os.mkdir(prefix)
except Exception as e:
print(e)
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = uuid()
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024 * 1024) - 1) # 1 MB
out.write(b'\0')
for f in MgrTestCases.files_local:
shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
protocol = rsemanager.create_protocol(rsemanager.get_rse_info(rse_id),
'write')
protocol.connect()
os.system('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' %
prefix)
cls.static_file = 'xroot://%s:%d/%s/data.raw' % (hostname, port,
prefix)
cmd = 'xrdcp %s/data.raw %s' % (prefix, cls.static_file)
execute(cmd)
for f in MgrTestCases.files_remote:
path = protocol.path2pfn(
prefix + protocol._get_path('user.%s' % cls.user, f))
cmd = 'xrdcp %s/data.raw %s' % (prefix, path)
execute(cmd)
for f in MgrTestCases.files_local_and_remote:
shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
path = protocol.path2pfn(
prefix + protocol._get_path('user.%s' % cls.user, f))
cmd = 'xrdcp %s/%s %s' % (cls.tmpdir, f, path)
execute(cmd)
def put(self, filename, target, source_dir):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
source_url = '%s/%s' % (source_dir, filename)
path = self.path2pfn(target)
if not os.path.exists(source_url):
raise exception.SourceNotFound()
try:
cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (source_url, path)
status, out, err = execute(cmd)
if not status == 0:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def put(self, filename, target, source_dir):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
source_url = '%s/%s' % (source_dir, filename)
path = self.path2pfn(target)
if not os.path.exists(source_url):
raise exception.SourceNotFound()
try:
cmd = 'xrdcp -f %s %s' % (source_url, path)
status, out, err = execute(cmd)
if not status == 0:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def rename(self, pfn, new_pfn):
""" Allows to rename a file stored inside the connected RSE.
:param pfn Current physical file name
:param new_pfn New physical file name
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
self.logger(logging.DEBUG, 'rclone.rename: pfn: {}'.format(pfn))
if not self.exists(pfn):
raise exception.SourceNotFound()
try:
path = self.pfn2path(pfn)
new_path = self.pfn2path(new_pfn)
cmd = 'rclone moveto %s:%s %s:%s' % (self.hostname, path,
self.hostname, new_path)
self.logger(logging.DEBUG,
'rclone.stat: rename cmd: {}'.format(cmd))
status, out, err = execute(cmd)
if status != 0:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def get_rse_info(cls):
"""
Detects if containerized rses for xrootd are available in the testing environment.
:return: A tuple (rse, prefix, hostname, port).
"""
cmd = "rucio list-rses --expression 'test_container_xrd=True'"
print(cmd)
exitcode, out, err = execute(cmd)
print(out, err)
rses = out.split()
data = load_test_conf_file('rse_repository.json')
prefix = data['WJ-XROOTD']['protocols']['supported']['xroot']['prefix']
port = data['WJ-XROOTD']['protocols']['supported']['xroot']['port']
if len(rses) == 0:
rse_id = 'WJ-XROOTD'
hostname = data['WJ-XROOTD']['protocols']['supported']['xroot']['hostname']
else:
rse_id = 'XRD1'
hostname = 'xrd1'
prefix = '/rucio/'
# TODO should read container info from a config file
return rse_id, prefix, hostname, port
def put(self, filename, target, source_dir, transfer_timeout=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
self.logger(
logging.DEBUG,
'rsync.put: filename: {} target: {}'.format(filename, target))
source_dir = source_dir or '.'
source_url = '%s/%s' % (source_dir, filename)
self.logger(logging.DEBUG,
'rsync.put: source url: {}'.format(source_url))
path = self.pfn2path(target)
pathdir = os.path.dirname(path)
if not os.path.exists(source_url):
raise exception.SourceNotFound()
try:
cmd = 'ssh -p %s %s%s "mkdir -p %s" && rsync -az -e "ssh -p %s" --append-verify %s %s%s:%s' % (
self.port, self.sshuser, self.hostname, pathdir, self.port,
source_url, self.sshuser, self.hostname, path)
self.logger(logging.DEBUG, 'rsync.put: cmd: {}'.format(cmd))
status, out, err = execute(cmd)
if status:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def teardownClass(cls):
"""SRM (RSE/PROTOCOLS): Removing created directorie s and files"""
with open('etc/rse_repository.json') as f:
data = json.load(f)
prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['prefix']
hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['hostname']
if hostname.count("://"):
hostname = hostname.split("://")[1]
if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys():
port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['port'])
else:
port = 0
if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys() and 'web_service_path' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes'].keys():
web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes']['web_service_path']
else:
web_service_path = ''
shutil.rmtree(cls.tmpdir)
clean_raw = '%s/data.raw' % prefix
if int(port) > 0:
srm_path = ''.join(["srm://", hostname, ":", port, web_service_path])
else:
srm_path = ''.join(["srm://", hostname, web_service_path])
list_files_cmd_user = '******' % (srm_path, prefix, cls.user)
clean_files = str(execute(list_files_cmd_user)[1]).split('\n')
list_files_cmd_user = '******' % (srm_path, prefix, cls.user)
clean_files += str(execute(list_files_cmd_user)[1]).split('\n')
clean_files.append("1024 " + clean_raw)
for files in clean_files:
if len(files.strip()) > 0:
file = files.split()[1]
if not file.endswith("/"):
clean_cmd = 'srmrm --debug=false -retry_num=0 %s/%s' % (srm_path, file)
execute(clean_cmd)
clean_directory = ['user', 'group']
for directory in clean_directory:
clean_cmd = 'srmrmdir --debug=false -retry_num=0 -recursive %s%s/%s/%s' % (srm_path, prefix, directory, cls.user)
execute(clean_cmd)
def tearDownClass(cls):
"""XROOTD (RSE/PROTOCOLS): Removing created directorie s and files"""
rse_id, prefix, hostname, port = cls.get_rse_info()
shutil.rmtree(prefix)
shutil.rmtree(cls.tmpdir)
clean_raw = '%s/data.raw' % prefix
list_files_cmd_user = '******' % (hostname, prefix, cls.user)
clean_files = str(execute(list_files_cmd_user)[1]).split('\n')
list_files_cmd_group = 'xrdfs %s ls %s/group.%s' % (hostname, prefix, cls.user)
clean_files += str(execute(list_files_cmd_group)[1]).split('\n')
clean_files.append(clean_raw)
for files in clean_files:
clean_cmd = 'xrdfs %s rm %s' % (hostname, files)
execute(clean_cmd)
clean_prefix = '%s' % prefix
list_directory = 'xrdfs %s ls %s' % (hostname, prefix)
clean_directory = str(execute(list_directory)[1]).split('\n')
clean_directory.append(clean_prefix)
for directory in clean_directory:
clean_cmd = 'xrdfs %s rmdir %s' % (hostname, directory)
execute(clean_cmd)
def generate_file(fname, size, logger=logging.log):
cmd = '/bin/dd if=/dev/urandom of=%s bs=%s count=1' % (fname, size)
exitcode, out, err = execute(cmd)
logger(logging.DEBUG, out)
logger(logging.DEBUG, err)
return exitcode
def mkdir(self, directory):
cmd = 'rfmkdir -p %(path)s' % locals()
status, out, err = execute(cmd)
return status == 0
def generate_file(fname, size):
cmd = '/bin/dd if=/dev/urandom of=%(fname)s bs=%(size)s count=1' % locals()
exitcode, out, err = execute(cmd)
logging.debug(out)
logging.debug(err)
return exitcode
def generate_file(fname, size):
cmd = '/bin/dd if=/dev/urandom of=%(fname)s bs=%(size)s count=1' % locals()
exitcode, out, err = execute(cmd)
logging.debug(out)
logging.debug(err)
return exitcode
def setUpClass(cls):
"""GFAL2 (RSE/PROTOCOLS): Creating necessary directories and files """
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = uuid()
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024 * 1024) - 1) # 1 MB
out.write('\0')
for f in MgrTestCases.files_local:
shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
with open('etc/rse_repository.json') as f:
data = json.load(f)
prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][
'prefix']
hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][
'srm']['hostname']
if hostname.count("://"):
hostname = hostname.split("://")[1]
if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][
'srm'].keys():
port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']
['srm']['port'])
else:
port = 0
if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols'][
'supported']['srm'].keys() and 'web_service_path' in data[
'FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][
'extended_attributes'].keys():
web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols'][
'supported']['srm']['extended_attributes']['web_service_path']
else:
web_service_path = ''
os.system('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' %
cls.tmpdir)
if port > 0:
cls.static_file = 'srm://%s:%s%s%s/data.raw' % (
hostname, port, web_service_path, prefix)
else:
cls.static_file = 'srm://%s%s%s/data.raw' % (
hostname, web_service_path, prefix)
cmd = 'srmcp -2 --debug=false -retry_num=0 file:///%s/data.raw %s' % (
cls.tmpdir, cls.static_file)
execute(cmd)
rse_settings = mgr.get_rse_info('FZK-LCG2_SCRATCHDISK')
for protocol in rse_settings['protocols']:
if protocol['scheme'] != "srm":
rse_settings['protocols'].remove(protocol)
if len(rse_settings['protocols']) > 0:
rse_settings['protocols'][0][
'impl'] = 'rucio.rse.protocols.gfal.Default'
for f in MgrTestCases.files_remote:
tmp = mgr.lfns2pfns(rse_settings, {
'name': f,
'scope': 'user.%s' % cls.user
},
scheme='srm').values()[0]
cmd = 'srmcp -2 --debug=false -retry_num=0 file:///%s/data.raw %s' % (
cls.tmpdir, tmp)
execute(cmd)
def mkdir(self, directory):
""" Create new directory. """
cmd = 'rfmkdir -p %(path)s' % locals()
status, out, err = execute(cmd)
return status == 0