def tearDownClass(cls):
"""SFTP (RSE/PROTOCOLS): Removing created directorie s and files """
# Load local creditentials from file
credentials = {}
data = load_test_conf_file('rse-accounts.cfg')
credentials = data['LXPLUS']
lxplus = pysftp.Connection(**credentials)
prefix = load_test_conf_file('rse_repository.json')['LXPLUS'][
'protocols']['supported']['sftp']['prefix']
lxplus.execute('rm -rf %s' % prefix)
lxplus.close()
shutil.rmtree(cls.tmpdir)
def __init__(self,
tmpdir,
rse_tag,
user,
static_file,
vo='def',
impl=None):
self.rse_settings = mgr.get_rse_info(rse=rse_tag, vo=vo)
try:
data = load_test_conf_file('rse-accounts.cfg.template')
self.rse_settings['credentials'] = data[rse_tag]
except KeyError:
print('No credentials found for this RSE.')
pass
self.tmpdir = tmpdir
self.gettmpdir = tempfile.mkdtemp()
self.user = user
self.static_file = static_file
self.impl = None
if impl and len(self.rse_settings['protocols']) > 0:
if len(impl.split('.')) == 1:
self.impl = 'rucio.rse.protocols.' + impl + '.Default'
else:
self.impl = 'rucio.rse.protocols.' + impl
self.vo = vo
def setUpClass(cls):
"""POSIX (RSE/PROTOCOLS): Creating necessary directories and files """
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = uuid()
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024 * 1024) - 1) # 1 MB
out.write(b'\0')
for f in MgrTestCases.files_local:
shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
data = load_test_conf_file('rse_repository.json')
prefix = data['MOCK-POSIX']['protocols']['supported']['file']['prefix']
try:
os.mkdir(prefix)
except Exception as e:
print(e)
os.system('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' %
prefix)
cls.static_file = '%s/data.raw' % prefix
for f in MgrTestCases.files_remote:
protocol = mgr.create_protocol(mgr.get_rse_info('MOCK-POSIX'),
'write')
pfn = mgr.lfns2pfns(mgr.get_rse_info('MOCK-POSIX'), {
'name': f,
'scope': 'user.%s' % cls.user
}).values()[0]
path = protocol.pfn2path(pfn)
dirs = os.path.dirname(path)
if not os.path.exists(dirs):
os.makedirs(dirs)
shutil.copy('%s/data.raw' % prefix, path)
def get_rse_info(cls):
"""
Detects if containerized rses for xrootd are available in the testing environment.
:return: A tuple (rse, prefix, hostname, port).
"""
cmd = "rucio list-rses --rses 'test_container_xrd=True'"
print(cmd)
exitcode, out, err = execute(cmd)
print(out, err)
rses = out.split()
data = load_test_conf_file('rse_repository.json')
prefix = data['WJ-XROOTD']['protocols']['supported']['xroot']['prefix']
port = data['WJ-XROOTD']['protocols']['supported']['xroot']['port']
if len(rses) == 0:
rse_id = 'WJ-XROOTD'
hostname = data['WJ-XROOTD']['protocols']['supported']['xroot'][
'hostname']
else:
rse_id = 'XRD1'
hostname = 'xrd1'
prefix = '/rucio/'
# TODO should read container info from a config file
return rse_id, prefix, hostname, port
def load_rse_info(request, containerized_rses):
"""
Detects if containerized rses for ssh are available in the testing environment.
:return: A tuple (rse, prefix, hostname, port, user).
"""
rses = [rse for rse in containerized_rses if rse[0] == 'SSH1']
data = load_test_conf_file('rse_repository.json')
request.cls.prefix = data['SSH_DISK']['protocols']['supported']['rsync'][
'prefix']
request.cls.port = data['SSH_DISK']['protocols']['supported']['rsync'][
'port']
request.cls.sshuser = data['SSH_DISK']['protocols']['supported']['rsync'][
'extended_attributes']['user']
if len(rses) == 0:
request.cls.rse_id = 'SSH-RSE'
request.cls.hostname = data['SSH-RSE']['protocols']['supported'][
'rsync']['hostname']
else:
request.cls.rse_id = 'SSH1'
request.cls.hostname = 'ssh1'
request.cls.prefix = '/rucio/'
request.cls.port = 22
request.cls.sshuser = '******'
def tearDownClass(cls):
"""S3 (RSE/PROTOCOLS): Removing created directories and files """
# Remove test files from storage
# rse_tag = 'AMAZON-BOTO'
rse_tag = 'BNL-BOTO'
rse_settings = mgr.get_rse_info(rse_tag)
try:
data = load_test_conf_file('rse-accounts.cfg')
rse_settings['credentials'] = data[rse_tag]
except KeyError:
print('No credentials found for this RSE.')
pass
for protocol in rse_settings['protocols']:
if protocol['scheme'] == 's3':
break
conn = boto.connect_s3(
host=protocol['hostname'],
port=int(protocol.get('port', 80)),
aws_access_key_id=rse_settings['credentials']['access_key'],
aws_secret_access_key=rse_settings['credentials']['secret_key'],
is_secure=rse_settings['credentials'].get('is_secure', False),
calling_format=boto.s3.connection.OrdinaryCallingFormat())
for protocol in rse_settings['protocols']:
if protocol['scheme'] == 's3':
break
bucket_name = protocol['prefix']
bucket = conn.get_bucket(bucket_name)
keys = bucket.list()
for key in keys:
key.delete()
def setUpClass(cls):
"""SRM (RSE/PROTOCOLS): Creating necessary directories and files """
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = uuid()
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024 * 1024) - 1) # 1 MB
out.write(b'\0')
for f in MgrTestCases.files_local:
shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
data = load_test_conf_file('rse_repository.json')
prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][
'prefix']
hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][
'srm']['hostname']
if hostname.count("://"):
hostname = hostname.split("://")[1]
if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][
'srm'].keys():
port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']
['srm']['port'])
else:
port = 0
if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols'][
'supported']['srm'].keys() and 'web_service_path' in data[
'FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][
'extended_attributes'].keys():
web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols'][
'supported']['srm']['extended_attributes']['web_service_path']
else:
web_service_path = ''
os.system('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' %
cls.tmpdir)
if port > 0:
cls.static_file = 'srm://%s:%s%s%s/data.raw' % (
hostname, port, web_service_path, prefix)
else:
cls.static_file = 'srm://%s%s%s/data.raw' % (
hostname, web_service_path, prefix)
cmd = 'srmcp --debug=false -retry_num=0 file:///%s/data.raw %s' % (
cls.tmpdir, cls.static_file)
execute(cmd)
for f in MgrTestCases.files_remote:
tmp = mgr.lfns2pfns(mgr.get_rse_info('FZK-LCG2_SCRATCHDISK'), {
'name': f,
'scope': 'user.%s' % cls.user
},
scheme='srm').values()[0]
cmd = 'srmcp --debug=false -retry_num=0 file:///%s/data.raw %s' % (
cls.tmpdir, tmp)
execute(cmd)
def tearDownClass(cls):
"""GFAL2 (RSE/PROTOCOLS): Removing created directorie s and files"""
data = load_test_conf_file('rse_repository.json')
prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][
'prefix']
hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][
'srm']['hostname']
if hostname.count("://"):
hostname = hostname.split("://")[1]
if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][
'srm'].keys():
port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']
['srm']['port'])
else:
port = 0
if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols'][
'supported']['srm'].keys() and 'web_service_path' in data[
'FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][
'extended_attributes'].keys():
web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols'][
'supported']['srm']['extended_attributes']['web_service_path']
else:
web_service_path = ''
shutil.rmtree(cls.tmpdir)
clean_raw = '%s/data.raw' % prefix
if int(port) > 0:
srm_path = ''.join(
["srm://", hostname, ":", port, web_service_path])
else:
srm_path = ''.join(["srm://", hostname, web_service_path])
list_files_cmd_user = '******' % (
srm_path, prefix, cls.user)
clean_files = str(execute(list_files_cmd_user)[1]).split('\n')
list_files_cmd_user = '******' % (
srm_path, prefix, cls.user)
clean_files += str(execute(list_files_cmd_user)[1]).split('\n')
clean_files.append("1024 " + clean_raw)
for files in clean_files:
if len(files.strip()) > 0:
file = files.split()[1]
if not file.endswith("/"):
clean_cmd = 'srmrm -2 --debug=false -retry_num=0 %s/%s' % (
srm_path, file)
execute(clean_cmd)
clean_directory = ['user', 'group']
for directory in clean_directory:
clean_cmd = 'srmrmdir -2 --debug=false -retry_num=0 -recursive %s%s/%s/%s' % (
srm_path, prefix, directory, cls.user)
execute(clean_cmd)
def __init__(self, tmpdir, rse_tag, user, static_file, vo='def'):
self.rse_settings = mgr.get_rse_info(rse=rse_tag, vo=vo)
try:
data = load_test_conf_file('rse-accounts.cfg')
self.rse_settings['credentials'] = data[rse_tag]
except KeyError:
print('No credentials found for this RSE.')
pass
self.tmpdir = tmpdir
self.gettmpdir = tempfile.mkdtemp()
self.user = user
self.static_file = static_file
def setUpClass(cls):
"""WebDAV (RSE/PROTOCOLS): Creating necessary directories and files """
session = requests.Session()
session.cert = os.getenv('X509_USER_PROXY')
session.verify = False
cls.site = 'FZK-LCG2_SCRATCHDISK'
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = '******'
data = load_test_conf_file('rse_repository.json')
scheme = data[cls.site]['protocols']['supported']['https']['scheme']
prefix = data[cls.site]['protocols']['supported']['https']['prefix']
hostname = data[
cls.site]['protocols']['supported']['https']['hostname']
port = data[cls.site]['protocols']['supported']['https']['port']
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024) - 1) # 1 kB
out.write(b'\0')
for f in MgrTestCases.files_local:
os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
cls.static_file = '%s://%s:%s%sdata.raw' % (scheme, hostname, port,
prefix)
rse_settings = rsemanager.get_rse_info(cls.site)
storage = rsemanager.create_protocol(rse_settings,
operation='write',
scheme='https')
storage.connect()
for f in MgrTestCases.files_remote:
os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
destfile = rsemanager.lfns2pfns(rse_settings, [
{
'name': f,
'scope': 'user.%s' % (cls.user)
},
],
operation='write',
scheme='https').values()[0]
try:
storage.put('%s/%s' % (cls.tmpdir, f), destfile)
except FileReplicaAlreadyExists as e:
print(e)
with open('%s/data.raw' % cls.tmpdir, 'rb') as f_file:
session.put(cls.static_file,
data=f_file.read(),
verify=False,
allow_redirects=True)
def setUpClass(cls):
"""SFTP (RSE/PROTOCOLS): Creating necessary directories and files """
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = uuid()
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024 * 1024) - 1) # 1 MB
out.write(b'\0')
for fil in MgrTestCases.files_local:
os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, fil))
# Load local credentials from file
data = load_test_conf_file('rse-accounts.cfg')
credentials = data['LXPLUS']
lxplus = pysftp.Connection(**credentials)
prefix = load_test_conf_file('rse_repository.json')['LXPLUS'][
'protocols']['supported']['sftp']['prefix']
lxplus.execute('mkdir %s' % prefix)
lxplus.execute('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' %
prefix)
cls.static_file = 'sftp://lxplus.cern.ch:22%sdata.raw' % prefix
protocol = mgr.create_protocol(mgr.get_rse_info('LXPLUS'), 'write')
for fil in MgrTestCases.files_remote:
tmp = protocol.parse_pfns(
protocol.lfns2pfns({
'name': fil,
'scope': 'user.%s' % cls.user
}).values()[0]).values()[0]
for cmd in [
'mkdir -p %s' % ''.join([tmp['prefix'], tmp['path']]),
'ln -s %sdata.raw %s' %
(prefix, ''.join([tmp['prefix'], tmp['path'], tmp['name']]))
]:
lxplus.execute(cmd)
lxplus.close()
def setUpClass(cls):
"""S3 (RSE/PROTOCOLS): Creating necessary directories and files """
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = uuid()
# cls.user = '******' # use again when latency issue with S3 storage is resolved
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024 * 1024) - 1) # 1 MB
out.write(b'\0')
for f in MgrTestCases.files_local:
os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
rse_tag = 'BNL-BOTO'
rse_settings = mgr.get_rse_info(rse_tag)
try:
data = load_test_conf_file('rse-accounts.cfg')
rse_settings['credentials'] = data[rse_tag]
except KeyError:
print('No credentials found for this RSE.')
pass
for protocol in rse_settings['protocols']:
if protocol['scheme'] == 's3':
break
conn = boto.connect_s3(
host=protocol['hostname'],
port=int(protocol.get('port', 80)),
aws_access_key_id=rse_settings['credentials']['access_key'],
aws_secret_access_key=rse_settings['credentials']['secret_key'],
is_secure=rse_settings['credentials'].get('is_secure', False),
calling_format=boto.s3.connection.OrdinaryCallingFormat())
cls.static_file = mgr.lfns2pfns(rse_settings, {
'name': 'data.raw',
'scope': 'user.%s' % cls.user
}).values()[0]
key = get_bucket_key(cls.static_file, conn, create=True)
key.set_contents_from_filename("%s/data.raw" % cls.tmpdir)
for f in MgrTestCases.files_remote:
pfn = mgr.lfns2pfns(rse_settings, {
'name': f,
'scope': 'user.%s' % cls.user
}).values()[0]
bucket_name, key_name = get_bucket_key_name(pfn)
key.copy(bucket_name, key_name)
def load_rse_info(request, containerized_rses):
"""
Detects if containerized rses for ssh are available in the testing environment.
:return: A tuple (rse, prefix, hostname).
"""
rses = [rse for rse in containerized_rses if rse[0] == 'SSH1']
data = load_test_conf_file('rse_repository.json')
request.cls.prefix = data['SSH_DISK']['protocols']['supported']['rclone'][
'prefix']
if len(rses) == 0:
request.cls.rse_id = 'SSH-RSE'
request.cls.hostname = data['SSH-RSE']['protocols']['supported'][
'rclone']['hostname']
else:
request.cls.rse_id = 'SSH1'
request.cls.hostname = 'ssh_rclone_rse'
request.cls.prefix = '/rucio/'
def tearDownClass(cls):
"""WebDAV (RSE/PROTOCOLS): Removing created directories and files """
rse_settings = rsemanager.get_rse_info(cls.site)
data = load_test_conf_file('rse_repository.json')
scheme = data[cls.site]['protocols']['supported']['https']['scheme']
prefix = data[cls.site]['protocols']['supported']['https']['prefix']
hostname = data[
cls.site]['protocols']['supported']['https']['hostname']
port = data[cls.site]['protocols']['supported']['https']['port']
storage = rsemanager.create_protocol(rse_settings,
operation='write',
scheme='https')
print(rse_settings)
storage.connect()
status1 = storage.delete('%s://%s:%s%suser/%s' %
(scheme, hostname, port, prefix, cls.user))
print(status1)
status2 = storage.delete('%s://%s:%s%sgroup/%s' %
(scheme, hostname, port, prefix, cls.user))
print(status2)
def tearDownClass(cls):
"""POSIX (RSE/PROTOCOLS): Removing created directorie s and files """
data = load_test_conf_file('rse_repository.json')
prefix = data['MOCK-POSIX']['protocols']['supported']['file']['prefix']
shutil.rmtree(prefix)
shutil.rmtree(cls.tmpdir)
