def tearDownClass(cls): """SFTP (RSE/PROTOCOLS): Removing created directorie s and files """ # Load local creditentials from file credentials = {} data = load_test_conf_file('rse-accounts.cfg') credentials = data['LXPLUS'] lxplus = pysftp.Connection(**credentials) prefix = load_test_conf_file('rse_repository.json')['LXPLUS'][ 'protocols']['supported']['sftp']['prefix'] lxplus.execute('rm -rf %s' % prefix) lxplus.close() shutil.rmtree(cls.tmpdir)
def __init__(self, tmpdir, rse_tag, user, static_file, vo='def', impl=None): self.rse_settings = mgr.get_rse_info(rse=rse_tag, vo=vo) try: data = load_test_conf_file('rse-accounts.cfg.template') self.rse_settings['credentials'] = data[rse_tag] except KeyError: print('No credentials found for this RSE.') pass self.tmpdir = tmpdir self.gettmpdir = tempfile.mkdtemp() self.user = user self.static_file = static_file self.impl = None if impl and len(self.rse_settings['protocols']) > 0: if len(impl.split('.')) == 1: self.impl = 'rucio.rse.protocols.' + impl + '.Default' else: self.impl = 'rucio.rse.protocols.' + impl self.vo = vo
def setUpClass(cls): """POSIX (RSE/PROTOCOLS): Creating necessary directories and files """ # Creating local files cls.tmpdir = tempfile.mkdtemp() cls.user = uuid() with open("%s/data.raw" % cls.tmpdir, "wb") as out: out.seek((1024 * 1024) - 1) # 1 MB out.write(b'\0') for f in MgrTestCases.files_local: shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f)) data = load_test_conf_file('rse_repository.json') prefix = data['MOCK-POSIX']['protocols']['supported']['file']['prefix'] try: os.mkdir(prefix) except Exception as e: print(e) os.system('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' % prefix) cls.static_file = '%s/data.raw' % prefix for f in MgrTestCases.files_remote: protocol = mgr.create_protocol(mgr.get_rse_info('MOCK-POSIX'), 'write') pfn = mgr.lfns2pfns(mgr.get_rse_info('MOCK-POSIX'), { 'name': f, 'scope': 'user.%s' % cls.user }).values()[0] path = protocol.pfn2path(pfn) dirs = os.path.dirname(path) if not os.path.exists(dirs): os.makedirs(dirs) shutil.copy('%s/data.raw' % prefix, path)
def get_rse_info(cls): """ Detects if containerized rses for xrootd are available in the testing environment. :return: A tuple (rse, prefix, hostname, port). """ cmd = "rucio list-rses --rses 'test_container_xrd=True'" print(cmd) exitcode, out, err = execute(cmd) print(out, err) rses = out.split() data = load_test_conf_file('rse_repository.json') prefix = data['WJ-XROOTD']['protocols']['supported']['xroot']['prefix'] port = data['WJ-XROOTD']['protocols']['supported']['xroot']['port'] if len(rses) == 0: rse_id = 'WJ-XROOTD' hostname = data['WJ-XROOTD']['protocols']['supported']['xroot'][ 'hostname'] else: rse_id = 'XRD1' hostname = 'xrd1' prefix = '/rucio/' # TODO should read container info from a config file return rse_id, prefix, hostname, port
def load_rse_info(request, containerized_rses): """ Detects if containerized rses for ssh are available in the testing environment. :return: A tuple (rse, prefix, hostname, port, user). """ rses = [rse for rse in containerized_rses if rse[0] == 'SSH1'] data = load_test_conf_file('rse_repository.json') request.cls.prefix = data['SSH_DISK']['protocols']['supported']['rsync'][ 'prefix'] request.cls.port = data['SSH_DISK']['protocols']['supported']['rsync'][ 'port'] request.cls.sshuser = data['SSH_DISK']['protocols']['supported']['rsync'][ 'extended_attributes']['user'] if len(rses) == 0: request.cls.rse_id = 'SSH-RSE' request.cls.hostname = data['SSH-RSE']['protocols']['supported'][ 'rsync']['hostname'] else: request.cls.rse_id = 'SSH1' request.cls.hostname = 'ssh1' request.cls.prefix = '/rucio/' request.cls.port = 22 request.cls.sshuser = '******'
def tearDownClass(cls): """S3 (RSE/PROTOCOLS): Removing created directories and files """ # Remove test files from storage # rse_tag = 'AMAZON-BOTO' rse_tag = 'BNL-BOTO' rse_settings = mgr.get_rse_info(rse_tag) try: data = load_test_conf_file('rse-accounts.cfg') rse_settings['credentials'] = data[rse_tag] except KeyError: print('No credentials found for this RSE.') pass for protocol in rse_settings['protocols']: if protocol['scheme'] == 's3': break conn = boto.connect_s3( host=protocol['hostname'], port=int(protocol.get('port', 80)), aws_access_key_id=rse_settings['credentials']['access_key'], aws_secret_access_key=rse_settings['credentials']['secret_key'], is_secure=rse_settings['credentials'].get('is_secure', False), calling_format=boto.s3.connection.OrdinaryCallingFormat()) for protocol in rse_settings['protocols']: if protocol['scheme'] == 's3': break bucket_name = protocol['prefix'] bucket = conn.get_bucket(bucket_name) keys = bucket.list() for key in keys: key.delete()
def setUpClass(cls): """SRM (RSE/PROTOCOLS): Creating necessary directories and files """ # Creating local files cls.tmpdir = tempfile.mkdtemp() cls.user = uuid() with open("%s/data.raw" % cls.tmpdir, "wb") as out: out.seek((1024 * 1024) - 1) # 1 MB out.write(b'\0') for f in MgrTestCases.files_local: shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f)) data = load_test_conf_file('rse_repository.json') prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][ 'prefix'] hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][ 'srm']['hostname'] if hostname.count("://"): hostname = hostname.split("://")[1] if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][ 'srm'].keys(): port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'] ['srm']['port']) else: port = 0 if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols'][ 'supported']['srm'].keys() and 'web_service_path' in data[ 'FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][ 'extended_attributes'].keys(): web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols'][ 'supported']['srm']['extended_attributes']['web_service_path'] else: web_service_path = '' os.system('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' % cls.tmpdir) if port > 0: cls.static_file = 'srm://%s:%s%s%s/data.raw' % ( hostname, port, web_service_path, prefix) else: cls.static_file = 'srm://%s%s%s/data.raw' % ( hostname, web_service_path, prefix) cmd = 'srmcp --debug=false -retry_num=0 file:///%s/data.raw %s' % ( cls.tmpdir, cls.static_file) execute(cmd) for f in MgrTestCases.files_remote: tmp = mgr.lfns2pfns(mgr.get_rse_info('FZK-LCG2_SCRATCHDISK'), { 'name': f, 'scope': 'user.%s' % cls.user }, scheme='srm').values()[0] cmd = 'srmcp --debug=false -retry_num=0 file:///%s/data.raw %s' % ( cls.tmpdir, tmp) execute(cmd)
def tearDownClass(cls): """GFAL2 (RSE/PROTOCOLS): Removing created directorie s and files""" data = load_test_conf_file('rse_repository.json') prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][ 'prefix'] hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][ 'srm']['hostname'] if hostname.count("://"): hostname = hostname.split("://")[1] if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'][ 'srm'].keys(): port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported'] ['srm']['port']) else: port = 0 if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols'][ 'supported']['srm'].keys() and 'web_service_path' in data[ 'FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'][ 'extended_attributes'].keys(): web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols'][ 'supported']['srm']['extended_attributes']['web_service_path'] else: web_service_path = '' shutil.rmtree(cls.tmpdir) clean_raw = '%s/data.raw' % prefix if int(port) > 0: srm_path = ''.join( ["srm://", hostname, ":", port, web_service_path]) else: srm_path = ''.join(["srm://", hostname, web_service_path]) list_files_cmd_user = '******' % ( srm_path, prefix, cls.user) clean_files = str(execute(list_files_cmd_user)[1]).split('\n') list_files_cmd_user = '******' % ( srm_path, prefix, cls.user) clean_files += str(execute(list_files_cmd_user)[1]).split('\n') clean_files.append("1024 " + clean_raw) for files in clean_files: if len(files.strip()) > 0: file = files.split()[1] if not file.endswith("/"): clean_cmd = 'srmrm -2 --debug=false -retry_num=0 %s/%s' % ( srm_path, file) execute(clean_cmd) clean_directory = ['user', 'group'] for directory in clean_directory: clean_cmd = 'srmrmdir -2 --debug=false -retry_num=0 -recursive %s%s/%s/%s' % ( srm_path, prefix, directory, cls.user) execute(clean_cmd)
def __init__(self, tmpdir, rse_tag, user, static_file, vo='def'): self.rse_settings = mgr.get_rse_info(rse=rse_tag, vo=vo) try: data = load_test_conf_file('rse-accounts.cfg') self.rse_settings['credentials'] = data[rse_tag] except KeyError: print('No credentials found for this RSE.') pass self.tmpdir = tmpdir self.gettmpdir = tempfile.mkdtemp() self.user = user self.static_file = static_file
def setUpClass(cls): """WebDAV (RSE/PROTOCOLS): Creating necessary directories and files """ session = requests.Session() session.cert = os.getenv('X509_USER_PROXY') session.verify = False cls.site = 'FZK-LCG2_SCRATCHDISK' # Creating local files cls.tmpdir = tempfile.mkdtemp() cls.user = '******' data = load_test_conf_file('rse_repository.json') scheme = data[cls.site]['protocols']['supported']['https']['scheme'] prefix = data[cls.site]['protocols']['supported']['https']['prefix'] hostname = data[ cls.site]['protocols']['supported']['https']['hostname'] port = data[cls.site]['protocols']['supported']['https']['port'] with open("%s/data.raw" % cls.tmpdir, "wb") as out: out.seek((1024) - 1) # 1 kB out.write(b'\0') for f in MgrTestCases.files_local: os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f)) cls.static_file = '%s://%s:%s%sdata.raw' % (scheme, hostname, port, prefix) rse_settings = rsemanager.get_rse_info(cls.site) storage = rsemanager.create_protocol(rse_settings, operation='write', scheme='https') storage.connect() for f in MgrTestCases.files_remote: os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f)) destfile = rsemanager.lfns2pfns(rse_settings, [ { 'name': f, 'scope': 'user.%s' % (cls.user) }, ], operation='write', scheme='https').values()[0] try: storage.put('%s/%s' % (cls.tmpdir, f), destfile) except FileReplicaAlreadyExists as e: print(e) with open('%s/data.raw' % cls.tmpdir, 'rb') as f_file: session.put(cls.static_file, data=f_file.read(), verify=False, allow_redirects=True)
def setUpClass(cls): """SFTP (RSE/PROTOCOLS): Creating necessary directories and files """ # Creating local files cls.tmpdir = tempfile.mkdtemp() cls.user = uuid() with open("%s/data.raw" % cls.tmpdir, "wb") as out: out.seek((1024 * 1024) - 1) # 1 MB out.write(b'\0') for fil in MgrTestCases.files_local: os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, fil)) # Load local credentials from file data = load_test_conf_file('rse-accounts.cfg') credentials = data['LXPLUS'] lxplus = pysftp.Connection(**credentials) prefix = load_test_conf_file('rse_repository.json')['LXPLUS'][ 'protocols']['supported']['sftp']['prefix'] lxplus.execute('mkdir %s' % prefix) lxplus.execute('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' % prefix) cls.static_file = 'sftp://lxplus.cern.ch:22%sdata.raw' % prefix protocol = mgr.create_protocol(mgr.get_rse_info('LXPLUS'), 'write') for fil in MgrTestCases.files_remote: tmp = protocol.parse_pfns( protocol.lfns2pfns({ 'name': fil, 'scope': 'user.%s' % cls.user }).values()[0]).values()[0] for cmd in [ 'mkdir -p %s' % ''.join([tmp['prefix'], tmp['path']]), 'ln -s %sdata.raw %s' % (prefix, ''.join([tmp['prefix'], tmp['path'], tmp['name']])) ]: lxplus.execute(cmd) lxplus.close()
def setUpClass(cls): """S3 (RSE/PROTOCOLS): Creating necessary directories and files """ # Creating local files cls.tmpdir = tempfile.mkdtemp() cls.user = uuid() # cls.user = '******' # use again when latency issue with S3 storage is resolved with open("%s/data.raw" % cls.tmpdir, "wb") as out: out.seek((1024 * 1024) - 1) # 1 MB out.write(b'\0') for f in MgrTestCases.files_local: os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f)) rse_tag = 'BNL-BOTO' rse_settings = mgr.get_rse_info(rse_tag) try: data = load_test_conf_file('rse-accounts.cfg') rse_settings['credentials'] = data[rse_tag] except KeyError: print('No credentials found for this RSE.') pass for protocol in rse_settings['protocols']: if protocol['scheme'] == 's3': break conn = boto.connect_s3( host=protocol['hostname'], port=int(protocol.get('port', 80)), aws_access_key_id=rse_settings['credentials']['access_key'], aws_secret_access_key=rse_settings['credentials']['secret_key'], is_secure=rse_settings['credentials'].get('is_secure', False), calling_format=boto.s3.connection.OrdinaryCallingFormat()) cls.static_file = mgr.lfns2pfns(rse_settings, { 'name': 'data.raw', 'scope': 'user.%s' % cls.user }).values()[0] key = get_bucket_key(cls.static_file, conn, create=True) key.set_contents_from_filename("%s/data.raw" % cls.tmpdir) for f in MgrTestCases.files_remote: pfn = mgr.lfns2pfns(rse_settings, { 'name': f, 'scope': 'user.%s' % cls.user }).values()[0] bucket_name, key_name = get_bucket_key_name(pfn) key.copy(bucket_name, key_name)
def load_rse_info(request, containerized_rses): """ Detects if containerized rses for ssh are available in the testing environment. :return: A tuple (rse, prefix, hostname). """ rses = [rse for rse in containerized_rses if rse[0] == 'SSH1'] data = load_test_conf_file('rse_repository.json') request.cls.prefix = data['SSH_DISK']['protocols']['supported']['rclone'][ 'prefix'] if len(rses) == 0: request.cls.rse_id = 'SSH-RSE' request.cls.hostname = data['SSH-RSE']['protocols']['supported'][ 'rclone']['hostname'] else: request.cls.rse_id = 'SSH1' request.cls.hostname = 'ssh_rclone_rse' request.cls.prefix = '/rucio/'
def tearDownClass(cls): """WebDAV (RSE/PROTOCOLS): Removing created directories and files """ rse_settings = rsemanager.get_rse_info(cls.site) data = load_test_conf_file('rse_repository.json') scheme = data[cls.site]['protocols']['supported']['https']['scheme'] prefix = data[cls.site]['protocols']['supported']['https']['prefix'] hostname = data[ cls.site]['protocols']['supported']['https']['hostname'] port = data[cls.site]['protocols']['supported']['https']['port'] storage = rsemanager.create_protocol(rse_settings, operation='write', scheme='https') print(rse_settings) storage.connect() status1 = storage.delete('%s://%s:%s%suser/%s' % (scheme, hostname, port, prefix, cls.user)) print(status1) status2 = storage.delete('%s://%s:%s%sgroup/%s' % (scheme, hostname, port, prefix, cls.user)) print(status2)
def tearDownClass(cls): """POSIX (RSE/PROTOCOLS): Removing created directorie s and files """ data = load_test_conf_file('rse_repository.json') prefix = data['MOCK-POSIX']['protocols']['supported']['file']['prefix'] shutil.rmtree(prefix) shutil.rmtree(cls.tmpdir)