def ls(self, filename):
""" Internal method to list files/directories
:param filename Name of the directory that needs to be created
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied
"""
path = self.path2pfn(filename)
headers = {'Depth': '1'}
self.exists(filename)
try:
result = self.session.request('PROPFIND',
path,
verify=False,
headers=headers,
timeout=self.timeout,
cert=self.cert)
if result.status_code in [
404,
]:
raise exception.SourceNotFound()
elif result.status_code in [
401,
]:
raise exception.RSEAccessDenied()
parser = Parser()
parser.feed(result.text)
list_files = [self.server + p_file for p_file in parser.list]
try:
list_files.remove(filename + '/')
except ValueError:
pass
try:
list_files.remove(filename)
except ValueError:
pass
parser.close()
return list_files
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ReadTimeout as error:
raise exception.ServiceUnavailable(error)
def rename(self, pfn, new_pfn):
""" Allows to rename a file stored inside the connected RSE.
:param pfn Current physical file name
:param new_pfn New physical file name
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied
"""
path = self.path2pfn(pfn)
new_path = self.path2pfn(new_pfn)
directories = new_path.split('/')
headers = {'Destination': new_path}
# Try the rename without testing the existence of the destination directory
try:
result = self.session.request('MOVE', path, verify=False, headers=headers, timeout=self.timeout, cert=self.cert)
if result.status_code == 201:
return
elif result.status_code in [404, ]:
raise exception.SourceNotFound()
else:
# Create the directories before issuing the MOVE
for directory_level in reversed(list(range(1, 4))):
upper_directory = "/".join(directories[:-directory_level])
self.mkdir(upper_directory)
try:
result = self.session.request('MOVE', path, verify=False, headers=headers, timeout=self.timeout, cert=self.cert)
if result.status_code == 201:
return
elif result.status_code in [404, ]:
raise exception.SourceNotFound()
elif result.status_code in [401, ]:
raise exception.RSEAccessDenied()
else:
# catchall exception
raise exception.RucioException(result.status_code, result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ReadTimeout as error:
raise exception.ServiceUnavailable(error)
def mkdir(self, directory):
""" Internal method to create directories
:param directory Name of the directory that needs to be created
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied
"""
path = self.path2pfn(directory)
try:
result = self.session.request('MKCOL', path, verify=False, timeout=self.timeout, cert=self.cert)
if result.status_code in [201, 405]: # Success or directory already exists
return
elif result.status_code in [404, ]:
raise exception.SourceNotFound()
elif result.status_code in [401, ]:
raise exception.RSEAccessDenied()
else:
# catchall exception
raise exception.RucioException(result.status_code, result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ReadTimeout as error:
raise exception.ServiceUnavailable(error)
def exists(self, pfn):
""" Checks if the requested file is known by the referred RSE.
:param pfn Physical file name
:returns: True if the file exists, False if it doesn't
:raise ServiceUnavailable, RSEAccessDenied
"""
path = self.path2pfn(pfn)
try:
result = self.session.request('HEAD', path, verify=False, timeout=self.timeout, cert=self.cert)
if result.status_code == 200:
return True
elif result.status_code in [401, ]:
raise exception.RSEAccessDenied()
elif result.status_code in [404, ]:
return False
else:
# catchall exception
raise exception.RucioException(result.status_code, result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
def get(self, pfn, dest='.', transfer_timeout=None):
""" Provides access to files stored inside connected the RSE.
:param pfn: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied
"""
path = self.path2pfn(pfn)
chunksize = 1024
try:
result = self.session.get(path, verify=False, stream=True, timeout=self.timeout, cert=self.cert)
if result and result.status_code in [200, ]:
length = None
if 'content-length' in result.headers:
length = int(result.headers['content-length'])
with open(dest, 'wb') as file_out:
nchunk = 0
if not length:
print('Malformed HTTP response (missing content-length header).')
for chunk in result.iter_content(chunksize):
file_out.write(chunk)
if length:
nchunk += 1
elif result.status_code in [404, ]:
raise exception.SourceNotFound()
elif result.status_code in [401, 403]:
raise exception.RSEAccessDenied()
else:
# catchall exception
raise exception.RucioException(result.status_code, result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ReadTimeout as error:
raise exception.ServiceUnavailable(error)
def put(self,
source,
target,
source_dir=None,
transfer_timeout=None,
progressbar=False):
""" Allows to store files inside the referred RSE.
:param source Physical file name
:param target Name of the file on the storage system e.g. with prefixed scope
:param source_dir Path where the to be transferred files are stored in the local file system
:param transfer_timeout Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied
"""
path = self.path2pfn(target)
full_name = source_dir + '/' + source if source_dir else source
directories = path.split('/')
# Try the upload without testing the existence of the destination directory
try:
if not os.path.exists(full_name):
raise exception.SourceNotFound()
it = UploadInChunks(full_name, 10000000, progressbar)
result = self.session.put(path,
data=IterableToFileAdapter(it),
verify=False,
allow_redirects=True,
timeout=self.timeout,
cert=self.cert)
if result.status_code in [200, 201]:
return
if result.status_code in [
409,
]:
raise exception.FileReplicaAlreadyExists()
else:
# Create the directories before issuing the PUT
for directory_level in reversed(list(range(1, 4))):
upper_directory = "/".join(directories[:-directory_level])
self.mkdir(upper_directory)
try:
if not os.path.exists(full_name):
raise exception.SourceNotFound()
it = UploadInChunks(full_name, 10000000, progressbar)
result = self.session.put(path,
data=IterableToFileAdapter(it),
verify=False,
allow_redirects=True,
timeout=self.timeout,
cert=self.cert)
if result.status_code in [200, 201]:
return
if result.status_code in [
409,
]:
raise exception.FileReplicaAlreadyExists()
elif result.status_code in [
401,
]:
raise exception.RSEAccessDenied()
else:
# catchall exception
raise exception.RucioException(result.status_code,
result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except IOError as error:
raise exception.SourceNotFound(error)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ReadTimeout as error:
raise exception.ServiceUnavailable(error)
except IOError as error:
raise exception.SourceNotFound(error)
def connect(self, credentials={}):
""" Establishes the actual connection to the referred RSE.
:param credentials Provides information to establish a connection
to the referred storage system. For WebDAV connections these are
ca_cert, cert, auth_type, timeout
:raises RSEAccessDenied
"""
try:
self.server = self.path2pfn('')
except KeyError:
raise exception.RSEAccessDenied('No specified Server')
try:
self.ca_cert = credentials['ca_cert']
except KeyError:
self.ca_cert = None
try:
self.auth_type = credentials['auth_type']
except KeyError:
self.auth_type = 'cert'
try:
self.cert = credentials['cert']
except KeyError:
x509 = os.getenv('X509_USER_PROXY')
if not x509:
# Trying to get the proxy from the default location
proxy_path = '/tmp/x509up_u%s' % os.geteuid()
if os.path.isfile(proxy_path):
x509 = proxy_path
elif self.auth_token:
pass
else:
raise exception.RSEAccessDenied(
'X509_USER_PROXY is not set')
self.cert = (x509, x509)
try:
self.timeout = credentials['timeout']
except KeyError:
self.timeout = 300
self.session = requests.Session()
self.session.mount('https://', TLSv1HttpAdapter())
if self.auth_token:
self.session.headers.update(
{'Authorization': 'Bearer ' + self.auth_token})
# "ping" to see if the server is available
try:
res = self.session.request('HEAD',
self.path2pfn(''),
verify=False,
timeout=self.timeout,
cert=self.cert)
if res.status_code != 200:
raise exception.ServiceUnavailable(
'Problem to connect %s : %s' %
(self.path2pfn(''), res.text))
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable('Problem to connect %s : %s' %
(self.path2pfn(''), error))
except requests.exceptions.ReadTimeout as error:
raise exception.ServiceUnavailable(error)
def put(self, source, target, source_dir=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
full_name = source_dir + '/' + source if source_dir else source
path = self._get_signed_url(target, operation='write')
full_name = source_dir + '/' + source if source_dir else source
try:
if not os.path.exists(full_name):
raise exception.SourceNotFound()
it = UploadInChunks(full_name, 10000000, progressbar=False)
result = self.session.put(path,
data=IterableToFileAdapter(it),
verify=False,
allow_redirects=True,
timeout=self.timeout,
cert=self.cert)
if result.status_code in [200, 201]:
return
if result.status_code in [
409,
]:
raise exception.FileReplicaAlreadyExists()
else:
try:
if not os.path.exists(full_name):
raise exception.SourceNotFound()
it = UploadInChunks(full_name, 10000000, progressbar=False)
result = self.session.put(path,
data=IterableToFileAdapter(it),
verify=False,
allow_redirects=True,
timeout=self.timeout,
cert=self.cert)
if result.status_code in [200, 201]:
return
if result.status_code in [
409,
]:
raise exception.FileReplicaAlreadyExists()
elif result.status_code in [
401,
]:
raise exception.RSEAccessDenied()
else:
# catchall exception
raise exception.RucioException(result.status_code,
result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except IOError as error:
raise exception.SourceNotFound(error)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except IOError as error:
raise exception.SourceNotFound(error)
