def ls(self, filename): """ Internal method to list files/directories :param filename Name of the directory that needs to be created :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied """ path = self.path2pfn(filename) headers = {'Depth': '1'} self.exists(filename) try: result = self.session.request('PROPFIND', path, verify=False, headers=headers, timeout=self.timeout, cert=self.cert) if result.status_code in [ 404, ]: raise exception.SourceNotFound() elif result.status_code in [ 401, ]: raise exception.RSEAccessDenied() parser = Parser() parser.feed(result.text) list_files = [self.server + p_file for p_file in parser.list] try: list_files.remove(filename + '/') except ValueError: pass try: list_files.remove(filename) except ValueError: pass parser.close() return list_files except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except requests.exceptions.ReadTimeout as error: raise exception.ServiceUnavailable(error)
def rename(self, pfn, new_pfn): """ Allows to rename a file stored inside the connected RSE. :param pfn Current physical file name :param new_pfn New physical file name :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied """ path = self.path2pfn(pfn) new_path = self.path2pfn(new_pfn) directories = new_path.split('/') headers = {'Destination': new_path} # Try the rename without testing the existence of the destination directory try: result = self.session.request('MOVE', path, verify=False, headers=headers, timeout=self.timeout, cert=self.cert) if result.status_code == 201: return elif result.status_code in [404, ]: raise exception.SourceNotFound() else: # Create the directories before issuing the MOVE for directory_level in reversed(list(range(1, 4))): upper_directory = "/".join(directories[:-directory_level]) self.mkdir(upper_directory) try: result = self.session.request('MOVE', path, verify=False, headers=headers, timeout=self.timeout, cert=self.cert) if result.status_code == 201: return elif result.status_code in [404, ]: raise exception.SourceNotFound() elif result.status_code in [401, ]: raise exception.RSEAccessDenied() else: # catchall exception raise exception.RucioException(result.status_code, result.text) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except requests.exceptions.ReadTimeout as error: raise exception.ServiceUnavailable(error)
def mkdir(self, directory): """ Internal method to create directories :param directory Name of the directory that needs to be created :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied """ path = self.path2pfn(directory) try: result = self.session.request('MKCOL', path, verify=False, timeout=self.timeout, cert=self.cert) if result.status_code in [201, 405]: # Success or directory already exists return elif result.status_code in [404, ]: raise exception.SourceNotFound() elif result.status_code in [401, ]: raise exception.RSEAccessDenied() else: # catchall exception raise exception.RucioException(result.status_code, result.text) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except requests.exceptions.ReadTimeout as error: raise exception.ServiceUnavailable(error)
def exists(self, pfn): """ Checks if the requested file is known by the referred RSE. :param pfn Physical file name :returns: True if the file exists, False if it doesn't :raise ServiceUnavailable, RSEAccessDenied """ path = self.path2pfn(pfn) try: result = self.session.request('HEAD', path, verify=False, timeout=self.timeout, cert=self.cert) if result.status_code == 200: return True elif result.status_code in [401, ]: raise exception.RSEAccessDenied() elif result.status_code in [404, ]: return False else: # catchall exception raise exception.RucioException(result.status_code, result.text) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error)
def get(self, pfn, dest='.', transfer_timeout=None): """ Provides access to files stored inside connected the RSE. :param pfn: Physical file name of requested file :param dest: Name and path of the files when stored at the client :param transfer_timeout: Transfer timeout (in seconds) - dummy :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied """ path = self.path2pfn(pfn) chunksize = 1024 try: result = self.session.get(path, verify=False, stream=True, timeout=self.timeout, cert=self.cert) if result and result.status_code in [200, ]: length = None if 'content-length' in result.headers: length = int(result.headers['content-length']) with open(dest, 'wb') as file_out: nchunk = 0 if not length: print('Malformed HTTP response (missing content-length header).') for chunk in result.iter_content(chunksize): file_out.write(chunk) if length: nchunk += 1 elif result.status_code in [404, ]: raise exception.SourceNotFound() elif result.status_code in [401, 403]: raise exception.RSEAccessDenied() else: # catchall exception raise exception.RucioException(result.status_code, result.text) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except requests.exceptions.ReadTimeout as error: raise exception.ServiceUnavailable(error)
def put(self, source, target, source_dir=None, transfer_timeout=None, progressbar=False): """ Allows to store files inside the referred RSE. :param source Physical file name :param target Name of the file on the storage system e.g. with prefixed scope :param source_dir Path where the to be transferred files are stored in the local file system :param transfer_timeout Transfer timeout (in seconds) - dummy :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied """ path = self.path2pfn(target) full_name = source_dir + '/' + source if source_dir else source directories = path.split('/') # Try the upload without testing the existence of the destination directory try: if not os.path.exists(full_name): raise exception.SourceNotFound() it = UploadInChunks(full_name, 10000000, progressbar) result = self.session.put(path, data=IterableToFileAdapter(it), verify=False, allow_redirects=True, timeout=self.timeout, cert=self.cert) if result.status_code in [200, 201]: return if result.status_code in [ 409, ]: raise exception.FileReplicaAlreadyExists() else: # Create the directories before issuing the PUT for directory_level in reversed(list(range(1, 4))): upper_directory = "/".join(directories[:-directory_level]) self.mkdir(upper_directory) try: if not os.path.exists(full_name): raise exception.SourceNotFound() it = UploadInChunks(full_name, 10000000, progressbar) result = self.session.put(path, data=IterableToFileAdapter(it), verify=False, allow_redirects=True, timeout=self.timeout, cert=self.cert) if result.status_code in [200, 201]: return if result.status_code in [ 409, ]: raise exception.FileReplicaAlreadyExists() elif result.status_code in [ 401, ]: raise exception.RSEAccessDenied() else: # catchall exception raise exception.RucioException(result.status_code, result.text) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except IOError as error: raise exception.SourceNotFound(error) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except requests.exceptions.ReadTimeout as error: raise exception.ServiceUnavailable(error) except IOError as error: raise exception.SourceNotFound(error)
def connect(self, credentials={}): """ Establishes the actual connection to the referred RSE. :param credentials Provides information to establish a connection to the referred storage system. For WebDAV connections these are ca_cert, cert, auth_type, timeout :raises RSEAccessDenied """ try: self.server = self.path2pfn('') except KeyError: raise exception.RSEAccessDenied('No specified Server') try: self.ca_cert = credentials['ca_cert'] except KeyError: self.ca_cert = None try: self.auth_type = credentials['auth_type'] except KeyError: self.auth_type = 'cert' try: self.cert = credentials['cert'] except KeyError: x509 = os.getenv('X509_USER_PROXY') if not x509: # Trying to get the proxy from the default location proxy_path = '/tmp/x509up_u%s' % os.geteuid() if os.path.isfile(proxy_path): x509 = proxy_path elif self.auth_token: pass else: raise exception.RSEAccessDenied( 'X509_USER_PROXY is not set') self.cert = (x509, x509) try: self.timeout = credentials['timeout'] except KeyError: self.timeout = 300 self.session = requests.Session() self.session.mount('https://', TLSv1HttpAdapter()) if self.auth_token: self.session.headers.update( {'Authorization': 'Bearer ' + self.auth_token}) # "ping" to see if the server is available try: res = self.session.request('HEAD', self.path2pfn(''), verify=False, timeout=self.timeout, cert=self.cert) if res.status_code != 200: raise exception.ServiceUnavailable( 'Problem to connect %s : %s' % (self.path2pfn(''), res.text)) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable('Problem to connect %s : %s' % (self.path2pfn(''), error)) except requests.exceptions.ReadTimeout as error: raise exception.ServiceUnavailable(error)
def put(self, source, target, source_dir=None): """ Allows to store files inside the referred RSE. :param source: path to the source file on the client file system :param target: path to the destination file on the storage :param source_dir: Path where the to be transferred files are stored in the local file system :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ full_name = source_dir + '/' + source if source_dir else source path = self._get_signed_url(target, operation='write') full_name = source_dir + '/' + source if source_dir else source try: if not os.path.exists(full_name): raise exception.SourceNotFound() it = UploadInChunks(full_name, 10000000, progressbar=False) result = self.session.put(path, data=IterableToFileAdapter(it), verify=False, allow_redirects=True, timeout=self.timeout, cert=self.cert) if result.status_code in [200, 201]: return if result.status_code in [ 409, ]: raise exception.FileReplicaAlreadyExists() else: try: if not os.path.exists(full_name): raise exception.SourceNotFound() it = UploadInChunks(full_name, 10000000, progressbar=False) result = self.session.put(path, data=IterableToFileAdapter(it), verify=False, allow_redirects=True, timeout=self.timeout, cert=self.cert) if result.status_code in [200, 201]: return if result.status_code in [ 409, ]: raise exception.FileReplicaAlreadyExists() elif result.status_code in [ 401, ]: raise exception.RSEAccessDenied() else: # catchall exception raise exception.RucioException(result.status_code, result.text) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except IOError as error: raise exception.SourceNotFound(error) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except IOError as error: raise exception.SourceNotFound(error)