def open(self, path, mode='r', **kwargs): if 'w' in mode or '+' in mode or 'a' in mode: logging.error('cannot use httpfs.open() in write mode: %s' % path) raise UnsupportedError('open', path=path) url = self._build_url(path) for attempt in (1, 2, 3): try: response = requests.get(url) except requests.RequestException as error: logging.warning('open attempt %d: %s %s' % (attempt, url, error)) else: break else: raise RemoteConnectionError('getinfo', path) if response.status_code == 200: return StringIO(response.content) elif response.status_code == 404: raise ResourceNotFoundError(path) else: logging.warning( 'open status %d for %s assumed as connection error.' % (response.status_code, url)) raise RemoteConnectionError('open', path)
def getpathurl(self, path, allow_none=False): """Returns a url that corresponds to the given path, if one exists. If the path does not have an equivalent URL form (and allow_none is False) then a :class:`~fs.errors.NoPathURLError` exception is thrown. Otherwise the URL will be returns as an unicode string. @param path: object id for which to return url path @param allow_none: if true, this method can return None if there is no URL form of the given path @type allow_none: bool @raises `fs.errors.NoPathURLError`: If no URL form exists, and allow_none is False (the default) @rtype: unicode """ url = None try: url = self.getinfo(path)['source'] except RemoteConnectionError as e: raise RemoteConnectionError(e) except: if not allow_none: raise NoPathURLError(path=path) return url
def xrd_ping(self): """Ping xrootd server. Specific to ``XRootDPyFS``. """ status, dummy = self._client.ping() if not status.ok: raise RemoteConnectionError(opname="ping", details=status) return True
def _getinfo(self, path): url = self._build_url(path) for attempt in (1, 2, 3): try: response = requests.head(url) except requests.RequestException as error: logging.warning('getinfo attempt %d: %s %s' % (attempt, url, error)) else: break else: raise RemoteConnectionError('getinfo', path) if response.status_code == 200: if 'content-length' in response.headers: size_in_bytes = int(response.headers['content-length']) if 'last-modified' in response.headers: last_modified = parse_http_date( response.headers['last-modified']) else: last_modified = None file_time = last_modified or datetime.datetime.now() return { 'size': size_in_bytes, 'created_time': file_time, 'accessed_time': file_time, 'modified_time': file_time, } logging.warning('getinfo missing required headers: %s' % response.headers) raise RemoteConnectionError('getinfo', path) elif response.status_code in (301, 302, 403, 404): # Pretend that redirects and forbiddens are 404s. raise ResourceNotFoundError(path) else: logging.warning( 'getinfo status %d for %s assumed as connection error.' % (response.status_code, url)) raise RemoteConnectionError('getinfo', path)
def file_delete(self, path): try: super(OneDriveClient, self).delete(path) except api_v5.ProtocolError as e: if e.code == 404: raise ResourceNotFoundError(path) raise OperationFailedError(opname='file_delete', msg=str(e)) except: raise RemoteConnectionError( "Most probable reasons: access token has expired " "or user credentials are invalid.") self.cache.pop(path, None)
def exists(self, path): """ Checks if a the specified path exists @param path: Id of the file/folder to check """ try: return self.client.metadata(path) except RemoteConnectionError as e: raise RemoteConnectionError(e) except: return False
def file_copy(self, src, dst): try: metadata = super(OneDriveClient, self).copy(src, dst, False) except api_v5.ProtocolError as e: if e.code == 404: raise ResourceNotFoundError( "Parent or source file don't exist") raise OperationFailedError(opname='file_copy', msg=str(e)) except: raise RemoteConnectionError( "Most probable reasons: access token has expired " "or user credentials are invalid.") self.cache.set(metadata['id'], metadata, parent=dst) return metadata['id']
def _runMethod(*args, **kwargs): def prepare_error(exp): return ''.join([ 'run command "', name, '", args: ', str(args), ', kwargs: ', str(kwargs), ', smbc exception: ', str(exp) ]) try: method = getattr(self._ctx, name) return method(*args, **kwargs) except smbc.TimedOutError, e: raise RemoteConnectionError(prepare_error(e), details=e)
def file_delete(self, path): """Delete a file of a give path.""" try: super(DropboxClient, self).file_delete(path) except rest.ErrorResponse as e: if e.status == 404: raise ResourceNotFoundError(path) if e.status == 400 and 'must not be empty' in str(e): raise DirectoryNotEmptyError(path) raise OperationFailedError(opname='file_copy', msg=str(e)) except: raise RemoteConnectionError( "Most probable reasons: access token has expired or user" " credentials are invalid.") self.cache.pop(path, None)
def _retry_operation(self, method, *args): """Method retries an operation. Sometimes access_token expires and we need to rebuild it using the refresh token. This method does that and retries the operation that failed. """ if self._retry < 5: self._retry += 1 self.service = self._build_service() return method(*args) else: raise RemoteConnectionError( "Most probable reasons: access token has expired " "or user credentials are invalid.")
def media(self, path): """Media.""" try: info = super(DropboxClient, self).media(path) return info.get('url', None) except rest.ErrorResponse as e: if e.status == 400: raise UnsupportedError("create a link to a folder") if e.status == 404: raise ResourceNotFoundError(path) raise OperationFailedError(opname='file_copy', msg=str(e)) except: raise RemoteConnectionError( "Most probable reasons: access token has expired or user" " credentials are invalid.")
def put_file(self, path, f, overwrite=False): """Upload a file.""" try: response = super(DropboxClient, self).put_file(path, f, overwrite=overwrite) except rest.ErrorResponse as e: raise OperationFailedError(opname='file_copy', msg=str(e)) except TypeError as e: raise ResourceInvalidError("put_file", path) except: raise RemoteConnectionError( "Most probable reasons: access token has expired or user" " credentials are invalid.") self.cache.pop(dirname(path), None) return response
def file_create_folder(self, path): """Add newly created directory to cache.""" try: metadata = super(DropboxClient, self).file_create_folder(path) except rest.ErrorResponse as e: if e.status == 403: raise DestinationExistsError(path) if e.status == 400: raise OperationFailedError(opname='file_create_folder', msg=str(e)) except: raise RemoteConnectionError( "Most probable reasons: access token has expired or user" " credentials are invalid.") self.cache.set(path, metadata) return metadata['path']
def put_file(self, parent_id, title, content, overwrite=False): try: metadata = super(OneDriveClient, self).put((title, content), parent_id, overwrite=overwrite) except api_v5.ProtocolError as e: if e.code == 404: raise ResourceNotFoundError(parent_id) raise OperationFailedError(opname='put_copy', msg=str(e)) except TypeError as e: raise ResourceInvalidError("put_file") except: raise RemoteConnectionError( "Most probable reasons: access token has expired " "or user credentials are invalid.") return metadata['id']
def update_file(self, file_id, new_file_info): try: metadata = super(OneDriveClient, self).info_update(file_id, new_file_info) except api_v5.ProtocolError as e: if e.resp.status == 404: raise ResourceNotFoundError(path=file_id) raise OperationFailedError(opname='update_file', msg=e.resp.reason) except: raise RemoteConnectionError( "Most probable reasons: access token has expired " "or user credentials are invalid.") self.cache.pop(file_id, None) self.cache.set(metadata['id'], metadata, parent=metadata['parent_id']) return metadata['id']
def file_create_folder(self, parent_id, title): "Add newly created directory to cache." try: metadata = super(OneDriveClient, self).mkdir(title, parent_id) except api_v5.ProtocolError as e: if e.code == 405: raise ResourceInvalidError(parent_id) if e.code == 404: raise ResourceNotFoundError(parent_id) raise OperationFailedError(opname='file_create_folder', msg=str(e)) except: raise RemoteConnectionError( "Most probable reasons: access token has expired " "or user credentials are invalid.") self.cache.set(metadata["id"], metadata, parent=parent_id) return metadata['id']
def children(self, path): """Get children of a given path.""" update = False hash_ = None item = self.cache.get(path) if item: if item.expired: update = True if item.metadata and item.children: hash_ = item.metadata['hash'] else: if not item.metadata.get('is_dir'): raise ResourceInvalidError(path) if not item.children: update = True else: update = True if update: try: metadata = super(DropboxClient, self).metadata(path, hash=hash_, include_deleted=False, list=True) children = [] contents = metadata.pop('contents') for child in contents: if child.get('is_deleted', False): continue children.append(basename(child['path'])) self.cache[child['path']] = CacheItem(child) item = self.cache[path] = CacheItem(metadata, children) except rest.ErrorResponse as e: if not item or e.status != 304: raise OperationFailedError(opname='metadata', path=path, msg=str(e)) # We have an item from cache (perhaps expired), but it's # hash is still valid (as far as Dropbox is concerned), # so just renew it and keep using it. item.renew() except: raise RemoteConnectionError( "Most probable reasons: access token has expired or user" " credentials are invalid.") return item.children
def file_copy(self, src, dst): """Copy a file to another location.""" try: metadata = super(DropboxClient, self).file_copy(src, dst) except rest.ErrorResponse as e: if e.status == 404: raise ResourceNotFoundError(src) if e.status == 403: raise DestinationExistsError(dst) if e.status == 503: raise OperationFailedError(opname='file_copy', msg="User over storage quota") raise OperationFailedError(opname='file_copy', msg=str(e)) except: raise RemoteConnectionError( "Most probable reasons: access token has expired or user" " credentials are invalid.") self.cache.set(dst, metadata) return metadata['path']
def metadata(self, path): "Gets metadata for a given path." item = self.cache.get(path) if not item or item.metadata is None or item.expired: try: metadata = super(OneDriveClient, self).info(path) except api_v5.ProtocolError as e: if e.code == 404: raise ResourceNotFoundError(path) raise OperationFailedError(opname='metadata', path=path, msg=str(e)) except: raise RemoteConnectionError( "Most probable reasons: access token has expired " "or user credentials are invalid.") item = self.cache[path] = CacheItem(metadata) # Copy the info so the caller cannot affect our cache. return dict(item.metadata.items())
def children(self, path): "Gets children of a given path." update = False item = self.cache.get(path) if item: if item.expired: update = True else: if item.metadata["type"] != "folder" and \ not ("folder" in path): raise ResourceInvalidError(path) if not item.children: update = True else: update = True if update: try: metadata = super(OneDriveClient, self).info(path) if metadata["type"] != "folder" and not ("folder" in path): raise ResourceInvalidError(path) children = [] contents = super(OneDriveClient, self).listdir(path) for child in contents: children.append(child['id']) self.cache[child['id']] = CacheItem(child, parent=path) item = self.cache[path] = CacheItem(metadata, children) except api_v5.ProtocolError as e: if e.code == 404: raise ResourceNotFoundError(path) if not item or e.resp.status != 304: raise OperationFailedError(opname='metadata', path=path, msg=str(e)) # We have an item from cache (perhaps expired), but it's # hash is still valid (as far as OneDrive is concerned), # so just renew it and keep using it. item.renew() except: raise RemoteConnectionError( "Most probable reasons: access token has expired " "or user credentials are invalid.") return item.children
def get_file(self, path): metadata = None if not self.cache.get(path, None): try: metadata = super(OneDriveClient, self).info(path) except api_v5.ProtocolError as e: if e.code == 404: raise ResourceNotFoundError("Source file doesn't exist") raise OperationFailedError(opname='get_file', msg=str(e)) except: raise RemoteConnectionError( "Most probable reasons: access token has expired " "or user credentials are invalid.") self.cache.set(metadata['id'], metadata, parent=metadata['parent_id']) else: item = self.cache[path] metadata = item.metadata return super(OneDriveClient, self).get(path)
def metadata(self, path): """Get metadata for a given path.""" item = self.cache.get(path) if not item or item.metadata is None or item.expired: try: metadata = super(DropboxClient, self).metadata(path, include_deleted=False, list=False) except rest.ErrorResponse as e: if e.status == 404: raise ResourceNotFoundError(path) raise OperationFailedError(opname='metadata', path=path, msg=str(e)) except: raise RemoteConnectionError( "Most probable reasons: access token has expired or user" " credentials are invalid.") if metadata.get('is_deleted', False): raise ResourceNotFoundError(path) item = self.cache[path] = CacheItem(metadata) # Copy the info so the caller cannot affect our cache. return dict(item.metadata.items())
def inner(*args, **kwargs): try: return outer(*args, **kwargs) except socket.gaierror as e: if e.errno == socket.EAI_NONAME: raise RemoteConnectionError(str(e), details=e) raise except socket.error as e: if e.errno in (errno.ECONNREFUSED, errno.EPIPE, errno.ETIMEDOUT): raise RemoteConnectionError(str(e), details=e) raise except OperationFailure as e: # Cycle through each message and map the first one to PyFilesystem # that is not a successful status (0x00000000). share = args[0] path = args[1] for msg in e.smb_messages: # Protocol versions expose the error values differently. if msg.protocol == 1: msg_status = msg.status.internal_value else: msg_status = msg.status if msg_status == 0x0: continue elif msg_status == 0x103: # Unknown error, but message says it is not found. raise ResourceNotFoundError(path=path) elif msg_status == 0xc000000f: raise ResourceNotFoundError(path=path) elif msg_status == 0xc0000022: raise PermissionDeniedError('access', path=path) elif msg_status == 0xc0000033: raise ResourceInvalidError(path=path) elif msg_status == 0xc0000034: raise ResourceNotFoundError(path=path) elif msg_status == 0xc0000035: raise DestinationExistsError(path=path) elif msg_status == 0xc000003a: raise ResourceNotFoundError(path=path) elif msg_status == 0xc0000056: raise DeletePendingError(path=path) elif msg_status == 0xc000006f: raise AccountTimeRestrictedError(path=path) elif msg_status == 0xc0000071: raise PasswordExpiredError(path=path) elif msg_status == 0xc00000ba: raise ResourceInvalidError(path=path) elif msg_status == 0xc00000cc: # Share does not exist. raise ResourceInvalidError(path=share) elif msg_status == 0xc00000d0: raise ResourceInvalidError(path=path) elif msg_status == 0xc0000101: raise DirectoryNotEmptyError(path=path) elif msg_status == 0xc0000103: raise ResourceInvalidError(path=path) elif msg_status == 0xc0000193: raise AccountExpiredError(path=path) elif msg_status == 0xc0000224: raise PasswordChangeRequiredError(path=path) elif msg_status == 0xc0000234: raise AccountLockedError(path=path) else: raise Exception('Unhandled SMB error: {0}'.format( hex(msg_status))) raise except NotConnectedError as e: # Connection does not exist or was disconnected. Using the wrong # NetBIOS name can cause this against a Windows server while Samba # will ignore it. raise RemoteConnectionError(details=e) except NotReadyError as e: # Connection has not been successfully authenticated. raise RemoteConnectionError(details=e)