def get_filesystem(path): # type: (str) -> FileSystems """Get the correct filesystem for the specified path """ try: path_scheme = FileSystems.get_scheme(path) systems = [ fs for fs in FileSystem.get_all_subclasses() if fs.scheme() == path_scheme ] if len(systems) == 0: raise ValueError( 'Unable to get filesystem from specified path, please use the ' 'correct path or ensure the required dependency is installed, ' 'e.g., pip install apache_beam[gcp]. Path specified: %s' % path) elif len(systems) == 1: # Pipeline options could come either from the Pipeline itself (using # direct runner), or via RuntimeValueProvider (other runners). options = (FileSystems._pipeline_options or RuntimeValueProvider.runtime_options) return systems[0](pipeline_options=options) else: raise ValueError('Found more than one filesystem for path %s' % path) except ValueError: raise except Exception as e: raise BeamIOError('Unable to get the Filesystem', {path: e})
def get_filesystem(path): """Get the correct filesystem for the specified path """ try: path_scheme = FileSystems.get_scheme(path) systems = [ fs for fs in FileSystem.get_all_subclasses() if fs.scheme() == path_scheme ] if len(systems) == 0: raise ValueError('Unable to get the Filesystem for path %s' % path) elif len(systems) == 1: # Pipeline options could come either from the Pipeline itself (using # direct runner), or via RuntimeValueProvider (other runners). options = (FileSystems._pipeline_options or RuntimeValueProvider.runtime_options) return systems[0](pipeline_options=options) else: raise ValueError('Found more than one filesystem for path %s' % path) except ValueError: raise except Exception as e: raise BeamIOError('Unable to get the Filesystem', {path: e})
def get_filesystem(path): """Get the correct filesystem for the specified path """ try: path_scheme = FileSystems.get_scheme(path) systems = [fs for fs in FileSystem.get_all_subclasses() if fs.scheme() == path_scheme] if len(systems) == 0: raise ValueError('Unable to get the Filesystem for path %s' % path) elif len(systems) == 1: return systems[0]() else: raise ValueError('Found more than one filesystem for path %s' % path) except ValueError: raise except Exception as e: raise BeamIOError('Unable to get the Filesystem', {path: e})
def get_filesystem(path): """Get the correct filesystem for the specified path """ try: path_scheme = FileSystems.get_scheme(path) systems = [fs for fs in FileSystem.get_all_subclasses() if fs.scheme() == path_scheme] if len(systems) == 0: raise ValueError('Unable to get the Filesystem for path %s' % path) elif len(systems) == 1: # Pipeline options could come either from the Pipeline itself (using # direct runner), or via RuntimeValueProvider (other runners). options = (FileSystems._pipeline_options or RuntimeValueProvider.runtime_options) return systems[0](pipeline_options=options) else: raise ValueError('Found more than one filesystem for path %s' % path) except ValueError: raise except Exception as e: raise BeamIOError('Unable to get the Filesystem', {path: e})
def get_filesystem(path): """Get the correct filesystem for the specified path """ try: path_scheme = FileSystems.get_scheme(path) systems = [ fs for fs in FileSystem.get_all_subclasses() if fs.scheme() == path_scheme ] if len(systems) == 0: raise ValueError('Unable to get the Filesystem for path %s' % path) elif len(systems) == 1: return systems[0]() else: raise ValueError('Found more than one filesystem for path %s' % path) except ValueError: raise except Exception as e: raise BeamIOError('Unable to get the Filesystem', {path: e})