def connect(data_item, cache=None): """ Connect to a :class:`DataItem <eoxserver.backends.models.DataItem>`. If the data item is not connectable but retrievable, this function uses :func:`retrieve` as a fallback. :param data_item: the :class:`DataItem <eoxserver.backends.models.DataItem>` to connect to :param cache: an instance of :class:`CacheContext <eoxserver.backends.cache.CacheContext>` or ``None`` if the caching shall be handled internally :returns: the connection string to retrieve data from or a local path if the ``DataItem`` was ``retrieved`` """ backend = BackendComponent(env) storage = data_item.storage if storage: component = backend.get_connected_storage_component( storage.storage_type ) if not storage or not component: return retrieve(data_item, cache) return component.connect(storage.url, data_item.location)
def connect(data_item, cache=None): """ Connect to a :class:`DataItem <eoxserver.backends.models.DataItem>`. If the data item is not connectable but retrievable, this function uses :func:`retrieve` as a fallback. :param data_item: the :class:`DataItem <eoxserver.backends.models.DataItem>` to connect to :param cache: an instance of :class:`CacheContext <eoxserver.backends.cache.CacheContext>` or ``None`` if the caching shall be handled internally :returns: the connection string to retrieve data from or a local path if the ``DataItem`` was ``retrieved`` """ backend = BackendComponent(env) storage = data_item.storage if storage: component = backend.get_connected_storage_component( storage.storage_type) if not storage or not component: return retrieve(data_item, cache) return component.connect(storage.url, data_item.location)
def _get_location_chain(self, items): """ Returns the tuple """ component = BackendComponent(env) storage = None package = None storage_type, url = self._split_location(items[0]) if storage_type: storage_component = component.get_storage_component(storage_type) else: storage_component = None if storage_component: storage, _ = backends.Storage.objects.get_or_create( url=url, storage_type=storage_type ) # packages for item in items[1 if storage else 0:-1]: package_component = component.get_package_component(type_or_format) format, location = self._split_location(item) if package_component: package, _ = backends.Package.objects.get_or_create( location=location, format=format, storage=storage, package=package ) storage = None # override here else: raise "Could not find package component" format, location = self._split_location(items[-1]) return storage, package, format, location
def retrieve(data_item, cache=None): """ """ backend = BackendComponent(env) if cache is None: cache = get_cache_context() # compute a cache path where the file *would* be cached with cache: item_id = generate_hash(data_item.location, data_item.format) path = cache.relative_path(item_id) logger.debug("Retrieving %s (ID: %s)" % (data_item, item_id)) if item_id in cache: logger.debug("Item %s is already in the cache." % item_id) return path if data_item.package is None and data_item.storage: return _retrieve_from_storage( backend, data_item, data_item.storage, item_id, path, cache ) elif data_item.package: return _extract_from_package( backend, data_item, data_item.package, item_id, path, cache ) else: return data_item.location
def connect(data_item, cache=None): """ return a connection string, either for a local (cached) data or something residing on a server of some kind """ backend = BackendComponent(env) storage = data_item.storage if storage: component = backend.get_connected_storage_component(storage.storage_type) if not storage or not component: return retrieve(data_item, cache) return component.connect(storage.url, data_item.location)
def _get_location_chain(self, items): """ Returns the tuple """ component = BackendComponent(env) storage = None package = None storage_type, url = self._split_location(items[0]) if storage_type: storage_component = component.get_storage_component(storage_type) else: storage_component = None if storage_component: storage, _ = backends.Storage.objects.get_or_create( url=url, storage_type=storage_type ) # packages for item in items[1 if storage else 0:-1]: type_or_format, location = self._split_location(item) package_component = component.get_package_component(type_or_format) if package_component: package, _ = backends.Package.objects.get_or_create( location=location, format=format, storage=storage, package=package ) storage = None # override here else: raise Exception( "Could not find package component for format '%s'" % type_or_format ) format, location = self._split_location(items[-1]) return storage, package, format, location
def retrieve(data_item, cache=None): """ Retrieve a :class:`DataItem <eoxserver.backends.models.DataItem>`, i.e: make it locally available. This takes into account any download from a :class:`Storage <eoxserver.backends.models.Storage>` and any unpacking from a :class:`Package <eoxserver.backends.models.Package>` the ``DataItem`` might be contained in. :param data_item: the :class:`DataItem <eoxserver.backends.models.DataItem>` to connect retrieve :param cache: an instance of :class:`CacheContext <eoxserver.backends.cache.CacheContext>` or ``None`` if the caching shall be handled internally """ backend = BackendComponent(env) if cache is None: cache = get_cache_context() # compute a cache path where the file *would* be cached with cache: item_id = generate_hash(data_item.location, data_item.format) path = cache.relative_path(item_id) logger.debug("Retrieving %s (ID: %s)" % (data_item, item_id)) if item_id in cache: logger.debug("Item %s is already in the cache." % item_id) return path if data_item.package is None and data_item.storage: return _retrieve_from_storage(backend, data_item, data_item.storage, item_id, path, cache) elif data_item.package: return _extract_from_package(backend, data_item, data_item.package, item_id, path, cache) else: return data_item.location
def _expand_data_item(data_item, cache=None): """ Helper function to expand a source data item to a list of file identifiers. """ backends = BackendComponent(env) storage = data_item.storage package = data_item.package if storage: # get list of files of that storage component = backends.get_storage_component(storage.storage_type) if not component: raise ValueError("No storage component for type '%s' found." % storage.storage_type) return component.list_files(storage.url, data_item.location) elif package: # get list of files of that package local_filename = retrieve(package, cache) component = backends.get_package_component(package.format) if not component: raise ValueError("No package component for type '%s' found." % package.format) return component.list_files(local_filename, data_item.location) else: # This is a local filename, expand it directly component = backends.get_storage_component("local") if not component: raise ValueError("No active local storage component found.") return component.list_files("", data_item.location)
def get_storage_type_choices(): backend_component = BackendComponent(env) return map(lambda p: (p.name, p.name), backend_component.storages)
def get_package_format_choices(): backend_component = BackendComponent(env) return map(lambda p: (p.name, p.name), backend_component.packages)
def get_format_choices(): backend_component = BackendComponent(env) return map(lambda r: (r.name, r.get_supported_formats()), backend_component.data_readers)