def connect(data_item, cache=None):
""" Connect to a :class:`DataItem <eoxserver.backends.models.DataItem>`.
If the data item is not connectable but retrievable, this function uses
:func:`retrieve` as a fallback.
:param data_item: the :class:`DataItem <eoxserver.backends.models.DataItem>`
to connect to
:param cache: an instance of :class:`CacheContext
<eoxserver.backends.cache.CacheContext>` or ``None``
if the caching shall be handled internally
:returns: the connection string to retrieve data from or a local path
if the ``DataItem`` was ``retrieved``
"""
backend = BackendComponent(env)
storage = data_item.storage
if storage:
component = backend.get_connected_storage_component(
storage.storage_type
)
if not storage or not component:
return retrieve(data_item, cache)
return component.connect(storage.url, data_item.location)
def connect(data_item, cache=None):
""" Connect to a :class:`DataItem <eoxserver.backends.models.DataItem>`.
If the data item is not connectable but retrievable, this function uses
:func:`retrieve` as a fallback.
:param data_item: the :class:`DataItem <eoxserver.backends.models.DataItem>`
to connect to
:param cache: an instance of :class:`CacheContext
<eoxserver.backends.cache.CacheContext>` or ``None``
if the caching shall be handled internally
:returns: the connection string to retrieve data from or a local path
if the ``DataItem`` was ``retrieved``
"""
backend = BackendComponent(env)
storage = data_item.storage
if storage:
component = backend.get_connected_storage_component(
storage.storage_type)
if not storage or not component:
return retrieve(data_item, cache)
return component.connect(storage.url, data_item.location)
def _get_location_chain(self, items):
""" Returns the tuple
"""
component = BackendComponent(env)
storage = None
package = None
storage_type, url = self._split_location(items[0])
if storage_type:
storage_component = component.get_storage_component(storage_type)
else:
storage_component = None
if storage_component:
storage, _ = backends.Storage.objects.get_or_create(
url=url, storage_type=storage_type
)
# packages
for item in items[1 if storage else 0:-1]:
package_component = component.get_package_component(type_or_format)
format, location = self._split_location(item)
if package_component:
package, _ = backends.Package.objects.get_or_create(
location=location, format=format,
storage=storage, package=package
)
storage = None # override here
else:
raise "Could not find package component"
format, location = self._split_location(items[-1])
return storage, package, format, location
def retrieve(data_item, cache=None):
"""
"""
backend = BackendComponent(env)
if cache is None:
cache = get_cache_context()
# compute a cache path where the file *would* be cached
with cache:
item_id = generate_hash(data_item.location, data_item.format)
path = cache.relative_path(item_id)
logger.debug("Retrieving %s (ID: %s)" % (data_item, item_id))
if item_id in cache:
logger.debug("Item %s is already in the cache." % item_id)
return path
if data_item.package is None and data_item.storage:
return _retrieve_from_storage(
backend, data_item, data_item.storage, item_id, path, cache
)
elif data_item.package:
return _extract_from_package(
backend, data_item, data_item.package, item_id, path, cache
)
else:
return data_item.location
def connect(data_item, cache=None):
""" return a connection string, either for a local (cached) data or something
residing on a server of some kind
"""
backend = BackendComponent(env)
storage = data_item.storage
if storage:
component = backend.get_connected_storage_component(storage.storage_type)
if not storage or not component:
return retrieve(data_item, cache)
return component.connect(storage.url, data_item.location)
def connect(data_item, cache=None):
""" return a connection string, either for a local (cached) data or something
residing on a server of some kind
"""
backend = BackendComponent(env)
storage = data_item.storage
if storage:
component = backend.get_connected_storage_component(storage.storage_type)
if not storage or not component:
return retrieve(data_item, cache)
return component.connect(storage.url, data_item.location)
def _get_location_chain(self, items):
""" Returns the tuple
"""
component = BackendComponent(env)
storage = None
package = None
storage_type, url = self._split_location(items[0])
if storage_type:
storage_component = component.get_storage_component(storage_type)
else:
storage_component = None
if storage_component:
storage, _ = backends.Storage.objects.get_or_create(
url=url, storage_type=storage_type
)
# packages
for item in items[1 if storage else 0:-1]:
type_or_format, location = self._split_location(item)
package_component = component.get_package_component(type_or_format)
if package_component:
package, _ = backends.Package.objects.get_or_create(
location=location, format=format,
storage=storage, package=package
)
storage = None # override here
else:
raise Exception(
"Could not find package component for format '%s'"
% type_or_format
)
format, location = self._split_location(items[-1])
return storage, package, format, location
def retrieve(data_item, cache=None):
""" Retrieve a :class:`DataItem <eoxserver.backends.models.DataItem>`, i.e:
make it locally available. This takes into account any download from a
:class:`Storage <eoxserver.backends.models.Storage>` and any unpacking from
a :class:`Package <eoxserver.backends.models.Package>` the ``DataItem``
might be contained in.
:param data_item: the :class:`DataItem <eoxserver.backends.models.DataItem>`
to connect retrieve
:param cache: an instance of :class:`CacheContext
<eoxserver.backends.cache.CacheContext>` or ``None``
if the caching shall be handled internally
"""
backend = BackendComponent(env)
if cache is None:
cache = get_cache_context()
# compute a cache path where the file *would* be cached
with cache:
item_id = generate_hash(data_item.location, data_item.format)
path = cache.relative_path(item_id)
logger.debug("Retrieving %s (ID: %s)" % (data_item, item_id))
if item_id in cache:
logger.debug("Item %s is already in the cache." % item_id)
return path
if data_item.package is None and data_item.storage:
return _retrieve_from_storage(backend, data_item,
data_item.storage, item_id, path,
cache)
elif data_item.package:
return _extract_from_package(backend, data_item, data_item.package,
item_id, path, cache)
else:
return data_item.location
def _expand_data_item(data_item, cache=None):
""" Helper function to expand a source data item to a list of file
identifiers.
"""
backends = BackendComponent(env)
storage = data_item.storage
package = data_item.package
if storage:
# get list of files of that storage
component = backends.get_storage_component(storage.storage_type)
if not component:
raise ValueError("No storage component for type '%s' found." %
storage.storage_type)
return component.list_files(storage.url, data_item.location)
elif package:
# get list of files of that package
local_filename = retrieve(package, cache)
component = backends.get_package_component(package.format)
if not component:
raise ValueError("No package component for type '%s' found." %
package.format)
return component.list_files(local_filename, data_item.location)
else:
# This is a local filename, expand it directly
component = backends.get_storage_component("local")
if not component:
raise ValueError("No active local storage component found.")
return component.list_files("", data_item.location)
def get_storage_type_choices():
backend_component = BackendComponent(env)
return map(lambda p: (p.name, p.name), backend_component.storages)
def get_package_format_choices():
backend_component = BackendComponent(env)
return map(lambda p: (p.name, p.name), backend_component.packages)
def get_format_choices():
backend_component = BackendComponent(env)
return map(lambda r: (r.name, r.get_supported_formats()),
backend_component.data_readers)