def load(self, obj):
'''Retrieve the Object and update it with the information we
receive.'''
# 0. Load src URI if not exist (mainly used for re-execution)
self._load_src_maybe_meta(obj)
# 1. Load object metadata from meta URI
if hasattr(obj, 'meta'):
uri = str(obj.meta)
scheme, path = split_scheme(uri)
if scheme == 'file':
self._load_attrbutes_localfile(obj, path)
else: # assume http
self._load_attributes(obj, uri)
# 2. Load object content from src URI
uri = str(obj.src)
scheme, path = split_scheme(uri)
if scheme == 'sha256':
self._load_blobcache(obj, path)
elif scheme == 'file':
self._load_localfile(obj, path)
else: # assume http
self._load_dataretriever(obj, uri)
# Set display name if not already in initial attributes
if ATTR_DISPLAY_NAME not in obj:
obj[ATTR_DISPLAY_NAME] = str(obj) + '\0'
def source_available(cls, state, uri):
'''Verify the URI to ensure that its data is accessible. Return
True if we can access the data, False if we can't and should inform
the client to that effect. Raise FilterUnsupportedSource if we don't
support the URI scheme.'''
scheme, path = split_scheme(uri)
if scheme == 'sha256':
return path in state.blob_cache
else:
raise FilterUnsupportedSource()
def source_available(cls, state, uri):
'''Verify the URI to ensure that its data is accessible. Return
True if we can access the data, False if we can't and should inform
the client to that effect. Raise FilterUnsupportedSource if we don't
support the URI scheme.'''
scheme, path = split_scheme(uri)
if scheme == 'sha256':
return path in state.blob_cache
else:
raise FilterUnsupportedSource()
def source_available(self, obj):
'''Examine the Object and return whether we think we will be able
to load it, without actually doing so. This can be used during
reexecution to determine whether we should return
DiamondRPCFCacheMiss to the client.'''
scheme, path = split_scheme(str(obj))
if scheme == 'sha256':
return path in self._blob_cache
# Assume we can always load other types of URLs
return True
def source_available(self, obj):
'''Examine the Object and return whether we think we will be able
to load it, without actually doing so. This can be used during
reexecution to determine whether we should return
DiamondRPCFCacheMiss to the client.'''
scheme, path = split_scheme(str(obj))
if scheme == 'sha256':
return path in self._blob_cache
else:
# Assume we can always load other types of URLs
return True
def _resolve_blob(self, state):
'''Returns (blob data, signature).'''
scheme, path = split_scheme(self.blob_source)
if scheme == 'sha256':
sig = path.lower()
try:
return (state.blob_cache[sig], sig)
except KeyError:
raise FilterDependencyError('Missing blob for filter ' +
self.name)
else:
raise FilterUnsupportedSource()
def _resolve_code(self, state):
'''Returns (code_path, signature).'''
scheme, path = split_scheme(self.code_source)
if scheme == 'sha256':
sig = path.lower()
try:
return (state.blob_cache.executable_path(sig), sig)
except KeyError:
raise FilterDependencyError('Missing code for filter ' +
self.name)
else:
raise FilterUnsupportedSource()
def load(self, obj):
'''Retrieve the Object and update it with the information we
receive.'''
uri = str(obj)
scheme, path = split_scheme(uri)
if scheme == 'sha256':
self._load_blobcache(obj, path)
else:
self._load_dataretriever(obj, uri)
# Set display name if not already in initial attributes
if ATTR_DISPLAY_NAME not in obj:
obj[ATTR_DISPLAY_NAME] = uri + '\0'
def _resolve_blob(self, state):
'''Returns (blob data, signature).'''
scheme, path = split_scheme(self.blob_source)
if scheme == 'sha256':
sig = path.lower()
try:
return (state.blob_cache[sig], sig)
except KeyError:
raise FilterDependencyError('Missing blob for filter ' +
self.name)
else:
raise FilterUnsupportedSource()
def _resolve_code(self, state):
'''Returns (code_path, signature).'''
scheme, path = split_scheme(self.code_source)
if scheme == 'sha256':
sig = path.lower()
try:
return (state.blob_cache.executable_path(sig), sig)
except KeyError:
raise FilterDependencyError('Missing code for filter ' +
self.name)
else:
raise FilterUnsupportedSource()
def _resolve_code(self, state):
'''Returns (code_path, signature).'''
scheme, path = split_scheme(self.code_source)
if scheme == 'sha256':
sig = path.lower()
try:
# FIXME with filter mode introduced, not necessary to set the
# executable bit so early.
return (state.blob_cache.executable_path(sig), sig)
except KeyError:
raise FilterDependencyError('Missing code for filter ' +
self.name)
else:
raise FilterUnsupportedSource()
def _load_src_maybe_meta(self, obj):
"""
Load the src and optional meta URIs from the object's id.
:param obj:
:return:
"""
if not hasattr(obj, 'src'):
uri = str(obj)
scheme, path = split_scheme(uri)
if scheme == 'sha256':
# Blob has no meta URI
obj.src = uri
else:
# Assume http
_, body = self._http.get(uri)
object_el = xmltodict.parse(body)
obj.src = urljoin(uri, object_el['object']['@src'])
if '@meta' in object_el['object']:
obj.meta = urljoin(uri, object_el['object']['@meta'])
return