def _add_all(self):
resource = Resource(self.resource_hash,
task_id=self.task_id,
path=self.resource_path,
files=['file'])
self.cache.add_resource(resource)
self.cache.set_prefix(self.task_id, self.prefix)
def pull_resource(self,
entry,
task_id,
success,
error,
client=None,
client_options=None,
async_=True):
resource_path = self.storage.get_path('', task_id)
resource = Resource(resource_hash=entry[0],
task_id=task_id,
files=entry[1],
path=resource_path)
if resource.files and self.storage.exists(resource):
success(entry, resource.files, task_id)
return
def success_wrapper(response, **_):
logger.debug("Downloaded resource. path=%s, hash=%s",
resource.path, resource.hash)
self._cache_resource(resource)
files = self._parse_pull_response(response, task_id)
success(entry, files, task_id)
def error_wrapper(exception, **_):
logger.warning(
"Error downloading resource."
"path=%s, hash=%s, error=%s", resource.path, resource.hash,
exception)
error(exception, entry, task_id)
logger.debug("Preparing to download resource. path=%s, hash=%s",
resource.path, resource.hash)
path = self.storage.get_path(resource.path, task_id)
local = self.storage.cache.get_by_hash(resource.hash)
os.makedirs(path, exist_ok=True)
logger.debug("Pulling resource. local=%r, hash=%s", local,
resource.hash)
if local:
try:
self.storage.copy(local.path, resource.path, task_id)
success_wrapper(entry)
except Exception as exc:
error_wrapper(exc)
else:
self._pull(resource,
task_id,
success=success_wrapper,
error=error_wrapper,
client=client,
client_options=client_options,
async_=async_)
def from_wire(serialized):
iterator = filter(lambda x: is_collection(x) and len(x) > 1,
serialized)
results = [
Resource.deserialize(entry) for entry in iterator
if is_collection(entry) and len(entry) > 1
]
if len(results) != len(serialized):
logger.warning("Errors occurred while deserializing %r",
serialized)
return results
def _cache_files(self, resource_hash: str, files: Iterable, task_id: str):
"""
Put the files in storage cache.
:param resource_hash: Hash that files are identified by
:param files: Collection of files
:param task_id: Task id that files are associated with
"""
resource_path = self.storage.get_path('', task_id)
resource = Resource(resource_hash,
task_id=task_id,
files=list(files),
path=resource_path)
self._cache_resource(resource)
def test_resources(self):
resource = Resource(self.resource_hash,
task_id=self.task_id,
files=self.resource_files,
path=self.resource_path)
new_task_file = 'new_name'
new_resource = Resource(str(uuid.uuid4()),
task_id=str(uuid.uuid4()),
files=[new_task_file],
path=self.resource_path)
def create_file(name):
directory = os.path.join(self.tempdir, self.resource_path, name)
os.makedirs(directory, exist_ok=True)
Path(os.path.join(directory, name)).touch()
self.cache.add_resource(resource)
assert self.cache.get_resources(self.task_id) == [resource]
assert self.cache.get_resources(new_resource.task_id) == []
assert self.cache.get_resources('unknown') == []
assert self.cache.has_resource(resource)
assert not self.cache.has_resource(new_resource)
assert not new_resource.exists
create_file(new_task_file)
self.cache.add_resource(new_resource)
assert new_resource.exists
assert self.cache.get_resources(self.task_id) == [resource]
assert self.cache.get_resources(new_resource.task_id) == [new_resource]
assert self.cache.get_resources('unknown') == []
assert self.cache.has_resource(resource)
assert self.cache.has_resource(new_resource)
assert self.cache.remove(self.task_id)
assert self.cache.remove('unknown') == []
assert self.cache.get_resources(new_resource.task_id) == [new_resource]
def test_prefix(self):
self.cache.set_prefix(self.task_id, self.prefix)
resource = Resource(self.resource_hash,
task_id=self.task_id,
files=self.resource_files,
path=self.resource_path)
assert self.cache.get_prefix(resource.task_id) == self.prefix
assert self.cache.get_prefix(str(uuid.uuid4())) == ''
assert self.cache.get_prefix(str(uuid.uuid4()), 'default_value') == \
'default_value'
self.cache.add_resource(resource)
self.cache.remove(resource.task_id)
assert self.cache.get_prefix(resource.task_id) == ''
def test_remove(self):
self._add_all()
self.cache.remove(self.task_id)
assert self._all_default_empty()
new_path = '/other/path'
new_resource = Resource(str(uuid.uuid4()),
task_id=str(uuid.uuid4()),
path=new_path,
files=[new_path])
self._add_all()
self.cache.add_resource(new_resource)
self.cache.remove(self.task_id)
assert self._all_default_empty()
assert self.cache.has_resource(new_resource)
assert self.cache.get_by_path(new_resource.path) == new_resource
assert self.cache.get_by_hash(new_resource.hash) == new_resource
def test_to_from_wire(self):
entries = []
for resource in self.joined_resources:
manager = Resource(str(uuid.uuid4()),
task_id="task",
path=os.path.dirname(resource),
files=[os.path.basename(resource)])
entries.append(manager)
resources_split = self.resource_manager.to_wire(entries)
resources_joined = self.resource_manager.from_wire(resources_split)
assert len(entries) == len(self.target_resources)
assert all([r[0] in self.split_resources for r in resources_split])
assert all([r[0] in self.joined_resources for r in resources_joined])
entries = [['resource', '1'], [None, '2'], None,
[['split', 'path'], '4']]
assert self.resource_manager.from_wire(entries) == [[
'resource', '1'
], [os.path.join('split', 'path'), '4']]