def do_file_query2(qq):
fname = qq.file
stream = s3.dir_dir(qq.base, qq.depth)
if skip_check:
yield from (SimpleNamespace(url=d + fname) for d in stream)
return
stream = (s3.head_object(d + fname) for d in stream)
for (f, _), _ in future_results(stream, 32):
if f is not None:
yield f
def __call__(self, urls, **kw):
"""Fetch a bunch of s3 urls concurrently.
urls -- sequence of <url | (url, range)> , where range is (in:int,out:int)|None
On output is a sequence of result objects, note that order is not
preserved, but one should get one result for every input.
Successful results object will contain:
.url = url
.data = bytes
.last_modified -- last modified timestamp
.range = None | (in,out)
.error = None
Failed result looks like this:
.url = url
.data = None
.last_modified = None
.range = None | (in, out)
.error = str| botocore.Exception class
"""
from odc.ppt import future_results
def generate_requests(urls, s3, **kw):
for url in urls:
if isinstance(url, tuple):
url, range = url
else:
range = None
yield self._async.submit(s3_fetch_object,
url,
s3=s3,
range=range,
**kw)
for rr, ee in future_results(generate_requests(urls, self._s3, **kw),
self._nconcurrent * 2):
if ee is not None:
assert (
not "s3_fetch_object should not raise exceptions, but did")
else:
yield rr