def run(self):
while not self.halt:
log.debug("waiting for pool {}....".format(self._id))
with self.pool:
url = self.request.get()
if url is not None:
try:
self.state(url)
r = url_get(url)
if self.content_handler is not None:
r = self.content_handler(r)
self.response.put(
{'response': r, 'url': url, 'exception': None, 'last_fetched': datetime.now()}
)
log.info("successfully fetched {}".format(url))
except Exception as ex:
self.response.put(
{'response': None, 'url': url, 'exception': ex, 'last_fetched': datetime.now()}
)
log.warning("error fetching {}".format(url))
log.warning(ex)
import traceback
log.debug(traceback.format_exc())
finally:
self.state('idle')
self.request.task_done()
def test_convert(self):
for fn in self.files:
(basename, _, ext) = fn.rpartition('.')
mime_type = TestImage.ext_to_mime.get(ext, None)
assert (mime_type is not None)
url = "file://{}".format(fn)
assert url
r = url_get(url)
assert r
assert r.content
img = img_to_data(r.content, mime_type)
assert img
print (img)