def fetch(self, q, count=100, batch_size: int = 200, timestamp=1):
try:
total_matches = self._get_total_matches(q)
logger.debug("{} images found ".format(total_matches))
result = []
for offset in range(0, total_matches, count):
images = self._make_api_call(q, offset, count)
result += images
time.sleep(timestamp)
for batch in more_itertools.chunked(result, batch_size):
delayed_tasks = []
for img_uri in batch:
try:
if FileUtil.exists_http_file(img_uri):
delayed_tasks.append(
dask.delayed(ImageUtil.url2img)(img_uri))
except Exception as ex:
if type(ex) in EXCEPTIONS:
logger.debug("skipping: {}".format(img_uri))
else:
logger.debug("skipping {}: {}".format(img_uri, ex))
continue
compute_result = dask.compute(*delayed_tasks)
yield [
img for img in compute_result
if isinstance(img, np.ndarray)
]
except Exception as ex:
logger.error("error fetching the images: {}".format(ex))
def _fetch_single_image(self, img_info, image_id, image_label):
try:
img_uri = img_info["coco_url"]
if FileUtil.exists_http_file(img_uri):
img_arr = ImageUtil.url2img(img_uri)
tagged_image=TaggedImage(img_arr)
tagged_image.id=image_id
if self.task == "detection":
tagged_image.regions = self._create_box_rois(img_info, image_label)
elif self.task == "segmentation":
tagged_image.regions=self._create_polygon_rois(img_info,image_label)
return tagged_image
except Exception as ex:
print(ex)
logger.error("error downloading the image with id {} : {}".format(image_id,ex))
return None