def _upload_single_image(self, image, label, filename: str, mode):
"""Uploads a single image to the Lightly platform.
"""
# check whether the filename is too long
basename = filename
if not check_filename(basename):
msg = (
f'Filename {basename} is longer than the allowed maximum of '
'characters and will be skipped.')
warnings.warn(msg)
return False
# calculate metadata, and check if corrupted
metadata = check_image(image)
# generate thumbnail if necessary
thumbname = None
if not metadata['is_corrupted'] and mode in ["thumbnails", "full"]:
thumbname = '.'.join(basename.split('.')[:-1]) + '_thumb.webp'
body = SampleCreateRequest(file_name=basename,
thumb_name=thumbname,
meta_data=metadata)
sample_id = retry(self.samples_api.create_sample_by_dataset_id,
body=body,
dataset_id=self.dataset_id).id
if not metadata['is_corrupted'] and mode in ["thumbnails", "full"]:
thumbnail = get_thumbnail_from_img(image)
image_to_upload = PIL_to_bytes(thumbnail, ext='webp', quality=90)
signed_url = retry(
self.samples_api.get_sample_image_write_url_by_id,
dataset_id=self.dataset_id,
sample_id=sample_id,
is_thumbnail=True)
# try to upload thumbnail
retry(self.upload_file_with_signed_url, image_to_upload,
signed_url)
thumbnail.close()
if not metadata['is_corrupted'] and mode == "full":
image_to_upload = PIL_to_bytes(image)
signed_url = retry(
self.samples_api.get_sample_image_write_url_by_id,
dataset_id=self.dataset_id,
sample_id=sample_id,
is_thumbnail=False)
# try to upload thumbnail
retry(self.upload_file_with_signed_url, image_to_upload,
signed_url)
image.close()
def test_PIL_to_bytes(self):
image = Image.new('RGB', (128, 128))
# test with quality=None
PIL_to_bytes(image)
# test with quality=90
PIL_to_bytes(image, quality=90)
# test with quality=90 and ext=jpg
PIL_to_bytes(image, ext='JPEG', quality=90)
def _upload_single_image(image, label, filename, dataset_id, token, mode):
"""Uploads a single image to the Lightly platform.
"""
# check whether the filename is too long
basename = filename
if not check_filename(basename):
msg = (f'Filename {basename} is longer than the allowed maximum of '
'characters and will be skipped.')
warnings.warn(msg)
return False
# calculate metadata, and check if corrupted
metadata = check_image(image)
# generate thumbnail if necessary
thumbname = None
thumbnail = None
if mode == 'thumbnails' and not metadata['is_corrupted']:
thumbname = '.'.join(basename.split('.')[:-1]) + '_thumb.webp'
thumbnail = get_thumbnail_from_img(image)
# upload sample with metadata
sample_upload_success = True
try:
sample_id = routes.users.datasets.samples.post(basename, thumbname,
metadata, dataset_id,
token)
except RuntimeError:
sample_upload_success = False
# upload thumbnail
thumbnail_upload_success = True
if mode == 'thumbnails' and not metadata['is_corrupted']:
try:
# try to get signed url for thumbnail
signed_url = routes.users.datasets.samples. \
get_presigned_upload_url(
thumbname, dataset_id, sample_id, token)
# try to upload thumbnail
upload_file_with_signed_url(
PIL_to_bytes(thumbnail, ext='webp', quality=70), signed_url)
# close thumbnail
thumbnail.close()
except RuntimeError:
thumbnail_upload_success = False
# upload full image
image_upload_success = True
if mode == 'full' and not metadata['is_corrupted']:
try:
# try to get signed url for image
signed_url = routes.users.datasets.samples. \
get_presigned_upload_url(
basename, dataset_id, sample_id, token)
# try to upload image
upload_file_with_signed_url(PIL_to_bytes(image), signed_url)
# close image
image.close()
except RuntimeError:
image_upload_success = False
success = sample_upload_success
success = success and thumbnail_upload_success
success = success and image_upload_success
return success
def _upload_single_image(fname):
# random delay of uniform[0, 0.01] seconds to prevent API bursts
rnd_delay = random.random() * 0.01
time.sleep(rnd_delay)
# get PIL image handles, metadata, and check if corrupted
metadata, is_corrupted = check_image(
os.path.join(path_to_folder, fname))
# filename is too long, cannot accept this file
if not metadata:
return False
# upload sample
basename = fname
thumbname = None
if mode in ['full', 'thumbnails'] and not is_corrupted:
thumbname = '.'.join(basename.split('.')[:-1]) + '_thumb.webp'
sample_upload_success = True
try:
sample_id = upload_sample_with_metadata(basename, thumbname,
metadata, dataset_id,
token)
except RuntimeError:
sample_upload_success = False
# upload thumbnail
thumbnail_upload_success = True
if mode == 'thumbnails' and not is_corrupted:
try:
# try to get signed url for thumbnail
signed_url = get_presigned_upload_url(thumbname, dataset_id,
sample_id, token)
# try to create thumbnail
image_path = os.path.join(path_to_folder, fname)
with Image.open(image_path) as temp_image:
thumbnail = get_thumbnail_from_img(temp_image)
# try to upload thumbnail
upload_file_with_signed_url(
PIL_to_bytes(thumbnail, ext='webp', quality=70),
signed_url)
except RuntimeError:
thumbnail_upload_success = False
# upload full image
image_upload_success = True
if mode == 'full' and not is_corrupted:
try:
# try to get signed url for image
signed_url = get_presigned_upload_url(basename, dataset_id,
sample_id, token)
# try to upload image
image_path = os.path.join(path_to_folder, fname)
with open(image_path, 'rb') as temp_image:
upload_file_with_signed_url(temp_image, signed_url)
except RuntimeError:
image_upload_success = False
success = sample_upload_success
success = success and thumbnail_upload_success
success = success and image_upload_success
return success