def _upload_single_image(self, image, label, filename: str, mode):
"""Uploads a single image to the Lightly platform.
"""
# check whether the filename is too long
basename = filename
if not check_filename(basename):
msg = (
f'Filename {basename} is longer than the allowed maximum of '
'characters and will be skipped.')
warnings.warn(msg)
return False
# calculate metadata, and check if corrupted
metadata = check_image(image)
# generate thumbnail if necessary
thumbname = None
if not metadata['is_corrupted'] and mode in ["thumbnails", "full"]:
thumbname = '.'.join(basename.split('.')[:-1]) + '_thumb.webp'
body = SampleCreateRequest(file_name=basename,
thumb_name=thumbname,
meta_data=metadata)
sample_id = retry(self.samples_api.create_sample_by_dataset_id,
body=body,
dataset_id=self.dataset_id).id
if not metadata['is_corrupted'] and mode in ["thumbnails", "full"]:
thumbnail = get_thumbnail_from_img(image)
image_to_upload = PIL_to_bytes(thumbnail, ext='webp', quality=90)
signed_url = retry(
self.samples_api.get_sample_image_write_url_by_id,
dataset_id=self.dataset_id,
sample_id=sample_id,
is_thumbnail=True)
# try to upload thumbnail
retry(self.upload_file_with_signed_url, image_to_upload,
signed_url)
thumbnail.close()
if not metadata['is_corrupted'] and mode == "full":
image_to_upload = PIL_to_bytes(image)
signed_url = retry(
self.samples_api.get_sample_image_write_url_by_id,
dataset_id=self.dataset_id,
sample_id=sample_id,
is_thumbnail=False)
# try to upload thumbnail
retry(self.upload_file_with_signed_url, image_to_upload,
signed_url)
image.close()
def _upload_single_image(image, label, filename, dataset_id, token, mode):
"""Uploads a single image to the Lightly platform.
"""
# check whether the filename is too long
basename = filename
if not check_filename(basename):
msg = (f'Filename {basename} is longer than the allowed maximum of '
'characters and will be skipped.')
warnings.warn(msg)
return False
# calculate metadata, and check if corrupted
metadata = check_image(image)
# generate thumbnail if necessary
thumbname = None
thumbnail = None
if mode == 'thumbnails' and not metadata['is_corrupted']:
thumbname = '.'.join(basename.split('.')[:-1]) + '_thumb.webp'
thumbnail = get_thumbnail_from_img(image)
# upload sample with metadata
sample_upload_success = True
try:
sample_id = routes.users.datasets.samples.post(basename, thumbname,
metadata, dataset_id,
token)
except RuntimeError:
sample_upload_success = False
# upload thumbnail
thumbnail_upload_success = True
if mode == 'thumbnails' and not metadata['is_corrupted']:
try:
# try to get signed url for thumbnail
signed_url = routes.users.datasets.samples. \
get_presigned_upload_url(
thumbname, dataset_id, sample_id, token)
# try to upload thumbnail
upload_file_with_signed_url(
PIL_to_bytes(thumbnail, ext='webp', quality=70), signed_url)
# close thumbnail
thumbnail.close()
except RuntimeError:
thumbnail_upload_success = False
# upload full image
image_upload_success = True
if mode == 'full' and not metadata['is_corrupted']:
try:
# try to get signed url for image
signed_url = routes.users.datasets.samples. \
get_presigned_upload_url(
basename, dataset_id, sample_id, token)
# try to upload image
upload_file_with_signed_url(PIL_to_bytes(image), signed_url)
# close image
image.close()
except RuntimeError:
image_upload_success = False
success = sample_upload_success
success = success and thumbnail_upload_success
success = success and image_upload_success
return success
def _upload_single_image(self,
image,
filename: str,
filepath: str,
mode: str,
custom_metadata: Union[Dict, None] = None,
datasource_type: str = 'LIGHTLY'):
"""Uploads a single image to the Lightly platform.
"""
# check whether the filepath is too long
if not check_filename(filepath):
msg = (
'Filepath {filepath} is longer than the allowed maximum of '
f'{MAXIMUM_FILENAME_LENGTH} characters and will be skipped.')
raise ValueError(msg)
# calculate metadata, and check if corrupted
metadata = image_processing.Metadata(image).to_dict()
metadata['sizeInBytes'] = os.path.getsize(filepath)
# try to get exif data
try:
exifdata = image_processing.Exifdata(image)
except Exception: # pylint disable=broad-except
exifdata = None
# generate thumbnail if necessary
thumbname = None
if not metadata['is_corrupted'] and mode in ['thumbnails', 'full']:
thumbname = '.'.join(filename.split('.')[:-1]) + '_thumb.webp'
body = SampleCreateRequest(
file_name=filename,
thumb_name=thumbname,
meta_data=metadata,
exif=exifdata if exifdata is None else exifdata.to_dict(),
custom_meta_data=custom_metadata,
)
sample_id = retry(self._samples_api.create_sample_by_dataset_id,
body=body,
dataset_id=self.dataset_id).id
if not metadata['is_corrupted'] and mode in ['thumbnails', 'full']:
def upload_thumbnail(image, signed_url):
thumbnail = image_processing.Thumbnail(image)
image_to_upload = thumbnail.to_bytes()
headers = None
if datasource_type == 'AZURE':
# build headers for Azure blob storage
size_in_bytes = str(image_to_upload.getbuffer().nbytes)
headers = build_azure_signed_url_write_headers(
size_in_bytes)
retry(
self.upload_file_with_signed_url,
image_to_upload,
signed_url,
headers=headers,
)
thumbnail.thumbnail.close()
def upload_full_image(filepath, signed_url):
with open(filepath, 'rb') as image_to_upload:
headers = None
if datasource_type == 'AZURE':
# build headers for Azure blob storage
image_to_upload.seek(0, 2)
size_in_bytes = str(image_to_upload.tell())
image_to_upload.seek(0, 0)
headers = build_azure_signed_url_write_headers(
size_in_bytes)
retry(self.upload_file_with_signed_url,
image_to_upload,
signed_url,
headers=headers)
if mode == 'thumbnails':
thumbnail_url = retry(
self._samples_api.get_sample_image_write_url_by_id,
dataset_id=self.dataset_id,
sample_id=sample_id,
is_thumbnail=True)
upload_thumbnail(image, thumbnail_url)
elif mode == 'full':
sample_write_urls: SampleWriteUrls = retry(
self._samples_api.get_sample_image_write_urls_by_id,
dataset_id=self.dataset_id,
sample_id=sample_id)
upload_thumbnail(image, sample_write_urls.thumb)
upload_full_image(filepath, sample_write_urls.full)
image.close()