def test_create_manifest_file_s3(make_stubber, monkeypatch, error_code):
s3_resource = boto3.resource('s3')
s3_stubber = make_stubber(s3_resource.meta.client)
image_bucket = 'image-bucket'
image_prefix = 'image-prefix/'
image_path = f'{image_bucket}/{image_prefix}'
mani_bucket = 'mani-bucket'
mani_prefix = 'mani-prefix/'
manifest_path = f'{mani_bucket}/{mani_prefix}'
monkeypatch.setattr(
s3_resource.meta.client, 'upload_file',
lambda Filename, Bucket, Key, ExtraArgs, Callback, Config: None)
s3_stubber.stub_list_objects(image_bucket,
[f'{image_prefix}anomaly/anomaly-test-key'],
f"{image_prefix}anomaly/", '/')
s3_stubber.stub_list_objects(image_bucket,
[f'{image_prefix}normal/normal-test-key'],
f"{image_prefix}normal/",
'/',
error_code=error_code)
with open("temp.manifest", 'w') as mani:
mani.write("Test manifest.")
if error_code is None:
Datasets.create_manifest_file_s3(s3_resource, image_path,
manifest_path)
else:
with pytest.raises(ClientError) as exc_info:
Datasets.create_manifest_file_s3(s3_resource, image_path,
manifest_path)
assert exc_info.value.response['Error']['Code'] == error_code
def create_dataset(lookoutvision_client, s3_resource, bucket, project_name,
dataset_images, dataset_type):
"""
Creates a manifest from images in the supplied bucket and then creates
a dataset.
:param lookoutvision_client: A Boto3 Lookout for Vision client.
:param s3_resource: A Boto3 Amazon S3 client.
:param bucket: The bucket that stores the manifest file.
:param project_name: The project in which to create the dataset.
:param dataset_images: The location of the images referenced by the dataset.
:param dataset_type: The type of dataset to create (train or test).
"""
print(f"Creating {dataset_type} dataset...")
manifest_file = f"s3://{bucket}/{project_name}/manifests/{dataset_type}.manifest"
logger.info("Creating %s manifest file in %s.", dataset_type,
manifest_file)
Datasets.create_manifest_file_s3(s3_resource, dataset_images,
manifest_file)
logger.info("Create %s dataset for project %s", dataset_type, project_name)
Datasets.create_dataset(lookoutvision_client, project_name, manifest_file,
dataset_type)
