def test_czi_parallel_processor(filename):
mock_ssm().start()
mock_s3().start()
init_ssm()
# init task
for x in range(2):
for y in range(2):
for z in range(2):
for c in range(2):
for t in range(2):
# Create sub_region file
open(
'sub_x_{}_2_y_{}_2_z_{}_2_c_{}_2_t_{}_2.txt'.
format(x, y, z, c, t), 'w').write("")
# Setup task with inputs
inputs = {
'file':
os.path.join('/test-resources', filename),
'sub_region_file':
'sub_x_{}_2_y_{}_2_z_{}_2_c_{}_2_t_{}_2.txt'.
format(x, y, z, c, t)
}
task = CZIProcessor(inputs=inputs)
setup_processor(task)
# run
task.run()
mock_s3().stop()
mock_ssm().stop()
def test_ometiff_processor(filename):
mock_ssm().start()
mock_s3().start()
init_ssm()
# init task
inputs = {'file': os.path.join('/test-resources', filename)}
task = OMETIFFProcessor(inputs=inputs)
setup_processor(task)
# run
task.run()
# Check outputs
assert os.path.isfile('view_asset_info.json')
json_dict = json.load(open('view_asset_info.json'))
assert 'size' in json_dict.keys()
assert 'fileType' in json_dict.keys()
assert os.path.isdir('%s-zoomed' %
os.path.basename(task.outputs[0].file_path))
assert os.path.isfile('%s-zoomed/dimensions.json' %
os.path.basename(task.outputs[0].file_path))
assert os.path.isfile('metadata.json')
# Check hack for backwards compatibility with viewer
assert os.path.isfile('%s-zoomed/slide.dzi' %
os.path.basename(task.outputs[0].file_path))
# assert os.path.isfile('%s-zoomed/slide.png' % os.path.basename(task.outputs[0].file_path))
mock_s3().stop()
mock_ssm().stop()
def test_sub_region_standard_image_processor(filename):
mock_ssm().start()
mock_s3().start()
init_ssm()
# init task
for a in range(1):
for b in range(1):
open('{}_{}.txt'.format(
a,
b,
), 'w').write("4,4")
inputs = {
'file': os.path.join('/test-resources', filename),
'sub_region_file': '{}_{}.txt'.format(
a,
b,
)
}
task = StandardImageProcessor(inputs=inputs)
setup_processor(task)
# run
task.run()
assert os.path.isdir('view')
assert os.path.isfile('view/slide.dzi')
def test_czi_processor(filename):
mock_ssm().start()
mock_s3().start()
init_ssm()
# init task
inputs = {
'file': os.path.join('/test-resources', filename),
'optimize_view': "y"
}
task = CZIProcessor(inputs=inputs)
setup_processor(task)
# run
task.run()
# Check outputs
first_slice_set = sorted(
glob.glob('%s-zoomed/*.dzi' % os.path.basename(task.file)))[0]
assert os.path.isfile('%s-zoomed/slide.dzi' % os.path.basename(task.file))
assert os.path.isfile('metadata.json')
mock_s3().stop()
mock_ssm().stop()
def test_standard_image_processor(filename):
mock_ssm().start()
mock_s3().start()
init_ssm()
# init task
inputs = {'file': os.path.join('/test-resources', filename)}
task = StandardImageProcessor(inputs=inputs)
setup_processor(task)
# run
task.run()
# Check outputs
assert os.path.isfile('view_asset_info.json')
json_dict = json.load(open('view_asset_info.json'))
print json_dict
if 'huge' in filename or 'tif' in filename.lower():
assert os.path.isdir('view')
assert os.path.isfile('view/slide.dzi')
assert os.path.isfile('view/dimensions.json')
elif filename.endswith('.jp2'):
assert os.path.isfile('output.png')
else:
assert not os.path.isdir('view')
assert not os.path.isfile('view/slide.dzi')
assert os.path.isfile('metadata.json')
def can_use_yaml_cfg_with_handler_override(self):
with mock_ssm():
options = {
'PARAMETER_STORE_AWS_REGION': 'us-west-2',
'PARAMETER_STORE_PREFIX': '/aumbry-test',
}
expected_cfg = SampleYamlConfig()
expected_cfg.nope = 'testing'
handler = GenericHandler()
# Save Sample Config
aumbry.save(aumbry.PARAM_STORE,
expected_cfg,
options,
handler=handler)
# Retrieve back the config
cfg = aumbry.load(aumbry.PARAM_STORE,
SampleGenericConfig,
options,
handler=handler)
expect(cfg.nope).to.equal(expected_cfg.nope)
def ssm(aws_credentials):
"""Mocks a SSM service for moto with a parameter holding value for testing bucket
Parameters
----------
aws_credentials : TYPE
Description
Yields
------
TYPE
Description
"""
with mock_ssm():
ssm = boto3.client('ssm')
ssm.put_parameter(Name='ConfigBucketName',
Value=os.environ.get('CONFIG_BUCKET'),
Type='String')
ssm.put_parameter(Name='InputBucketName',
Value=os.environ.get('INPUT_BUCKET'),
Type='String')
ssm.put_parameter(Name='ExternalBucketName',
Value=os.environ.get('EXPORT_BUCKET'),
Type='String')
ssm.put_parameter(Name='LogBucketName',
Value=os.environ.get('LOG_BUCKET'),
Type='String')
yield ssm
def ssm(aws_credentials):
with mock_ssm():
ssm = boto3.client('ssm')
for parameter in [YT_API_KEY_SSM, YT_LIST_ID_SSM, YT_NEXT_PAGE_TOKEN_SSM, TELEGRAM_BOT_TOKEN_SSM, TELEGRAM_CHAT_ID_SSM]:
ssm.put_parameter(
Name=parameter,
Value='test',
Type='String'
)
yield ssm
def aws_parameter_fixtures(random_string):
mock = mock_ssm()
mock.start()
os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
client = boto3.client("ssm")
client.put_parameter(Name="/foo/bar/baz",
Value=random_string,
Type="String")
yield random_string
mock.stop()
def moto_session(monkeypatch, moto_sagemaker):
for k in list(os.environ):
if k.startswith("AWS_"):
monkeypatch.delitem(os.environ, k)
# The environment variables duplicate what happens when an AWS Lambda
# is executed. See
# https://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html
monkeypatch.setitem(os.environ, "AWS_ACCESS_KEY_ID", "dummy-access-key")
monkeypatch.setitem(os.environ, "AWS_SECRET_ACCESS_KEY", "dummy-access-key-secret")
monkeypatch.setitem(os.environ, "AWS_SECURITY_TOKEN", "dummy-security-token")
monkeypatch.setitem(os.environ, "AWS_SESSION_TOKEN", "dummy-session-token")
monkeypatch.setitem(os.environ, "AWS_REGION", MOTO_TEST_REGION)
with moto.mock_s3(), moto.mock_ssm(), moto_sagemaker.mock_sagemaker():
yield boto3.Session(region_name=MOTO_TEST_REGION)
def test_extract_structure_as_json(filename):
inputs = {'file': os.path.join('/test-resources', filename)}
mock_ssm().start()
mock_s3().start()
init_ssm()
task = HDF5StructureProcessor(inputs=inputs)
setup_processor(task)
task.run()
# Verify the payload JSON is written out:
assert task.payload_output_path and os.path.exists(
task.payload_output_path)
with open(task.payload_output_path, 'r') as f:
payload = json.load(f)
assert len(payload) > 0
entry = payload[0]
assert 'path' in entry and isinstance(entry['path'], list)
assert 'path_key' in entry and isinstance(entry['path_key'],
string_types)
assert 'name' in entry and isinstance(entry['name'], string_types)
assert 'metadata' in entry and isinstance(entry['metadata'], list)
assert 'type' in entry and isinstance(entry['type'],
(dict, string_types))
if isinstance(entry['type'], dict) and \
entry['type']['dimensions'] == 1 and \
entry['type']['size'] == 1:
assert 'value' in entry
# Verify the asset JSON is written out:
assert task.asset_output_path and os.path.exists(task.asset_output_path)
with open(task.asset_output_path, 'r') as f:
asset_info = json.load(f)
print(asset_info)
def ssm_client(pytestconfig):
with mock_ssm():
ssm = client('ssm')
for e in pytestconfig.getini('dataplattform-aws-ssm'):
key, _, value = e.partition('=')
key = key.strip()
type_, _, value = value.partition(':')
ssm.put_parameter(Name=key,
Value=value.strip(),
Type=type_.strip(),
Tier='Standard')
yield ssm
def ssm(aws_credentials):
mock = mock_ssm()
mock.start()
# There is currently a bug on moto, this line is needed as a workaround
# Ref: https://github.com/spulec/moto/issues/1926
# boto3.setup_default_session()
ssm_client = boto3.client('ssm', 'us-west-2')
token_name = "TEST_TOKEN"
token_value = "TEST_VALUE"
response = ssm_client.put_parameter(
Name=token_name, Description="A test parameter", Value=token_value, Type="SecureString"
)
yield (ssm_client, token_name, token_value)
mock.stop()
def mock_aws_environment(secret_key):
with contextlib.ExitStack() as stack:
# fmt: off
# Add AWS services to mock in every test...
mock_aws_service_context_managers = (
moto.mock_ssm(),
moto.mock_dynamodb2(),
)
# fmt: on
for service_mock in mock_aws_service_context_managers:
stack.enter_context(service_mock)
# Perform (mocked) AWS service calls to prepare the environment for each test...
boto3.client('ssm').put_parameter(
Name=os.environ['THOR_API_SECRET_KEY__SSM_KEY'], Type='SecureString', Value=secret_key
)
yield
def ssm(self, scope="session", autouse=True):
mock = mock_ssm()
mock.start()
# There is currently a bug on moto, this line is needed as a workaround
# Ref: https://github.com/spulec/moto/issues/1926
boto3.setup_default_session()
ssm_client = boto3.client('ssm', 'us-west-2')
ssm_client.put_parameter(Name="TENABLEIO_ACCESS_KEY",
Description="Bogus access key.",
Value="TEST",
Type="SecureString")
ssm_client.put_parameter(Name="TENABLEIO_SECRET_KEY",
Description="Bogus secret key.",
Value="TEST",
Type="SecureString")
yield ssm_client
mock.stop()
def ssm_client(aws_credentials: Any) -> Any:
with mock_ssm():
client = boto3.client("ssm", region_name="eu-west-2")
values = [
dict(
Name="/test",
Type="string",
Value="9ce10572-a1a4-422c-9cf1-d4b45ecd93c2",
),
dict(
Name="/parent/child",
Type="string",
Value="27698a22-c47c-4457-a6b6-907051b4534a",
),
dict(
Name="/a/1",
Type="string",
Value="49f48a53-e592-4bab-bf3a-7cfb08c584fb",
),
dict(
Name="/a/2",
Type="string",
Value="ee11a634-5d43-4f16-a584-82752ce24591",
),
dict(
Name="/b/3",
Type="string",
Value="fef47e26-b153-4ab8-a195-c7958e000491",
),
dict(
Name="/c/2",
Type="string",
Value="e6c0fcf4-a67b-4375-a477-e7d72fce3420",
),
]
for value in values:
client.put_parameter(**value)
yield client
def can_successfully_save_and_load(self):
with mock_ssm():
options = {
'PARAMETER_STORE_AWS_REGION': 'us-west-2',
'PARAMETER_STORE_PREFIX': '/aumbry-test',
}
expected_cfg = SampleGenericConfig()
expected_cfg.nope = 'testing'
expected_cfg.sample_list = ['trace']
expected_cfg.sample_dict = {'trace': 'boom'}
expected_cfg.sample_model = SampleJsonConfig()
expected_cfg.sample_model.nope = 'testing2'
# Save Sample Config
aumbry.save(aumbry.PARAM_STORE, expected_cfg, options)
# Retrieve back the config
cfg = aumbry.load(aumbry.PARAM_STORE, SampleGenericConfig, options)
expect(cfg.nope).to.equal(expected_cfg.nope)
expect(cfg.sample_dict).to.equal({'trace': 'boom'})
expect(cfg.sample_list).to.equal(expected_cfg.sample_list)
expect(cfg.sample_model.nope).to.equal(expected_cfg.sample_model.nope)
def ssm():
with mock_ssm():
yield boto3.client('ssm')
def ssm():
with mock_ssm():
yield boto3.client('ssm', region_name='us-east-1')
def ssm(aws_credentials):
with moto.mock_ssm():
yield boto3.client("ssm", region_name="us-east-1")
def ssm():
"""Mock SSM."""
with mock_ssm():
yield
def test_explode_assets(filename):
print "~" * 60
print " Using test file %s to test exploding PNG assets " % filename
print "~" * 60
mock_dynamodb2().start()
mock_ssm().start()
mock_s3().start()
init_ssm()
# init task
inputs = {'file': os.path.join('/test-resources', filename)}
task = OMETIFFProcessor(inputs=inputs)
setup_processor(task)
# Load image
output_file = OMETIFFOutputFile()
output_file.file_path = task.file
output_file.load_and_save_assets(output_file.file_path,
i_xyzct=(-1, -1, -1, -1, -1),
n_xyzct=(-1, -1, -1, -1, -1),
asset_format='png')
# Since we know first two dimensions are are X and Y
assert output_file.img_dimensions['dimensions'][0]['assignment'] == 'X'
assert output_file.img_dimensions['dimensions'][1]['assignment'] == 'Y'
# Get number of files generated
num_png_files = len(
glob.glob('%s-zoomed/*.png' % os.path.basename(output_file.file_path)))
print glob.glob('%s-zoomed/*.png' %
os.path.basename(output_file.file_path))
# Ensure correct number of png outputs
if output_file.isRGB:
# Compute combinatorial number of files expected
shape = [
output_file.SizeX, output_file.SizeY, output_file.SizeZ,
output_file.SizeC, output_file.SizeT
]
# Remove RGB dimension from combinatorial count
shape.pop(output_file.RGBDimension)
else:
# Compute combinatorial number of files expected
shape = [
output_file.SizeX, output_file.SizeY, output_file.SizeZ,
output_file.SizeC, output_file.SizeT
]
total_count = np.prod(shape[2:])
print shape, output_file.isRGB
# Make sure same number of files generated
assert num_png_files == total_count
# Save to local storage output deepzoom files
output_file.load_and_save_assets(output_file.file_path,
i_xyzct=(-1, -1, -1, -1, -1),
n_xyzct=(-1, -1, -1, -1, -1),
asset_format='dzi')
# Get number of files generated
num_dzi_files = len(
glob.glob('%s-zoomed/*.dzi' % os.path.basename(output_file.file_path)))
print glob.glob('%s-zoomed/*.dzi' %
os.path.basename(output_file.file_path))
# Ensure correct number of dzi outputs
if output_file.isRGB:
# Compute combinatorial number of files expected
shape = [
output_file.SizeX, output_file.SizeY, output_file.SizeZ,
output_file.SizeC, output_file.SizeT
]
# Remove RGB dimension from combinatorial count
shape.pop(output_file.RGBDimension)
else:
# Compute combinatorial number of files expected
shape = [
output_file.SizeX, output_file.SizeY, output_file.SizeZ,
output_file.SizeC, output_file.SizeT
]
total_count = np.prod(shape[2:])
print shape
# Make sure same number of files generated
assert num_dzi_files == total_count
# Clean up
os.system('rm %s-zoomed/*.png' % os.path.basename(output_file.file_path))
os.system('rm %s-zoomed/*.dzi' % os.path.basename(output_file.file_path))
mock_s3().stop()
mock_ssm().stop()
mock_dynamodb2().stop()
def ssm_client():
mock = mock_ssm()
mock.start()
ssm_client = boto3.client("ssm")
yield ssm_client
mock.stop()
def ssm():
with mock_ssm():
yield
def fake_ssm():
"""Provide a consistent faked SSM context for running an entire test case."""
with mock_ssm():
yield None
def ssm(aws_creds):
with moto.mock_ssm():
yield boto3.client('ssm', region_name='eu-west-1')
def ssm():
with mock_ssm():
yield boto3.client("ssm", region_name="us-east-1")
def ssm_client(aws_credentials):
"""SSM Mock Client"""
with mock_ssm():
connection = boto3.client("ssm", region_name="us-east-1")
yield connection
def ssm() -> Iterator[None]:
"""Mock SSM."""
with mock_ssm():
yield
def ssm_client(aws_credentials):
with mock_ssm():
yield boto3.client("ssm")
def ssm():
with mock_ssm():
yield
