def test_czi_parallel_processor(filename): mock_ssm().start() mock_s3().start() init_ssm() # init task for x in range(2): for y in range(2): for z in range(2): for c in range(2): for t in range(2): # Create sub_region file open( 'sub_x_{}_2_y_{}_2_z_{}_2_c_{}_2_t_{}_2.txt'. format(x, y, z, c, t), 'w').write("") # Setup task with inputs inputs = { 'file': os.path.join('/test-resources', filename), 'sub_region_file': 'sub_x_{}_2_y_{}_2_z_{}_2_c_{}_2_t_{}_2.txt'. format(x, y, z, c, t) } task = CZIProcessor(inputs=inputs) setup_processor(task) # run task.run() mock_s3().stop() mock_ssm().stop()
def test_ometiff_processor(filename): mock_ssm().start() mock_s3().start() init_ssm() # init task inputs = {'file': os.path.join('/test-resources', filename)} task = OMETIFFProcessor(inputs=inputs) setup_processor(task) # run task.run() # Check outputs assert os.path.isfile('view_asset_info.json') json_dict = json.load(open('view_asset_info.json')) assert 'size' in json_dict.keys() assert 'fileType' in json_dict.keys() assert os.path.isdir('%s-zoomed' % os.path.basename(task.outputs[0].file_path)) assert os.path.isfile('%s-zoomed/dimensions.json' % os.path.basename(task.outputs[0].file_path)) assert os.path.isfile('metadata.json') # Check hack for backwards compatibility with viewer assert os.path.isfile('%s-zoomed/slide.dzi' % os.path.basename(task.outputs[0].file_path)) # assert os.path.isfile('%s-zoomed/slide.png' % os.path.basename(task.outputs[0].file_path)) mock_s3().stop() mock_ssm().stop()
def test_sub_region_standard_image_processor(filename): mock_ssm().start() mock_s3().start() init_ssm() # init task for a in range(1): for b in range(1): open('{}_{}.txt'.format( a, b, ), 'w').write("4,4") inputs = { 'file': os.path.join('/test-resources', filename), 'sub_region_file': '{}_{}.txt'.format( a, b, ) } task = StandardImageProcessor(inputs=inputs) setup_processor(task) # run task.run() assert os.path.isdir('view') assert os.path.isfile('view/slide.dzi')
def test_czi_processor(filename): mock_ssm().start() mock_s3().start() init_ssm() # init task inputs = { 'file': os.path.join('/test-resources', filename), 'optimize_view': "y" } task = CZIProcessor(inputs=inputs) setup_processor(task) # run task.run() # Check outputs first_slice_set = sorted( glob.glob('%s-zoomed/*.dzi' % os.path.basename(task.file)))[0] assert os.path.isfile('%s-zoomed/slide.dzi' % os.path.basename(task.file)) assert os.path.isfile('metadata.json') mock_s3().stop() mock_ssm().stop()
def test_standard_image_processor(filename): mock_ssm().start() mock_s3().start() init_ssm() # init task inputs = {'file': os.path.join('/test-resources', filename)} task = StandardImageProcessor(inputs=inputs) setup_processor(task) # run task.run() # Check outputs assert os.path.isfile('view_asset_info.json') json_dict = json.load(open('view_asset_info.json')) print json_dict if 'huge' in filename or 'tif' in filename.lower(): assert os.path.isdir('view') assert os.path.isfile('view/slide.dzi') assert os.path.isfile('view/dimensions.json') elif filename.endswith('.jp2'): assert os.path.isfile('output.png') else: assert not os.path.isdir('view') assert not os.path.isfile('view/slide.dzi') assert os.path.isfile('metadata.json')
def can_use_yaml_cfg_with_handler_override(self): with mock_ssm(): options = { 'PARAMETER_STORE_AWS_REGION': 'us-west-2', 'PARAMETER_STORE_PREFIX': '/aumbry-test', } expected_cfg = SampleYamlConfig() expected_cfg.nope = 'testing' handler = GenericHandler() # Save Sample Config aumbry.save(aumbry.PARAM_STORE, expected_cfg, options, handler=handler) # Retrieve back the config cfg = aumbry.load(aumbry.PARAM_STORE, SampleGenericConfig, options, handler=handler) expect(cfg.nope).to.equal(expected_cfg.nope)
def ssm(aws_credentials): """Mocks a SSM service for moto with a parameter holding value for testing bucket Parameters ---------- aws_credentials : TYPE Description Yields ------ TYPE Description """ with mock_ssm(): ssm = boto3.client('ssm') ssm.put_parameter(Name='ConfigBucketName', Value=os.environ.get('CONFIG_BUCKET'), Type='String') ssm.put_parameter(Name='InputBucketName', Value=os.environ.get('INPUT_BUCKET'), Type='String') ssm.put_parameter(Name='ExternalBucketName', Value=os.environ.get('EXPORT_BUCKET'), Type='String') ssm.put_parameter(Name='LogBucketName', Value=os.environ.get('LOG_BUCKET'), Type='String') yield ssm
def ssm(aws_credentials): with mock_ssm(): ssm = boto3.client('ssm') for parameter in [YT_API_KEY_SSM, YT_LIST_ID_SSM, YT_NEXT_PAGE_TOKEN_SSM, TELEGRAM_BOT_TOKEN_SSM, TELEGRAM_CHAT_ID_SSM]: ssm.put_parameter( Name=parameter, Value='test', Type='String' ) yield ssm
def aws_parameter_fixtures(random_string): mock = mock_ssm() mock.start() os.environ["AWS_DEFAULT_REGION"] = "us-east-1" client = boto3.client("ssm") client.put_parameter(Name="/foo/bar/baz", Value=random_string, Type="String") yield random_string mock.stop()
def moto_session(monkeypatch, moto_sagemaker): for k in list(os.environ): if k.startswith("AWS_"): monkeypatch.delitem(os.environ, k) # The environment variables duplicate what happens when an AWS Lambda # is executed. See # https://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html monkeypatch.setitem(os.environ, "AWS_ACCESS_KEY_ID", "dummy-access-key") monkeypatch.setitem(os.environ, "AWS_SECRET_ACCESS_KEY", "dummy-access-key-secret") monkeypatch.setitem(os.environ, "AWS_SECURITY_TOKEN", "dummy-security-token") monkeypatch.setitem(os.environ, "AWS_SESSION_TOKEN", "dummy-session-token") monkeypatch.setitem(os.environ, "AWS_REGION", MOTO_TEST_REGION) with moto.mock_s3(), moto.mock_ssm(), moto_sagemaker.mock_sagemaker(): yield boto3.Session(region_name=MOTO_TEST_REGION)
def test_extract_structure_as_json(filename): inputs = {'file': os.path.join('/test-resources', filename)} mock_ssm().start() mock_s3().start() init_ssm() task = HDF5StructureProcessor(inputs=inputs) setup_processor(task) task.run() # Verify the payload JSON is written out: assert task.payload_output_path and os.path.exists( task.payload_output_path) with open(task.payload_output_path, 'r') as f: payload = json.load(f) assert len(payload) > 0 entry = payload[0] assert 'path' in entry and isinstance(entry['path'], list) assert 'path_key' in entry and isinstance(entry['path_key'], string_types) assert 'name' in entry and isinstance(entry['name'], string_types) assert 'metadata' in entry and isinstance(entry['metadata'], list) assert 'type' in entry and isinstance(entry['type'], (dict, string_types)) if isinstance(entry['type'], dict) and \ entry['type']['dimensions'] == 1 and \ entry['type']['size'] == 1: assert 'value' in entry # Verify the asset JSON is written out: assert task.asset_output_path and os.path.exists(task.asset_output_path) with open(task.asset_output_path, 'r') as f: asset_info = json.load(f) print(asset_info)
def ssm_client(pytestconfig): with mock_ssm(): ssm = client('ssm') for e in pytestconfig.getini('dataplattform-aws-ssm'): key, _, value = e.partition('=') key = key.strip() type_, _, value = value.partition(':') ssm.put_parameter(Name=key, Value=value.strip(), Type=type_.strip(), Tier='Standard') yield ssm
def ssm(aws_credentials): mock = mock_ssm() mock.start() # There is currently a bug on moto, this line is needed as a workaround # Ref: https://github.com/spulec/moto/issues/1926 # boto3.setup_default_session() ssm_client = boto3.client('ssm', 'us-west-2') token_name = "TEST_TOKEN" token_value = "TEST_VALUE" response = ssm_client.put_parameter( Name=token_name, Description="A test parameter", Value=token_value, Type="SecureString" ) yield (ssm_client, token_name, token_value) mock.stop()
def mock_aws_environment(secret_key): with contextlib.ExitStack() as stack: # fmt: off # Add AWS services to mock in every test... mock_aws_service_context_managers = ( moto.mock_ssm(), moto.mock_dynamodb2(), ) # fmt: on for service_mock in mock_aws_service_context_managers: stack.enter_context(service_mock) # Perform (mocked) AWS service calls to prepare the environment for each test... boto3.client('ssm').put_parameter( Name=os.environ['THOR_API_SECRET_KEY__SSM_KEY'], Type='SecureString', Value=secret_key ) yield
def ssm(self, scope="session", autouse=True): mock = mock_ssm() mock.start() # There is currently a bug on moto, this line is needed as a workaround # Ref: https://github.com/spulec/moto/issues/1926 boto3.setup_default_session() ssm_client = boto3.client('ssm', 'us-west-2') ssm_client.put_parameter(Name="TENABLEIO_ACCESS_KEY", Description="Bogus access key.", Value="TEST", Type="SecureString") ssm_client.put_parameter(Name="TENABLEIO_SECRET_KEY", Description="Bogus secret key.", Value="TEST", Type="SecureString") yield ssm_client mock.stop()
def ssm_client(aws_credentials: Any) -> Any: with mock_ssm(): client = boto3.client("ssm", region_name="eu-west-2") values = [ dict( Name="/test", Type="string", Value="9ce10572-a1a4-422c-9cf1-d4b45ecd93c2", ), dict( Name="/parent/child", Type="string", Value="27698a22-c47c-4457-a6b6-907051b4534a", ), dict( Name="/a/1", Type="string", Value="49f48a53-e592-4bab-bf3a-7cfb08c584fb", ), dict( Name="/a/2", Type="string", Value="ee11a634-5d43-4f16-a584-82752ce24591", ), dict( Name="/b/3", Type="string", Value="fef47e26-b153-4ab8-a195-c7958e000491", ), dict( Name="/c/2", Type="string", Value="e6c0fcf4-a67b-4375-a477-e7d72fce3420", ), ] for value in values: client.put_parameter(**value) yield client
def can_successfully_save_and_load(self): with mock_ssm(): options = { 'PARAMETER_STORE_AWS_REGION': 'us-west-2', 'PARAMETER_STORE_PREFIX': '/aumbry-test', } expected_cfg = SampleGenericConfig() expected_cfg.nope = 'testing' expected_cfg.sample_list = ['trace'] expected_cfg.sample_dict = {'trace': 'boom'} expected_cfg.sample_model = SampleJsonConfig() expected_cfg.sample_model.nope = 'testing2' # Save Sample Config aumbry.save(aumbry.PARAM_STORE, expected_cfg, options) # Retrieve back the config cfg = aumbry.load(aumbry.PARAM_STORE, SampleGenericConfig, options) expect(cfg.nope).to.equal(expected_cfg.nope) expect(cfg.sample_dict).to.equal({'trace': 'boom'}) expect(cfg.sample_list).to.equal(expected_cfg.sample_list) expect(cfg.sample_model.nope).to.equal(expected_cfg.sample_model.nope)
def ssm(): with mock_ssm(): yield boto3.client('ssm')
def ssm(): with mock_ssm(): yield boto3.client('ssm', region_name='us-east-1')
def ssm(aws_credentials): with moto.mock_ssm(): yield boto3.client("ssm", region_name="us-east-1")
def ssm(): """Mock SSM.""" with mock_ssm(): yield
def test_explode_assets(filename): print "~" * 60 print " Using test file %s to test exploding PNG assets " % filename print "~" * 60 mock_dynamodb2().start() mock_ssm().start() mock_s3().start() init_ssm() # init task inputs = {'file': os.path.join('/test-resources', filename)} task = OMETIFFProcessor(inputs=inputs) setup_processor(task) # Load image output_file = OMETIFFOutputFile() output_file.file_path = task.file output_file.load_and_save_assets(output_file.file_path, i_xyzct=(-1, -1, -1, -1, -1), n_xyzct=(-1, -1, -1, -1, -1), asset_format='png') # Since we know first two dimensions are are X and Y assert output_file.img_dimensions['dimensions'][0]['assignment'] == 'X' assert output_file.img_dimensions['dimensions'][1]['assignment'] == 'Y' # Get number of files generated num_png_files = len( glob.glob('%s-zoomed/*.png' % os.path.basename(output_file.file_path))) print glob.glob('%s-zoomed/*.png' % os.path.basename(output_file.file_path)) # Ensure correct number of png outputs if output_file.isRGB: # Compute combinatorial number of files expected shape = [ output_file.SizeX, output_file.SizeY, output_file.SizeZ, output_file.SizeC, output_file.SizeT ] # Remove RGB dimension from combinatorial count shape.pop(output_file.RGBDimension) else: # Compute combinatorial number of files expected shape = [ output_file.SizeX, output_file.SizeY, output_file.SizeZ, output_file.SizeC, output_file.SizeT ] total_count = np.prod(shape[2:]) print shape, output_file.isRGB # Make sure same number of files generated assert num_png_files == total_count # Save to local storage output deepzoom files output_file.load_and_save_assets(output_file.file_path, i_xyzct=(-1, -1, -1, -1, -1), n_xyzct=(-1, -1, -1, -1, -1), asset_format='dzi') # Get number of files generated num_dzi_files = len( glob.glob('%s-zoomed/*.dzi' % os.path.basename(output_file.file_path))) print glob.glob('%s-zoomed/*.dzi' % os.path.basename(output_file.file_path)) # Ensure correct number of dzi outputs if output_file.isRGB: # Compute combinatorial number of files expected shape = [ output_file.SizeX, output_file.SizeY, output_file.SizeZ, output_file.SizeC, output_file.SizeT ] # Remove RGB dimension from combinatorial count shape.pop(output_file.RGBDimension) else: # Compute combinatorial number of files expected shape = [ output_file.SizeX, output_file.SizeY, output_file.SizeZ, output_file.SizeC, output_file.SizeT ] total_count = np.prod(shape[2:]) print shape # Make sure same number of files generated assert num_dzi_files == total_count # Clean up os.system('rm %s-zoomed/*.png' % os.path.basename(output_file.file_path)) os.system('rm %s-zoomed/*.dzi' % os.path.basename(output_file.file_path)) mock_s3().stop() mock_ssm().stop() mock_dynamodb2().stop()
def ssm_client(): mock = mock_ssm() mock.start() ssm_client = boto3.client("ssm") yield ssm_client mock.stop()
def ssm(): with mock_ssm(): yield
def fake_ssm(): """Provide a consistent faked SSM context for running an entire test case.""" with mock_ssm(): yield None
def ssm(aws_creds): with moto.mock_ssm(): yield boto3.client('ssm', region_name='eu-west-1')
def ssm(): with mock_ssm(): yield boto3.client("ssm", region_name="us-east-1")
def ssm_client(aws_credentials): """SSM Mock Client""" with mock_ssm(): connection = boto3.client("ssm", region_name="us-east-1") yield connection
def ssm() -> Iterator[None]: """Mock SSM.""" with mock_ssm(): yield
def ssm_client(aws_credentials): with mock_ssm(): yield boto3.client("ssm")
def ssm(): with mock_ssm(): yield