def cluster(): boto3.setup_default_session(region_name='eu-west-1') moto.mock_ecs().start() moto.mock_ec2().start() ec2 = boto3.resource('ec2', region_name='eu-west-1') ecs = boto3.client('ecs', region_name='eu-west-1') test_instance = ec2.create_instances( ImageId="ami-1234abcd", MinCount=1, MaxCount=1, )[0] instance_id_document = json.dumps( ec2_utils.generate_instance_identity_document(test_instance)) cluster = ecs.create_cluster(clusterName='default') ecs.register_container_instance( cluster='default', instanceIdentityDocument=instance_id_document) yield cluster moto.mock_ecs().stop() moto.mock_ec2().stop()
def ecs(): """ Starts moto ecs server. Creates ecs mock resources. Auto uses the setup across all test functions. See https://github.com/spulec/moto/issues/620#issuecomment-224339087 """ mock_ecs().start() conn = boto3.client('ecs', region_name='us-east-1') conn.create_cluster(clusterName='production') conn.create_cluster(clusterName='staging') conn.create_cluster(clusterName='development') for service in ['app2', 'worker1', 'app1']: # Create task definitions revisions for n in range(1, 4): conn.register_task_definition( family='production-' + service, containerDefinitions=create_container_definitions('app1'), ) conn.create_service( cluster='production', serviceName=service, taskDefinition='production-' + service, desiredCount=1 ) yield mock_ecs().stop()
def moto_start(set_region): mock_autoscaling().start() mock_ec2().start() mock_ecs().start() mock_sns().start() mock_sqs().start() yield mock_autoscaling().stop() mock_ec2().stop() mock_ecs().stop() mock_sns().stop() mock_sqs().stop()
def test_create_multiple_clusters(self): with mock_ecs(): clusters = ("test-cluster0", "test-cluster1") logger = "stacker.hooks.ecs" client = boto3.client("ecs", region_name=REGION) response = client.list_clusters() self.assertEqual(len(response["clusterArns"]), 0) for cluster in clusters: with LogCapture(logger) as logs: self.assertTrue( create_clusters( provider=self.provider, context=self.context, clusters=cluster, ) ) logs.check( ( logger, "DEBUG", "Creating ECS cluster: %s" % cluster ) ) response = client.list_clusters() self.assertEqual(len(response["clusterArns"]), 2)
def test_ecs(): cl = EcsCluster(cluster_name="name", task_definitions=["one", "two"]) with mock_ecs(), mock_ec2(): resources.ecs_cluster(cl) ecs = boto3.client("ecs") ran = ecs.run_task(cluster=cl.cluster_name, taskDefinition=cl.task_definitions[0]) print(ran)
def test_fail_create_cluster(self) -> None: """Test fail create cluster.""" with mock_ecs(): client = boto3.client("ecs", region_name=REGION) response = client.list_clusters() self.assertEqual(len(response["clusterArns"]), 0) with self.assertRaises(TypeError): create_clusters(context=self.context) # type: ignore pylint: disable=E
def setUp(self): if use_moto: self.mock_ecs = mock_ecs() self.mock_ecs.start() self.addCleanup(self.mock_ecs.stop) self.client = boto3.client('ecs', region_name='us-east-1', aws_access_key_id='XXXXXXXXXXXX', aws_secret_access_key='XXXXXXXXXXXXX') import luigi.contrib.ecs luigi.contrib.ecs.client = self.client # necessary because contrib/ecs.py uses a module-level client. else: self.client = client # Register the test task definition response = self.client.register_task_definition(**TEST_TASK_DEF) self.arn = response['taskDefinition']['taskDefinitionArn']
def setUp(self): # Use moto to setup a mock AWS service that works inmemory self.mock_ec2 = moto.mock_ec2() self.mock_ec2.start() self.mock_ecs = moto.mock_ecs() self.mock_ecs.start() self.mock_elbv2 = moto.mock_elbv2() self.mock_elbv2.start() self.ecs_client = boto3.client('ecs', region_name='us-east-1') self.ec2_client = boto3.resource('ec2', region_name='us-east-1') self.elb_client = boto3.client('elbv2', region_name='us-east-1') self.cluster, self.service, self.task_definition, self.container_instance_id = self.setup_ecs_resources( ) self.start_task(self.container_instance_id)
def test_fail_create_cluster(self): """Test fail create cluster.""" with mock_ecs(): logger = "runway.cfngin.hooks.ecs" client = boto3.client("ecs", region_name=REGION) response = client.list_clusters() self.assertEqual(len(response["clusterArns"]), 0) with LogCapture(logger) as logs: create_clusters(provider=self.provider, context=self.context) logs.check((logger, "ERROR", "clusters argument required but not provided")) response = client.list_clusters() self.assertEqual(len(response["clusterArns"]), 0)
def test_fail_create_cluster(self): with mock_ecs(): logger = "stacker.hooks.ecs" client = boto3.client("ecs", region_name=REGION) response = client.list_clusters() self.assertEqual(len(response["clusterArns"]), 0) with LogCapture(logger) as logs: create_clusters(provider=self.provider, context=self.context) logs.check( (logger, "ERROR", "setup_clusters hook missing \"clusters\" argument")) response = client.list_clusters() self.assertEqual(len(response["clusterArns"]), 0)
def requireMocking(): """ method which should be called before all other methods in tests. It basically configures our mocking context for stasis """ bucket = moto.mock_s3() bucket.start() sns = moto.mock_sns() sns.start() sqs = moto.mock_sqs() sqs.start() dynamo = moto.mock_dynamodb2() dynamo.start() lamb = moto.mock_lambda() lamb.start() ecs = moto.mock_ecs() ecs.start() ec2 = moto.mock_ec2() ec2.start() ecr = moto.mock_ecr() ecr.start() session = boto3.session.Session() session.client('sns') session.client('s3') dynamodb = boto3.resource('dynamodb') yield sqs.stop() sns.stop() dynamo.stop() lamb.stop() bucket.stop() ecs.stop() ec2.stop() ecr.stop() pass
def test_create_single_cluster(self) -> None: """Test create single cluster.""" with mock_ecs(): logger = "runway.cfngin.hooks.ecs" client = boto3.client("ecs", region_name=REGION) response = client.list_clusters() self.assertEqual(len(response["clusterArns"]), 0) with LogCapture(logger) as logs: cluster = "test-cluster" self.assertTrue( create_clusters(provider=self.provider, context=self.context, clusters=cluster)) logs.check( (logger, "DEBUG", "creating ECS cluster: %s" % cluster)) response = client.list_clusters() self.assertEqual(len(response["clusterArns"]), 1)
def test_create_single_cluster(self): with mock_ecs(): cluster = "test-cluster" logger = "stacker.hooks.ecs" client = boto3.client("ecs", region_name=REGION) response = client.list_clusters() self.assertEqual(len(response["clusterArns"]), 0) with LogCapture(logger) as logs: self.assertTrue( create_clusters(region=REGION, namespace="fake", mappings={}, parameters={}, clusters=cluster)) logs.check( (logger, "DEBUG", "Creating ECS cluster: %s" % cluster)) response = client.list_clusters() self.assertEqual(len(response["clusterArns"]), 1)
def test_create_single_cluster(self): with mock_ecs(): cluster = "test-cluster" logger = "stacker.hooks.ecs" client = boto3.client("ecs", region_name=REGION) response = client.list_clusters() self.assertEqual(len(response["clusterArns"]), 0) with LogCapture(logger) as logs: self.assertTrue( create_clusters( provider=self.provider, context=self.context, clusters=cluster, )) logs.check( (logger, "DEBUG", "Creating ECS cluster: %s" % cluster)) response = client.list_clusters() self.assertEqual(len(response["clusterArns"]), 1)
def cluster(): with moto.mock_ecs(), moto.mock_ec2(): boto3.setup_default_session(region_name="eu-west-1") ec2 = boto3.resource("ec2", region_name="eu-west-1") ecs = boto3.client("ecs", region_name="eu-west-1") known_amis = list(ec2_backend.describe_images()) test_instance = ec2.create_instances(ImageId=known_amis[0].id, MinCount=1, MaxCount=1)[0] instance_id_document = json.dumps( ec2_utils.generate_instance_identity_document(test_instance)) cluster = ecs.create_cluster(clusterName="default") ecs.register_container_instance( cluster="default", instanceIdentityDocument=instance_id_document) yield cluster
def test_create_multiple_clusters(self): """Test create multiple clusters.""" with mock_ecs(): clusters = ("test-cluster0", "test-cluster1") logger = "runway.cfngin.hooks.ecs" client = boto3.client("ecs", region_name=REGION) response = client.list_clusters() self.assertEqual(len(response["clusterArns"]), 0) for cluster in clusters: with LogCapture(logger) as logs: self.assertTrue( create_clusters( provider=self.provider, context=self.context, clusters=cluster, )) logs.check((logger, "DEBUG", "creating ECS cluster: %s" % cluster)) response = client.list_clusters() self.assertEqual(len(response["clusterArns"]), 2)
def cluster(): """Create the mock Airflow cluster. moto doesn't support the Fargate launch type, so we have to pretend like we're going to launch our containers in EC2. There's a little hand waving to make this work. moto comes with some predefined images that seem to work fine. Also see the ``patch_cluster_config`` fixture below. """ C = namedtuple('C', ['name', 'scheduler', 'worker', 'web']) cluster = C('airflow-test', 'airflow-test-scheduler', 'airflow-test-worker', 'airflow-test-web') with mock_ecs(), mock_ec2(): ec2_client = boto3.client('ec2') ec2 = boto3.resource('ec2') ecs = boto3.client('ecs') image = ec2_client.describe_images()['Images'][0] instance = ec2.create_instances(ImageId=image['ImageId'], MinCount=1, MaxCount=1)[0] doc = json.dumps(generate_instance_identity_document(instance)) ecs.create_cluster(clusterName=cluster.name) ecs.register_container_instance(cluster=cluster.name, instanceIdentityDocument=doc) for service in cluster[1:]: ecs.register_task_definition(family=service, containerDefinitions=[]) ecs.create_service(cluster=cluster.name, serviceName=service, desiredCount=1, taskDefinition=f'{service}:1') ecs.update_service(cluster=cluster.name, service=cluster.worker, desiredCount=3) yield cluster
def ecs_client(aws_region): with mock_ecs(): yield boto3.client("ecs", region_name=aws_region)
def mock_ecs_client(): with mock_ecs(): yield boto3.client("ecs", region_name="us-east-2")
def ecs(aws_credentials) -> Type[Boto]: with mock_ecs(): yield Boto(client=boto3.client("ecs", region_name="us-east-1"), resource=None)
def test_lambda_execute_ecs_task(mock=False): """ Test for lambda function that creates ecs tasks :param mock: True: tests using moto lib, False: tests using real cloud env (make sure you have your default creds set) :return: """ ms3 = mock_s3() ml = mock_lambda() mecs = mock_ecs() mlogs = mock_logs() if mock: ms3.start() ml.start() mecs.start() mlogs.start() bucket_name = 'testamolbucket' key_name = 'taskarn.list' # path to file that contains task ARNs script_dir = os.getcwd() lambda_function_filepath = r'..\..\scripts' lambda_function_file = 'LambdaExecuteEcsTask.py' assert os.path.exists( os.path.join(lambda_function_filepath, lambda_function_file)), 'File does not exist!' lambda_function_name = 'testfunction' lambda_execution_role = 'arn:aws:iam::123456789:role/test-amol-role-lambda' lambda_zip_file = '.'.join([lambda_function_file.split('.')[0], 'zip']) os.chdir(lambda_function_filepath) lambda_zip_filepath = r'..\test\outputs\{0}'.format(lambda_zip_file) with zipfile.ZipFile(lambda_zip_filepath, 'w') as myzip: myzip.write(lambda_function_file) _upload_file_to_bucket(lambda_zip_filepath, bucket_name, lambda_zip_file) _create_lambda_function(lambda_function_name, 'LambdaExecuteEcsTask.lambda_handler', lambda_execution_role, lambda_zip_file, bucket_name) s3_event_payload_data = { 'Records': [{ 's3': { 'object': { 'key': 'testkey' }, 'bucket': { 'name': bucket_name } } }] } os.chdir(script_dir) _upload_file_to_bucket(key_name, bucket_name, key_name) if not mock: # lambda function which uses boto lib cannot be mocked, but invoking the lambda can be mocked in any case # Note: This lambda function invocation has already been validated in real AWS environment. _invoke_lambda_function('testfunction', json.dumps(s3_event_payload_data))
def setup(): mock_ec2 = moto.mock_ec2() mock_ecs = moto.mock_ecs() mock_ec2.start() mock_ecs.start() ecs = client('ecs', region_name="us-east-1") ecs.create_cluster(clusterName='UploadEventCluster', tags=[ { 'key': 'Cost', 'value': 'AI' }, ], capacityProviders=[ 'FARGATE', ]) ec2_resource = resource("ec2", "us-east-1") response = ec2_resource.create_instances( ImageId='ami_id', MinCount=1, MaxCount=1, BlockDeviceMappings=[{ "DeviceName": "/dev/sda1", "Ebs": { "VolumeSize": 50, "DeleteOnTermination": False }, }], ) instance_id_document = json.dumps( ec2_utils.generate_instance_identity_document(response[0])) ecs.register_container_instance( cluster='UploadEventCluster', instanceIdentityDocument=instance_id_document) ecs.register_task_definition( family='kairon-task', taskRoleArn='arn:aws:iam::014936247795:role/ecsTaskExecutionRole', networkMode='awsvpc', containerDefinitions=[ { 'name': 'kairon-task', 'image': 'digite/kairon-data-importer:latest', 'cpu': 4096, 'memory': 8192, 'essential': True, 'environment': [{ 'name': 'BOT', 'value': 'demo' }, { 'name': 'USER', 'value': 'demo' }, { 'name': 'IMPORT_DATA', 'value': 'demo' }, { 'name': 'OVERWRITE', 'value': 'demo' }] }, ], requiresCompatibilities=[ 'FARGATE', ], tags=[ { 'key': 'Cost', 'value': 'AI' }, ])
def ecs_client(aws_credentials, region_name): with moto.mock_ecs(): yield boto3.client("ecs", region_name=region_name)