def cluster():
boto3.setup_default_session(region_name='eu-west-1')
moto.mock_ecs().start()
moto.mock_ec2().start()
ec2 = boto3.resource('ec2', region_name='eu-west-1')
ecs = boto3.client('ecs', region_name='eu-west-1')
test_instance = ec2.create_instances(
ImageId="ami-1234abcd",
MinCount=1,
MaxCount=1,
)[0]
instance_id_document = json.dumps(
ec2_utils.generate_instance_identity_document(test_instance))
cluster = ecs.create_cluster(clusterName='default')
ecs.register_container_instance(
cluster='default',
instanceIdentityDocument=instance_id_document)
yield cluster
moto.mock_ecs().stop()
moto.mock_ec2().stop()
def ecs():
"""
Starts moto ecs server. Creates ecs mock resources. Auto uses the setup
across all test functions.
See https://github.com/spulec/moto/issues/620#issuecomment-224339087
"""
mock_ecs().start()
conn = boto3.client('ecs', region_name='us-east-1')
conn.create_cluster(clusterName='production')
conn.create_cluster(clusterName='staging')
conn.create_cluster(clusterName='development')
for service in ['app2', 'worker1', 'app1']:
# Create task definitions revisions
for n in range(1, 4):
conn.register_task_definition(
family='production-' + service,
containerDefinitions=create_container_definitions('app1'),
)
conn.create_service(
cluster='production',
serviceName=service,
taskDefinition='production-' + service,
desiredCount=1
)
yield
mock_ecs().stop()
def moto_start(set_region):
mock_autoscaling().start()
mock_ec2().start()
mock_ecs().start()
mock_sns().start()
mock_sqs().start()
yield
mock_autoscaling().stop()
mock_ec2().stop()
mock_ecs().stop()
mock_sns().stop()
mock_sqs().stop()
def test_create_multiple_clusters(self):
with mock_ecs():
clusters = ("test-cluster0", "test-cluster1")
logger = "stacker.hooks.ecs"
client = boto3.client("ecs", region_name=REGION)
response = client.list_clusters()
self.assertEqual(len(response["clusterArns"]), 0)
for cluster in clusters:
with LogCapture(logger) as logs:
self.assertTrue(
create_clusters(
provider=self.provider,
context=self.context,
clusters=cluster,
)
)
logs.check(
(
logger,
"DEBUG",
"Creating ECS cluster: %s" % cluster
)
)
response = client.list_clusters()
self.assertEqual(len(response["clusterArns"]), 2)
def test_ecs():
cl = EcsCluster(cluster_name="name", task_definitions=["one", "two"])
with mock_ecs(), mock_ec2():
resources.ecs_cluster(cl)
ecs = boto3.client("ecs")
ran = ecs.run_task(cluster=cl.cluster_name,
taskDefinition=cl.task_definitions[0])
print(ran)
def test_fail_create_cluster(self) -> None:
"""Test fail create cluster."""
with mock_ecs():
client = boto3.client("ecs", region_name=REGION)
response = client.list_clusters()
self.assertEqual(len(response["clusterArns"]), 0)
with self.assertRaises(TypeError):
create_clusters(context=self.context) # type: ignore pylint: disable=E
def setUp(self):
if use_moto:
self.mock_ecs = mock_ecs()
self.mock_ecs.start()
self.addCleanup(self.mock_ecs.stop)
self.client = boto3.client('ecs', region_name='us-east-1', aws_access_key_id='XXXXXXXXXXXX', aws_secret_access_key='XXXXXXXXXXXXX')
import luigi.contrib.ecs
luigi.contrib.ecs.client = self.client # necessary because contrib/ecs.py uses a module-level client.
else:
self.client = client
# Register the test task definition
response = self.client.register_task_definition(**TEST_TASK_DEF)
self.arn = response['taskDefinition']['taskDefinitionArn']
def setUp(self):
# Use moto to setup a mock AWS service that works inmemory
self.mock_ec2 = moto.mock_ec2()
self.mock_ec2.start()
self.mock_ecs = moto.mock_ecs()
self.mock_ecs.start()
self.mock_elbv2 = moto.mock_elbv2()
self.mock_elbv2.start()
self.ecs_client = boto3.client('ecs', region_name='us-east-1')
self.ec2_client = boto3.resource('ec2', region_name='us-east-1')
self.elb_client = boto3.client('elbv2', region_name='us-east-1')
self.cluster, self.service, self.task_definition, self.container_instance_id = self.setup_ecs_resources(
)
self.start_task(self.container_instance_id)
def test_fail_create_cluster(self):
"""Test fail create cluster."""
with mock_ecs():
logger = "runway.cfngin.hooks.ecs"
client = boto3.client("ecs", region_name=REGION)
response = client.list_clusters()
self.assertEqual(len(response["clusterArns"]), 0)
with LogCapture(logger) as logs:
create_clusters(provider=self.provider, context=self.context)
logs.check((logger, "ERROR",
"clusters argument required but not provided"))
response = client.list_clusters()
self.assertEqual(len(response["clusterArns"]), 0)
def test_fail_create_cluster(self):
with mock_ecs():
logger = "stacker.hooks.ecs"
client = boto3.client("ecs", region_name=REGION)
response = client.list_clusters()
self.assertEqual(len(response["clusterArns"]), 0)
with LogCapture(logger) as logs:
create_clusters(provider=self.provider, context=self.context)
logs.check(
(logger, "ERROR",
"setup_clusters hook missing \"clusters\" argument"))
response = client.list_clusters()
self.assertEqual(len(response["clusterArns"]), 0)
def requireMocking():
"""
method which should be called before all other methods in tests. It basically configures our
mocking context for stasis
"""
bucket = moto.mock_s3()
bucket.start()
sns = moto.mock_sns()
sns.start()
sqs = moto.mock_sqs()
sqs.start()
dynamo = moto.mock_dynamodb2()
dynamo.start()
lamb = moto.mock_lambda()
lamb.start()
ecs = moto.mock_ecs()
ecs.start()
ec2 = moto.mock_ec2()
ec2.start()
ecr = moto.mock_ecr()
ecr.start()
session = boto3.session.Session()
session.client('sns')
session.client('s3')
dynamodb = boto3.resource('dynamodb')
yield
sqs.stop()
sns.stop()
dynamo.stop()
lamb.stop()
bucket.stop()
ecs.stop()
ec2.stop()
ecr.stop()
pass
def test_create_single_cluster(self) -> None:
"""Test create single cluster."""
with mock_ecs():
logger = "runway.cfngin.hooks.ecs"
client = boto3.client("ecs", region_name=REGION)
response = client.list_clusters()
self.assertEqual(len(response["clusterArns"]), 0)
with LogCapture(logger) as logs:
cluster = "test-cluster"
self.assertTrue(
create_clusters(provider=self.provider,
context=self.context,
clusters=cluster))
logs.check(
(logger, "DEBUG", "creating ECS cluster: %s" % cluster))
response = client.list_clusters()
self.assertEqual(len(response["clusterArns"]), 1)
def test_create_single_cluster(self):
with mock_ecs():
cluster = "test-cluster"
logger = "stacker.hooks.ecs"
client = boto3.client("ecs", region_name=REGION)
response = client.list_clusters()
self.assertEqual(len(response["clusterArns"]), 0)
with LogCapture(logger) as logs:
self.assertTrue(
create_clusters(region=REGION,
namespace="fake",
mappings={},
parameters={},
clusters=cluster))
logs.check(
(logger, "DEBUG", "Creating ECS cluster: %s" % cluster))
response = client.list_clusters()
self.assertEqual(len(response["clusterArns"]), 1)
def test_create_single_cluster(self):
with mock_ecs():
cluster = "test-cluster"
logger = "stacker.hooks.ecs"
client = boto3.client("ecs", region_name=REGION)
response = client.list_clusters()
self.assertEqual(len(response["clusterArns"]), 0)
with LogCapture(logger) as logs:
self.assertTrue(
create_clusters(
provider=self.provider,
context=self.context,
clusters=cluster,
))
logs.check(
(logger, "DEBUG", "Creating ECS cluster: %s" % cluster))
response = client.list_clusters()
self.assertEqual(len(response["clusterArns"]), 1)
def cluster():
with moto.mock_ecs(), moto.mock_ec2():
boto3.setup_default_session(region_name="eu-west-1")
ec2 = boto3.resource("ec2", region_name="eu-west-1")
ecs = boto3.client("ecs", region_name="eu-west-1")
known_amis = list(ec2_backend.describe_images())
test_instance = ec2.create_instances(ImageId=known_amis[0].id,
MinCount=1,
MaxCount=1)[0]
instance_id_document = json.dumps(
ec2_utils.generate_instance_identity_document(test_instance))
cluster = ecs.create_cluster(clusterName="default")
ecs.register_container_instance(
cluster="default", instanceIdentityDocument=instance_id_document)
yield cluster
def test_create_multiple_clusters(self):
"""Test create multiple clusters."""
with mock_ecs():
clusters = ("test-cluster0", "test-cluster1")
logger = "runway.cfngin.hooks.ecs"
client = boto3.client("ecs", region_name=REGION)
response = client.list_clusters()
self.assertEqual(len(response["clusterArns"]), 0)
for cluster in clusters:
with LogCapture(logger) as logs:
self.assertTrue(
create_clusters(
provider=self.provider,
context=self.context,
clusters=cluster,
))
logs.check((logger, "DEBUG",
"creating ECS cluster: %s" % cluster))
response = client.list_clusters()
self.assertEqual(len(response["clusterArns"]), 2)
def cluster():
"""Create the mock Airflow cluster.
moto doesn't support the Fargate launch type, so we have to pretend
like we're going to launch our containers in EC2. There's a little
hand waving to make this work. moto comes with some predefined images
that seem to work fine.
Also see the ``patch_cluster_config`` fixture below.
"""
C = namedtuple('C', ['name', 'scheduler', 'worker', 'web'])
cluster = C('airflow-test', 'airflow-test-scheduler',
'airflow-test-worker', 'airflow-test-web')
with mock_ecs(), mock_ec2():
ec2_client = boto3.client('ec2')
ec2 = boto3.resource('ec2')
ecs = boto3.client('ecs')
image = ec2_client.describe_images()['Images'][0]
instance = ec2.create_instances(ImageId=image['ImageId'],
MinCount=1,
MaxCount=1)[0]
doc = json.dumps(generate_instance_identity_document(instance))
ecs.create_cluster(clusterName=cluster.name)
ecs.register_container_instance(cluster=cluster.name,
instanceIdentityDocument=doc)
for service in cluster[1:]:
ecs.register_task_definition(family=service,
containerDefinitions=[])
ecs.create_service(cluster=cluster.name,
serviceName=service,
desiredCount=1,
taskDefinition=f'{service}:1')
ecs.update_service(cluster=cluster.name,
service=cluster.worker,
desiredCount=3)
yield cluster
def ecs_client(aws_region):
with mock_ecs():
yield boto3.client("ecs", region_name=aws_region)
def mock_ecs_client():
with mock_ecs():
yield boto3.client("ecs", region_name="us-east-2")
def ecs(aws_credentials) -> Type[Boto]:
with mock_ecs():
yield Boto(client=boto3.client("ecs", region_name="us-east-1"),
resource=None)
def test_lambda_execute_ecs_task(mock=False):
"""
Test for lambda function that creates ecs tasks
:param mock: True: tests using moto lib, False: tests using real cloud env (make sure you have your default creds
set)
:return:
"""
ms3 = mock_s3()
ml = mock_lambda()
mecs = mock_ecs()
mlogs = mock_logs()
if mock:
ms3.start()
ml.start()
mecs.start()
mlogs.start()
bucket_name = 'testamolbucket'
key_name = 'taskarn.list' # path to file that contains task ARNs
script_dir = os.getcwd()
lambda_function_filepath = r'..\..\scripts'
lambda_function_file = 'LambdaExecuteEcsTask.py'
assert os.path.exists(
os.path.join(lambda_function_filepath,
lambda_function_file)), 'File does not exist!'
lambda_function_name = 'testfunction'
lambda_execution_role = 'arn:aws:iam::123456789:role/test-amol-role-lambda'
lambda_zip_file = '.'.join([lambda_function_file.split('.')[0], 'zip'])
os.chdir(lambda_function_filepath)
lambda_zip_filepath = r'..\test\outputs\{0}'.format(lambda_zip_file)
with zipfile.ZipFile(lambda_zip_filepath, 'w') as myzip:
myzip.write(lambda_function_file)
_upload_file_to_bucket(lambda_zip_filepath, bucket_name, lambda_zip_file)
_create_lambda_function(lambda_function_name,
'LambdaExecuteEcsTask.lambda_handler',
lambda_execution_role, lambda_zip_file,
bucket_name)
s3_event_payload_data = {
'Records': [{
's3': {
'object': {
'key': 'testkey'
},
'bucket': {
'name': bucket_name
}
}
}]
}
os.chdir(script_dir)
_upload_file_to_bucket(key_name, bucket_name, key_name)
if not mock:
# lambda function which uses boto lib cannot be mocked, but invoking the lambda can be mocked in any case
# Note: This lambda function invocation has already been validated in real AWS environment.
_invoke_lambda_function('testfunction',
json.dumps(s3_event_payload_data))
def setup():
mock_ec2 = moto.mock_ec2()
mock_ecs = moto.mock_ecs()
mock_ec2.start()
mock_ecs.start()
ecs = client('ecs', region_name="us-east-1")
ecs.create_cluster(clusterName='UploadEventCluster',
tags=[
{
'key': 'Cost',
'value': 'AI'
},
],
capacityProviders=[
'FARGATE',
])
ec2_resource = resource("ec2", "us-east-1")
response = ec2_resource.create_instances(
ImageId='ami_id',
MinCount=1,
MaxCount=1,
BlockDeviceMappings=[{
"DeviceName": "/dev/sda1",
"Ebs": {
"VolumeSize": 50,
"DeleteOnTermination": False
},
}],
)
instance_id_document = json.dumps(
ec2_utils.generate_instance_identity_document(response[0]))
ecs.register_container_instance(
cluster='UploadEventCluster',
instanceIdentityDocument=instance_id_document)
ecs.register_task_definition(
family='kairon-task',
taskRoleArn='arn:aws:iam::014936247795:role/ecsTaskExecutionRole',
networkMode='awsvpc',
containerDefinitions=[
{
'name':
'kairon-task',
'image':
'digite/kairon-data-importer:latest',
'cpu':
4096,
'memory':
8192,
'essential':
True,
'environment': [{
'name': 'BOT',
'value': 'demo'
}, {
'name': 'USER',
'value': 'demo'
}, {
'name': 'IMPORT_DATA',
'value': 'demo'
}, {
'name': 'OVERWRITE',
'value': 'demo'
}]
},
],
requiresCompatibilities=[
'FARGATE',
],
tags=[
{
'key': 'Cost',
'value': 'AI'
},
])
def ecs_client(aws_credentials, region_name):
with moto.mock_ecs():
yield boto3.client("ecs", region_name=region_name)
