def aws_credentials_api_GET(request):
client = boto3.client('sts')
role_arn, _ = create_s3_role(request.user.email,
str(request.user.profile.sso_id))
# Creating new credentials unfortunately sometimes fails
max_attempts = 3
for i in range(0, 3):
try:
credentials = client.assume_role(
RoleArn=role_arn,
RoleSessionName='s3_access_' +
str(request.user.profile.sso_id),
DurationSeconds=60 * 60,
)['Credentials']
except Exception:
if i == max_attempts - 1:
raise
else:
break
return JsonResponse(
{
'AccessKeyId': credentials['AccessKeyId'],
'SecretAccessKey': credentials['SecretAccessKey'],
'SessionToken': credentials['SessionToken'],
'Expiration': credentials['Expiration'],
},
status=200,
)
def spawn(
user_email_address,
user_sso_id,
public_host_data,
application_instance_id,
spawner_options,
db_credentials,
):
try:
task_arn = None
options = json.loads(spawner_options)
cluster_name = options['CLUSTER_NAME']
container_name = options['CONTAINER_NAME']
definition_arn = options['DEFINITION_ARN']
security_groups = options['SECURITY_GROUPS']
subnets = options['SUBNETS']
cmd = options['CMD'] if 'CMD' in options else []
env = options.get('ENV', {})
port = options['PORT']
s3_sync = options['S3_SYNC'] == 'true'
s3_region = options['S3_REGION']
s3_host = options['S3_HOST']
s3_bucket = options['S3_BUCKET']
database_env = {
f'DATABASE_DSN__{database["memorable_name"]}': f'host={database["db_host"]} port={database["db_port"]} sslmode=require dbname={database["db_name"]} '
f'user={database["db_user"]} password={database["db_password"]}'
for database in db_credentials
}
logger.info('Starting %s', cmd)
role_arn, s3_prefix = create_s3_role(user_email_address, user_sso_id)
s3_env = {
'S3_PREFIX': s3_prefix,
'S3_REGION': s3_region,
'S3_HOST': s3_host,
'S3_BUCKET': s3_bucket,
}
application_instance = ApplicationInstance.objects.get(
id=application_instance_id
)
# If CONTAINER_TAG_PATTERN is given, the data from the public_host is used to fill
# CONTAINER_TAG_PATTERN to work out what Docker tag should be launched
try:
tag = options['CONTAINER_TAG_PATTERN']
except KeyError:
definition_arn_with_image = definition_arn
else:
# Not robust, but the pattern is specified by config rather than user-provided
for key, value in public_host_data.items():
tag = tag.replace(f'<{key}>', value)
definition_arn_with_image = _fargate_task_definition_with_tag(
definition_arn, container_name, tag
)
# If memory or cpu are given, create a new task definition.
cpu = application_instance.cpu
memory = application_instance.memory
cpu_or_mem = cpu is not None or memory is not None
definition_arn_with_cpu_memory_image = (
_fargate_task_definition_with_cpu_memory(
definition_arn_with_image, cpu, memory
)
if cpu_or_mem
else definition_arn_with_image
)
for i in range(0, 10):
# Sometimes there is an error assuming the new role: both IAM and ECS are
# eventually consistent
try:
start_task_response = _fargate_task_run(
role_arn,
cluster_name,
container_name,
definition_arn_with_cpu_memory_image,
security_groups,
subnets,
cmd,
{**s3_env, **database_env, **env},
s3_sync,
)
except ClientError:
gevent.sleep(3)
if i == 9:
raise
else:
break
task = (
start_task_response['tasks'][0]
if 'tasks' in start_task_response
else start_task_response['task']
)
task_arn = task['taskArn']
application_instance.spawner_application_instance_id = json.dumps(
{'task_arn': task_arn}
)
application_instance.spawner_created_at = task['createdAt']
application_instance.spawner_cpu = task['cpu']
application_instance.spawner_memory = task['memory']
application_instance.save(
update_fields=[
'spawner_application_instance_id',
'spawner_created_at',
'spawner_cpu',
'spawner_memory',
]
)
application_instance.refresh_from_db()
if application_instance.state == 'STOPPED':
raise Exception('Application set to stopped before spawning complete')
for _ in range(0, 60):
ip_address = _fargate_task_ip(options['CLUSTER_NAME'], task_arn)
if ip_address:
application_instance.proxy_url = f'http://{ip_address}:{port}'
application_instance.save(update_fields=['proxy_url'])
return
gevent.sleep(3)
raise Exception('Spawner timed out before finding ip address')
except Exception:
logger.exception('FARGATE %s %s', application_instance_id, spawner_options)
if task_arn:
_fargate_task_stop(cluster_name, task_arn)
def spawn(
user_email_address,
user_sso_id,
tag,
application_instance_id,
spawner_options,
db_credentials,
app_schema,
):
try:
pipeline_id = None
task_arn = None
options = json.loads(spawner_options)
cluster_name = options['CLUSTER_NAME']
container_name = options['CONTAINER_NAME']
definition_arn = options['DEFINITION_ARN']
ecr_repository_name = options.get('ECR_REPOSITORY_NAME')
security_groups = options['SECURITY_GROUPS']
subnets = options['SUBNETS']
cmd = options['CMD'] if 'CMD' in options else []
env = options.get('ENV', {})
port = options['PORT']
s3_sync = options['S3_SYNC'] == 'true'
s3_region = options['S3_REGION']
s3_host = options['S3_HOST']
s3_bucket = options['S3_BUCKET']
database_env = {
f'DATABASE_DSN__{database["memorable_name"]}':
f'host={database["db_host"]} '
f'port={database["db_port"]} sslmode=require dbname={database["db_name"]} '
f'user={database["db_user"]} password={database["db_password"]}'
for database in db_credentials
}
schema_env = {'APP_SCHEMA': app_schema}
logger.info('Starting %s', cmd)
role_arn, s3_prefix = create_s3_role(user_email_address,
user_sso_id)
s3_env = {
'S3_PREFIX': s3_prefix,
'S3_REGION': s3_region,
'S3_HOST': s3_host,
'S3_BUCKET': s3_bucket,
}
application_instance = ApplicationInstance.objects.get(
id=application_instance_id)
# Build tag if we can and it doesn't already exist
if (ecr_repository_name and tag and application_instance.commit_id
and
application_instance.application_template.gitlab_project_id
and not _ecr_tag_exists(ecr_repository_name, tag)):
pipeline = gitlab_api_v4_ecr_pipeline_trigger(
ECR_PROJECT_ID,
application_instance.application_template.
gitlab_project_id,
application_instance.commit_id,
ecr_repository_name,
tag,
)
if 'id' not in pipeline:
raise Exception(
'Unable to start pipeline: {}'.format(pipeline))
pipeline_id = pipeline['id']
application_instance.spawner_application_instance_id = json.dumps(
{
'pipeline_id': pipeline_id,
'task_arn': None
})
application_instance.save(
update_fields=['spawner_application_instance_id'])
for _ in range(0, 600):
gevent.sleep(3)
pipeline = _gitlab_ecr_pipeline_get(pipeline_id)
logger.info('Fetched pipeline %s', pipeline)
if (pipeline['status'] not in RUNNING_PIPELINE_STATUSES
and pipeline['status']
not in SUCCESS_PIPELINE_STATUSES):
raise Exception('Pipeline failed {}'.format(pipeline))
if pipeline['status'] in SUCCESS_PIPELINE_STATUSES:
break
else:
logger.error('Pipeline took too long, cancelling: %s',
pipeline)
_gitlab_ecr_pipeline_cancel(pipeline_id)
raise Exception(
'Pipeline {} took too long'.format(pipeline))
# Tag is given, create a new task definition
definition_arn_with_image = (_fargate_task_definition_with_tag(
definition_arn, container_name, tag)
if tag else definition_arn)
for i in range(0, 10):
# Sometimes there is an error assuming the new role: both IAM and ECS are
# eventually consistent
try:
start_task_response = _fargate_task_run(
role_arn,
cluster_name,
container_name,
definition_arn_with_image,
security_groups,
subnets,
application_instance.cpu,
application_instance.memory,
cmd,
{
**s3_env,
**database_env,
**schema_env,
**env
},
s3_sync,
)
except ClientError:
gevent.sleep(3)
if i == 9:
raise
else:
break
task = (start_task_response['tasks'][0] if 'tasks'
in start_task_response else start_task_response['task'])
task_arn = task['taskArn']
application_instance.spawner_application_instance_id = json.dumps({
'pipeline_id':
pipeline_id,
'task_arn':
task_arn
})
application_instance.spawner_created_at = task['createdAt']
application_instance.spawner_cpu = task['cpu']
application_instance.spawner_memory = task['memory']
application_instance.save(update_fields=[
'spawner_application_instance_id',
'spawner_created_at',
'spawner_cpu',
'spawner_memory',
])
application_instance.refresh_from_db()
if application_instance.state == 'STOPPED':
raise Exception(
'Application set to stopped before spawning complete')
for _ in range(0, 60):
ip_address = _fargate_task_ip(options['CLUSTER_NAME'],
task_arn)
if ip_address:
application_instance.proxy_url = f'http://{ip_address}:{port}'
application_instance.save(update_fields=['proxy_url'])
return
gevent.sleep(3)
raise Exception('Spawner timed out before finding ip address')
except Exception:
logger.exception(
'Spawning %s %s %s',
pipeline_id,
application_instance_id,
spawner_options,
)
if task_arn:
_fargate_task_stop(cluster_name, task_arn)
if pipeline_id:
_gitlab_ecr_pipeline_cancel(pipeline_id)