def deduct_credit(user_name):
# For expServer to invoke - it only has access to deduct credit.
# No other configurable params - to avoid potential securty issue with juypter
# To-do auth logic to make sure the caller is updating his credit only
user_info = get_user_info(dynamodb_client, user_name, user_table)
if 'Item' in user_info and 'cfn_id' in user_info['Item']:
cfn_id = user_info['Item']['cfn_id']['S']
else:
return _make_reply(
_http_status(user_info), {
'status': Status.NO_STACK,
'error': '%s does not have a stack' % user_name
})
stack_info = get_stack_info(cfn_client, cfn_id)
if 'errorCode' in stack_info:
return _make_reply(
stack_info['errorCode'], {
'status': Status.STACK_NOT_FOUND,
'error': 'Stack %s not found' % cfn_id
})
elif 'size' not in stack_info or 'type' not in stack_info or stack_info[
'size'] == 0:
return _make_reply(200, {
'status': Status.NO_RUNNING_CLUSTER,
'error': 'No running cluster'
})
credit_change = str(-1 * get_price(stack_info['type'], stack_info['size']))
return update_credit(user_name, credit_change)
def stop_cluster(user_name):
user_info = get_user_info(dynamodb_client, user_name, user_table)
if 'Item' not in user_info:
return _make_reply(_http_status(user_info), {
"status": Status.USER_NOT_FOUND,
"error": "%s does not exist" % user_name
})
if 'cfn_id' not in user_info['Item']:
return _make_reply(_http_status(user_info), {
'status': Status.NO_STACK,
'error': '%s does not have a stack' % user_name
})
cfn_id = user_info['Item']['cfn_id']['S']
stack_params = get_stack_params(cfn_client, cfn_id)
for param in stack_params:
if param['ParameterKey'] == 'ClusterSize':
param.update(ParameterValue='0')
if param['ParameterKey'] == 'AdminPassword':
param.pop('ParameterValue', None)
param['UsePreviousValue']=True
response = cfn_client.update_stack(
StackName = cfn_id,
UsePreviousTemplate = True,
Parameters = stack_params,
Capabilities=IamCapabilities,
RoleARN=cfn_role_arn
)
return _make_reply(_http_status(response), {'status': Status.OK})
def lambda_handler(event, context):
try:
path = event['path']
headers = event['headers']
headers_origin = '*'
headers_cookies = None
for key, headerLine in headers.items():
if (key.lower() == "origin"):
headers_origin = headerLine
if (key.lower() == "cookie"):
headers_cookies = headerLine
if headers_origin == '*':
data = json.loads(event['body'])
if 'username' not in data or \
'instanceId' not in data or \
validate_user_instance(ec2_client,
data['username'],
data['instanceId']) != True:
return _make_reply(401, {
'status': Status.AUTH_ERROR,
'error': 'Authentication failed'
}, headers_origin)
elif re.match('^https://\w+.'+domain, headers_origin, re.M|re.I):
if (event['httpMethod'] == 'OPTIONS'):
return _make_options_reply(200, headers_origin)
data = json.loads(event['body'])
credential, username = check_user_credential(dynamodb_client,
session_table,
creds_table,
headers_cookies)
if credential is None or username != data['username']:
return _make_reply(401, {
'status': Status.AUTH_ERROR,
'error': "Authentication Failed"
}, headers_origin)
else:
return _make_reply(403, "Forbidden", headers_origin)
if path == '/cluster/start':
reply = start_cluster(data['username'], data['clusterParams'])
elif path == '/cluster/stop':
reply = stop_cluster(data['username'])
elif path == '/cluster/get':
reply = get_cluster(data['username'])
else:
reply = _make_reply(400, "Invalid endpoint: %s" % path)
except Exception as e:
traceback.print_exc()
reply = _make_reply(400, "Exception has occurred: {}".format(e))
reply = _replace_headers_origin(reply, headers_origin)
return reply
def lambda_handler(event, context):
try:
path = event['path']
if path == '/config':
reply = _make_reply(200, {
'config': configResult
})
else:
reply = _make_reply(400, "Invalid endpoint: %s" % path)
except Exception as e:
traceback.print_exc()
reply = _make_reply(400, "Exception has occurred: {}".format(e))
return reply
def delete_file(delete_params):
user_name = delete_params['username']
file_name = delete_params['fileName']
bucket_resp = get_bucket(user_name)
if type(bucket_resp) != str:
return bucket_resp
s3_client = boto3.client('s3', region_name=region)
s3_client.delete_object(Bucket=bucket_resp, Key=file_name)
return _make_reply(200, {'status': Status.OK})
def lambda_handler(event, context):
try:
path = event['path']
headers = event['headers']
headers_origin = '*'
headers_cookies = None
for key, headerLine in headers.items():
if (key.lower() == "origin"):
headers_origin = headerLine
if (key.lower() == "cookie"):
headers_cookies = headerLine
if re.match('^https://\w+.' + domain, headers_origin, re.M | re.I):
if (event['httpMethod'] == 'OPTIONS'):
return _make_options_reply(200, headers_origin)
data = json.loads(event['body'])
credential, username = check_user_credential(
dynamodb_client, session_table, creds_table, headers_cookies)
if credential == None or username != data['username']:
return _make_reply(401, {
'status': Status.AUTH_ERROR,
'error': "Authentication Failed"
}, headers_origin)
else:
return _make_reply(403, {'error': "Forbidden"}, headers_origin)
if path == '/s3/uploadurl':
reply = get_upload_file_url(data)
elif path == '/s3/delete':
reply = delete_file(data)
elif path == '/s3/describe':
reply = bucket_info(data['username'])
elif path == '/s3/corsconfig':
reply = put_cors_config(data['username'])
else:
reply = _make_reply(400, {'error': "Invalid endpoint: %s" % path})
except Exception as e:
traceback.print_exc()
reply = _make_reply(400,
{'error': "Exception has occurred: {}".format(e)})
reply = _replace_headers_origin(reply, headers_origin)
return reply
def get_bucket(user_name):
response = get_user_info(dynamodb_client, user_name, user_table)
if 'Item' not in response:
return _make_reply(
_http_status(response), {
'status': Status.USER_NOT_FOUND,
'error': "%s does not exist" % user_name
})
cfn_id = response['Item']['cfn_id']['S']
stack_info = get_stack_info(cfn_client, cfn_id)
if 'errorCode' in stack_info:
return _make_reply(
stack_info['errorCode'], {
'status': Status.STACK_NOT_FOUND,
'error': 'Stack %s not found' % cfn_id
})
if 's3_url' not in stack_info:
return _make_reply(
200, {
'status': Status.S3_BUCKET_NOT_EXIST,
'error': 'Cloud not find s3 given stack %s' % cfn_id
})
return stack_info['s3_url']
def get_credit(user_name):
response = dynamodb_client.query(
TableName=billing_table,
ScanIndexForward=True,
ProjectionExpression='credit_change',
KeyConditionExpression='user_name = :uname',
ExpressionAttributeValues={':uname': {
'S': user_name
}})
credit = 0
if 'Items' in response and len(response['Items']) > 0:
for row in response['Items']:
credit += float(row['credit_change']['N'])
else:
return _make_reply(
_http_status(response), {
'status': Status.NO_CREDIT_HISTORY,
'error': 'No credit history for user: %s' % user_name
})
while 'LastEvaluatedKey' in response:
response = dynamodb_client.query(
TableName=billing_table,
ScanIndexForward=True,
ExclusiveStartKey=response['LastEvaluatedKey'],
ProjectionExpression='credit_change',
KeyConditionExpression='user_name = :uname',
ExpressionAttributeValues={':uname': {
'S': user_name
}})
if 'Items' in response and len(response['Items']) > 0:
for row in response['Items']:
credit += float(row['credit_change']['N'])
return _make_reply(_http_status(response), {
'status': Status.OK,
'credits': credit,
})
def update_credit(user_name, credit_change):
transaction = {
'user_name': {
'S': user_name
},
'timestamp': {
'N': str(round(time.time() * 1000))
},
'credit_change': {
'N': credit_change
}
}
response = dynamodb_client.put_item(TableName=billing_table,
Item=transaction)
return _make_reply(_http_status(response), {'status': Status.OK})
def get_upload_file_url(upload_params):
user_name = upload_params['username']
file_name = upload_params['fileName']
fields = upload_params['fields']
conditions = upload_params['conditions']
bucket_resp = get_bucket(user_name)
if type(bucket_resp) != str:
return bucket_resp
expiration = 3600
s3_client = boto3.client('s3', region_name=region)
res_dict = s3_client.generate_presigned_post(bucket_resp,
file_name,
Fields=fields,
Conditions=conditions,
ExpiresIn=expiration)
return _make_reply(200, {'status': Status.OK, 'responseDict': res_dict})
def put_cors_config(user_name):
# Hard code it here
cors_config = {
"CORSRules": [{
"AllowedHeaders": ["*"],
"AllowedMethods": ["POST"],
"AllowedOrigins": ["*"],
"MaxAgeSeconds": 3000
}]
}
bucket_resp = get_bucket(user_name)
if type(bucket_resp) != str:
return bucket_resp
s3_client = boto3.client('s3', region_name=region)
s3_client.put_bucket_cors(Bucket=bucket_resp,
CORSConfiguration=cors_config)
return _make_reply(200, {'status': Status.OK})
def get_cluster(user_name):
user_info = get_user_info(dynamodb_client, user_name, user_table)
if 'Item' not in user_info:
response = init_user(dynamodb_client, user_name, default_credit, user_table, billing_table)
return _make_reply(_http_status(response), {
'status': Status.OK,
'isPending': False
})
elif 'cfn_id' not in user_info['Item']:
return _make_reply(200, {
'status': Status.OK,
'isPending': False
})
cfn_id = user_info['Item']['cfn_id']['S']
stack_info = get_stack_info(cfn_client, cfn_id)
if 'errorCode' in stack_info:
if _http_status(reset_user_cfn(dynamodb_client, user_name, user_table)) != 200:
error = 'Stack %s not found and failed to clean user table' % cfn_id
else:
error = 'Stack %s not found' % cfn_id
return _make_reply(stack_info['errorCode'], {
'status': Status.STACK_NOT_FOUND,
'error': error
})
# To-do more detailed stack status
else:
# in progresss
if stack_info['stack_status'].endswith('IN_PROGRESS'):
return _make_reply(200, {
'status': Status.OK,
'isPending': True,
'isStarting': False if stack_info['size'] == 0 else True
})
#updated completed, then check cluster status
elif stack_info['stack_status'] == 'UPDATE_COMPLETE':
cluster_status = check_cluster_status(user_name, stack_info)
credit_change = str(-1 * get_price(stack_info['type'], stack_info['size']))
return _make_reply(200, cluster_status)
#error(more detailed failure check)
else:
return _make_reply(200, {
'status': Status.STACK_ERROR,
'error': 'Stack has error: %s' % stack_info['stack_status'],
})
def start_cluster(user_name, cluster_params):
# if the user has a cfn stack
response = get_user_info(dynamodb_client, user_name, user_table)
if 'Item' not in response:
return _make_reply(_http_status(response), {
'status': Status.USER_NOT_FOUND,
'error': '%s does not exist' % user_name
})
user_info = response['Item']
parameters = []
is_new = False # whether this is a new user
tags = cfn_id = cluster_type = response = None
if 'cfn_id' in user_info:
cfn_id = user_info['cfn_id']['S']
else:
#or we give him an available one
stack_info = get_available_stack(user_name)
if stack_info is None:
return _make_reply(200, {
'status': Status.NO_AVAILABLE_STACK,
'error': 'No available stack at this moment'
})
cfn_id = stack_info['cfn_id']
tags = stack_info['tags']
is_new = True
if 'type' in cluster_params and cluster_params['type'] in cluster_type_table:
cluster_type = cluster_type_table[cluster_params['type']]
else:
# default to use 'XS'
cluster_type = cluster_type_table['XS']
stack_params = get_stack_params(cfn_client, cfn_id)
for param in stack_params:
if param['ParameterKey'] == 'ClusterSize':
param.update(ParameterValue=cluster_type['clusterSize'])
if param['ParameterKey'] == 'InstanceType':
param.update(ParameterValue=cluster_type['instanceType'])
if param['ParameterKey'] == 'ImageId' and 'AMI' in cluster_params:
param.update(ParameterValue=cluster_params['AMI'])
if param['ParameterKey'] == 'AdminPassword':
param.pop('ParameterValue', None)
param['UsePreviousValue']=True
is_test_cluster = check_test_cluster(cfn_id)
if is_test_cluster is None:
error = 'Stack %s not found' % cfn_id
return _make_reply(200, {
'status': Status.STACK_NOT_FOUND,
'error': error
})
elif is_test_cluster:
if is_new == False:
response = cfn_client.update_stack(
StackName=cfn_id,
UsePreviousTemplate=True,
Parameters=stack_params,
Capabilities=IamCapabilities,
RoleARN=cfn_role_arn
)
else:
cfn_client.update_stack(
StackName=cfn_id,
UsePreviousTemplate=True,
Parameters=stack_params ,
Capabilities=IamCapabilities,
RoleARN=cfn_role_arn,
Tags=tags
)
updates = {
'cfn_id': {
'S': cfn_id
}
}
response = update_user_info(dynamodb_client, user_info, updates, user_table)
else:
template = ssm_client.get_parameter(Name=ssm_key)['Parameter']['Value']
if is_new == False:
response = cfn_client.update_stack(
StackName=cfn_id,
TemplateURL=template,
UsePreviousTemplate=False,
Parameters=stack_params,
Capabilities=IamCapabilities,
RoleARN=cfn_role_arn
)
else:
cfn_client.update_stack(
StackName=cfn_id,
TemplateURL=template,
UsePreviousTemplate=False,
Parameters=stack_params ,
Capabilities=IamCapabilities,
RoleARN=cfn_role_arn,
Tags=tags
)
updates = {
'cfn_id': {
'S': cfn_id
}
}
response = update_user_info(dynamodb_client, user_info, updates, user_table)
return _make_reply(_http_status(response), {
'status': Status.OK
})
def bucket_info(user_name):
bucket_resp = get_bucket(user_name)
if type(bucket_resp) != str:
return bucket_resp
return _make_reply(200, {'status': Status.OK, 'bucketName': bucket_resp})