def get_metric_statistics(Namespace, MetricName, Dimensions,
Period=60, StartTime=None, EndTime=None, Statistics=None):
if not StartTime:
StartTime = datetime.now() - timedelta(minutes=5)
if not EndTime:
EndTime = datetime.now()
if Statistics is None:
Statistics = ['Sum']
cloudwatch_url = aws_stack.get_local_service_url('cloudwatch')
url = '%s/?Action=GetMetricValues' % cloudwatch_url
all_metrics = make_http_request(url)
assert all_metrics.status_code == 200
datapoints = []
for datapoint in json.loads(to_str(all_metrics.content)):
if datapoint['Namespace'] == Namespace and datapoint['Name'] == MetricName:
dp_dimensions = datapoint['Dimensions']
all_present = all(m in dp_dimensions for m in Dimensions)
no_additional = all(m in Dimensions for m in dp_dimensions)
if all_present and no_additional:
datapoints.append(datapoint)
result = {
'Label': '%s/%s' % (Namespace, MetricName),
'Datapoints': datapoints
}
return result
def forward_request(self, method, path, data, headers):
req_data = None
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(to_str(data))
action = req_data.get('Action')[0]
if req_data:
if action == 'CreateStack':
return create_stack(req_data)
if action == 'CreateChangeSet':
return create_change_set(req_data)
elif action == 'DescribeChangeSet':
return describe_change_set(req_data)
elif action == 'ExecuteChangeSet':
return execute_change_set(req_data)
elif action == 'UpdateStack' and req_data.get('TemplateURL'):
# Temporary fix until the moto CF backend can handle TemplateURL (currently fails)
url = re.sub(r'https?://s3\.amazonaws\.com', aws_stack.get_local_service_url('s3'),
req_data.get('TemplateURL')[0])
req_data['TemplateBody'] = requests.get(url).content
modified_data = urlparse.urlencode(req_data, doseq=True)
return Request(data=modified_data, headers=headers, method=method)
elif action == 'ValidateTemplate':
return validate_template(req_data)
return True
def replay_command(command):
function = getattr(requests, command['m'].lower())
data = command['d']
if data:
data = base64.b64decode(data)
endpoint = aws_stack.get_local_service_url(command['a'])
full_url = (endpoint[:-1] if endpoint.endswith('/') else endpoint) + command['p']
result = function(full_url, data=data, headers=command['h'])
return result
def aws_cmd(service, aws_endpoint):
# TODO: use boto3 instead of running aws-cli commands here!
cmd = '. .venv/bin/activate; aws'
if aws_endpoint == 'local':
aws_endpoint = aws_stack.get_local_service_url(service)
if aws_endpoint:
cmd = '%s --endpoint-url="%s"' % (cmd, aws_endpoint)
cmd = '%s %s' % (cmd, service)
return cmd
def aws_cmd(service, env):
# TODO: use boto3 instead of running aws-cli commands here!
cmd = '. .venv/bin/activate; aws'
endpoint_url = None
if env.region == REGION_LOCAL:
endpoint_url = aws_stack.get_local_service_url(service)
if endpoint_url:
cmd = '%s --endpoint-url="%s"' % (cmd, endpoint_url)
cmd = '%s %s' % (cmd, service)
return cmd
def replay_command(command):
function = getattr(requests, command['m'].lower())
data = prepare_replay_data(command)
endpoint = aws_stack.get_local_service_url(command['a'])
full_url = (endpoint[:-1]
if endpoint.endswith('/') else endpoint) + command['p']
response = function(full_url,
data=data,
headers=command['h'],
verify=False)
return response
def get_command(backend_port):
cmd = (
"cd %s; PORT=%s java -Dcom.amazonaws.sdk.disableCertChecking -Xmx%s -jar StepFunctionsLocal.jar "
"--aws-region %s --aws-account %s"
) % (
install.INSTALL_DIR_STEPFUNCTIONS,
backend_port,
MAX_HEAP_SIZE,
default_region, # doesn't matter, because we patch multi-region. just needs to correspond to patches in stepfunctions_listener
TEST_AWS_ACCOUNT_ID,
)
if config.STEPFUNCTIONS_LAMBDA_ENDPOINT.lower() != "default":
lambda_endpoint = config.STEPFUNCTIONS_LAMBDA_ENDPOINT or aws_stack.get_local_service_url(
"lambda")
cmd += f" --lambda-endpoint {lambda_endpoint}"
# add service endpoint flags
services = [
"athena",
"batch",
"dynamodb",
"ecs",
"eks",
"events",
"glue",
"sagemaker",
"sns",
"sqs",
"stepfunctions",
]
for service in services:
flag = f"--{service}-endpoint"
if service == "stepfunctions":
flag = "--step-functions-endpoint"
elif service == "events":
flag = "--eventbridge-endpoint"
elif service in ["athena", "eks"]:
flag = f"--step-functions-{service}"
endpoint = aws_stack.get_local_service_url(service)
cmd += f" {flag} {endpoint}"
return cmd
def get_command(backend_port):
cmd = (
"cd %s; PORT=%s java -Dcom.amazonaws.sdk.disableCertChecking -Xmx%s -jar StepFunctionsLocal.jar "
"--aws-region %s --aws-account %s"
) % (
install.INSTALL_DIR_STEPFUNCTIONS,
backend_port,
MAX_HEAP_SIZE,
aws_stack.get_region(),
TEST_AWS_ACCOUNT_ID,
)
if config.STEPFUNCTIONS_LAMBDA_ENDPOINT.lower() != "default":
lambda_endpoint = config.STEPFUNCTIONS_LAMBDA_ENDPOINT or aws_stack.get_local_service_url(
"lambda"
)
cmd += (" --lambda-endpoint %s") % (lambda_endpoint)
# add service endpoint flags
services = [
"athena",
"batch",
"dynamodb",
"ecs",
"eks",
"events",
"glue",
"sagemaker",
"sns",
"sqs",
"stepfunctions",
]
for service in services:
flag = "--%s-endpoint" % service
if service == "stepfunctions":
flag = "--step-functions-endpoint"
elif service == "events":
flag = "--eventbridge-endpoint"
elif service in ["athena", "eks"]:
flag = "--step-functions-%s" % service
endpoint = aws_stack.get_local_service_url(service)
cmd += " %s %s" % (flag, endpoint)
return cmd
def get_command(backend_port):
cmd = ("cd %s; PORT=%s java "
"-javaagent:aspectjweaver-1.9.7.jar "
"-Dorg.aspectj.weaver.loadtime.configuration=META-INF/aop.xml "
"-Dcom.amazonaws.sdk.disableCertChecking -Xmx%s "
"-jar StepFunctionsLocal.jar --aws-account %s") % (
install.INSTALL_DIR_STEPFUNCTIONS,
backend_port,
MAX_HEAP_SIZE,
TEST_AWS_ACCOUNT_ID,
)
if config.STEPFUNCTIONS_LAMBDA_ENDPOINT.lower() != "default":
lambda_endpoint = config.STEPFUNCTIONS_LAMBDA_ENDPOINT or aws_stack.get_local_service_url(
"lambda")
cmd += f" --lambda-endpoint {lambda_endpoint}"
# add service endpoint flags
services = [
"athena",
"batch",
"dynamodb",
"ecs",
"eks",
"events",
"glue",
"sagemaker",
"sns",
"sqs",
"stepfunctions",
]
for service in services:
flag = f"--{service}-endpoint"
if service == "stepfunctions":
flag = "--step-functions-endpoint"
elif service == "events":
flag = "--eventbridge-endpoint"
elif service in ["athena", "eks"]:
flag = f"--step-functions-{service}"
endpoint = aws_stack.get_local_service_url(service)
cmd += f" {flag} {endpoint}"
return cmd
def setUpClass(cls):
cls.es_url = aws_stack.get_local_service_url('elasticsearch')
# create ES domain
cls._create_domain()
document = {
'first_name': 'Jane',
'last_name': 'Smith',
'age': 32,
'about': 'I like to collect rock albums',
'interests': ['music']
}
resp = cls._add_document(TEST_DOC_ID, document)
assert_equal(resp.status_code, 201, msg='Request failed({}): {}'.format(resp.status_code, resp.text))
def start_stepfunctions(port=None, asynchronous=False, update_listener=None):
port = port or config.PORT_STEPFUNCTIONS
install.install_stepfunctions_local()
backend_port = DEFAULT_PORT_STEPFUNCTIONS_BACKEND
# TODO: local port is currently hard coded in Step Functions Local :/
backend_port = 8083
lambda_endpoint = aws_stack.get_local_service_url('lambda')
dynamodb_endpoint = aws_stack.get_local_service_url('dynamodb')
sns_endpoint = aws_stack.get_local_service_url('sns')
sqs_endpoint = aws_stack.get_local_service_url('sqs')
cmd = (
'cd %s; java -Dcom.amazonaws.sdk.disableCertChecking -Xmx%s -jar StepFunctionsLocal.jar '
'--lambda-endpoint %s --dynamodb-endpoint %s --sns-endpoint %s '
'--sqs-endpoint %s --aws-region %s --aws-account %s') % (
install.INSTALL_DIR_STEPFUNCTIONS, MAX_HEAP_SIZE, lambda_endpoint,
dynamodb_endpoint, sns_endpoint, sqs_endpoint, DEFAULT_REGION,
TEST_AWS_ACCOUNT_ID)
print('Starting mock StepFunctions (%s port %s)...' %
(get_service_protocol(), port))
start_proxy_for_service('stepfunctions', port, backend_port,
update_listener)
return do_run(cmd, asynchronous)
def aws_cmd(service, env):
# TODO: use boto3 instead of running aws-cli commands here!
cmd = '{ test `which aws` || . .venv/bin/activate; }; aws'
endpoint_url = None
env = aws_stack.get_environment(env)
if env.region == REGION_LOCAL:
endpoint_url = aws_stack.get_local_service_url(service)
if endpoint_url:
cmd = '%s --endpoint-url="%s"' % (cmd, endpoint_url)
if not is_port_open(endpoint_url):
raise socket.error()
cmd = '%s %s' % (cmd, service)
return cmd
def get_command(backend_port):
cmd = (
'cd %s; PORT=%s java -Dcom.amazonaws.sdk.disableCertChecking -Xmx%s -jar StepFunctionsLocal.jar '
'--aws-region %s --aws-account %s') % (
install.INSTALL_DIR_STEPFUNCTIONS, backend_port, MAX_HEAP_SIZE,
aws_stack.get_region(), TEST_AWS_ACCOUNT_ID)
if config.STEPFUNCTIONS_LAMBDA_ENDPOINT.lower() != 'default':
lambda_endpoint = config.STEPFUNCTIONS_LAMBDA_ENDPOINT or aws_stack.get_local_service_url(
'lambda')
cmd += (' --lambda-endpoint %s') % (lambda_endpoint)
# add service endpoint flags
services = [
'athena', 'batch', 'dynamodb', 'ecs', 'eks', 'glue', 'sagemaker',
'sns', 'sqs', 'stepfunctions'
]
for service in services:
flag = '--%s-endpoint' % service
if service == 'stepfunctions':
flag = '--step-functions-endpoint'
elif service in ['athena', 'eks']:
flag = '--step-functions-%s' % service
endpoint = aws_stack.get_local_service_url(service)
cmd += ' %s %s' % (flag, endpoint)
return cmd
def aws_cmd(service, env):
# TODO: use boto3 instead of running aws-cli commands here!
cmd = '{ test `which aws` || . .venv/bin/activate; }; aws'
endpoint_url = None
env = aws_stack.get_environment(env)
if aws_stack.is_local_env(env):
endpoint_url = aws_stack.get_local_service_url(service)
if endpoint_url:
if endpoint_url.startswith('https://'):
cmd += ' --no-verify-ssl'
cmd = '%s --endpoint-url="%s"' % (cmd, endpoint_url)
if not is_port_open(endpoint_url):
raise socket.error()
cmd = '%s %s' % (cmd, service)
return cmd
def replay_command(command):
api = command['a']
if not is_api_enabled(api):
return
function = getattr(requests, command['m'].lower())
data = prepare_replay_data(command)
endpoint = aws_stack.get_local_service_url(api)
full_url = (endpoint[:-1] if endpoint.endswith('/') else endpoint) + command['p']
try:
# fix an error when calling requests with invalid payload encoding
data and hasattr(data, 'encode') and data.encode('latin-1')
except UnicodeEncodeError:
if hasattr(data, 'encode'):
data = data.encode('utf-8')
response = function(full_url, data=data, headers=command['h'], verify=False)
return response
def aws_cmd(service, env):
# TODO: use boto3 instead of running aws-cli commands here!
cmd = '{ test `which aws` || . .venv/bin/activate; }; aws'
endpoint_url = None
env = aws_stack.get_environment(env)
if env.region == REGION_LOCAL:
endpoint_url = aws_stack.get_local_service_url(service)
if endpoint_url:
if endpoint_url.startswith('https://'):
cmd += ' --no-verify-ssl'
cmd = '%s --endpoint-url="%s"' % (cmd, endpoint_url)
if not is_port_open(endpoint_url):
raise socket.error()
cmd = '%s %s' % (cmd, service)
return cmd
def replay_command(command):
api = command["a"]
if not is_api_enabled(api):
return
function = getattr(requests, command["m"].lower())
data = prepare_replay_data(command)
endpoint = aws_stack.get_local_service_url(api)
full_url = (endpoint[:-1] if endpoint.endswith("/") else endpoint) + command["p"]
headers = aws_stack.set_internal_auth(command["h"])
try:
# fix an error when calling requests with invalid payload encoding
data and hasattr(data, "encode") and data.encode("latin-1")
except UnicodeEncodeError:
if hasattr(data, "encode"):
data = data.encode("utf-8")
response = function(full_url, data=data, headers=headers, verify=False)
return response
def setUpClass(cls):
then = time.time()
LOG.info("waiting for initialization lock")
with INIT_LOCK:
LOG.info("initialization lock acquired in %.2f seconds",
time.time() - then)
cls.es_url = aws_stack.get_local_service_url("elasticsearch")
# create ES domain
cls._create_domain(name=TEST_DOMAIN_NAME)
document = {
"first_name": "Jane",
"last_name": "Smith",
"age": 32,
"about": "I like to collect rock albums",
"interests": ["music"],
}
resp = cls._add_document(TEST_DOC_ID, document)
assert resp.status_code == 201, "Request failed({}): {}".format(
resp.status_code, resp.text)
def forward_request(self, method, path, data, headers):
req_data = None
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(to_str(data))
action = req_data.get('Action')[0]
if req_data:
if action == 'CreateChangeSet':
return create_change_set(req_data)
elif action == 'DescribeChangeSet':
return describe_change_set(req_data)
elif action == 'ExecuteChangeSet':
return execute_change_set(req_data)
elif action == 'UpdateStack' and req_data.get('TemplateURL'):
# Temporary fix until the moto CF backend can handle TemplateURL (currently fails)
url = re.sub(r'https?://s3\.amazonaws\.com', aws_stack.get_local_service_url('s3'),
req_data.get('TemplateURL')[0])
req_data['TemplateBody'] = requests.get(url).content
modified_data = urlparse.urlencode(req_data, doseq=True)
return Request(data=modified_data, headers=headers, method=method)
elif action == 'ValidateTemplate':
return validate_template(req_data)
return True
def update_cloudformation(method,
path,
data,
headers,
response=None,
return_forward_info=False):
req_data = None
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(data)
action = req_data.get('Action')[0]
if return_forward_info:
if req_data:
if action == 'CreateChangeSet':
return create_change_set(req_data)
elif action == 'DescribeChangeSet':
return describe_change_set(req_data)
elif action == 'ExecuteChangeSet':
return execute_change_set(req_data)
elif action == 'UpdateStack' and req_data.get('TemplateURL'):
# Temporary fix until the moto CF backend can handle TemplateURL (currently fails)
url = re.sub(r'https?://s3\.amazonaws\.com',
aws_stack.get_local_service_url('s3'),
req_data.get('TemplateURL')[0])
req_data['TemplateBody'] = requests.get(url).content
modified_data = urlparse.urlencode(req_data, doseq=True)
return Request(data=modified_data,
headers=headers,
method=method)
return True
if req_data:
if action == 'DescribeStackResources':
if response.status_code < 300:
response_dict = xmltodict.parse(
response.content)['DescribeStackResourcesResponse']
resources = response_dict['DescribeStackResourcesResult'][
'StackResources']
if not resources:
# Check if stack exists
stack_name = req_data.get('StackName')[0]
cloudformation_client = aws_stack.connect_to_service(
'cloudformation')
try:
cloudformation_client.describe_stacks(
StackName=stack_name)
except Exception as e:
return error_response(
'Stack with id %s does not exist' % stack_name,
code=404)
if action == 'DescribeStackResource':
if response.status_code >= 500:
# fix an error in moto where it fails with 500 if the stack does not exist
return error_response('Stack resource does not exist',
code=404)
elif action == 'CreateStack' or action == 'UpdateStack':
# run the actual deployment
template = template_deployer.template_to_json(
req_data.get('TemplateBody')[0])
template_deployer.deploy_template(template,
req_data.get('StackName')[0])
if response.status_code >= 400:
return make_response(action)
import json
import time
from botocore.exceptions import ClientError
from nose.tools import assert_raises, assert_equal, assert_true, assert_false
from localstack.utils.aws import aws_stack
from localstack.utils.common import safe_requests as requests
ES_URL = aws_stack.get_local_service_url('elasticsearch')
TEST_INDEX = 'megacorp'
TEST_DOC_ID = 1
COMMON_HEADERS = {
'content-type': 'application/json',
'Accept-encoding': 'identity'
}
TEST_DOMAIN_NAME = 'test_es_domain_1'
TEST_ENDPOINT_URL = 'http://localhost:4571'
def setUp():
document = {
'first_name': 'Jane',
'last_name': 'Smith',
'age': 32,
'about': 'I like to collect rock albums',
'interests': ['music']
}
resp = add_document(TEST_DOC_ID, document)
assert_equal(201, resp.status_code,
msg='Request failed({}): {}'.format(resp.status_code, resp.text))
import boto3
from nose.tools import assert_raises, assert_equal, assert_true
from botocore.exceptions import ClientError
from localstack import config
import json
import requests
import time
from localstack.utils.aws import aws_stack
es_url = aws_stack.get_local_service_url('elasticsearch')
test_index = 'megacorp'
test_doc_id = 1
common_headers = {
'content-type': 'application/json',
'Accept-encoding': 'identity'
}
def setUp():
document = {
'first_name': 'Jane',
'last_name': 'Smith',
'age': 32,
'about': 'I like to collect rock albums',
'interests': ['music']
}
resp = add_document(test_doc_id, document)
assert_equal(201,
resp.status_code,
msg='Request failed({}): {}'.format(resp.status_code,
resp.text))