def record_service_health(api, status): data = {api: status} health_url = '%s://%s:%s/health' % (get_service_protocol(), config.LOCALHOST, config.PORT_WEB_UI) try: requests.put(health_url, data=json.dumps(data)) except Exception: # ignore for now, if the service is not running pass
def get_domain_status(domain_name, deleted=False): status = ES_DOMAINS.get(domain_name) or {} cluster_cfg = status.get("ElasticsearchClusterConfig") or {} default_cfg = DEFAULT_ES_CLUSTER_CONFIG endpoint = "%s://%s:%s" % ( get_service_protocol(), config.HOSTNAME_EXTERNAL, config.PORT_ELASTICSEARCH, ) return { "DomainStatus": { "ARN": "arn:aws:es:%s:%s:domain/%s" % (aws_stack.get_region(), TEST_AWS_ACCOUNT_ID, domain_name), "Created": status.get("Created", False), "Deleted": deleted, "DomainId": "%s/%s" % (TEST_AWS_ACCOUNT_ID, domain_name), "DomainName": domain_name, "ElasticsearchClusterConfig": { "DedicatedMasterCount": cluster_cfg.get("DedicatedMasterCount", default_cfg["DedicatedMasterCount"]), "DedicatedMasterEnabled": cluster_cfg.get("DedicatedMasterEnabled", default_cfg["DedicatedMasterEnabled"]), "DedicatedMasterType": cluster_cfg.get("DedicatedMasterType", default_cfg["DedicatedMasterType"]), "InstanceCount": cluster_cfg.get("InstanceCount", default_cfg["InstanceCount"]), "InstanceType": cluster_cfg.get("InstanceType", default_cfg["InstanceType"]), "ZoneAwarenessEnabled": cluster_cfg.get("ZoneAwarenessEnabled", default_cfg["ZoneAwarenessEnabled"]), }, "ElasticsearchVersion": status.get("ElasticsearchVersion") or DEFAULT_ES_VERSION, "Endpoint": endpoint, "Processing": False, "EBSOptions": { "EBSEnabled": True, "VolumeType": "gp2", "VolumeSize": 10, "Iops": 0, }, "CognitoOptions": { "Enabled": False }, } }
def start_local_api(name, port, method, asynchronous=False): print( 'Starting mock %s service in %s ports %s (recommended) and %s (deprecated)...' % (name, get_service_protocol(), config.EDGE_PORT, port)) if asynchronous: thread = start_thread(method, port, quiet=True) return thread else: method(port)
def get_local_service_url(service_name_or_port): """ Return the local service URL for the given service name or port. """ if isinstance(service_name_or_port, int): return '%s://%s:%s' % (get_service_protocol(), LOCALHOST, service_name_or_port) service_name = service_name_or_port if service_name == 's3api': service_name = 's3' return os.environ['TEST_%s_URL' % (service_name.upper().replace('-', '_'))]
def _queue_url(path, req_data, headers): queue_url = req_data.get("QueueUrl") if queue_url: return queue_url url = config.service_url("sqs") if headers.get("Host"): url = "%s://%s" % (get_service_protocol(), headers["Host"]) queue_url = "%s%s" % (url, path.partition("?")[0]) return queue_url
def start_local_api(name, port, method, asynchronous=False): print('Starting mock %s service (%s port %s)...' % (name, get_service_protocol(), port)) if asynchronous: thread = FuncThread(method, port, quiet=True) thread.start() TMP_THREADS.append(thread) return thread else: method(port)
def _update_location(content, bucket_name): bucket_name = normalize_bucket_name(bucket_name) host = config.HOSTNAME_EXTERNAL if ':' not in host: host = '%s:%s' % (host, config.PORT_S3) return re.sub(r'<Location>\s*([a-zA-Z0-9\-]+)://[^/]+/([^<]+)\s*</Location>', r'<Location>%s://%s/%s/\2</Location>' % (get_service_protocol(), host, bucket_name), content, flags=re.MULTILINE)
def test_schedule_expression_event_with_http_endpoint(self): class HttpEndpointListener(ProxyListener): def forward_request(self, method, path, data, headers): event = json.loads(to_str(data)) events.append(event) return 200 topic_name = 'topic-{}'.format(short_uid()) rule_name = 'rule-{}'.format(short_uid()) target_id = 'target-{}'.format(short_uid()) events = [] local_port = get_free_tcp_port() proxy = start_proxy(local_port, backend_url=None, update_listener=HttpEndpointListener()) wait_for_port_open(local_port) endpoint = '{}://{}:{}'.format(get_service_protocol(), config.LOCALSTACK_HOSTNAME, local_port) topic_arn = self.sns_client.create_topic(Name=topic_name)['TopicArn'] self.sns_client.subscribe(TopicArn=topic_arn, Protocol='http', Endpoint=endpoint) event = {'env': 'testing'} self.events_client.put_rule(Name=rule_name, ScheduleExpression='rate(1 minutes)') self.events_client.put_targets(Rule=rule_name, Targets=[{ 'Id': target_id, 'Arn': topic_arn, 'Input': json.dumps(event) }]) def received(): self.assertGreaterEqual(len(events), 2) notifications = [ event['Message'] for event in events if event['Type'] == 'Notification' ] self.assertGreaterEqual(len(notifications), 1) return notifications[0] notification = retry(received, retries=2, sleep=40) self.assertEqual(json.loads(notification), event) proxy.stop() self.events_client.remove_targets(Rule=rule_name, Ids=[target_id], Force=True) self.events_client.delete_rule(Name=rule_name, Force=True) self.sns_client.delete_topic(TopicArn=topic_arn)
def _queue_url(path, req_data, headers): queue_url = req_data.get('QueueUrl') if queue_url: return queue_url url = config.TEST_SQS_URL if headers.get('Host'): url = '%s://%s' % (get_service_protocol(), headers['Host']) queue_url = '%s%s' % (url, path.partition('?')[0]) return queue_url
def forward_request(self, method, path, data, headers): if method == 'OPTIONS': return 200 if path.split('?')[0] == '/health': return serve_health_endpoint(method, path, data) # kill the process if we receive this header headers.get(HEADER_KILL_SIGNAL) and os._exit(0) target = headers.get('x-amz-target', '') auth_header = headers.get('authorization', '') host = headers.get('host', '') headers[HEADER_LOCALSTACK_EDGE_URL] = 'https://%s' % host # extract API details api, port, path, host = get_api_from_headers(headers, path) if port and int(port) < 0: return 404 if not port: port = get_port_from_custom_rules(method, path, data, headers) or port if not port: if api in ['', None, '_unknown_']: truncated = truncate(data) LOG.info(( 'Unable to find forwarding rule for host "%s", path "%s", ' 'target header "%s", auth header "%s", data "%s"') % (host, path, target, auth_header, truncated)) else: LOG.info(( 'Unable to determine forwarding port for API "%s" - please ' 'make sure this API is enabled via the SERVICES configuration' ) % api) response = Response() response.status_code = 404 response._content = '{"status": "running"}' return response connect_host = '%s:%s' % (config.HOSTNAME, port) url = '%s://%s%s' % (get_service_protocol(), connect_host, path) headers['Host'] = host function = getattr(requests, method.lower()) if isinstance(data, dict): data = json.dumps(data) response = function(url, data=data, headers=headers, verify=False, stream=True) return response
def do_forward_request_network(port, method, path, data, headers): connect_host = '%s:%s' % (config.HOSTNAME, port) url = '%s://%s%s' % (get_service_protocol(), connect_host, path) function = getattr(requests, method.lower()) response = function(url, data=data, headers=headers, verify=False, stream=True) return response
def get_domain_status(domain_name, deleted=False): status = ES_DOMAINS.get(domain_name) or {} cluster_cfg = status.get('ElasticsearchClusterConfig') or {} default_cfg = DEFAULT_ES_CLUSTER_CONFIG endpoint = '%s://%s:%s' % (get_service_protocol(), config.HOSTNAME_EXTERNAL, config.PORT_ELASTICSEARCH) return { 'DomainStatus': { 'ARN': 'arn:aws:es:%s:%s:domain/%s' % (aws_stack.get_region(), TEST_AWS_ACCOUNT_ID, domain_name), 'Created': status.get('Created', True), 'Deleted': deleted, 'DomainId': '%s/%s' % (TEST_AWS_ACCOUNT_ID, domain_name), 'DomainName': domain_name, 'ElasticsearchClusterConfig': { 'DedicatedMasterCount': cluster_cfg.get('DedicatedMasterCount', default_cfg['DedicatedMasterCount']), 'DedicatedMasterEnabled': cluster_cfg.get('DedicatedMasterEnabled', default_cfg['DedicatedMasterEnabled']), 'DedicatedMasterType': cluster_cfg.get('DedicatedMasterType', default_cfg['DedicatedMasterType']), 'InstanceCount': cluster_cfg.get('InstanceCount', default_cfg['InstanceCount']), 'InstanceType': cluster_cfg.get('InstanceType', default_cfg['InstanceType']), 'ZoneAwarenessEnabled': cluster_cfg.get('ZoneAwarenessEnabled', default_cfg['ZoneAwarenessEnabled']), }, 'ElasticsearchVersion': status.get('ElasticsearchVersion') or DEFAULT_ES_VERSION, 'Endpoint': endpoint, 'Processing': False, 'EBSOptions': { 'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10, 'Iops': 0 }, 'CognitoOptions': { 'Enabled': False }, } }
def get_local_service_url(service_name_or_port): """Return the local service URL for the given service name or port.""" if isinstance(service_name_or_port, int): return "%s://%s:%s" % (get_service_protocol(), LOCALHOST, service_name_or_port) service_name = service_name_or_port if service_name == "s3api": service_name = "s3" elif service_name == "runtime.sagemaker": service_name = "sagemaker-runtime" service_name_upper = service_name.upper().replace("-", "_").replace(".", "_") return os.environ["TEST_%s_URL" % service_name_upper]
def start_local_api(name, port, api, method, asynchronous=False): print('Starting mock %s service on %s port %s ...' % (name, get_service_protocol(), config.EDGE_PORT)) if config.FORWARD_EDGE_INMEM: port = get_free_tcp_port() PROXY_LISTENERS[api] = (api, port, None) if asynchronous: thread = start_thread(method, port, quiet=True) return thread else: method(port)
def start_moto_server(key, port, name=None, backend_port=None, asynchronous=False, update_listener=None): if not name: name = key print('Starting mock %s (%s port %s)...' % (name, get_service_protocol(), port)) if config.USE_SSL and not backend_port: backend_port = get_free_tcp_port() if backend_port: start_proxy_for_service(key, port, backend_port, update_listener) if config.BUNDLE_API_PROCESSES: return multiserver.start_api_server(key, backend_port or port) return start_moto_server_separate(key, port, name=name, backend_port=backend_port, asynchronous=asynchronous)
def do_forward_request_network(port, method, path, data, headers): # TODO: enable per-service endpoints, to allow deploying in distributed settings connect_host = '%s:%s' % (LOCALHOST, port) url = '%s://%s%s' % (get_service_protocol(), connect_host, path) function = getattr(requests, method.lower()) response = function(url, data=data, headers=headers, verify=False, stream=True) return response
def start_moto_server(key, port, name=None, backend_port=None, asynchronous=False, update_listener=None): if not name: name = key print('Starting mock %s service in %s ports %s (recommended) and %s (deprecated)...' % ( name, get_service_protocol(), config.EDGE_PORT, port)) if not backend_port and (config.USE_SSL or update_listener): backend_port = get_free_tcp_port() if backend_port: start_proxy_for_service(key, port, backend_port, update_listener) if config.BUNDLE_API_PROCESSES: return multiserver.start_api_server(key, backend_port or port) return start_moto_server_separate(key, port, name=name, backend_port=backend_port, asynchronous=asynchronous)
def get_forward_url(self, method, path, data, headers): def sub(match): # make sure to convert any bucket names to lower case bucket_name = normalize_bucket_name(match.group(1)) return '/%s%s' % (bucket_name, match.group(2) or '') path_new = re.sub(r'/([^?/]+)([?/].*)?', sub, path) if path == path_new: return url = '%s://%s:%s%s' % (get_service_protocol(), constants.LOCALHOST, constants.DEFAULT_PORT_S3_BACKEND, path_new) return url
def start_moto_server(key, port, name=None, backend_port=None, asynchronous=False, update_listener=None): moto_server_cmd = '%s/bin/moto_server' % LOCALSTACK_VENV_FOLDER if not os.path.exists(moto_server_cmd): moto_server_cmd = run('which moto_server').strip() if USE_SSL and not backend_port: backend_port = get_free_tcp_port() cmd = 'VALIDATE_LAMBDA_S3=0 %s %s -p %s -H %s' % (moto_server_cmd, key, backend_port or port, constants.BIND_HOST) if not name: name = key print('Starting mock %s (%s port %s)...' % (name, get_service_protocol(), port)) if backend_port: start_proxy_for_service(key, port, backend_port, update_listener) return do_run(cmd, asynchronous)
def test_forward_to_fallback_url_http(self): class MyUpdateListener(ProxyListener): def forward_request(self, method, path, data, headers): records.append({"data": data, "headers": headers, "method": method, "path": path}) return lambda_result lambda_result = {"result": "test123"} local_port = get_free_tcp_port() proxy = start_proxy(local_port, backend_url=None, update_listener=MyUpdateListener()) local_url = "%s://localhost:%s" % (get_service_protocol(), local_port) # test 1: forward to LAMBDA_FALLBACK_URL records = [] self._run_forward_to_fallback_url(local_url) items_after = len(records) for record in records: self.assertIn("non-existing-lambda", record["headers"]["lambda-function-name"]) self.assertEqual(3, items_after) # create test Lambda lambda_name = "test-%s" % short_uid() testutil.create_lambda_function( handler_file=TEST_LAMBDA_PYTHON, func_name=lambda_name, libs=TEST_LAMBDA_LIBS, ) # test 2: forward to LAMBDA_FORWARD_URL records = [] inv_results = self._run_forward_to_fallback_url( local_url, lambda_name=lambda_name, fallback=False ) items_after = len(records) for record in records: headers = record["headers"] self.assertIn("/lambda/", headers["Authorization"]) self.assertEqual("POST", record["method"]) self.assertIn("/functions/%s/invocations" % lambda_name, record["path"]) self.assertTrue(headers.get("X-Amz-Client-Context")) self.assertEqual("RequestResponse", headers.get("X-Amz-Invocation-Type")) self.assertEqual({"foo": "bar"}, json.loads(to_str(record["data"]))) self.assertEqual(3, items_after) # assert result payload matches response_payload = inv_results[0]["Payload"].read() self.assertEqual(lambda_result, json.loads(response_payload)) # clean up / shutdown lambda_client = aws_stack.create_external_boto_client("lambda") lambda_client.delete_function(FunctionName=lambda_name) proxy.stop()
def test_redrive_policy_http_subscription(self): self.unsubscribe_all_from_sns() # create HTTP endpoint and connect it to SNS topic class MyUpdateListener(ProxyListener): def forward_request(self, method, path, data, headers): records.append((json.loads(to_str(data)), headers)) return 200 records = [] local_port = get_free_tcp_port() proxy = start_proxy(local_port, backend_url=None, update_listener=MyUpdateListener()) wait_for_port_open(local_port) http_endpoint = "%s://localhost:%s" % (get_service_protocol(), local_port) subscription = self.sns_client.subscribe(TopicArn=self.topic_arn, Protocol="http", Endpoint=http_endpoint) self.sns_client.set_subscription_attributes( SubscriptionArn=subscription["SubscriptionArn"], AttributeName="RedrivePolicy", AttributeValue=json.dumps({ "deadLetterTargetArn": aws_stack.sqs_queue_arn(TEST_QUEUE_DLQ_NAME) }), ) proxy.stop() # for some reason, it takes a long time to stop the proxy thread -> TODO investigate time.sleep(5) self.sns_client.publish( TopicArn=self.topic_arn, Message=json.dumps({"message": "test_redrive_policy"}), ) def receive_dlq(): result = self.sqs_client.receive_message( QueueUrl=self.dlq_url, MessageAttributeNames=["All"]) self.assertGreater(len(result["Messages"]), 0) self.assertEqual( json.loads( json.loads(result["Messages"][0]["Body"])["Message"][0]) ["message"], "test_redrive_policy", ) retry(receive_dlq, retries=7, sleep=2.5)
def test_invoke_apis_via_edge(self): edge_url = '%s://localhost:%s' % (get_service_protocol(), config.EDGE_PORT) if is_api_enabled('s3'): self._invoke_s3_via_edge(edge_url) if is_api_enabled('kinesis'): self._invoke_kinesis_via_edge(edge_url) if is_api_enabled('dynamodbstreams'): self._invoke_dynamodbstreams_via_edge(edge_url) if is_api_enabled('firehose'): self._invoke_firehose_via_edge(edge_url) if is_api_enabled('stepfunctions'): self._invoke_stepfunctions_via_edge(edge_url)
def check_elasticsearch(expect_shutdown=False, print_error=True): # Check internal endpoint for health endpoint = '%s://%s:%s' % (get_service_protocol(), constants.LOCALHOST, config.PORT_ELASTICSEARCH) try: req = requests.get(endpoint + '/_cluster/health') es_status = json.loads(req.text) es_status = es_status['status'] return es_status == 'green' or es_status == 'yellow' except ValueError as e: if print_error: LOG.error( 'Elasticsearch health check to endpoint %s failed (retrying...): %s %s' % ( endpoint, e, traceback.format_exc())) pass
def test_forward_to_fallback_url_http(self): class MyUpdateListener(ProxyListener): def forward_request(self, method, path, data, headers): records.append(data) return 200 records = [] local_port = get_free_tcp_port() proxy = start_proxy(local_port, backend_url=None, update_listener=MyUpdateListener()) items_before = len(records) self._run_forward_to_fallback_url('%s://localhost:%s' % (get_service_protocol(), local_port)) items_after = len(records) self.assertEqual(items_after, items_before + 3) proxy.stop()
def test_invoke_apis_via_edge(self): edge_port = config.EDGE_PORT_HTTP or config.EDGE_PORT edge_url = "%s://localhost:%s" % (get_service_protocol(), edge_port) if is_api_enabled("s3"): self._invoke_s3_via_edge(edge_url) self._invoke_s3_via_edge_multipart_form(edge_url) if is_api_enabled("kinesis"): self._invoke_kinesis_via_edge(edge_url) if is_api_enabled("dynamodbstreams"): self._invoke_dynamodbstreams_via_edge(edge_url) if is_api_enabled("firehose"): self._invoke_firehose_via_edge(edge_url) if is_api_enabled("stepfunctions"): self._invoke_stepfunctions_via_edge(edge_url)
def start_elasticsearch(port=None, version=None, delete_data=True, asynchronous=False, update_listener=None): if STATE.get('_thread_'): return STATE['_thread_'] port = port or config.PORT_ELASTICSEARCH # delete Elasticsearch data that may be cached locally from a previous test run delete_all_elasticsearch_data(version) install.install_elasticsearch(version) backend_port = get_free_tcp_port() base_dir = install.get_elasticsearch_install_dir(version) es_data_dir = os.path.join(base_dir, 'data') es_tmp_dir = os.path.join(base_dir, 'tmp') es_mods_dir = os.path.join(base_dir, 'modules') if config.DATA_DIR: delete_data = False es_data_dir = '%s/elasticsearch' % config.DATA_DIR # Elasticsearch 5.x cannot be bound to 0.0.0.0 in some Docker environments, # hence we use the default bind address 127.0.0.0 and put a proxy in front of it backup_dir = os.path.join(config.TMP_FOLDER, 'es_backup') cmd = (('%s/bin/elasticsearch ' + '-E http.port=%s -E http.publish_port=%s -E http.compression=false ' + '-E path.data=%s -E path.repo=%s') % (base_dir, backend_port, backend_port, es_data_dir, backup_dir)) if os.path.exists(os.path.join(es_mods_dir, 'x-pack-ml')): cmd += ' -E xpack.ml.enabled=false' env_vars = { 'ES_JAVA_OPTS': os.environ.get('ES_JAVA_OPTS', '-Xms200m -Xmx600m'), 'ES_TMPDIR': es_tmp_dir } LOG.debug('Starting local Elasticsearch (%s port %s)' % (get_service_protocol(), port)) if delete_data: rm_rf(es_data_dir) # fix permissions chmod_r(base_dir, 0o777) mkdir(es_data_dir) chmod_r(es_data_dir, 0o777) mkdir(es_tmp_dir) chmod_r(es_tmp_dir, 0o777) # start proxy and ES process proxy = start_proxy_for_service('elasticsearch', port, backend_port, update_listener, quiet=True, params={'protocol_version': 'HTTP/1.0'}) STATE['_proxy_'] = proxy if is_root(): cmd = "su localstack -c '%s'" % cmd thread = do_run(cmd, asynchronous, env_vars=env_vars) STATE['_thread_'] = thread return thread
def get_201_reponse(self, key, bucket_name): return """ <PostResponse> <Location>{protocol}://{host}/{encoded_key}</Location> <Bucket>{bucket}</Bucket> <Key>{key}</Key> <ETag>{etag}</ETag> </PostResponse> """.format( protocol=get_service_protocol(), host=config.HOSTNAME_EXTERNAL, encoded_key=urlparse.quote(key, safe=''), key=key, bucket=bucket_name, etag='d41d8cd98f00b204e9800998ecf8427f', )
def do_forward_request_network(port, method, path, data, headers, target_url=None): # TODO: enable per-service endpoints, to allow deploying in distributed settings target_url = target_url or "%s://%s:%s" % (get_service_protocol(), LOCALHOST, port) url = "%s%s" % (target_url, path) response = requests.request(method, url, data=data, headers=headers, verify=False, stream=True) return response
def test_redrive_policy_http_subscription(self): self.unsubscribe_all_from_sns() # create HTTP endpoint and connect it to SNS topic class MyUpdateListener(ProxyListener): def forward_request(self, method, path, data, headers): records.append((json.loads(to_str(data)), headers)) return 200 records = [] local_port = get_free_tcp_port() proxy = start_proxy(local_port, backend_url=None, update_listener=MyUpdateListener()) wait_for_port_open(local_port) http_endpoint = '%s://localhost:%s' % (get_service_protocol(), local_port) subscription = self.sns_client.subscribe(TopicArn=self.topic_arn, Protocol='http', Endpoint=http_endpoint) self.sns_client.set_subscription_attributes( SubscriptionArn=subscription['SubscriptionArn'], AttributeName='RedrivePolicy', AttributeValue=json.dumps({ 'deadLetterTargetArn': aws_stack.sqs_queue_arn(TEST_QUEUE_DLQ_NAME) })) proxy.stop() self.sns_client.publish(TopicArn=self.topic_arn, Message=json.dumps( {'message': 'test_redrive_policy'})) def receive_dlq(): result = self.sqs_client.receive_message( QueueUrl=self.dlq_url, MessageAttributeNames=['All']) self.assertGreater(len(result['Messages']), 0) self.assertEqual( json.loads( json.loads(result['Messages'][0]['Body'])['Message'][0]) ['message'], 'test_redrive_policy') retry(receive_dlq, retries=10, sleep=2)
def list_queues_with_auth_in_presigned_url(self, method): base_url = "{}://{}:{}".format(get_service_protocol(), config.LOCALSTACK_HOSTNAME, config.PORT_SQS) req = AWSRequest( method=method, url=base_url, data={ "Action": "ListQueues", "Version": "2012-11-05" }, ) # boto doesn't support querystring-style auth, so we have to do some # weird logic to use boto's signing functions, to understand what's # going on here look at the internals of the SigV4Auth.add_auth # method. datetime_now = datetime.datetime.utcnow() req.context["timestamp"] = datetime_now.strftime(SIGV4_TIMESTAMP) signer = SigV4Auth( Credentials(TEST_AWS_ACCESS_KEY_ID, TEST_AWS_SECRET_ACCESS_KEY), "sqs", aws_stack.get_region(), ) canonical_request = signer.canonical_request(req) string_to_sign = signer.string_to_sign(req, canonical_request) payload = { "Action": "ListQueues", "Version": "2012-11-05", "X-Amz-Algorithm": "AWS4-HMAC-SHA256", "X-Amz-Credential": signer.scope(req), "X-Amz-SignedHeaders": ";".join(signer.headers_to_sign(req).keys()), "X-Amz-Signature": signer.signature(string_to_sign, req), } if method == "GET": return requests.get(url=base_url, params=payload) else: return requests.post(url=base_url, data=urlencode(payload))