def start_cloudformation(port=None, asynchronous=False, update_listener=None): port = port or config.PORT_CLOUDFORMATION print( 'Starting mock CloudFormation service in %s ports %s (recommended) and %s (deprecated)...' % (get_service_protocol(), config.EDGE_PORT, port)) backend_port = get_free_tcp_port() start_proxy_for_service('cloudformation', port, backend_port, update_listener) if RUN_SERVER_IN_PROCESS: cmd = 'python "%s" cloudformation -p %s -H 0.0.0.0' % (__file__, backend_port) env_vars = {'PYTHONPATH': ':'.join(sys.path)} return do_run(cmd, asynchronous, env_vars=env_vars) else: argv = ['cloudformation', '-p', str(backend_port), '-H', '0.0.0.0'] thread = FuncThread(start_up, argv) thread.start() return thread
def start_http_server( test_port: int = None, invocations: List = None, invocation_handler: Callable = None ) -> Tuple[int, List, FuncThread]: # Note: leave imports here to avoid import errors (e.g., "flask") for CLI commands from localstack.services.generic_proxy import ProxyListener from localstack.services.infra import start_proxy class TestListener(ProxyListener): def forward_request(self, **kwargs): if invocation_handler: kwargs = invocation_handler(**kwargs) invocations.append(kwargs) return 200 test_port = test_port or get_free_tcp_port() invocations = invocations or [] proxy = start_proxy(test_port, update_listener=TestListener()) return test_port, invocations, proxy
def start_kms_local(port=None, backend_port=None, asynchronous=None, update_listener=None): port = port or config.PORT_KMS backend_port = get_free_tcp_port() kms_binary = INSTALL_PATH_KMS_BINARY_PATTERN.replace("<arch>", get_os()) log_startup_message("KMS") start_proxy_for_service("kms", port, backend_port, update_listener) env_vars = { "PORT": str(backend_port), "KMS_REGION": config.DEFAULT_REGION, "REGION": config.DEFAULT_REGION, "KMS_ACCOUNT_ID": TEST_AWS_ACCOUNT_ID, "ACCOUNT_ID": TEST_AWS_ACCOUNT_ID, } if config.dirs.data: env_vars["KMS_DATA_PATH"] = config.dirs.data result = do_run(kms_binary, asynchronous, env_vars=env_vars) wait_for_port_open(backend_port) return result
def start_moto_server( key, port, name=None, backend_port=None, asynchronous=False, update_listener=None ): if not name: name = key log_startup_message(name) if not backend_port: if config.FORWARD_EDGE_INMEM: backend_port = multiserver.get_moto_server_port() elif config.USE_SSL or update_listener: backend_port = get_free_tcp_port() if backend_port or config.FORWARD_EDGE_INMEM: start_proxy_for_service(key, port, backend_port, update_listener) if config.BUNDLE_API_PROCESSES: return multiserver.start_api_server(key, backend_port or port) return start_moto_server_separate( key, port, name=name, backend_port=backend_port, asynchronous=asynchronous )
def start_kms(port=None, backend_port=None, asynchronous=None, update_listener=None): port = port or config.PORT_KMS backend_port = get_free_tcp_port() kms_binary = INSTALL_PATH_KMS_BINARY_PATTERN.replace('<arch>', get_arch()) log_startup_message('KMS') start_proxy_for_service('kms', port, backend_port, update_listener) env_vars = { 'PORT': str(backend_port), 'KMS_REGION': config.DEFAULT_REGION, 'REGION': config.DEFAULT_REGION, 'KMS_ACCOUNT_ID': TEST_AWS_ACCOUNT_ID, 'ACCOUNT_ID': TEST_AWS_ACCOUNT_ID } if config.DATA_DIR: env_vars['KMS_DATA_PATH'] = config.DATA_DIR result = do_run(kms_binary, asynchronous, env_vars=env_vars) wait_for_port_open(backend_port) return result
def start_swf(port=None, backend_port=None, asynchronous=None, update_listener=None): port = port or config.PORT_SWF if not backend_port: if config.FORWARD_EDGE_INMEM: backend_port = multiserver.get_moto_server_port() else: backend_port = get_free_tcp_port() swf_listener.PORT_SWF_BACKEND = backend_port return start_moto_server(key='swf', name='SWF', asynchronous=asynchronous, port=port, backend_port=backend_port, update_listener=update_listener)
def test_http2_traffic(self): port = get_free_tcp_port() class MyListener(ProxyListener): def forward_request(self, method, path, data, headers): return {"method": method, "path": path, "data": data} url = "https://localhost:%s/foo/bar" % port listener = MyListener() proxy = start_proxy_server(port, update_listener=listener, use_ssl=True) time.sleep(1) response = requests.post(url, verify=False) expected = {"method": "POST", "path": "/foo/bar", "data": ""} assert json.loads(to_str(response.content)) == expected proxy.stop()
def _create_cluster(self, arn, url, version, create_domain_request) -> Server: with self.mutex: if not self.cluster: # startup routine for the singleton cluster instance self.cluster = ElasticsearchCluster( port=get_free_tcp_port(), directories=resolve_directories(version, arn)) def _start_async(*_): LOG.info("starting %s on %s", type(self.cluster), self.cluster.url) self.cluster.start() # start may block during install start_thread(_start_async) return ClusterEndpoint(self.cluster, EndpointProxy(url, self.cluster.url))
def test_forward_to_fallback_url_http(self): class MyUpdateListener(ProxyListener): def forward_request(self, method, path, data, headers): records.append(data) response = Response() response.status_code = 200 response._content = '' return response records = [] local_port = get_free_tcp_port() proxy = start_proxy(local_port, backend_url=None, update_listener=MyUpdateListener()) items_before = len(records) self.run_forward_to_fallback_url('%s://localhost:%s' % (get_service_protocol(), local_port)) items_after = len(records) self.assertEqual(items_after, items_before + 3) proxy.stop()
def start_kinesis(port=None, asynchronous=False, update_listener=None): port = port or config.PORT_KINESIS install.install_kinesalite() backend_port = get_free_tcp_port() latency = config.KINESIS_LATENCY kinesis_data_dir_param = '' if config.DATA_DIR: kinesis_data_dir = '%s/kinesis' % config.DATA_DIR mkdir(kinesis_data_dir) kinesis_data_dir_param = '--path %s' % kinesis_data_dir cmd = ('%s/node_modules/kinesalite/cli.js --shardLimit %s --port %s' ' --createStreamMs %s --deleteStreamMs %s --updateStreamMs %s %s' ) % (ROOT_PATH, config.KINESIS_SHARD_LIMIT, backend_port, latency, latency, latency, kinesis_data_dir_param) print('Starting mock Kinesis service on %s port %s...' % (get_service_protocol(), config.EDGE_PORT)) start_proxy_for_service('kinesis', port, backend_port, update_listener) return do_run(cmd, asynchronous)
def test_download_with_timeout(): class DownloadListener(ProxyListener): def forward_request(self, method, path, data, headers): if path == "/sleep": time.sleep(2) return {} port = get_free_tcp_port() proxy = start_proxy_server(port, update_listener=DownloadListener()) tmp_file = new_tmp_file() download(f"http://localhost:{port}/", tmp_file) assert load_file(tmp_file) == "{}" with pytest.raises(TimeoutError): download(f"http://localhost:{port}/sleep", tmp_file, timeout=1) # clean up proxy.stop() rm_rf(tmp_file)
def test_run_and_stop_server(self): port = get_free_tcp_port() host = "127.0.0.1" LOG.info("%.2f starting server on port %d", time.time(), port) thread = run_server(port=port, bind_address=host, asynchronous=True) try: url = f"http://{host}:{port}" assert poll_condition( lambda: is_port_open(url, http_path="/"), timeout=15), f"gave up waiting for port {port}" finally: LOG.info("%.2f stopping server on port %d", time.time(), port) thread.stop() LOG.info("%.2f waiting on server to shut down", time.time()) thread.join(timeout=15) assert not is_port_open(port), "port is still open after stop" LOG.info("%.2f port stopped %d", time.time(), port)
def get_moto_server(timeout=10) -> MotoServer: """ Returns the MotoServer singleton or creates it and waits for it to become ready. """ global _server, _mutex with _mutex: if _server: return _server _server = MotoServer(port=get_free_tcp_port(), host=constants.BIND_HOST) _server.start() if not _server.wait_is_up(timeout): raise TimeoutError("gave up waiting for moto server on %s" % _server.url) return _server
def start_dynamodb(port=None, asynchronous=False, update_listener=None): global PORT_DYNAMODB_BACKEND PORT_DYNAMODB_BACKEND = get_free_tcp_port() port = port or config.PORT_DYNAMODB install.install_dynamodb_local() ddb_data_dir_param = '-inMemory' if config.DATA_DIR: ddb_data_dir = '%s/dynamodb' % config.DATA_DIR mkdir(ddb_data_dir) # as the service command cds into a different directory, the absolute # path of the DATA_DIR is needed as the -dbPath absolute_path = os.path.abspath(ddb_data_dir) ddb_data_dir_param = '-dbPath %s' % absolute_path cmd = ('cd %s/infra/dynamodb/; java -Djava.library.path=./DynamoDBLocal_lib ' + '-Xmx%s -jar DynamoDBLocal.jar -port %s %s') % ( ROOT_PATH, config.DYNAMODB_HEAP_SIZE, PORT_DYNAMODB_BACKEND, ddb_data_dir_param) log_startup_message('DynamoDB') start_proxy_for_service('dynamodb', port, backend_port=PORT_DYNAMODB_BACKEND, update_listener=update_listener) return do_run(cmd, asynchronous, auto_restart=True)
def do_run(self): self.cluster_port = get_free_tcp_port() self.cluster = ElasticsearchCluster( port=self.cluster_port, host=DEFAULT_BACKEND_HOST, version=self.version, directories=self.directories, ) self.cluster.start() self.proxy = EndpointProxy(self.url, self.cluster.url) LOG.info("registering an endpoint proxy for %s => %s", self.url, self.cluster.url) self.proxy.register() self.cluster.wait_is_up() LOG.info("elasticsearch cluster on %s is ready", self.cluster.url) return self.cluster.join()
def start_s3(port=None, backend_port=None, asynchronous=None, update_listener=None): port = port or config.PORT_S3 if not backend_port: if config.FORWARD_EDGE_INMEM: backend_port = multiserver.get_moto_server_port() else: backend_port = get_free_tcp_port() s3_listener.PORT_S3_BACKEND = backend_port apply_patches() return start_moto_server( key="s3", name="S3", asynchronous=asynchronous, port=port, backend_port=backend_port, update_listener=update_listener, )
def create_kinesalite_server(port=None) -> KinesaliteServer: """ Creates a new Kinesalite server instance. Installs Kinesalite on the host first if necessary. Introspects on the host config to determine server configuration: config.dirs.data -> if set, the server runs with persistence using the path to store data config.KINESIS_LATENCY -> configure stream latency (in milliseconds) """ port = port or get_free_tcp_port() install.install_kinesalite() if config.dirs.data: kinesis_data_dir = "%s/kinesis" % config.dirs.data mkdir(kinesis_data_dir) else: kinesis_data_dir = None return KinesaliteServer(port=port, latency=config.KINESIS_LATENCY, data_dir=kinesis_data_dir)
def start_kinesis_mock(port=None, asynchronous=False, update_listener=None): kinesis_mock_bin = install.install_kinesis_mock() port = port or config.PORT_KINESIS backend_port = get_free_tcp_port() kinesis_data_dir_param = '' if config.DATA_DIR: kinesis_data_dir = '%s/kinesis' % config.DATA_DIR mkdir(kinesis_data_dir) kinesis_data_dir_param = 'SHOULD_PERSIST_DATA=true PERSIST_PATH=%s' % kinesis_data_dir if not config.LS_LOG: log_level = 'INFO' elif config.LS_LOG == 'warning': log_level = 'WARN' else: log_level = config.LS_LOG.upper() log_level_param = 'LOG_LEVEL=%s' % log_level latency = config.KINESIS_LATENCY + 'ms' latency_param = 'CREATE_STREAM_DURATION=%s DELETE_STREAM_DURATION=%s REGISTER_STREAM_CONSUMER_DURATION=%s ' \ 'START_STREAM_ENCRYPTION_DURATION=%s STOP_STREAM_ENCRYPTION_DURATION=%s ' \ 'DEREGISTER_STREAM_CONSUMER_DURATION=%s MERGE_SHARDS_DURATION=%s SPLIT_SHARD_DURATION=%s ' \ 'UPDATE_SHARD_COUNT_DURATION=%s' \ % (latency, latency, latency, latency, latency, latency, latency, latency, latency) if config.KINESIS_INITIALIZE_STREAMS != '': initialize_streams_param = 'INITIALIZE_STREAMS=%s' % ( config.KINESIS_INITIALIZE_STREAMS) else: initialize_streams_param = '' if kinesis_mock_bin.endswith('.jar'): cmd = 'KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s java -XX:+UseG1GC -jar %s' \ % (backend_port, config.KINESIS_SHARD_LIMIT, latency_param, kinesis_data_dir_param, log_level_param, initialize_streams_param, kinesis_mock_bin) else: chmod_r(kinesis_mock_bin, 0o777) cmd = 'KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s %s --gc=G1' \ % (backend_port, config.KINESIS_SHARD_LIMIT, latency_param, kinesis_data_dir_param, log_level_param, initialize_streams_param, kinesis_mock_bin) LOGGER.info('starting kinesis-mock proxy %d:%d with cmd: %s', port, backend_port, cmd) start_proxy_for_service('kinesis', port, backend_port, update_listener) return do_run(cmd, asynchronous)
def start_kms(port=None, backend_port=None, asynchronous=None, update_listener=None): port = port or config.PORT_KMS backend_port = get_free_tcp_port() kms_binary = INSTALL_PATH_KMS_BINARY_PATTERN.replace('<arch>', get_arch()) print('Starting mock KMS service on %s ...' % edge_ports_info()) start_proxy_for_service('kms', port, backend_port, update_listener) env_vars = { 'PORT': str(backend_port), 'KMS_REGION': config.DEFAULT_REGION, 'REGION': config.DEFAULT_REGION, 'KMS_ACCOUNT_ID': TEST_AWS_ACCOUNT_ID, 'ACCOUNT_ID': TEST_AWS_ACCOUNT_ID } result = do_run(kms_binary, asynchronous, env_vars=env_vars) wait_for_port_open(backend_port) return result
def create_kinesis_mock_server(port=None, persist_path: Optional[str] = None ) -> KinesisMockServer: """ Creates a new Kinesis Mock server instance. Installs Kinesis Mock on the host first if necessary. Introspects on the host config to determine server configuration: config.dirs.data -> if set, the server runs with persistence using the path to store data config.LS_LOG -> configure kinesis mock log level (defaults to INFO) config.KINESIS_LATENCY -> configure stream latency (in milliseconds) config.KINESIS_INITIALIZE_STREAMS -> Initialize the given streams on startup """ port = port or get_free_tcp_port() is_kinesis_mock_installed, kinesis_mock_bin_path = install.get_is_kinesis_mock_installed( ) if not is_kinesis_mock_installed: install.install_kinesis_mock(kinesis_mock_bin_path) persist_path = (f"{config.dirs.data}/kinesis" if not persist_path and config.dirs.data else persist_path) if persist_path: mkdir(persist_path) if config.LS_LOG: if config.LS_LOG == "warning": log_level = "WARN" else: log_level = config.LS_LOG.upper() else: log_level = "INFO" latency = config.KINESIS_LATENCY + "ms" initialize_streams = (config.KINESIS_INITIALIZE_STREAMS if config.KINESIS_INITIALIZE_STREAMS else None) server = KinesisMockServer( port=port, bin_path=kinesis_mock_bin_path, log_level=log_level, latency=latency, initialize_streams=initialize_streams, data_dir=persist_path, ) return server
def start_kinesalite(port=None, asynchronous=False, update_listener=None): # install and apply patches install.install_kinesalite() apply_patches_kinesalite() # start up process port = port or config.PORT_KINESIS backend_port = get_free_tcp_port() latency = config.KINESIS_LATENCY kinesis_data_dir_param = '' if config.DATA_DIR: kinesis_data_dir = '%s/kinesis' % config.DATA_DIR mkdir(kinesis_data_dir) kinesis_data_dir_param = '--path %s' % kinesis_data_dir cmd = ('%s/node_modules/kinesalite/cli.js --shardLimit %s --port %s' ' --createStreamMs %s --deleteStreamMs %s --updateStreamMs %s %s' ) % (MODULE_MAIN_PATH, config.KINESIS_SHARD_LIMIT, backend_port, latency, latency, latency, kinesis_data_dir_param) log_startup_message('Kinesis') start_proxy_for_service('kinesis', port, backend_port, update_listener) return do_run(cmd, asynchronous)
def start_moto_server(key, port, name=None, backend_port=None, asynchronous=False, update_listener=None): moto_server_cmd = '%s/bin/moto_server' % LOCALSTACK_VENV_FOLDER if not os.path.exists(moto_server_cmd): moto_server_cmd = run('which moto_server').strip() if USE_SSL and not backend_port: backend_port = get_free_tcp_port() cmd = 'VALIDATE_LAMBDA_S3=0 %s %s -p %s -H %s' % ( moto_server_cmd, key, backend_port or port, constants.BIND_HOST) if not name: name = key print('Starting mock %s (%s port %s)...' % (name, get_service_protocol(), port)) if backend_port: start_proxy_for_service(key, port, backend_port, update_listener) return do_run(cmd, asynchronous)
def test_subscribe_http_endpoint(self): # create HTTP endpoint and connect it to SNS topic class MyUpdateListener(ProxyListener): def forward_request(self, method, path, data, headers): records.append((json.loads(to_str(data)), headers)) return 200 records = [] local_port = get_free_tcp_port() proxy = start_proxy(local_port, backend_url=None, update_listener=MyUpdateListener()) wait_for_port_open(local_port) queue_arn = '%s://localhost:%s' % (get_service_protocol(), local_port) self.sns_client.subscribe(TopicArn=self.topic_arn, Protocol='http', Endpoint=queue_arn) def received(): assert records[0][0]['Type'] == 'SubscriptionConfirmation' assert records[0][1]['x-amz-sns-message-type'] == 'SubscriptionConfirmation' retry(received, retries=5, sleep=1) proxy.stop()
def test_http2_traffic(self): port = get_free_tcp_port() class MyListener(ProxyListener): def forward_request(self, method, path, data, headers): return {'method': method, 'path': path, 'data': data} url = 'https://localhost:%s/foo/bar' % port listener = MyListener() run_proxy_server_http2(port, listener=listener, asynchronous=True, use_ssl=True) time.sleep(1) response = requests.post(url, verify=False) self.assertEqual({ 'method': 'POST', 'path': '/foo/bar', 'data': '' }, json.loads(to_str(response.content)))
def test_forward_raw_path(self, monkeypatch): class MyListener(ProxyListener): def forward_request(self, method, path, data, headers): _path = get_full_raw_path(quart_request) return {"method": method, "path": _path} # start listener and configure EDGE_FORWARD_URL port = get_free_tcp_port() forward_url = f"http://localhost:{port}" listener = MyListener() proxy = start_proxy_server(port, update_listener=listener, use_ssl=True) monkeypatch.setattr(config, "EDGE_FORWARD_URL", forward_url) # run test request, assert that raw request path is forwarded test_arn = "arn:aws:test:resource/test" raw_path = f"/test/{urllib.parse.quote(test_arn)}/bar?q1=foo&q2=bar" url = f"{config.get_edge_url()}{raw_path}" response = requests.get(url) expected = {"method": "GET", "path": raw_path} assert json.loads(to_str(response.content)) == expected proxy.stop()
def start_moto_server(key, port, name=None, backend_port=None, asynchronous=False, update_listener=None): if not name: name = key print('Starting mock %s (%s port %s)...' % (name, get_service_protocol(), port)) if config.USE_SSL and not backend_port: backend_port = get_free_tcp_port() if backend_port: start_proxy_for_service(key, port, backend_port, update_listener) if config.BUNDLE_API_PROCESSES: return multiserver.start_api_server(key, backend_port or port) return start_moto_server_separate(key, port, name=name, backend_port=backend_port, asynchronous=asynchronous)
def start_moto_server( key, port, name=None, backend_port=None, asynchronous=False, update_listener=None ) -> MotoServerProperties: # TODO: refactor this method! the name and parameters suggest that a server is started, but it actually only adds # a proxy listener around the already started motoserver singleton. # TODO: remove asynchronous parameter (from all calls to this function) # TODO: re-think backend_port parameter (still needed since determined by motoserver singleton?) if not name: name = key log_startup_message(name) if not backend_port: if config.FORWARD_EDGE_INMEM: backend_port = motoserver.get_moto_server().port elif config.USE_SSL or update_listener: backend_port = get_free_tcp_port() if backend_port or config.FORWARD_EDGE_INMEM: start_proxy_for_service(key, port, backend_port, update_listener) server = motoserver.get_moto_server() return MotoServerProperties(server._thread, server.port)
def test_api_gateway_http_integration_with_path_request_parmeter(self): client = aws_stack.connect_to_service('apigateway') test_port = get_free_tcp_port() backend_url = 'http://localhost:%s/person/{id}' % (test_port) # start test HTTP backend proxy = self.start_http_backend(test_port) # create rest api api_rest = client.create_rest_api(name='test') api_id = api_rest['id'] parent_response = client.get_resources(restApiId=api_id) parent_id = parent_response['items'][0]['id'] resource_1 = client.create_resource(restApiId=api_id, parentId=parent_id, pathPart='person') resource_1_id = resource_1['id'] resource_2 = client.create_resource(restApiId=api_id, parentId=resource_1_id, pathPart='{id}') resource_2_id = resource_2['id'] client.put_method( restApiId=api_id, resourceId=resource_2_id, httpMethod='GET', authorizationType='NONE', apiKeyRequired=False, requestParameters={'method.request.path.id': True}) client.put_integration( restApiId=api_id, resourceId=resource_2_id, httpMethod='GET', integrationHttpMethod='GET', type='HTTP', uri=backend_url, timeoutInMillis=3000, contentHandling='CONVERT_TO_BINARY', requestParameters={'integration.request.path.id': 'method.request.path.id'}) client.create_deployment(restApiId=api_id, stageName='test') url = f'http://localhost:{config.EDGE_PORT}/restapis/{api_id}/test/_user_request_/person/123' result = requests.get(url) content = json.loads(result._content) self.assertEqual(result.status_code, 200) self.assertEqual(content['headers'].get(HEADER_LOCALSTACK_REQUEST_URL), f'http://localhost:{config.EDGE_PORT}/person/123') # clean up client.delete_rest_api(restApiId=api_id) proxy.stop()
def test_static_route(self): class MyListener(ProxyListener): def forward_request(self, method, path, *args, **kwargs): return {"method": method, "path": path} # start proxy server listener = MyListener() port = get_free_tcp_port() server = start_proxy_server(port, update_listener=listener) wait_for_port_open(port) # request a /static/... path from the server and assert result url = f"http://{LOCALHOST_HOSTNAME}:{port}/static/index.html" response = requests.get(url, verify=False) assert response.ok assert json.loads(to_str(response.content)) == { "method": "GET", "path": "/static/index.html", } # clean up server.stop()
def start_moto_server(key, port, name=None, backend_port=None, asynchronous=False, update_listener=None): if not name: name = key print( 'Starting mock %s service on %s ports %s (recommended) and %s (deprecated)...' % (name, get_service_protocol(), config.EDGE_PORT, port)) if not backend_port and (config.USE_SSL or update_listener): backend_port = get_free_tcp_port() if backend_port: start_proxy_for_service(key, port, backend_port, update_listener) if config.BUNDLE_API_PROCESSES: return multiserver.start_api_server(key, backend_port or port) return start_moto_server_separate(key, port, name=name, backend_port=backend_port, asynchronous=asynchronous)