def forward_request(self, method, path, data, headers): if method == 'OPTIONS': return 200 if path.split('?')[0] == '/health': return serve_health_endpoint(method, path, data) if method == 'POST' and path == '/graph': return serve_resource_graph(data) # kill the process if we receive this header headers.get(HEADER_KILL_SIGNAL) and os._exit(0) target = headers.get('x-amz-target', '') auth_header = headers.get('authorization', '') host = headers.get('host', '') headers[HEADER_LOCALSTACK_EDGE_URL] = 'https://%s' % host # extract API details api, port, path, host = get_api_from_headers(headers, path) set_default_region_in_headers(headers) if port and int(port) < 0: return 404 if not port: port = get_port_from_custom_rules(method, path, data, headers) or port if not port: if api in ['', None, '_unknown_']: truncated = truncate(data) LOG.info(( 'Unable to find forwarding rule for host "%s", path "%s", ' 'target header "%s", auth header "%s", data "%s"') % (host, path, target, auth_header, truncated)) else: LOG.info(( 'Unable to determine forwarding port for API "%s" - please ' 'make sure this API is enabled via the SERVICES configuration' ) % api) response = Response() response.status_code = 404 response._content = '{"status": "running"}' return response connect_host = '%s:%s' % (config.HOSTNAME, port) url = '%s://%s%s' % (get_service_protocol(), connect_host, path) headers['Host'] = host function = getattr(requests, method.lower()) if isinstance(data, dict): data = json.dumps(data) response = function(url, data=data, headers=headers, verify=False, stream=True) return response
def handle_connection(self, conn): socket_file = conn.makefile() while self.running: line = socket_file.readline() line = line[:-1] if line == '': # end of socket input stream break else: try: event = json.loads(line) records = event['records'] shard_id = event['shard_id'] method_args = inspect.getargspec(self.callback)[0] if len(method_args) > 2: self.callback(records, shard_id=shard_id, fh_d_stream=self.fh_d_stream) elif len(method_args) > 1: self.callback(records, shard_id=shard_id) else: self.callback(records) except Exception as e: LOGGER.warning( "Unable to process JSON line: '%s': %s %s. Callback: %s" % (truncate(line), e, traceback.format_exc(), self.callback)) conn.close()
def _put_to_search_db( self, db_flavor, db_description, delivery_stream_name, records, unprocessed_records ): """ sends Firehose records to an ElasticSearch or Opensearch database """ search_db_index = db_description["IndexName"] search_db_type = db_description.get("TypeName") region = aws_stack.get_region() domain_arn = db_description.get("DomainARN") cluster_endpoint = db_description.get("ClusterEndpoint") if cluster_endpoint is None: cluster_endpoint = aws_stack.get_opensearch_endpoint(domain_arn) db_connection = get_search_db_connection(cluster_endpoint, region) if db_description.get("S3BackupMode") == ElasticsearchS3BackupMode.AllDocuments: s3_dest_desc = db_description.get("S3DestinationDescription") if s3_dest_desc: try: self._put_records_to_s3_bucket( stream_name=delivery_stream_name, records=unprocessed_records, s3_destination_description=s3_dest_desc, ) except Exception as e: LOG.warning("Unable to backup unprocessed records to S3. Error: %s", e) else: LOG.warning("Passed S3BackupMode without S3Configuration. Cannot backup...") elif db_description.get("S3BackupMode") == ElasticsearchS3BackupMode.FailedDocumentsOnly: # TODO support FailedDocumentsOnly as well LOG.warning("S3BackupMode FailedDocumentsOnly is set but currently not supported.") for record in records: obj_id = uuid.uuid4() data = "{}" # DirectPut if "Data" in record: data = base64.b64decode(record["Data"]) # KinesisAsSource elif "data" in record: data = base64.b64decode(record["data"]) try: body = json.loads(data) except Exception as e: LOG.warning(f"{db_flavor} only allows json input data!") raise e LOG.debug( "Publishing to {} destination. Data: {}".format( db_flavor, truncate(data, max_length=300) ) ) try: db_connection.create( index=search_db_index, doc_type=search_db_type, id=obj_id, body=body ) except Exception as e: LOG.exception(f"Unable to put record to stream {delivery_stream_name}.") raise e
def raise_exception_if_error_response(response): if not is_response_obj(response): return if response.status_code < 400: return content = "..." try: content = truncate(to_str(response.content or "")) except Exception: pass # ignore if content has non-printable bytes raise Exception("Received error response (code %s): %s" % (response.status_code, content))
def forward_request(self, method, path, data, headers): if path.split('?')[0] == '/health': return serve_health_endpoint(method, path, data) if method == 'POST' and path == '/graph': return serve_resource_graph(data) # kill the process if we receive this header headers.get(HEADER_KILL_SIGNAL) and os._exit(0) target = headers.get('x-amz-target', '') auth_header = headers.get('authorization', '') host = headers.get('host', '') headers[HEADER_LOCALSTACK_EDGE_URL] = 'https://%s' % host # extract API details api, port, path, host = get_api_from_headers(headers, method=method, path=path, data=data) set_default_region_in_headers(headers) if port and int(port) < 0: return 404 if not port: api, port = get_api_from_custom_rules(method, path, data, headers) or (api, port) if not port: if method == 'OPTIONS': return 200 if api in ['', None, '_unknown_']: truncated = truncate(data) LOG.info(('Unable to find forwarding rule for host "%s", path "%s %s", ' 'target header "%s", auth header "%s", data "%s"') % ( host, method, path, target, auth_header, truncated)) else: LOG.info(('Unable to determine forwarding port for API "%s" - please ' 'make sure this API is enabled via the SERVICES configuration') % api) response = Response() response.status_code = 404 response._content = '{"status": "running"}' return response if api and not headers.get('Authorization'): headers['Authorization'] = aws_stack.mock_aws_request_headers(api)['Authorization'] headers['Host'] = host if isinstance(data, dict): data = json.dumps(data) return do_forward_request(api, port, method, path, data, headers)
def test_truncate(self): env = common.truncate('foobar', 3) self.assertEqual(env, 'foo...')
def forward_request(self, method, path, data, headers): if path.split('?')[0] == '/health': return serve_health_endpoint(method, path, data) if method == 'POST' and path == '/graph': return serve_resource_graph(data) # kill the process if we receive this header headers.get(HEADER_KILL_SIGNAL) and os._exit(0) target = headers.get('x-amz-target', '') auth_header = headers.get('authorization', '') host = headers.get('host', '') headers[HEADER_LOCALSTACK_EDGE_URL] = 'https://%s' % host # extract API details api, port, path, host = get_api_from_headers(headers, method=method, path=path, data=data) set_default_region_in_headers(headers) if port and int(port) < 0: return 404 if not port: api, port = get_api_from_custom_rules(method, path, data, headers) or (api, port) if not port: if method == 'OPTIONS': return 200 if api in ['', None, '_unknown_']: truncated = truncate(data) if auth_header or target or data or path not in ['/', '/favicon.ico']: LOG.info(('Unable to find forwarding rule for host "%s", path "%s %s", ' 'target header "%s", auth header "%s", data "%s"') % ( host, method, path, target, auth_header, truncated)) else: LOG.info(('Unable to determine forwarding port for API "%s" - please ' 'make sure this API is enabled via the SERVICES configuration') % api) response = Response() response.status_code = 404 response._content = '{"status": "running"}' return response if api and not headers.get('Authorization'): headers['Authorization'] = aws_stack.mock_aws_request_headers(api)['Authorization'] headers['Host'] = host if isinstance(data, dict): data = json.dumps(data) encoding_type = headers.get('content-encoding') or '' if encoding_type.upper() == GZIP_ENCODING and api is not S3: headers.set('content-encoding', IDENTITY_ENCODING) data = gzip.decompress(data) lock_ctx = BOOTSTRAP_LOCK if persistence.API_CALLS_RESTORED or is_internal_call_context(headers): lock_ctx = empty_context_manager() with lock_ctx: return do_forward_request(api, method, path, data, headers, port=port)
def test_truncate(self): env = common.truncate("foobar", 3) assert env == "foo..."
def forward_request(self, method, path, data, headers): if common.INFRA_STOPPED: return 503 if config.EDGE_FORWARD_URL: return do_forward_request_network( 0, method, path, data, headers, target_url=config.EDGE_FORWARD_URL) # kill the process if we receive this header headers.get(HEADER_KILL_SIGNAL) and sys.exit(0) target = headers.get("x-amz-target", "") auth_header = get_auth_string(method, path, headers, data) if auth_header and not headers.get("authorization"): headers["authorization"] = auth_header host = headers.get("host", "") orig_req_url = headers.pop(HEADER_LOCALSTACK_REQUEST_URL, "") headers[HEADER_LOCALSTACK_EDGE_URL] = (re.sub( r"^([^:]+://[^/]+).*", r"\1", orig_req_url) or "http://%s" % host) # extract API details api, port, path, host = get_api_from_headers(headers, method=method, path=path, data=data) set_default_region_in_headers(headers) if port and int(port) < 0: return 404 if not port: api, port = get_api_from_custom_rules(method, path, data, headers) or ( api, port, ) should_log_trace = is_trace_logging_enabled(headers) if api and should_log_trace: # print request trace for debugging, if enabled LOG.debug('IN(%s): "%s %s" - headers: %s - data: %s', api, method, path, dict(headers), data) if not port: if method == "OPTIONS": if api and should_log_trace: # print request trace for debugging, if enabled LOG.debug('IN(%s): "%s %s" - status: %s', api, method, path, 200) return 200 if api in ["", None, API_UNKNOWN]: truncated = truncate(data) if auth_header or target or data or path not in [ "/", "/favicon.ico" ]: LOG.info( ('Unable to find forwarding rule for host "%s", path "%s %s", ' 'target header "%s", auth header "%s", data "%s"'), host, method, path, target, auth_header, truncated, ) else: LOG.info( ('Unable to determine forwarding port for API "%s" - please ' "make sure this API is enabled via the SERVICES configuration" ), api, ) response = Response() response.status_code = 404 response._content = '{"status": "running"}' return response if api and not headers.get("Authorization"): headers["Authorization"] = aws_stack.mock_aws_request_headers( api)["Authorization"] headers[HEADER_TARGET_API] = str(api) headers["Host"] = host if isinstance(data, dict): data = json.dumps(data) encoding_type = headers.get("Content-Encoding") or "" if encoding_type.upper() == GZIP_ENCODING.upper( ) and api not in SKIP_GZIP_APIS: headers.set("Content-Encoding", IDENTITY_ENCODING) data = gzip.decompress(data) is_internal_call = is_internal_call_context(headers) self._require_service(api) lock_ctx = BOOTSTRAP_LOCK if is_internal_call or persistence.is_persistence_restored(): lock_ctx = empty_context_manager() with lock_ctx: result = do_forward_request(api, method, path, data, headers, port=port) if should_log_trace and result not in [None, False, True]: result_status_code = getattr(result, "status_code", result) result_headers = getattr(result, "headers", {}) result_content = getattr(result, "content", "") LOG.debug( 'OUT(%s): "%s %s" - status: %s - response headers: %s - response: %s', api, method, path, result_status_code, dict(result_headers or {}), result_content, ) return result
def test_truncate(self): env = common.truncate("foobar", 3) self.assertEqual("foo...", env)
def forward_request(self, method, path, data, headers): if path.split('?')[0] == '/health': return serve_health_endpoint(method, path, data) if method == 'POST' and path == '/graph': return serve_resource_graph(data) # kill the process if we receive this header headers.get(HEADER_KILL_SIGNAL) and sys.exit(0) target = headers.get('x-amz-target', '') auth_header = get_auth_string(method, path, headers, data) if auth_header and not headers.get('authorization'): headers['authorization'] = auth_header host = headers.get('host', '') orig_req_url = headers.pop(HEADER_LOCALSTACK_REQUEST_URL, '') headers[HEADER_LOCALSTACK_EDGE_URL] = (re.sub( r'^([^:]+://[^/]+).*', r'\1', orig_req_url) or 'http://%s' % host) # extract API details api, port, path, host = get_api_from_headers(headers, method=method, path=path, data=data) if api and config.LS_LOG: # print request trace for debugging, if enabled LOG.debug('IN(%s): "%s %s" - headers: %s - data: %s' % (api, method, path, dict(headers), data)) set_default_region_in_headers(headers) if port and int(port) < 0: return 404 if not port: api, port = get_api_from_custom_rules(method, path, data, headers) or (api, port) if not port: if method == 'OPTIONS': if api and config.LS_LOG: # print request trace for debugging, if enabled LOG.debug('OUT(%s): "%s %s" - status: %s' % (api, method, path, 200)) return 200 if api in ['', None, API_UNKNOWN]: truncated = truncate(data) if auth_header or target or data or path not in [ '/', '/favicon.ico' ]: LOG.info(( 'Unable to find forwarding rule for host "%s", path "%s %s", ' 'target header "%s", auth header "%s", data "%s"') % (host, method, path, target, auth_header, truncated)) else: LOG.info(( 'Unable to determine forwarding port for API "%s" - please ' 'make sure this API is enabled via the SERVICES configuration' ) % api) response = Response() response.status_code = 404 response._content = '{"status": "running"}' return response if api and not headers.get('Authorization'): headers['Authorization'] = aws_stack.mock_aws_request_headers( api)['Authorization'] headers[HEADER_TARGET_API] = str(api) headers['Host'] = host if isinstance(data, dict): data = json.dumps(data) encoding_type = headers.get('Content-Encoding') or '' if encoding_type.upper() == GZIP_ENCODING.upper() and api not in [S3]: headers.set('Content-Encoding', IDENTITY_ENCODING) data = gzip.decompress(data) lock_ctx = BOOTSTRAP_LOCK if persistence.API_CALLS_RESTORED or is_internal_call_context(headers): lock_ctx = empty_context_manager() with lock_ctx: return do_forward_request(api, method, path, data, headers, port=port)