def get_stream_info(stream_name, log_file=None, shards=None, env=None, endpoint_url=None, ddb_lease_table_suffix=None, env_vars={}): if not ddb_lease_table_suffix: ddb_lease_table_suffix = DEFAULT_DDB_LEASE_TABLE_SUFFIX # construct stream info env = aws_stack.get_environment(env) props_file = os.path.join(tempfile.gettempdir(), 'kclipy.%s.properties' % short_uid()) # make sure to convert stream ARN to stream name stream_name = aws_stack.kinesis_stream_name(stream_name) app_name = '%s%s' % (stream_name, ddb_lease_table_suffix) stream_info = { 'name': stream_name, 'region': aws_stack.get_region(), 'shards': shards, 'properties_file': props_file, 'log_file': log_file, 'app_name': app_name, 'env_vars': env_vars } # set local connection if aws_stack.is_local_env(env): stream_info['conn_kwargs'] = { 'host': HOSTNAME, 'port': config.PORT_KINESIS, 'is_secure': bool(USE_SSL) } if endpoint_url: if 'conn_kwargs' not in stream_info: stream_info['conn_kwargs'] = {} url = urlparse(endpoint_url) stream_info['conn_kwargs']['host'] = url.hostname stream_info['conn_kwargs']['port'] = url.port stream_info['conn_kwargs']['is_secure'] = url.scheme == 'https' return stream_info
def start_kcl_client_process(stream_name, listener_script, log_file=None, env=None, configs={}, endpoint_url=None, ddb_lease_table_suffix=None, env_vars={}, region_name=None, kcl_log_level=DEFAULT_KCL_LOG_LEVEL, log_subscribers=[]): env = aws_stack.get_environment(env) # make sure to convert stream ARN to stream name stream_name = aws_stack.kinesis_stream_name(stream_name) # decide which credentials provider to use credentialsProvider = None if (('AWS_ASSUME_ROLE_ARN' in os.environ or 'AWS_ASSUME_ROLE_ARN' in env_vars) and ('AWS_ASSUME_ROLE_SESSION_NAME' in os.environ or 'AWS_ASSUME_ROLE_SESSION_NAME' in env_vars)): # use special credentials provider that can assume IAM roles and handle temporary STS auth tokens credentialsProvider = 'cloud.localstack.DefaultSTSAssumeRoleSessionCredentialsProvider' # pass through env variables to child process for var_name in ['AWS_ASSUME_ROLE_ARN', 'AWS_ASSUME_ROLE_SESSION_NAME', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN']: if var_name in os.environ and var_name not in env_vars: env_vars[var_name] = os.environ[var_name] if aws_stack.is_local_env(env): # need to disable CBOR protocol, enforce use of plain JSON, # see https://github.com/mhart/kinesalite/issues/31 env_vars['AWS_CBOR_DISABLE'] = 'true' if kcl_log_level or (len(log_subscribers) > 0): if not log_file: log_file = LOG_FILE_PATTERN.replace('*', short_uid()) TMP_FILES.append(log_file) run('touch %s' % log_file) # start log output reader thread which will read the KCL log # file and print each line to stdout of this process... reader_thread = OutputReaderThread({'file': log_file, 'level': kcl_log_level, 'log_prefix': 'KCL', 'log_subscribers': log_subscribers}) reader_thread.start() # construct stream info stream_info = get_stream_info(stream_name, log_file, env=env, endpoint_url=endpoint_url, ddb_lease_table_suffix=ddb_lease_table_suffix, env_vars=env_vars) props_file = stream_info['properties_file'] # set kcl config options kwargs = { 'metricsLevel': 'NONE', 'initialPositionInStream': 'LATEST' } # set parameters for local connection if aws_stack.is_local_env(env): kwargs['kinesisEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_KINESIS) kwargs['dynamodbEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_DYNAMODB) kwargs['kinesisProtocol'] = get_service_protocol() kwargs['dynamodbProtocol'] = get_service_protocol() kwargs['disableCertChecking'] = 'true' kwargs.update(configs) # create config file kclipy_helper.create_config_file(config_file=props_file, executableName=listener_script, streamName=stream_name, applicationName=stream_info['app_name'], credentialsProvider=credentialsProvider, region_name=region_name, **kwargs) TMP_FILES.append(props_file) # start stream consumer stream = KinesisStream(id=stream_name, params=stream_info) thread_consumer = KinesisProcessorThread.start_consumer(stream) TMP_THREADS.append(thread_consumer) return thread_consumer
def put_log_events_model(self, log_group_name, log_stream_name, log_events, sequence_token): # TODO: call/patch upstream method here, instead of duplicating the code! self.last_ingestion_time = int(unix_time_millis()) self.stored_bytes += sum( [len(log_event["message"]) for log_event in log_events]) events = [ logs_models.LogEvent(self.last_ingestion_time, log_event) for log_event in log_events ] self.events += events self.upload_sequence_token += 1 log_events = [{ "id": event.event_id, "timestamp": event.timestamp, "message": event.message, } for event in events] data = { "messageType": "DATA_MESSAGE", "owner": aws_stack.get_account_id(), "logGroup": log_group_name, "logStream": log_stream_name, "subscriptionFilters": [self.filter_name], "logEvents": log_events, } output = io.BytesIO() with GzipFile(fileobj=output, mode="w") as f: f.write(json.dumps(data, separators=(",", ":")).encode("utf-8")) payload_gz_encoded = base64.b64encode( output.getvalue()).decode("utf-8") event = {"awslogs": {"data": payload_gz_encoded}} if self.destination_arn: if ":lambda:" in self.destination_arn: client = aws_stack.connect_to_service("lambda") lambda_name = aws_stack.lambda_function_name( self.destination_arn) client.invoke(FunctionName=lambda_name, Payload=json.dumps(event)) if ":kinesis:" in self.destination_arn: client = aws_stack.connect_to_service("kinesis") stream_name = aws_stack.kinesis_stream_name( self.destination_arn) client.put_record( StreamName=stream_name, Data=json.dumps(payload_gz_encoded), PartitionKey=log_group_name, ) if ":firehose:" in self.destination_arn: client = aws_stack.connect_to_service("firehose") firehose_name = aws_stack.firehose_name(self.destination_arn) client.put_record( DeliveryStreamName=firehose_name, Record={"Data": json.dumps(payload_gz_encoded)}, )
def put_subscription_filter(*args, **kwargs): log_group_name = args[0] filter_name = args[1] filter_pattern = args[2] destination_arn = args[3] role_arn = args[4] log_group = logs_models.logs_backends[ aws_stack.get_region()].groups.get(log_group_name) if not log_group: raise ResourceNotFoundException( 'The specified log group does not exist.') if ':lambda:' in destination_arn: client = aws_stack.connect_to_service('lambda') lambda_name = aws_stack.lambda_function_name(destination_arn) try: client.get_function(FunctionName=lambda_name) except Exception: raise InvalidParameterException( 'destinationArn for vendor lambda cannot be used with roleArn' ) elif ':kinesis:' in destination_arn: client = aws_stack.connect_to_service('kinesis') stream_name = aws_stack.kinesis_stream_name(destination_arn) try: client.describe_stream(StreamName=stream_name) except Exception: raise InvalidParameterException( 'Could not deliver test message to specified Kinesis stream. ' 'Check if the given kinesis stream is in ACTIVE state. ' ) elif ':firehose:' in destination_arn: client = aws_stack.connect_to_service('firehose') firehose_name = aws_stack.firehose_name(destination_arn) try: client.describe_delivery_stream( DeliveryStreamName=firehose_name) except Exception: raise InvalidParameterException( 'Could not deliver test message to specified Firehose stream. ' 'Check if the given Firehose stream is in ACTIVE state.' ) else: service = aws_stack.extract_service_from_arn(destination_arn) raise InvalidParameterException( 'PutSubscriptionFilter operation cannot work with destinationArn for vendor %s' % service) log_group.put_subscription_filter(filter_name, filter_pattern, destination_arn, role_arn)
def moto_put_subscription_filter(fn, self, *args, **kwargs): log_group_name = args[0] filter_name = args[1] filter_pattern = args[2] destination_arn = args[3] role_arn = args[4] log_group = self.groups.get(log_group_name) if not log_group: raise ResourceNotFoundException("The specified log group does not exist.") if ":lambda:" in destination_arn: client = aws_stack.connect_to_service("lambda") lambda_name = aws_stack.lambda_function_name(destination_arn) try: client.get_function(FunctionName=lambda_name) except Exception: raise InvalidParameterException( "destinationArn for vendor lambda cannot be used with roleArn" ) elif ":kinesis:" in destination_arn: client = aws_stack.connect_to_service("kinesis") stream_name = aws_stack.kinesis_stream_name(destination_arn) try: client.describe_stream(StreamName=stream_name) except Exception: raise InvalidParameterException( "Could not deliver test message to specified Kinesis stream. " "Check if the given kinesis stream is in ACTIVE state. " ) elif ":firehose:" in destination_arn: client = aws_stack.connect_to_service("firehose") firehose_name = aws_stack.firehose_name(destination_arn) try: client.describe_delivery_stream(DeliveryStreamName=firehose_name) except Exception: raise InvalidParameterException( "Could not deliver test message to specified Firehose stream. " "Check if the given Firehose stream is in ACTIVE state." ) else: service = aws_stack.extract_service_from_arn(destination_arn) raise InvalidParameterException( f"PutSubscriptionFilter operation cannot work with destinationArn for vendor {service}" ) if filter_pattern: for stream in log_group.streams.values(): stream.filter_pattern = filter_pattern log_group.put_subscription_filter(filter_name, filter_pattern, destination_arn, role_arn)
def put_log_events_model(self, log_group_name, log_stream_name, log_events, sequence_token): self.lastIngestionTime = int(unix_time_millis()) self.storedBytes += sum( [len(log_event['message']) for log_event in log_events]) events = [ logs_models.LogEvent(self.lastIngestionTime, log_event) for log_event in log_events ] self.events += events self.uploadSequenceToken += 1 log_events = [{ 'id': event.eventId, 'timestamp': event.timestamp, 'message': event.message, } for event in events] data = { 'messageType': 'DATA_MESSAGE', 'owner': aws_stack.get_account_id(), 'logGroup': log_group_name, 'logStream': log_stream_name, 'subscriptionFilters': [self.filter_name], 'logEvents': log_events, } output = io.BytesIO() with GzipFile(fileobj=output, mode='w') as f: f.write(json.dumps(data, separators=(',', ':')).encode('utf-8')) payload_gz_encoded = base64.b64encode( output.getvalue()).decode('utf-8') event = {'awslogs': {'data': payload_gz_encoded}} if self.destination_arn: if ':lambda:' in self.destination_arn: client = aws_stack.connect_to_service('lambda') lambda_name = aws_stack.lambda_function_name( self.destination_arn) client.invoke(FunctionName=lambda_name, Payload=json.dumps(event)) if ':kinesis:' in self.destination_arn: client = aws_stack.connect_to_service('kinesis') stream_name = aws_stack.kinesis_stream_name( self.destination_arn) client.put_record(StreamName=stream_name, Data=json.dumps(payload_gz_encoded), PartitionKey=log_group_name) if ':firehose:' in self.destination_arn: client = aws_stack.connect_to_service('firehose') firehose_name = aws_stack.firehose_name(self.destination_arn) client.put_record( DeliveryStreamName=firehose_name, Record={'Data': json.dumps(payload_gz_encoded)})
def get_stream_info( stream_name, log_file=None, shards=None, env=None, endpoint_url=None, ddb_lease_table_suffix=None, env_vars=None, ): if env_vars is None: env_vars = {} if not ddb_lease_table_suffix: ddb_lease_table_suffix = DEFAULT_DDB_LEASE_TABLE_SUFFIX # construct stream info env = aws_stack.get_environment(env) props_file = os.path.join(tempfile.gettempdir(), "kclipy.%s.properties" % short_uid()) # make sure to convert stream ARN to stream name stream_name = aws_stack.kinesis_stream_name(stream_name) app_name = "%s%s" % (stream_name, ddb_lease_table_suffix) stream_info = { "name": stream_name, "region": aws_stack.get_region(), "shards": shards, "properties_file": props_file, "log_file": log_file, "app_name": app_name, "env_vars": env_vars, } # set local connection if aws_stack.is_local_env(env): stream_info["conn_kwargs"] = { "host": LOCALHOST, "port": config.service_port("kinesis"), "is_secure": bool(config.USE_SSL), } if endpoint_url: if "conn_kwargs" not in stream_info: stream_info["conn_kwargs"] = {} url = urlparse(endpoint_url) stream_info["conn_kwargs"]["host"] = url.hostname stream_info["conn_kwargs"]["port"] = url.port stream_info["conn_kwargs"]["is_secure"] = url.scheme == "https" return stream_info
def start_kcl_client_process( stream_name, listener_script, log_file=None, env=None, configs=None, endpoint_url=None, ddb_lease_table_suffix=None, env_vars=None, region_name=None, kcl_log_level=DEFAULT_KCL_LOG_LEVEL, log_subscribers=None, ): if configs is None: configs = {} if env_vars is None: env_vars = {} if log_subscribers is None: log_subscribers = [] env = aws_stack.get_environment(env) # make sure to convert stream ARN to stream name stream_name = aws_stack.kinesis_stream_name(stream_name) # decide which credentials provider to use credentialsProvider = None if ("AWS_ASSUME_ROLE_ARN" in os.environ or "AWS_ASSUME_ROLE_ARN" in env_vars) and ("AWS_ASSUME_ROLE_SESSION_NAME" in os.environ or "AWS_ASSUME_ROLE_SESSION_NAME" in env_vars): # use special credentials provider that can assume IAM roles and handle temporary STS auth tokens credentialsProvider = "cloud.localstack.DefaultSTSAssumeRoleSessionCredentialsProvider" # pass through env variables to child process for var_name in [ "AWS_ASSUME_ROLE_ARN", "AWS_ASSUME_ROLE_SESSION_NAME", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", ]: if var_name in os.environ and var_name not in env_vars: env_vars[var_name] = os.environ[var_name] if aws_stack.is_local_env(env): # need to disable CBOR protocol, enforce use of plain JSON, # see https://github.com/mhart/kinesalite/issues/31 env_vars["AWS_CBOR_DISABLE"] = "true" if kcl_log_level or (len(log_subscribers) > 0): if not log_file: log_file = LOG_FILE_PATTERN.replace("*", short_uid()) TMP_FILES.append(log_file) run("touch %s" % log_file) # start log output reader thread which will read the KCL log # file and print each line to stdout of this process... reader_thread = OutputReaderThread({ "file": log_file, "level": kcl_log_level, "log_prefix": "KCL", "log_subscribers": log_subscribers, }) reader_thread.start() # construct stream info stream_info = get_stream_info( stream_name, log_file, env=env, endpoint_url=endpoint_url, ddb_lease_table_suffix=ddb_lease_table_suffix, env_vars=env_vars, ) props_file = stream_info["properties_file"] # set kcl config options kwargs = {"metricsLevel": "NONE", "initialPositionInStream": "LATEST"} # set parameters for local connection if aws_stack.is_local_env(env): kwargs[ "kinesisEndpoint"] = f"{LOCALHOST}:{config.service_port('kinesis')}" kwargs[ "dynamodbEndpoint"] = f"{LOCALHOST}:{config.service_port('dynamodb')}" kwargs["kinesisProtocol"] = config.get_protocol() kwargs["dynamodbProtocol"] = config.get_protocol() kwargs["disableCertChecking"] = "true" kwargs.update(configs) # create config file kclipy_helper.create_config_file( config_file=props_file, executableName=listener_script, streamName=stream_name, applicationName=stream_info["app_name"], credentialsProvider=credentialsProvider, region_name=region_name, **kwargs, ) TMP_FILES.append(props_file) # start stream consumer stream = KinesisStream(id=stream_name, params=stream_info) thread_consumer = KinesisProcessorThread.start_consumer(stream) TMP_THREADS.append(thread_consumer) return thread_consumer