def __init__(self, region, name, tags):
self.name = name
self.region = region
self.arn = "arn:aws:logs:{region}:1:log-group:{log_group}".format(
region=region, log_group=name)
self.creationTime = unix_time_millis()
self.tags = tags
self.streams = dict() # {name: LogStream}
def __init__(self, region, name, tags):
self.name = name
self.region = region
self.arn = "arn:aws:logs:{region}:1:log-group:{log_group}".format(
region=region, log_group=name)
self.creationTime = unix_time_millis()
self.tags = tags
self.streams = dict() # {name: LogStream}
self.retentionInDays = None # AWS defaults to Never Expire for log group retention
def put_log_events(self, log_group_name, log_stream_name, log_events, sequence_token):
# TODO: ensure sequence_token
# TODO: to be thread safe this would need a lock
self.lastIngestionTime = unix_time_millis()
# TODO: make this match AWS if possible
self.storedBytes += sum([len(log_event["message"]) for log_event in log_events])
self.events += [LogEvent(self.lastIngestionTime, log_event) for log_event in log_events]
self.uploadSequenceToken += 1
return self.uploadSequenceToken
def __init__(self, region, log_group, name):
self.region = region
self.arn = "arn:aws:logs:{region}:{id}:log-group:{log_group}:log-stream:{log_stream}".format(
region=region, id=self.__class__._log_ids, log_group=log_group, log_stream=name)
self.creationTime = unix_time_millis()
self.firstEventTimestamp = None
self.lastEventTimestamp = None
self.lastIngestionTime = None
self.logStreamName = name
self.storedBytes = 0
self.uploadSequenceToken = 0 # I'm guessing this is token needed for sequenceToken by put_events
self.events = []
self.__class__._log_ids += 1
def mark_received(self, visibility_timeout=None):
"""
When a message is received we will set the first receive timestamp,
tap the ``approximate_receive_count`` and the ``visible_at`` time.
"""
if visibility_timeout:
visibility_timeout = int(visibility_timeout)
else:
visibility_timeout = 0
if not self.approximate_first_receive_timestamp:
self.approximate_first_receive_timestamp = unix_time_millis()
self.approximate_receive_count += 1
# Make message visible again in the future unless its
# destroyed.
if visibility_timeout:
self.change_visibility(visibility_timeout)
self.receipt_handle = generate_receipt_handle()
def delay(self, delay_seconds):
delay_msec = int(delay_seconds) * 1000
self.delayed_until = unix_time_millis() + delay_msec
def test_put_subscription_filter_with_firehose():
# given
region_name = "us-east-1"
client_firehose = boto3.client("firehose", region_name)
client_logs = boto3.client("logs", region_name)
log_group_name = "/firehose-test"
log_stream_name = "delivery-stream"
client_logs.create_log_group(logGroupName=log_group_name)
client_logs.create_log_stream(logGroupName=log_group_name,
logStreamName=log_stream_name)
# Create a S3 bucket.
bucket_name = "firehosetestbucket"
s3_client = boto3.client("s3", region_name=region_name)
s3_client.create_bucket(
Bucket=bucket_name,
CreateBucketConfiguration={"LocationConstraint": "us-west-1"},
)
# Create the Firehose delivery stream that uses that S3 bucket as
# the destination.
delivery_stream_name = "firehose_log_test"
firehose_arn = client_firehose.create_delivery_stream(
DeliveryStreamName=delivery_stream_name,
ExtendedS3DestinationConfiguration={
"RoleARN": _get_role_name(region_name),
"BucketARN": f"arn:aws:s3::{bucket_name}",
},
)["DeliveryStreamARN"]
# when
client_logs.put_subscription_filter(
logGroupName=log_group_name,
filterName="firehose-test",
filterPattern="",
destinationArn=firehose_arn,
)
# then
response = client_logs.describe_subscription_filters(
logGroupName=log_group_name)
response["subscriptionFilters"].should.have.length_of(1)
_filter = response["subscriptionFilters"][0]
_filter["creationTime"].should.be.a(int)
_filter["destinationArn"] = firehose_arn
_filter["distribution"] = "ByLogStream"
_filter["logGroupName"] = "/firehose-test"
_filter["filterName"] = "firehose-test"
_filter["filterPattern"] = ""
# when
ts_0 = int(unix_time_millis(datetime.utcnow()))
ts_1 = int(unix_time_millis(datetime.utcnow()))
client_logs.put_log_events(
logGroupName=log_group_name,
logStreamName=log_stream_name,
logEvents=[
{
"timestamp": ts_0,
"message": "test"
},
{
"timestamp": ts_1,
"message": "test 2"
},
],
)
# then
bucket_objects = s3_client.list_objects_v2(Bucket=bucket_name)
message = s3_client.get_object(Bucket=bucket_name,
Key=bucket_objects["Contents"][0]["Key"])
response = json.loads(
zlib.decompress(message["Body"].read(),
16 + zlib.MAX_WBITS).decode("utf-8"))
response["messageType"].should.equal("DATA_MESSAGE")
response["owner"].should.equal("123456789012")
response["logGroup"].should.equal("/firehose-test")
response["logStream"].should.equal("delivery-stream")
response["subscriptionFilters"].should.equal(["firehose-test"])
log_events = sorted(response["logEvents"],
key=lambda log_event: log_event["id"])
log_events.should.have.length_of(2)
log_events[0]["id"].should.be.a(int)
log_events[0]["message"].should.equal("test")
log_events[0]["timestamp"].should.equal(ts_0)
log_events[1]["id"].should.be.a(int)
log_events[1]["message"].should.equal("test 2")
log_events[1]["timestamp"].should.equal(ts_1)
def mark_sent(self, delay_seconds=None):
self.sent_timestamp = unix_time_millis()
if delay_seconds:
self.delay(delay_seconds=delay_seconds)
def _invoke_lambda(self, code, event=None, context=None):
# TODO: context not yet implemented
if event is None:
event = dict()
if context is None:
context = {}
output = None
try:
# TODO: I believe we can keep the container running and feed events as needed
# also need to hook it up to the other services so it can make kws/s3 etc calls
# Should get invoke_id /RequestId from invocation
env_vars = {
"_HANDLER": self.handler,
"AWS_EXECUTION_ENV": "AWS_Lambda_{}".format(self.run_time),
"AWS_LAMBDA_FUNCTION_TIMEOUT": self.timeout,
"AWS_LAMBDA_FUNCTION_NAME": self.function_name,
"AWS_LAMBDA_FUNCTION_MEMORY_SIZE": self.memory_size,
"AWS_LAMBDA_FUNCTION_VERSION": self.version,
"AWS_REGION": self.region,
"AWS_ACCESS_KEY_ID": "role-account-id",
"AWS_SECRET_ACCESS_KEY": "role-secret-key",
"AWS_SESSION_TOKEN": "session-token",
}
env_vars.update(self.environment_vars)
container = exit_code = None
log_config = docker.types.LogConfig(
type=docker.types.LogConfig.types.JSON)
with _DockerDataVolumeContext(self) as data_vol:
try:
self.docker_client.ping() # Verify Docker is running
run_kwargs = (dict(links={"motoserver": "motoserver"})
if settings.TEST_SERVER_MODE else {})
container = self.docker_client.containers.run(
"lambci/lambda:{}".format(self.run_time),
[self.handler, json.dumps(event)],
remove=False,
mem_limit="{}m".format(self.memory_size),
volumes=["{}:/var/task".format(data_vol.name)],
environment=env_vars,
detach=True,
log_config=log_config,
**run_kwargs)
finally:
if container:
try:
exit_code = container.wait(timeout=300)
except requests.exceptions.ReadTimeout:
exit_code = -1
container.stop()
container.kill()
else:
if docker_3:
exit_code = exit_code["StatusCode"]
output = container.logs(stdout=False, stderr=True)
output += container.logs(stdout=True, stderr=False)
container.remove()
output = output.decode("utf-8")
# Send output to "logs" backend
invoke_id = uuid.uuid4().hex
log_stream_name = "{date.year}/{date.month:02d}/{date.day:02d}/[{version}]{invoke_id}".format(
date=datetime.datetime.utcnow(),
version=self.version,
invoke_id=invoke_id,
)
self.logs_backend.create_log_stream(self.logs_group_name,
log_stream_name)
log_events = [{
"timestamp": unix_time_millis(),
"message": line
} for line in output.splitlines()]
self.logs_backend.put_log_events(self.logs_group_name,
log_stream_name, log_events, None)
if exit_code != 0:
raise Exception(
"lambda invoke failed output: {}".format(output))
# We only care about the response from the lambda
# Which is the last line of the output, according to https://github.com/lambci/docker-lambda/issues/25
resp = output.splitlines()[-1]
logs = os.linesep.join(
[line for line in self.convert(output).splitlines()[:-1]])
return resp, False, logs
except docker.errors.DockerException as e:
# Docker itself is probably not running - there will be no Lambda-logs to handle
return "error running docker: {}".format(e), True, ""
except BaseException as e:
traceback.print_exc()
logs = os.linesep.join(
[line for line in self.convert(output).splitlines()[:-1]])
return "error running lambda: {}".format(e), True, logs
def test_put_subscription_filter_with_lambda():
# given
region_name = "us-east-1"
client_lambda = boto3.client("lambda", region_name)
client_logs = boto3.client("logs", region_name)
log_group_name = "/test"
log_stream_name = "stream"
client_logs.create_log_group(logGroupName=log_group_name)
client_logs.create_log_stream(logGroupName=log_group_name,
logStreamName=log_stream_name)
function_arn = client_lambda.create_function(
FunctionName="test",
Runtime="python3.8",
Role=_get_role_name(region_name),
Handler="lambda_function.lambda_handler",
Code={"ZipFile": _get_test_zip_file()},
Description="test lambda function",
Timeout=3,
MemorySize=128,
Publish=True,
)["FunctionArn"]
# when
client_logs.put_subscription_filter(
logGroupName=log_group_name,
filterName="test",
filterPattern="",
destinationArn=function_arn,
)
# then
response = client_logs.describe_subscription_filters(
logGroupName=log_group_name)
response["subscriptionFilters"].should.have.length_of(1)
sub_filter = response["subscriptionFilters"][0]
sub_filter["creationTime"].should.be.a(int)
sub_filter[
"destinationArn"] = "arn:aws:lambda:us-east-1:123456789012:function:test"
sub_filter["distribution"] = "ByLogStream"
sub_filter["logGroupName"] = "/test"
sub_filter["filterName"] = "test"
sub_filter["filterPattern"] = ""
# when
ts_0 = int(unix_time_millis(datetime.utcnow()))
ts_1 = int(unix_time_millis(datetime.utcnow())) + 10
client_logs.put_log_events(
logGroupName=log_group_name,
logStreamName=log_stream_name,
logEvents=[
{
"timestamp": ts_0,
"message": "test"
},
{
"timestamp": ts_1,
"message": "test 2"
},
],
)
# then
msg_showed_up, received_message = _wait_for_log_msg(
client_logs, "/aws/lambda/test", "awslogs")
assert msg_showed_up, "CloudWatch log event was not found. All logs: {}".format(
received_message)
data = json.loads(received_message)["awslogs"]["data"]
response = json.loads(
zlib.decompress(base64.b64decode(data),
16 + zlib.MAX_WBITS).decode("utf-8"))
response["messageType"].should.equal("DATA_MESSAGE")
response["owner"].should.equal("123456789012")
response["logGroup"].should.equal("/test")
response["logStream"].should.equal("stream")
response["subscriptionFilters"].should.equal(["test"])
log_events = sorted(response["logEvents"],
key=lambda log_event: log_event["id"])
log_events.should.have.length_of(2)
log_events[0]["id"].should.be.a(int)
log_events[0]["message"].should.equal("test")
log_events[0]["timestamp"].should.equal(ts_0)
log_events[1]["id"].should.be.a(int)
log_events[1]["message"].should.equal("test 2")
log_events[1]["timestamp"].should.equal(ts_1)
def _invoke_lambda(self, code, event=None, context=None):
# TODO: context not yet implemented
if event is None:
event = dict()
if context is None:
context = {}
try:
# TODO: I believe we can keep the container running and feed events as needed
# also need to hook it up to the other services so it can make kws/s3 etc calls
# Should get invoke_id /RequestId from invovation
env_vars = {
"AWS_LAMBDA_FUNCTION_TIMEOUT": self.timeout,
"AWS_LAMBDA_FUNCTION_NAME": self.function_name,
"AWS_LAMBDA_FUNCTION_MEMORY_SIZE": self.memory_size,
"AWS_LAMBDA_FUNCTION_VERSION": self.version,
"AWS_REGION": self.region,
}
env_vars.update(self.environment_vars)
container = output = exit_code = None
with _DockerDataVolumeContext(self) as data_vol:
try:
run_kwargs = dict(links={'motoserver': 'motoserver'}
) if settings.TEST_SERVER_MODE else {}
container = self.docker_client.containers.run(
"lambci/lambda:{}".format(self.run_time),
[self.handler, json.dumps(event)],
remove=False,
mem_limit="{}m".format(self.memory_size),
volumes=["{}:/var/task".format(data_vol.name)],
environment=env_vars,
detach=True,
**run_kwargs)
finally:
if container:
try:
exit_code = container.wait(timeout=300)
except requests.exceptions.ReadTimeout:
exit_code = -1
container.stop()
container.kill()
else:
if docker_3:
exit_code = exit_code['StatusCode']
output = container.logs(stdout=False, stderr=True)
output += container.logs(stdout=True, stderr=False)
container.remove()
output = output.decode('utf-8')
# Send output to "logs" backend
invoke_id = uuid.uuid4().hex
log_stream_name = "{date.year}/{date.month:02d}/{date.day:02d}/[{version}]{invoke_id}".format(
date=datetime.datetime.utcnow(),
version=self.version,
invoke_id=invoke_id)
self.logs_backend.create_log_stream(self.logs_group_name,
log_stream_name)
log_events = [{
'timestamp': unix_time_millis(),
"message": line
} for line in output.splitlines()]
self.logs_backend.put_log_events(self.logs_group_name,
log_stream_name, log_events, None)
if exit_code != 0:
raise Exception(
'lambda invoke failed output: {}'.format(output))
# strip out RequestId lines
output = os.linesep.join([
line for line in self.convert(output).splitlines()
if not _stderr_regex.match(line)
])
return output, False
except BaseException as e:
traceback.print_exc()
return "error running lambda: {}".format(e), True
def _invoke_lambda(self, code, event=None, context=None):
# TODO: context not yet implemented
if event is None:
event = dict()
if context is None:
context = {}
try:
# TODO: I believe we can keep the container running and feed events as needed
# also need to hook it up to the other services so it can make kws/s3 etc calls
# Should get invoke_id /RequestId from invovation
env_vars = {
"AWS_LAMBDA_FUNCTION_TIMEOUT": self.timeout,
"AWS_LAMBDA_FUNCTION_NAME": self.function_name,
"AWS_LAMBDA_FUNCTION_MEMORY_SIZE": self.memory_size,
"AWS_LAMBDA_FUNCTION_VERSION": self.version,
"AWS_REGION": self.region,
}
env_vars.update(self.environment_vars)
container = output = exit_code = None
with _DockerDataVolumeContext(self) as data_vol:
try:
run_kwargs = dict(links={'motoserver': 'motoserver'}) if settings.TEST_SERVER_MODE else {}
container = self.docker_client.containers.run(
"lambci/lambda:{}".format(self.run_time),
[self.handler, json.dumps(event)], remove=False,
mem_limit="{}m".format(self.memory_size),
volumes=["{}:/var/task".format(data_vol.name)], environment=env_vars, detach=True, **run_kwargs)
finally:
if container:
try:
exit_code = container.wait(timeout=300)
except requests.exceptions.ReadTimeout:
exit_code = -1
container.stop()
container.kill()
else:
if docker_3:
exit_code = exit_code['StatusCode']
output = container.logs(stdout=False, stderr=True)
output += container.logs(stdout=True, stderr=False)
container.remove()
output = output.decode('utf-8')
# Send output to "logs" backend
invoke_id = uuid.uuid4().hex
log_stream_name = "{date.year}/{date.month:02d}/{date.day:02d}/[{version}]{invoke_id}".format(
date=datetime.datetime.utcnow(), version=self.version, invoke_id=invoke_id
)
self.logs_backend.create_log_stream(self.logs_group_name, log_stream_name)
log_events = [{'timestamp': unix_time_millis(), "message": line}
for line in output.splitlines()]
self.logs_backend.put_log_events(self.logs_group_name, log_stream_name, log_events, None)
if exit_code != 0:
raise Exception(
'lambda invoke failed output: {}'.format(output))
# strip out RequestId lines
output = os.linesep.join([line for line in self.convert(output).splitlines() if not _stderr_regex.match(line)])
return output, False
except BaseException as e:
traceback.print_exc()
return "error running lambda: {}".format(e), True
def visible(self):
current_time = unix_time_millis()
if current_time > self.visible_at:
return True
return False
def moto_put_log_events(self, log_group_name, log_stream_name, log_events):
# TODO: call/patch upstream method here, instead of duplicating the code!
self.last_ingestion_time = int(unix_time_millis())
self.stored_bytes += sum(
[len(log_event["message"]) for log_event in log_events])
events = [
logs_models.LogEvent(self.last_ingestion_time, log_event)
for log_event in log_events
]
self.events += events
self.upload_sequence_token += 1
# apply filterpattern -> only forward what matches the pattern
if self.filter_pattern:
# TODO only patched in pro
matches = get_pattern_matcher(self.filter_pattern)
events = [
logs_models.LogEvent(self.last_ingestion_time, event)
for event in log_events if matches(self.filter_pattern, event)
]
if events and self.destination_arn:
log_events = [{
"id": str(event.event_id),
"timestamp": event.timestamp,
"message": event.message,
} for event in events]
data = {
"messageType": "DATA_MESSAGE",
"owner": aws_stack.get_account_id(),
"logGroup": log_group_name,
"logStream": log_stream_name,
"subscriptionFilters": [self.filter_name],
"logEvents": log_events,
}
output = io.BytesIO()
with GzipFile(fileobj=output, mode="w") as f:
f.write(json.dumps(data, separators=(",", ":")).encode("utf-8"))
payload_gz_encoded = output.getvalue()
event = {
"awslogs": {
"data": base64.b64encode(output.getvalue()).decode("utf-8")
}
}
if ":lambda:" in self.destination_arn:
client = aws_stack.connect_to_service("lambda")
lambda_name = aws_stack.lambda_function_name(self.destination_arn)
client.invoke(FunctionName=lambda_name, Payload=json.dumps(event))
if ":kinesis:" in self.destination_arn:
client = aws_stack.connect_to_service("kinesis")
stream_name = aws_stack.kinesis_stream_name(self.destination_arn)
client.put_record(
StreamName=stream_name,
Data=payload_gz_encoded,
PartitionKey=log_group_name,
)
if ":firehose:" in self.destination_arn:
client = aws_stack.connect_to_service("firehose")
firehose_name = aws_stack.firehose_name(self.destination_arn)
client.put_record(
DeliveryStreamName=firehose_name,
Record={"Data": payload_gz_encoded},
)
return "{:056d}".format(self.upload_sequence_token)
def change_visibility(self, visibility_timeout):
# We're dealing with milliseconds internally
visibility_timeout_msec = int(visibility_timeout) * 1000
self.visible_at = unix_time_millis() + visibility_timeout_msec
def change_visibility(self, visibility_timeout):
# We're dealing with milliseconds internally
visibility_timeout_msec = int(visibility_timeout) * 1000
self.visible_at = unix_time_millis() + visibility_timeout_msec
def mark_sent(self, delay_seconds=None):
self.sent_timestamp = int(unix_time_millis())
if delay_seconds:
self.delay(delay_seconds=delay_seconds)
def __init__(self, policy_name, policy_document):
self.policy_name = policy_name
self.policy_document = policy_document
self.last_updated_time = int(unix_time_millis())
def update(self, policy_document):
self.policy_document = policy_document
self.last_updated_time = int(unix_time_millis())
def delay(self, delay_seconds):
delay_msec = int(delay_seconds) * 1000
self.delayed_until = unix_time_millis() + delay_msec
def visible(self):
current_time = unix_time_millis()
if current_time > self.visible_at:
return True
return False
def delayed(self):
current_time = unix_time_millis()
if current_time < self.delayed_until:
return True
return False
def delayed(self):
current_time = unix_time_millis()
if current_time < self.delayed_until:
return True
return False
