def new():
"""
Select our event input based on the `EVENT_SOURCE` environment variable setting. `EVENT_SOURCE` can currently
have one of a these settings:
* ``commandline``: accept a single event as a string on the `docker` command line
* ``environment``: Read a single event from the `AWS_LAMBDA_EVENT_BODY` environment variable
* ``filesystem``: Read one or more events from a folder in the Docker filesystem; either built
in to the image or mounted through a volume mount
* ``kinesis``: read events from a Kinesis stream, or from a kinesalite stream
:rtype: an event source class
"""
source = os.environ.get('EVENT_SOURCE', 'commandline')
if source not in [
'filesystem', 'kinesis', 'environment', 'commandline'
]:
eprint('docker-lambda.source.unknown source=%s' % source)
sys.exit(1)
else:
log('docker-lambda.source.selected source=%s' % source)
if source == 'filesystem':
source = FilesystemEventSource()
elif source == 'kinesis':
source = KinesisEventSource()
elif source == 'environment':
source = EnvironmentEventSource()
elif source == 'commandline':
source = CommandLineEventSource()
return source
def poll(self):
if len(sys.argv) > 2:
event = sys.argv[2]
else:
log("docker-lambda.source.commandline.no-event")
event = "{}"
self.is_done = True
return event
def poll(self):
if len(self.files) > 0:
if self.index < len(self.files):
with open(self.files[self.index], 'r') as fd:
events = fd.read()
log("docker-lambda.source.filesystem.done.event-loaded folder=%s file=%s"
% (self.folder_name, self.files[self.index]))
self.index += 1
if self.index >= len(self.files):
if self.loop:
self.index = 0
log("docker-lambda.source.filesystem.done.rewind folder=%s loop=%s"
% (self.folder_name, self.loop))
else:
log("docker-lambda.source.filesystem.done.no-more-files "
"folder=%s loop=%s exit-policy=%s" %
(self.folder_name, self.loop, self.exit_policy))
if self.exit_policy == "wait":
while True:
time.sleep(30)
else:
log("docker-lambda.source.filesystem.done")
self.is_done = True
else:
eprint("docker-lambda.source.filesystem.error.no-files folder=%s" %
self.folder_name)
sys.exit(1)
time.sleep(self.period)
return events
def __init__(self):
self.folder_name = os.environ['FILESYSTEM_FOLDER']
self.period = int(os.environ.get('FILESYSTEM_SLEEP_SECONDS', 1))
self.loop = os.environ.get('FILESYSTEM_LOOP', "False") == "True"
# Either "exit" or "wait"
self.exit_policy = os.environ.get('FILESYSTEM_EXIT_POLICY', "exit")
self.files = [
os.path.join(self.folder_name, f)
for f in os.listdir(self.folder_name)
if os.path.isfile(os.path.join(self.folder_name, f))
]
self.files.sort()
self.index = 0
log("docker-lambda.source.filesystem.start folder=%s files=%s loop=%s period=%s"
% (self.folder_name, len(self.files), self.loop, self.period))
self.is_done = False
def poll(self):
"""
This is what receive_invoke() calls. This will block until records are available, return all records found and
go back to waiting for records.
"""
while 1:
records = self.kinesis.get_records(ShardIterator=self.shard_iterator['ShardIterator'])
if len(records['Records']) > 0:
break
log(
"docker-lambda.source.kinesis.stream.poll.no-events stream_name=%s" % (self.stream_name)
)
# For standard iterators, Lambda polls each shard in your Kinesis stream for records at a base rate of once
# per second. When more records are available, Lambda keeps processing batches until it receives a batch
# that's smaller than the configured maximum batch size. The function shares read throughput with other
# consumers of the shard.
# Ref: https://docs.aws.amazon.com/lambda/latest/dg/with-kinesis.html
time.sleep(1)
events = {'Records': []}
for krecord in records['Records']:
events['Records'].append({
"kinesis": {
"kinesisSchemaVersion": "1.0",
"partitionKey": "1",
"sequenceNumber": krecord['SequenceNumber'],
"data": b64encode(krecord['Data']).decode(),
"approximateArrivalTimestamp": krecord['ApproximateArrivalTimestamp'].timestamp()
},
"eventSource": "aws:kinesis",
"eventVersion": "1.0",
"eventID": f"{self.shard_id}:{krecord['SequenceNumber']}",
"eventName": "aws:kinesis:record",
"invokeIdentityArn": "my_fake_arn",
"awsRegion": self.region,
"eventSourceARN": self.stream_arn
})
log(
"docker-lambda.source.kinesis.stream.poll stream_name=%s nevents=%s" % (self.stream_name, len(events['Records']))
)
return json.dumps(events).encode('utf8')
def __init__(self):
self.stream_name = os.environ['KINESIS_STREAM_NAME']
self.region = os.environ.get('AWS_REGION', os.environ.get('AWS_DEFAULT_REGION', 'us-east-1'))
access_key_id = os.environ.get('AWS_ACCESS_KEY_ID', 'SOME_ACCESS_KEY_ID')
secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY', 'SOME_SECRET_ACCESS_KEY')
endpoint_url = os.environ.get('KINESIS_ENDPOINT_URL', None)
log("docker-lambda.source.kinesis.start stream_name=%s %s" % (
self.stream_name,
"endpoint_url=%s" % endpoint_url if endpoint_url else ''
))
self.kinesis = boto3.client(
'kinesis',
aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key,
region_name=self.region,
endpoint_url=os.environ.get('KINESIS_ENDPOINT_URL', None)
)
waiter = self.kinesis.get_waiter('stream_exists')
started = False
count = 0
while not started:
try:
waiter.wait(StreamName=self.stream_name)
except botocore.exceptions.WaiterError:
# If we're using a Kinesalite container, it may take a little while for it to
# boot and create its stream. Give it a
log("docker-lambda.source.kinesis.stream.not-ready stream_name=%s")
time.sleep(2)
count += 1
if count == 10:
raise
else:
started = True
stream = self.kinesis.describe_stream(StreamName=self.stream_name)
self.stream_arn = stream['StreamDescription']['StreamARN']
shards = self.kinesis.list_shards(StreamName=self.stream_name)
self.shard_id = shards['Shards'][0]['ShardId']
self.shard_iterator = self.kinesis.get_shard_iterator(
StreamName=self.stream_name,
ShardId=self.shard_id,
ShardIteratorType='LATEST'
)
log("docker-lambda.source.kinesis.stream.ready stream_name=%s" % self.stream_name)
def done(self):
if self.is_done:
log("docker-lambda.source.environment.done")
return self.is_done
def poll(self):
event = os.environ.get('AWS_LAMBDA_EVENT_BODY', {})
if not event:
log("docker-lambda.source.environment.no-event")
self.is_done = True
return event
def done(self):
if self.is_done:
log("docker-lambda.source.commandline.done")
return self.is_done