def default_attributes(self) -> QueueAttributeMap:
return {
QueueAttributeName.QueueArn:
self.arn,
QueueAttributeName.ApproximateNumberOfMessages:
self.visible._qsize,
QueueAttributeName.ApproximateNumberOfMessagesNotVisible:
lambda: len(self.inflight),
QueueAttributeName.ApproximateNumberOfMessagesDelayed:
"0", # FIXME: this should also be callable
QueueAttributeName.CreatedTimestamp:
str(now()),
QueueAttributeName.LastModifiedTimestamp:
str(now()),
QueueAttributeName.VisibilityTimeout:
"30",
QueueAttributeName.MaximumMessageSize:
"262144",
QueueAttributeName.MessageRetentionPeriod:
"345600",
QueueAttributeName.DelaySeconds:
"0",
QueueAttributeName.ReceiveMessageWaitTimeSeconds:
"0",
}
def _execute(self,
func_arn,
func_details,
event,
context=None,
version=None):
lambda_cwd = func_details.cwd
environment = self._prepare_environment(func_details)
# execute the Lambda function in a forked sub-process, sync result via queue
queue = Queue()
lambda_function = func_details.function(version)
def do_execute():
# now we're executing in the child process, safe to change CWD and ENV
path_before = sys.path
try:
if lambda_cwd:
os.chdir(lambda_cwd)
sys.path = [lambda_cwd] + sys.path
if environment:
os.environ.update(environment)
result = lambda_function(event, context)
queue.put(result)
finally:
sys.path = path_before
process = Process(target=do_execute)
start_time = now(millis=True)
with CaptureOutput() as c:
process.run()
result = queue.get()
end_time = now(millis=True)
# Make sure to keep the log line below, to ensure the log stream gets created
request_id = long_uid()
log_output = 'START %s: Lambda %s started via "local" executor ...' % (
request_id, func_arn)
# TODO: Interweaving stdout/stderr currently not supported
for stream in (c.stdout(), c.stderr()):
if stream:
log_output += ('\n' if log_output else '') + stream
log_output += '\nEND RequestId: %s' % request_id
log_output += '\nREPORT RequestId: %s Duration: %s ms' % (
request_id, int((end_time - start_time) * 1000))
# store logs to CloudWatch
_store_logs(func_details, log_output)
result = result.result if isinstance(result,
InvocationResult) else result
invocation_result = InvocationResult(result, log_output=log_output)
return invocation_result
def process_records(self, records, checkpointer):
if self.processor_func:
self.processor_func(records=records, checkpointer=checkpointer, shard_id=self.shard_id)
for record in records:
seq = int(record.sequence_number)
sub_seq = record.sub_sequence_number
if self.should_update_sequence(seq, sub_seq):
self._largest_seq = (seq, sub_seq)
if self.auto_checkpoint:
time_now = now()
if (time_now - CHECKPOINT_FREQ_SECS) > self.last_checkpoint_time:
self.checkpoint(checkpointer, str(self._largest_seq[0]), self._largest_seq[1])
self.last_checkpoint_time = time_now
def process_records(self, records, checkpointer):
if self.processor_func:
self.processor_func(records=records,
checkpointer=checkpointer, shard_id=self.shard_id)
for record in records:
seq = int(record.sequence_number)
sub_seq = record.sub_sequence_number
if self.should_update_sequence(seq, sub_seq):
self._largest_seq = (seq, sub_seq)
if self.auto_checkpoint:
time_now = now()
if (time_now - CHECKPOINT_FREQ_SECS) > self.last_checkpoint_time:
self.checkpoint(checkpointer, str(self._largest_seq[0]), self._largest_seq[1])
self.last_checkpoint_time = time_now
def start_session(self, metadata: ClientMetadata) -> SessionResponse:
# FIXME: re-using Event as request object this way is kind of a hack
request = Event("session",
EventMetadata(self.localstack_session_id, str(now())),
payload=metadata)
response = requests.post(self.endpoint_session,
headers=self._create_headers(),
json=request.asdict())
if not response.ok:
raise ValueError(
"error during session initiation with analytics backend")
return SessionResponse(response.json())
def _create_message_attributes(
self,
context: RequestContext,
message_system_attributes: MessageBodySystemAttributeMap = None,
) -> Dict[MessageSystemAttributeName, str]:
result: Dict[MessageSystemAttributeName, str] = {
MessageSystemAttributeName.SenderId: context.account_id,
MessageSystemAttributeName.SentTimestamp: str(now()),
}
if message_system_attributes is not None:
for attr in message_system_attributes:
result[attr] = message_system_attributes[attr]["StringValue"]
return result
def clean_cache(file_pattern=CACHE_FILE_PATTERN,
last_clean_time=None,
max_age=CACHE_MAX_AGE):
if last_clean_time is None:
last_clean_time = last_cache_clean_time
with MUTEX_CLEAN:
time_now = now()
if last_clean_time["time"] > time_now - CACHE_CLEAN_TIMEOUT:
return
for cache_file in set(glob.glob(file_pattern)):
mod_time = os.path.getmtime(cache_file)
if time_now > mod_time + max_age:
rm_rf(cache_file)
last_clean_time["time"] = time_now
return time_now
def test_now(self):
env = common.now()
test = common.mktime(datetime.now())
self.assertEqual(env, test)
def test_now(self):
env = common.now()
test = time.time()
assert test == pytest.approx(env, 1)
def test_now(self):
env = common.now()
test = time.time()
self.assertAlmostEqual(env, test, delta=1)
def _execute(self,
func_arn,
func_details,
event,
context=None,
version=None):
lambda_cwd = func_details.cwd
environment = self._prepare_environment(func_details)
# execute the Lambda function in a forked sub-process, sync result via queue
queue = Queue()
lambda_function = func_details.function(version)
def do_execute():
# now we're executing in the child process, safe to change CWD and ENV
result = None
try:
if lambda_cwd:
os.chdir(lambda_cwd)
sys.path.insert(0, '')
if environment:
os.environ.update(environment)
result = lambda_function(event, context)
except Exception as e:
result = str(e)
sys.stderr.write('%s %s' % (e, traceback.format_exc()))
raise
finally:
queue.put(result)
process = Process(target=do_execute)
start_time = now(millis=True)
error = None
with CaptureOutput() as c:
try:
process.run()
except Exception as e:
error = e
result = queue.get()
end_time = now(millis=True)
# Make sure to keep the log line below, to ensure the log stream gets created
request_id = long_uid()
log_output = 'START %s: Lambda %s started via "local" executor ...' % (
request_id, func_arn)
# TODO: Interweaving stdout/stderr currently not supported
for stream in (c.stdout(), c.stderr()):
if stream:
log_output += ('\n' if log_output else '') + stream
log_output += '\nEND RequestId: %s' % request_id
log_output += '\nREPORT RequestId: %s Duration: %s ms' % (
request_id, int((end_time - start_time) * 1000))
# store logs to CloudWatch
_store_logs(func_details, log_output)
result = result.result if isinstance(result,
InvocationResult) else result
if error:
LOG.info('Error executing Lambda "%s": %s %s' %
(func_arn, error, ''.join(
traceback.format_tb(error.__traceback__))))
raise InvocationException(result, log_output)
invocation_result = InvocationResult(result, log_output=log_output)
return invocation_result
def update_last_modified(self, timestamp: int = None):
if timestamp is None:
timestamp = now()
self.attributes[QueueAttributeName.LastModifiedTimestamp] = str(
timestamp)
