def _do_start_ssl_proxy_with_client_auth(port: int, target: str,
client_cert_key: Tuple[str, str]):
base_url = f"{'https://' if '://' not in target else ''}{target.rstrip('/')}"
# prepare cert files (TODO: check whether/how we can pass cert strings to requests.request(..) directly)
cert_file = client_cert_key[0]
if not os.path.exists(cert_file):
cert_file = new_tmp_file()
save_file(cert_file, client_cert_key[0])
key_file = client_cert_key[1]
if not os.path.exists(key_file):
key_file = new_tmp_file()
save_file(key_file, client_cert_key[1])
cert_params = (cert_file, key_file)
# define forwarding listener
class Listener(ProxyListener):
def forward_request(self, method, path, data, headers):
url = f"{base_url}{path}"
result = requests.request(method=method,
url=url,
data=data,
headers=headers,
cert=cert_params,
verify=False)
return result
proxy_thread = start_proxy_server(port,
update_listener=Listener(),
use_ssl=True)
return proxy_thread
def download_and_extract(archive_url,
target_dir,
retries=0,
sleep=3,
tmp_archive=None):
mkdir(target_dir)
tmp_archive = tmp_archive or new_tmp_file()
if not os.path.exists(tmp_archive):
# create temporary placeholder file, to avoid duplicate parallel downloads
save_file(tmp_archive, '')
for i in range(retries + 1):
try:
download(archive_url, tmp_archive)
break
except Exception:
time.sleep(sleep)
_, ext = os.path.splitext(tmp_archive)
if ext == '.zip':
unzip(tmp_archive, target_dir)
elif ext == '.gz' or ext == '.bz2':
untar(tmp_archive, target_dir)
else:
raise Exception('Unsupported archive format: %s' % ext)
def test_s3_event_notification_with_sqs(self):
key_by_path = 'aws/bucket=2020/test1.txt'
queue_url, queue_attributes = self._create_test_queue()
self._create_test_notification_bucket(queue_attributes)
self.s3_client.put_bucket_versioning(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
VersioningConfiguration={'Status': 'Enabled'})
body = 'Lorem ipsum dolor sit amet, ... ' * 30
# put an object
self.s3_client.put_object(Bucket=TEST_BUCKET_WITH_NOTIFICATION, Key=key_by_path, Body=body)
self.assertEqual(self._get_test_queue_message_count(queue_url), '1')
rs = self.sqs_client.receive_message(QueueUrl=queue_url)
record = [json.loads(to_str(m['Body'])) for m in rs['Messages']][0]['Records'][0]
download_file = new_tmp_file()
self.s3_client.download_file(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
Key=key_by_path, Filename=download_file)
self.assertEqual(record['s3']['object']['size'], os.path.getsize(download_file))
# clean up
self.s3_client.put_bucket_versioning(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
VersioningConfiguration={'Status': 'Disabled'})
self.sqs_client.delete_queue(QueueUrl=queue_url)
self._delete_bucket(TEST_BUCKET_WITH_NOTIFICATION, [key_by_path])
def test_s3_upload_fileobj_with_large_file_notification(self):
queue_url, queue_attributes = self._create_test_queue()
self._create_test_notification_bucket(queue_attributes)
# has to be larger than 64MB to be broken up into a multipart upload
file_size = 75000000
large_file = self.generate_large_file(file_size)
download_file = new_tmp_file()
try:
self.s3_client.upload_file(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
Key=large_file.name, Filename=large_file.name)
self.assertEqual(self._get_test_queue_message_count(queue_url), '1')
# ensure that the first message's eventName is ObjectCreated:CompleteMultipartUpload
messages = self.sqs_client.receive_message(QueueUrl=queue_url, AttributeNames=['All'])
message = json.loads(messages['Messages'][0]['Body'])
self.assertEqual(message['Records'][0]['eventName'], 'ObjectCreated:CompleteMultipartUpload')
# download the file, check file size
self.s3_client.download_file(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
Key=large_file.name, Filename=download_file)
self.assertEqual(os.path.getsize(download_file), file_size)
# clean up
self.sqs_client.delete_queue(QueueUrl=queue_url)
self._delete_bucket(TEST_BUCKET_WITH_NOTIFICATION, large_file.name)
finally:
# clean up large files
large_file.close()
rm_rf(large_file.name)
rm_rf(download_file)
def get_java_handler(zip_file_content, handler, main_file):
"""Creates a Java handler from an uploaded ZIP or JAR.
:type zip_file_content: bytes
:param zip_file_content: ZIP file bytes.
:type handler: str
:param handler: The lambda handler path.
:type main_file: str
:param main_file: Filepath to the uploaded ZIP or JAR file.
:returns: function or flask.Response
"""
if not is_jar_archive(zip_file_content):
with zipfile.ZipFile(BytesIO(zip_file_content)) as zip_ref:
jar_entries = [e for e in zip_ref.infolist() if e.filename.endswith('.jar')]
if len(jar_entries) != 1:
raise Exception('Expected exactly one *.jar entry in zip file, found %s' % len(jar_entries))
zip_file_content = zip_ref.read(jar_entries[0].filename)
LOG.info('Found jar file %s with %s bytes in Lambda zip archive' %
(jar_entries[0].filename, len(zip_file_content)))
main_file = new_tmp_file()
save_file(main_file, zip_file_content)
if is_jar_archive(zip_file_content):
def execute(event, context):
result, log_output = lambda_executors.EXECUTOR_LOCAL.execute_java_lambda(
event, context, handler=handler, main_file=main_file)
return result
return execute
return error_response(
'Unable to extract Java Lambda handler - file is not a valid zip/jar files', 400, error_type='ValidationError')
def download_and_extract(archive_url,
target_dir,
retries=0,
sleep=3,
tmp_archive=None):
mkdir(target_dir)
if tmp_archive:
_, ext = os.path.splitext(tmp_archive)
else:
_, ext = os.path.splitext(archive_url)
tmp_archive = tmp_archive or new_tmp_file()
if not os.path.exists(tmp_archive) or os.path.getsize(tmp_archive) <= 0:
# create temporary placeholder file, to avoid duplicate parallel downloads
save_file(tmp_archive, "")
for i in range(retries + 1):
try:
download(archive_url, tmp_archive)
break
except Exception:
time.sleep(sleep)
if ext == ".zip":
unzip(tmp_archive, target_dir)
elif ext == ".gz" or ext == ".bz2":
untar(tmp_archive, target_dir)
else:
raise Exception("Unsupported archive format: %s" % ext)
def run_cached(cmd, cache_duration_secs=None):
if cache_duration_secs is None:
cache_duration_secs = AWS_CACHE_TIMEOUT
env_vars = os.environ.copy()
env_vars.update({
'AWS_ACCESS_KEY_ID':
os.environ.get('AWS_ACCESS_KEY_ID') or 'foobar',
'AWS_SECRET_ACCESS_KEY':
os.environ.get('AWS_SECRET_ACCESS_KEY') or 'foobar',
'AWS_DEFAULT_REGION':
os.environ.get('AWS_DEFAULT_REGION') or DEFAULT_REGION,
'PYTHONWARNINGS':
'ignore:Unverified HTTPS request'
})
tmp_file_path = new_tmp_file()
error = None
with open(tmp_file_path, 'w') as err_file:
try:
return run(cmd,
cache_duration_secs=cache_duration_secs,
env_vars=env_vars,
stderr=err_file)
except Exception as e:
error = e
if error:
LOG.warning('Error running command: %s %s %s' %
(cmd, error, load_file(tmp_file_path)))
raise error
def test_parse_config_file(input_type, sections):
config_string = CONFIG_FILE_SECTION.lstrip()
# generate config string
if sections == 0:
config_string = config_string.partition("\n")[2]
config_string = "\n".join([
config_string.replace("{section}", str(i))
for i in range(max(sections, 1))
])
# parse result
config_input = config_string
if input_type == "file":
config_input = new_tmp_file() # deleted on shutdown
save_file(config_input, config_string)
result = parse_config_file(config_input)
# run assertions
expected = {
"var1": "foo bar 123",
"var2": "123.45",
"var3": "Test string' <with% special { chars!",
}
if sections <= 1:
assert expected == result
else:
assert sections == len(result)
for section in result.values():
assert expected == section
def test_train_tensorflow():
sagemaker_client = aws_stack.connect_to_service('sagemaker')
iam_client = aws_stack.connect_to_service('iam')
sagemaker_session = sagemaker.Session(boto_session=aws_stack.Boto3Session(),
sagemaker_client=sagemaker_client)
try:
response = iam_client.create_role(RoleName='r1', AssumeRolePolicyDocument='{}')
except Exception:
response = iam_client.get_role(RoleName='r1')
role_arn = response['Role']['Arn']
test_data = 'testdata'
if not os.path.exists(test_data):
data_sets = input_data.read_data_sets(test_data,
dtype=tf.uint8, reshape=False, validation_size=5000)
convert_to(data_sets.train, 'train', test_data)
convert_to(data_sets.validation, 'validation', test_data)
convert_to(data_sets.test, 'test', test_data)
inputs = sagemaker_session.upload_data(path=test_data, key_prefix='data/mnist')
tmp_file = new_tmp_file()
download(TF_MNIST_URL, tmp_file)
mnist_estimator = TensorFlow(entry_point=tmp_file, role=role_arn, framework_version='1.12.0',
training_steps=10, evaluation_steps=10, sagemaker_session=sagemaker_session,
train_instance_count=1, train_instance_type='local')
mnist_estimator.fit(inputs, logs=False)
def test_s3_upload_fileobj_with_large_file_notification(self):
# create test queue
queue_url = self.sqs_client.create_queue(
QueueName=TEST_QUEUE_FOR_BUCKET_WITH_NOTIFICATION)['QueueUrl']
queue_attributes = self.sqs_client.get_queue_attributes(
QueueUrl=queue_url, AttributeNames=['QueueArn'])
# create test bucket
self.s3_client.create_bucket(Bucket=TEST_BUCKET_WITH_NOTIFICATION)
self.s3_client.put_bucket_notification_configuration(
Bucket=TEST_BUCKET_WITH_NOTIFICATION,
NotificationConfiguration={
'QueueConfigurations': [{
'QueueArn':
queue_attributes['Attributes']['QueueArn'],
'Events': ['s3:ObjectCreated:*']
}]
})
# has to be larger than 64MB to be broken up into a multipart upload
file_size = 75000000
large_file = self.generate_large_file(file_size)
download_file = new_tmp_file()
try:
self.s3_client.upload_file(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
Key=large_file.name,
Filename=large_file.name)
queue_attributes = self.sqs_client.get_queue_attributes(
QueueUrl=queue_url,
AttributeNames=['ApproximateNumberOfMessages'])
message_count = queue_attributes['Attributes'][
'ApproximateNumberOfMessages']
# the ApproximateNumberOfMessages attribute is a string
self.assertEqual(message_count, '1')
# ensure that the first message's eventName is ObjectCreated:CompleteMultipartUpload
messages = self.sqs_client.receive_message(QueueUrl=queue_url,
AttributeNames=['All'])
message = json.loads(messages['Messages'][0]['Body'])
self.assertEqual(message['Records'][0]['eventName'],
'ObjectCreated:CompleteMultipartUpload')
# download the file, check file size
self.s3_client.download_file(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
Key=large_file.name,
Filename=download_file)
self.assertEqual(os.path.getsize(download_file), file_size)
# clean up
self.sqs_client.delete_queue(QueueUrl=queue_url)
self.s3_client.delete_object(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
Key=large_file.name)
self.s3_client.delete_bucket(Bucket=TEST_BUCKET_WITH_NOTIFICATION)
finally:
# clean up large files
large_file.close()
rm_rf(large_file.name)
rm_rf(download_file)
def _do_start_ssl_proxy_with_client_auth(port: int, target: PortOrUrl,
client_cert_key: Tuple[str, str]):
# prepare cert files (TODO: check whether/how we can pass cert strings to requests.request(..) directly)
cert_file = client_cert_key[0]
if not os.path.exists(cert_file):
cert_file = new_tmp_file()
save_file(cert_file, client_cert_key[0])
key_file = client_cert_key[1]
if not os.path.exists(key_file):
key_file = new_tmp_file()
save_file(key_file, client_cert_key[1])
cert_params = (cert_file, key_file)
# start proxy
requests_kwargs = {"cert": cert_params}
result = _do_start_ssl_proxy_with_listener(port,
target,
requests_kwargs=requests_kwargs)
return result
def setUpClass(cls):
# Note: create scheduled Lambda here - assertions will be run in test_scheduled_lambda() below..
# create test Lambda
cls.scheduled_lambda_name = 'scheduled-%s' % short_uid()
handler_file = new_tmp_file()
save_file(handler_file, TEST_HANDLER)
resp = testutil.create_lambda_function(handler_file=handler_file, func_name=cls.scheduled_lambda_name)
func_arn = resp['CreateFunctionResponse']['FunctionArn']
# create scheduled Lambda function
rule_name = 'rule-%s' % short_uid()
events = aws_stack.connect_to_service('events')
events.put_rule(Name=rule_name, ScheduleExpression='rate(1 minutes)')
events.put_targets(Rule=rule_name, Targets=[{'Id': 'target-%s' % short_uid(), 'Arn': func_arn}])
def test_s3_event_notification_with_sqs(self):
key_by_path = 'aws/bucket=2020/test1.txt'
queue_url, queue_attributes = self._create_test_queue()
self._create_test_notification_bucket(queue_attributes)
self.s3_client.put_bucket_versioning(
Bucket=TEST_BUCKET_WITH_NOTIFICATION,
VersioningConfiguration={'Status': 'Enabled'})
# flake8: noqa: W291
body = """ Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor.
Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus.
Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim.
Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a,.
Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi.
Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim.
Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet.
Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui.
Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sed ipsum. """
# put an object
self.s3_client.put_object(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
Key=key_by_path,
Body=body)
self.assertEqual(self._get_test_queue_message_count(queue_url), '1')
rs = self.sqs_client.receive_message(QueueUrl=queue_url)
record = [json.loads(to_str(m['Body']))
for m in rs['Messages']][0]['Records'][0]
download_file = new_tmp_file()
self.s3_client.download_file(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
Key=key_by_path,
Filename=download_file)
self.assertEqual(record['s3']['object']['size'],
os.path.getsize(download_file))
# clean up
self.s3_client.put_bucket_versioning(
Bucket=TEST_BUCKET_WITH_NOTIFICATION,
VersioningConfiguration={'Status': 'Disabled'})
self.sqs_client.delete_queue(QueueUrl=queue_url)
self._delete_bucket(TEST_BUCKET_WITH_NOTIFICATION, [key_by_path])
def test_download_with_timeout():
class DownloadListener(ProxyListener):
def forward_request(self, method, path, data, headers):
if path == "/sleep":
time.sleep(2)
return {}
port = get_free_tcp_port()
proxy = start_proxy_server(port, update_listener=DownloadListener())
tmp_file = new_tmp_file()
download(f"http://localhost:{port}/", tmp_file)
assert load_file(tmp_file) == "{}"
with pytest.raises(TimeoutError):
download(f"http://localhost:{port}/sleep", tmp_file, timeout=1)
# clean up
proxy.stop()
rm_rf(tmp_file)
def setUpClass(cls):
# Note: create scheduled Lambda here - assertions will be run in test_scheduled_lambda() below..
# create test Lambda
cls.scheduled_lambda_name = "scheduled-%s" % short_uid()
handler_file = new_tmp_file()
save_file(handler_file, TEST_HANDLER)
resp = testutil.create_lambda_function(
handler_file=handler_file, func_name=cls.scheduled_lambda_name
)
func_arn = resp["CreateFunctionResponse"]["FunctionArn"]
# create scheduled Lambda function
rule_name = "rule-%s" % short_uid()
events = aws_stack.connect_to_service("events")
events.put_rule(Name=rule_name, ScheduleExpression="rate(1 minutes)")
events.put_targets(
Rule=rule_name, Targets=[{"Id": "target-%s" % short_uid(), "Arn": func_arn}]
)
def test_generate_ssl_cert():
def _assert(cert, key):
# assert that file markers are in place
assert PEM_CERT_START in cert
assert PEM_CERT_END in cert
assert re.match(PEM_KEY_START_REGEX, key.replace("\n", " "))
assert re.match(fr".*{PEM_KEY_END_REGEX}", key.replace("\n", " "))
# generate cert and get content directly
cert = generate_ssl_cert()
_assert(cert, cert)
# generate cert to file and load content from there
target_file, cert_file_name, key_file_name = generate_ssl_cert(
target_file=new_tmp_file(), overwrite=True)
_assert(load_file(cert_file_name), load_file(key_file_name))
# clean up
rm_rf(cert_file_name)
rm_rf(key_file_name)