def download_and_extract_with_retry(archive_url, tmp_archive, target_dir):
mkdir(target_dir)
def download_and_extract():
if not os.path.exists(tmp_archive):
# create temporary placeholder file, to avoid duplicate parallel downloads
save_file(tmp_archive, '')
download(archive_url, tmp_archive)
_, ext = os.path.splitext(tmp_archive)
if ext == '.zip':
unzip(tmp_archive, target_dir)
elif ext == '.gz' or ext == '.bz2':
untar(tmp_archive, target_dir)
else:
raise Exception('Unsupported archive format: %s' % ext)
try:
download_and_extract()
except Exception as e:
# try deleting and re-downloading the zip file
LOG.info('Unable to extract file, re-downloading ZIP archive %s: %s' %
(tmp_archive, e))
rm_rf(tmp_archive)
download_and_extract()
def _check(fname, is_dir):
test_entry = os.path.join(tmp_dir, fname)
mkdir(test_entry) if is_dir else save_file(test_entry, "test content")
assert not is_empty_dir(tmp_dir)
assert is_empty_dir(tmp_dir, ignore_hidden=True) == (fname == ".hidden")
rm_rf(test_entry)
assert is_empty_dir(tmp_dir)
def install_dynamodb_local():
if OVERWRITE_DDB_FILES_IN_DOCKER and in_docker():
rm_rf(INSTALL_DIR_DDB)
is_in_alpine = is_alpine()
if not os.path.exists(INSTALL_PATH_DDB_JAR):
log_install_msg("DynamoDB")
# download and extract archive
tmp_archive = os.path.join(tempfile.gettempdir(), "localstack.ddb.zip")
dynamodb_url = DYNAMODB_JAR_URL_ALPINE if is_in_alpine else DYNAMODB_JAR_URL
download_and_extract_with_retry(dynamodb_url, tmp_archive,
INSTALL_DIR_DDB)
# fix logging configuration for DynamoDBLocal
log4j2_config = """<Configuration status="WARN">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="WARN"><AppenderRef ref="Console"/></Root>
</Loggers>
</Configuration>"""
log4j2_file = os.path.join(INSTALL_DIR_DDB, "log4j2.xml")
save_file(log4j2_file, log4j2_config)
run('cd "%s" && zip -u DynamoDBLocal.jar log4j2.xml || true' %
INSTALL_DIR_DDB)
def get_lambda_code_param(params, _include_arch=False, **kwargs):
code = params.get("Code", {})
zip_file = code.get("ZipFile")
if zip_file and not is_base64(zip_file) and not is_zip_file(
to_bytes(zip_file)):
tmp_dir = new_tmp_dir()
handler_file = get_handler_file_from_name(
params["Handler"], runtime=params["Runtime"])
tmp_file = os.path.join(tmp_dir, handler_file)
save_file(tmp_file, zip_file)
# add 'cfn-response' module to archive - see:
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-lambda-function-code-cfnresponsemodule.html
cfn_response_tmp_file = get_cfn_response_mod_file()
cfn_response_mod_dir = os.path.join(tmp_dir, "node_modules",
"cfn-response")
mkdir(cfn_response_mod_dir)
cp_r(
cfn_response_tmp_file,
os.path.join(cfn_response_mod_dir, "index.js"),
)
# create zip file
zip_file = create_zip_file(tmp_dir, get_content=True)
code["ZipFile"] = zip_file
rm_rf(tmp_dir)
if _include_arch and "Architectures" in params:
code["Architectures"] = params.get("Architectures")
return code
def test_s3_upload_fileobj_with_large_file_notification(self):
queue_url, queue_attributes = self._create_test_queue()
self._create_test_notification_bucket(queue_attributes)
# has to be larger than 64MB to be broken up into a multipart upload
file_size = 75000000
large_file = self.generate_large_file(file_size)
download_file = new_tmp_file()
try:
self.s3_client.upload_file(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
Key=large_file.name, Filename=large_file.name)
self.assertEqual(self._get_test_queue_message_count(queue_url), '1')
# ensure that the first message's eventName is ObjectCreated:CompleteMultipartUpload
messages = self.sqs_client.receive_message(QueueUrl=queue_url, AttributeNames=['All'])
message = json.loads(messages['Messages'][0]['Body'])
self.assertEqual(message['Records'][0]['eventName'], 'ObjectCreated:CompleteMultipartUpload')
# download the file, check file size
self.s3_client.download_file(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
Key=large_file.name, Filename=download_file)
self.assertEqual(os.path.getsize(download_file), file_size)
# clean up
self.sqs_client.delete_queue(QueueUrl=queue_url)
self._delete_bucket(TEST_BUCKET_WITH_NOTIFICATION, large_file.name)
finally:
# clean up large files
large_file.close()
rm_rf(large_file.name)
rm_rf(download_file)
def install_stepfunctions_local():
if not os.path.exists(INSTALL_PATH_STEPFUNCTIONS_JAR):
# pull the JAR file from the Docker image, which is more up-to-date than the downloadable JAR file
log_install_msg("Step Functions")
mkdir(INSTALL_DIR_STEPFUNCTIONS)
run("{dc} pull {img}".format(dc=config.DOCKER_CMD,
img=IMAGE_NAME_SFN_LOCAL))
docker_name = "tmp-ls-sfn"
run(("{dc} run --name={dn} --entrypoint= -d --rm {img} sleep 15"
).format(dc=config.DOCKER_CMD,
dn=docker_name,
img=IMAGE_NAME_SFN_LOCAL))
time.sleep(5)
run("{dc} cp {dn}:/home/stepfunctionslocal/ {tgt}".format(
dc=config.DOCKER_CMD, dn=docker_name, tgt=INSTALL_DIR_INFRA))
run("mv %s/stepfunctionslocal/*.jar %s" %
(INSTALL_DIR_INFRA, INSTALL_DIR_STEPFUNCTIONS))
rm_rf("%s/stepfunctionslocal" % INSTALL_DIR_INFRA)
# apply patches
patch_class_file = os.path.join(INSTALL_DIR_STEPFUNCTIONS, SFN_PATCH_CLASS)
if not os.path.exists(patch_class_file):
download(SFN_PATCH_CLASS_URL, patch_class_file)
cmd = 'cd "%s"; zip %s %s' % (
INSTALL_DIR_STEPFUNCTIONS,
INSTALL_PATH_STEPFUNCTIONS_JAR,
SFN_PATCH_CLASS,
)
run(cmd)
def create_lambda_archive(
script: str,
get_content: bool = False,
libs: List[str] = None,
runtime: str = None,
file_name: str = None,
exclude_func: Callable[[str], bool] = None,
):
"""Utility method to create a Lambda function archive"""
if libs is None:
libs = []
runtime = runtime or LAMBDA_DEFAULT_RUNTIME
with tempfile.TemporaryDirectory(prefix=ARCHIVE_DIR_PREFIX) as tmp_dir:
file_name = file_name or get_handler_file_from_name(LAMBDA_DEFAULT_HANDLER, runtime=runtime)
script_file = os.path.join(tmp_dir, file_name)
if os.path.sep in script_file:
mkdir(os.path.dirname(script_file))
# create __init__.py files along the path to allow Python imports
path = file_name.split(os.path.sep)
for i in range(1, len(path)):
save_file(os.path.join(tmp_dir, *(path[:i] + ["__init__.py"])), "")
save_file(script_file, script)
chmod_r(script_file, 0o777)
# copy libs
for lib in libs:
paths = [lib, "%s.py" % lib]
try:
module = importlib.import_module(lib)
paths.append(module.__file__)
except Exception:
pass
target_dir = tmp_dir
root_folder = os.path.join(LOCALSTACK_VENV_FOLDER, "lib/python*/site-packages")
if lib == "localstack":
paths = ["localstack/*.py", "localstack/utils"]
root_folder = LOCALSTACK_ROOT_FOLDER
target_dir = os.path.join(tmp_dir, lib)
mkdir(target_dir)
for path in paths:
file_to_copy = path if path.startswith("/") else os.path.join(root_folder, path)
for file_path in glob.glob(file_to_copy):
name = os.path.join(target_dir, file_path.split(os.path.sep)[-1])
if os.path.isdir(file_path):
copy_dir(file_path, name)
else:
shutil.copyfile(file_path, name)
if exclude_func:
for dirpath, folders, files in os.walk(tmp_dir):
for name in list(folders) + list(files):
full_name = os.path.join(dirpath, name)
relative = os.path.relpath(full_name, start=tmp_dir)
if exclude_func(relative):
rm_rf(full_name)
# create zip file
result = create_zip_file(tmp_dir, get_content=get_content)
return result
def download_and_extract_with_retry(archive_url, tmp_archive, target_dir):
try:
download_and_extract(archive_url, target_dir, tmp_archive=tmp_archive)
except Exception as e:
# try deleting and re-downloading the zip file
LOG.info('Unable to extract file, re-downloading ZIP archive %s: %s' % (tmp_archive, e))
rm_rf(tmp_archive)
download_and_extract(archive_url, target_dir, tmp_archive=tmp_archive)
def test_s3_upload_fileobj_with_large_file_notification(self):
# create test queue
queue_url = self.sqs_client.create_queue(
QueueName=TEST_QUEUE_FOR_BUCKET_WITH_NOTIFICATION)['QueueUrl']
queue_attributes = self.sqs_client.get_queue_attributes(
QueueUrl=queue_url, AttributeNames=['QueueArn'])
# create test bucket
self.s3_client.create_bucket(Bucket=TEST_BUCKET_WITH_NOTIFICATION)
self.s3_client.put_bucket_notification_configuration(
Bucket=TEST_BUCKET_WITH_NOTIFICATION,
NotificationConfiguration={
'QueueConfigurations': [{
'QueueArn':
queue_attributes['Attributes']['QueueArn'],
'Events': ['s3:ObjectCreated:*']
}]
})
# has to be larger than 64MB to be broken up into a multipart upload
file_size = 75000000
large_file = self.generate_large_file(file_size)
download_file = new_tmp_file()
try:
self.s3_client.upload_file(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
Key=large_file.name,
Filename=large_file.name)
queue_attributes = self.sqs_client.get_queue_attributes(
QueueUrl=queue_url,
AttributeNames=['ApproximateNumberOfMessages'])
message_count = queue_attributes['Attributes'][
'ApproximateNumberOfMessages']
# the ApproximateNumberOfMessages attribute is a string
self.assertEqual(message_count, '1')
# ensure that the first message's eventName is ObjectCreated:CompleteMultipartUpload
messages = self.sqs_client.receive_message(QueueUrl=queue_url,
AttributeNames=['All'])
message = json.loads(messages['Messages'][0]['Body'])
self.assertEqual(message['Records'][0]['eventName'],
'ObjectCreated:CompleteMultipartUpload')
# download the file, check file size
self.s3_client.download_file(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
Key=large_file.name,
Filename=download_file)
self.assertEqual(os.path.getsize(download_file), file_size)
# clean up
self.sqs_client.delete_queue(QueueUrl=queue_url)
self.s3_client.delete_object(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
Key=large_file.name)
self.s3_client.delete_bucket(Bucket=TEST_BUCKET_WITH_NOTIFICATION)
finally:
# clean up large files
large_file.close()
rm_rf(large_file.name)
rm_rf(download_file)
def start_elasticsearch(port=None,
version=None,
delete_data=True,
asynchronous=False,
update_listener=None):
if STATE.get('_thread_'):
return STATE['_thread_']
port = port or config.PORT_ELASTICSEARCH
# delete Elasticsearch data that may be cached locally from a previous test run
delete_all_elasticsearch_data(version)
install.install_elasticsearch(version)
backend_port = get_free_tcp_port()
base_dir = install.get_elasticsearch_install_dir(version)
es_data_dir = os.path.join(base_dir, 'data')
es_tmp_dir = os.path.join(base_dir, 'tmp')
es_mods_dir = os.path.join(base_dir, 'modules')
if config.DATA_DIR:
delete_data = False
es_data_dir = '%s/elasticsearch' % config.DATA_DIR
# Elasticsearch 5.x cannot be bound to 0.0.0.0 in some Docker environments,
# hence we use the default bind address 127.0.0.0 and put a proxy in front of it
backup_dir = os.path.join(config.TMP_FOLDER, 'es_backup')
cmd = (
('%s/bin/elasticsearch ' +
'-E http.port=%s -E http.publish_port=%s -E http.compression=false ' +
'-E path.data=%s -E path.repo=%s') %
(base_dir, backend_port, backend_port, es_data_dir, backup_dir))
if os.path.exists(os.path.join(es_mods_dir, 'x-pack-ml')):
cmd += ' -E xpack.ml.enabled=false'
env_vars = {
'ES_JAVA_OPTS': os.environ.get('ES_JAVA_OPTS', '-Xms200m -Xmx600m'),
'ES_TMPDIR': es_tmp_dir
}
LOG.debug('Starting local Elasticsearch (%s port %s)' %
(get_service_protocol(), port))
if delete_data:
rm_rf(es_data_dir)
# fix permissions
chmod_r(base_dir, 0o777)
mkdir(es_data_dir)
chmod_r(es_data_dir, 0o777)
mkdir(es_tmp_dir)
chmod_r(es_tmp_dir, 0o777)
# start proxy and ES process
proxy = start_proxy_for_service('elasticsearch',
port,
backend_port,
update_listener,
quiet=True,
params={'protocol_version': 'HTTP/1.0'})
STATE['_proxy_'] = proxy
if is_root():
cmd = "su localstack -c '%s'" % cmd
thread = do_run(cmd, asynchronous, env_vars=env_vars)
STATE['_thread_'] = thread
return thread
def _run(*args):
with INIT_LOCK:
base_dir = cls.get_base_dir()
if not os.path.exists(os.path.join(base_dir, '.terraform', 'plugins')):
run('cd %s; terraform init -input=false' % base_dir)
# remove any cache files from previous runs
for tf_file in ['tfplan', 'terraform.tfstate', 'terraform.tfstate.backup']:
rm_rf(os.path.join(base_dir, tf_file))
# create TF plan
run('cd %s; terraform plan -out=tfplan -input=false' % base_dir)
def get_lambda_code(func_name,
retries=1,
cache_time=None,
env=None,
region=None):
if MOCK_OBJ:
return ""
env = aws_stack.get_environment(env)
if cache_time is None and not aws_stack.is_local_env(env):
cache_time = AWS_LAMBDA_CODE_CACHE_TIMEOUT
lambda_client = _connect("lambda", env=env, region=region)
out = lambda_client.get_function(FunctionName=func_name)
loc = out["Code"]["Location"]
hash = md5(loc)
folder = TMP_DOWNLOAD_FILE_PATTERN.replace("*", hash)
filename = "archive.zip"
archive = "%s/%s" % (folder, filename)
try:
mkdir(folder)
if not os.path.isfile(archive):
download(loc, archive, verify_ssl=False)
if len(os.listdir(folder)) <= 1:
zip_path = os.path.join(folder, filename)
unzip(zip_path, folder)
except Exception as e:
print("WARN: %s" % e)
rm_rf(archive)
if retries > 0:
return get_lambda_code(func_name,
retries=retries - 1,
cache_time=1,
env=env)
else:
print("WARNING: Unable to retrieve lambda code: %s" % e)
# traverse subdirectories and get script sources
result = {}
for root, subdirs, files in os.walk(folder):
for file in files:
prefix = root.split(folder)[-1]
key = "%s/%s" % (prefix, file)
if re.match(r".+\.py$", key) or re.match(r".+\.js$", key):
codefile = "%s/%s" % (root, file)
result[key] = load_file(codefile)
# cleanup cache
clean_cache(
file_pattern=TMP_DOWNLOAD_FILE_PATTERN,
last_clean_time=last_cache_cleanup_time,
max_age=TMP_DOWNLOAD_CACHE_MAX_AGE,
)
# TODO: delete only if cache_time is over
rm_rf(folder)
return result
def get_lambda_code_param(params, **kwargs):
code = params.get('Code', {})
zip_file = code.get('ZipFile')
if zip_file and not common.is_base64(zip_file):
tmp_dir = common.new_tmp_dir()
handler_file = get_handler_file_from_name(params['Handler'], runtime=params['Runtime'])
tmp_file = os.path.join(tmp_dir, handler_file)
common.save_file(tmp_file, zip_file)
zip_file = create_zip_file(tmp_file, get_content=True)
code['ZipFile'] = zip_file
common.rm_rf(tmp_dir)
return code
def install_elasticsearch():
if not os.path.exists(INSTALL_DIR_ES):
log_install_msg('Elasticsearch')
mkdir(INSTALL_DIR_INFRA)
# download and extract archive
tmp_archive = os.path.join(tempfile.gettempdir(), 'localstack.es.zip')
download_and_extract_with_retry(ELASTICSEARCH_JAR_URL, tmp_archive,
INSTALL_DIR_INFRA)
elasticsearch_dir = glob.glob(
os.path.join(INSTALL_DIR_INFRA, 'elasticsearch*'))
if not elasticsearch_dir:
raise Exception('Unable to find Elasticsearch folder in %s' %
INSTALL_DIR_INFRA)
shutil.move(elasticsearch_dir[0], INSTALL_DIR_ES)
for dir_name in ('data', 'logs', 'modules', 'plugins',
'config/scripts'):
dir_path = '%s/%s' % (INSTALL_DIR_ES, dir_name)
mkdir(dir_path)
chmod_r(dir_path, 0o777)
# install default plugins
for plugin in ELASTICSEARCH_PLUGIN_LIST:
if is_alpine():
# https://github.com/pires/docker-elasticsearch/issues/56
os.environ['ES_TMPDIR'] = '/tmp'
plugin_binary = os.path.join(INSTALL_DIR_ES, 'bin',
'elasticsearch-plugin')
print('install elasticsearch-plugin %s' % (plugin))
run('%s install -b %s' % (plugin_binary, plugin))
# delete some plugins to free up space
for plugin in ELASTICSEARCH_DELETE_MODULES:
module_dir = os.path.join(INSTALL_DIR_ES, 'modules', plugin)
rm_rf(module_dir)
# disable x-pack-ml plugin (not working on Alpine)
xpack_dir = os.path.join(INSTALL_DIR_ES, 'modules', 'x-pack-ml',
'platform')
rm_rf(xpack_dir)
# patch JVM options file - replace hardcoded heap size settings
jvm_options_file = os.path.join(INSTALL_DIR_ES, 'config', 'jvm.options')
if os.path.exists(jvm_options_file):
jvm_options = load_file(jvm_options_file)
jvm_options_replaced = re.sub(r'(^-Xm[sx][a-zA-Z0-9\.]+$)',
r'# \1',
jvm_options,
flags=re.MULTILINE)
if jvm_options != jvm_options_replaced:
save_file(jvm_options_file, jvm_options_replaced)
def install_elasticsearch(version=None):
version = get_elasticsearch_install_version(version)
install_dir = get_elasticsearch_install_dir(version)
installed_executable = os.path.join(install_dir, 'bin', 'elasticsearch')
if not os.path.exists(installed_executable):
log_install_msg('Elasticsearch (%s)' % version)
es_url = ELASTICSEARCH_URLS.get(version)
if not es_url:
raise Exception('Unable to find download URL for Elasticsearch version "%s"' % version)
install_dir_parent = os.path.dirname(install_dir)
mkdir(install_dir_parent)
# download and extract archive
tmp_archive = os.path.join(config.TMP_FOLDER, 'localstack.%s' % os.path.basename(es_url))
download_and_extract_with_retry(es_url, tmp_archive, install_dir_parent)
elasticsearch_dir = glob.glob(os.path.join(install_dir_parent, 'elasticsearch*'))
if not elasticsearch_dir:
raise Exception('Unable to find Elasticsearch folder in %s' % install_dir_parent)
shutil.move(elasticsearch_dir[0], install_dir)
for dir_name in ('data', 'logs', 'modules', 'plugins', 'config/scripts'):
dir_path = os.path.join(install_dir, dir_name)
mkdir(dir_path)
chmod_r(dir_path, 0o777)
# install default plugins
for plugin in ELASTICSEARCH_PLUGIN_LIST:
if is_alpine():
# https://github.com/pires/docker-elasticsearch/issues/56
os.environ['ES_TMPDIR'] = '/tmp'
plugin_binary = os.path.join(install_dir, 'bin', 'elasticsearch-plugin')
plugin_dir = os.path.join(install_dir, 'plugins', plugin)
if not os.path.exists(plugin_dir):
LOG.info('Installing Elasticsearch plugin %s' % (plugin))
run('%s install -b %s' % (plugin_binary, plugin))
# delete some plugins to free up space
for plugin in ELASTICSEARCH_DELETE_MODULES:
module_dir = os.path.join(install_dir, 'modules', plugin)
rm_rf(module_dir)
# disable x-pack-ml plugin (not working on Alpine)
xpack_dir = os.path.join(install_dir, 'modules', 'x-pack-ml', 'platform')
rm_rf(xpack_dir)
# patch JVM options file - replace hardcoded heap size settings
jvm_options_file = os.path.join(install_dir, 'config', 'jvm.options')
if os.path.exists(jvm_options_file):
jvm_options = load_file(jvm_options_file)
jvm_options_replaced = re.sub(r'(^-Xm[sx][a-zA-Z0-9\.]+$)', r'# \1', jvm_options, flags=re.MULTILINE)
if jvm_options != jvm_options_replaced:
save_file(jvm_options_file, jvm_options_replaced)
def download_and_extract_with_retry(archive_url, tmp_archive, target_dir):
def download_and_extract():
if not os.path.exists(tmp_archive):
download(archive_url, tmp_archive)
unzip(tmp_archive, target_dir)
try:
download_and_extract()
except Exception:
# try deleting and re-downloading the zip file
LOGGER.info('Unable to extract file, re-downloading ZIP archive: %s' %
tmp_archive)
rm_rf(tmp_archive)
download_and_extract()
def _run(*args):
with INIT_LOCK:
base_dir = cls.get_base_dir()
if not os.path.exists(os.path.join(base_dir, ".terraform", "plugins")):
run("cd %s; terraform init -input=false" % base_dir)
# remove any cache files from previous runs
for tf_file in [
"tfplan",
"terraform.tfstate",
"terraform.tfstate.backup",
]:
rm_rf(os.path.join(base_dir, tf_file))
# create TF plan
run("cd %s; terraform plan -out=tfplan -input=false" % base_dir)
def get_lambda_code(func_name, retries=1, cache_time=None, env=None):
if MOCK_OBJ:
return ''
env = aws_stack.get_environment(env)
if cache_time is None and not aws_stack.is_local_env(env):
cache_time = AWS_LAMBDA_CODE_CACHE_TIMEOUT
out = cmd_lambda('get-function --function-name %s' % func_name, env,
cache_time)
out = json.loads(out)
loc = out['Code']['Location']
hash = md5(loc)
folder = TMP_DOWNLOAD_FILE_PATTERN.replace('*', hash)
filename = 'archive.zip'
archive = '%s/%s' % (folder, filename)
try:
mkdir(folder)
if not os.path.isfile(archive):
download(loc, archive, verify_ssl=False)
if len(os.listdir(folder)) <= 1:
zip_path = os.path.join(folder, filename)
unzip(zip_path, folder)
except Exception as e:
print('WARN: %s' % e)
rm_rf(archive)
if retries > 0:
return get_lambda_code(func_name,
retries=retries - 1,
cache_time=1,
env=env)
else:
print('WARNING: Unable to retrieve lambda code: %s' % e)
# traverse subdirectories and get script sources
result = {}
for root, subdirs, files in os.walk(folder):
for file in files:
prefix = root.split(folder)[-1]
key = '%s/%s' % (prefix, file)
if re.match(r'.+\.py$', key) or re.match(r'.+\.js$', key):
codefile = '%s/%s' % (root, file)
result[key] = load_file(codefile)
# cleanup cache
clean_cache(file_pattern=TMP_DOWNLOAD_FILE_PATTERN,
last_clean_time=last_cache_cleanup_time,
max_age=TMP_DOWNLOAD_CACHE_MAX_AGE)
# TODO: delete only if cache_time is over
rm_rf(folder)
return result
def download_and_extract_with_retry(archive_url, tmp_archive, target_dir):
def download_and_extract():
if not os.path.exists(tmp_archive):
download(archive_url, tmp_archive)
unzip(tmp_archive, target_dir)
try:
download_and_extract()
except Exception:
# try deleting and re-downloading the zip file
LOGGER.info('Unable to extract file, re-downloading ZIP archive: %s' % tmp_archive)
rm_rf(tmp_archive)
download_and_extract()
def install_stepfunctions_local():
if not os.path.exists(INSTALL_PATH_STEPFUNCTIONS_JAR):
# pull the JAR file from the Docker image, which is more up-to-date than the downloadable JAR file
log_install_msg('Step Functions')
mkdir(INSTALL_DIR_STEPFUNCTIONS)
run('{dc} pull {img}'.format(dc=config.DOCKER_CMD, img=IMAGE_NAME_SFN_LOCAL))
docker_name = 'tmp-ls-sfn'
run(('{dc} run --name={dn} --entrypoint= -d --rm {img} sleep 15').format(
dc=config.DOCKER_CMD, dn=docker_name, img=IMAGE_NAME_SFN_LOCAL))
time.sleep(5)
run('{dc} cp {dn}:/home/stepfunctionslocal/ {tgt}'.format(dc=config.DOCKER_CMD,
dn=docker_name, tgt=INSTALL_DIR_INFRA))
run('mv %s/stepfunctionslocal/*.jar %s' % (INSTALL_DIR_INFRA, INSTALL_DIR_STEPFUNCTIONS))
rm_rf('%s/stepfunctionslocal' % INSTALL_DIR_INFRA)
def listen_to_kinesis(stream_name, listener_func=None, processor_script=None,
events_file=None, endpoint_url=None, log_file=None, configs={}, env=None,
ddb_lease_table_suffix=None, env_vars={}, kcl_log_level=DEFAULT_KCL_LOG_LEVEL,
log_subscribers=[], wait_until_started=False, fh_d_stream=None, region_name=None):
"""
High-level function that allows to subscribe to a Kinesis stream
and receive events in a listener function. A KCL client process is
automatically started in the background.
"""
env = aws_stack.get_environment(env)
if not events_file:
events_file = EVENTS_FILE_PATTERN.replace('*', short_uid())
TMP_FILES.append(events_file)
if not processor_script:
processor_script = generate_processor_script(events_file, log_file=log_file)
rm_rf(events_file)
# start event reader thread (this process)
ready_mutex = threading.Semaphore(0)
thread = EventFileReaderThread(events_file, listener_func, ready_mutex=ready_mutex, fh_d_stream=fh_d_stream)
thread.start()
# Wait until the event reader thread is ready (to avoid 'Connection refused' error on the UNIX socket)
ready_mutex.acquire()
# start KCL client (background process)
if processor_script[-4:] == '.pyc':
processor_script = processor_script[0:-1]
# add log listener that notifies when KCL is started
if wait_until_started:
listener = KclStartedLogListener()
log_subscribers.append(listener)
process = start_kcl_client_process(stream_name, processor_script,
endpoint_url=endpoint_url, log_file=log_file, configs=configs, env=env,
ddb_lease_table_suffix=ddb_lease_table_suffix, env_vars=env_vars, kcl_log_level=kcl_log_level,
log_subscribers=log_subscribers, region_name=region_name)
if wait_until_started:
# Wait at most 90 seconds for initialization. Note that creating the DDB table can take quite a bit
try:
listener.sync_init.get(block=True, timeout=90)
except Exception:
raise Exception('Timeout when waiting for KCL initialization.')
# wait at most 30 seconds for shard lease notification
try:
listener.sync_take_shard.get(block=True, timeout=30)
except Exception:
# this merely means that there is no shard available to take. Do nothing.
pass
return process
def install_dynamodb_local():
if OVERWRITE_DDB_FILES_IN_DOCKER and in_docker():
rm_rf(INSTALL_DIR_DDB)
if not os.path.exists(INSTALL_PATH_DDB_JAR):
log_install_msg('DynamoDB')
# download and extract archive
tmp_archive = os.path.join(tempfile.gettempdir(), 'localstack.ddb.zip')
dynamodb_url = DYNAMODB_JAR_URL_ALPINE if in_docker(
) else DYNAMODB_JAR_URL
download_and_extract_with_retry(dynamodb_url, tmp_archive,
INSTALL_DIR_DDB)
# fix for Alpine, otherwise DynamoDBLocal fails with:
# DynamoDBLocal_lib/libsqlite4java-linux-amd64.so: __memcpy_chk: symbol not found
if is_alpine():
ddb_libs_dir = '%s/DynamoDBLocal_lib' % INSTALL_DIR_DDB
patched_marker = '%s/alpine_fix_applied' % ddb_libs_dir
if APPLY_DDB_ALPINE_FIX and not os.path.exists(patched_marker):
patched_lib = (
'https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/'
+
'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/libsqlite4java-linux-amd64.so'
)
patched_jar = (
'https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/'
+
'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/sqlite4java.jar')
run("curl -L -o %s/libsqlite4java-linux-amd64.so '%s'" %
(ddb_libs_dir, patched_lib))
run("curl -L -o %s/sqlite4java.jar '%s'" %
(ddb_libs_dir, patched_jar))
save_file(patched_marker, '')
# fix logging configuration for DynamoDBLocal
log4j2_config = """<Configuration status="WARN">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="WARN"><AppenderRef ref="Console"/></Root>
</Loggers>
</Configuration>"""
log4j2_file = os.path.join(INSTALL_DIR_DDB, 'log4j2.xml')
save_file(log4j2_file, log4j2_config)
run('cd "%s" && zip -u DynamoDBLocal.jar log4j2.xml || true' %
INSTALL_DIR_DDB)
def clean_cache(file_pattern=CACHE_FILE_PATTERN,
last_clean_time=None,
max_age=CACHE_MAX_AGE):
if last_clean_time is None:
last_clean_time = last_cache_clean_time
with MUTEX_CLEAN:
time_now = now()
if last_clean_time["time"] > time_now - CACHE_CLEAN_TIMEOUT:
return
for cache_file in set(glob.glob(file_pattern)):
mod_time = os.path.getmtime(cache_file)
if time_now > mod_time + max_age:
rm_rf(cache_file)
last_clean_time["time"] = time_now
return time_now
def _init_directories(self):
dirs = self.directories
LOG.debug("initializing elasticsearch directories %s", dirs)
chmod_r(dirs.base, 0o777)
if not dirs.data.startswith(config.DATA_DIR):
# only clear previous data if it's not in DATA_DIR
rm_rf(dirs.data)
mkdir(dirs.data)
chmod_r(dirs.data, 0o777)
rm_rf(dirs.tmp)
mkdir(dirs.tmp)
chmod_r(dirs.tmp, 0o777)
def _run(*args):
with INIT_LOCK:
install_terraform()
base_dir = get_base_dir()
if not os.path.exists(os.path.join(base_dir, ".terraform", "plugins")):
run(f"cd {base_dir}; {TERRAFORM_BIN} init -input=false")
# remove any cache files from previous runs
for tf_file in [
"tfplan",
"terraform.tfstate",
"terraform.tfstate.backup",
]:
rm_rf(os.path.join(base_dir, tf_file))
# create TF plan
run(f"cd {base_dir}; {TERRAFORM_BIN} plan -out=tfplan -input=false")
def test_create_archive(self):
# create archive from empty directory
tmp_dir = new_tmp_dir()
content = create_zip_file(tmp_dir, get_content=True)
zip_obj = zipfile.ZipFile(io.BytesIO(content))
assert zip_obj.infolist() == []
rm_rf(tmp_dir)
# create archive from non-empty directory
tmp_dir = new_tmp_dir()
save_file(os.path.join(tmp_dir, "testfile"), "content 123")
content = create_zip_file(tmp_dir, get_content=True)
zip_obj = zipfile.ZipFile(io.BytesIO(content))
assert len(zip_obj.infolist()) == 1
assert zip_obj.infolist()[0].filename == "testfile"
rm_rf(tmp_dir)
def get_lambda_code(func_name, retries=1, cache_time=None, env=None):
if MOCK_OBJ:
return ''
env = aws_stack.get_environment(env)
if cache_time is None and env.region != REGION_LOCAL:
cache_time = AWS_LAMBDA_CODE_CACHE_TIMEOUT
out = cmd_lambda('get-function --function-name %s' % func_name, env, cache_time)
out = json.loads(out)
loc = out['Code']['Location']
hash = md5(loc)
folder = TMP_DOWNLOAD_FILE_PATTERN.replace('*', hash)
filename = 'archive.zip'
archive = '%s/%s' % (folder, filename)
try:
mkdir(folder)
if not os.path.isfile(archive):
download(loc, archive, verify_ssl=False)
if len(os.listdir(folder)) <= 1:
zip_path = os.path.join(folder, filename)
unzip(zip_path, folder)
except Exception as e:
print('WARN: %s' % e)
rm_rf(archive)
if retries > 0:
return get_lambda_code(func_name, retries=retries - 1, cache_time=1, env=env)
else:
print('WARNING: Unable to retrieve lambda code: %s' % e)
# traverse subdirectories and get script sources
result = {}
for root, subdirs, files in os.walk(folder):
for file in files:
prefix = root.split(folder)[-1]
key = '%s/%s' % (prefix, file)
if re.match(r'.+\.py$', key) or re.match(r'.+\.js$', key):
codefile = '%s/%s' % (root, file)
result[key] = load_file(codefile)
# cleanup cache
clean_cache(file_pattern=TMP_DOWNLOAD_FILE_PATTERN,
last_clean_time=last_cache_cleanup_time,
max_age=TMP_DOWNLOAD_CACHE_MAX_AGE)
# TODO: delete only if cache_time is over
rm_rf(folder)
return result
def rm_env_vars_file(env_vars_file_flag):
if not env_vars_file_flag or "--env-file" not in env_vars_file_flag:
return
if isinstance(env_vars_file_flag, list):
env_vars_file = env_vars_file_flag[env_vars_file_flag.index("--env-file") + 1]
else:
env_vars_file = env_vars_file_flag.replace("--env-file", "").strip()
return rm_rf(env_vars_file)
def test_download_with_timeout():
class DownloadListener(ProxyListener):
def forward_request(self, method, path, data, headers):
if path == "/sleep":
time.sleep(2)
return {}
port = get_free_tcp_port()
proxy = start_proxy_server(port, update_listener=DownloadListener())
tmp_file = new_tmp_file()
download(f"http://localhost:{port}/", tmp_file)
assert load_file(tmp_file) == "{}"
with pytest.raises(TimeoutError):
download(f"http://localhost:{port}/sleep", tmp_file, timeout=1)
# clean up
proxy.stop()
rm_rf(tmp_file)
def test_unzip_bad_crc(self):
"""Test unzipping of files with incorrect CRC codes - usually works with native `unzip` command,
but seems to fail with zipfile module under certain Python versions (extracts 0-bytes files)"""
# base64-encoded zip file with a single entry with incorrect CRC (created by Node.js 18 / Serverless)
zip_base64 = """
UEsDBBQAAAAIAAAAIQAAAAAAJwAAAAAAAAAjAAAAbm9kZWpzL25vZGVfbW9kdWxlcy9sb2Rhc2gvaW5k
ZXguanPLzU8pzUnVS60oyC8qKVawVShKLSzNLErVUNfTz8lPSSzOUNe0BgBQSwECLQMUAAAACAAAACEA
AAAAACcAAAAAAAAAIwAAAAAAAAAAACAApIEAAAAAbm9kZWpzL25vZGVfbW9kdWxlcy9sb2Rhc2gvaW5k
ZXguanNQSwUGAAAAAAEAAQBRAAAAaAAAAAAA
"""
tmp_dir = new_tmp_dir()
zip_file = os.path.join(tmp_dir, "test.zip")
save_file(zip_file, base64.b64decode(zip_base64))
unzip(zip_file, tmp_dir)
content = load_file(
os.path.join(tmp_dir, "nodejs", "node_modules", "lodash",
"index.js"))
assert content.strip() == "module.exports = require('./lodash');"
rm_rf(tmp_dir)
def test_generate_ssl_cert():
def _assert(cert, key):
# assert that file markers are in place
assert PEM_CERT_START in cert
assert PEM_CERT_END in cert
assert re.match(PEM_KEY_START_REGEX, key.replace("\n", " "))
assert re.match(fr".*{PEM_KEY_END_REGEX}", key.replace("\n", " "))
# generate cert and get content directly
cert = generate_ssl_cert()
_assert(cert, cert)
# generate cert to file and load content from there
target_file, cert_file_name, key_file_name = generate_ssl_cert(
target_file=new_tmp_file(), overwrite=True)
_assert(load_file(cert_file_name), load_file(key_file_name))
# clean up
rm_rf(cert_file_name)
rm_rf(key_file_name)
def install_stepfunctions_local():
if not os.path.exists(INSTALL_PATH_STEPFUNCTIONS_JAR):
# pull the JAR file from the Docker image, which is more up-to-date than the downloadable JAR file
# TODO: works only when running on the host, outside of Docker -> add a fallback if running in Docker?
log_install_msg("Step Functions")
mkdir(INSTALL_DIR_STEPFUNCTIONS)
DOCKER_CLIENT.pull_image(IMAGE_NAME_SFN_LOCAL)
docker_name = "tmp-ls-sfn"
DOCKER_CLIENT.run_container(
IMAGE_NAME_SFN_LOCAL,
remove=True,
entrypoint="",
name=docker_name,
detach=True,
command=["sleep", "15"],
)
time.sleep(5)
DOCKER_CLIENT.copy_from_container(
docker_name,
local_path=INSTALL_DIR_INFRA,
container_path="/home/stepfunctionslocal/")
path = Path(f"{INSTALL_DIR_INFRA}/stepfunctionslocal/")
for file in path.glob("*.jar"):
file.rename(Path(INSTALL_DIR_STEPFUNCTIONS) / file.name)
rm_rf("%s/stepfunctionslocal" % INSTALL_DIR_INFRA)
# apply patches
for patch_class, patch_url in (
(SFN_PATCH_CLASS1, SFN_PATCH_CLASS_URL1),
(SFN_PATCH_CLASS2, SFN_PATCH_CLASS_URL2),
):
patch_class_file = os.path.join(INSTALL_DIR_STEPFUNCTIONS, patch_class)
if not os.path.exists(patch_class_file):
download(patch_url, patch_class_file)
cmd = 'cd "%s"; zip %s %s' % (
INSTALL_DIR_STEPFUNCTIONS,
INSTALL_PATH_STEPFUNCTIONS_JAR,
patch_class,
)
run(cmd)
