def install_dynamodb_local():
if OVERWRITE_DDB_FILES_IN_DOCKER and in_docker():
rm_rf(INSTALL_DIR_DDB)
if not os.path.exists(INSTALL_PATH_DDB_JAR):
log_install_msg('DynamoDB')
# download and extract archive
tmp_archive = os.path.join(tempfile.gettempdir(), 'localstack.ddb.zip')
dynamodb_url = DYNAMODB_JAR_URL_ALPINE if in_docker(
) else DYNAMODB_JAR_URL
download_and_extract_with_retry(dynamodb_url, tmp_archive,
INSTALL_DIR_DDB)
# fix for Alpine, otherwise DynamoDBLocal fails with:
# DynamoDBLocal_lib/libsqlite4java-linux-amd64.so: __memcpy_chk: symbol not found
if is_alpine():
ddb_libs_dir = '%s/DynamoDBLocal_lib' % INSTALL_DIR_DDB
patched_marker = '%s/alpine_fix_applied' % ddb_libs_dir
if APPLY_DDB_ALPINE_FIX and not os.path.exists(patched_marker):
patched_lib = (
'https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/'
+
'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/libsqlite4java-linux-amd64.so'
)
patched_jar = (
'https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/'
+
'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/sqlite4java.jar')
run("curl -L -o %s/libsqlite4java-linux-amd64.so '%s'" %
(ddb_libs_dir, patched_lib))
run("curl -L -o %s/sqlite4java.jar '%s'" %
(ddb_libs_dir, patched_jar))
save_file(patched_marker, '')
# fix logging configuration for DynamoDBLocal
log4j2_config = """<Configuration status="WARN">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="WARN"><AppenderRef ref="Console"/></Root>
</Loggers>
</Configuration>"""
log4j2_file = os.path.join(INSTALL_DIR_DDB, 'log4j2.xml')
save_file(log4j2_file, log4j2_config)
run('cd "%s" && zip -u DynamoDBLocal.jar log4j2.xml || true' %
INSTALL_DIR_DDB)
def install_dynamodb_local():
if OVERWRITE_DDB_FILES_IN_DOCKER and in_docker():
rm_rf(INSTALL_DIR_DDB)
is_in_alpine = is_alpine()
if not os.path.exists(INSTALL_PATH_DDB_JAR):
log_install_msg("DynamoDB")
# download and extract archive
tmp_archive = os.path.join(tempfile.gettempdir(), "localstack.ddb.zip")
dynamodb_url = DYNAMODB_JAR_URL_ALPINE if is_in_alpine else DYNAMODB_JAR_URL
download_and_extract_with_retry(dynamodb_url, tmp_archive,
INSTALL_DIR_DDB)
# fix logging configuration for DynamoDBLocal
log4j2_config = """<Configuration status="WARN">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="WARN"><AppenderRef ref="Console"/></Root>
</Loggers>
</Configuration>"""
log4j2_file = os.path.join(INSTALL_DIR_DDB, "log4j2.xml")
save_file(log4j2_file, log4j2_config)
run('cd "%s" && zip -u DynamoDBLocal.jar log4j2.xml || true' %
INSTALL_DIR_DDB)
def start_infra(asynchronous=False, apis=None):
try:
os.environ[LOCALSTACK_INFRA_PROCESS] = '1'
is_in_docker = in_docker()
# print a warning if we're not running in Docker but using Docker based LAMBDA_EXECUTOR
if not is_in_docker and 'docker' in config.LAMBDA_EXECUTOR and not is_linux():
print(('!WARNING! - Running outside of Docker with $LAMBDA_EXECUTOR=%s can lead to '
'problems on your OS. The environment variable $LOCALSTACK_HOSTNAME may not '
'be properly set in your Lambdas.') % config.LAMBDA_EXECUTOR)
if is_in_docker and config.LAMBDA_REMOTE_DOCKER and not os.environ.get('HOST_TMP_FOLDER'):
print('!WARNING! - Looks like you have configured $LAMBDA_REMOTE_DOCKER=1 - '
"please make sure to configure $HOST_TMP_FOLDER to point to your host's $TMPDIR")
# apply patches
patch_urllib3_connection_pool(maxsize=128)
# load plugins
load_plugins()
# with plugins loaded, now start the infrastructure
do_start_infra(asynchronous, apis, is_in_docker)
except KeyboardInterrupt:
print('Shutdown')
except Exception as e:
print('Error starting infrastructure: %s %s' % (e, traceback.format_exc()))
sys.stdout.flush()
raise e
finally:
if not asynchronous:
stop_infra()
def start_infra(asynchronous=False, apis=None):
try:
# load plugins
load_plugins()
event_publisher.fire_event(event_publisher.EVENT_START_INFRA,
{'d': in_docker() and 1 or 0, 'c': in_ci() and 1 or 0})
# set up logging
setup_logging()
# prepare APIs
apis = canonicalize_api_names(apis)
# set environment
os.environ['AWS_REGION'] = config.DEFAULT_REGION
os.environ['ENV'] = ENV_DEV
# register signal handlers
if not os.environ.get(ENV_INTERNAL_TEST_RUN):
register_signal_handlers()
# make sure AWS credentials are configured, otherwise boto3 bails on us
check_aws_credentials()
# install libs if not present
install.install_components(apis)
# Some services take a bit to come up
sleep_time = 5
# start services
thread = None
if 'elasticsearch' in apis or 'es' in apis:
sleep_time = max(sleep_time, 10)
# loop through plugins and start each service
for name, plugin in SERVICE_PLUGINS.items():
if name in apis:
t1 = plugin.start(asynchronous=True)
thread = thread or t1
time.sleep(sleep_time)
# ensure that all infra components are up and running
check_infra(apis=apis)
# restore persisted data
restore_persisted_data(apis=apis)
print('Ready.')
sys.stdout.flush()
if not asynchronous and thread:
# this is a bit of an ugly hack, but we need to make sure that we
# stay in the execution context of the main thread, otherwise our
# signal handlers don't work
while True:
time.sleep(1)
return thread
except KeyboardInterrupt:
print('Shutdown')
except Exception as e:
print('Error starting infrastructure: %s %s' % (e, traceback.format_exc()))
sys.stdout.flush()
raise e
finally:
if not asynchronous:
stop_infra()
def start_infra(asynchronous=False, apis=None):
try:
os.environ[LOCALSTACK_INFRA_PROCESS] = "1"
is_in_docker = in_docker()
# print a warning if we're not running in Docker but using Docker based LAMBDA_EXECUTOR
if not is_in_docker and "docker" in config.LAMBDA_EXECUTOR and not is_linux():
print(
(
"!WARNING! - Running outside of Docker with $LAMBDA_EXECUTOR=%s can lead to "
"problems on your OS. The environment variable $LOCALSTACK_HOSTNAME may not "
"be properly set in your Lambdas."
)
% config.LAMBDA_EXECUTOR
)
if (
is_in_docker
and not config.LAMBDA_REMOTE_DOCKER
and not os.environ.get("HOST_TMP_FOLDER")
):
print(
"!WARNING! - Looks like you have configured $LAMBDA_REMOTE_DOCKER=0 - "
"please make sure to configure $HOST_TMP_FOLDER to point to your host's $TMPDIR"
)
print_runtime_information(is_in_docker)
# apply patches
patch_urllib3_connection_pool(maxsize=128)
patch_instance_tracker_meta()
# load plugins
load_plugins()
# with plugins loaded, now start the infrastructure
thread = do_start_infra(asynchronous, apis, is_in_docker)
if not asynchronous and thread:
# We're making sure that we stay in the execution context of the
# main thread, otherwise our signal handlers don't work
SHUTDOWN_INFRA.wait()
return thread
except KeyboardInterrupt:
print("Shutdown")
except Exception as e:
print("Error starting infrastructure: %s %s" % (e, traceback.format_exc()))
sys.stdout.flush()
raise e
finally:
if not asynchronous:
stop_infra()
def get_docker_host_from_container():
global DOCKER_MAIN_CONTAINER_IP
if DOCKER_MAIN_CONTAINER_IP is None:
DOCKER_MAIN_CONTAINER_IP = False
try:
if in_docker():
DOCKER_MAIN_CONTAINER_IP = bootstrap.get_main_container_ip()
except Exception as e:
LOG.info('Unable to get IP address of main Docker container "%s": %s' %
(bootstrap.MAIN_CONTAINER_NAME, e))
# return main container IP, or fall back to Docker host (bridge IP, or host DNS address)
return DOCKER_MAIN_CONTAINER_IP or config.DOCKER_HOST_FROM_CONTAINER
def get_main_endpoint_from_container():
global DOCKER_MAIN_CONTAINER_IP
if DOCKER_MAIN_CONTAINER_IP is None:
DOCKER_MAIN_CONTAINER_IP = False
try:
if in_docker():
DOCKER_MAIN_CONTAINER_IP = bootstrap.get_main_container_ip()
LOG.info('Determined main container target IP: %s' % DOCKER_MAIN_CONTAINER_IP)
except Exception as e:
container_name = bootstrap.get_main_container_name()
LOG.info('Unable to get IP address of main Docker container "%s": %s' %
(container_name, e))
# return main container IP, or fall back to Docker host (bridge IP, or host DNS address)
return DOCKER_MAIN_CONTAINER_IP or config.DOCKER_HOST_FROM_CONTAINER
def get_main_endpoint_from_container():
global DOCKER_MAIN_CONTAINER_IP
if not config.HOSTNAME_FROM_LAMBDA and DOCKER_MAIN_CONTAINER_IP is None:
DOCKER_MAIN_CONTAINER_IP = False
try:
if in_docker():
DOCKER_MAIN_CONTAINER_IP = bootstrap.get_main_container_ip()
LOG.info('Determined main container target IP: %s' % DOCKER_MAIN_CONTAINER_IP)
except Exception as e:
container_name = bootstrap.get_main_container_name()
LOG.info('Unable to get IP address of main Docker container "%s": %s' %
(container_name, e))
# return (1) predefined endpoint host, or (2) main container IP, or (3) Docker host (e.g., bridge IP)
return config.HOSTNAME_FROM_LAMBDA or DOCKER_MAIN_CONTAINER_IP or config.DOCKER_HOST_FROM_CONTAINER
def terminate_all_processes_in_docker():
if not in_docker():
# make sure we only run this inside docker!
return
print('INFO: Received command to restart all processes ...')
cmd = ('ps aux | grep -v supervisor | grep -v docker-entrypoint.sh | grep -v "make infra" | '
"grep -v localstack_infra.log | awk '{print $1}' | grep -v PID")
pids = run(cmd).strip()
pids = re.split(r'\s+', pids)
pids = [int(pid) for pid in pids]
this_pid = os.getpid()
for pid in pids:
if pid != this_pid:
try:
# kill spawned process
os.kill(pid, signal.SIGKILL)
except Exception:
pass
# kill the process itself
os._exit(0)
def start_infra(asynchronous=False, apis=None):
try:
os.environ[LOCALSTACK_INFRA_PROCESS] = '1'
is_in_docker = in_docker()
# print a warning if we're not running in Docker but using Docker based LAMBDA_EXECUTOR
if not is_in_docker and 'docker' in config.LAMBDA_EXECUTOR and not is_linux(
):
print((
'!WARNING! - Running outside of Docker with $LAMBDA_EXECUTOR=%s can lead to '
'problems on your OS. The environment variable $LOCALSTACK_HOSTNAME may not '
'be properly set in your Lambdas.') % config.LAMBDA_EXECUTOR)
if is_in_docker and config.LAMBDA_REMOTE_DOCKER and not os.environ.get(
'HOST_TMP_FOLDER'):
print(
'!WARNING! - Looks like you have configured $LAMBDA_REMOTE_DOCKER=1 - '
"please make sure to configure $HOST_TMP_FOLDER to point to your host's $TMPDIR"
)
# apply patches
patch_urllib3_connection_pool(maxsize=128)
# load plugins
load_plugins()
event_publisher.fire_event(event_publisher.EVENT_START_INFRA, {
'd': is_in_docker and 1 or 0,
'c': in_ci() and 1 or 0
})
# set up logging
setup_logging()
# prepare APIs
apis = canonicalize_api_names(apis)
# set environment
os.environ['AWS_REGION'] = config.DEFAULT_REGION
os.environ['ENV'] = ENV_DEV
# register signal handlers
if not os.environ.get(ENV_INTERNAL_TEST_RUN):
register_signal_handlers()
# make sure AWS credentials are configured, otherwise boto3 bails on us
check_aws_credentials()
# install libs if not present
install.install_components(apis)
# Some services take a bit to come up
sleep_time = 5
# start services
thread = None
# loop through plugins and start each service
for name, plugin in SERVICE_PLUGINS.items():
if plugin.is_enabled(api_names=apis):
record_service_health(name, 'starting')
t1 = plugin.start(asynchronous=True)
thread = thread or t1
time.sleep(sleep_time)
# ensure that all infra components are up and running
check_infra(apis=apis)
# restore persisted data
persistence.restore_persisted_data(apis=apis)
print('Ready.')
sys.stdout.flush()
if not asynchronous and thread:
# this is a bit of an ugly hack, but we need to make sure that we
# stay in the execution context of the main thread, otherwise our
# signal handlers don't work
while True:
time.sleep(1)
return thread
except KeyboardInterrupt:
print('Shutdown')
except Exception as e:
print('Error starting infrastructure: %s %s' %
(e, traceback.format_exc()))
sys.stdout.flush()
raise e
finally:
if not asynchronous:
stop_infra()
