def install_dynamodb_local(): if OVERWRITE_DDB_FILES_IN_DOCKER and in_docker(): rm_rf(INSTALL_DIR_DDB) if not os.path.exists(INSTALL_PATH_DDB_JAR): log_install_msg('DynamoDB') # download and extract archive tmp_archive = os.path.join(tempfile.gettempdir(), 'localstack.ddb.zip') dynamodb_url = DYNAMODB_JAR_URL_ALPINE if in_docker( ) else DYNAMODB_JAR_URL download_and_extract_with_retry(dynamodb_url, tmp_archive, INSTALL_DIR_DDB) # fix for Alpine, otherwise DynamoDBLocal fails with: # DynamoDBLocal_lib/libsqlite4java-linux-amd64.so: __memcpy_chk: symbol not found if is_alpine(): ddb_libs_dir = '%s/DynamoDBLocal_lib' % INSTALL_DIR_DDB patched_marker = '%s/alpine_fix_applied' % ddb_libs_dir if APPLY_DDB_ALPINE_FIX and not os.path.exists(patched_marker): patched_lib = ( 'https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/' + 'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/libsqlite4java-linux-amd64.so' ) patched_jar = ( 'https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/' + 'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/sqlite4java.jar') run("curl -L -o %s/libsqlite4java-linux-amd64.so '%s'" % (ddb_libs_dir, patched_lib)) run("curl -L -o %s/sqlite4java.jar '%s'" % (ddb_libs_dir, patched_jar)) save_file(patched_marker, '') # fix logging configuration for DynamoDBLocal log4j2_config = """<Configuration status="WARN"> <Appenders> <Console name="Console" target="SYSTEM_OUT"> <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/> </Console> </Appenders> <Loggers> <Root level="WARN"><AppenderRef ref="Console"/></Root> </Loggers> </Configuration>""" log4j2_file = os.path.join(INSTALL_DIR_DDB, 'log4j2.xml') save_file(log4j2_file, log4j2_config) run('cd "%s" && zip -u DynamoDBLocal.jar log4j2.xml || true' % INSTALL_DIR_DDB)
def install_dynamodb_local(): if OVERWRITE_DDB_FILES_IN_DOCKER and in_docker(): rm_rf(INSTALL_DIR_DDB) is_in_alpine = is_alpine() if not os.path.exists(INSTALL_PATH_DDB_JAR): log_install_msg("DynamoDB") # download and extract archive tmp_archive = os.path.join(tempfile.gettempdir(), "localstack.ddb.zip") dynamodb_url = DYNAMODB_JAR_URL_ALPINE if is_in_alpine else DYNAMODB_JAR_URL download_and_extract_with_retry(dynamodb_url, tmp_archive, INSTALL_DIR_DDB) # fix logging configuration for DynamoDBLocal log4j2_config = """<Configuration status="WARN"> <Appenders> <Console name="Console" target="SYSTEM_OUT"> <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/> </Console> </Appenders> <Loggers> <Root level="WARN"><AppenderRef ref="Console"/></Root> </Loggers> </Configuration>""" log4j2_file = os.path.join(INSTALL_DIR_DDB, "log4j2.xml") save_file(log4j2_file, log4j2_config) run('cd "%s" && zip -u DynamoDBLocal.jar log4j2.xml || true' % INSTALL_DIR_DDB)
def start_infra(asynchronous=False, apis=None): try: os.environ[LOCALSTACK_INFRA_PROCESS] = '1' is_in_docker = in_docker() # print a warning if we're not running in Docker but using Docker based LAMBDA_EXECUTOR if not is_in_docker and 'docker' in config.LAMBDA_EXECUTOR and not is_linux(): print(('!WARNING! - Running outside of Docker with $LAMBDA_EXECUTOR=%s can lead to ' 'problems on your OS. The environment variable $LOCALSTACK_HOSTNAME may not ' 'be properly set in your Lambdas.') % config.LAMBDA_EXECUTOR) if is_in_docker and config.LAMBDA_REMOTE_DOCKER and not os.environ.get('HOST_TMP_FOLDER'): print('!WARNING! - Looks like you have configured $LAMBDA_REMOTE_DOCKER=1 - ' "please make sure to configure $HOST_TMP_FOLDER to point to your host's $TMPDIR") # apply patches patch_urllib3_connection_pool(maxsize=128) # load plugins load_plugins() # with plugins loaded, now start the infrastructure do_start_infra(asynchronous, apis, is_in_docker) except KeyboardInterrupt: print('Shutdown') except Exception as e: print('Error starting infrastructure: %s %s' % (e, traceback.format_exc())) sys.stdout.flush() raise e finally: if not asynchronous: stop_infra()
def start_infra(asynchronous=False, apis=None): try: # load plugins load_plugins() event_publisher.fire_event(event_publisher.EVENT_START_INFRA, {'d': in_docker() and 1 or 0, 'c': in_ci() and 1 or 0}) # set up logging setup_logging() # prepare APIs apis = canonicalize_api_names(apis) # set environment os.environ['AWS_REGION'] = config.DEFAULT_REGION os.environ['ENV'] = ENV_DEV # register signal handlers if not os.environ.get(ENV_INTERNAL_TEST_RUN): register_signal_handlers() # make sure AWS credentials are configured, otherwise boto3 bails on us check_aws_credentials() # install libs if not present install.install_components(apis) # Some services take a bit to come up sleep_time = 5 # start services thread = None if 'elasticsearch' in apis or 'es' in apis: sleep_time = max(sleep_time, 10) # loop through plugins and start each service for name, plugin in SERVICE_PLUGINS.items(): if name in apis: t1 = plugin.start(asynchronous=True) thread = thread or t1 time.sleep(sleep_time) # ensure that all infra components are up and running check_infra(apis=apis) # restore persisted data restore_persisted_data(apis=apis) print('Ready.') sys.stdout.flush() if not asynchronous and thread: # this is a bit of an ugly hack, but we need to make sure that we # stay in the execution context of the main thread, otherwise our # signal handlers don't work while True: time.sleep(1) return thread except KeyboardInterrupt: print('Shutdown') except Exception as e: print('Error starting infrastructure: %s %s' % (e, traceback.format_exc())) sys.stdout.flush() raise e finally: if not asynchronous: stop_infra()
def start_infra(asynchronous=False, apis=None): try: os.environ[LOCALSTACK_INFRA_PROCESS] = "1" is_in_docker = in_docker() # print a warning if we're not running in Docker but using Docker based LAMBDA_EXECUTOR if not is_in_docker and "docker" in config.LAMBDA_EXECUTOR and not is_linux(): print( ( "!WARNING! - Running outside of Docker with $LAMBDA_EXECUTOR=%s can lead to " "problems on your OS. The environment variable $LOCALSTACK_HOSTNAME may not " "be properly set in your Lambdas." ) % config.LAMBDA_EXECUTOR ) if ( is_in_docker and not config.LAMBDA_REMOTE_DOCKER and not os.environ.get("HOST_TMP_FOLDER") ): print( "!WARNING! - Looks like you have configured $LAMBDA_REMOTE_DOCKER=0 - " "please make sure to configure $HOST_TMP_FOLDER to point to your host's $TMPDIR" ) print_runtime_information(is_in_docker) # apply patches patch_urllib3_connection_pool(maxsize=128) patch_instance_tracker_meta() # load plugins load_plugins() # with plugins loaded, now start the infrastructure thread = do_start_infra(asynchronous, apis, is_in_docker) if not asynchronous and thread: # We're making sure that we stay in the execution context of the # main thread, otherwise our signal handlers don't work SHUTDOWN_INFRA.wait() return thread except KeyboardInterrupt: print("Shutdown") except Exception as e: print("Error starting infrastructure: %s %s" % (e, traceback.format_exc())) sys.stdout.flush() raise e finally: if not asynchronous: stop_infra()
def get_docker_host_from_container(): global DOCKER_MAIN_CONTAINER_IP if DOCKER_MAIN_CONTAINER_IP is None: DOCKER_MAIN_CONTAINER_IP = False try: if in_docker(): DOCKER_MAIN_CONTAINER_IP = bootstrap.get_main_container_ip() except Exception as e: LOG.info('Unable to get IP address of main Docker container "%s": %s' % (bootstrap.MAIN_CONTAINER_NAME, e)) # return main container IP, or fall back to Docker host (bridge IP, or host DNS address) return DOCKER_MAIN_CONTAINER_IP or config.DOCKER_HOST_FROM_CONTAINER
def get_main_endpoint_from_container(): global DOCKER_MAIN_CONTAINER_IP if DOCKER_MAIN_CONTAINER_IP is None: DOCKER_MAIN_CONTAINER_IP = False try: if in_docker(): DOCKER_MAIN_CONTAINER_IP = bootstrap.get_main_container_ip() LOG.info('Determined main container target IP: %s' % DOCKER_MAIN_CONTAINER_IP) except Exception as e: container_name = bootstrap.get_main_container_name() LOG.info('Unable to get IP address of main Docker container "%s": %s' % (container_name, e)) # return main container IP, or fall back to Docker host (bridge IP, or host DNS address) return DOCKER_MAIN_CONTAINER_IP or config.DOCKER_HOST_FROM_CONTAINER
def get_main_endpoint_from_container(): global DOCKER_MAIN_CONTAINER_IP if not config.HOSTNAME_FROM_LAMBDA and DOCKER_MAIN_CONTAINER_IP is None: DOCKER_MAIN_CONTAINER_IP = False try: if in_docker(): DOCKER_MAIN_CONTAINER_IP = bootstrap.get_main_container_ip() LOG.info('Determined main container target IP: %s' % DOCKER_MAIN_CONTAINER_IP) except Exception as e: container_name = bootstrap.get_main_container_name() LOG.info('Unable to get IP address of main Docker container "%s": %s' % (container_name, e)) # return (1) predefined endpoint host, or (2) main container IP, or (3) Docker host (e.g., bridge IP) return config.HOSTNAME_FROM_LAMBDA or DOCKER_MAIN_CONTAINER_IP or config.DOCKER_HOST_FROM_CONTAINER
def terminate_all_processes_in_docker(): if not in_docker(): # make sure we only run this inside docker! return print('INFO: Received command to restart all processes ...') cmd = ('ps aux | grep -v supervisor | grep -v docker-entrypoint.sh | grep -v "make infra" | ' "grep -v localstack_infra.log | awk '{print $1}' | grep -v PID") pids = run(cmd).strip() pids = re.split(r'\s+', pids) pids = [int(pid) for pid in pids] this_pid = os.getpid() for pid in pids: if pid != this_pid: try: # kill spawned process os.kill(pid, signal.SIGKILL) except Exception: pass # kill the process itself os._exit(0)
def start_infra(asynchronous=False, apis=None): try: os.environ[LOCALSTACK_INFRA_PROCESS] = '1' is_in_docker = in_docker() # print a warning if we're not running in Docker but using Docker based LAMBDA_EXECUTOR if not is_in_docker and 'docker' in config.LAMBDA_EXECUTOR and not is_linux( ): print(( '!WARNING! - Running outside of Docker with $LAMBDA_EXECUTOR=%s can lead to ' 'problems on your OS. The environment variable $LOCALSTACK_HOSTNAME may not ' 'be properly set in your Lambdas.') % config.LAMBDA_EXECUTOR) if is_in_docker and config.LAMBDA_REMOTE_DOCKER and not os.environ.get( 'HOST_TMP_FOLDER'): print( '!WARNING! - Looks like you have configured $LAMBDA_REMOTE_DOCKER=1 - ' "please make sure to configure $HOST_TMP_FOLDER to point to your host's $TMPDIR" ) # apply patches patch_urllib3_connection_pool(maxsize=128) # load plugins load_plugins() event_publisher.fire_event(event_publisher.EVENT_START_INFRA, { 'd': is_in_docker and 1 or 0, 'c': in_ci() and 1 or 0 }) # set up logging setup_logging() # prepare APIs apis = canonicalize_api_names(apis) # set environment os.environ['AWS_REGION'] = config.DEFAULT_REGION os.environ['ENV'] = ENV_DEV # register signal handlers if not os.environ.get(ENV_INTERNAL_TEST_RUN): register_signal_handlers() # make sure AWS credentials are configured, otherwise boto3 bails on us check_aws_credentials() # install libs if not present install.install_components(apis) # Some services take a bit to come up sleep_time = 5 # start services thread = None # loop through plugins and start each service for name, plugin in SERVICE_PLUGINS.items(): if plugin.is_enabled(api_names=apis): record_service_health(name, 'starting') t1 = plugin.start(asynchronous=True) thread = thread or t1 time.sleep(sleep_time) # ensure that all infra components are up and running check_infra(apis=apis) # restore persisted data persistence.restore_persisted_data(apis=apis) print('Ready.') sys.stdout.flush() if not asynchronous and thread: # this is a bit of an ugly hack, but we need to make sure that we # stay in the execution context of the main thread, otherwise our # signal handlers don't work while True: time.sleep(1) return thread except KeyboardInterrupt: print('Shutdown') except Exception as e: print('Error starting infrastructure: %s %s' % (e, traceback.format_exc())) sys.stdout.flush() raise e finally: if not asynchronous: stop_infra()