def start_dns_server(asynchronous=False):
try:
# start local DNS server, if present
from localstack_ext import config as config_ext
from localstack_ext.services import dns_server
if config_ext.DNS_ADDRESS in config.FALSE_STRINGS:
return
if is_port_open(PORT_DNS):
return
if is_root():
result = dns_server.start_servers()
if not asynchronous:
sleep_forever()
return result
env_vars = {}
for env_var in config.CONFIG_ENV_VARS:
if env_var.startswith("DNS_"):
value = os.environ.get(env_var, None)
if value is not None:
env_vars[env_var] = value
# note: running in a separate process breaks integration with Route53 (to be fixed for local dev mode!)
return run_process_as_sudo("dns",
PORT_DNS,
asynchronous=asynchronous,
env_vars=env_vars)
except Exception:
pass
def start_edge(port=None, use_ssl=True, asynchronous=False):
if not port:
port = config.EDGE_PORT
if config.EDGE_PORT_HTTP:
do_start_edge(config.EDGE_PORT_HTTP, use_ssl=False, asynchronous=True)
if port > 1024 or is_root():
return do_start_edge(port, use_ssl, asynchronous=asynchronous)
# process requires priviledged port but we're not root -> try running as sudo
class Terminator(object):
def stop(self, quiet=True):
try:
url = 'http%s://localhost:%s' % ('s' if use_ssl else '', port)
requests.verify_ssl = False
requests.post(url, headers={HEADER_KILL_SIGNAL: 'kill'})
except Exception:
pass
# make sure we can run sudo commands
ensure_can_use_sudo()
# register a signal handler to terminate the sudo process later on
TMP_THREADS.append(Terminator())
# start the process as sudo
sudo_cmd = 'sudo '
python_cmd = sys.executable
cmd = '%sPYTHONPATH=.:%s %s %s %s' % (sudo_cmd, LOCALSTACK_ROOT_FOLDER,
python_cmd, __file__, port)
process = run(cmd, asynchronous=asynchronous)
return process
def do_start_thread(self) -> FuncThread:
# FIXME: if this fails the cluster could be left in a wonky state
# FIXME: this is not a good place to run install, and it only works because we're
# assuming that there will only ever be one running Elasticsearch cluster
install.install_elasticsearch(self.version)
self._init_directories()
cmd = self._create_run_command(
additional_settings=self.command_settings)
cmd = " ".join(cmd)
user = constants.OS_USER_ELASTICSEARCH
if is_root() and user:
# run the elasticsearch process as a non-root user (when running in docker)
cmd = f"su {user} -c '{cmd}'"
env_vars = self._create_env_vars()
LOG.info("starting elasticsearch: %s with env %s", cmd, env_vars)
t = ShellCommandThread(
cmd,
env_vars=env_vars,
strip_color=True,
log_listener=self._log_listener,
)
t.start()
return t
def start_edge(port=None, use_ssl=True, asynchronous=False):
if not port:
port = config.EDGE_PORT
if config.EDGE_PORT_HTTP and config.EDGE_PORT_HTTP != port:
do_start_edge(
config.EDGE_BIND_HOST,
config.EDGE_PORT_HTTP,
use_ssl=False,
asynchronous=True,
)
if port > 1024 or is_root():
return do_start_edge(config.EDGE_BIND_HOST,
port,
use_ssl,
asynchronous=asynchronous)
# process requires privileged port but we're not root -> try running as sudo
class Terminator(object):
def stop(self, quiet=True):
try:
url = "http%s://%s:%s" % ("s" if use_ssl else "", LOCALHOST,
port)
requests.verify_ssl = False
requests.post(url, headers={HEADER_KILL_SIGNAL: "kill"})
except Exception:
pass
# register a signal handler to terminate the sudo process later on
TMP_THREADS.append(Terminator())
return run_process_as_sudo("edge", port, asynchronous=asynchronous)
def _run_elasticsearch(self, *args):
# *args is necessary for start_thread to work
with self._lifecycle_lock:
if self._elasticsearch_thread:
return
# FIXME: if this fails the cluster could be left in a wonky state
# FIXME: this is not a good place to run install, and it only works because we're
# assuming that there will only ever be one running Elasticsearch cluster
install.install_elasticsearch(self.version)
self._init_directories()
cmd = self._create_run_command(additional_settings=self.command_settings)
cmd = " ".join(cmd)
user = constants.OS_USER_ELASTICSEARCH
if is_root() and user:
# run the elasticsearch process as a non-root user (when running in docker)
cmd = f"su {user} -c '{cmd}'"
env_vars = self._create_env_vars()
LOG.info("starting elasticsearch: %s with env %s", cmd, env_vars)
# use asynchronous=True to get a ShellCommandThread
self._elasticsearch_thread = do_run(cmd, asynchronous=True, env_vars=env_vars)
self._starting.set()
# block until the thread running the command is done
try:
self._elasticsearch_thread.join()
finally:
LOG.info("elasticsearch process ended")
self._stopped.set()
def start_elasticsearch(port=None, delete_data=True, asynchronous=False, update_listener=None):
port = port or config.PORT_ELASTICSEARCH
# delete Elasticsearch data that may be cached locally from a previous test run
delete_all_elasticsearch_data()
install.install_elasticsearch()
backend_port = DEFAULT_PORT_ELASTICSEARCH_BACKEND
es_data_dir = '%s/infra/elasticsearch/data' % (ROOT_PATH)
es_tmp_dir = '%s/infra/elasticsearch/tmp' % (ROOT_PATH)
if config.DATA_DIR:
es_data_dir = '%s/elasticsearch' % config.DATA_DIR
# Elasticsearch 5.x cannot be bound to 0.0.0.0 in some Docker environments,
# hence we use the default bind address 127.0.0.0 and put a proxy in front of it
cmd = (('%s/infra/elasticsearch/bin/elasticsearch ' +
'-E http.port=%s -E http.publish_port=%s -E http.compression=false -E path.data=%s') %
(ROOT_PATH, backend_port, backend_port, es_data_dir))
env_vars = {
'ES_JAVA_OPTS': os.environ.get('ES_JAVA_OPTS', '-Xms200m -Xmx600m'),
'ES_TMPDIR': es_tmp_dir
}
print('Starting local Elasticsearch (%s port %s)...' % (get_service_protocol(), port))
if delete_data:
run('rm -rf %s' % es_data_dir)
# fix permissions
chmod_r('%s/infra/elasticsearch' % ROOT_PATH, 0o777)
mkdir(es_data_dir)
chmod_r(es_data_dir, 0o777)
# start proxy and ES process
start_proxy_for_service('elasticsearch', port, backend_port,
update_listener, quiet=True, params={'protocol_version': 'HTTP/1.0'})
if is_root():
cmd = "su -c '%s' localstack" % cmd
thread = do_run(cmd, asynchronous, env_vars=env_vars)
return thread
def ensure_can_use_sudo():
if not is_root() and not can_use_sudo():
if not sys.stdin.isatty():
raise IOError("cannot get sudo password from non-tty input")
print(
"Please enter your sudo password (required to configure local network):"
)
run("sudo -v", stdin=True)
def start_elasticsearch(port=None,
version=None,
delete_data=True,
asynchronous=False,
update_listener=None):
if STATE.get('_thread_'):
return STATE['_thread_']
port = port or config.PORT_ELASTICSEARCH
# delete Elasticsearch data that may be cached locally from a previous test run
delete_all_elasticsearch_data(version)
install.install_elasticsearch(version)
backend_port = get_free_tcp_port()
base_dir = install.get_elasticsearch_install_dir(version)
es_data_dir = os.path.join(base_dir, 'data')
es_tmp_dir = os.path.join(base_dir, 'tmp')
es_mods_dir = os.path.join(base_dir, 'modules')
if config.DATA_DIR:
delete_data = False
es_data_dir = '%s/elasticsearch' % config.DATA_DIR
# Elasticsearch 5.x cannot be bound to 0.0.0.0 in some Docker environments,
# hence we use the default bind address 127.0.0.0 and put a proxy in front of it
backup_dir = os.path.join(config.TMP_FOLDER, 'es_backup')
cmd = (
('%s/bin/elasticsearch ' +
'-E http.port=%s -E http.publish_port=%s -E http.compression=false ' +
'-E path.data=%s -E path.repo=%s') %
(base_dir, backend_port, backend_port, es_data_dir, backup_dir))
if os.path.exists(os.path.join(es_mods_dir, 'x-pack-ml')):
cmd += ' -E xpack.ml.enabled=false'
env_vars = {
'ES_JAVA_OPTS': os.environ.get('ES_JAVA_OPTS', '-Xms200m -Xmx600m'),
'ES_TMPDIR': es_tmp_dir
}
LOG.debug('Starting local Elasticsearch (%s port %s)' %
(get_service_protocol(), port))
if delete_data:
rm_rf(es_data_dir)
# fix permissions
chmod_r(base_dir, 0o777)
mkdir(es_data_dir)
chmod_r(es_data_dir, 0o777)
mkdir(es_tmp_dir)
chmod_r(es_tmp_dir, 0o777)
# start proxy and ES process
proxy = start_proxy_for_service('elasticsearch',
port,
backend_port,
update_listener,
quiet=True,
params={'protocol_version': 'HTTP/1.0'})
STATE['_proxy_'] = proxy
if is_root():
cmd = "su localstack -c '%s'" % cmd
thread = do_run(cmd, asynchronous, env_vars=env_vars)
STATE['_thread_'] = thread
return thread
def start_edge(port=None, use_ssl=True, asynchronous=False):
if not port:
port = config.EDGE_PORT
if config.EDGE_PORT_HTTP:
do_start_edge(
config.EDGE_BIND_HOST,
config.EDGE_PORT_HTTP,
use_ssl=False,
asynchronous=True,
)
if port > 1024 or is_root():
return do_start_edge(config.EDGE_BIND_HOST,
port,
use_ssl,
asynchronous=asynchronous)
# process requires priviledged port but we're not root -> try running as sudo
class Terminator(object):
def stop(self, quiet=True):
try:
url = "http%s://%s:%s" % ("s" if use_ssl else "", LOCALHOST,
port)
requests.verify_ssl = False
requests.post(url, headers={HEADER_KILL_SIGNAL: "kill"})
except Exception:
pass
# make sure we can run sudo commands
try:
ensure_can_use_sudo()
except Exception as e:
LOG.error("cannot start edge proxy on privileged port %s: %s", port,
str(e))
return
# register a signal handler to terminate the sudo process later on
TMP_THREADS.append(Terminator())
# start the process as sudo
sudo_cmd = "sudo "
python_cmd = sys.executable
cmd = "%sPYTHONPATH=.:%s %s %s %s" % (
sudo_cmd,
LOCALSTACK_ROOT_FOLDER,
python_cmd,
__file__,
port,
)
process = run(cmd, asynchronous=asynchronous)
return process
def start_dns_server(asynchronous=False):
try:
# start local DNS server, if present
from localstack_ext import config as config_ext
from localstack_ext.services import dns_server
if config_ext.DNS_ADDRESS in config.FALSE_STRINGS:
return
if is_port_open(PORT_DNS):
return
if is_root():
result = dns_server.start_servers()
if not asynchronous:
sleep_forever()
return result
# note: running in a separate process breaks integration with Route53 (to be fixed for local dev mode!)
return run_process_as_sudo("dns", PORT_DNS, asynchronous=asynchronous)
except Exception:
pass
def do_start_thread(self) -> FuncThread:
self._ensure_installed()
self._init_directories()
cmd = self._create_run_command(additional_settings=self.command_settings)
cmd = " ".join(cmd)
if is_root() and self.os_user:
# run the opensearch process as a non-root user (when running in docker)
cmd = f"su {self.os_user} -c '{cmd}'"
env_vars = self._create_env_vars()
LOG.info("starting %s: %s with env %s", self.bin_name, cmd, env_vars)
t = ShellCommandThread(
cmd,
env_vars=env_vars,
strip_color=True,
log_listener=self._log_listener,
)
t.start()
return t
def ensure_can_use_sudo():
if not is_root() and not can_use_sudo():
print(
'Please enter your sudo password (required to configure local network):'
)
run('sudo echo', stdin=True)
print('Starting local Elasticsearch (%s port %s)...' %
(get_service_protocol(), port))
if delete_data:
run('rm -rf %s' % es_data_dir)
# fix permissions
chmod_r('%s/infra/elasticsearch' % ROOT_PATH, 0o777)
mkdir(es_data_dir)
chmod_r(es_data_dir, 0o777)
# start proxy and ES process
start_proxy_for_service('elasticsearch',
port,
backend_port,
update_listener,
quiet=True,
params={'protocol_version': 'HTTP/1.0'})
if is_root():
cmd = "su -c '%s' localstack" % cmd
thread = do_run(cmd, async)
return thread
def check_elasticsearch(expect_shutdown=False, print_error=False):
out = None
try:
# check Elasticsearch
es = aws_stack.connect_elasticsearch()
out = es.cat.aliases()
except Exception as e:
if print_error:
LOGGER.error(
'Elasticsearch health check failed (retrying...): %s %s' %
