def hash_sharding_config() -> str:
config_file = get_config_file()
if not config_file.has_section("tornado_sharding"):
return hashlib.sha256(b'').hexdigest()
contents = subprocess.check_output([
"crudini",
"--get",
"--format=lines",
"/etc/zulip/zulip.conf",
"tornado_sharding",
])
return hashlib.sha256(contents).hexdigest()
def write_updated_configs() -> None:
config_file = get_config_file()
ports = get_tornado_ports(config_file)
expected_ports = list(range(9800, max(ports) + 1))
assert (
sorted(ports) == expected_ports
), f"ports ({sorted(ports)}) must be contiguous, starting with 9800"
with open("/etc/zulip/nginx_sharding.conf.tmp",
"w") as nginx_sharding_conf_f, open(
"/etc/zulip/sharding.json.tmp", "w") as sharding_json_f:
if len(ports) == 1:
nginx_sharding_conf_f.write(
"set $tornado_server http://tornado;\n")
sharding_json_f.write("{}\n")
return
nginx_sharding_conf_f.write(
"set $tornado_server http://tornado9800;\n")
shard_map: Dict[str, int] = {}
external_host = subprocess.check_output(
[
os.path.join(BASE_DIR, "scripts/get-django-setting"),
"EXTERNAL_HOST"
],
text=True,
).strip()
for port in config_file["tornado_sharding"]:
shards = config_file["tornado_sharding"][port].strip()
if shards:
for shard in shards.split(" "):
if "." in shard:
host = shard
else:
host = f"{shard}.{external_host}"
assert host not in shard_map, f"host {host} duplicated"
shard_map[host] = int(port)
write_realm_nginx_config_line(nginx_sharding_conf_f, host,
port)
nginx_sharding_conf_f.write("\n")
sharding_json_f.write(json.dumps(shard_map) + "\n")
def write_realm_nginx_config_line(f: Any, host: str, port: str) -> None:
f.write("""if ($host = '%s') {
set $tornado_server http://tornado%s;
}\n""" % (host, port))
# Basic system to do Tornado sharding. Writes two output .tmp files that need
# to be renamed to the following files to finalize the changes:
# * /etc/zulip/nginx_sharding.conf; nginx needs to be reloaded after changing.
# * /etc/zulip/sharding.json; supervisor Django process needs to be reloaded
# after changing. TODO: We can probably make this live-reload by statting the file.
#
# TODO: Restructure this to automatically generate a sharding layout.
with open('/etc/zulip/nginx_sharding.conf.tmp', 'w') as nginx_sharding_conf_f, \
open('/etc/zulip/sharding.json.tmp', 'w') as sharding_json_f:
config_file = get_config_file()
if not config_file.has_section("tornado_sharding"):
nginx_sharding_conf_f.write("set $tornado_server http://tornado;\n")
sharding_json_f.write('{}\n')
sys.exit(0)
nginx_sharding_conf_f.write("set $tornado_server http://tornado9800;\n")
shard_map: Dict[str, int] = {}
external_host = subprocess.check_output([os.path.join(BASE_DIR, 'scripts/get-django-setting'),
'EXTERNAL_HOST'],
universal_newlines=True).strip()
for port in config_file["tornado_sharding"]:
shards = config_file["tornado_sharding"][port].strip().split(' ')
for shard in shards:
if '.' in shard:
def generate_secrets(development: bool = False) -> None:
if development:
OUTPUT_SETTINGS_FILENAME = "zproject/dev-secrets.conf"
else:
OUTPUT_SETTINGS_FILENAME = "/etc/zulip/zulip-secrets.conf"
current_conf = get_old_conf(OUTPUT_SETTINGS_FILENAME)
lines: List[str] = []
if len(current_conf) == 0:
lines = ["[secrets]\n"]
def need_secret(name: str) -> bool:
return name not in current_conf
def add_secret(name: str, value: str) -> None:
lines.append(f"{name} = {value}\n")
current_conf[name] = value
for name in AUTOGENERATED_SETTINGS:
if need_secret(name):
add_secret(name, random_token())
# These secrets are exclusive to a Zulip development environment.
# We use PostgreSQL peer authentication by default in production,
# and initial_password_salt is used to generate passwords for the
# test/development database users. See `manage.py
# print_initial_password`.
if development and need_secret("initial_password_salt"):
add_secret("initial_password_salt", random_token())
if development and need_secret("local_database_password"):
add_secret("local_database_password", random_token())
# We only need a secret if the database username does not match
# the OS username, as identd auth works in that case.
if get_config(get_config_file(), "postgresql", "database_user",
"zulip") != "zulip" and need_secret("postgres_password"):
add_secret("postgres_password", random_token())
# The core Django SECRET_KEY setting, used by Django internally to
# secure sessions. If this gets changed, all users will be logged out.
if need_secret("secret_key"):
secret_key = generate_django_secretkey()
add_secret("secret_key", secret_key)
# To prevent Django ImproperlyConfigured error
from zproject import settings
settings.SECRET_KEY = secret_key
# Secret key for the Camo HTTPS proxy.
if need_secret("camo_key"):
add_secret("camo_key", random_string(64))
if not development:
# The memcached_password and redis_password secrets are only
# required/relevant in production.
# Password for authentication to memcached.
if need_secret("memcached_password"):
# We defer importing settings unless we need it, because
# importing settings is expensive (mostly because of
# django-auth-ldap) and we want the noop case to be fast.
from zproject import settings
if settings.MEMCACHED_LOCATION == "127.0.0.1:11211":
add_secret("memcached_password", random_token())
# Password for authentication to Redis.
if need_secret("redis_password"):
# We defer importing settings unless we need it, because
# importing settings is expensive (mostly because of
# django-auth-ldap) and we want the noop case to be fast.
from zproject import settings
if settings.REDIS_HOST == "127.0.0.1":
# To prevent Puppet from restarting Redis, which would lose
# data because we configured Redis to disable persistence, set
# the Redis password on the running server and edit the config
# file directly.
import redis
from zerver.lib.redis_utils import get_redis_client
redis_password = random_token()
for filename in [
"/etc/redis/zuli-redis.conf",
"/etc/redis/zulip-redis.conf"
]:
if os.path.exists(filename):
with open(filename, "a") as f:
f.write(
"# Set a Redis password based on zulip-secrets.conf\n"
f"requirepass '{redis_password}'\n", )
break
try:
get_redis_client().config_set("requirepass",
redis_password)
except redis.exceptions.ConnectionError:
pass
add_secret("redis_password", redis_password)
# Random id and secret used to identify this installation when
# accessing the Zulip mobile push notifications service.
# * zulip_org_key is generated using os.urandom().
# * zulip_org_id only needs to be unique, so we use a UUID.
if need_secret("zulip_org_key"):
add_secret("zulip_org_key", random_string(64))
if need_secret("zulip_org_id"):
add_secret("zulip_org_id", str(uuid.uuid4()))
if len(lines) == 0:
print("generate_secrets: No new secrets to generate.")
return
with open(OUTPUT_SETTINGS_FILENAME, "a") as f:
# Write a newline at the start, in case there was no newline at
# the end of the file due to human editing.
f.write("\n" + "".join(lines))
print(f"Generated new secrets in {OUTPUT_SETTINGS_FILENAME}.")