def get_venv_dependencies(vendor: str, os_version: str) -> List[str]:
if "debian" in os_families():
return VENV_DEPENDENCIES
elif "rhel" in os_families():
return REDHAT_VENV_DEPENDENCIES
elif "fedora" in os_families():
return FEDORA_VENV_DEPENDENCIES
else:
raise AssertionError("Invalid vendor")
def get_venv_dependencies(vendor: str, os_version: str) -> List[str]:
if vendor == 'ubuntu' and os_version == '20.04':
return VENV_DEPENDENCIES + [PYTHON_DEV_DEPENDENCY.format("2")]
elif "debian" in os_families():
return VENV_DEPENDENCIES + [PYTHON_DEV_DEPENDENCY.format("")]
elif "rhel" in os_families():
return REDHAT_VENV_DEPENDENCIES
elif "fedora" in os_families():
return FEDORA_VENV_DEPENDENCIES
else:
raise AssertionError("Invalid vendor")
def install_system_deps() -> None:
# By doing list -> set -> list conversion, we remove duplicates.
deps_to_install = sorted(set(SYSTEM_DEPENDENCIES))
if "fedora" in os_families():
install_yum_deps(deps_to_install)
elif "debian" in os_families():
install_apt_deps(deps_to_install)
else:
raise AssertionError("Invalid vendor")
# For some platforms, there aren't published PGroonga
# packages available, so we build them from source.
if BUILD_PGROONGA_FROM_SOURCE:
run_as_root(["./scripts/lib/build-pgroonga"])
def install_yum_deps(deps_to_install):
# type: (List[str]) -> None
print(WARNING + "RedHat support is still experimental.")
run_as_root(["./scripts/lib/setup-yum-repo"])
# Hack specific to unregistered RHEL system. The moreutils
# package requires a perl module package, which isn't available in
# the unregistered RHEL repositories.
#
# Error: Package: moreutils-0.49-2.el7.x86_64 (epel)
# Requires: perl(IPC::Run)
yum_extra_flags = [] # type: List[str]
if vendor == "rhel":
exitcode, subs_status = subprocess.getstatusoutput("sudo subscription-manager status")
if exitcode == 1:
# TODO this might overkill since `subscription-manager` is already
# called in setup-yum-repo
if 'Status' in subs_status:
# The output is well-formed
yum_extra_flags = ["--skip-broken"]
else:
print("Unrecognized output. `subscription-manager` might not be available")
run_as_root(["yum", "install", "-y"] + yum_extra_flags + deps_to_install)
if "rhel" in os_families():
# This is how a pip3 is installed to /usr/bin in CentOS/RHEL
# for python35 and later.
run_as_root(["python36", "-m", "ensurepip"])
# `python36` is not aliased to `python3` by default
run_as_root(["ln", "-nsf", "/usr/bin/python36", "/usr/bin/python3"])
postgres_dir = 'pgsql-%s' % (POSTGRES_VERSION,)
for cmd in ['pg_config', 'pg_isready', 'psql']:
# Our tooling expects these postgres scripts to be at
# well-known paths. There's an argument for eventually
# making our tooling auto-detect, but this is simpler.
run_as_root(["ln", "-nsf", "/usr/%s/bin/%s" % (postgres_dir, cmd),
"/usr/bin/%s" % (cmd,)])
# From here, we do the first-time setup/initialization for the postgres database.
pg_datadir = "/var/lib/pgsql/%s/data" % (POSTGRES_VERSION,)
pg_hba_conf = os.path.join(pg_datadir, "pg_hba.conf")
# We can't just check if the file exists with os.path, since the
# current user likely doesn't have permission to read the
# pg_datadir directory.
if subprocess.call(["sudo", "test", "-e", pg_hba_conf]) == 0:
# Skip setup if it has been applied previously
return
run_as_root(["/usr/%s/bin/postgresql-%s-setup" % (postgres_dir, POSTGRES_VERSION), "initdb"],
sudo_args = ['-H'])
# Use vendored pg_hba.conf, which enables password authentication.
run_as_root(["cp", "-a", "puppet/zulip/files/postgresql/centos_pg_hba.conf", pg_hba_conf])
# Later steps will ensure postgres is started
# Link in tsearch data files
overwrite_symlink("/usr/share/myspell/en_US.dic", "/usr/pgsql-%s/share/tsearch_data/en_us.dict"
% (POSTGRES_VERSION,))
overwrite_symlink("/usr/share/myspell/en_US.aff", "/usr/pgsql-%s/share/tsearch_data/en_us.affix"
% (POSTGRES_VERSION,))
def main(options):
# type: (argparse.Namespace) -> NoReturn
# yarn and management commands expect to be run from the root of the
# project.
os.chdir(ZULIP_PATH)
# hash the apt dependencies
sha_sum = hashlib.sha1()
for apt_depedency in SYSTEM_DEPENDENCIES:
sha_sum.update(apt_depedency.encode('utf8'))
if "debian" in os_families():
sha_sum.update(open('scripts/lib/setup-apt-repo', 'rb').read())
else:
# hash the content of setup-yum-repo*
sha_sum.update(open('scripts/lib/setup-yum-repo', 'rb').read())
# hash the content of build-pgroonga if pgroonga is built from source
if BUILD_PGROONGA_FROM_SOURCE:
sha_sum.update(open('scripts/lib/build-pgroonga', 'rb').read())
new_apt_dependencies_hash = sha_sum.hexdigest()
last_apt_dependencies_hash = None
apt_hash_file_path = os.path.join(UUID_VAR_PATH, "apt_dependencies_hash")
with open(apt_hash_file_path, 'a+') as hash_file:
hash_file.seek(0)
last_apt_dependencies_hash = hash_file.read()
if (new_apt_dependencies_hash != last_apt_dependencies_hash):
try:
install_system_deps()
except subprocess.CalledProcessError:
# Might be a failure due to network connection issues. Retrying...
print(WARNING +
"Installing system dependencies failed; retrying..." + ENDC)
install_system_deps()
with open(apt_hash_file_path, 'w') as hash_file:
hash_file.write(new_apt_dependencies_hash)
else:
print("No changes to apt dependencies, so skipping apt operations.")
# Here we install node.
proxy_env = [
"env",
"http_proxy=" + os.environ.get("http_proxy", ""),
"https_proxy=" + os.environ.get("https_proxy", ""),
"no_proxy=" + os.environ.get("no_proxy", ""),
]
run_as_root(proxy_env + ["scripts/lib/install-node"], sudo_args=['-H'])
if not os.access(NODE_MODULES_CACHE_PATH, os.W_OK):
run_as_root(["mkdir", "-p", NODE_MODULES_CACHE_PATH])
run_as_root([
"chown",
"%s:%s" % (os.getuid(), os.getgid()), NODE_MODULES_CACHE_PATH
])
# This is a wrapper around `yarn`, which we run last since
# it can often fail due to network issues beyond our control.
try:
setup_node_modules(prefer_offline=True)
except subprocess.CalledProcessError:
print(WARNING + "`yarn install` failed; retrying..." + ENDC)
try:
setup_node_modules()
except subprocess.CalledProcessError:
print(
FAIL +
"`yarn install` is failing; check your network connection (and proxy settings)."
+ ENDC)
sys.exit(1)
# Install shellcheck.
run_as_root(["tools/setup/install-shellcheck"])
setup_venvs.main()
run_as_root(["cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])
if is_circleci or (is_travis and not options.is_production_travis):
run_as_root(["service", "rabbitmq-server", "restart"])
run_as_root(["service", "redis-server", "restart"])
run_as_root(["service", "memcached", "restart"])
run_as_root(["service", "postgresql", "restart"])
elif "fedora" in os_families():
# These platforms don't enable and start services on
# installing their package, so we do that here.
for service in [
"postgresql-%s" % (POSTGRES_VERSION, ), "rabbitmq-server",
"memcached", "redis"
]:
run_as_root(["systemctl", "enable", service], sudo_args=['-H'])
run_as_root(["systemctl", "start", service], sudo_args=['-H'])
# If we imported modules after activating the virtualenv in this
# Python process, they could end up mismatching with modules we’ve
# already imported from outside the virtualenv. That seems like a
# bad idea, and empirically it can cause Python to segfault on
# certain cffi-related imports. Instead, start a new Python
# process inside the virtualenv.
activate_this = "/srv/zulip-py3-venv/bin/activate_this.py"
provision_inner = os.path.join(ZULIP_PATH, "tools", "lib",
"provision_inner.py")
exec(open(activate_this).read(), dict(__file__=activate_this))
os.execvp(provision_inner, [
provision_inner,
*(["--force"] if options.is_force else []),
*(["--production-travis"] if options.is_production_travis else []),
])
if vendor == 'debian' and os_version in [] or vendor == 'ubuntu' and os_version in []:
# For platforms without a pgroonga release, we need to build it
# from source.
BUILD_PGROONGA_FROM_SOURCE = True
SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [
pkg.format(POSTGRES_VERSION) for pkg in [
"postgresql-{0}",
# Dependency for building pgroonga from source
"postgresql-server-dev-{0}",
"libgroonga-dev",
"libmsgpack-dev",
"clang-9",
"llvm-9-dev"
]
] + VENV_DEPENDENCIES
elif "debian" in os_families():
SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [
pkg.format(POSTGRES_VERSION) for pkg in [
"postgresql-{0}",
"postgresql-{0}-pgroonga",
]
] + VENV_DEPENDENCIES
elif "rhel" in os_families():
SYSTEM_DEPENDENCIES = COMMON_YUM_DEPENDENCIES + [
pkg.format(POSTGRES_VERSION) for pkg in [
"postgresql{0}-server",
"postgresql{0}",
"postgresql{0}-devel",
"postgresql{0}-pgroonga",
]
] + VENV_DEPENDENCIES
def main(options: argparse.Namespace) -> "NoReturn":
# yarn and management commands expect to be run from the root of the
# project.
os.chdir(ZULIP_PATH)
# hash the apt dependencies
sha_sum = hashlib.sha1()
for apt_depedency in SYSTEM_DEPENDENCIES:
sha_sum.update(apt_depedency.encode())
if "debian" in os_families():
with open("scripts/lib/setup-apt-repo", "rb") as fb:
sha_sum.update(fb.read())
else:
# hash the content of setup-yum-repo*
with open("scripts/lib/setup-yum-repo", "rb") as fb:
sha_sum.update(fb.read())
# hash the content of build-pgroonga if PGroonga is built from source
if BUILD_PGROONGA_FROM_SOURCE:
with open("scripts/lib/build-pgroonga", "rb") as fb:
sha_sum.update(fb.read())
new_apt_dependencies_hash = sha_sum.hexdigest()
last_apt_dependencies_hash = None
apt_hash_file_path = os.path.join(UUID_VAR_PATH, "apt_dependencies_hash")
with open(apt_hash_file_path, "a+") as hash_file:
hash_file.seek(0)
last_apt_dependencies_hash = hash_file.read()
if new_apt_dependencies_hash != last_apt_dependencies_hash:
try:
install_system_deps()
except subprocess.CalledProcessError:
try:
# Might be a failure due to network connection issues. Retrying...
print(WARNING +
"Installing system dependencies failed; retrying..." +
ENDC)
install_system_deps()
except BaseException as e:
# Suppress exception chaining
raise e from None
with open(apt_hash_file_path, "w") as hash_file:
hash_file.write(new_apt_dependencies_hash)
else:
print("No changes to apt dependencies, so skipping apt operations.")
# Binary-patch ARM64 assembly bug in OpenSSL 1.1.1b through 1.1.1h.
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=989604
# https://bugs.launchpad.net/ubuntu/+source/openssl/+bug/1951279
try:
with open("/usr/lib/aarch64-linux-gnu/libcrypto.so.1.1", "rb") as fb:
if b"\xbf#\x03\xd5\xfd\x07E\xf8" in fb.read():
run_as_root(
[
"sed",
"-i",
r"s/\(\xbf#\x03\xd5\)\(\xfd\x07E\xf8\)/\2\1/",
"/usr/lib/aarch64-linux-gnu/libcrypto.so.1.1",
],
env={
**os.environ, "LC_ALL": "C"
},
)
except FileNotFoundError:
pass
# Here we install node.
proxy_env = [
"env",
"http_proxy=" + os.environ.get("http_proxy", ""),
"https_proxy=" + os.environ.get("https_proxy", ""),
"no_proxy=" + os.environ.get("no_proxy", ""),
]
run_as_root([*proxy_env, "scripts/lib/install-node"], sudo_args=["-H"])
run_as_root([*proxy_env, "scripts/lib/install-yarn"])
if not os.access(NODE_MODULES_CACHE_PATH, os.W_OK):
run_as_root(["mkdir", "-p", NODE_MODULES_CACHE_PATH])
run_as_root(
["chown", f"{os.getuid()}:{os.getgid()}", NODE_MODULES_CACHE_PATH])
# This is a wrapper around `yarn`, which we run last since
# it can often fail due to network issues beyond our control.
try:
setup_node_modules(prefer_offline=True)
except subprocess.CalledProcessError:
print(WARNING + "`yarn install` failed; retrying..." + ENDC)
try:
setup_node_modules()
except subprocess.CalledProcessError:
print(
FAIL +
"`yarn install` is failing; check your network connection (and proxy settings)."
+ ENDC)
sys.exit(1)
# Install shellcheck.
run_as_root(["tools/setup/install-shellcheck"])
# Install shfmt.
run_as_root(["tools/setup/install-shfmt"])
setup_venvs.main()
run_as_root(["cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])
if CONTINUOUS_INTEGRATION and not options.is_build_release_tarball_only:
run_as_root(["service", "redis-server", "start"])
run_as_root(["service", "memcached", "start"])
run_as_root(["service", "rabbitmq-server", "start"])
run_as_root(["service", "postgresql", "start"])
elif "fedora" in os_families():
# These platforms don't enable and start services on
# installing their package, so we do that here.
for service in [
f"postgresql-{POSTGRESQL_VERSION}",
"rabbitmq-server",
"memcached",
"redis",
]:
run_as_root(["systemctl", "enable", service], sudo_args=["-H"])
run_as_root(["systemctl", "start", service], sudo_args=["-H"])
# If we imported modules after activating the virtualenv in this
# Python process, they could end up mismatching with modules we’ve
# already imported from outside the virtualenv. That seems like a
# bad idea, and empirically it can cause Python to segfault on
# certain cffi-related imports. Instead, start a new Python
# process inside the virtualenv.
activate_this = "/srv/zulip-py3-venv/bin/activate_this.py"
provision_inner = os.path.join(ZULIP_PATH, "tools", "lib",
"provision_inner.py")
with open(activate_this) as f:
exec(f.read(), dict(__file__=activate_this))
os.execvp(
provision_inner,
[
provision_inner,
*(["--force"] if options.is_force else []),
*(["--build-release-tarball-only"]
if options.is_build_release_tarball_only else []),
*(["--skip-dev-db-build"] if options.skip_dev_db_build else []),
],
)
