def do_setup_virtualenv(venv_path, requirements_file, virtualenv_args):
# type: (str, str, List[str]) -> None
# Setup Python virtualenv
new_packages = set(get_package_names(requirements_file))
run(["sudo", "rm", "-rf", venv_path])
if not try_to_copy_venv(venv_path, new_packages):
# Create new virtualenv.
run(["sudo", "mkdir", "-p", venv_path])
run(["sudo", "virtualenv"] + virtualenv_args + [venv_path])
run(["sudo", "chown", "-R",
"{}:{}".format(os.getuid(), os.getgid()), venv_path])
create_log_entry(get_logfile_name(venv_path), "", set(), new_packages)
create_requirements_index_file(venv_path, requirements_file)
# Switch current Python context to the virtualenv.
activate_this = os.path.join(venv_path, "bin", "activate_this.py")
exec(open(activate_this).read(), {}, dict(__file__=activate_this))
try:
install_venv_deps(requirements_file)
except subprocess.CalledProcessError:
# Might be a failure due to network connection issues. Retrying...
print(WARNING + "`pip install` failed; retrying..." + ENDC)
install_venv_deps(requirements_file)
run(["sudo", "chmod", "-R", "a+rX", venv_path])
def setup_node_modules(npm_args=None, stdout=None, stderr=None, copy_modules=False):
# type: (Optional[List[str]], Optional[IO], Optional[IO], Optional[bool]) -> None
sha1sum = hashlib.sha1()
sha1sum.update(subprocess_text_output(["cat", "package.json"]).encode("utf8"))
sha1sum.update(subprocess_text_output(["npm", "--version"]).encode("utf8"))
sha1sum.update(subprocess_text_output(["node", "--version"]).encode("utf8"))
if npm_args is not None:
sha1sum.update("".join(sorted(npm_args)).encode("utf8"))
npm_cache = os.path.join(NPM_CACHE_PATH, sha1sum.hexdigest())
cached_node_modules = os.path.join(npm_cache, "node_modules")
success_stamp = os.path.join(cached_node_modules, ".success-stamp")
# Check if a cached version already exists
if not os.path.exists(success_stamp):
do_npm_install(
npm_cache,
npm_args or [],
stdout=stdout,
stderr=stderr,
success_stamp=success_stamp,
copy_modules=copy_modules,
)
print("Using cached node modules from %s" % (cached_node_modules,))
cmds = [["rm", "-rf", "node_modules"], ["ln", "-nsf", cached_node_modules, "node_modules"]]
for cmd in cmds:
run(cmd, stdout=stdout, stderr=stderr)
def install_apt_deps():
# type: () -> None
# setup-apt-repo does an `apt-get update`
run(["sudo", "./scripts/lib/setup-apt-repo"])
# By doing list -> set -> list conversion we remove duplicates.
deps_to_install = list(set(APT_DEPENDENCIES[codename]))
run(["sudo", "apt-get", "-y", "install", "--no-install-recommends"] + deps_to_install)
def setup_node_modules(production=DEFAULT_PRODUCTION, stdout=None, stderr=None, copy_modules=False,
prefer_offline=False):
# type: (bool, Optional[IO], Optional[IO], bool, bool) -> None
yarn_args = get_yarn_args(production=production)
if prefer_offline:
yarn_args.append("--prefer-offline")
sha1sum = generate_sha1sum_node_modules(production=production)
target_path = os.path.join(NODE_MODULES_CACHE_PATH, sha1sum)
cached_node_modules = os.path.join(target_path, 'node_modules')
success_stamp = os.path.join(target_path, '.success-stamp')
# Check if a cached version already exists
if not os.path.exists(success_stamp):
do_yarn_install(target_path,
yarn_args,
success_stamp,
stdout=stdout,
stderr=stderr,
copy_modules=copy_modules)
print("Using cached node modules from %s" % (cached_node_modules,))
cmds = [
['rm', '-rf', 'node_modules'],
["ln", "-nsf", cached_node_modules, 'node_modules'],
]
for cmd in cmds:
run(cmd, stdout=stdout, stderr=stderr)
def do_yarn_install(target_path, yarn_args, success_stamp, stdout=None, stderr=None,
copy_modules=False):
# type: (str, List[str], str, Optional[IO[Any]], Optional[IO[Any]], bool) -> None
cmds = [
['mkdir', '-p', target_path],
['cp', 'package.json', "yarn.lock", target_path],
]
cached_node_modules = os.path.join(target_path, 'node_modules')
if copy_modules:
print("Cached version not found! Copying node modules.")
cmds.append(["cp", "-rT", "prod-static/serve/node_modules", cached_node_modules])
else:
print("Cached version not found! Installing node modules.")
# Copy the existing node_modules to speed up install
if os.path.exists("node_modules"):
cmds.append(["cp", "-R", "node_modules/", cached_node_modules])
cd_exec = os.path.join(ZULIP_PATH, "scripts/lib/cd_exec")
if os.environ.get('CUSTOM_CA_CERTIFICATES'):
cmds.append([YARN_BIN, "config", "set", "cafile", os.environ['CUSTOM_CA_CERTIFICATES']])
cmds.append([cd_exec, target_path, YARN_BIN, "install", "--non-interactive"] +
yarn_args)
cmds.append(['touch', success_stamp])
for cmd in cmds:
run(cmd, stdout=stdout, stderr=stderr)
def do_setup_virtualenv(venv_path, requirements_file, virtualenv_args):
# type: (str, str, List[str]) -> None
# Setup Python virtualenv
new_packages = set(get_package_names(requirements_file))
run_as_root(["rm", "-rf", venv_path])
if not try_to_copy_venv(venv_path, new_packages):
# Create new virtualenv.
run_as_root(["mkdir", "-p", venv_path])
run_as_root(["virtualenv"] + virtualenv_args + [venv_path])
run_as_root(["chown", "-R",
"{}:{}".format(os.getuid(), os.getgid()), venv_path])
create_log_entry(get_logfile_name(venv_path), "", set(), new_packages)
create_requirements_index_file(venv_path, requirements_file)
pip = os.path.join(venv_path, "bin", "pip")
# use custom certificate if needed
if os.environ.get('CUSTOM_CA_CERTIFICATES'):
print("Configuring pip to use custom CA certificates...")
add_cert_to_pipconf()
# CentOS-specific hack/workaround
# Install pycurl with custom flag due to this error when installing
# via pip:
# __main__.ConfigurationError: Curl is configured to use SSL, but
# we have not been able to determine which SSL backend it is using.
# Please see PycURL documentation for how to specify the SSL
# backend manually.
# See https://github.com/pycurl/pycurl/issues/526
# The fix exists on pycurl master, but not yet in any release
# We can likely remove this when pycurl > 7.43.0.2 comes out.
if os.path.exists("/etc/redhat-release"):
pycurl_env = os.environ.copy()
pycurl_env["PYCURL_SSL_LIBRARY"] = "nss"
run([pip, "install", "pycurl==7.43.0.2", "--compile", "--no-cache-dir"],
env=pycurl_env)
try:
install_venv_deps(pip, requirements_file)
except subprocess.CalledProcessError:
# Might be a failure due to network connection issues. Retrying...
print(WARNING + "`pip install` failed; retrying..." + ENDC)
install_venv_deps(pip, requirements_file)
# The typing module has been included in stdlib since 3.5.
# Installing a pypi version of it has been harmless until a bug
# "AttributeError: type object 'Callable' has no attribute
# '_abc_registry'" happens in 3.7. And so just to be safe, it is
# disabled from now on for all >= 3.5 versions.
# Remove this once 3.4 is no longer supported.
at_least_35 = (sys.version_info.major == 3) and (sys.version_info.minor >= 5)
if at_least_35 and ('python2.7' not in virtualenv_args):
run([pip, "uninstall", "-y", "typing"])
run_as_root(["chmod", "-R", "a+rX", venv_path])
def do_npm_install(target_path, npm_args, stdout=None, stderr=None, copy_modules=False, success_stamp=None):
# type: (str, List[str], Optional[IO], Optional[IO], Optional[bool], Optional[str]) -> None
cmds = [["rm", "-rf", target_path], ["mkdir", "-p", target_path], ["cp", "package.json", target_path]]
if copy_modules:
print("Cached version not found! Copying node modules.")
cmds.append(["cp", "-rT", "prod-static/serve/node_modules", os.path.join(target_path, "node_modules")])
else:
print("Cached version not found! Installing node modules.")
cmds.append(["npm", "install"] + npm_args + ["--prefix", target_path])
cmds.append(["touch", success_stamp])
for cmd in cmds:
run(cmd, stdout=stdout, stderr=stderr)
def main(is_travis=False):
# type: (bool) -> None
# Get the correct Python interpreter. If we don't do this and use
# `virtualenv -p python3` to create the venv in Travis, the venv
# starts referring to the system Python interpreter.
python_interpreter = subprocess_text_output(['which', 'python3'])
if is_travis:
setup_virtualenv(VENV_PATH, DEV_REQS_FILE, patch_activate_script=True,
virtualenv_args=['-p', python_interpreter])
else:
run(['sudo', 'rm', '-f', OLD_VENV_PATH])
setup_virtualenv(VENV_PATH, DEV_REQS_FILE, patch_activate_script=True,
virtualenv_args=['-p', python_interpreter])
def handle(self, *args: Any, **options: Any) -> None:
timestamp = timezone_now().strftime(TIMESTAMP_FORMAT)
with tempfile.TemporaryDirectory(
prefix="zulip-backup-%s-" % (timestamp,)
) as tmp:
os.mkdir(os.path.join(tmp, "zulip-backup"))
members = []
if settings.DEVELOPMENT:
os.symlink(
os.path.join(settings.DEPLOY_ROOT, "zproject"),
os.path.join(tmp, "zulip-backup", "zproject"),
)
members.append("zulip-backup/zproject/dev-secrets.conf")
else:
os.symlink("/etc/zulip", os.path.join(tmp, "zulip-backup", "settings"))
members.append("zulip-backup/settings")
db_name = settings.DATABASES["default"]["NAME"]
db_dir = os.path.join(tmp, "zulip-backup", "database")
run(["pg_dump", "--format=directory", db_name, "--file", db_dir])
members.append("zulip-backup/database")
if settings.LOCAL_UPLOADS_DIR is not None and os.path.exists(
os.path.join(settings.DEPLOY_ROOT, settings.LOCAL_UPLOADS_DIR)
):
os.symlink(
os.path.join(settings.DEPLOY_ROOT, settings.LOCAL_UPLOADS_DIR),
os.path.join(tmp, "zulip-backup", "uploads"),
)
members.append("zulip-backup/uploads")
try:
if options["output"] is None:
tarball_path = tempfile.NamedTemporaryFile(
prefix="zulip-backup-%s-" % (timestamp,),
suffix=".tar.gz",
delete=False,
).name
else:
tarball_path = options["output"]
run(["tar", "-C", tmp, "-chzf", tarball_path, "--"] + members)
print("Backup tarball written to %s" % (tarball_path,))
except BaseException:
if options["output"] is None:
os.unlink(tarball_path)
raise
def run_db_migrations(platform: str) -> None:
if platform == 'dev':
migration_status_file = 'migration_status_dev'
settings = 'zproject.settings'
elif platform == 'test':
migration_status_file = 'migration_status_test'
settings = 'zproject.test_settings'
# We shell out to `manage.py` and pass `DJANGO_SETTINGS_MODULE` on
# the command line rather than just calling the migration
# functions, because Django doesn't support changing settings like
# what the database is as runtime.
run(['env', ('DJANGO_SETTINGS_MODULE=%s' % settings), './manage.py',
'migrate', '--no-input'])
run(['env', ('DJANGO_SETTINGS_MODULE=%s' % settings), './manage.py',
'get_migration_status', '--output=%s' % (migration_status_file)])
def do_npm_install(target_path, npm_args, stdout=None, stderr=None, copy_modules=False):
# type: (str, List[str], Optional[str], Optional[str], Optional[bool]) -> None
cmds = [
["sudo", "rm", "-rf", target_path],
['sudo', 'mkdir', '-p', target_path],
["sudo", "chown", "{}:{}".format(os.getuid(), os.getgid()), target_path],
['cp', 'package.json', target_path],
]
if copy_modules:
print("Cached version not found! Copying node modules.")
cmds.append(["mv", "node_modules", target_path])
else:
print("Cached version not found! Installing node modules.")
cmds.append(['npm', 'install'] + npm_args + ['--prefix', target_path])
for cmd in cmds:
run(cmd, stdout=stdout, stderr=stderr)
def do_npm_install(target_path, npm_args, stdout=None, stderr=None, copy_modules=False,
success_stamp=None):
# type: (str, List[str], Optional[IO], Optional[IO], Optional[bool], Optional[str]) -> None
cmds = [
["rm", "-rf", target_path],
['mkdir', '-p', target_path],
['cp', 'package.json', target_path],
]
if copy_modules:
print("Cached version not found! Copying node modules.")
cmds.append(["mv", "node_modules", target_path])
else:
print("Cached version not found! Installing node modules.")
cmds.append(['npm', 'install'] + npm_args + ['--prefix', target_path])
cmds.append(['touch', success_stamp])
for cmd in cmds:
run(cmd, stdout=stdout, stderr=stderr)
def run_db_migrations(platform: str) -> None:
if platform == 'dev':
migration_status_file = 'migration_status_dev'
settings = 'zproject.settings'
db_name = 'ZULIP_DB_NAME=zulip'
elif platform == 'test':
migration_status_file = 'migration_status_test'
settings = 'zproject.test_settings'
db_name = 'ZULIP_DB_NAME=zulip_test_template'
# We shell out to `manage.py` and pass `DJANGO_SETTINGS_MODULE` on
# the command line rather than just calling the migration
# functions, because Django doesn't support changing settings like
# what the database is as runtime.
# Also we export DB_NAME which is ignored by dev platform but
# recognised by test platform and used to migrate correct db.
run(['env', ('DJANGO_SETTINGS_MODULE=%s' % settings), db_name,
'./manage.py', 'migrate', '--no-input'])
run(['env', ('DJANGO_SETTINGS_MODULE=%s' % settings), db_name,
'./manage.py', 'get_migration_status',
'--output=%s' % (migration_status_file)])
def setup_virtualenv(target_venv_path, requirements_file, virtualenv_args=None, patch_activate_script=False):
# type: (Optional[str], str, Optional[List[str]], bool) -> str
# Check if a cached version already exists
path = os.path.join(ZULIP_PATH, 'scripts', 'lib', 'hash_reqs.py')
output = subprocess.check_output([path, requirements_file], universal_newlines=True)
sha1sum = output.split()[0]
if target_venv_path is None:
cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, 'venv')
else:
cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, os.path.basename(target_venv_path))
success_stamp = os.path.join(cached_venv_path, "success-stamp")
if not os.path.exists(success_stamp):
do_setup_virtualenv(cached_venv_path, requirements_file, virtualenv_args or [])
open(success_stamp, 'w').close()
print("Using cached Python venv from %s" % (cached_venv_path,))
if target_venv_path is not None:
run(["sudo", "ln", "-nsf", cached_venv_path, target_venv_path])
if patch_activate_script:
do_patch_activate_script(target_venv_path)
return cached_venv_path
def setup_node_modules(npm_args=None, stdout=None, stderr=None, copy_modules=False):
# type: (Optional[List[str]], Optional[IO], Optional[IO], Optional[bool]) -> None
sha1sum = generate_sha1sum_node_modules(npm_args)
npm_cache = os.path.join(NPM_CACHE_PATH, sha1sum)
cached_node_modules = os.path.join(npm_cache, 'node_modules')
success_stamp = os.path.join(cached_node_modules, '.success-stamp')
# Check if a cached version already exists
if not os.path.exists(success_stamp):
do_npm_install(npm_cache,
npm_args or [],
success_stamp,
stdout=stdout,
stderr=stderr,
copy_modules=copy_modules)
print("Using cached node modules from %s" % (cached_node_modules,))
cmds = [
['rm', '-rf', 'node_modules'],
["ln", "-nsf", cached_node_modules, 'node_modules'],
]
for cmd in cmds:
run(cmd, stdout=stdout, stderr=stderr)
def install_npm():
# type: () -> None
if not TRAVIS:
if subprocess_text_output(['npm', '--version']) != NPM_VERSION:
run(["sudo", "npm", "install", "-g", "npm@{}".format(NPM_VERSION)])
return
run(['mkdir', '-p', TRAVIS_NODE_PATH])
npm_exe = os.path.join(TRAVIS_NODE_PATH, 'bin', 'npm')
travis_npm = subprocess_text_output(['which', 'npm'])
if os.path.exists(npm_exe):
run(['sudo', 'ln', '-sf', npm_exe, travis_npm])
version = subprocess_text_output(['npm', '--version'])
if os.path.exists(npm_exe) and version == NPM_VERSION:
print("Using cached npm")
return
run(["npm", "install", "-g", "--prefix", TRAVIS_NODE_PATH, "npm@{}".format(NPM_VERSION)])
run(['sudo', 'ln', '-sf', npm_exe, travis_npm])
def setup_node_modules(npm_args=None, stdout=None, stderr=None, copy_modules=False):
# type: (Optional[List[str]], Optional[str], Optional[str], Optional[bool]) -> None
sha1sum = hashlib.sha1()
sha1sum.update(subprocess_text_output(['cat', 'package.json']).encode('utf8'))
sha1sum.update(subprocess_text_output(['npm', '--version']).encode('utf8'))
sha1sum.update(subprocess_text_output(['node', '--version']).encode('utf8'))
if npm_args is not None:
sha1sum.update(''.join(sorted(npm_args)).encode('utf8'))
npm_cache = os.path.join(NPM_CACHE_PATH, sha1sum.hexdigest())
cached_node_modules = os.path.join(npm_cache, 'node_modules')
success_stamp = os.path.join(cached_node_modules, '.success-stamp')
# Check if a cached version already exists
if not os.path.exists(success_stamp):
do_npm_install(npm_cache, npm_args or [], stdout, stderr, copy_modules)
print("Using cached node modules from %s" % (cached_node_modules,))
cmds = [
['rm', '-rf', 'node_modules'],
["ln", "-nsf", cached_node_modules, 'node_modules'],
['touch', success_stamp],
]
for cmd in cmds:
run(cmd, stdout=stdout, stderr=stderr)
def setup_virtualenv(target_venv_path, requirements_file, virtualenv_args=None, patch_activate_script=False):
# type: (Optional[str], str, Optional[List[str]], bool) -> str
# Check if a cached version already exists
path = os.path.join(ZULIP_PATH, 'scripts', 'lib', 'hash_reqs.py')
output = subprocess.check_output([path, requirements_file], universal_newlines=True)
sha1sum = output.split()[0]
if target_venv_path is None:
cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, 'venv')
else:
cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, os.path.basename(target_venv_path))
success_stamp = os.path.join(cached_venv_path, "success-stamp")
if not os.path.exists(success_stamp):
do_setup_virtualenv(cached_venv_path, requirements_file, virtualenv_args or [])
run(["touch", success_stamp])
print("Using cached Python venv from %s" % (cached_venv_path,))
if target_venv_path is not None:
run(["sudo", "ln", "-nsf", cached_venv_path, target_venv_path])
if patch_activate_script:
do_patch_activate_script(target_venv_path)
activate_this = os.path.join(cached_venv_path, "bin", "activate_this.py")
exec(open(activate_this).read(), {}, dict(__file__=activate_this)) # type: ignore # https://github.com/python/mypy/issues/1577
return cached_venv_path
def install_apt_deps(deps_to_install, retry=False):
# type: (List[str], bool) -> None
if retry:
print(WARNING + "`apt-get -y install` failed while installing dependencies; retrying..." + ENDC)
# Since a common failure mode is for the caching in
# `setup-apt-repo` to optimize the fast code path to skip
# running `apt-get update` when the target apt repository
# is out of date, we run it explicitly here so that we
# recover automatically.
run(['sudo', 'apt-get', 'update'])
# setup-apt-repo does an `apt-get update`
run(["sudo", "./scripts/lib/setup-apt-repo"])
run(["sudo", "apt-get", "-y", "install", "--no-install-recommends"] + deps_to_install)
def copy_parent_log(source_log, target_log):
# type: (str, str) -> None
if os.path.exists(source_log):
run('cp {} {}'.format(source_log, target_log).split())
def main(options: argparse.Namespace) -> int:
setup_shell_profile('~/.bash_profile')
setup_shell_profile('~/.zprofile')
# This needs to happen before anything that imports zproject.settings.
run(["scripts/setup/generate_secrets.py", "--development"])
# create log directory `zulip/var/log`
os.makedirs(LOG_DIR_PATH, exist_ok=True)
# create upload directory `var/uploads`
os.makedirs(UPLOAD_DIR_PATH, exist_ok=True)
# create test upload directory `var/test_upload`
os.makedirs(TEST_UPLOAD_DIR_PATH, exist_ok=True)
# create coverage directory `var/coverage`
os.makedirs(COVERAGE_DIR_PATH, exist_ok=True)
# create linecoverage directory `var/node-coverage`
os.makedirs(NODE_TEST_COVERAGE_DIR_PATH, exist_ok=True)
# create XUnit XML test results directory`var/xunit-test-results`
os.makedirs(XUNIT_XML_TEST_RESULTS_DIR_PATH, exist_ok=True)
# The `build_emoji` script requires `emoji-datasource` package
# which we install via npm; thus this step is after installing npm
# packages.
if not os.access(EMOJI_CACHE_PATH, os.W_OK):
run_as_root(["mkdir", "-p", EMOJI_CACHE_PATH])
run_as_root(
["chown",
"%s:%s" % (os.getuid(), os.getgid()), EMOJI_CACHE_PATH])
run(["tools/setup/emoji/build_emoji"])
# copy over static files from the zulip_bots package
generate_zulip_bots_static_files()
build_pygments_data_paths = [
"tools/setup/build_pygments_data", "tools/setup/lang.json"
]
from pygments import __version__ as pygments_version
if file_or_package_hash_updated(build_pygments_data_paths,
"build_pygments_data_hash",
options.is_force, [pygments_version]):
run(["tools/setup/build_pygments_data"])
else:
print("No need to run `tools/setup/build_pygments_data`.")
update_authors_json_paths = [
"tools/update-authors-json", "zerver/tests/fixtures/authors.json"
]
if file_or_package_hash_updated(update_authors_json_paths,
"update_authors_json_hash",
options.is_force):
run(["tools/update-authors-json", "--use-fixture"])
else:
print("No need to run `tools/update-authors-json`.")
email_source_paths = [
"tools/inline-email-css", "templates/zerver/emails/email.css"
]
email_source_paths += glob.glob('templates/zerver/emails/*.source.html')
if file_or_package_hash_updated(email_source_paths,
"last_email_source_files_hash",
options.is_force):
run(["tools/inline-email-css"])
else:
print("No need to run `tools/inline-email-css`.")
if not options.is_production_travis:
# The following block is skipped for the production Travis
# suite, because that suite doesn't make use of these elements
# of the development environment (it just uses the development
# environment to build a release tarball).
# Need to set up Django before using template_database_status
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
import django
django.setup()
from zerver.lib.test_fixtures import template_database_status, run_db_migrations, \
destroy_leaked_test_databases
try:
from zerver.lib.queue import SimpleQueueClient
SimpleQueueClient()
rabbitmq_is_configured = True
except Exception:
rabbitmq_is_configured = False
if options.is_force or not rabbitmq_is_configured:
run(["scripts/setup/configure-rabbitmq"])
else:
print("RabbitMQ is already configured.")
migration_status_path = os.path.join(UUID_VAR_PATH,
"migration_status_dev")
dev_template_db_status = template_database_status(
migration_status=migration_status_path,
settings="zproject.settings",
database_name="zulip",
)
if options.is_force or dev_template_db_status == 'needs_rebuild':
run(["tools/setup/postgres-init-dev-db"])
run(["tools/do-destroy-rebuild-database"])
elif dev_template_db_status == 'run_migrations':
run_db_migrations('dev')
elif dev_template_db_status == 'current':
print("No need to regenerate the dev DB.")
test_template_db_status = template_database_status()
if options.is_force or test_template_db_status == 'needs_rebuild':
run(["tools/setup/postgres-init-test-db"])
run(["tools/do-destroy-rebuild-test-database"])
elif test_template_db_status == 'run_migrations':
run_db_migrations('test')
elif test_template_db_status == 'current':
print("No need to regenerate the test DB.")
# Consider updating generated translations data: both `.mo`
# files and `language-options.json`.
paths = ['zerver/management/commands/compilemessages.py']
paths += glob.glob('locale/*/LC_MESSAGES/*.po')
paths += glob.glob('locale/*/translations.json')
if file_or_package_hash_updated(paths, "last_compilemessages_hash",
options.is_force):
run(["./manage.py", "compilemessages"])
else:
print("No need to run `manage.py compilemessages`.")
destroyed = destroy_leaked_test_databases()
if destroyed:
print("Dropped %s stale test databases!" % (destroyed, ))
run(["scripts/lib/clean-unused-caches", "--threshold=6"])
# Keeping this cache file around can cause eslint to throw
# random TypeErrors when new/updated dependencies are added
if os.path.isfile('.eslintcache'):
# Remove this block when
# https://github.com/eslint/eslint/issues/11639 is fixed
# upstream.
os.remove('.eslintcache')
# Clean up the root of the `var/` directory for various
# testing-related files that we have migrated to
# `var/<uuid>/test-backend`.
print("Cleaning var/ directory files...")
var_paths = glob.glob('var/test*')
var_paths.append('var/bot_avatar')
for path in var_paths:
try:
if os.path.isdir(path):
shutil.rmtree(path)
else:
os.remove(path)
except FileNotFoundError:
pass
version_file = os.path.join(UUID_VAR_PATH, 'provision_version')
print('writing to %s\n' % (version_file, ))
open(version_file, 'w').write(PROVISION_VERSION + '\n')
print()
print(OKBLUE + "Zulip development environment setup succeeded!" + ENDC)
return 0
def do_setup_virtualenv(venv_path, requirements_file, virtualenv_args):
# type: (str, str, List[str]) -> None
# Setup Python virtualenv
new_packages = set(get_package_names(requirements_file))
run(["sudo", "rm", "-rf", venv_path])
if not try_to_copy_venv(venv_path, new_packages):
# Create new virtualenv.
run(["sudo", "mkdir", "-p", venv_path])
run(["sudo", "virtualenv"] + virtualenv_args + [venv_path])
run(["sudo", "chown", "-R",
"{}:{}".format(os.getuid(), os.getgid()), venv_path])
create_log_entry(get_logfile_name(venv_path), "", set(), new_packages)
create_requirements_index_file(venv_path, requirements_file)
# Switch current Python context to the virtualenv.
activate_this = os.path.join(venv_path, "bin", "activate_this.py")
exec(open(activate_this).read(), {}, dict(__file__=activate_this)) # type: ignore # https://github.com/python/mypy/issues/1577
run(["pip", "install", "-U", "setuptools"])
run(["pip", "install", "--upgrade", "pip", "wheel"])
run(["pip", "install", "--no-deps", "--requirement", requirements_file])
run(["sudo", "chmod", "-R", "a+rX", venv_path])
def main(options):
# type: (Any) -> int
# yarn and management commands expect to be run from the root of the
# project.
os.chdir(ZULIP_PATH)
# setup-apt-repo does an `apt-get update`
# hash the apt dependencies
sha_sum = hashlib.sha1()
for apt_depedency in APT_DEPENDENCIES[codename]:
sha_sum.update(apt_depedency.encode('utf8'))
# hash the content of setup-apt-repo
sha_sum.update(open('scripts/lib/setup-apt-repo', 'rb').read())
new_apt_dependencies_hash = sha_sum.hexdigest()
last_apt_dependencies_hash = None
apt_hash_file_path = os.path.join(UUID_VAR_PATH, "apt_dependencies_hash")
try:
hash_file = open(apt_hash_file_path, 'r')
last_apt_dependencies_hash = hash_file.read()
except IOError:
run(['touch', apt_hash_file_path])
if (new_apt_dependencies_hash != last_apt_dependencies_hash):
try:
install_apt_deps()
except subprocess.CalledProcessError:
# Might be a failure due to network connection issues. Retrying...
print(WARNING + "`apt-get -y install` failed while installing dependencies; retrying..." + ENDC)
# Since a common failure mode is for the caching in
# `setup-apt-repo` to optimize the fast code path to skip
# running `apt-get update` when the target apt repository
# is out of date, we run it explicitly here so that we
# recover automatically.
run(['sudo', 'apt-get', 'update'])
install_apt_deps()
hash_file = open(apt_hash_file_path, 'w')
hash_file.write(new_apt_dependencies_hash)
else:
print("No changes to apt dependencies, so skipping apt operations.")
# Here we install node.
run(["sudo", "-H", "scripts/lib/install-node"])
# This is a wrapper around `yarn`, which we run last since
# it can often fail due to network issues beyond our control.
try:
# Hack: We remove `node_modules` as root to work around an
# issue with the symlinks being improperly owned by root.
if os.path.islink("node_modules"):
run(["sudo", "rm", "-f", "node_modules"])
run(["sudo", "mkdir", "-p", NODE_MODULES_CACHE_PATH])
run(["sudo", "chown", "%s:%s" % (user_id, user_id), NODE_MODULES_CACHE_PATH])
setup_node_modules(prefer_offline=True)
except subprocess.CalledProcessError:
print(WARNING + "`yarn install` failed; retrying..." + ENDC)
setup_node_modules()
# Import tools/setup_venv.py instead of running it so that we get an
# activated virtualenv for the rest of the provisioning process.
from tools.setup import setup_venvs
setup_venvs.main()
setup_shell_profile('~/.bash_profile')
setup_shell_profile('~/.zprofile')
run(["sudo", "cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])
# create log directory `zulip/var/log`
run(["mkdir", "-p", LOG_DIR_PATH])
# create upload directory `var/uploads`
run(["mkdir", "-p", UPLOAD_DIR_PATH])
# create test upload directory `var/test_upload`
run(["mkdir", "-p", TEST_UPLOAD_DIR_PATH])
# create coverage directory`var/coverage`
run(["mkdir", "-p", COVERAGE_DIR_PATH])
# create linecoverage directory`var/linecoverage-report`
run(["mkdir", "-p", LINECOVERAGE_DIR_PATH])
# create linecoverage directory`var/node-coverage`
run(["mkdir", "-p", NODE_TEST_COVERAGE_DIR_PATH])
# `build_emoji` script requires `emoji-datasource` package which we install
# via npm and hence it should be executed after we are done installing npm
# packages.
if not os.path.isdir(EMOJI_CACHE_PATH):
run(["sudo", "mkdir", EMOJI_CACHE_PATH])
run(["sudo", "chown", "%s:%s" % (user_id, user_id), EMOJI_CACHE_PATH])
run(["tools/setup/emoji/build_emoji"])
# copy over static files from the zulip_bots package
run(["tools/setup/generate_zulip_bots_static_files"])
run(["tools/generate-custom-icon-webfont"])
run(["tools/setup/build_pygments_data"])
run(["scripts/setup/generate_secrets.py", "--development"])
run(["tools/update-authors-json", "--use-fixture"])
run(["tools/inline-email-css"])
if is_circleci or (is_travis and not options.is_production_travis):
run(["sudo", "service", "rabbitmq-server", "restart"])
run(["sudo", "service", "redis-server", "restart"])
run(["sudo", "service", "memcached", "restart"])
run(["sudo", "service", "postgresql", "restart"])
elif options.is_docker:
run(["sudo", "service", "rabbitmq-server", "restart"])
run(["sudo", "pg_dropcluster", "--stop", POSTGRES_VERSION, "main"])
run(["sudo", "pg_createcluster", "-e", "utf8", "--start", POSTGRES_VERSION, "main"])
run(["sudo", "service", "redis-server", "restart"])
run(["sudo", "service", "memcached", "restart"])
if not options.is_production_travis:
# The following block is skipped for the production Travis
# suite, because that suite doesn't make use of these elements
# of the development environment (it just uses the development
# environment to build a release tarball).
# Need to set up Django before using is_template_database_current
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
import django
django.setup()
from zerver.lib.test_fixtures import is_template_database_current
try:
from zerver.lib.queue import SimpleQueueClient
SimpleQueueClient()
rabbitmq_is_configured = True
except Exception:
rabbitmq_is_configured = False
if options.is_force or not rabbitmq_is_configured:
run(["scripts/setup/configure-rabbitmq"])
else:
print("RabbitMQ is already configured.")
migration_status_path = os.path.join(UUID_VAR_PATH, "migration_status_dev")
if options.is_force or not is_template_database_current(
migration_status=migration_status_path,
settings="zproject.settings",
database_name="zulip",
):
run(["tools/setup/postgres-init-dev-db"])
run(["tools/do-destroy-rebuild-database"])
else:
print("No need to regenerate the dev DB.")
if options.is_force or not is_template_database_current():
run(["tools/setup/postgres-init-test-db"])
run(["tools/do-destroy-rebuild-test-database"])
else:
print("No need to regenerate the test DB.")
# Consider updating generated translations data: both `.mo`
# files and `language-options.json`.
sha1sum = hashlib.sha1()
paths = ['zerver/management/commands/compilemessages.py']
paths += glob.glob('static/locale/*/LC_MESSAGES/*.po')
paths += glob.glob('static/locale/*/translations.json')
for path in paths:
with open(path, 'rb') as file_to_hash:
sha1sum.update(file_to_hash.read())
compilemessages_hash_path = os.path.join(UUID_VAR_PATH, "last_compilemessages_hash")
new_compilemessages_hash = sha1sum.hexdigest()
run(['touch', compilemessages_hash_path])
with open(compilemessages_hash_path, 'r') as hash_file:
last_compilemessages_hash = hash_file.read()
if options.is_force or (new_compilemessages_hash != last_compilemessages_hash):
with open(compilemessages_hash_path, 'w') as hash_file:
hash_file.write(new_compilemessages_hash)
run(["./manage.py", "compilemessages"])
else:
print("No need to run `manage.py compilemessages`.")
run(["./manage.py", "create_realm_internal_bots"]) # Creates realm internal bots if required.
run(["scripts/lib/clean-unused-caches"])
version_file = os.path.join(UUID_VAR_PATH, 'provision_version')
print('writing to %s\n' % (version_file,))
open(version_file, 'w').write(PROVISION_VERSION + '\n')
print()
print(OKBLUE + "Zulip development environment setup succeeded!" + ENDC)
return 0
def try_to_copy_venv(venv_path, new_packages):
# type: (str, Set[str]) -> bool
"""
Tries to copy packages from an old virtual environment in the cache
to the new virtual environment. The algorithm works as follows:
1. Find a virtual environment, v, from the cache that has the
highest overlap with the new requirements such that:
a. The new requirements only add to the packages of v.
b. The new requirements only upgrade packages of v.
2. Copy the contents of v to the new virtual environment using
virtualenv-clone.
3. Delete all .pyc files in the new virtual environment.
"""
if not os.path.exists(VENV_CACHE_PATH):
return False
venv_name = os.path.basename(venv_path)
overlaps = [] # type: List[Tuple[int, str, Set[str]]]
old_packages = set() # type: Set[str]
for sha1sum in os.listdir(VENV_CACHE_PATH):
curr_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, venv_name)
if (curr_venv_path == venv_path or
not os.path.exists(get_index_filename(curr_venv_path))):
continue
old_packages = get_venv_packages(curr_venv_path)
# We only consider using using old virtualenvs that only
# contain packages that we want in our new virtualenv.
if not (old_packages - new_packages):
overlap = new_packages & old_packages
overlaps.append((len(overlap), curr_venv_path, overlap))
target_log = get_logfile_name(venv_path)
source_venv_path = None
if overlaps:
# Here, we select the old virtualenv with the largest overlap
overlaps = sorted(overlaps)
_, source_venv_path, copied_packages = overlaps[-1]
print('Copying packages from {}'.format(source_venv_path))
clone_ve = "{}/bin/virtualenv-clone".format(source_venv_path)
cmd = "sudo {exe} {source} {target}".format(exe=clone_ve,
source=source_venv_path,
target=venv_path).split()
try:
# TODO: We can probably remove this in a few months, now
# that we can expect that virtualenv-clone is present in
# all of our recent virtualenvs.
run(cmd)
except Exception:
# Virtualenv-clone is not installed. Install it and try running
# the command again.
try:
run("{}/bin/pip install --no-deps virtualenv-clone".format(
source_venv_path).split())
run(cmd)
except Exception:
# virtualenv-clone isn't working, so just make a new venv
return False
run(["sudo", "chown", "-R",
"{}:{}".format(os.getuid(), os.getgid()), venv_path])
source_log = get_logfile_name(source_venv_path)
copy_parent_log(source_log, target_log)
create_log_entry(target_log, source_venv_path, copied_packages,
new_packages - copied_packages)
return True
return False
def add_cert_to_pipconf():
# type: () -> None
conffile = os.path.expanduser("~/.pip/pip.conf")
confdir = os.path.expanduser("~/.pip/")
os.makedirs(confdir, exist_ok=True)
run(["crudini", "--set", conffile, "global", "cert", os.environ["CUSTOM_CA_CERTIFICATES"]])
def install_apt_deps():
# type: () -> None
# setup-apt-repo does an `apt-get update`
run(["sudo", "./scripts/lib/setup-apt-repo"])
run(["sudo", "apt-get", "-y", "install", "--no-install-recommends"] +
APT_DEPENDENCIES[codename])
def handle(self, *args: Any, **options: Any) -> None:
timestamp = timezone_now().strftime(TIMESTAMP_FORMAT)
with tempfile.TemporaryDirectory(
prefix=f"zulip-backup-{timestamp}-", ) as tmp:
os.mkdir(os.path.join(tmp, "zulip-backup"))
members = []
paths = []
with open(os.path.join(tmp, "zulip-backup", "zulip-version"),
"w") as f:
print(ZULIP_VERSION, file=f)
git = try_git_describe()
if git:
print(git, file=f)
members.append("zulip-backup/zulip-version")
with open(os.path.join(tmp, "zulip-backup", "os-version"),
"w") as f:
print(
"{ID} {VERSION_ID}".format(**parse_os_release()),
file=f,
)
members.append("zulip-backup/os-version")
with open(os.path.join(tmp, "zulip-backup", "postgres-version"),
"w") as f:
print(connection.pg_version, file=f)
members.append("zulip-backup/postgres-version")
if settings.DEVELOPMENT:
members.append(
os.path.join(settings.DEPLOY_ROOT, "zproject",
"dev-secrets.conf"), )
paths.append(
("zproject", os.path.join(settings.DEPLOY_ROOT,
"zproject")), )
else:
members.append("/etc/zulip")
paths.append(("settings", "/etc/zulip"))
if not options['skip_db']:
pg_dump_command = [
"pg_dump",
"--format=directory",
"--file",
os.path.join(tmp, "zulip-backup", "database"),
"--host",
settings.DATABASES["default"]["HOST"],
"--port",
settings.DATABASES["default"]["PORT"],
"--username",
settings.DATABASES["default"]["USER"],
"--dbname",
settings.DATABASES["default"]["NAME"],
"--no-password",
]
os.environ["PGPASSWORD"] = settings.DATABASES["default"][
"PASSWORD"]
run(
pg_dump_command,
cwd=tmp,
)
members.append("zulip-backup/database")
if not options[
'skip_uploads'] and settings.LOCAL_UPLOADS_DIR is not None and os.path.exists(
os.path.join(settings.DEPLOY_ROOT,
settings.LOCAL_UPLOADS_DIR), ):
members.append(
os.path.join(settings.DEPLOY_ROOT,
settings.LOCAL_UPLOADS_DIR), )
paths.append((
"uploads",
os.path.join(settings.DEPLOY_ROOT,
settings.LOCAL_UPLOADS_DIR),
), )
assert not any("|" in name or "|" in path for name, path in paths)
transform_args = [
r"--transform=s|^{}(/.*)?$|zulip-backup/{}\1|x".format(
re.escape(path),
name.replace("\\", r"\\"),
) for name, path in paths
]
try:
if options["output"] is None:
tarball_path = tempfile.NamedTemporaryFile(
prefix=f"zulip-backup-{timestamp}-",
suffix=".tar.gz",
delete=False,
).name
else:
tarball_path = options["output"]
run([
"tar", "-C", tmp, "-cPzf", tarball_path, *transform_args,
"--", *members
])
print(f"Backup tarball written to {tarball_path}")
except BaseException:
if options["output"] is None:
os.unlink(tarball_path)
raise
def copy_parent_log(source_log, target_log):
# type: (str, str) -> None
if os.path.exists(source_log):
run('cp {} {}'.format(source_log, target_log).split())
# Check the RAM on the user's system, and throw an effort if <1.5GB.
# This avoids users getting segfaults running `pip install` that are
# generally more annoying to debug.
with open("/proc/meminfo") as meminfo:
ram_size = meminfo.readlines()[0].strip().split(" ")[-2]
ram_gb = float(ram_size) / 1024.0 / 1024.0
if ram_gb < 1.5:
print("You have insufficient RAM (%s GB) to run the Zulip development environment." % (
round(ram_gb, 2),))
print("We recommend at least 2 GB of RAM, and require at least 1.5 GB.")
sys.exit(1)
try:
UUID_VAR_PATH = get_dev_uuid_var_path(create_if_missing=True)
run(["mkdir", "-p", UUID_VAR_PATH])
if os.path.exists(os.path.join(VAR_DIR_PATH, 'zulip-test-symlink')):
os.remove(os.path.join(VAR_DIR_PATH, 'zulip-test-symlink'))
os.symlink(
os.path.join(ZULIP_PATH, 'README.md'),
os.path.join(VAR_DIR_PATH, 'zulip-test-symlink')
)
os.remove(os.path.join(VAR_DIR_PATH, 'zulip-test-symlink'))
except OSError:
print(FAIL + "Error: Unable to create symlinks."
"Make sure you have permission to create symbolic links." + ENDC)
print("See this page for more information:")
print(" https://zulip.readthedocs.io/en/latest/development/setup-vagrant.html#os-symlink-error")
sys.exit(1)
if platform.architecture()[0] == '64bit':
def main(options):
# type: (Any) -> int
# yarn and management commands expect to be run from the root of the
# project.
os.chdir(ZULIP_PATH)
# setup-apt-repo does an `apt-get update`
# hash the apt dependencies
sha_sum = hashlib.sha1()
for apt_depedency in APT_DEPENDENCIES[codename]:
sha_sum.update(apt_depedency.encode('utf8'))
# hash the content of setup-apt-repo
sha_sum.update(open('scripts/lib/setup-apt-repo', 'rb').read())
new_apt_dependencies_hash = sha_sum.hexdigest()
last_apt_dependencies_hash = None
apt_hash_file_path = os.path.join(UUID_VAR_PATH, "apt_dependencies_hash")
try:
hash_file = open(apt_hash_file_path, 'r')
last_apt_dependencies_hash = hash_file.read()
except IOError:
run(['touch', apt_hash_file_path])
if (new_apt_dependencies_hash != last_apt_dependencies_hash):
try:
install_apt_deps()
except subprocess.CalledProcessError:
# Might be a failure due to network connection issues. Retrying...
print(WARNING + "`apt-get -y install` failed while installing dependencies; retrying..." + ENDC)
# Since a common failure mode is for the caching in
# `setup-apt-repo` to optimize the fast code path to skip
# running `apt-get update` when the target apt repository
# is out of date, we run it explicitly here so that we
# recover automatically.
run(['sudo', 'apt-get', 'update'])
install_apt_deps()
hash_file = open(apt_hash_file_path, 'w')
hash_file.write(new_apt_dependencies_hash)
else:
print("No changes to apt dependencies, so skipping apt operations.")
# Here we install node.
run(["sudo", "-H", "scripts/lib/install-node"])
# This is a wrapper around `yarn`, which we run last since
# it can often fail due to network issues beyond our control.
try:
# Hack: We remove `node_modules` as root to work around an
# issue with the symlinks being improperly owned by root.
if os.path.islink("node_modules"):
run(["sudo", "rm", "-f", "node_modules"])
run(["sudo", "mkdir", "-p", NODE_MODULES_CACHE_PATH])
run(["sudo", "chown", "%s:%s" % (user_id, user_id), NODE_MODULES_CACHE_PATH])
setup_node_modules(prefer_offline=True)
except subprocess.CalledProcessError:
print(WARNING + "`yarn install` failed; retrying..." + ENDC)
setup_node_modules()
# Import tools/setup_venv.py instead of running it so that we get an
# activated virtualenv for the rest of the provisioning process.
from tools.setup import setup_venvs
setup_venvs.main()
setup_shell_profile('~/.bash_profile')
setup_shell_profile('~/.zprofile')
run(["sudo", "cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])
# create log directory `zulip/var/log`
run(["mkdir", "-p", LOG_DIR_PATH])
# create upload directory `var/uploads`
run(["mkdir", "-p", UPLOAD_DIR_PATH])
# create test upload directory `var/test_upload`
run(["mkdir", "-p", TEST_UPLOAD_DIR_PATH])
# create coverage directory`var/coverage`
run(["mkdir", "-p", COVERAGE_DIR_PATH])
# create linecoverage directory`var/linecoverage-report`
run(["mkdir", "-p", LINECOVERAGE_DIR_PATH])
# create linecoverage directory`var/node-coverage`
run(["mkdir", "-p", NODE_TEST_COVERAGE_DIR_PATH])
# `build_emoji` script requires `emoji-datasource` package which we install
# via npm and hence it should be executed after we are done installing npm
# packages.
if not os.path.isdir(EMOJI_CACHE_PATH):
run(["sudo", "mkdir", EMOJI_CACHE_PATH])
run(["sudo", "chown", "%s:%s" % (user_id, user_id), EMOJI_CACHE_PATH])
run(["tools/setup/emoji/build_emoji"])
# copy over static files from the zulip_bots package
run(["tools/setup/generate_zulip_bots_static_files"])
run(["tools/generate-custom-icon-webfont"])
run(["tools/setup/build_pygments_data"])
run(["scripts/setup/generate_secrets.py", "--development"])
run(["tools/update-authors-json", "--use-fixture"])
run(["tools/inline-email-css"])
if is_circleci or (is_travis and not options.is_production_travis):
run(["sudo", "service", "rabbitmq-server", "restart"])
run(["sudo", "service", "redis-server", "restart"])
run(["sudo", "service", "memcached", "restart"])
run(["sudo", "service", "postgresql", "restart"])
elif options.is_docker:
run(["sudo", "service", "rabbitmq-server", "restart"])
run(["sudo", "pg_dropcluster", "--stop", POSTGRES_VERSION, "main"])
run(["sudo", "pg_createcluster", "-e", "utf8", "--start", POSTGRES_VERSION, "main"])
run(["sudo", "service", "redis-server", "restart"])
run(["sudo", "service", "memcached", "restart"])
if not options.is_production_travis:
# The following block is skipped for the production Travis
# suite, because that suite doesn't make use of these elements
# of the development environment (it just uses the development
# environment to build a release tarball).
# Need to set up Django before using is_template_database_current
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
import django
django.setup()
from zerver.lib.test_fixtures import is_template_database_current
try:
from zerver.lib.queue import SimpleQueueClient
SimpleQueueClient()
rabbitmq_is_configured = True
except Exception:
rabbitmq_is_configured = False
if options.is_force or not rabbitmq_is_configured:
run(["scripts/setup/configure-rabbitmq"])
else:
print("RabbitMQ is already configured.")
migration_status_path = os.path.join(UUID_VAR_PATH, "migration_status_dev")
if options.is_force or not is_template_database_current(
migration_status=migration_status_path,
settings="zproject.settings",
database_name="zulip",
):
run(["tools/setup/postgres-init-dev-db"])
run(["tools/do-destroy-rebuild-database"])
else:
print("No need to regenerate the dev DB.")
if options.is_force or not is_template_database_current():
run(["tools/setup/postgres-init-test-db"])
run(["tools/do-destroy-rebuild-test-database"])
else:
print("No need to regenerate the test DB.")
# Consider updating generated translations data: both `.mo`
# files and `language-options.json`.
sha1sum = hashlib.sha1()
paths = ['zerver/management/commands/compilemessages.py']
paths += glob.glob('static/locale/*/LC_MESSAGES/*.po')
paths += glob.glob('static/locale/*/translations.json')
for path in paths:
with open(path, 'rb') as file_to_hash:
sha1sum.update(file_to_hash.read())
compilemessages_hash_path = os.path.join(UUID_VAR_PATH, "last_compilemessages_hash")
new_compilemessages_hash = sha1sum.hexdigest()
run(['touch', compilemessages_hash_path])
with open(compilemessages_hash_path, 'r') as hash_file:
last_compilemessages_hash = hash_file.read()
if options.is_force or (new_compilemessages_hash != last_compilemessages_hash):
with open(compilemessages_hash_path, 'w') as hash_file:
hash_file.write(new_compilemessages_hash)
run(["./manage.py", "compilemessages"])
else:
print("No need to run `manage.py compilemessages`.")
run(["scripts/lib/clean-unused-caches"])
version_file = os.path.join(UUID_VAR_PATH, 'provision_version')
print('writing to %s\n' % (version_file,))
open(version_file, 'w').write(PROVISION_VERSION + '\n')
print()
print(OKBLUE + "Zulip development environment setup succeeded!" + ENDC)
return 0
def main(options: argparse.Namespace) -> int:
setup_bash_profile()
setup_shell_profile("~/.zprofile")
# This needs to happen before anything that imports zproject.settings.
run(["scripts/setup/generate_secrets.py", "--development"])
create_var_directories()
# The `build_emoji` script requires `emoji-datasource` package
# which we install via npm; thus this step is after installing npm
# packages.
run(["tools/setup/emoji/build_emoji"])
# copy over static files from the zulip_bots package
generate_zulip_bots_static_files()
if options.is_force or need_to_run_build_pygments_data():
run(["tools/setup/build_pygments_data"])
write_new_digest(
"build_pygments_data_hash",
build_pygments_data_paths(),
[pygments_version],
)
else:
print("No need to run `tools/setup/build_pygments_data`.")
if options.is_force or need_to_run_build_timezone_data():
run(["tools/setup/build_timezone_values"])
write_new_digest(
"build_timezones_data_hash",
build_timezones_data_paths(),
[timezones_version],
)
else:
print("No need to run `tools/setup/build_timezone_values`.")
if options.is_force or need_to_run_inline_email_css():
run(["scripts/setup/inline_email_css.py"])
write_new_digest(
"last_email_source_files_hash",
inline_email_css_paths(),
)
else:
print("No need to run `scripts/setup/inline_email_css.py`.")
if not options.is_build_release_tarball_only:
# The following block is skipped when we just need the development
# environment to build a release tarball.
# Need to set up Django before using template_status
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
import django
django.setup()
from django.conf import settings
from zerver.lib.test_fixtures import (
DEV_DATABASE,
TEST_DATABASE,
destroy_leaked_test_databases,
)
if options.is_force or need_to_run_configure_rabbitmq(
[settings.RABBITMQ_PASSWORD]):
run(["scripts/setup/configure-rabbitmq"])
write_new_digest(
"last_configure_rabbitmq_hash",
configure_rabbitmq_paths(),
[settings.RABBITMQ_PASSWORD],
)
else:
print("No need to run `scripts/setup/configure-rabbitmq.")
dev_template_db_status = DEV_DATABASE.template_status()
if options.is_force or dev_template_db_status == "needs_rebuild":
run(["tools/setup/postgresql-init-dev-db"])
if options.skip_dev_db_build:
# We don't need to build the manual development
# database on continuous integration for running tests, so we can
# just leave it as a template db and save a minute.
#
# Important: We don't write a digest as that would
# incorrectly claim that we ran migrations.
pass
else:
run(["tools/rebuild-dev-database"])
DEV_DATABASE.write_new_db_digest()
elif dev_template_db_status == "run_migrations":
DEV_DATABASE.run_db_migrations()
elif dev_template_db_status == "current":
print("No need to regenerate the dev DB.")
test_template_db_status = TEST_DATABASE.template_status()
if options.is_force or test_template_db_status == "needs_rebuild":
run(["tools/setup/postgresql-init-test-db"])
run(["tools/rebuild-test-database"])
TEST_DATABASE.write_new_db_digest()
elif test_template_db_status == "run_migrations":
TEST_DATABASE.run_db_migrations()
elif test_template_db_status == "current":
print("No need to regenerate the test DB.")
if options.is_force or need_to_run_compilemessages():
run(["./manage.py", "compilemessages"])
write_new_digest(
"last_compilemessages_hash",
compilemessages_paths(),
)
else:
print("No need to run `manage.py compilemessages`.")
destroyed = destroy_leaked_test_databases()
if destroyed:
print(f"Dropped {destroyed} stale test databases!")
clean_unused_caches.main(
argparse.Namespace(
threshold_days=6,
# The defaults here should match parse_cache_script_args in zulip_tools.py
dry_run=False,
verbose=False,
no_headings=True,
))
# Keeping this cache file around can cause eslint to throw
# random TypeErrors when new/updated dependencies are added
if os.path.isfile(".eslintcache"):
# Remove this block when
# https://github.com/eslint/eslint/issues/11639 is fixed
# upstream.
os.remove(".eslintcache")
# Clean up the root of the `var/` directory for various
# testing-related files that we have migrated to
# `var/<uuid>/test-backend`.
print("Cleaning var/ directory files...")
var_paths = glob.glob("var/test*")
var_paths.append("var/bot_avatar")
for path in var_paths:
try:
if os.path.isdir(path):
shutil.rmtree(path)
else:
os.remove(path)
except FileNotFoundError:
pass
version_file = os.path.join(UUID_VAR_PATH, "provision_version")
print(f"writing to {version_file}\n")
with open(version_file, "w") as f:
f.write(PROVISION_VERSION + "\n")
print()
print(OKBLUE + "Zulip development environment setup succeeded!" + ENDC)
return 0
def handle(self, *args: Any, **options: Any) -> None:
timestamp = timezone_now().strftime(TIMESTAMP_FORMAT)
with tempfile.TemporaryDirectory(prefix="zulip-backup-%s-" %
(timestamp, )) as tmp:
os.mkdir(os.path.join(tmp, "zulip-backup"))
members = []
paths = []
with open(os.path.join(tmp, "zulip-backup", "zulip-version"),
"w") as f:
print(ZULIP_VERSION, file=f)
git = try_git_describe()
if git:
print(git, file=f)
members.append("zulip-backup/zulip-version")
with open(os.path.join(tmp, "zulip-backup", "os-version"),
"w") as f:
print(
"{DISTRIB_ID} {DISTRIB_CODENAME}".format(
**parse_lsb_release()),
file=f,
)
members.append("zulip-backup/os-version")
with open(os.path.join(tmp, "zulip-backup", "postgres-version"),
"w") as f:
print(connection.pg_version, file=f)
members.append("zulip-backup/postgres-version")
if settings.DEVELOPMENT:
members.append(
os.path.join(settings.DEPLOY_ROOT, "zproject",
"dev-secrets.conf"))
paths.append(
("zproject", os.path.join(settings.DEPLOY_ROOT,
"zproject")))
else:
members.append("/etc/zulip")
paths.append(("settings", "/etc/zulip"))
if not options['skip_db']:
db_name = settings.DATABASES["default"]["NAME"]
db_dir = os.path.join(tmp, "zulip-backup", "database")
run(
[
"pg_dump", "--format=directory", "--file", db_dir,
"--", db_name
],
cwd=tmp,
)
members.append("zulip-backup/database")
if not options[
'skip_uploads'] and settings.LOCAL_UPLOADS_DIR is not None and os.path.exists(
os.path.join(settings.DEPLOY_ROOT,
settings.LOCAL_UPLOADS_DIR)):
members.append(
os.path.join(settings.DEPLOY_ROOT,
settings.LOCAL_UPLOADS_DIR))
paths.append((
"uploads",
os.path.join(settings.DEPLOY_ROOT,
settings.LOCAL_UPLOADS_DIR),
))
assert not any("|" in name or "|" in path for name, path in paths)
transform_args = [
r"--transform=s|^{}(/.*)?$|zulip-backup/{}\1|x".format(
re.escape(path), name.replace("\\", r"\\"))
for name, path in paths
]
try:
if options["output"] is None:
tarball_path = tempfile.NamedTemporaryFile(
prefix="zulip-backup-%s-" % (timestamp, ),
suffix=".tar.gz",
delete=False,
).name
else:
tarball_path = options["output"]
run(["tar", "-C", tmp, "-cPzf", tarball_path] +
transform_args + ["--"] + members)
print("Backup tarball written to %s" % (tarball_path, ))
except BaseException:
if options["output"] is None:
os.unlink(tarball_path)
raise
def install_venv_deps(requirements_file):
# type: (str) -> None
pip_requirements = os.path.join(ZULIP_PATH, "requirements", "pip.txt")
run(["pip", "install", "-U", "--requirement", pip_requirements])
run(["pip", "install", "--no-deps", "--requirement", requirements_file])
def install_yum_deps(deps_to_install, retry=False):
# type: (List[str], bool) -> None
print(WARNING + "RedHat support is still experimental.")
run(["sudo", "./scripts/lib/setup-yum-repo"])
# Hack specific to unregistered RHEL system. The moreutils
# package requires a perl module package, which isn't available in
# the unregistered RHEL repositories.
#
# Error: Package: moreutils-0.49-2.el7.x86_64 (epel)
# Requires: perl(IPC::Run)
yum_extra_flags = [] # type: List[str]
if vendor == 'RedHat':
exitcode, subs_status = subprocess.getstatusoutput(
"sudo subscription-manager status")
if exitcode == 1:
# TODO this might overkill since `subscription-manager` is already
# called in setup-yum-repo
if 'Status' in subs_status:
# The output is well-formed
yum_extra_flags = ["--skip-broken"]
else:
print(
"Unrecognized output. `subscription-manager` might not be available"
)
run(["sudo", "yum", "install", "-y"] + yum_extra_flags + deps_to_install)
if vendor in ["CentOS", "RedHat"]:
# This is how a pip3 is installed to /usr/bin in CentOS/RHEL
# for python35 and later.
run(["sudo", "python36", "-m", "ensurepip"])
# `python36` is not aliased to `python3` by default
run(["sudo", "ln", "-nsf", "/usr/bin/python36", "/usr/bin/python3"])
postgres_dir = 'pgsql-%s' % (POSTGRES_VERSION, )
for cmd in ['pg_config', 'pg_isready', 'psql']:
# Our tooling expects these postgres scripts to be at
# well-known paths. There's an argument for eventually
# making our tooling auto-detect, but this is simpler.
run([
"sudo", "ln", "-nsf",
"/usr/%s/bin/%s" % (postgres_dir, cmd),
"/usr/bin/%s" % (cmd, )
])
# Compile tsearch-extras from scratch, since we maintain the
# package and haven't built an RPM package for it.
run(["sudo", "./scripts/lib/build-tsearch-extras"])
if vendor == "Fedora":
# Compile PGroonga from scratch, since pgroonga upstream
# doesn't provide Fedora packages.
run(["sudo", "./scripts/lib/build-pgroonga"])
# From here, we do the first-time setup/initialization for the postgres database.
pg_datadir = "/var/lib/pgsql/%s/data" % (POSTGRES_VERSION, )
pg_hba_conf = os.path.join(pg_datadir, "pg_hba.conf")
# We can't just check if the file exists with os.path, since the
# current user likely doesn't have permission to read the
# pg_datadir directory.
if subprocess.call(["sudo", "test", "-e", pg_hba_conf]) == 0:
# Skip setup if it has been applied previously
return
run([
"sudo", "-H",
"/usr/%s/bin/postgresql-%s-setup" % (postgres_dir, POSTGRES_VERSION),
"initdb"
])
# Use vendored pg_hba.conf, which enables password authentication.
run([
"sudo", "cp", "-a", "puppet/zulip/files/postgresql/centos_pg_hba.conf",
pg_hba_conf
])
def handle(self, *args: Any, **options: Any) -> None:
timestamp = timezone_now().strftime(TIMESTAMP_FORMAT)
with tempfile.TemporaryDirectory(
prefix="zulip-backup-%s-" % (timestamp,)
) as tmp:
os.mkdir(os.path.join(tmp, "zulip-backup"))
members = []
with open(os.path.join(tmp, "zulip-backup", "zulip-version"), "w") as f:
print(ZULIP_VERSION, file=f)
git = try_git_describe()
if git:
print(git, file=f)
members.append("zulip-backup/zulip-version")
with open(os.path.join(tmp, "zulip-backup", "os-version"), "w") as f:
print(
"{DISTRIB_ID} {DISTRIB_CODENAME}".format(**parse_lsb_release()),
file=f,
)
members.append("zulip-backup/os-version")
with open(os.path.join(tmp, "zulip-backup", "postgres-version"), "w") as f:
print(connection.pg_version, file=f)
members.append("zulip-backup/postgres-version")
if settings.DEVELOPMENT:
os.symlink(
os.path.join(settings.DEPLOY_ROOT, "zproject"),
os.path.join(tmp, "zulip-backup", "zproject"),
)
members.append("zulip-backup/zproject/dev-secrets.conf")
else:
os.symlink("/etc/zulip", os.path.join(tmp, "zulip-backup", "settings"))
members.append("zulip-backup/settings")
db_name = settings.DATABASES["default"]["NAME"]
db_dir = os.path.join(tmp, "zulip-backup", "database")
run(
["pg_dump", "--format=directory", "--file", db_dir, "--", db_name],
cwd=tmp,
)
members.append("zulip-backup/database")
if settings.LOCAL_UPLOADS_DIR is not None and os.path.exists(
os.path.join(settings.DEPLOY_ROOT, settings.LOCAL_UPLOADS_DIR)
):
os.symlink(
os.path.join(settings.DEPLOY_ROOT, settings.LOCAL_UPLOADS_DIR),
os.path.join(tmp, "zulip-backup", "uploads"),
)
members.append("zulip-backup/uploads")
try:
if options["output"] is None:
tarball_path = tempfile.NamedTemporaryFile(
prefix="zulip-backup-%s-" % (timestamp,),
suffix=".tar.gz",
delete=False,
).name
else:
tarball_path = options["output"]
run(["tar", "-C", tmp, "-chzf", tarball_path, "--"] + members)
print("Backup tarball written to %s" % (tarball_path,))
except BaseException:
if options["output"] is None:
os.unlink(tarball_path)
raise
def do_setup_virtualenv(venv_path, requirements_file, virtualenv_args):
# type: (str, str, List[str]) -> None
# Setup Python virtualenv
new_packages = set(get_package_names(requirements_file))
run(["sudo", "rm", "-rf", venv_path])
if not try_to_copy_venv(venv_path, new_packages):
# Create new virtualenv.
run(["sudo", "mkdir", "-p", venv_path])
run(["sudo", "virtualenv"] + virtualenv_args + [venv_path])
run(["sudo", "chown", "-R",
"{}:{}".format(os.getuid(), os.getgid()), venv_path])
create_log_entry(get_logfile_name(venv_path), "", set(), new_packages)
create_requirements_index_file(venv_path, requirements_file)
# Switch current Python context to the virtualenv.
activate_this = os.path.join(venv_path, "bin", "activate_this.py")
exec(open(activate_this).read(), {}, dict(__file__=activate_this))
# use custom certificate if needed
if os.environ.get('CUSTOM_CA_CERTIFICATES'):
print("Configuring pip to use custom CA certificates...")
add_cert_to_pipconf()
# CentOS-specific hack/workaround
# Install pycurl with custom flag due to this error when installing
# via pip:
# __main__.ConfigurationError: Curl is configured to use SSL, but
# we have not been able to determine which SSL backend it is using.
# Please see PycURL documentation for how to specify the SSL
# backend manually.
# See https://github.com/pycurl/pycurl/issues/526
# The fix exists on pycurl master, but not yet in any release
# We can likely remove this when pycurl > 7.43.0.2 comes out.
if os.path.exists("/etc/redhat-release"):
pycurl_env = os.environ.copy()
pycurl_env["PYCURL_SSL_LIBRARY"] = "nss"
run(["pip", "install", "pycurl==7.43.0.2", "--compile", "--no-cache-dir"],
env=pycurl_env)
try:
install_venv_deps(requirements_file)
except subprocess.CalledProcessError:
# Might be a failure due to network connection issues. Retrying...
print(WARNING + "`pip install` failed; retrying..." + ENDC)
install_venv_deps(requirements_file)
run(["sudo", "chmod", "-R", "a+rX", venv_path])
# Check the RAM on the user's system, and throw an effort if <1.5GB.
# This avoids users getting segfaults running `pip install` that are
# generally more annoying to debug.
with open("/proc/meminfo") as meminfo:
ram_size = meminfo.readlines()[0].strip().split(" ")[-2]
ram_gb = float(ram_size) / 1024.0 / 1024.0
if ram_gb < 1.5:
print("You have insufficient RAM (%s GB) to run the Zulip development environment." % (
round(ram_gb, 2),))
print("We recommend at least 2 GB of RAM, and require at least 1.5 GB.")
sys.exit(1)
try:
UUID_VAR_PATH = get_dev_uuid_var_path(create_if_missing=True)
run(["mkdir", "-p", UUID_VAR_PATH])
if os.path.exists(os.path.join(VAR_DIR_PATH, 'zulip-test-symlink')):
os.remove(os.path.join(VAR_DIR_PATH, 'zulip-test-symlink'))
os.symlink(
os.path.join(ZULIP_PATH, 'README.md'),
os.path.join(VAR_DIR_PATH, 'zulip-test-symlink')
)
os.remove(os.path.join(VAR_DIR_PATH, 'zulip-test-symlink'))
except OSError as err:
print(FAIL + "Error: Unable to create symlinks."
"Make sure you have permission to create symbolic links." + ENDC)
print("See this page for more information:")
print(" https://zulip.readthedocs.io/en/latest/development/setup-vagrant.html#os-symlink-error")
sys.exit(1)
if platform.architecture()[0] == '64bit':
def main():
# type: () -> int
# npm install and management commands expect to be run from the root of the
# project.
os.chdir(ZULIP_PATH)
run(["sudo", "./scripts/lib/setup-apt-repo"])
# Add groonga repository to get the pgroonga packages
run(["sudo", "add-apt-repository", "-y", "ppa:groonga/ppa"])
run(["sudo", "apt-get", "update"])
run(["sudo", "apt-get", "-y", "install", "--no-install-recommends"] + APT_DEPENDENCIES[codename])
if TRAVIS:
if PY2:
MYPY_REQS_FILE = os.path.join(ZULIP_PATH, "requirements", "mypy.txt")
setup_virtualenv(PY3_VENV_PATH, MYPY_REQS_FILE, patch_activate_script=True,
virtualenv_args=['-p', 'python3'])
DEV_REQS_FILE = os.path.join(ZULIP_PATH, "requirements", "py2_dev.txt")
setup_virtualenv(PY2_VENV_PATH, DEV_REQS_FILE, patch_activate_script=True)
else:
TWISTED_REQS_FILE = os.path.join(ZULIP_PATH, "requirements", "twisted.txt")
setup_virtualenv("/srv/zulip-py2-twisted-venv", TWISTED_REQS_FILE,
patch_activate_script=True)
DEV_REQS_FILE = os.path.join(ZULIP_PATH, "requirements", "py3_dev.txt")
setup_virtualenv(VENV_PATH, DEV_REQS_FILE, patch_activate_script=True,
virtualenv_args=['-p', 'python3'])
else:
# Import tools/setup_venv.py instead of running it so that we get an
# activated virtualenv for the rest of the provisioning process.
from tools.setup import setup_venvs
setup_venvs.main()
# Put Python2 virtualenv activation in our .bash_profile.
with open(os.path.expanduser('~/.bash_profile'), 'w+') as bash_profile:
bash_profile.writelines([
"source .bashrc\n",
"source %s\n" % (os.path.join(VENV_PATH, "bin", "activate"),),
])
run(["sudo", "cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])
# create log directory `zulip/var/log`
run(["mkdir", "-p", LOG_DIR_PATH])
# create upload directory `var/uploads`
run(["mkdir", "-p", UPLOAD_DIR_PATH])
# create test upload directory `var/test_upload`
run(["mkdir", "-p", TEST_UPLOAD_DIR_PATH])
# create coverage directory`var/coverage`
run(["mkdir", "-p", COVERAGE_DIR_PATH])
# create linecoverage directory`var/linecoverage-report`
run(["mkdir", "-p", LINECOVERAGE_DIR_PATH])
# create linecoverage directory`var/node-coverage`
run(["mkdir", "-p", NODE_TEST_COVERAGE_DIR_PATH])
if TRAVIS:
run(["tools/setup/install-phantomjs", "--travis"])
else:
run(["tools/setup/install-phantomjs"])
run(["tools/setup/download-zxcvbn"])
run(["tools/setup/emoji_dump/build_emoji"])
run(["scripts/setup/generate_secrets.py", "-d"])
if TRAVIS and not PRODUCTION_TRAVIS:
run(["sudo", "service", "rabbitmq-server", "restart"])
run(["sudo", "service", "redis-server", "restart"])
run(["sudo", "service", "memcached", "restart"])
elif "--docker" in sys.argv:
run(["sudo", "service", "rabbitmq-server", "restart"])
run(["sudo", "pg_dropcluster", "--stop", POSTGRES_VERSION, "main"])
run(["sudo", "pg_createcluster", "-e", "utf8", "--start", POSTGRES_VERSION, "main"])
run(["sudo", "service", "redis-server", "restart"])
run(["sudo", "service", "memcached", "restart"])
if not PRODUCTION_TRAVIS:
# These won't be used anyway
run(["scripts/setup/configure-rabbitmq"])
run(["tools/setup/postgres-init-dev-db"])
run(["tools/do-destroy-rebuild-database"])
run(["tools/setup/postgres-init-test-db"])
run(["tools/do-destroy-rebuild-test-database"])
run(["python", "./manage.py", "compilemessages"])
# Install the pinned version of npm.
install_npm()
# Run npm install last because it can be flaky, and that way one
# only needs to rerun `npm install` to fix the installation.
try:
setup_node_modules()
except subprocess.CalledProcessError:
print(WARNING + "`npm install` failed; retrying..." + ENDC)
setup_node_modules()
print()
print(OKBLUE + "Zulip development environment setup succeeded!" + ENDC)
return 0
def do_setup_virtualenv(venv_path, requirements_file, virtualenv_args):
# type: (str, str, List[str]) -> None
# Setup Python virtualenv
new_packages = set(get_package_names(requirements_file))
run_as_root(["rm", "-rf", venv_path])
if not try_to_copy_venv(venv_path, new_packages):
# Create new virtualenv.
run_as_root(["mkdir", "-p", venv_path])
run_as_root(["virtualenv"] + virtualenv_args + [venv_path])
run_as_root([
"chown", "-R", "{}:{}".format(os.getuid(), os.getgid()), venv_path
])
create_log_entry(get_logfile_name(venv_path), "", set(), new_packages)
create_requirements_index_file(venv_path, requirements_file)
pip = os.path.join(venv_path, "bin", "pip")
# use custom certificate if needed
if os.environ.get('CUSTOM_CA_CERTIFICATES'):
print("Configuring pip to use custom CA certificates...")
add_cert_to_pipconf()
# CentOS-specific hack/workaround
# Install pycurl with custom flag due to this error when installing
# via pip:
# __main__.ConfigurationError: Curl is configured to use SSL, but
# we have not been able to determine which SSL backend it is using.
# Please see PycURL documentation for how to specify the SSL
# backend manually.
# See https://github.com/pycurl/pycurl/issues/526
# The fix exists on pycurl master, but not yet in any release
# We can likely remove this when pycurl > 7.43.0.2 comes out.
if os.path.exists("/etc/redhat-release"):
pycurl_env = os.environ.copy()
pycurl_env["PYCURL_SSL_LIBRARY"] = "nss"
run([
pip, "install", "pycurl==7.43.0.2", "--compile", "--no-cache-dir"
],
env=pycurl_env)
try:
install_venv_deps(pip, requirements_file)
except subprocess.CalledProcessError:
# Might be a failure due to network connection issues. Retrying...
print(WARNING + "`pip install` failed; retrying..." + ENDC)
install_venv_deps(pip, requirements_file)
# The typing module has been included in stdlib since 3.5.
# Installing a pypi version of it has been harmless until a bug
# "AttributeError: type object 'Callable' has no attribute
# '_abc_registry'" happens in 3.7. And so just to be safe, it is
# disabled from now on for all >= 3.5 versions.
# Remove this once 3.4 is no longer supported.
at_least_35 = (sys.version_info.major
== 3) and (sys.version_info.minor >= 5)
if at_least_35 and ('python2.7' not in virtualenv_args):
run([pip, "uninstall", "-y", "typing"])
run_as_root(["chmod", "-R", "a+rX", venv_path])
def install_apt_deps():
# type: () -> None
# setup-apt-repo does an `apt-get update`
run(["sudo", "./scripts/lib/setup-apt-repo"])
run(["sudo", "apt-get", "-y", "install", "--no-install-recommends"] + APT_DEPENDENCIES[codename])
def main(options):
# type: (Any) -> int
# yarn and management commands expect to be run from the root of the
# project.
os.chdir(ZULIP_PATH)
# hash the apt dependencies
sha_sum = hashlib.sha1()
for apt_depedency in SYSTEM_DEPENDENCIES:
sha_sum.update(apt_depedency.encode('utf8'))
if vendor in ["Ubuntu", "Debian"]:
sha_sum.update(open('scripts/lib/setup-apt-repo', 'rb').read())
else:
# hash the content of setup-yum-repo and build-*
sha_sum.update(open('scripts/lib/setup-yum-repo', 'rb').read())
build_paths = glob.glob("scripts/lib/build-")
for bp in build_paths:
sha_sum.update(open(bp, 'rb').read())
new_apt_dependencies_hash = sha_sum.hexdigest()
last_apt_dependencies_hash = None
apt_hash_file_path = os.path.join(UUID_VAR_PATH, "apt_dependencies_hash")
with open(apt_hash_file_path, 'a+') as hash_file:
hash_file.seek(0)
last_apt_dependencies_hash = hash_file.read()
if (new_apt_dependencies_hash != last_apt_dependencies_hash):
try:
install_system_deps()
except subprocess.CalledProcessError:
# Might be a failure due to network connection issues. Retrying...
print(WARNING +
"Installing system dependencies failed; retrying..." + ENDC)
install_system_deps()
with open(apt_hash_file_path, 'w') as hash_file:
hash_file.write(new_apt_dependencies_hash)
else:
print("No changes to apt dependencies, so skipping apt operations.")
# Here we install node.
proxy_env = [
"env",
"http_proxy=" + os.environ.get("http_proxy", ""),
"https_proxy=" + os.environ.get("https_proxy", ""),
"no_proxy=" + os.environ.get("no_proxy", ""),
]
run_as_root(proxy_env + ["scripts/lib/install-node"], sudo_args=['-H'])
# This is a wrapper around `yarn`, which we run last since
# it can often fail due to network issues beyond our control.
try:
# Hack: We remove `node_modules` as root to work around an
# issue with the symlinks being improperly owned by root.
if os.path.islink("node_modules"):
run_as_root(["rm", "-f", "node_modules"])
run_as_root(["mkdir", "-p", NODE_MODULES_CACHE_PATH])
run_as_root(
["chown",
"%s:%s" % (user_id, user_id), NODE_MODULES_CACHE_PATH])
setup_node_modules(prefer_offline=True)
except subprocess.CalledProcessError:
print(WARNING + "`yarn install` failed; retrying..." + ENDC)
try:
setup_node_modules()
except subprocess.CalledProcessError:
print(
FAIL +
"`yarn install` is failing; check your network connection (and proxy settings)."
+ ENDC)
sys.exit(1)
# Install shellcheck.
run_as_root(["scripts/lib/install-shellcheck"])
from tools.setup import setup_venvs
setup_venvs.main()
activate_this = "/srv/zulip-py3-venv/bin/activate_this.py"
exec(open(activate_this).read(), dict(__file__=activate_this))
setup_shell_profile('~/.bash_profile')
setup_shell_profile('~/.zprofile')
# This needs to happen before anything that imports zproject.settings.
run(["scripts/setup/generate_secrets.py", "--development"])
run_as_root(["cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])
# create log directory `zulip/var/log`
os.makedirs(LOG_DIR_PATH, exist_ok=True)
# create upload directory `var/uploads`
os.makedirs(UPLOAD_DIR_PATH, exist_ok=True)
# create test upload directory `var/test_upload`
os.makedirs(TEST_UPLOAD_DIR_PATH, exist_ok=True)
# create coverage directory `var/coverage`
os.makedirs(COVERAGE_DIR_PATH, exist_ok=True)
# create linecoverage directory `var/node-coverage`
os.makedirs(NODE_TEST_COVERAGE_DIR_PATH, exist_ok=True)
# create XUnit XML test results directory`var/xunit-test-results`
os.makedirs(XUNIT_XML_TEST_RESULTS_DIR_PATH, exist_ok=True)
# The `build_emoji` script requires `emoji-datasource` package
# which we install via npm; thus this step is after installing npm
# packages.
if not os.path.isdir(EMOJI_CACHE_PATH):
run_as_root(["mkdir", EMOJI_CACHE_PATH])
run_as_root(["chown", "%s:%s" % (user_id, user_id), EMOJI_CACHE_PATH])
run(["tools/setup/emoji/build_emoji"])
# copy over static files from the zulip_bots package
generate_zulip_bots_static_files()
webfont_paths = [
"tools/setup/generate-custom-icon-webfont",
"static/icons/fonts/template.hbs"
]
webfont_paths += glob.glob('static/assets/icons/*')
if file_or_package_hash_updated(webfont_paths, "webfont_files_hash",
options.is_force):
run(["tools/setup/generate-custom-icon-webfont"])
else:
print("No need to run `tools/setup/generate-custom-icon-webfont`.")
build_pygments_data_paths = [
"tools/setup/build_pygments_data", "tools/setup/lang.json"
]
from pygments import __version__ as pygments_version
if file_or_package_hash_updated(build_pygments_data_paths,
"build_pygments_data_hash",
options.is_force, [pygments_version]):
run(["tools/setup/build_pygments_data"])
else:
print("No need to run `tools/setup/build_pygments_data`.")
update_authors_json_paths = [
"tools/update-authors-json", "zerver/tests/fixtures/authors.json"
]
if file_or_package_hash_updated(update_authors_json_paths,
"update_authors_json_hash",
options.is_force):
run(["tools/update-authors-json", "--use-fixture"])
else:
print("No need to run `tools/update-authors-json`.")
email_source_paths = [
"tools/inline-email-css", "templates/zerver/emails/email.css"
]
email_source_paths += glob.glob('templates/zerver/emails/*.source.html')
if file_or_package_hash_updated(email_source_paths,
"last_email_source_files_hash",
options.is_force):
run(["tools/inline-email-css"])
else:
print("No need to run `tools/inline-email-css`.")
if is_circleci or (is_travis and not options.is_production_travis):
run_as_root(["service", "rabbitmq-server", "restart"])
run_as_root(["service", "redis-server", "restart"])
run_as_root(["service", "memcached", "restart"])
run_as_root(["service", "postgresql", "restart"])
elif family == 'redhat':
for service in [
"postgresql-%s" % (POSTGRES_VERSION, ), "rabbitmq-server",
"memcached", "redis"
]:
run_as_root(["systemctl", "enable", service], sudo_args=['-H'])
run_as_root(["systemctl", "start", service], sudo_args=['-H'])
if not options.is_production_travis:
# The following block is skipped for the production Travis
# suite, because that suite doesn't make use of these elements
# of the development environment (it just uses the development
# environment to build a release tarball).
# Need to set up Django before using template_database_status
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
import django
django.setup()
from zerver.lib.test_fixtures import template_database_status, run_db_migrations, \
destroy_leaked_test_databases
try:
from zerver.lib.queue import SimpleQueueClient
SimpleQueueClient()
rabbitmq_is_configured = True
except Exception:
rabbitmq_is_configured = False
if options.is_force or not rabbitmq_is_configured:
run(["scripts/setup/configure-rabbitmq"])
else:
print("RabbitMQ is already configured.")
migration_status_path = os.path.join(UUID_VAR_PATH,
"migration_status_dev")
dev_template_db_status = template_database_status(
migration_status=migration_status_path,
settings="zproject.settings",
database_name="zulip",
)
if options.is_force or dev_template_db_status == 'needs_rebuild':
run(["tools/setup/postgres-init-dev-db"])
run(["tools/do-destroy-rebuild-database"])
elif dev_template_db_status == 'run_migrations':
run_db_migrations('dev')
elif dev_template_db_status == 'current':
print("No need to regenerate the dev DB.")
test_template_db_status = template_database_status()
if options.is_force or test_template_db_status == 'needs_rebuild':
run(["tools/setup/postgres-init-test-db"])
run(["tools/do-destroy-rebuild-test-database"])
elif test_template_db_status == 'run_migrations':
run_db_migrations('test')
elif test_template_db_status == 'current':
print("No need to regenerate the test DB.")
# Consider updating generated translations data: both `.mo`
# files and `language-options.json`.
paths = ['zerver/management/commands/compilemessages.py']
paths += glob.glob('locale/*/LC_MESSAGES/*.po')
paths += glob.glob('locale/*/translations.json')
if file_or_package_hash_updated(paths, "last_compilemessages_hash",
options.is_force):
run(["./manage.py", "compilemessages"])
else:
print("No need to run `manage.py compilemessages`.")
destroyed = destroy_leaked_test_databases()
if destroyed:
print("Dropped %s stale test databases!" % (destroyed, ))
run(["scripts/lib/clean-unused-caches"])
# Keeping this cache file around can cause eslint to throw
# random TypeErrors when new/updated dependencies are added
if os.path.isfile('.eslintcache'):
# Remove this block when
# https://github.com/eslint/eslint/issues/11639 is fixed
# upstream.
os.remove('.eslintcache')
# Clean up the root of the `var/` directory for various
# testing-related files that we have migrated to
# `var/<uuid>/test-backend`.
print("Cleaning var/ directory files...")
var_paths = glob.glob('var/test*')
var_paths.append('var/bot_avatar')
for path in var_paths:
try:
if os.path.isdir(path):
shutil.rmtree(path)
else:
os.remove(path)
except FileNotFoundError:
pass
version_file = os.path.join(UUID_VAR_PATH, 'provision_version')
print('writing to %s\n' % (version_file, ))
open(version_file, 'w').write(PROVISION_VERSION + '\n')
print()
print(OKBLUE + "Zulip development environment setup succeeded!" + ENDC)
return 0
def main(options: argparse.Namespace) -> int:
setup_bash_profile()
setup_shell_profile('~/.zprofile')
# This needs to happen before anything that imports zproject.settings.
run(["scripts/setup/generate_secrets.py", "--development"])
create_var_directories()
# The `build_emoji` script requires `emoji-datasource` package
# which we install via npm; thus this step is after installing npm
# packages.
run(["tools/setup/emoji/build_emoji"])
# copy over static files from the zulip_bots package
generate_zulip_bots_static_files()
if options.is_force or need_to_run_build_pygments_data():
run(["tools/setup/build_pygments_data"])
write_new_digest(
'build_pygments_data_hash',
build_pygments_data_paths(),
[pygments_version]
)
else:
print("No need to run `tools/setup/build_pygments_data`.")
if options.is_force or need_to_run_inline_email_css():
run(["scripts/setup/inline_email_css.py"])
write_new_digest(
"last_email_source_files_hash",
inline_email_css_paths(),
)
else:
print("No need to run `scripts/setup/inline_email_css.py`.")
if not options.is_build_release_tarball_only:
# The following block is skipped when we just need the development
# environment to build a release tarball.
# Need to set up Django before using template_status
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
import django
django.setup()
from zerver.lib.test_fixtures import (
DEV_DATABASE,
TEST_DATABASE,
destroy_leaked_test_databases,
)
from django.conf import settings
if options.is_force or need_to_run_configure_rabbitmq(
[settings.RABBITMQ_PASSWORD]):
run(["scripts/setup/configure-rabbitmq"])
write_new_digest(
'last_configure_rabbitmq_hash',
configure_rabbitmq_paths(),
[settings.RABBITMQ_PASSWORD]
)
else:
print("No need to run `scripts/setup/configure-rabbitmq.")
dev_template_db_status = DEV_DATABASE.template_status()
if options.is_force or dev_template_db_status == 'needs_rebuild':
run(["tools/setup/postgres-init-dev-db"])
run(["tools/rebuild-dev-database"])
DEV_DATABASE.write_new_db_digest()
elif dev_template_db_status == 'run_migrations':
DEV_DATABASE.run_db_migrations()
elif dev_template_db_status == 'current':
print("No need to regenerate the dev DB.")
test_template_db_status = TEST_DATABASE.template_status()
if options.is_force or test_template_db_status == 'needs_rebuild':
run(["tools/setup/postgres-init-test-db"])
run(["tools/rebuild-test-database"])
TEST_DATABASE.write_new_db_digest()
elif test_template_db_status == 'run_migrations':
TEST_DATABASE.run_db_migrations()
elif test_template_db_status == 'current':
print("No need to regenerate the test DB.")
if options.is_force or need_to_run_compilemessages():
run(["./manage.py", "compilemessages"])
write_new_digest(
"last_compilemessages_hash",
compilemessages_paths(),
)
else:
print("No need to run `manage.py compilemessages`.")
destroyed = destroy_leaked_test_databases()
if destroyed:
print("Dropped %s stale test databases!" % (destroyed,))
clean_unused_caches()
# Keeping this cache file around can cause eslint to throw
# random TypeErrors when new/updated dependencies are added
if os.path.isfile('.eslintcache'):
# Remove this block when
# https://github.com/eslint/eslint/issues/11639 is fixed
# upstream.
os.remove('.eslintcache')
# Clean up the root of the `var/` directory for various
# testing-related files that we have migrated to
# `var/<uuid>/test-backend`.
print("Cleaning var/ directory files...")
var_paths = glob.glob('var/test*')
var_paths.append('var/bot_avatar')
for path in var_paths:
try:
if os.path.isdir(path):
shutil.rmtree(path)
else:
os.remove(path)
except FileNotFoundError:
pass
version_file = os.path.join(UUID_VAR_PATH, 'provision_version')
print('writing to %s\n' % (version_file,))
open(version_file, 'w').write(PROVISION_VERSION + '\n')
print()
print(OKBLUE + "Zulip development environment setup succeeded!" + ENDC)
return 0
def main(options):
# type: (Any) -> int
# yarn and management commands expect to be run from the root of the
# project.
os.chdir(ZULIP_PATH)
# hash the apt dependencies
sha_sum = hashlib.sha1()
for apt_depedency in SYSTEM_DEPENDENCIES[codename]:
sha_sum.update(apt_depedency.encode('utf8'))
# hash the content of setup-apt-repo
sha_sum.update(open('scripts/lib/setup-apt-repo', 'rb').read())
new_apt_dependencies_hash = sha_sum.hexdigest()
last_apt_dependencies_hash = None
apt_hash_file_path = os.path.join(UUID_VAR_PATH, "apt_dependencies_hash")
with open(apt_hash_file_path, 'a+') as hash_file:
hash_file.seek(0)
last_apt_dependencies_hash = hash_file.read()
if (new_apt_dependencies_hash != last_apt_dependencies_hash):
try:
install_system_deps()
except subprocess.CalledProcessError:
# Might be a failure due to network connection issues. Retrying...
install_system_deps(retry=True)
with open(apt_hash_file_path, 'w') as hash_file:
hash_file.write(new_apt_dependencies_hash)
else:
print("No changes to apt dependencies, so skipping apt operations.")
# Here we install node.
proxy_env = [
"env",
"http_proxy=" + os.environ.get("http_proxy", ""),
"https_proxy=" + os.environ.get("https_proxy", ""),
"no_proxy=" + os.environ.get("no_proxy", ""),
]
run(["sudo", "-H"] + proxy_env + ["scripts/lib/install-node"])
# This is a wrapper around `yarn`, which we run last since
# it can often fail due to network issues beyond our control.
try:
# Hack: We remove `node_modules` as root to work around an
# issue with the symlinks being improperly owned by root.
if os.path.islink("node_modules"):
run(["sudo", "rm", "-f", "node_modules"])
run(["sudo", "mkdir", "-p", NODE_MODULES_CACHE_PATH])
run([
"sudo", "chown",
"%s:%s" % (user_id, user_id), NODE_MODULES_CACHE_PATH
])
setup_node_modules(prefer_offline=True)
except subprocess.CalledProcessError:
print(WARNING + "`yarn install` failed; retrying..." + ENDC)
setup_node_modules()
# Install shellcheck.
run(["sudo", "scripts/lib/install-shellcheck"])
# Import tools/setup_venv.py instead of running it so that we get an
# activated virtualenv for the rest of the provisioning process.
from tools.setup import setup_venvs
setup_venvs.main()
setup_shell_profile('~/.bash_profile')
setup_shell_profile('~/.zprofile')
run(["sudo", "cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])
# create log directory `zulip/var/log`
os.makedirs(LOG_DIR_PATH, exist_ok=True)
# create upload directory `var/uploads`
os.makedirs(UPLOAD_DIR_PATH, exist_ok=True)
# create test upload directory `var/test_upload`
os.makedirs(TEST_UPLOAD_DIR_PATH, exist_ok=True)
# create coverage directory`var/coverage`
os.makedirs(COVERAGE_DIR_PATH, exist_ok=True)
# create linecoverage directory`var/node-coverage`
os.makedirs(NODE_TEST_COVERAGE_DIR_PATH, exist_ok=True)
# `build_emoji` script requires `emoji-datasource` package which we install
# via npm and hence it should be executed after we are done installing npm
# packages.
if not os.path.isdir(EMOJI_CACHE_PATH):
run(["sudo", "mkdir", EMOJI_CACHE_PATH])
run(["sudo", "chown", "%s:%s" % (user_id, user_id), EMOJI_CACHE_PATH])
run(["tools/setup/emoji/build_emoji"])
# copy over static files from the zulip_bots package
generate_zulip_bots_static_files()
webfont_paths = [
"tools/setup/generate-custom-icon-webfont",
"static/icons/fonts/template.hbs"
]
webfont_paths += glob.glob('static/assets/icons/*')
if file_or_package_hash_updated(webfont_paths, "webfont_files_hash",
options.is_force):
run(["tools/setup/generate-custom-icon-webfont"])
else:
print("No need to run `tools/setup/generate-custom-icon-webfont`.")
build_pygments_data_paths = [
"tools/setup/build_pygments_data", "tools/setup/lang.json"
]
from pygments import __version__ as pygments_version
if file_or_package_hash_updated(build_pygments_data_paths,
"build_pygments_data_hash",
options.is_force, [pygments_version]):
run(["tools/setup/build_pygments_data"])
else:
print("No need to run `tools/setup/build_pygments_data`.")
run(["scripts/setup/generate_secrets.py", "--development"])
update_authors_json_paths = [
"tools/update-authors-json", "zerver/tests/fixtures/authors.json"
]
if file_or_package_hash_updated(update_authors_json_paths,
"update_authors_json_hash",
options.is_force):
run(["tools/update-authors-json", "--use-fixture"])
else:
print("No need to run `tools/update-authors-json`.")
email_source_paths = [
"tools/inline-email-css", "templates/zerver/emails/email.css"
]
email_source_paths += glob.glob('templates/zerver/emails/*.source.html')
if file_or_package_hash_updated(email_source_paths,
"last_email_source_files_hash",
options.is_force):
run(["tools/inline-email-css"])
else:
print("No need to run `tools/inline-email-css`.")
if is_circleci or (is_travis and not options.is_production_travis):
run(["sudo", "service", "rabbitmq-server", "restart"])
run(["sudo", "service", "redis-server", "restart"])
run(["sudo", "service", "memcached", "restart"])
run(["sudo", "service", "postgresql", "restart"])
elif vendor == 'CentOS':
for service in [
"postgresql-%s" % (POSTGRES_VERSION, ), "rabbitmq server",
"memcached", "redis"
]:
run(["sudo", "-H", "systemctl", "enable", service])
run(["sudo", "-H", "systemctl", "start", service])
elif options.is_docker:
run(["sudo", "service", "rabbitmq-server", "restart"])
run(["sudo", "pg_dropcluster", "--stop", POSTGRES_VERSION, "main"])
run([
"sudo", "pg_createcluster", "-e", "utf8", "--start",
POSTGRES_VERSION, "main"
])
run(["sudo", "service", "redis-server", "restart"])
run(["sudo", "service", "memcached", "restart"])
if not options.is_production_travis:
# The following block is skipped for the production Travis
# suite, because that suite doesn't make use of these elements
# of the development environment (it just uses the development
# environment to build a release tarball).
# Need to set up Django before using template_database_status
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
import django
django.setup()
from zerver.lib.test_fixtures import template_database_status, run_db_migrations
try:
from zerver.lib.queue import SimpleQueueClient
SimpleQueueClient()
rabbitmq_is_configured = True
except Exception:
rabbitmq_is_configured = False
if options.is_force or not rabbitmq_is_configured:
run(["scripts/setup/configure-rabbitmq"])
else:
print("RabbitMQ is already configured.")
migration_status_path = os.path.join(UUID_VAR_PATH,
"migration_status_dev")
dev_template_db_status = template_database_status(
migration_status=migration_status_path,
settings="zproject.settings",
database_name="zulip",
)
if options.is_force or dev_template_db_status == 'needs_rebuild':
run(["tools/setup/postgres-init-dev-db"])
run(["tools/do-destroy-rebuild-database"])
elif dev_template_db_status == 'run_migrations':
run_db_migrations('dev')
elif dev_template_db_status == 'current':
print("No need to regenerate the dev DB.")
test_template_db_status = template_database_status()
if options.is_force or test_template_db_status == 'needs_rebuild':
run(["tools/setup/postgres-init-test-db"])
run(["tools/do-destroy-rebuild-test-database"])
elif test_template_db_status == 'run_migrations':
run_db_migrations('test')
elif test_template_db_status == 'current':
print("No need to regenerate the test DB.")
# Consider updating generated translations data: both `.mo`
# files and `language-options.json`.
paths = ['zerver/management/commands/compilemessages.py']
paths += glob.glob('static/locale/*/LC_MESSAGES/*.po')
paths += glob.glob('static/locale/*/translations.json')
if file_or_package_hash_updated(paths, "last_compilemessages_hash",
options.is_force):
run(["./manage.py", "compilemessages"])
else:
print("No need to run `manage.py compilemessages`.")
run(["scripts/lib/clean-unused-caches"])
version_file = os.path.join(UUID_VAR_PATH, 'provision_version')
print('writing to %s\n' % (version_file, ))
open(version_file, 'w').write(PROVISION_VERSION + '\n')
print()
print(OKBLUE + "Zulip development environment setup succeeded!" + ENDC)
return 0
def do_setup_virtualenv(venv_path, requirements_file, virtualenv_args):
# type: (str, str, List[str]) -> None
# Setup Python virtualenv
new_packages = set(get_package_names(requirements_file))
run(["sudo", "rm", "-rf", venv_path])
if not try_to_copy_venv(venv_path, new_packages):
# Create new virtualenv.
run(["sudo", "mkdir", "-p", venv_path])
run(["sudo", "virtualenv"] + virtualenv_args + [venv_path])
run([
"sudo", "chown", "-R", "{}:{}".format(os.getuid(), os.getgid()),
venv_path
])
create_log_entry(get_logfile_name(venv_path), "", set(), new_packages)
create_requirements_index_file(venv_path, requirements_file)
# Switch current Python context to the virtualenv.
activate_this = os.path.join(venv_path, "bin", "activate_this.py")
exec(open(activate_this).read(), {}, dict(__file__=activate_this)
) # type: ignore # https://github.com/python/mypy/issues/1577
run(["pip", "install", "-U", "setuptools"])
run(["pip", "install", "--upgrade", "pip", "wheel"])
run(["pip", "install", "--no-deps", "--requirement", requirements_file])
run(["sudo", "chmod", "-R", "a+rX", venv_path])
def try_to_copy_venv(venv_path, new_packages):
# type: (str, Set[str]) -> bool
"""
Tries to copy packages from an old virtual environment in the cache
to the new virtual environment. The algorithm works as follows:
1. Find a virtual environment, v, from the cache that has the
highest overlap with the new requirements such that:
a. The new requirements only add to the packages of v.
b. The new requirements only upgrade packages of v.
2. Copy the contents of v to the new virtual environment using
virtualenv-clone.
3. Delete all .pyc files in the new virtual environment.
"""
if not os.path.exists(VENV_CACHE_PATH):
return False
venv_name = os.path.basename(venv_path)
overlaps = [] # type: List[Tuple[int, str, Set[str]]]
old_packages = set() # type: Set[str]
for sha1sum in os.listdir(VENV_CACHE_PATH):
curr_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, venv_name)
if (curr_venv_path == venv_path
or not os.path.exists(get_index_filename(curr_venv_path))):
continue
old_packages = get_venv_packages(curr_venv_path)
# We only consider using using old virtualenvs that only
# contain packages that we want in our new virtualenv.
if not (old_packages - new_packages):
overlap = new_packages & old_packages
overlaps.append((len(overlap), curr_venv_path, overlap))
target_log = get_logfile_name(venv_path)
source_venv_path = None
if overlaps:
# Here, we select the old virtualenv with the largest overlap
overlaps = sorted(overlaps)
_, source_venv_path, copied_packages = overlaps[-1]
print('Copying packages from {}'.format(source_venv_path))
clone_ve = "{}/bin/virtualenv-clone".format(source_venv_path)
cmd = "sudo {exe} {source} {target}".format(exe=clone_ve,
source=source_venv_path,
target=venv_path).split()
try:
run(cmd)
except Exception:
# Virtualenv-clone is not installed. Install it and try running
# the command again.
try:
run("{}/bin/pip install --no-deps virtualenv-clone".format(
source_venv_path).split())
run(cmd)
except Exception:
# virtualenv-clone isn't working, so just make a new venv
return False
run([
"sudo", "chown", "-R", "{}:{}".format(os.getuid(), os.getgid()),
venv_path
])
source_log = get_logfile_name(source_venv_path)
copy_parent_log(source_log, target_log)
create_log_entry(target_log, source_venv_path, copied_packages,
new_packages - copied_packages)
return True
return False
def handle(self, *args: Any, **options: Any) -> None:
timestamp = timezone_now().strftime(TIMESTAMP_FORMAT)
with tempfile.TemporaryDirectory(
prefix="zulip-backup-%s-" % (timestamp,)
) as tmp:
os.mkdir(os.path.join(tmp, "zulip-backup"))
members = []
with open(os.path.join(tmp, "zulip-backup", "zulip-version"), "w") as f:
print(ZULIP_VERSION, file=f)
git = try_git_describe()
if git:
print(git, file=f)
members.append("zulip-backup/zulip-version")
with open(os.path.join(tmp, "zulip-backup", "os-version"), "w") as f:
print(
"{DISTRIB_ID} {DISTRIB_CODENAME}".format(**parse_lsb_release()),
file=f,
)
members.append("zulip-backup/os-version")
with open(os.path.join(tmp, "zulip-backup", "postgres-version"), "w") as f:
print(connection.pg_version, file=f)
members.append("zulip-backup/postgres-version")
if settings.DEVELOPMENT:
os.symlink(
os.path.join(settings.DEPLOY_ROOT, "zproject"),
os.path.join(tmp, "zulip-backup", "zproject"),
)
members.append("zulip-backup/zproject/dev-secrets.conf")
else:
os.symlink("/etc/zulip", os.path.join(tmp, "zulip-backup", "settings"))
members.append("zulip-backup/settings")
db_name = settings.DATABASES["default"]["NAME"]
db_dir = os.path.join(tmp, "zulip-backup", "database")
run(["pg_dump", "--format=directory", db_name, "--file", db_dir])
members.append("zulip-backup/database")
if settings.LOCAL_UPLOADS_DIR is not None and os.path.exists(
os.path.join(settings.DEPLOY_ROOT, settings.LOCAL_UPLOADS_DIR)
):
os.symlink(
os.path.join(settings.DEPLOY_ROOT, settings.LOCAL_UPLOADS_DIR),
os.path.join(tmp, "zulip-backup", "uploads"),
)
members.append("zulip-backup/uploads")
try:
if options["output"] is None:
tarball_path = tempfile.NamedTemporaryFile(
prefix="zulip-backup-%s-" % (timestamp,),
suffix=".tar.gz",
delete=False,
).name
else:
tarball_path = options["output"]
run(["tar", "-C", tmp, "-chzf", tarball_path, "--"] + members)
print("Backup tarball written to %s" % (tarball_path,))
except BaseException:
if options["output"] is None:
os.unlink(tarball_path)
raise
def install_venv_deps(pip: str, requirements_file: str, python2: bool) -> None:
pip_requirements = os.path.join(ZULIP_PATH, "requirements", "pip2.txt" if python2 else "pip.txt")
run([pip, "install", "--force-reinstall", "--require-hashes", "--requirement", pip_requirements])
run([pip, "install", "--no-deps", "--require-hashes", "--requirement", requirements_file])
def install_venv_deps(requirements_file):
# type: (str) -> None
pip_requirements = os.path.join(ZULIP_PATH, "requirements", "pip.txt")
run(["pip", "install", "-U", "--requirement", pip_requirements])
run(["pip", "install", "--no-deps", "--requirement", requirements_file])
def try_to_copy_venv(venv_path, new_packages):
# type: (str, Set[str]) -> bool
"""
Tries to copy packages from an old virtual environment in the cache
to the new virtual environment. The algorithm works as follows:
1. Find a virtual environment, v, from the cache that has the
highest overlap with the new requirements such that:
a. The new requirements only add to the packages of v.
b. The new requirements only upgrade packages of v.
2. Copy the contents of v to the new virtual environment using
virtualenv-clone.
3. Delete all .pyc files in the new virtual environment.
"""
if not os.path.exists(VENV_CACHE_PATH):
return False
venv_name = os.path.basename(venv_path)
overlaps = [] # type: List[Tuple[int, str, Set[str]]]
old_packages = set() # type: Set[str]
for sha1sum in os.listdir(VENV_CACHE_PATH):
curr_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, venv_name)
if (curr_venv_path == venv_path
or not os.path.exists(get_index_filename(curr_venv_path))):
continue
old_packages = get_venv_packages(curr_venv_path)
# We only consider using using old virtualenvs that only
# contain packages that we want in our new virtualenv.
if not (old_packages - new_packages):
overlap = new_packages & old_packages
overlaps.append((len(overlap), curr_venv_path, overlap))
target_log = get_logfile_name(venv_path)
source_venv_path = None
if overlaps:
# Here, we select the old virtualenv with the largest overlap
overlaps = sorted(overlaps)
_, source_venv_path, copied_packages = overlaps[-1]
print('Copying packages from {}'.format(source_venv_path))
clone_ve = "{}/bin/virtualenv-clone".format(source_venv_path)
cmd = "sudo {exe} {source} {target}".format(exe=clone_ve,
source=source_venv_path,
target=venv_path).split()
try:
# TODO: We can probably remove this in a few months, now
# that we can expect that virtualenv-clone is present in
# all of our recent virtualenvs.
run(cmd)
except Exception:
# Virtualenv-clone is not installed. Install it and try running
# the command again.
try:
run("{}/bin/pip install --no-deps virtualenv-clone".format(
source_venv_path).split())
run(cmd)
except Exception:
# virtualenv-clone isn't working, so just make a new venv
return False
# virtualenv-clone, unfortunately, copies the success stamp,
# which means if the upcoming `pip install` phase were to
# fail, we'd end up with a broken half-provisioned virtualenv
# that's incorrectly tagged as properly provisioned. The
# right fix is to use
# https://github.com/edwardgeorge/virtualenv-clone/pull/38,
# but this rm is almost as good.
success_stamp_path = os.path.join(venv_path, 'success-stamp')
run_as_root(["rm", "-f", success_stamp_path])
run_as_root([
"chown", "-R", "{}:{}".format(os.getuid(), os.getgid()), venv_path
])
source_log = get_logfile_name(source_venv_path)
copy_parent_log(source_log, target_log)
create_log_entry(target_log, source_venv_path, copied_packages,
new_packages - copied_packages)
return True
return False
def add_cert_to_pipconf():
# type: () -> None
conffile = os.path.expanduser("~/.pip/pip.conf")
confdir = os.path.expanduser("~/.pip/")
os.makedirs(confdir, exist_ok=True)
run(["crudini", "--set", conffile, "global", "cert", os.environ["CUSTOM_CA_CERTIFICATES"]])
def main(options: argparse.Namespace) -> int:
setup_bash_profile()
setup_shell_profile('~/.zprofile')
# This needs to happen before anything that imports zproject.settings.
run(["scripts/setup/generate_secrets.py", "--development"])
create_var_directories()
# The `build_emoji` script requires `emoji-datasource` package
# which we install via npm; thus this step is after installing npm
# packages.
run(["tools/setup/emoji/build_emoji"])
# copy over static files from the zulip_bots package
generate_zulip_bots_static_files()
if options.is_force or need_to_run_build_pygments_data():
run(["tools/setup/build_pygments_data"])
else:
print("No need to run `tools/setup/build_pygments_data`.")
if options.is_force or need_to_run_inline_email_css():
run(["scripts/setup/inline_email_css.py"])
else:
print("No need to run `scripts/setup/inline_email_css.py`.")
if not options.is_production_travis:
# The following block is skipped for the production Travis
# suite, because that suite doesn't make use of these elements
# of the development environment (it just uses the development
# environment to build a release tarball).
# Need to set up Django before using template_database_status
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
import django
django.setup()
from zerver.lib.test_fixtures import template_database_status, run_db_migrations, \
destroy_leaked_test_databases
try:
from zerver.lib.queue import SimpleQueueClient
SimpleQueueClient()
rabbitmq_is_configured = True
except Exception:
rabbitmq_is_configured = False
if options.is_force or not rabbitmq_is_configured:
run(["scripts/setup/configure-rabbitmq"])
else:
print("No need to run `scripts/setup/configure-rabbitmq.")
dev_template_db_status = template_database_status('dev')
if options.is_force or dev_template_db_status == 'needs_rebuild':
run(["tools/setup/postgres-init-dev-db"])
run(["tools/do-destroy-rebuild-database"])
elif dev_template_db_status == 'run_migrations':
run_db_migrations('dev')
elif dev_template_db_status == 'current':
print("No need to regenerate the dev DB.")
test_template_db_status = template_database_status('test')
if options.is_force or test_template_db_status == 'needs_rebuild':
run(["tools/setup/postgres-init-test-db"])
run(["tools/do-destroy-rebuild-test-database"])
elif test_template_db_status == 'run_migrations':
run_db_migrations('test')
elif test_template_db_status == 'current':
print("No need to regenerate the test DB.")
if options.is_force or need_to_run_compilemessages():
run(["./manage.py", "compilemessages"])
else:
print("No need to run `manage.py compilemessages`.")
destroyed = destroy_leaked_test_databases()
if destroyed:
print("Dropped %s stale test databases!" % (destroyed, ))
run(["scripts/lib/clean-unused-caches", "--threshold=6"])
# Keeping this cache file around can cause eslint to throw
# random TypeErrors when new/updated dependencies are added
if os.path.isfile('.eslintcache'):
# Remove this block when
# https://github.com/eslint/eslint/issues/11639 is fixed
# upstream.
os.remove('.eslintcache')
# Clean up the root of the `var/` directory for various
# testing-related files that we have migrated to
# `var/<uuid>/test-backend`.
print("Cleaning var/ directory files...")
var_paths = glob.glob('var/test*')
var_paths.append('var/bot_avatar')
for path in var_paths:
try:
if os.path.isdir(path):
shutil.rmtree(path)
else:
os.remove(path)
except FileNotFoundError:
pass
version_file = os.path.join(UUID_VAR_PATH, 'provision_version')
print('writing to %s\n' % (version_file, ))
open(version_file, 'w').write(PROVISION_VERSION + '\n')
print()
print(OKBLUE + "Zulip development environment setup succeeded!" + ENDC)
return 0
def main(options):
# type: (Any) -> int
# yarn and management commands expect to be run from the root of the
# project.
os.chdir(ZULIP_PATH)
# hash the apt dependencies
sha_sum = hashlib.sha1()
for apt_depedency in SYSTEM_DEPENDENCIES:
sha_sum.update(apt_depedency.encode('utf8'))
if vendor in ["Ubuntu", "Debian"]:
sha_sum.update(open('scripts/lib/setup-apt-repo', 'rb').read())
else:
# hash the content of setup-yum-repo and build-*
sha_sum.update(open('scripts/lib/setup-yum-repo', 'rb').read())
build_paths = glob.glob("scripts/lib/build-")
for bp in build_paths:
sha_sum.update(open(bp, 'rb').read())
new_apt_dependencies_hash = sha_sum.hexdigest()
last_apt_dependencies_hash = None
apt_hash_file_path = os.path.join(UUID_VAR_PATH, "apt_dependencies_hash")
with open(apt_hash_file_path, 'a+') as hash_file:
hash_file.seek(0)
last_apt_dependencies_hash = hash_file.read()
if (new_apt_dependencies_hash != last_apt_dependencies_hash):
try:
install_system_deps()
except subprocess.CalledProcessError:
# Might be a failure due to network connection issues. Retrying...
install_system_deps(retry=True)
with open(apt_hash_file_path, 'w') as hash_file:
hash_file.write(new_apt_dependencies_hash)
else:
print("No changes to apt dependencies, so skipping apt operations.")
# Here we install node.
proxy_env = [
"env",
"http_proxy=" + os.environ.get("http_proxy", ""),
"https_proxy=" + os.environ.get("https_proxy", ""),
"no_proxy=" + os.environ.get("no_proxy", ""),
]
run_as_root(proxy_env + ["scripts/lib/install-node"], sudo_args = ['-H'])
# This is a wrapper around `yarn`, which we run last since
# it can often fail due to network issues beyond our control.
try:
# Hack: We remove `node_modules` as root to work around an
# issue with the symlinks being improperly owned by root.
if os.path.islink("node_modules"):
run_as_root(["rm", "-f", "node_modules"])
run_as_root(["mkdir", "-p", NODE_MODULES_CACHE_PATH])
run_as_root(["chown", "%s:%s" % (user_id, user_id), NODE_MODULES_CACHE_PATH])
setup_node_modules(prefer_offline=True)
except subprocess.CalledProcessError:
print(WARNING + "`yarn install` failed; retrying..." + ENDC)
setup_node_modules()
# Install shellcheck.
run_as_root(["scripts/lib/install-shellcheck"])
from tools.setup import setup_venvs
setup_venvs.main()
activate_this = "/srv/zulip-py3-venv/bin/activate_this.py"
exec(open(activate_this).read(), {}, dict(__file__=activate_this))
setup_shell_profile('~/.bash_profile')
setup_shell_profile('~/.zprofile')
# This needs to happen before anything that imports zproject.settings.
run(["scripts/setup/generate_secrets.py", "--development"])
run_as_root(["cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])
# create log directory `zulip/var/log`
os.makedirs(LOG_DIR_PATH, exist_ok=True)
# create upload directory `var/uploads`
os.makedirs(UPLOAD_DIR_PATH, exist_ok=True)
# create test upload directory `var/test_upload`
os.makedirs(TEST_UPLOAD_DIR_PATH, exist_ok=True)
# create coverage directory `var/coverage`
os.makedirs(COVERAGE_DIR_PATH, exist_ok=True)
# create linecoverage directory `var/node-coverage`
os.makedirs(NODE_TEST_COVERAGE_DIR_PATH, exist_ok=True)
# The `build_emoji` script requires `emoji-datasource` package
# which we install via npm; thus this step is after installing npm
# packages.
if not os.path.isdir(EMOJI_CACHE_PATH):
run_as_root(["mkdir", EMOJI_CACHE_PATH])
run_as_root(["chown", "%s:%s" % (user_id, user_id), EMOJI_CACHE_PATH])
run(["tools/setup/emoji/build_emoji"])
# copy over static files from the zulip_bots package
generate_zulip_bots_static_files()
webfont_paths = ["tools/setup/generate-custom-icon-webfont", "static/icons/fonts/template.hbs"]
webfont_paths += glob.glob('static/assets/icons/*')
if file_or_package_hash_updated(webfont_paths, "webfont_files_hash", options.is_force):
run(["tools/setup/generate-custom-icon-webfont"])
else:
print("No need to run `tools/setup/generate-custom-icon-webfont`.")
build_pygments_data_paths = ["tools/setup/build_pygments_data", "tools/setup/lang.json"]
from pygments import __version__ as pygments_version
if file_or_package_hash_updated(build_pygments_data_paths, "build_pygments_data_hash", options.is_force,
[pygments_version]):
run(["tools/setup/build_pygments_data"])
else:
print("No need to run `tools/setup/build_pygments_data`.")
update_authors_json_paths = ["tools/update-authors-json", "zerver/tests/fixtures/authors.json"]
if file_or_package_hash_updated(update_authors_json_paths, "update_authors_json_hash", options.is_force):
run(["tools/update-authors-json", "--use-fixture"])
else:
print("No need to run `tools/update-authors-json`.")
email_source_paths = ["tools/inline-email-css", "templates/zerver/emails/email.css"]
email_source_paths += glob.glob('templates/zerver/emails/*.source.html')
if file_or_package_hash_updated(email_source_paths, "last_email_source_files_hash", options.is_force):
run(["tools/inline-email-css"])
else:
print("No need to run `tools/inline-email-css`.")
if is_circleci or (is_travis and not options.is_production_travis):
run_as_root(["service", "rabbitmq-server", "restart"])
run_as_root(["service", "redis-server", "restart"])
run_as_root(["service", "memcached", "restart"])
run_as_root(["service", "postgresql", "restart"])
elif family == 'redhat':
for service in ["postgresql-%s" % (POSTGRES_VERSION,), "rabbitmq-server", "memcached", "redis"]:
run_as_root(["systemctl", "enable", service], sudo_args = ['-H'])
run_as_root(["systemctl", "start", service], sudo_args = ['-H'])
elif options.is_docker:
run_as_root(["service", "rabbitmq-server", "restart"])
run_as_root(["pg_dropcluster", "--stop", POSTGRES_VERSION, "main"])
run_as_root(["pg_createcluster", "-e", "utf8", "--start", POSTGRES_VERSION, "main"])
run_as_root(["service", "redis-server", "restart"])
run_as_root(["service", "memcached", "restart"])
if not options.is_production_travis:
# The following block is skipped for the production Travis
# suite, because that suite doesn't make use of these elements
# of the development environment (it just uses the development
# environment to build a release tarball).
# Need to set up Django before using template_database_status
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
import django
django.setup()
from zerver.lib.test_fixtures import template_database_status, run_db_migrations
try:
from zerver.lib.queue import SimpleQueueClient
SimpleQueueClient()
rabbitmq_is_configured = True
except Exception:
rabbitmq_is_configured = False
if options.is_force or not rabbitmq_is_configured:
run(["scripts/setup/configure-rabbitmq"])
else:
print("RabbitMQ is already configured.")
migration_status_path = os.path.join(UUID_VAR_PATH, "migration_status_dev")
dev_template_db_status = template_database_status(
migration_status=migration_status_path,
settings="zproject.settings",
database_name="zulip",
)
if options.is_force or dev_template_db_status == 'needs_rebuild':
run(["tools/setup/postgres-init-dev-db"])
run(["tools/do-destroy-rebuild-database"])
elif dev_template_db_status == 'run_migrations':
run_db_migrations('dev')
elif dev_template_db_status == 'current':
print("No need to regenerate the dev DB.")
test_template_db_status = template_database_status()
if options.is_force or test_template_db_status == 'needs_rebuild':
run(["tools/setup/postgres-init-test-db"])
run(["tools/do-destroy-rebuild-test-database"])
elif test_template_db_status == 'run_migrations':
run_db_migrations('test')
elif test_template_db_status == 'current':
print("No need to regenerate the test DB.")
# Consider updating generated translations data: both `.mo`
# files and `language-options.json`.
paths = ['zerver/management/commands/compilemessages.py']
paths += glob.glob('static/locale/*/LC_MESSAGES/*.po')
paths += glob.glob('static/locale/*/translations.json')
if file_or_package_hash_updated(paths, "last_compilemessages_hash", options.is_force):
run(["./manage.py", "compilemessages"])
else:
print("No need to run `manage.py compilemessages`.")
run(["scripts/lib/clean-unused-caches"])
version_file = os.path.join(UUID_VAR_PATH, 'provision_version')
print('writing to %s\n' % (version_file,))
open(version_file, 'w').write(PROVISION_VERSION + '\n')
print()
print(OKBLUE + "Zulip development environment setup succeeded!" + ENDC)
return 0
def do_setup_virtualenv(venv_path, requirements_file, virtualenv_args):
# type: (str, str, List[str]) -> None
# Setup Python virtualenv
run(["sudo", "rm", "-rf", venv_path])
run(["sudo", "mkdir", "-p", venv_path])
run(["sudo", "chown", "{}:{}".format(os.getuid(), os.getgid()), venv_path])
run(["virtualenv"] + virtualenv_args + [venv_path])
# Switch current Python context to the virtualenv.
activate_this = os.path.join(venv_path, "bin", "activate_this.py")
exec(open(activate_this).read(), {}, dict(__file__=activate_this)
) # type: ignore # https://github.com/python/mypy/issues/1577
run(["pip", "install", "-U", "setuptools"]),
run(["pip", "install", "--upgrade", "pip", "wheel"])
run(["pip", "install", "--no-deps", "--requirement", requirements_file])
run(["sudo", "chmod", "-R", "a+rX", venv_path])
