def setup_virtualenv(target_venv_path,
requirements_file,
virtualenv_args=None):
# type: (Optional[str], str, Optional[List[str]]) -> str
# Check if a cached version already exists
path = os.path.join(ZULIP_PATH, 'scripts', 'lib', 'hash_reqs.py')
output = subprocess.check_output([path, requirements_file])
sha1sum = output.split()[0]
if target_venv_path is None:
cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, 'venv')
else:
cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum,
os.path.basename(target_venv_path))
success_stamp = os.path.join(cached_venv_path, "success-stamp")
if not os.path.exists(success_stamp):
do_setup_virtualenv(cached_venv_path, requirements_file,
virtualenv_args or [])
run(["touch", success_stamp])
print("Using cached Python venv from %s" % (cached_venv_path, ))
if target_venv_path is not None:
run(["sudo", "ln", "-nsf", cached_venv_path, target_venv_path])
activate_this = os.path.join(cached_venv_path, "bin", "activate_this.py")
exec(open(activate_this).read(), {}, dict(__file__=activate_this)
) # type: ignore # https://github.com/python/mypy/issues/1577
return cached_venv_path
def setup_virtualenv(target_venv_path, requirements_file, virtualenv_args=[]):
# Check if a cached version already exists
output = subprocess.check_output(['sha1sum', requirements_file])
sha1sum = output.split()[0]
cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, os.path.basename(target_venv_path))
success_stamp = os.path.join(cached_venv_path, "success-stamp")
if not os.path.exists(success_stamp):
do_setup_virtualenv(cached_venv_path, requirements_file, virtualenv_args)
run(["touch", success_stamp])
print("Using cached Python venv from %s" % (cached_venv_path,))
run(["sudo", "ln", "-nsf", cached_venv_path, target_venv_path])
activate_this = os.path.join(target_venv_path, "bin", "activate_this.py")
exec(open(activate_this).read(), {}, dict(__file__=activate_this)) # type: ignore # https://github.com/python/mypy/issues/1577
def setup_virtualenv(target_venv_path, requirements_file, virtualenv_args=None):
# type: (str, str, List[str]) -> None
# Check if a cached version already exists
output = subprocess.check_output(['sha1sum', requirements_file])
sha1sum = output.split()[0]
cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, os.path.basename(target_venv_path))
success_stamp = os.path.join(cached_venv_path, "success-stamp")
if not os.path.exists(success_stamp):
do_setup_virtualenv(cached_venv_path, requirements_file, virtualenv_args or [])
run(["touch", success_stamp])
print("Using cached Python venv from %s" % (cached_venv_path,))
run(["sudo", "ln", "-nsf", cached_venv_path, target_venv_path])
activate_this = os.path.join(target_venv_path, "bin", "activate_this.py")
exec(open(activate_this).read(), {}, dict(__file__=activate_this)) # type: ignore # https://github.com/python/mypy/issues/1577
def install_npm():
# type: () -> None
if "--travis" not in sys.argv:
if subprocess.check_output(['npm', '--version']).strip() != NPM_VERSION:
run(["sudo", "npm", "install", "-g", "npm@{}".format(NPM_VERSION)])
return
run(['mkdir', '-p', TRAVIS_NODE_PATH])
npm_exe = os.path.join(TRAVIS_NODE_PATH, 'bin', 'npm')
travis_npm = subprocess.check_output(['which', 'npm']).strip()
if os.path.exists(npm_exe):
run(['sudo', 'ln', '-sf', npm_exe, travis_npm])
version = subprocess.check_output(['npm', '--version']).strip()
if os.path.exists(npm_exe) and version == NPM_VERSION:
print("Using cached npm")
return
run(["npm", "install", "-g", "--prefix", TRAVIS_NODE_PATH, "npm@{}".format(NPM_VERSION)])
run(['sudo', 'ln', '-sf', npm_exe, travis_npm])
def setup_virtualenv(target_venv_path, requirements_file, virtualenv_args=None):
# type: (Optional[str], str, Optional[List[str]]) -> str
# Check if a cached version already exists
path = os.path.join(ZULIP_PATH, 'scripts', 'lib', 'hash_reqs.py')
output = subprocess.check_output([path, requirements_file])
sha1sum = output.split()[0]
if target_venv_path is None:
cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, 'venv')
else:
cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, os.path.basename(target_venv_path))
success_stamp = os.path.join(cached_venv_path, "success-stamp")
if not os.path.exists(success_stamp):
do_setup_virtualenv(cached_venv_path, requirements_file, virtualenv_args or [])
run(["touch", success_stamp])
print("Using cached Python venv from %s" % (cached_venv_path,))
if target_venv_path is not None:
run(["sudo", "ln", "-nsf", cached_venv_path, target_venv_path])
activate_this = os.path.join(cached_venv_path, "bin", "activate_this.py")
exec(open(activate_this).read(), {}, dict(__file__=activate_this)) # type: ignore # https://github.com/python/mypy/issues/1577
return cached_venv_path
def install_npm():
# type: () -> None
if "--travis" not in sys.argv:
if subprocess.check_output(['npm', '--version']).strip() != NPM_VERSION:
run(["sudo", "npm", "install", "-g", "npm@{}".format(NPM_VERSION)])
return
run(['mkdir', '-p', TRAVIS_NODE_PATH])
npm_exe = os.path.join(TRAVIS_NODE_PATH, 'bin', 'npm')
travis_npm = subprocess.check_output(['which', 'npm']).strip()
if os.path.exists(npm_exe):
run(['sudo', 'ln', '-sf', npm_exe, travis_npm])
version = subprocess.check_output(['npm', '--version']).strip()
if os.path.exists(npm_exe) and version == NPM_VERSION:
print("Using cached npm")
return
run(["npm", "install", "-g", "--prefix", TRAVIS_NODE_PATH, "npm@{}".format(NPM_VERSION)])
run(['sudo', 'ln', '-sf', npm_exe, travis_npm])
def setup_node_modules():
output = subprocess.check_output(['sha1sum', 'package.json'])
sha1sum = output.split()[0]
success_stamp = os.path.join('node_modules', '.npm-success-stamp', sha1sum)
if not os.path.exists(success_stamp):
print("Deleting cached version")
run(["rm", "-rf", "node_modules"])
print("Installing node modules")
run(["npm", "install"])
run(["mkdir", "-p", success_stamp])
else:
print("Using cached version of node_modules")
def setup_node_modules():
output = subprocess.check_output(['sha1sum', 'package.json'])
sha1sum = output.split()[0]
success_stamp = os.path.join('node_modules', '.npm-success-stamp', sha1sum)
if not os.path.exists(success_stamp):
print("Deleting cached version")
run(["rm", "-rf", "node_modules"])
print("Installing node modules")
run(["npm", "install"])
run(["mkdir", "-p", success_stamp])
else:
print("Using cached version of node_modules")
def do_setup_virtualenv(venv_path, requirements_file, virtualenv_args):
# type: (str, str, List[str]) -> None
# Setup Python virtualenv
run(["sudo", "rm", "-rf", venv_path])
run(["sudo", "mkdir", "-p", venv_path])
run(["sudo", "chown", "{}:{}".format(os.getuid(), os.getgid()), venv_path])
run(["virtualenv"] + virtualenv_args + [venv_path])
# Switch current Python context to the virtualenv.
activate_this = os.path.join(venv_path, "bin", "activate_this.py")
exec(open(activate_this).read(), {}, dict(__file__=activate_this)
) # type: ignore # https://github.com/python/mypy/issues/1577
run(["pip", "install", "--upgrade", "pip"])
run(["pip", "install", "--no-deps", "--requirement", requirements_file])
run(["sudo", "chmod", "-R", "a+rX", venv_path])
def main():
run(["sudo", "apt-get", "update"])
run(["sudo", "apt-get", "-y", "install"] + APT_DEPENDENCIES[codename])
temp_deb_path = subprocess.check_output(
["mktemp", "package_XXXXXX.deb", "--tmpdir"])
run(["wget", "-O", temp_deb_path, TSEARCH_URL])
run(["sudo", "dpkg", "--install", temp_deb_path])
setup_virtualenv(PY3_VENV_PATH,
os.path.join(ZULIP_PATH, "tools", "setup",
"py3_test_reqs.txt"),
virtualenv_args=['-p', 'python3'])
setup_virtualenv(VENV_PATH, os.path.join(ZULIP_PATH, "requirements.txt"))
# Put Python2 virtualenv activation in our .bash_profile.
with open(os.path.expanduser('~/.bash_profile'), 'w+') as bash_profile:
bash_profile.writelines([
"source .bashrc\n",
"source %s\n" % (os.path.join(VENV_PATH, "bin", "activate"), ),
])
run(["sudo", "cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])
# npm install and management commands expect to be run from the root of the
# project.
os.chdir(ZULIP_PATH)
if "--travis" in sys.argv:
run(["tools/setup/install-phantomjs", "--travis"])
else:
run(["tools/setup/install-phantomjs"])
run(["tools/setup/download-zxcvbn"])
run(["tools/setup/emoji_dump/build_emoji"])
run(["scripts/setup/generate_secrets.py", "-d"])
if "--travis" in sys.argv:
run(["sudo", "service", "rabbitmq-server", "restart"])
run(["sudo", "service", "redis-server", "restart"])
run(["sudo", "service", "memcached", "restart"])
elif "--docker" in sys.argv:
run(["sudo", "service", "rabbitmq-server", "restart"])
run(["sudo", "pg_dropcluster", "--stop", POSTGRES_VERSION, "main"])
run([
"sudo", "pg_createcluster", "-e", "utf8", "--start",
POSTGRES_VERSION, "main"
])
run(["sudo", "service", "redis-server", "restart"])
run(["sudo", "service", "memcached", "restart"])
run(["scripts/setup/configure-rabbitmq"])
run(["tools/setup/postgres-init-dev-db"])
run(["tools/do-destroy-rebuild-database"])
run(["tools/setup/postgres-init-test-db"])
run(["tools/do-destroy-rebuild-test-database"])
run(["python", "./manage.py", "compilemessages"])
# Install the latest npm.
run(["sudo", "npm", "install", "-g", "npm"])
# Run npm install last because it can be flaky, and that way one
# only needs to rerun `npm install` to fix the installation.
run(["npm", "install"])
return 0
def main():
# type: () -> int
run(["sudo", "apt-get", "update"])
run(["sudo", "apt-get", "-y", "install"] + APT_DEPENDENCIES[codename])
if subprocess.call(['dpkg', '-s', TSEARCH_PACKAGE_NAME]):
temp_deb_path = subprocess.check_output(["mktemp", "package_XXXXXX.deb", "--tmpdir"])
run(["wget", "-O", temp_deb_path, TSEARCH_URL])
run(["sudo", "dpkg", "--install", temp_deb_path])
setup_virtualenv(PY3_VENV_PATH,
os.path.join(ZULIP_PATH, "requirements", "mypy.txt"),
virtualenv_args=['-p', 'python3'])
setup_virtualenv(VENV_PATH, os.path.join(ZULIP_PATH, "requirements.txt"))
# Put Python2 virtualenv activation in our .bash_profile.
with open(os.path.expanduser('~/.bash_profile'), 'w+') as bash_profile:
bash_profile.writelines([
"source .bashrc\n",
"source %s\n" % (os.path.join(VENV_PATH, "bin", "activate"),),
])
run(["sudo", "cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])
# npm install and management commands expect to be run from the root of the
# project.
os.chdir(ZULIP_PATH)
if "--travis" in sys.argv:
run(["tools/setup/install-phantomjs", "--travis"])
else:
run(["tools/setup/install-phantomjs"])
run(["tools/setup/download-zxcvbn"])
run(["tools/setup/emoji_dump/build_emoji"])
run(["scripts/setup/generate_secrets.py", "-d"])
if "--travis" in sys.argv:
run(["sudo", "service", "rabbitmq-server", "restart"])
run(["sudo", "service", "redis-server", "restart"])
run(["sudo", "service", "memcached", "restart"])
elif "--docker" in sys.argv:
run(["sudo", "service", "rabbitmq-server", "restart"])
run(["sudo", "pg_dropcluster", "--stop", POSTGRES_VERSION, "main"])
run(["sudo", "pg_createcluster", "-e", "utf8", "--start", POSTGRES_VERSION, "main"])
run(["sudo", "service", "redis-server", "restart"])
run(["sudo", "service", "memcached", "restart"])
run(["scripts/setup/configure-rabbitmq"])
run(["tools/setup/postgres-init-dev-db"])
run(["tools/do-destroy-rebuild-database"])
run(["tools/setup/postgres-init-test-db"])
run(["tools/do-destroy-rebuild-test-database"])
run(["python", "./manage.py", "compilemessages"])
# Install the pinned version of npm.
install_npm()
# Run npm install last because it can be flaky, and that way one
# only needs to rerun `npm install` to fix the installation.
setup_node_modules()
return 0
def do_setup_virtualenv(venv_path, requirements_file, virtualenv_args):
# type: (str, str, List[str]) -> None
# Setup Python virtualenv
run(["sudo", "rm", "-rf", venv_path])
run(["sudo", "mkdir", "-p", venv_path])
run(["sudo", "chown", "{}:{}".format(os.getuid(), os.getgid()), venv_path])
run(["virtualenv"] + virtualenv_args + [venv_path])
# Switch current Python context to the virtualenv.
activate_this = os.path.join(venv_path, "bin", "activate_this.py")
exec(open(activate_this).read(), {}, dict(__file__=activate_this)) # type: ignore # https://github.com/python/mypy/issues/1577
run(["pip", "install", "--upgrade", "pip"])
run(["pip", "install", "--no-deps", "--requirement", requirements_file])
def do_setup_virtualenv(venv_path, requirements_file, virtualenv_args):
# type: (str, str, List[str]) -> None
# Setup Python virtualenv
run(["sudo", "rm", "-rf", venv_path])
run(["sudo", "mkdir", "-p", venv_path])
run(["sudo", "chown", "{}:{}".format(os.getuid(), os.getgid()), venv_path])
run(["virtualenv"] + virtualenv_args + [venv_path])
# Switch current Python context to the virtualenv.
activate_this = os.path.join(venv_path, "bin", "activate_this.py")
exec(open(activate_this).read(), {}, dict(__file__=activate_this)) # type: ignore # https://github.com/python/mypy/issues/1577
run(["pip", "install", "--upgrade", "pip", "wheel"])
# Currently, Scikit-Learn and Numpy cannot be installed with pip in one
# install pass. See https://github.com/scikit-learn/scikit-learn/issues/4164
# for further details.
if 'dev.txt' in requirements_file or 'prod.txt' in requirements_file:
numpy = os.path.join(ZULIP_PATH, 'requirements', 'numpy.txt')
run(["pip", "install", "--no-deps", "--requirement", numpy])
run(["pip", "install", "--no-deps", "--requirement", requirements_file])
run(["sudo", "chmod", "-R", "a+rX", venv_path])
def do_setup_virtualenv(venv_path, requirements_file, virtualenv_args):
# Setup Python virtualenv
run(["sudo", "rm", "-rf", venv_path])
run(["sudo", "mkdir", "-p", venv_path])
run(["sudo", "chown", "{}:{}".format(os.getuid(), os.getgid()), venv_path])
run(["virtualenv"] + virtualenv_args + [venv_path])
# Switch current Python context to the virtualenv.
activate_this = os.path.join(venv_path, "bin", "activate_this.py")
execfile(activate_this, dict(__file__=activate_this))
run(["pip", "install", "--upgrade", "pip"])
run(["pip", "install", "--no-deps", "--requirement", requirements_file])
def main():
run(["sudo", "apt-get", "update"])
run(["sudo", "apt-get", "-y", "install"] + APT_DEPENDENCIES[codename])
temp_deb_path = subprocess.check_output(["mktemp", "package_XXXXXX.deb", "--tmpdir"])
run(["wget", "-O", temp_deb_path, TSEARCH_URL])
run(["sudo", "dpkg", "--install", temp_deb_path])
run(["sudo", "rm", "-rf", VENV_PATH])
run(["sudo", "mkdir", "-p", VENV_PATH])
run(["sudo", "chown", "{}:{}".format(os.getuid(), os.getgid()), VENV_PATH])
run(["virtualenv", VENV_PATH])
# Put Python virtualenv activation in our .bash_profile.
with open(os.path.expanduser('~/.bash_profile'), 'w+') as bash_profile:
bash_profile.writelines([
"source .bashrc\n",
"source %s\n" % (os.path.join(VENV_PATH, "bin", "activate"),),
])
# Switch current Python context to the virtualenv.
activate_this = os.path.join(VENV_PATH, "bin", "activate_this.py")
execfile(activate_this, dict(__file__=activate_this))
run(["pip", "install", "--no-deps", "--requirement",
os.path.join(ZULIP_PATH, "requirements.txt")])
run(["sudo", "cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])
# npm install and management commands expect to be run from the root of the
# project.
os.chdir(ZULIP_PATH)
run(["tools/install-phantomjs"])
run(["tools/download-zxcvbn"])
run(["tools/emoji_dump/build_emoji"])
run(["scripts/setup/generate_secrets.py", "-d"])
if "--travis" in sys.argv:
run(["sudo", "service", "rabbitmq-server", "restart"])
run(["sudo", "service", "redis-server", "restart"])
run(["sudo", "service", "memcached", "restart"])
elif "--docker" in sys.argv:
run(["sudo", "service", "rabbitmq-server", "restart"])
run(["sudo", "pg_dropcluster", "--stop", POSTGRES_VERSION, "main"])
run(["sudo", "pg_createcluster", "-e", "utf8", "--start", POSTGRES_VERSION, "main"])
run(["sudo", "service", "redis-server", "restart"])
run(["sudo", "service", "memcached", "restart"])
run(["scripts/setup/configure-rabbitmq"])
run(["tools/postgres-init-dev-db"])
run(["tools/do-destroy-rebuild-database"])
run(["tools/postgres-init-test-db"])
run(["tools/do-destroy-rebuild-test-database"])
# Install the latest npm.
run(["sudo", "npm", "install", "-g", "npm"])
# Run npm install last because it can be flaky, and that way one
# only needs to rerun `npm install` to fix the installation.
run(["npm", "install"])
return 0