Beispiel #1
0
def setup_virtualenv(target_venv_path,
                     requirements_file,
                     virtualenv_args=None):
    # type: (Optional[str], str, Optional[List[str]]) -> str

    # Check if a cached version already exists
    path = os.path.join(ZULIP_PATH, 'scripts', 'lib', 'hash_reqs.py')
    output = subprocess.check_output([path, requirements_file])
    sha1sum = output.split()[0]
    if target_venv_path is None:
        cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, 'venv')
    else:
        cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum,
                                        os.path.basename(target_venv_path))
    success_stamp = os.path.join(cached_venv_path, "success-stamp")
    if not os.path.exists(success_stamp):
        do_setup_virtualenv(cached_venv_path, requirements_file,
                            virtualenv_args or [])
        run(["touch", success_stamp])

    print("Using cached Python venv from %s" % (cached_venv_path, ))
    if target_venv_path is not None:
        run(["sudo", "ln", "-nsf", cached_venv_path, target_venv_path])
    activate_this = os.path.join(cached_venv_path, "bin", "activate_this.py")
    exec(open(activate_this).read(), {}, dict(__file__=activate_this)
         )  # type: ignore # https://github.com/python/mypy/issues/1577
    return cached_venv_path
Beispiel #2
0
def setup_virtualenv(target_venv_path, requirements_file, virtualenv_args=[]):
    # Check if a cached version already exists
    output = subprocess.check_output(['sha1sum', requirements_file])
    sha1sum = output.split()[0]
    cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, os.path.basename(target_venv_path))
    success_stamp = os.path.join(cached_venv_path, "success-stamp")
    if not os.path.exists(success_stamp):
        do_setup_virtualenv(cached_venv_path, requirements_file, virtualenv_args)
        run(["touch", success_stamp])

    print("Using cached Python venv from %s" % (cached_venv_path,))
    run(["sudo", "ln", "-nsf", cached_venv_path, target_venv_path])
    activate_this = os.path.join(target_venv_path, "bin", "activate_this.py")
    exec(open(activate_this).read(), {}, dict(__file__=activate_this)) # type: ignore # https://github.com/python/mypy/issues/1577
Beispiel #3
0
def setup_virtualenv(target_venv_path, requirements_file, virtualenv_args=None):
    # type: (str, str, List[str]) -> None

    # Check if a cached version already exists
    output = subprocess.check_output(['sha1sum', requirements_file])
    sha1sum = output.split()[0]
    cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, os.path.basename(target_venv_path))
    success_stamp = os.path.join(cached_venv_path, "success-stamp")
    if not os.path.exists(success_stamp):
        do_setup_virtualenv(cached_venv_path, requirements_file, virtualenv_args or [])
        run(["touch", success_stamp])

    print("Using cached Python venv from %s" % (cached_venv_path,))
    run(["sudo", "ln", "-nsf", cached_venv_path, target_venv_path])
    activate_this = os.path.join(target_venv_path, "bin", "activate_this.py")
    exec(open(activate_this).read(), {}, dict(__file__=activate_this)) # type: ignore # https://github.com/python/mypy/issues/1577
Beispiel #4
0
def install_npm():
    # type: () -> None
    if "--travis" not in sys.argv:
        if subprocess.check_output(['npm', '--version']).strip() != NPM_VERSION:
            run(["sudo", "npm", "install", "-g", "npm@{}".format(NPM_VERSION)])

        return

    run(['mkdir', '-p', TRAVIS_NODE_PATH])

    npm_exe = os.path.join(TRAVIS_NODE_PATH, 'bin', 'npm')
    travis_npm = subprocess.check_output(['which', 'npm']).strip()
    if os.path.exists(npm_exe):
        run(['sudo', 'ln', '-sf', npm_exe, travis_npm])

    version = subprocess.check_output(['npm', '--version']).strip()
    if os.path.exists(npm_exe) and version == NPM_VERSION:
        print("Using cached npm")
        return

    run(["npm", "install", "-g", "--prefix", TRAVIS_NODE_PATH, "npm@{}".format(NPM_VERSION)])
    run(['sudo', 'ln', '-sf', npm_exe, travis_npm])
Beispiel #5
0
def setup_virtualenv(target_venv_path, requirements_file, virtualenv_args=None):
    # type: (Optional[str], str, Optional[List[str]]) -> str

    # Check if a cached version already exists
    path = os.path.join(ZULIP_PATH, 'scripts', 'lib', 'hash_reqs.py')
    output = subprocess.check_output([path, requirements_file])
    sha1sum = output.split()[0]
    if target_venv_path is None:
        cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, 'venv')
    else:
        cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, os.path.basename(target_venv_path))
    success_stamp = os.path.join(cached_venv_path, "success-stamp")
    if not os.path.exists(success_stamp):
        do_setup_virtualenv(cached_venv_path, requirements_file, virtualenv_args or [])
        run(["touch", success_stamp])

    print("Using cached Python venv from %s" % (cached_venv_path,))
    if target_venv_path is not None:
        run(["sudo", "ln", "-nsf", cached_venv_path, target_venv_path])
    activate_this = os.path.join(cached_venv_path, "bin", "activate_this.py")
    exec(open(activate_this).read(), {}, dict(__file__=activate_this)) # type: ignore # https://github.com/python/mypy/issues/1577
    return cached_venv_path
Beispiel #6
0
def install_npm():
    # type: () -> None
    if "--travis" not in sys.argv:
        if subprocess.check_output(['npm', '--version']).strip() != NPM_VERSION:
            run(["sudo", "npm", "install", "-g", "npm@{}".format(NPM_VERSION)])

        return

    run(['mkdir', '-p', TRAVIS_NODE_PATH])

    npm_exe = os.path.join(TRAVIS_NODE_PATH, 'bin', 'npm')
    travis_npm = subprocess.check_output(['which', 'npm']).strip()
    if os.path.exists(npm_exe):
        run(['sudo', 'ln', '-sf', npm_exe, travis_npm])

    version = subprocess.check_output(['npm', '--version']).strip()
    if os.path.exists(npm_exe) and version == NPM_VERSION:
        print("Using cached npm")
        return

    run(["npm", "install", "-g", "--prefix", TRAVIS_NODE_PATH, "npm@{}".format(NPM_VERSION)])
    run(['sudo', 'ln', '-sf', npm_exe, travis_npm])
Beispiel #7
0
def setup_node_modules():
    output = subprocess.check_output(['sha1sum', 'package.json'])
    sha1sum = output.split()[0]
    success_stamp = os.path.join('node_modules', '.npm-success-stamp', sha1sum)
    if not os.path.exists(success_stamp):
        print("Deleting cached version")
        run(["rm", "-rf", "node_modules"])
        print("Installing node modules")
        run(["npm", "install"])
        run(["mkdir", "-p", success_stamp])
    else:
        print("Using cached version of node_modules")
Beispiel #8
0
def setup_node_modules():
    output = subprocess.check_output(['sha1sum', 'package.json'])
    sha1sum = output.split()[0]
    success_stamp = os.path.join('node_modules', '.npm-success-stamp', sha1sum)
    if not os.path.exists(success_stamp):
        print("Deleting cached version")
        run(["rm", "-rf", "node_modules"])
        print("Installing node modules")
        run(["npm", "install"])
        run(["mkdir", "-p", success_stamp])
    else:
        print("Using cached version of node_modules")
Beispiel #9
0
def do_setup_virtualenv(venv_path, requirements_file, virtualenv_args):
    # type: (str, str, List[str]) -> None

    # Setup Python virtualenv
    run(["sudo", "rm", "-rf", venv_path])
    run(["sudo", "mkdir", "-p", venv_path])
    run(["sudo", "chown", "{}:{}".format(os.getuid(), os.getgid()), venv_path])
    run(["virtualenv"] + virtualenv_args + [venv_path])

    # Switch current Python context to the virtualenv.
    activate_this = os.path.join(venv_path, "bin", "activate_this.py")
    exec(open(activate_this).read(), {}, dict(__file__=activate_this)
         )  # type: ignore # https://github.com/python/mypy/issues/1577

    run(["pip", "install", "--upgrade", "pip"])
    run(["pip", "install", "--no-deps", "--requirement", requirements_file])
    run(["sudo", "chmod", "-R", "a+rX", venv_path])
Beispiel #10
0
def main():
    run(["sudo", "apt-get", "update"])
    run(["sudo", "apt-get", "-y", "install"] + APT_DEPENDENCIES[codename])

    temp_deb_path = subprocess.check_output(
        ["mktemp", "package_XXXXXX.deb", "--tmpdir"])
    run(["wget", "-O", temp_deb_path, TSEARCH_URL])
    run(["sudo", "dpkg", "--install", temp_deb_path])

    setup_virtualenv(PY3_VENV_PATH,
                     os.path.join(ZULIP_PATH, "tools", "setup",
                                  "py3_test_reqs.txt"),
                     virtualenv_args=['-p', 'python3'])
    setup_virtualenv(VENV_PATH, os.path.join(ZULIP_PATH, "requirements.txt"))

    # Put Python2 virtualenv activation in our .bash_profile.
    with open(os.path.expanduser('~/.bash_profile'), 'w+') as bash_profile:
        bash_profile.writelines([
            "source .bashrc\n",
            "source %s\n" % (os.path.join(VENV_PATH, "bin", "activate"), ),
        ])

    run(["sudo", "cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])

    # npm install and management commands expect to be run from the root of the
    # project.
    os.chdir(ZULIP_PATH)

    if "--travis" in sys.argv:
        run(["tools/setup/install-phantomjs", "--travis"])
    else:
        run(["tools/setup/install-phantomjs"])
    run(["tools/setup/download-zxcvbn"])
    run(["tools/setup/emoji_dump/build_emoji"])
    run(["scripts/setup/generate_secrets.py", "-d"])
    if "--travis" in sys.argv:
        run(["sudo", "service", "rabbitmq-server", "restart"])
        run(["sudo", "service", "redis-server", "restart"])
        run(["sudo", "service", "memcached", "restart"])
    elif "--docker" in sys.argv:
        run(["sudo", "service", "rabbitmq-server", "restart"])
        run(["sudo", "pg_dropcluster", "--stop", POSTGRES_VERSION, "main"])
        run([
            "sudo", "pg_createcluster", "-e", "utf8", "--start",
            POSTGRES_VERSION, "main"
        ])
        run(["sudo", "service", "redis-server", "restart"])
        run(["sudo", "service", "memcached", "restart"])
    run(["scripts/setup/configure-rabbitmq"])
    run(["tools/setup/postgres-init-dev-db"])
    run(["tools/do-destroy-rebuild-database"])
    run(["tools/setup/postgres-init-test-db"])
    run(["tools/do-destroy-rebuild-test-database"])
    run(["python", "./manage.py", "compilemessages"])
    # Install the latest npm.
    run(["sudo", "npm", "install", "-g", "npm"])
    # Run npm install last because it can be flaky, and that way one
    # only needs to rerun `npm install` to fix the installation.
    run(["npm", "install"])
    return 0
Beispiel #11
0
def main():
    # type: () -> int
    run(["sudo", "apt-get", "update"])
    run(["sudo", "apt-get", "-y", "install"] + APT_DEPENDENCIES[codename])

    if subprocess.call(['dpkg', '-s', TSEARCH_PACKAGE_NAME]):
        temp_deb_path = subprocess.check_output(["mktemp", "package_XXXXXX.deb", "--tmpdir"])
        run(["wget", "-O", temp_deb_path, TSEARCH_URL])
        run(["sudo", "dpkg", "--install", temp_deb_path])

    setup_virtualenv(PY3_VENV_PATH,
                     os.path.join(ZULIP_PATH, "requirements", "mypy.txt"),
                     virtualenv_args=['-p', 'python3'])
    setup_virtualenv(VENV_PATH, os.path.join(ZULIP_PATH, "requirements.txt"))

    # Put Python2 virtualenv activation in our .bash_profile.
    with open(os.path.expanduser('~/.bash_profile'), 'w+') as bash_profile:
        bash_profile.writelines([
            "source .bashrc\n",
            "source %s\n" % (os.path.join(VENV_PATH, "bin", "activate"),),
        ])

    run(["sudo", "cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])

    # npm install and management commands expect to be run from the root of the
    # project.
    os.chdir(ZULIP_PATH)

    if "--travis" in sys.argv:
        run(["tools/setup/install-phantomjs", "--travis"])
    else:
        run(["tools/setup/install-phantomjs"])
    run(["tools/setup/download-zxcvbn"])
    run(["tools/setup/emoji_dump/build_emoji"])
    run(["scripts/setup/generate_secrets.py", "-d"])
    if "--travis" in sys.argv:
        run(["sudo", "service", "rabbitmq-server", "restart"])
        run(["sudo", "service", "redis-server", "restart"])
        run(["sudo", "service", "memcached", "restart"])
    elif "--docker" in sys.argv:
        run(["sudo", "service", "rabbitmq-server", "restart"])
        run(["sudo", "pg_dropcluster", "--stop", POSTGRES_VERSION, "main"])
        run(["sudo", "pg_createcluster", "-e", "utf8", "--start", POSTGRES_VERSION, "main"])
        run(["sudo", "service", "redis-server", "restart"])
        run(["sudo", "service", "memcached", "restart"])
    run(["scripts/setup/configure-rabbitmq"])
    run(["tools/setup/postgres-init-dev-db"])
    run(["tools/do-destroy-rebuild-database"])
    run(["tools/setup/postgres-init-test-db"])
    run(["tools/do-destroy-rebuild-test-database"])
    run(["python", "./manage.py", "compilemessages"])
    # Install the pinned version of npm.
    install_npm()
    # Run npm install last because it can be flaky, and that way one
    # only needs to rerun `npm install` to fix the installation.
    setup_node_modules()
    return 0
Beispiel #12
0
def do_setup_virtualenv(venv_path, requirements_file, virtualenv_args):
    # type: (str, str, List[str]) -> None

    # Setup Python virtualenv
    run(["sudo", "rm", "-rf", venv_path])
    run(["sudo", "mkdir", "-p", venv_path])
    run(["sudo", "chown", "{}:{}".format(os.getuid(), os.getgid()), venv_path])
    run(["virtualenv"] + virtualenv_args + [venv_path])

    # Switch current Python context to the virtualenv.
    activate_this = os.path.join(venv_path, "bin", "activate_this.py")
    exec(open(activate_this).read(), {}, dict(__file__=activate_this)) # type: ignore # https://github.com/python/mypy/issues/1577

    run(["pip", "install", "--upgrade", "pip"])
    run(["pip", "install", "--no-deps", "--requirement", requirements_file])
Beispiel #13
0
def do_setup_virtualenv(venv_path, requirements_file, virtualenv_args):
    # type: (str, str, List[str]) -> None

    # Setup Python virtualenv
    run(["sudo", "rm", "-rf", venv_path])
    run(["sudo", "mkdir", "-p", venv_path])
    run(["sudo", "chown", "{}:{}".format(os.getuid(), os.getgid()), venv_path])
    run(["virtualenv"] + virtualenv_args + [venv_path])

    # Switch current Python context to the virtualenv.
    activate_this = os.path.join(venv_path, "bin", "activate_this.py")
    exec(open(activate_this).read(), {}, dict(__file__=activate_this)) # type: ignore # https://github.com/python/mypy/issues/1577

    run(["pip", "install", "--upgrade", "pip", "wheel"])
    # Currently, Scikit-Learn and Numpy cannot be installed with pip in one
    # install pass. See https://github.com/scikit-learn/scikit-learn/issues/4164
    # for further details.
    if 'dev.txt' in requirements_file or 'prod.txt' in requirements_file:
        numpy = os.path.join(ZULIP_PATH, 'requirements', 'numpy.txt')
        run(["pip", "install", "--no-deps", "--requirement", numpy])

    run(["pip", "install", "--no-deps", "--requirement", requirements_file])
    run(["sudo", "chmod", "-R", "a+rX", venv_path])
Beispiel #14
0
def do_setup_virtualenv(venv_path, requirements_file, virtualenv_args):
    # Setup Python virtualenv
    run(["sudo", "rm", "-rf", venv_path])
    run(["sudo", "mkdir", "-p", venv_path])
    run(["sudo", "chown", "{}:{}".format(os.getuid(), os.getgid()), venv_path])
    run(["virtualenv"] + virtualenv_args + [venv_path])

    # Switch current Python context to the virtualenv.
    activate_this = os.path.join(venv_path, "bin", "activate_this.py")
    execfile(activate_this, dict(__file__=activate_this))

    run(["pip", "install", "--upgrade", "pip"])
    run(["pip", "install", "--no-deps", "--requirement", requirements_file])
Beispiel #15
0
def main():
    run(["sudo", "apt-get", "update"])
    run(["sudo", "apt-get", "-y", "install"] + APT_DEPENDENCIES[codename])

    temp_deb_path = subprocess.check_output(["mktemp", "package_XXXXXX.deb", "--tmpdir"])
    run(["wget", "-O", temp_deb_path, TSEARCH_URL])
    run(["sudo", "dpkg", "--install", temp_deb_path])

    run(["sudo", "rm", "-rf", VENV_PATH])
    run(["sudo", "mkdir", "-p", VENV_PATH])
    run(["sudo", "chown", "{}:{}".format(os.getuid(), os.getgid()), VENV_PATH])

    run(["virtualenv", VENV_PATH])

    # Put Python virtualenv activation in our .bash_profile.
    with open(os.path.expanduser('~/.bash_profile'), 'w+') as bash_profile:
        bash_profile.writelines([
            "source .bashrc\n",
            "source %s\n" % (os.path.join(VENV_PATH, "bin", "activate"),),
        ])

    # Switch current Python context to the virtualenv.
    activate_this = os.path.join(VENV_PATH, "bin", "activate_this.py")
    execfile(activate_this, dict(__file__=activate_this))

    run(["pip", "install", "--no-deps", "--requirement",
         os.path.join(ZULIP_PATH, "requirements.txt")])

    run(["sudo", "cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])

    # npm install and management commands expect to be run from the root of the
    # project.
    os.chdir(ZULIP_PATH)

    run(["tools/install-phantomjs"])
    run(["tools/download-zxcvbn"])
    run(["tools/emoji_dump/build_emoji"])
    run(["scripts/setup/generate_secrets.py", "-d"])
    if "--travis" in sys.argv:
        run(["sudo", "service", "rabbitmq-server", "restart"])
        run(["sudo", "service", "redis-server", "restart"])
        run(["sudo", "service", "memcached", "restart"])
    elif "--docker" in sys.argv:
        run(["sudo", "service", "rabbitmq-server", "restart"])
        run(["sudo", "pg_dropcluster", "--stop", POSTGRES_VERSION, "main"])
        run(["sudo", "pg_createcluster", "-e", "utf8", "--start", POSTGRES_VERSION, "main"])
        run(["sudo", "service", "redis-server", "restart"])
        run(["sudo", "service", "memcached", "restart"])
    run(["scripts/setup/configure-rabbitmq"])
    run(["tools/postgres-init-dev-db"])
    run(["tools/do-destroy-rebuild-database"])
    run(["tools/postgres-init-test-db"])
    run(["tools/do-destroy-rebuild-test-database"])
    # Install the latest npm.
    run(["sudo", "npm", "install", "-g", "npm"])
    # Run npm install last because it can be flaky, and that way one
    # only needs to rerun `npm install` to fix the installation.
    run(["npm", "install"])
    return 0