def deploy():
local('make clean')
local('pipenv run python setup.py sdist bdist_wheel --universal')
local('tar cf %(stamptar)s run.sh' % env)
local('(cd dist && tar rf ../%(stamptar)s *.tar.gz)' % env)
local('gzip %(stamptar)s' % env)
put(stampzip, '/tmp/%(stampzip)s' % env)
local('rm %(stampzip)s' % env)
with settings(sudo_user='******'):
with cd('/home/reddit_tagtrain/run'):
sudo('mkdir -p %(stamp)s/src' % env)
sudo('mkdir -p %(stamp)s/venv' % env)
with cd('/home/reddit_tagtrain/run/%(stamp)s' % env):
sudo('tar xfz /tmp/%(stampzip)s -C ./src/' % env)
sudo('rm /tmp/%(stampzip)s' % env)
with settings(sudo_user='******'):
with cd('/home/reddit_tagtrain/run/%(stamp)s' % env):
with shell_env(PATH='/opt/pyenv/bin/:$PATH',
PYENV_ROOT='/opt/pyenv'):
sudo('virtualenv venv -p $(pyenv prefix 3.6.1)/bin/python' %
env)
with path('./venv/bin', behavior='prepend'):
sudo(
'pip install --quiet --no-cache-dir -r ./src/requirements/prod.txt'
% env)
with path('./venv/bin', behavior='prepend'):
sudo(
'python -m peewee_migrate migrate --database=sqlite:///../tagtrain.db --directory=src/migrations'
)
with cd('/home/reddit_tagtrain'):
sudo('run/current/src/run.sh stop')
with cd('/home/reddit_tagtrain/run'):
sudo('ln -nsf $(basename $(readlink -f current)) previous' % env)
sudo('ln -nsf %(stamp)s current' % env)
with cd('/home/reddit_tagtrain'):
sudo('run/current/src/run.sh start')
def install_python_modules(self):
if platform.system() == 'Linux':
local('echo -e "[install]\ninstall-purelib=\$base/lib64/python" > setup.cfg')
options = dict(requirements=self.requirements, lib_path=self.lib_path)
with lcd(BASE_PATH), path(os.path.join(self.install_prefix, 'bin'), behavior='prepend'):
local('pip install --upgrade -r {requirements} -t {lib_path}'.format(**options))
def install_python_modules(self):
if platform.system() == 'Linux':
local('echo -e "[install]\ninstall-purelib=\$base/lib64/python" > setup.cfg')
options = dict(requirements=self.requirements, lib_path=self.lib_path)
with lcd(BASE_PATH), path(os.path.join(self.install_prefix, 'bin'), behavior='prepend'):
local('pip install --upgrade -r {requirements} -t {lib_path}'.format(**options))
def build(self):
# if we're running a virtualenv the we need to reload the defaults
virtualenv_name = env.get("virtualenv", None)
if (virtualenv_name is not None):
# make a place for the virtualenv to exist
local("{} -p {}".format(env.tools['mkdir'], env.python_virtualenv_root_dir))
# remember where the default python installation went
system_python_virtualenv = env.python_virtualenv
# create the virtualenv
with lcd(env.python_virtualenv_root_dir):
local("{} --python={} {}".format(system_python_virtualenv, env.python, virtualenv_name))
with settings(path("{}/{}/bin".format(env.python_virtualenv_root_dir, virtualenv_name), behavior="prepend"),
shell_env(VIRTUAL_ENV="{}/{}".format(env.python_virtualenv_root_dir, virtualenv_name))):
# re-load the default paths to make it uses the virtualenv python
load_defaults()
# load requirements into virtualenv
if (os.path.isfile("{}/requirements.txt".format(env.build_dir))):
local("{} install -r {}/requirements.txt".format(env.python_pip, env.build_dir))
# really build
self._build()
# make it so that we can move the virtualenv
with lcd(env.python_virtualenv_root_dir):
local("{} --relocatable {}".format(system_python_virtualenv, virtualenv_name))
else:
# really build
self._build()
def install_mecab_neologd(self, pkg_name, mecab_ipadic):
local('git clone --depth 1 https://github.com/neologd/{}.git'.format(pkg_name))
local('xz -dkv {}/seed/mecab-user-dict-seed.*.csv.xz'.format(pkg_name))
local('mv {}/seed/mecab-user-dict-seed.*.csv {}/'.format(pkg_name, mecab_ipadic))
with lcd(mecab_ipadic), path(os.path.join(self.install_prefix, 'bin'), behavior='prepend'):
local('{}/libexec/mecab/mecab-dict-index -f utf-8 -t utf-8'.format(self.install_prefix))
local('make install')
def create_runtime(pythonver, conda_packages, pip_packages,
pip_upgrade_packages):
conda_pkgs_default_channel = []
conda_pkgs_custom_channel = []
for c in conda_packages:
if isinstance(c, tuple):
conda_pkgs_custom_channel.append(c)
else:
conda_pkgs_default_channel.append(c)
conda_default_pkg_str = " ".join(conda_pkgs_default_channel)
pip_pkg_str = " ".join(pip_packages)
pip_pkg_upgrade_str = " ".join(pip_upgrade_packages)
python_base_ver = pythonver.split(".")[0]
run("rm -Rf {}".format(CONDA_BUILD_DIR))
run("rm -Rf {}".format(CONDA_INSTALL_DIR))
run("mkdir -p {}".format(CONDA_BUILD_DIR))
with cd(CONDA_BUILD_DIR):
run("wget https://repo.continuum.io/miniconda/Miniconda{}-latest-Linux-x86_64.sh -O miniconda.sh "
.format(python_base_ver))
run("bash miniconda.sh -b -p {}".format(CONDA_INSTALL_DIR))
with path("{}/bin".format(CONDA_INSTALL_DIR), behavior="prepend"):
run("conda install -q -y python={}".format(pythonver))
run("conda install -q -y {}".format(conda_default_pkg_str))
for chan, pkg in conda_pkgs_custom_channel:
run("conda install -q -y -c {} {}".format(chan, pkg))
run("pip install {}".format(pip_pkg_str))
run("pip install --upgrade {}".format(pip_pkg_upgrade_str))
def do():
with open(config, 'r') as ip:
config_data = json.load(ip)
dag_class = config_data['dag']
# push the toast config to the remote machine
toast_config_worker_path = os.path.join(
eggo_config.get('worker_env', 'work_path'),
build_dest_filename(config))
put(local_path=config,
remote_path=toast_config_worker_path)
# TODO: run on central scheduler instead
toast_cmd = ('toaster.py --local-scheduler {clazz} '
'--ToastConfig-config {toast_config}'.format(
clazz=dag_class,
toast_config=toast_config_worker_path))
hadoop_bin = os.path.join(eggo_config.get('worker_env', 'hadoop_home'), 'bin')
toast_env = {'EGGO_HOME': eggo_config.get('worker_env', 'eggo_home'), # toaster.py imports eggo_config, which needs EGGO_HOME on worker
'EGGO_CONFIG': eggo_config.get('worker_env', 'eggo_config_path'), # bc toaster.py imports eggo_config which must be init on the worker
'LUIGI_CONFIG_PATH': eggo_config.get('worker_env', 'luigi_config_path'),
'AWS_ACCESS_KEY_ID': eggo_config.get('aws', 'aws_access_key_id'), # bc dataset dnload pushes data to S3 TODO: should only be added if the dfs is S3
'AWS_SECRET_ACCESS_KEY': eggo_config.get('aws', 'aws_secret_access_key'), # TODO: should only be added if the dfs is S3
'SPARK_HOME': eggo_config.get('worker_env', 'spark_home')}
if exec_ctx == 'local':
# this should copy vars that maintain venv info
env_copy = os.environ.copy()
env_copy.update(toast_env)
toast_env = env_copy
with path(hadoop_bin):
with shell_env(**toast_env):
wrun(toast_cmd)
def deploy():
_install_depends()
_install_pydepends()
sudo("mkdir -p /opt/archives")
cuda_path = _install_cuda()
cuda_lib = "{0}/cuda/lib64/".format(PREFIX)
# because of licensing issue, we should delete unredistributable files.
sudo("mkdir -p {0}/cuda".format(PREFIX))
sudo("cp -pr {0}/lib64 {1}".format(cuda_path, cuda_lib))
sudo("rm {0}/libcuinj64.so*".format(cuda_lib))
sudo("rm {0}/libnvToolsExt.so*".format(cuda_lib))
sudo("rm {0}/libOpenCL.so".format(cuda_lib))
sudo("rm {0}/stubs/libcuda.so".format(cuda_lib))
with path("{0}/bin".format(cuda_path), behavior="prepend"):
with shell_env(LIBRARY_PATH=cuda_lib, LD_LIBRARY_PATH=cuda_lib, CPATH="{0}/include".format(cuda_path)):
caffe_path = _install_caffe()
sudo("rm -rf /opt/archives")
sudo("rm -rf {0}".format(cuda_path))
sudo("rm -rf /usr/local/cuda")
sudo("ln -fs {0}/build {1}/build".format(caffe_path, PREFIX))
sudo("ln -fs {0}/python {1}/python".format(caffe_path, PREFIX))
# suppress: libdc1394 error: Failed to initialize libdc1394
if not exists("/dev/raw1394"):
sudo("ln -s /dev/null /dev/raw1394")
sudo("sed --in-place -e \"/^exit 0$/i ln -s /dev/null /dev/raw1394\" /etc/rc.local")
_configure_paths("{0}/build/tools".format(PREFIX), "{0}/python".format(PREFIX), "{0}/cuda/lib64".format(PREFIX))
sudo("pip install labellio-cli")
def install_pip():
with cd('/tmp'):
with path('{}/bin'.format(INSTALL_PREFIX)):
run('curl -O https://bitbucket.org/pypa/setuptools/raw/bootstrap/'
'ez_setup.py')
sudo('python{} ez_setup.py --insecure'.format(PYTHON_SUFFIX))
sudo('easy_install-{} pip'.format(PYTHON_SUFFIX))
def run_install_requirements():
with cd(fab_settings.REMOTE_APP_ROOT):
# site isn't deployed yet, so copy requirements.txt as one-off
with lcd(fab_settings.PROJECT_ROOT):
put(local_path='requirements.txt',
remote_path ='requirements.txt')
with path('{0}/bin'.format(fab_settings.VENV_NAME), behavior='prepend'):
run('pip install -r requirements.txt')
def install_mecab_ipadic(self, pkg_name):
local('mv {} {}'.format(os.path.join('/root', pkg_name), self.tempdir))
local('nkf --overwrite -Ew {}/*'.format(pkg_name))
with lcd(pkg_name), path(os.path.join(self.install_prefix, 'bin'), behavior='prepend'):
local('{}/libexec/mecab/mecab-dict-index -f utf-8 -t utf-8'.format(self.install_prefix))
local('./configure')
local('make install')
with lcd(DICT_PATH), path(os.path.join(self.install_prefix, 'bin'), behavior='prepend'):
local('cp -r {} {}'.format(os.path.join(self.tempdir,pkg_name), DICT_PATH))
local('{}/libexec/mecab/mecab-dict-index -m {} -d {} -u {} -f utf8 -t utf8 -a {}'.format(self.install_prefix, 'mecab-ipadic-2.7.0-20070801.model', os.path.join(DICT_PATH, pkg_name), 'PacPacDev.csv', 'PacPac.csv'), capture=False)
local('mv PacPac.csv {}'.format(os.path.join(self.tempdir, pkg_name)))
with lcd(pkg_name), path(os.path.join(self.install_prefix, 'bin'), behavior='prepend'):
local('{}/libexec/mecab/mecab-dict-index -f utf-8 -t utf-8'.format(self.install_prefix))
local('make install')
def install_mecab_ipadic(self, pkg_name):
local('wget http://mecab.googlecode.com/files/{}.tar.gz'.format(pkg_name))
local('tar zvxf {}.tar.gz'.format(pkg_name))
local('nkf --overwrite -Ew {}/*'.format(pkg_name))
with lcd(pkg_name), path(os.path.join(self.install_prefix, 'bin'), behavior='prepend'):
local('{}/libexec/mecab/mecab-dict-index -f utf-8 -t utf-8'.format(self.install_prefix))
local('./configure')
local('make install')
def env_set():
with path('/opt/'):
run('echo $PATH')
run('echo $PATH')
#fixme ALL para you can use http://docs.fabfile.org/en/1.13/usage/env.html
with settings(warn_only=True):
run('echo $USER')
with prefix('free'):
with shell_env(JAVA_HOME='/opt/java'):
run('echo $JAVA_HOME')
def setup_virtualenv(virtualenv_path):
virtualenv_cmd = ["virtualenv",
"--distribute",
"--no-site-packages",
"-p %s" % os.path.join(env.virtualenv_python_dir, 'python'),
"%s" % virtualenv_path]
with fab.path(env.virtualenv_python_dir, behavior='prepend'):
with fab.prefix("umask 0002"):
helpers.remote(" ".join(virtualenv_cmd))
def conda_setup_minimal():
run("rm -Rf /tmp/conda")
run("mkdir -p /tmp/conda")
with cd("/tmp/conda"):
run("wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh "
)
run("bash miniconda.sh -b -p /tmp/conda/condaruntime")
with path("/tmp/conda/condaruntime/bin", behavior="prepend"):
run("conda install -q -y nomkl numpy boto3 boto"
) # Numpy is required
def install_release(self):
flags_file = "/tmp/feature-flags-rules.json"
if exists(flags_file, use_sudo=True):
print "{:80} {:}".format("Installation already completed. ", '[ ' + colored('SKIPPING DCM INSTALL', 'yellow') + ' ]')
else:
print "{:80}".format("Running DCM install"),
with path('/opt/dmcm-base/bin:/opt/dmcm-base/embedded/bin:/opt/dmcm-base/bin:/opt/dmcm-base/embedded/bin'):
with cd(self.release_path):
sudo("chef-solo -j local_settings/handsfree/single_node.json -c solo.rb -o 'role[single_node]'")
print "{:}".format('[ ' + colored('OK', 'green') + ' ]')
def setup_virtualenv(virtualenv_path):
virtualenv_cmd = ["virtualenv",
"--distribute",
"--no-site-packages",
"-p python2.7",
"%s" % virtualenv_path]
# TODO: extract to variable
with fab.path("/seq/annotation/development/tools/python/2.7.1/bin", behavior='prepend'):
with fab.prefix("umask 0002"):
helpers.remote(" ".join(virtualenv_cmd))
def install_mecab_neologd(self, pkg_name, mecab_ipadic):
local('git clone --depth 1 https://github.com/neologd/{}.git'.format(
pkg_name))
local('xz -dkv {}/seed/mecab-user-dict-seed.*.csv.xz'.format(pkg_name))
local('mv {}/seed/mecab-user-dict-seed.*.csv {}/'.format(
pkg_name, mecab_ipadic))
with lcd(mecab_ipadic), path(os.path.join(self.install_prefix, 'bin'),
behavior='prepend'):
local('{}/libexec/mecab/mecab-dict-index -f utf-8 -t utf-8'.format(
self.install_prefix))
local('make install')
def build_tournament_manager():
''' build tournament manager '''
with cd('/home/vagrant/StarcraftAITournamentManager/src/'):
with path("/cygdrive/c/Program\ Files/Java/jdk" + env.jdk_version + "/bin",
behavior='append'):
run('chmod +x make.bat')
run('cmd /c make.bat')
get('/home/vagrant/StarcraftAITournamentManager/server/server.jar',
env.server_jar)
get('/home/vagrant/StarcraftAITournamentManager/client/client.jar',
env.client_jar)
def install_mecab_ipadic(self, pkg_name):
local(
'wget -O mecab-ipadic-2.7.0-20070801.tar.gz "https://drive.google.com/uc?export=download&id=0B4y35FiV1wh7MWVlSDBCSXZMTXM"'
)
local('tar zvxf {}.tar.gz'.format(pkg_name))
local('nkf --overwrite -Ew {}/*'.format(pkg_name))
with lcd(pkg_name), path(os.path.join(self.install_prefix, 'bin'),
behavior='prepend'):
local('{}/libexec/mecab/mecab-dict-index -f utf-8 -t utf-8'.format(
self.install_prefix))
local('./configure')
local('make install')
def openblas():
sudo("sudo yum install -q -y git gcc g++ gfortran libgfortran")
with path("/tmp/conda/condaruntime/bin", behavior="prepend"):
with cd("/tmp/conda"):
run("rm -Rf openblas-build")
run("mkdir openblas-build")
with cd('openblas-build'):
run('git clone https://github.com/xianyi/OpenBLAS.git')
with cd('OpenBLAS'):
run('make -j4')
run('make install PREFIX=/tmp/conda/condaruntime/openblas-install'
)
def conda_setup_nomkl():
run("rm -Rf /tmp/conda")
run("mkdir -p /tmp/conda")
with cd("/tmp/conda"):
run("wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh "
)
run("bash miniconda.sh -b -p /tmp/conda/condaruntime")
with path("/tmp/conda/condaruntime/bin", behavior="prepend"):
run("conda install -q -y nomkl numpy enum34 pytest Click numba boto3 PyYAML cython"
)
run("conda list")
run("pip install --upgrade cloudpickle glob2")
def putcode():
release_dir = 'releases/%s' % env.datetag
src_dir = 'src'
with path('~'):
put('/tmp/%s' % env.packed_filename, 'tmp/')
# run('mkdir -p %s' % release_dir)
# with cd('%s' % release_dir):
# run('tar -xzf ~/tmp/%s' % env.packed_filename)
with cd('%s' % src_dir):
run('tar -xzf ~/tmp/%s' % env.packed_filename)
run('rm ~/tmp/%s' % env.packed_filename)
local("rm /tmp/%s" % env.packed_filename)
def putcode():
release_dir = 'releases/%s' % env.datetag
src_dir = 'src'
with path('~'):
put('/tmp/%s' % env.packed_filename, 'tmp/')
# run('mkdir -p %s' % release_dir)
# with cd('%s' % release_dir):
# run('tar -xzf ~/tmp/%s' % env.packed_filename)
with cd('%s' % src_dir):
run('tar -xzf ~/tmp/%s' % env.packed_filename)
run('rm ~/tmp/%s' % env.packed_filename)
local("rm /tmp/%s" % env.packed_filename)
def test(self):
# if we're running a virtualenv then we need to reload the defaults
virtualenv_name = env.get("virtualenv", None)
if (virtualenv_name is not None):
with settings(path("{}/{}/bin".format(env.python_virtualenv_root_dir, virtualenv_name), behavior="prepend"),
shell_env(VIRTUAL_ENV="{}/{}".format(env.python_virtualenv_root_dir, virtualenv_name))):
# re-load the default paths to make it uses the virtualenv python
load_defaults()
# really test
self._test()
else:
# really test
self._test()
def conda_setup_mkl():
run("rm -Rf /tmp/conda")
run("mkdir -p /tmp/conda")
with cd("/tmp/conda"):
run("wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh "
)
run("bash miniconda.sh -b -p /tmp/conda/condaruntime")
with path("/tmp/conda/condaruntime/bin", behavior="prepend"):
run("conda install -q -y numpy enum34 pytest Click numba boto3 PyYAML cython"
)
run("conda list")
run("pip install --upgrade cloudpickle")
run("rm -Rf /tmp/conda/condaruntime/pkgs/mkl-11.3.3-0/*")
with cd("/tmp/conda/condaruntime/lib"):
run("rm *_mc.so *_mc2.so *_mc3.so *_avx512* *_avx2*")
def conda_setup_mkl_avx2():
run("rm -Rf /tmp/conda")
run("mkdir -p /tmp/conda")
with cd("/tmp/conda"):
run("wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh "
)
run("bash miniconda.sh -b -p /tmp/conda/condaruntime")
with path("/tmp/conda/condaruntime/bin", behavior="prepend"):
run("conda install -q -y numpy enum34 pytest Click numba boto3 PyYAML cython boto scipy pillow cvxopt scikit-learn tblib"
)
run("conda list")
#run("conda clean -y -i -t -p")
run("pip install --upgrade cloudpickle")
run("pip install cvxpy")
run("pip install redis")
def workon(env=None):
"""Context manager which sets the PATH to that of the named virtualenv
If no argument or an empty argument (e.g. '') is specified, the Fabric env
will be checked for a "virtualenv". If neither is specified, a functional
context manager will be returned, but the PATH will be functionally
unchanged.
"""
env = env or fab.env.get('virtualenv')
# In the future we might want to support more via the actual "workon"
# command, (using "fabric.api.prefix"), but that appears to require also
# prefixing "source /../virtualenvwrapper" (to make that command
# available). Prepending to the PATH, though it requires knowledge of the
# env's full path, is much lighter weight.
return fab.path(virtualenv_path(env), behavior='prepend')
def test_install_and_uninstall_global_package(nodejs):
from fabtools.nodejs import install_package, package_version, uninstall_package
# This is not in root's PATH on RedHat systems
with path('/usr/local/bin'):
if not package_version('underscore'):
install_package('underscore', version='1.4.2')
assert package_version('underscore') == '1.4.2'
assert is_file('/usr/local/lib/node_modules/underscore/underscore.js')
uninstall_package('underscore')
assert package_version('underscore') is None
assert not is_file('/usr/local/lib/node_modules/underscore/underscore.js')
def test_install_and_uninstall_global_package(nodejs):
from fabtools.nodejs import install_package, package_version, uninstall_package
# This is not in root's PATH on RedHat systems
with path("/usr/local/bin"):
if not package_version("underscore"):
install_package("underscore", version="1.4.2")
assert package_version("underscore") == "1.4.2"
assert is_file("/usr/local/lib/node_modules/underscore/underscore.js")
uninstall_package("underscore")
assert package_version("underscore") is None
assert not is_file("/usr/local/lib/node_modules/underscore/underscore.js")
def test_install_and_uninstall_global_package(nodejs):
from burlap.nodejs import install_package, package_version, uninstall_package
# This is not in root's PATH on RedHat systems
with path('/usr/local/bin'):
if not package_version('underscore'):
install_package('underscore', version='1.4.2')
assert package_version('underscore') == '1.4.2'
assert is_file('/usr/local/lib/node_modules/underscore/underscore.js')
uninstall_package('underscore')
assert package_version('underscore') is None
assert not is_file('/usr/local/lib/node_modules/underscore/underscore.js')
def deploy():
local('find . \( -name "*.pyc" -or -name "*.pyo" -or -name "*py.class" \) -delete')
local('tar cf %(stamptar)s _requirements/' % env)
local('tar rf %(stamptar)s firebase/' % env)
local('tar rf %(stamptar)s frontpage/' % env)
local('tar rf %(stamptar)s services/' % env)
local('tar rf %(stamptar)s main.py' % env)
local('tar rf %(stamptar)s run.sh' % env)
local('gzip %(stamptar)s' % env)
put(stampzip, '/tmp/%(stampzip)s' % env)
local('rm %(stampzip)s' % env)
with settings(sudo_user='******'):
with cd('/home/hn_frontpage/run'):
sudo('mkdir -p %(stamp)s/src' % env)
sudo('mkdir -p %(stamp)s/venv' % env)
with cd('/home/hn_frontpage/run/%(stamp)s' % env):
sudo('tar xfz /tmp/%(stampzip)s -C ./src/' % env)
sudo('rm /tmp/%(stampzip)s' % env)
with settings(sudo_user='******'):
with cd('/home/hn_frontpage/run/%(stamp)s' % env):
with shell_env(PATH='/opt/pyenv/bin/:$PATH', PYENV_ROOT='/opt/pyenv'):
sudo('virtualenv venv -p $(pyenv prefix 2.7.11)/bin/python' % env)
with path('./venv/bin', behavior='prepend'):
sudo('pip install --quiet --no-cache-dir -r ./src/_requirements/default.txt' % env)
with cd('/home/hn_frontpage/run/current/src/'):
sudo('./run.sh stop')
with cd('/home/hn_frontpage/run'):
sudo('ln -nsf $(basename $(readlink -f current)) previous' % env)
sudo('ln -nsf %(stamp)s current' % env)
with cd('/home/hn_frontpage/run/current/src/'):
sudo('./run.sh start')
def create_runtime(pythonver, conda_packages, pip_packages,
pip_upgrade_packages):
conda_pkg_str = " ".join(conda_packages)
pip_pkg_str = " ".join(pip_packages)
pip_pkg_upgrade_str = " ".join(pip_upgrade_packages)
python_base_ver = pythonver.split(".")[0]
run("rm -Rf {}".format(CONDA_BUILD_DIR))
run("mkdir -p {}".format(CONDA_BUILD_DIR))
with cd(CONDA_BUILD_DIR):
run("wget https://repo.continuum.io/miniconda/Miniconda{}-latest-Linux-x86_64.sh -O miniconda.sh "
.format(python_base_ver))
run("bash miniconda.sh -b -p {}".format(CONDA_INSTALL_DIR))
with path("{}/bin".format(CONDA_INSTALL_DIR), behavior="prepend"):
run("conda install -q -y python={}".format(pythonver))
run("conda install -q -y {}".format(conda_pkg_str))
run("pip install {}".format(pip_pkg_str))
run("pip install --upgrade {}".format(pip_pkg_upgrade_str))
def deploy(directory=BASE_DIR):
dist_file = '{}.tar.gz'.format(_dist())
dist = 'dist/' + dist_file
with cd(directory):
mkvirtualenv(directory + '/venv')
put(dist, '.')
put('wsgi.py', '.')
put('requirements.txt', '.')
with path('venv/bin', 'prepend'):
run('pip install --download-cache=cache -r requirements.txt')
run('pip install --download-cache=cache ' + dist_file)
# run('rm ' + dist_file)
print_modwsgi_info(directory)
def deploy():
_install_depends()
_install_pydepends()
sudo("mkdir -p /opt/archives")
cuda_path = _install_cuda()
cuda_lib = "{0}/cuda/lib64/".format(PREFIX)
# because of licensing issue, we should delete unredistributable files.
sudo("mkdir -p {0}/cuda".format(PREFIX))
sudo("cp -pr {0}/lib64 {1}".format(cuda_path, cuda_lib))
sudo("rm {0}/libcuinj64.so*".format(cuda_lib))
sudo("rm {0}/libnvToolsExt.so*".format(cuda_lib))
sudo("rm {0}/libOpenCL.so".format(cuda_lib))
sudo("rm {0}/stubs/libcuda.so".format(cuda_lib))
with path("{0}/bin".format(cuda_path), behavior="prepend"):
with shell_env(LIBRARY_PATH=cuda_lib,
LD_LIBRARY_PATH=cuda_lib,
CPATH="{0}/include".format(cuda_path)):
caffe_path = _install_caffe()
sudo("rm -rf /opt/archives")
sudo("rm -rf {0}".format(cuda_path))
sudo("rm -rf /usr/local/cuda")
sudo("ln -fs {0}/build {1}/build".format(caffe_path, PREFIX))
sudo("ln -fs {0}/python {1}/python".format(caffe_path, PREFIX))
# suppress: libdc1394 error: Failed to initialize libdc1394
if not exists("/dev/raw1394"):
sudo("ln -s /dev/null /dev/raw1394")
sudo(
"sed --in-place -e \"/^exit 0$/i ln -s /dev/null /dev/raw1394\" /etc/rc.local"
)
_configure_paths("{0}/build/tools".format(PREFIX),
"{0}/python".format(PREFIX),
"{0}/cuda/lib64".format(PREFIX))
sudo("pip install labellio-cli")
def deploy():
local('find . \( -name "*.pyc" -or -name "*.pyo" -or -name "*py.class" \) -delete')
local("tar cf %(stamptar)s _requirements/" % env)
local("tar rf %(stamptar)s _project/" % env)
local("tar rf %(stamptar)s app/" % env)
local("tar rf %(stamptar)s manage.py" % env)
local("tar rf %(stamptar)s gunicorn.conf.py" % env)
local("gzip %(stamptar)s" % env)
put(stampzip, "/tmp/%(stampzip)s" % env)
local("rm %(stampzip)s" % env)
with settings(sudo_user="******"):
with cd("/home/hiring/web"):
sudo("mkdir -p %(stamp)s/src" % env)
sudo("mkdir -p %(stamp)s/venv" % env)
with cd("/home/hiring/web/%(stamp)s/" % env):
sudo("tar xfz /tmp/%(stampzip)s -C ./src/" % env)
sudo("perl -pi -e 's/base/%(env)s/ig' src/manage.py" % env)
with shell_env(PATH='/opt/pyenv/bin/:$PATH', PYENV_ROOT='/opt/pyenv'):
sudo("virtualenv venv -p $(pyenv prefix 3.6.1)/bin/python")
with path("./venv/bin", behavior="prepend"):
sudo("pip install --quiet --no-cache-dir -r ./src/_requirements/prod.txt")
sudo("python src/manage.py migrate")
sudo("python src/manage.py collectstatic --noinput")
sudo("python src/manage.py staticsitegen")
with cd("/home/hiring/web"):
sudo("ln -nsf $(basename $(readlink -f current)) previous")
sudo("ln -nsf %(stamp)s current" % env)
sudo("%(supervisor)s restart hnhiring" % env)
sudo("%(nginx)s -s reload" % env)
sudo("rm /tmp/%(stampzip)s" % env)
def local_init_flask_project():
with lcd(fab_settings.PROJECT_ROOT):
# download remote default Apache httpd.conf
local('rm -rf apache2/conf')
local('mkdir -p apache2/conf')
with cd(fab_settings.REMOTE_APP_ROOT):
get(remote_path='apache2/conf/httpd.conf',
local_path ='apache2/conf/httpd.conf')
# prepare httpd.conf
app_root_fullpath = fab_settings.REMOTE_APP_ROOT_FULLPATH
python_home = '{0}/{1}'.format(app_root_fullpath, fab_settings.VENV_NAME)
# strip '/lib/'... from python-path so app dir is in python path
local("sed -i -e '/^\s*WSGIDaemonProcess.*/ s@\(python-path=[^ ]\+\)/lib/[^ ]\+@\\1@'"
" apache2/conf/httpd.conf")
# add python-home pointed at the virtualenv
local("sed -i -e '/^\s*WSGIDaemonProcess.*/ s@$@ python-home={0}@'"
" apache2/conf/httpd.conf".format(python_home))
# add script alias for index.py
local("sed -i -e '/^\s*WSGIDaemonProcess.*/a "
"WSGIScriptAlias / {0}/htdocs/index.py'"
" apache2/conf/httpd.conf".format(app_root_fullpath))
# prepare webfaction.py with APP_URL
local("sed -i -e 's!@APP_URL@!{0}!'"
" htdocs/webfaction.py".format(fab_settings.APP_URL))
# initialize local Flask project
with path('{0}/bin'.format(fab_settings.VENV_NAME), behavior='prepend'):
local('pip install Flask')
local('pip freeze > requirements.txt')
# generate random SECRET_KEYs for config
local('sed -e "s/@DEVKEY@/{0}/"'
' -e "s/@TESTKEY@/{1}/"'
' -e "s/@PRODKEY@/{2}/"'
' config/myapp/config.py > myapp/config.py'.format(
binascii.hexlify(os.urandom(24)).decode('ascii'),
binascii.hexlify(os.urandom(24)).decode('ascii'),
binascii.hexlify(os.urandom(24)).decode('ascii'),
))
def deploy():
local('pipenv run python setup.py sdist --formats=gztar')
local("tar cf %(stamptar)s gunicorn.conf.py" % env)
local("tar rf %(stamptar)s wsgi.py" % env)
local('(cd dist && tar rf ../%(stamptar)s *.tar.gz)' % env)
local('gzip %(stamptar)s' % env)
put(stampzip, "/tmp/%(stampzip)s" % env)
local("rm %(stampzip)s" % env)
with settings(sudo_user=server_user):
with cd("/home/%(server_user)s/web" % env):
sudo("mkdir -p %(stamp)s/src" % env)
sudo("mkdir -p %(stamp)s/venv" % env)
with cd("/home/%(server_user)s/web/%(stamp)s/" % env):
sudo("tar xfz /tmp/%(stampzip)s -C ./src/" % env)
sudo('rm /tmp/%(stampzip)s' % env)
with settings(sudo_user=server_user):
with cd("/home/%(server_user)s/web/%(stamp)s/" % env):
with shell_env(PATH='/opt/pyenv/bin/:$PATH', PYENV_ROOT='/opt/pyenv'):
sudo("virtualenv venv -p $(pyenv prefix 3.6.2)/bin/python")
with path("./venv/bin", behavior="prepend"):
sudo("pip install --quiet --no-cache-dir ./src/*.tar.gz")
with cd("/home/%(server_user)s/web" % env):
sudo("ln -nsf $(basename $(readlink -f current)) previous")
sudo("ln -nsf %(stamp)s current" % env)
sudo("%(supervisor)s restart aturan-calendar-web" % env)
sudo("%(nginx)s -s reload" % env)
def test_require_global_package(nodejs):
from burlap.require.nodejs import package as require_package
from burlap.nodejs import package_version, uninstall_package
# This is not in root's PATH on RedHat systems
with path('/usr/local/bin'):
try:
# Require specific version
require_package('underscore', version='1.4.1')
assert package_version('underscore') == '1.4.1'
# Downgrade
require_package('underscore', version='1.4.0')
assert package_version('underscore') == '1.4.0'
# Upgrade
require_package('underscore', version='1.4.2')
assert package_version('underscore') == '1.4.2'
finally:
uninstall_package('underscore')
def test_require_global_package(nodejs):
from fabtools.require.nodejs import package as require_package
from fabtools.nodejs import package_version, uninstall_package
# This is not in root's PATH on RedHat systems
with path("/usr/local/bin"):
try:
# Require specific version
require_package("underscore", version="1.4.1")
assert package_version("underscore") == "1.4.1"
# Downgrade
require_package("underscore", version="1.4.0")
assert package_version("underscore") == "1.4.0"
# Upgrade
require_package("underscore", version="1.4.2")
assert package_version("underscore") == "1.4.2"
finally:
uninstall_package("underscore")
def test_require_global_package(nodejs):
from burlap.require.nodejs import package as require_package
from burlap.nodejs import package_version, uninstall_package
# This is not in root's PATH on RedHat systems
with path('/usr/local/bin'):
try:
# Require specific version
require_package('underscore', version='1.4.1')
assert package_version('underscore') == '1.4.1'
# Downgrade
require_package('underscore', version='1.4.0')
assert package_version('underscore') == '1.4.0'
# Upgrade
require_package('underscore', version='1.4.2')
assert package_version('underscore') == '1.4.2'
finally:
uninstall_package('underscore')
def numpy():
"""
http://stackoverflow.com/questions/11443302/compiling-numpy-with-openblas-integration
"""
with path("/tmp/conda/condaruntime/bin", behavior="prepend"):
# git clone
run("rm -Rf /tmp/conda/numpy")
with cd("/tmp/conda"):
run("git clone https://github.com/numpy/numpy")
with cd("numpy"):
run("cp site.cfg.example site.cfg")
config = """
[openblas]
libraries = openblas
library_dirs = /tmp/conda/condaruntime/openblas-install/lib
include_dirs = /tmp/conda/condaruntime/openblas-install/install
runtime_library_dirs = /tmp/conda/condaruntime/openblas-install/lib
"""
for l in config.split("\n"):
run("echo '{}' >> {}".format(l.strip(), 'site.cfg'))
run("python setup.py config") # check this output
run("pip install .")
def test_no_trailing_space_in_shell_path_in_run(self):
put(StringIO("#!/bin/bash\necho hi"),
"%s/myapp" % self.dirpath,
mode="0755")
with path(self.dirpath):
assert run('myapp').stdout == 'hi'
def compile_project():
with cd("/opt/gush/gush-sbt"), path("/opt/sbt/bin"):
run("sbt compile")
def test_no_trailing_space_in_shell_path_in_run(self):
put(StringIO("#!/bin/bash\necho hi"), "%s/myapp" % self.dirpath, mode="0755")
with path(self.dirpath):
assert run('myapp').stdout == 'hi'
def compile_project():
with cd("/opt/gush/gush-sbt"), path("/opt/sbt/bin"):
run("sbt compile")
def virtualenv(virtualenv_path):
with path(virtualenv_path, 'prepend'):
yield
def install_python_requirements():
with path('{}/bin:/usr/pgsql-9.1/bin'.format(INSTALL_PREFIX)):
sudo('pip{} install virtualenv'.format(PYTHON_SUFFIX))
sudo('pip{} install mercurial'.format(PYTHON_SUFFIX))
sudo('pip{} install psycopg2'.format(PYTHON_SUFFIX))
def pyenv(python_path):
with path(python_path, 'prepend'):
yield
def virtualenv(virtualenv_path):
with path(virtualenv_path, 'prepend'):
yield
def pyenv(python_path):
with path(python_path, 'prepend'):
yield