def deploy():
# figure out the release name and version
dist = local('python setup.py --fullname', capture=True).strip()
# upload the source tarball to the temporary folder on the server
put('dist/{}.tar.gz'.format(dist), '/tmp/{}.tar.gz'.format(dist))
# create a place where we can unzip the tarball, then enter
# that directory and unzip it
with cd("/tmp"):
run('rm -rf {}/*'.format(dist))
run('tar xzf /tmp/{}.tar.gz'.format(dist))
with cd(dist):
run("tox")
run("cp -r /tmp/{}/* {}".format(dist, env.app_root))
install_deps()
with cd(env.app_root):
with virtualenv(env.app_env):
run("py.test")
with cd(env.app_root):
with virtualenv(env.app_env):
run("./manage.py create_db")
refresh_uwsgi()
run('rm -f /tmp/{}.tar.gz'.format(dist))
def install():
sudo('rm -rf {}'.format(SETTINGS['proj']['remote-root']))
sudo('rm -rf {}'.format(SETTINGS['proj']['remote-venv']))
git.clone('https://github.com/ccortezia/featuring.git',
path=SETTINGS['proj']['remote-root'],
use_sudo=False)
git.checkout(SETTINGS['proj']['remote-root'], branch='api-revamp')
require.python.virtualenv(SETTINGS['proj']['remote-venv'],
venv_python='/usr/bin/python3.7')
require.python.pip()
with python.virtualenv(SETTINGS['proj']['remote-venv']):
python.install_requirements(os.path.join(COMPROOT, 'requirements.txt'))
upload_template(
filename=SETTINGS['supervisor']['proj-local'],
destination=SETTINGS['supervisor']['proj-remote'],
context={},
use_sudo=True)
sudo('mkdir -p {}'.format(os.path.dirname(SETTINGS['gunicorn']['proj-remote'])))
put(SETTINGS['gunicorn']['proj-local'], SETTINGS['gunicorn']['proj-remote'], use_sudo=True)
with python.virtualenv(SETTINGS['proj']['remote-venv']):
require.python.package('gunicorn')
def setup():
require.git.working_copy('[email protected]:ArthurPBressan/hotsite-sa-bilac-2015.git',
path=HOTSITE_WORK_PATH, update=True)
with cd(HOTSITE_WORK_PATH):
require.files.directories(['instance', 'tmp'], use_sudo=True)
require.python.virtualenv(HOTSITE_WORK_PATH)
with virtualenv(HOTSITE_WORK_PATH):
python.install('uwsgi')
deploy()
with cd(HOTSITE_WORK_PATH), virtualenv(HOTSITE_WORK_PATH):
sudo('uwsgi --socket :8080 --module="hotsite:create_app()" --touch-reload="/root/uwsgi_file" &')
def setup():
require.git.working_copy('[email protected]:HardDiskD/TCMCidadeIluminada.git',
path=CIDADEILUMINADA_WORK_PATH,
update=True)
with cd(CIDADEILUMINADA_WORK_PATH):
require.files.directories(['instance', 'tmp'], use_sudo=True)
require.python.virtualenv(CIDADEILUMINADA_WORK_PATH)
with virtualenv(CIDADEILUMINADA_WORK_PATH):
python.install('uwsgi')
deploy()
with cd(CIDADEILUMINADA_WORK_PATH), virtualenv(CIDADEILUMINADA_WORK_PATH):
sudo('python manage.py ci criar_usuario admin 123456')
sudo(
'uwsgi --socket :8080 --module="cidadeiluminada:create_app()" --touch-reload="/root/uwsgi_file"'
)
def create_settings_local():
with cd(CIDADEILUMINADA_WORK_PATH), virtualenv(CIDADEILUMINADA_WORK_PATH):
instance_path = sudo('python manage.py instance_path')
require.files.directories([instance_path], use_sudo=True)
settings_path = os.path.join(CIDADEILUMINADA_WORK_PATH, 'settings.py')
settings_local_path = os.path.join(instance_path, 'settings_local.py')
files.copy(settings_path, settings_local_path, use_sudo=True)
def runserver():
"""Starts the default django server"""
with virtualenv(env.virtualenv_path):
with cd(env.source_root):
sudo("./manage.py runserver 0.0.0.0:8000")
def alembic_upgrade_head(app, ctx):
with python.virtualenv(str(app.venv_dir)), cd(str(app.src_dir)):
sudo('%s -n production upgrade head' % (app.alembic), user=app.name)
if confirm('Vacuum database?', default=False):
flag = '-f ' if confirm('VACUUM FULL?', default=False) else ''
sudo('vacuumdb %s-z -d %s' % (flag, app.name), user='******')
def deploy():
"""
TODO: add all related commands
"""
if confirm('Create tag for this release?', default=False):
create_tag()
if confirm('Backup DB?', default=False):
dump_db()
if confirm("Start deploying to remote server?"):
with cd("cgi-bin"):
run("git pull")
alter_sql_path = "alter.sql"
if confirm('Show alter.sql file?', default=False):
print 40 * "-"
run("cat %s" % alter_sql_path)
print 40 * "-"
if confirm('Execute alter.sql file?', default=False):
run("psql -f %s" % alter_sql_path)
with virtualenv(env.virtualenv_path):
run("compilemessages.sh")
run("./reload_webserver.py")
def migrate():
"""Executes all the migrations"""
with virtualenv(env.virtualenv_path):
with cd(env.source_root):
run("./manage.py migrate")
def start():
"""
Run application
"""
with virtualenv(env.virtualenv_path, local=True):
with lcd("./openode/"):
local("python manage.py runserver 0.0.0.0:8000")
def app_deploy():
# Pull repository
run('eval `ssh-agent -s` && ssh-add')
with cd("/home/developer/scheduler"):
run("git fetch origin")
run("git reset origin/master --hard")
# Create / update config
if files.is_file('/etc/nginx/sites-enabled/default'):
files.remove('/etc/nginx/sites-enabled/default', use_sudo=True)
update_file('/etc/systemd/system/gunicorn.service',
source='conf/gunicorn.service')
update_file('/etc/nginx/sites-available/www.azacili.com',
source='conf/www.azacili.com')
# Create / update Python environment
with python.virtualenv("/home/developer/.pyenv/versions/azacili/"):
with cd('/home/developer/scheduler'):
run("pip install -r requirements.txt --upgrade --upgrade-strategy eager"
)
run(r"find . -name '*.pyc' -exec rm -rf {} \;")
run("python manage.py collectstatic -c --noinput --settings=core.settings.prod"
)
sudo("systemctl daemon-reload")
sudo("service gunicorn restart")
sudo("service nginx restart")
return True
def get_file_stuff(file_id):
"""
get_file_stuff
"""
with virtualenv(env.virtenv):
with cd(env.code_root):
run("./manage.py get_file_stuff %s" % file_id)
def setup():
require.git.working_copy(CIDADEILUMINADA_REPO_PATH, path=CIDADEILUMINADA_WORK_PATH, update=True)
with cd(CIDADEILUMINADA_WORK_PATH):
require.python.virtualenv(CIDADEILUMINADA_WORK_PATH)
with virtualenv(CIDADEILUMINADA_WORK_PATH):
python.install('uwsgi')
create_settings_local()
def setup_rfm69serial_service():
print_title('Installing RFM69 Serial Service')
with virtualenv(Settings.DIR_VENV):
install('pyserial', use_sudo=False)
conf = '''[Unit]
Description=Serial daemon
After=network.target
[Service]
User={USER}
Group={GRP}
WorkingDirectory={PATH}
Restart=always
ExecStart={VIRTUALENV_PATH}bin/python rfmSerial.py
[Install]
WantedBy=multi-user.target
'''.format(
PATH=Settings.DIR_CODE,
USER=env.user,
GRP=Settings.DEPLOY_GRP,
VIRTUALENV_PATH=Settings.DIR_VENV
)
service = "/etc/systemd/system/rfmserial.service"
files.append(service, conf, use_sudo=True)
sudo('systemctl enable rfmserial')
sudo('systemctl start rfmserial')
def install_indexer():
"""
Deploy Cozy Data Indexer. Use supervisord to daemonize it.
"""
indexer_dir = "%s/cozy-data-indexer" % cozy_home
indexer_env_dir = "%s/virtualenv" % indexer_dir
python_exe = indexer_dir + "/virtualenv/bin/python"
indexer_exe = "server.py"
process_name = "cozy-indexer"
with cd(cozy_home):
delete_if_exists("cozy-data-indexer")
cozydo('git clone git://github.com/mycozycloud/cozy-data-indexer.git')
require.python.virtualenv(indexer_env_dir, use_sudo=True, user="******")
with python.virtualenv(indexer_env_dir):
cozydo("pip install --use-mirrors -r %s/requirements/common.txt" % \
indexer_dir)
require.supervisor.process(process_name,
command='%s %s' % (python_exe, indexer_exe),
directory=indexer_dir,
user=cozy_user
)
supervisor.restart_process(process_name)
print(green("Data Indexer successfully started"))
def web():
def jade(source_name, out_dir):
pwd = path.join(path.dirname(__file__), 'website')
jade_loc = path.join(pwd, 'src', 'jade', source_name)
out_loc = path.join(pwd, 'site', out_dir)
local('jade -P %s --out %s' % (jade_loc, out_loc))
with virtualenv(VENV_DIR):
local(
'./website/create_code_samples spacy/tests/website/ website/src/code/'
)
jade('404.jade', '')
jade('home/index.jade', '')
jade('docs/index.jade', 'docs/')
jade('blog/index.jade', 'blog/')
for collection in ('blog', 'tutorials'):
for post_dir in (Path(__file__).parent / 'website' / 'src' / 'jade' /
collection).iterdir():
if post_dir.is_dir() \
and (post_dir / 'index.jade').exists() \
and (post_dir / 'meta.jade').exists():
jade(str(post_dir / 'index.jade'),
path.join(collection, post_dir.parts[-1]))
def create_settings_local():
with cd(CIDADEILUMINADA_WORK_PATH), virtualenv(CIDADEILUMINADA_WORK_PATH):
instance_path = sudo('python manage.py instance_path')
require.files.directories([instance_path], use_sudo=True)
settings_path = os.path.join(CIDADEILUMINADA_WORK_PATH, 'settings.py')
settings_local_path = os.path.join(instance_path, 'settings_local.py')
files.copy(settings_path, settings_local_path, use_sudo=True)
def deploy_api(commit='master'):
now = int(time.time())
require.files.directory(env.deployment_dir(now))
with cd(env.deployment_dir(now)):
run(u'wget {}'.format(env.apitaxi_archive.format(commit)))
run('unzip {}.zip'.format(commit))
if commit != 'master':
run('mv APITaxi-{} APITaxi-master'.format(commit))
with cd(env.apitaxi_dir(now)):
require.python.virtualenv(env.apitaxi_venv_path(now))
with python.virtualenv(env.apitaxi_venv_path(now)):
python.install_pip(use_sudo=False)
require.python.package('uwsgi')
python.install_requirements('requirements.txt')
put(environ['APITAXI_CONFIG_FILE'], env.apitaxi_config_path(now))
with shell_env(APITAXI_CONFIG_FILE=env.apitaxi_config_path(now)):
for i in range(1, 30):
if service.is_running('supervisor'):
break
time.sleep(1)
run('python manage.py db upgrade')
install_admin_user()
deploy_front(now)
deploy_nginx_api_site(now)
if not service.is_running('nginx'):
service.start('nginx')
clean_directories(now)
stop_old_processes(now)
restart_stats_workers(now)
def deploy_front(now):
with cd(env.deployment_dir(now)):
run(u'wget {} -O front.zip'.format(env.fronttaxi_archive))
run('unzip front.zip')
with cd(env.fronttaxi_dir(now)), python.virtualenv(env.apitaxi_venv_path(now)):
python.install_requirements('requirements.txt')
put(environ['APITAXI_CONFIG_FILE'], env.fronttaxi_config_path(now))
def stop_old_processes(now):
def stop_process(name, visitor):
l = run('for i in /etc/supervisor/conf.d/{}_*; do echo $i; done'.format(name)).split("\n")
for f in [f.replace('\r', '') for f in l]:
print 'To remove: {}'.format(f)
if str(now) in f:
continue
file_ = f.split('/')[-1]
process = file_[:-len('.conf')]
visitor(process)
files.remove(f, use_sudo=True)
stop_process('uwsgi', lambda p:supervisor.stop_process(p))
def stop_queues(process):
#Request' status is failure after 15 secs in received
#So even if queue is not empty we can shutdown the process
for i in range(1, 17):
res = run('python manage.py active_tasks {}'.format(process))
if res == '':
break
time.sleep(1)
supervisor.stop_process(process)
with cd(env.apitaxi_dir(now)):
with python.virtualenv(env.apitaxi_venv_path(now)),\
shell_env(APITAXI_CONFIG_FILE=env.apitaxi_config_path(now)):
stop_process('send_hail', stop_queues)
def finish_setup():
deploy()
with cd(CIDADEILUMINADA_WORK_PATH), virtualenv(CIDADEILUMINADA_WORK_PATH):
sudo('python manage.py ci criar_usuarios')
sudo(
'uwsgi --socket :8080 --module="cidadeiluminada:create_app()" --touch-reload="/root/uwsgi_file"'
)
def dev():
# Allow this to persist, since we aren't as rigorous about keeping state clean
if not file_exists('.denv'):
local('virtualenv .denv')
with virtualenv(DEV_ENV_DIR):
local('pip install -r requirements.txt')
def migrate():
"""
Migrate the database
"""
with virtualenv(env.virtenv):
with cd(env.code_root):
run("./manage.py migrate")
def bootstrap():
set_env()
install_system_packages()
make_all_required_dir()
setup_postgresql()
create_db()
create_virtualenv()
with cd(env.src_path):
get_latest_source()
with virtualenv(env.virtualenv_path):
install_requirements()
with shell_env(
DJANGO_SECRET_KEY=env.get('DJANGO_SECRET_KEY'),
db_name=env.get('db_name'),
db_user=env.get('db_user'),
db_password=env.get('db_password'),
):
update_database()
create_superuser()
update_static_files()
create_gunicorn_conf()
create_nginx_conf()
restart_gunicorn()
enable_gunicorn()
restart_nginx()
def deploy():
"""
deploy xsendfile_example application
"""
if not env.hosts:
abort(USAGE)
# create package from code
commit = local('git stash create', capture=True)
if not commit:
commit = 'HEAD'
local('git archive %s > %s' % (commit, PACKAGE_NAME))
# copy package to VM
put(PACKAGE_NAME, '~')
with cd('~'):
# install package
run('mkdir xsendfile_example', quiet=True)
with cd('xsendfile_example'):
run('tar -xf ~/%s' % PACKAGE_NAME)
# install application dependencies
with python.virtualenv('/home/vagrant/venv'):
python.install_requirements('requirements.txt')
# create sample media file
run('mkdir media', quiet=True)
with cd('media'):
run('echo "%s" | base64 -d > red_dot.png' % RED_DOT)
sudo('service apache2 restart')
def pip_freeze(app):
"""write installed versions to <app_name>/requirements.txt"""
with python.virtualenv(str(app.venv_dir)):
stdout = run('pip freeze', combine_stderr=False)
def iterlines(lines):
warning = ('\x1b[33m', 'You should ')
warning_within = ('SNIMissingWarning', 'InsecurePlatformWarning')
app_git = '%s.git' % app.name.lower()
ignore = {
'babel', 'fabric', 'fabtools', 'newrelic', 'paramiko', 'pycrypto',
'pyx'
}
for line in lines:
if line.startswith(warning) or any(w in line
for w in warning_within):
continue # https://github.com/pypa/pip/issues/2470
elif app_git in line or line.partition('==')[0].lower() in ignore:
continue
elif 'clld.git' in line:
line = 'clld'
elif 'clldmpg.git' in line:
line = 'clldmpg'
yield line + '\n'
target = APPS_DIR / app.name / 'requirements.txt'
with target.open('w', encoding='ascii') as fp:
fp.writelines(iterlines(stdout.splitlines()))
def make():
with virtualenv(VENV_DIR):
with lcd(path.dirname(__file__)):
local('pip install cython')
local('pip install murmurhash')
local('pip install -r requirements.txt')
local('python setup.py build_ext --inplace')
def run_supervisor(**kwargs):
www = "/home/{user}/www/".format(user=env.project_user)
venv = www+'venv'
git_dir = www+'uber_school/'
backend_dir = git_dir+'backend/'
log_dir = git_dir + 'logs/'
python_vbin = venv + '/bin/gunicorn'
with settings(user=env.project_user):
if not files.exists(log_dir):
run('mkdir -p ' + log_dir)
run('touch {}django_stdout.log'.format(log_dir))
run('touch {}django_stderr.log'.format(log_dir))
# "environment=DJANGO_SETTINGS_MODULE=%(ENV_DJANGO_SETTINGS_MODULE)s,SECRET_KEY=%(ENV_SECRET_KEY)s,DB_USER=%(ENV_DB_USER)s,DB_PASSWORD=%(ENV_DB_PASSWORD)s,AWS_ACCESS_KEY_ID=%(ENV_AWS_ACCESS_KEY_ID)s,AWS_SECRET_ACCESS_KEY=%(ENV_AWS_SECRET_ACCESS_KEY)s " +
with cd(backend_dir), virtualenv(venv):
concat = ",".join([key+"=\""+kwargs[key]+"\"" for key in kwargs])
print concat
print log_dir + 'django_stdout.log'
#with prefix(concat):
with shell_env(**kwargs):
fabtools.require.supervisor.process('django',
environment=concat,#"DJANGO_SETTINGS_MODULE=%(ENV_DJANGO_SETTINGS_MODULE)s,SECRET_KEY=%(ENV_SECRET_KEY)s,DB_USER=%(ENV_DB_USER)s,DB_PASSWORD=%(ENV_DB_PASSWORD)s,AWS_ACCESS_KEY_ID=%(ENV_AWS_ACCESS_KEY_ID)s,AWS_SECRET_ACCESS_KEY=%(ENV_AWS_SECRET_ACCESS_KEY)s",
command=python_vbin + ' backend.wsgi:application -w 1 -b 127.0.0.1:8000 -t 300 --max-requests=100',
directory=backend_dir,
user=env.project_user,
stdout_logfile=log_dir + 'django_stdout.log',
stderr_logfile=log_dir + 'django_stderr.log',
autorestart=True,
redirect_stderr=True,
)
def install_dependencies():
www = "/home/{user}/www/".format(user=env.project_user)
venv = www+'venv'
git_dir = www+'uber_school/'
with cd(git_dir), virtualenv(venv):
fabtools.python.install_requirements('{git_dir}backend/requirements.txt'.format(git_dir=git_dir))
def install_indexer():
"""
Install Cozy Data Indexer. Use supervisord to daemonize it.
"""
home = "/usr/local/cozy-indexer"
indexer_dir = "%s/cozy-data-indexer" % home
indexer_env_dir = "%s/virtualenv" % indexer_dir
python_exe = indexer_dir + "/virtualenv/bin/python"
indexer_exe = "server.py"
process_name = "cozy-indexer"
require.files.directory(home, use_sudo=True)
with cd(home):
delete_if_exists("cozy-data-indexer")
sudo('git clone git://github.com/mycozycloud/cozy-data-indexer.git')
require.python.virtualenv(indexer_env_dir, use_sudo=True)
with python.virtualenv(indexer_env_dir):
sudo("pip install --use-mirrors -r %s/requirements/common.txt" % \
indexer_dir)
sudo("chown -R cozy:cozy %s" % home)
require.supervisor.process(process_name,
command='%s %s' % (python_exe, indexer_exe),
directory=indexer_dir,
user="******")
supervisor.restart_process(process_name)
result = run('curl -X GET http://127.0.0.1:9102/')
installedController = result.find("Cozy Data Indexer")
if installedController == -1:
print_failed("cozy-data-indexer")
print(green("Data Indexer successfully started"))
def setup_gunicorn():
print_title('Installing Gunicorn')
with virtualenv(Settings.DIR_VENV):
install('gunicorn', use_sudo=False)
gunicorn_conf = '''[Unit]
Description=gunicorn daemon
After=network.target
[Service]
User={USER}
Group={GRP}
WorkingDirectory={PATH}
Restart=always
ExecStart={VIRTUALENV_PATH}/bin/gunicorn --workers 3 --bind unix:{SOCKET_FILES_PATH}{PROJECT_NAME}.sock {APP_ENTRY_POINT}
[Install]
WantedBy=multi-user.target
'''.format(
APP_NAME=Settings.ROOT_NAME,
PROJECT_NAME=Settings.ROOT_NAME,
PATH=Settings.DIR_CODE,
USER=env.user,
GRP=Settings.DEPLOY_GRP,
VIRTUALENV_PATH=Settings.DIR_VENV,
SOCKET_FILES_PATH=Settings.DIR_SOCK,
APP_ENTRY_POINT=Settings.APP_ENTRY_POINT
)
gunicorn_service = "/etc/systemd/system/gunicorn.service"
files.append(gunicorn_service, gunicorn_conf, use_sudo=True)
sudo('systemctl enable gunicorn')
sudo('systemctl start gunicorn')
def zpar():
with virtualenv(str(VENV_DIR)):
local('pip install python-zpar')
download(Path(FILENAMES['zpar']), URLS['zpar'])
with lcd('models'):
local('unzip english.zip')
local('mv english/ zpar/')
def upgrade_requirements():
"""upgrades all the requirements. basically a pip install -U -r"""
with virtualenv(env.virtualenv_path):
with cd(env.app_root):
fab_install_requirements("requirements.txt", upgrade=True, use_sudo=True)
def make():
with virtualenv(VENV_DIR):
with lcd(path.dirname(__file__)):
local('pip install cython')
local('pip install murmurhash')
local('pip install -r requirements.txt')
local('python setup.py build_ext --inplace')
def env(lang="python2.7"):
if VENV_DIR.exists():
local('rm -rf %s' % VENV_DIR)
local('python -m virtualenv -p %s %s' % (lang, VENV_DIR))
with virtualenv(str(VENV_DIR)):
local('pip install setuptools==9.0')
local('pip install plac')
def install_indexer():
"""
Install Cozy Data Indexer. Use supervisord to daemonize it.
"""
home = "/usr/local/cozy-indexer"
indexer_dir = "%s/cozy-data-indexer" % home
indexer_env_dir = "%s/virtualenv" % indexer_dir
python_exe = indexer_dir + "/virtualenv/bin/python"
indexer_exe = "server.py"
process_name = "cozy-indexer"
require.files.directory(home, use_sudo=True)
with cd(home):
delete_if_exists("cozy-data-indexer")
sudo('git clone git://github.com/mycozycloud/cozy-data-indexer.git')
require.python.virtualenv(indexer_env_dir, use_sudo=True)
with python.virtualenv(indexer_env_dir):
sudo("pip install --use-mirrors -r %s/requirements/common.txt" % \
indexer_dir)
sudo("chown -R cozy:cozy %s" % home)
require.supervisor.process(process_name,
command='%s %s' % (python_exe, indexer_exe),
directory=indexer_dir,
user="******"
)
supervisor.restart_process(process_name)
result = run('curl -X GET http://127.0.0.1:9102/')
installedController = result.find("Cozy Data Indexer")
if installedController == -1:
print_failed("cozy-data-indexer")
print(green("Data Indexer successfully started"))
def run_supervisor(**kwargs):
www = "/home/{user}/www/".format(user=env.project_user)
venv = www + 'venv'
git_dir = www + 'uber_school/'
backend_dir = git_dir + 'backend/'
log_dir = git_dir + 'logs/'
python_vbin = venv + '/bin/gunicorn'
with settings(user=env.project_user):
if not files.exists(log_dir):
run('mkdir -p ' + log_dir)
run('touch {}django_stdout.log'.format(log_dir))
run('touch {}django_stderr.log'.format(log_dir))
# "environment=DJANGO_SETTINGS_MODULE=%(ENV_DJANGO_SETTINGS_MODULE)s,SECRET_KEY=%(ENV_SECRET_KEY)s,DB_USER=%(ENV_DB_USER)s,DB_PASSWORD=%(ENV_DB_PASSWORD)s,AWS_ACCESS_KEY_ID=%(ENV_AWS_ACCESS_KEY_ID)s,AWS_SECRET_ACCESS_KEY=%(ENV_AWS_SECRET_ACCESS_KEY)s " +
with cd(backend_dir), virtualenv(venv):
concat = ",".join(
[key + "=\"" + kwargs[key] + "\"" for key in kwargs])
print concat
print log_dir + 'django_stdout.log'
#with prefix(concat):
with shell_env(**kwargs):
fabtools.require.supervisor.process(
'django',
environment=
concat, #"DJANGO_SETTINGS_MODULE=%(ENV_DJANGO_SETTINGS_MODULE)s,SECRET_KEY=%(ENV_SECRET_KEY)s,DB_USER=%(ENV_DB_USER)s,DB_PASSWORD=%(ENV_DB_PASSWORD)s,AWS_ACCESS_KEY_ID=%(ENV_AWS_ACCESS_KEY_ID)s,AWS_SECRET_ACCESS_KEY=%(ENV_AWS_SECRET_ACCESS_KEY)s",
command=python_vbin +
' backend.wsgi:application -w 1 -b 127.0.0.1:8000 -t 300 --max-requests=100',
directory=backend_dir,
user=env.project_user,
stdout_logfile=log_dir + 'django_stdout.log',
stderr_logfile=log_dir + 'django_stderr.log',
autorestart=True,
redirect_stderr=True,
)
def collect_static():
"""
Collect the static files
"""
with virtualenv(env.virtenv):
run("cd %s && python manage.py collectstatic --noinput" %
env.code_root)
def setup_memcache():
fabtools.require.deb.packages(
['memcached', 'libmemcached-dev', 'python-pylibmc'])
with virtualenv(env.venv):
fabtools.require.python.packages(
['django-memcached', 'pylibmc', 'django-pylibmc'])
# env.cache_str = """{
# 'default': {
# 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
# 'LOCATION': '127.0.0.1:%s' ,
# }
# }""" % env.memcache['port']
# pylibmc alternative
env.cache_str = """{
'default': {
'BACKEND': 'django_pylibmc.memcached.PyLibMCCache',
'LOCATION': 'localhost:11211',
'TIMEOUT': 500,
'BINARY': True,
'OPTIONS': { # Maps to pylibmc "behaviors"
'tcp_nodelay': True,
'ketama': True
}
}}"""
env.installed_apps.append('"django-pylibmc"')
print(green('Memcache installed.'))
def deploy():
"""
deploy xsendfile_example application
"""
if not env.hosts:
abort(USAGE)
# create package from code
commit = local('git stash create', capture=True)
if not commit:
commit = 'HEAD'
local('git archive %s > %s' % (commit, PACKAGE_NAME))
# copy package to VM
put(PACKAGE_NAME, '~')
with cd('~'):
# install package
run('mkdir xsendfile_example', quiet=True)
with cd('xsendfile_example'):
run('tar -xf ~/%s' % PACKAGE_NAME)
# install application dependencies
with python.virtualenv('/home/vagrant/venv'):
python.install_requirements('requirements.txt')
# create sample media file
run('mkdir media', quiet=True)
with cd('media'):
run('echo "%s" | base64 -d > red_dot.png' % RED_DOT)
sudo('service apache2 restart')
def install():
"""
Install a jobcatcher in python2 virtualenv
"""
if not env.host_string:
env.host_string = 'localhost'
pkgs = [
'python2',
'git',
]
require.arch.packages(pkgs)
project_root = '$HOME/projects'
project_name = 'JobCatcher'
use_python = 'python2.7'
virtualenv = '.virtualenvs/jobcatcher'
gitproject = '[email protected]:yoannsculo/JobCatcher.git'
require.python.pip(python_cmd=use_python)
require.python.package(
'virtualenv',
python_cmd=use_python,
use_sudo=True,
)
require.python.package(
'virtualenvwrapper',
python_cmd=use_python,
use_sudo=True,
)
require.python.virtualenv(
virtualenv,
python_cmd=use_python,
venv_python='python2.7',
)
# Get a jobcatcher github repository
cloned = False
project = '%s/%s' % (project_root, project_name)
if not is_dir(project):
require.files.directory(project_root)
cmd = 'cd %s ; git clone %s ; cd jobcatcher ; git checkout server' % (project_root, gitproject)
run(cmd)
cloned = True
if not cloned and not is_dir(project):
cmd = 'cd (%project)s ; git pull' % locals()
run(cmd)
# Install jobcatcher
with python.virtualenv(virtualenv):
here = os.path.dirname(__file__)
requirements = '%(here)s/requirements.txt' % locals()
put(requirements, '/tmp/requirements.txt')
require.python.requirements(
'/tmp/requirements.txt',
)
def install_dependencies():
www = "/home/{user}/www/".format(user=env.project_user)
venv = www + 'venv'
git_dir = www + 'uber_school/'
with cd(git_dir), virtualenv(venv):
fabtools.python.install_requirements(
'{git_dir}backend/requirements.txt'.format(git_dir=git_dir))
def migrate():
"""Executes all the migrations"""
with virtualenv(env.virtualenv_path):
with cd(env.source_root):
run("./manage.py migrate")
def env():
if file_exists(VENV_DIR):
local('rm -rf {env}'.format(env=VENV_DIR))
local('virtualenv {env}'.format(env=VENV_DIR))
with virtualenv(VENV_DIR):
local('pip install --upgrade setuptools')
local('pip install -r requirements.txt')
local('pip install pytest')
def upgrade_requirements():
"""upgrades all the requirements. basically a pip install -U -r"""
with virtualenv(env.virtualenv_path):
with cd(env.app_root):
fab_install_requirements("requirements.txt",
upgrade=True,
use_sudo=True)
def publish(version):
with virtualenv(VENV_DIR):
local('git push origin master')
local('git tag -a %s' % version)
local('git push origin %s' % version)
local('python setup.py sdist')
local('python setup.py register')
local('twine upload dist/%s.tar.gz' % version)
def django_migrate():
print_title('Database migration')
with virtualenv(Settings.DIR_VENV):
run('python {}manage.py makemigrations sd_store'.format(Settings.DIR_CODE))
run('python {}manage.py migrate sd_store'.format(Settings.DIR_CODE))
run('python {}manage.py makemigrations'.format(Settings.DIR_CODE))
run('python {}manage.py migrate'.format(Settings.DIR_CODE))
def publish(version):
with virtualenv(VENV_DIR):
local('git push origin master')
local('git tag -a %s' % version)
local('git push origin %s' % version)
local('python setup.py sdist')
local('python setup.py register')
local('twine upload dist/%s.tar.gz' % version)
def deploy():
require.git.working_copy('[email protected]:ArthurPBressan/hotsite-sa-bilac-2015.git',
path=HOTSITE_WORK_PATH, update=True)
with cd(HOTSITE_WORK_PATH), virtualenv(HOTSITE_WORK_PATH):
require.python.requirements('requirements.txt')
require.python.requirements('requirements-db.txt')
sudo('python manage.py db upgrade')
reload()
def runserver():
"""Starts the default django server"""
with virtualenv(env.virtualenv_path):
with cd(env.source_root):
sudo("./manage.py runserver 0.0.0.0:8000")
def install_python_dependencies():
"""Install Python dependencies."""
require.python.virtualenv(newebe_dir + "/virtualenv",
use_sudo=True, user=newebe_user)
with python.virtualenv(newebe_dir + "/virtualenv"):
newebedo("pip install --use-mirrors -r %s/deploy/requirements.txt" % \
newebe_dir)
def env():
if file_exists(VENV_DIR):
local('rm -rf {env}'.format(env=VENV_DIR))
local('python -m virtualenv {env}'.format(env=VENV_DIR))
with virtualenv(VENV_DIR):
local('python -m pip install --upgrade setuptools')
local('python -m pip install -r requirements.txt')
local('python -m pip install pytest')
def setup_server():
"""
Setup project on clean Ubuntu server
"""
sudo('apt-get update')
require.deb.packages([
'sudo',
'mc',
'git',
'nginx',
'supervisor',
'uwsgi',
'uwsgi-plugin-python',
'libpq-dev',
])
# Creating project paths
sudo('mkdir {env.project_path:s} -p'.format(env=env))
sudo('chown {env.project_user:s} {env.project_path:s}'.format(env=env))
sudo('mkdir {env.venv_path:s} -p'.format(env=env))
sudo('chown {env.project_user:s} {env.venv_path:s}'.format(env=env))
git.clone(env.repository_url, path=env.project_path, use_sudo=False, user=env.project_user)
git.checkout(path=env.project_path, branch=env.branch, use_sudo=False, user=env.project_user)
require.python.virtualenv(env.venv_path, use_sudo=False)
with virtualenv(env.venv_path):
require.python.requirements(os.path.join(env.project_path, 'requirements.txt'))
require.postgres.server()
require.postgres.user(env.db_user, password=env.db_pass, createdb=False, createrole=True)
require.postgres.database(env.db_name, env.db_user)
upload_template(
filename='conf/server_local.py',
destination='%(settings_path)s/local.py' % env,
context={
'db_name': env.db_name,
'db_pass': env.db_pass,
'db_user': env.db_user,
},
use_jinja=True
)
with cd(env.manage_path):
run('chmod ogu+x manage.py')
uwsgi_setup()
supervisor_setup()
sudo('rm /etc/nginx/sites-enabled/default')
nginx_setup()
manage('syncdb')
manage('migrate')
manage('collectstatic --noinput')
def train(json_dir=None, dev_loc=None, model_dir=None):
if json_dir is None:
json_dir = 'corpora/en/json'
if model_dir is None:
model_dir = 'models/en/'
with virtualenv(VENV_DIR):
with lcd(path.dirname(__file__)):
local('python bin/init_model.py lang_data/en/ corpora/en/ ' + model_dir)
local('python bin/parser/train.py %s %s' % (json_dir, model_dir))
def manage(command, noinput=True):
"""
Run manage command.
"""
with virtualenv(env.virtualenv_path):
if noinput:
run('%(manage_path)s/manage.py ' % env + command + ' --noinput')
else:
run('%(manage_path)s/manage.py ' % env + command)
def sentry(command, noinput=True):
"""
Run manage command.
"""
with virtualenv(env.virtualenv_path):
if noinput:
run('sentry --config=/srv/sites/sentry/sentry.conf.py ' % env + command + ' --noinput')
else:
run('sentry --config=/srv/sites/sentry/sentry.conf.py ' % env + command)
