def refresh_env():
# TODO: is there a way to get around having to put this at the top of every task?
server = None
for k, v in servers.itervalues():
if v['ip'] == env.host:
server = v
env.server_name = k
break
if server is None:
error('Could not find the current attempted server in config: %s' % env.host)
if 'stage' in server:
env.stage = server['stage']
else:
env.stage = 'prod'
django.settings_module('conf.%(stage)s.settings' % env)
from django.conf import settings
env.django_settings = settings
if 'os' in server:
env.os = server['os']
else:
env.os = 'ubuntu'
if env.os == 'ubuntu':
env.sudoers = 'wheel'
if 'deploy_type' in server:
env.deploy_type = server['deploy_type']
else:
env.deploy_type = 'shell'
def build_dev():
"""
Build the application for a development environment.
"""
require_secrets()
django.settings_module('texas_choropleth.settings.local')
build()
def django_syncdb():
"""
Synchronize/create database on remote host
"""
_setup_env()
sys.path.append(env.base_path)
django.settings_module('%(project_name)s.settings_%(role)s' % env)
settings = __import__('%(project_name)s.settings_%(role)s' % env)
# Not sure what's going on here .. but it fixes a problem..
if getattr(settings, 'settings_%(role)s' % env, False):
settings = getattr(settings, 'settings_%(role)s' % env, False)
dbconf = settings.DATABASES['default']
#if dbconf['ENGINE'].endswith('mysql'):
# _create_mysqldb(dbconf)
# Give apache write permission to the project directory
if dbconf['ENGINE'].endswith('sqlite3') and env.role != 'dev':
user = '******' in env.stage and env.stage['user'] or 'www-data'
sudo('chown -R %s %s' % (user, os.path.dirname(dbconf['NAME'])))
sudo('chmod 777 %s' % dbconf['NAME'])
if env.role in ['prod', 'demo']:
path = env.stage['path']
do = run
else:
path = env.base_path
do = local
if (yes_no_prompt("Do you want to syncdb ?")):
with (cd(os.path.join(path, env.project_name))):
do(env.venv_activate +
' && %s manage.py syncdb --noinput --settings=settings_%s' %
(env.venv_python, env.role))
def add_photos_to_db():
### DJANGO CONFIGURATION SETUP ###
import sys
from fabric.contrib import django
sys.path.append(PROJECT_ROOT)
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
django.settings_module('bostongreenmap.settings')
from parks.models import Park, Parkimage
### END DJANGO SETUP #############
db = anydbm.open('has_been_downloaded_2', 'r')
for url, park_id in curated_park_info():
if url in db:
try:
park = Park.objects.get(id=park_id)
except Exception as e:
print e
continue
### Create the image row and relate it to the Park
filename = db[url]
new_image_path = CURATE_PATH + '/' + filename
p_img, created = Parkimage.objects.get_or_create(image=new_image_path)
if created:
park.images.add(p_img)
# new_image = Parkimage.objects.get_or_create()
print new_image_path + " added to " + park.name + "with id " + str(park.id)
def settings(filepath, proj_name):
sys.path.append(os.path.join(
os.path.dirname(os.path.abspath(filepath)), proj_name))
from fabric.contrib.django import settings_module # NOQA
settings_module('settings')
from django.conf import settings as djsettings # NOQA
return djsettings
def set_db_data(env):
"""
set database variables to env
"""
# if using django override db settings
if env.settings_django_package:
import_non_local('django','django_std')
from django_std.conf import settings as django_settings
from fabric.contrib import django
django.settings_module(env.settings_django_package)
env.db_user = django_settings.DB_USER
env.db_pass = django_settings.DB_PASSWORD
env.db_name = django_settings.DB_NAME
env.site_url = django_settings.SITE_URL
# SET BACKUP VARS
from datetime import date
day = str(date.today()).split('-')[2]
month = str(date.today()).split('-')[1]
year = str(date.today()).split('-')[0]
env.year_backup_folder = env.backup_path + '/' + year
env.month_backup_folder = env.year_backup_folder + '/' + month
env.today_backup_folder = env.month_backup_folder + '/' + day
env.today_backup_gzip = env.today_backup_folder + '.tgz';
# SET CRON VARS
env.crontab_path_tmp = env.crontab_path + '.tmp'
def setup_jenkins_server():
"""
Complete setup for the ci server
"""
install_jenkins()
install_git()
install_nginx()
install_virtualenvwrapper()
install_postgres()
install_npm()
install_jshint()
install_csslint()
install_sloccount()
install('ttf-dejavu')
install_additional_packages()
upload_ci_nginx_settings()
django.settings_module(fabconf.CI_SETTINGS_PATH)
setup_database_from_settings()
add_pub_key_as_jenkins()
upload_env_keys()
nginx('restart')
def add_photos_to_db():
### DJANGO CONFIGURATION SETUP ###
import sys
from fabric.contrib import django
sys.path.append(PROJECT_ROOT)
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
django.settings_module('bostongreenmap.settings')
from parks.models import Park, Parkimage
### END DJANGO SETUP #############
db = anydbm.open('has_been_downloaded_2', 'r')
for img_url, os_id in curated_park_info():
if img_url in db:
try:
park = Park.objects.get(os_id=os_id)
except Exception as e:
print e
continue
### Create the image row and relate it to the Park
filename = db[img_url]
new_image_path = CURATE_PATH + '/' + filename
p_img, created = Parkimage.objects.get_or_create(image=new_image_path, default=True)
if created:
park.images.add(p_img)
# new_image = Parkimage.objects.get_or_create()
print new_image_path + " added to " + park.name + "with id " + str(park.id)
def compress_static():
pgreen('*** Check compress static files...')
# Add project dir to PYTHONPATH so "config.settings" object is found
# TODO: Since this code is extracted from project it doesn't work anymore
# trying to get project path relative to python package path. Refactor!!
# sys.path.append(os.path.join(
# os.path.dirname(os.path.realpath(__file__)),
# os.pardir,
# os.pardir))
current_path = '/home/vagrant/Development/projects/cirujanos'
sys.path.append(current_path)
django.settings_module('config.settings.%s' % env.environment)
from django.conf import settings as django_settings
# current_path = os.path.dirname(os.path.realpath(__file__))
pcyan('CURRENT PATH: %s\n' % current_path)
pcyan('COMPRESS_ENABLED: %s\n' % django_settings.COMPRESS_ENABLED)
pcyan('DEBUG: %s\n' % django_settings.DEBUG)
if (django_settings.COMPRESS_ENABLED and not django_settings.DEBUG):
pred('Compressing files..')
command = './manage.py compress --settings="config.settings.%s"; '
command_params = (env.environment, )
sudo_command(command, command_params)
else:
pred('WARNING: Django settings NOT allow compression')
def django_syncdb():
"""
Synchronize/create database on remote host
"""
_setup_env()
sys.path.append(env.base_path)
django.settings_module('%(project_name)s.settings_%(role)s' % env)
settings = __import__('%(project_name)s.settings_%(role)s' % env)
# Not sure what's going on here .. but it fixes a problem..
if getattr(settings, 'settings_%(role)s' % env, False):
settings = getattr(settings, 'settings_%(role)s' % env, False)
dbconf = settings.DATABASES['default']
if dbconf['ENGINE'].endswith('mysql'):
_create_mysqldb(dbconf)
# Give apache write permission to the project directory
elif dbconf['ENGINE'].endswith('sqlite3') and env.role != 'dev':
user = '******' in env.stage and env.stage['user'] or 'www-data'
sudo('chown -R %s %s' % (user, os.path.dirname(dbconf['NAME'])))
sudo('chmod 777 %s' % dbconf['NAME'])
if env.role in ['prod', 'demo']:
path = env.stage['path']
do = run
else:
path = env.base_path
do = local
if (yes_no_prompt("Do you want to syncdb ?")):
with(cd(os.path.join(path, env.project_name))):
do(env.venv_activate +' && %s manage.py syncdb --noinput --settings=settings_%s' % (env.venv_python, env.role))
def wrapper(*args, **kwargs):
if env.host == "localhost":
_env_set("localhost")
for key, value in fabsettings.PROJECT_SITES.iteritems():
# key is the identifier, e.g. "dev", "prod", etc etc
for k, v in value.iteritems():
if v == env.host:
# iterate in the nested dictionary
# if the nested dictionary's name is equivalent to env.host
# we will set our global state env with _env_set(key)
_env_set(key)
with prefix(env.activate):
from fabric.contrib import django
if env.host == "localhost":
_settings = '.'.join(
[env.project_name, 'settings'])
else:
_settings = '.'.join(
[env.project_name, 'settings', env.target])
django.settings_module(_settings)
if not env.get('distro'):
_env_set_distro()
return f(*args, **kwargs)
def insertIntoGallery(
gallery_title,
gallery_slug,
screenshot,
title,
slug,
gallery_description="",
gallery_tags="",
caption="",
tags="",
fab_dir='%s/.fabric-bolt' % (os.path.expanduser('~/'))
):
# Add custom fabric-bolt settings directory
sys.path.insert(0, fab_dir)
# Utilize django within fabfile
# Load custom fabric-bolt settings file
django.settings_module('settings')
# Loads the django Models
get_wsgi_application()
# Once loaded we can reference them
from photologue.models import Photo
from photologue.models import Gallery
file = open(screenshot, 'rb')
data = file.read()
# First Generate or Retrieve the Photo Model and save or update it
try:
photo = Photo.objects.get(slug=slug)
photo.date_added = datetime.now()
photo.date_taken = datetime.now()
print("~~~ FOUND existing Screenshot ~~~")
except Photo.DoesNotExist:
photo = Photo(title=title, slug=slug, caption=caption, is_public=True, tags=tags,)
print("~~~ CREATED new Screenshot ~~~")
try:
photo.image.save(os.path.basename(screenshot), ContentFile(data))
except FieldError:
# For some reason a field, 'photo,' is being passed to model as a field.
pass
print("~~~ SAVED Screenshot ~~~")
# Now Create or Retrieve the named Gallery and add the photo to it.
gallery = None
try:
gallery = Gallery.objects.get(title=gallery_title)
print("~~~ FOUND existing Screenshot Gallery ~~~")
except Gallery.DoesNotExist:
gallery = Gallery(title=gallery_title, slug=gallery_slug, description=gallery_description, is_public=True, tags=gallery_tags,)
gallery.save()
print("~~~ CREATED new Screenshot Gallery ~~~")
if gallery:
gallery.photos.add(photo)
print("~~~ Added Screenshot to Gallery ~~~")
print("<a target=\"_parent\" href=\"/photologue/gallery/%s\">View Screenshot Gallery %s</a>") % (gallery_title, gallery_title)
# Reset the syspath
sys.path.remove(fab_dir)
def import_remote_db():
print(green('IMPORTING REMOTE DB ...'))
sys.path.append(os.path.dirname(__file__))
django.settings_module('{{ project_name }}.settings')
from django.conf import settings
engine = settings.DATABASES['default']['ENGINE'].split('.')[-1]
if engine == 'mysql':
with cd(env.tmp_dir):
db_file = 'mysql-%s.gz' % env.db_name
run('mysqldump -uadmin -p %s | gzip -9 > %s' % (env.db_name, db_file))
get(db_file, '../')
run('rm %s' % db_file)
local('gunzip ../%s' % db_file)
local('mysql -uroot -p %s < %s' % (env.db_name, '../%s' % db_file.replace('.gz', '')))
elif engine == 'postgresql_psycopg2':
with cd(env.tmp_dir):
db_file = 'psql-%s.gz' % env.db_name
run('pg_dump -O -F p -c -U postgres %s | gzip -9 > %s' % (env.db_name, db_file))
get(db_file, '../../')
run('rm %s' % db_file)
local('gunzip ../../%s' % db_file)
local('psql -U postgres -d %s -f %s' % (env.db_name, '../../%s' % db_file.replace('.gz', '')))
def loc():
'''Set config to local (requires SSH server running on localhost)
'''
env.dev_settings_file = "settings_local.py"
django.settings_module(os.path.splitext(env.dev_settings_file)[0])
env.os = env.local_os
def local_clear_database():
"""Recreate schema"""
django.settings_module(env.django_project)
from django.conf import settings
env.role = settings.DATABASES['default']['USER']
env.db_password = settings.DATABASES['default']['PASSWORD']
local('echo "DROP SCHEMA public CASCADE;CREATE SCHEMA public AUTHORIZATION %(role)s;GRANT ALL ON SCHEMA public TO %(role)s;" | PGPASSWORD="******" bin/django dbshell' % env)
def load_base_env():
django.settings_module('bcpp.settings')
CONFIG_FILENAME = 'bcpp.conf'
DOWNLOADS_DIR = '~/Downloads'
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
ETC_CONFIG_PATH = os.path.join(BASE_DIR, 'fabfile', 'etc')
FABRIC_CONFIG_PATH = os.path.join(BASE_DIR, 'fabfile', 'conf',
'fabric.conf')
timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
env.log_folder = os.path.expanduser('~/fabric/{}'.format(timestamp))
if not os.path.exists(env.log_folder):
os.makedirs(env.log_folder)
print('log_folder', env.log_folder)
update_env_secrets(path=ETC_CONFIG_PATH)
env.roledefs = roledefs
env.hosts, env.passwords = get_hosts(path=ETC_CONFIG_PATH,
gpg_filename='hosts.conf.gpg')
env.hostname_pattern = hostname_pattern
env.device_ids = get_device_ids()
env.prompts = prompts
env.prompts.update({'Enter password: '******'hosts.txt'), 'a') as f:
f.write('{}\n'.format(',\n'.join([h for h in env.hosts])))
def _get_settings(conf=None):
# do this here. django settings cannot be imported more than once...probably.
# still dont really get the mess here.
if not conf:
conf = env.project_conf
django.settings_module(conf)
from django.conf import settings
return settings
def installdev():
django.settings_module('django-heroku.settings.dev')
local('pip install -r requirements.txt')
local('python manage.py migrate')
local('python manage.py installwatson')
local('python manage.py buildwatson')
# fixture_dev()
local('bower install --config.interactive=false')
def batch():
django.settings_module('django-heroku.settings.dev')
local('python manage.py batch batch_passage_en_presouscription')
local('python manage.py batch batch_fin_presouscription')
local('python manage.py batch batch_souscription')
local('python manage.py batch batch_fin_campagne')
local('python manage.py batch batch_task_mail_auteur_demandes_new_souscription')
local('python manage.py batch batch_detecte_nouveaux_livres')
def _get_database_name():
"""Gets database dictionary either from ENVS or form Django settings.py"""
_require_environment()
database = env.project.get('database', None)
if not database:
django.settings_module(_interpolate('%(project)s.%(settings)s'))
database = django_settings.DATABASES['default']
return database
def recreate_initial_data():
django.settings_module('settings_debug')
if os.path.isfile('inhouse/fixtures/_initial_data.json'):
local('mv inhouse/fixtures/_initial_data.json inhouse/fixtures/_initial_data.json.old')
local('rm -f dev.db')
local('python manage.py syncdb --noinput')
local('python manage.py loaddata %s' % ' '.join(INITIAL_FIXTURES))
local('python manage.py dumpdata -a -e inhouse.AuthUserGroup -e contenttypes.ContentType --indent=2> inhouse/fixtures/_initial_data.json')
def _get_settings(conf=None):
# do this here. django settings cannot be imported more than once...probably.
# still dont really get the mess here.
if not conf:
conf = env.project_conf
django.settings_module(conf)
from django.conf import settings
return settings
def import_sightings(json_file='ias_ess_dump.json',
skip_images=False,
pks__gte=0,
pks__lte=None):
"""Put sightings from a fixture file into db.
:param settings: change to change settings file inside ashtag.settings...
:param json_file: change to use locally dumped json
:param skip_images: set True to ignore images
:param pks__gte: set > 0 to limit import (or do an update, etc.)
:param pks__lte: set not None to limit import (or do an update, etc.)
"""
django.settings_module(os.environ.get('DJANGO_SETTINGS_MODULE'))
from ashtag.apps.core.models import Sighting, Tree
dumped_json = ""
with open(json_file, 'r') as fp:
dumped_json = fp.read()
sightings = json.loads(dumped_json)
ash_sightings = filter(lambda x: x['fields']['taxon'] == 100004, sightings)
not_rejects = filter(lambda x: not x['fields']['rejected'], ash_sightings)
have_emails = filter(lambda x: x['fields']['email'], not_rejects)
only_pks = sorted(filter(lambda x: x['pk'] >= int(pks__gte), have_emails),
key=lambda x: x['pk'])
if pks__lte is not None:
only_pks = filter(lambda x: x['pk'] <= int(pks__lte), only_pks)
print red("%s sightings don't have emails" %
(len(not_rejects) - len(have_emails)))
unknown = Sighting.DISEASE_STATE.unknown
diseased = Sighting.DISEASE_STATE.diseased
print green("Processing %s sightings with %s <= PK <= %s" %
(len(only_pks), pks__gte, pks__lte))
for sighting in only_pks:
fields = sighting['fields']
tree = Tree.objects.create(location=fields['location'],
creator_email=fields['email'])
s = Sighting.objects.create(
tree=tree,
location=fields['location'],
notes="Imported from IAS-ESS: http://ias-ess.org/ias/sighting/%s" %
sighting['pk'],
disease_state=diseased if fields['verified'] else unknown,
created=fields['datetime'],
creator_email=fields['email'])
if not skip_images:
image = urllib.urlretrieve("http://ias-ess.org/static/media/%s" %
fields['photo'])
s.image.save(os.path.basename(fields['photo']),
File(open(image[0])))
s.save()
if len(only_pks):
print green("Made %s Trees and Sightings, latest PK = %s" %
(len(only_pks), only_pks[-1]['pk']))
else:
print red("Nothing made!")
def erase_all_local():
"""Nuke the Tree/Sighting objects."""
django.settings_module(os.environ.get('DJANGO_SETTINGS_MODULE'))
from ashtag.apps.core.models import Sighting, Tree
if confirm(red("Really delete all local Trees and Sightings?"), default=False):
Tree.objects.all().delete()
Sighting.objects.all().delete()
else:
abort("Not continuing with anything.")
def erase_all_local(settings=SETTINGS):
"""Nuke the Tree/Sighting objects."""
django.settings_module('ashtag.settings.%s' % settings)
from ashtag.apps.core.models import Sighting, Tree
if confirm(red("Really delete all local Trees and Sightings?"), default=False):
Tree.objects.all().delete()
Sighting.objects.all().delete()
else:
abort("Not continuing with anything.")
def test(module='tests'):
"""
Runs all unit tests for OpenSlides using coverage.
The settings file in the tests directory is used, therefor the
environment variable DJANGO_SETTINGS_MODULE is set to 'tests.settings'.
"""
django.settings_module('tests.settings')
local('coverage run ./manage.py django test %s' % module)
def env_setup(config=env.CONFIG,settings_module='codalab.settings'):
if config is None:
sys.path.append('.')
os.environ['DJANGO_CONFIGURATION'] = config
os.environ["DJANGO_SETTINGS_MODULE"] = settings_module
from fabric.contrib import django
from configurations import importer
importer.install()
django.settings_module(settings_module)
from django.conf import settings as django_settings
env.roledefs = django_settings.DEPLOY_ROLES
env.django_settings = django_settings
@task
def local(**kwargs):
env.run = lrun
env_setup(**kwargs)
@task
def remote(**kwargs):
env.run = run
env_setup(**kwargs)
@task
def clone_repo(repo_url='https://github.com/codalab/codalab.git',path=env.conf.DEPLOY_PATH):
env.run('git clone %s %s' % (repo_url, path))
@task
def provision(config='Dev'):
clone_repo()
with cd(env.conf.DEPLOY_PATH):
sudo('/bin/bash codalab/scripts/provision %s' % env.conf.DEPLOY_USER)
sudo('python manage.py config_gen --configuration=%s' % config,
user=env.conf.DEPLOY_USER)
@task
def bootstrap():
make_virtualenv(path='venv', system_site_packages=False)
with virtualenv('venv'):
run('pip install --upgrade pip')
run('pip install --upgrade distribute')
run('rm -rf codalab_src')
run("git clone %s codalab_src" % env.django_settings.SOURCE_GIT_URL)
put(os.path.join(THIS_SETTINGS_DIR,'deploy.py'), 'codalab_src/codalab/codalab/settings')
with virtualenv('venv'):
run('cd codalab_src && bash ./requirements')
@task
def install():
env.run('deploymentcmd')
def download_media(dirname):
remote_media_path = os.path.join(os.path.dirname(env.code_dir), 'media')
remote_dir = os.path.join(remote_media_path, dirname)
sys.path.append(os.path.dirname(__file__))
django.settings_module('bridges.settings')
from django.conf import settings
get(remote_dir, settings.MEDIA_ROOT)
def test(module='tests'):
"""
Runs all unit tests for OpenSlides using coverage.
The settings file in the tests directory is used, therefor the
environment variable DJANGO_SETTINGS_MODULE is set to 'tests.settings'.
"""
django.settings_module('tests.settings')
local('coverage run ./manage.py django test %s' % module)
def import_sightings(settings=SETTINGS, json_file='ias_ess_dump.json',
skip_images=False, pks__gte=0):
"""Put sightings from a fixture file into db.
:param settings: change to change settings file inside ashtag.settings...
:param json_file: change to use locally dumped json
:param skip_images: set True to ignore images
:param pks__gte: set > 0 to limit import (or do an update, etc.)
"""
django.settings_module('ashtag.settings.%s' % settings)
from ashtag.apps.core.models import Sighting, Tree
dumped_json = ""
with open(json_file, 'r') as fp:
dumped_json = fp.read()
sightings = json.loads(dumped_json)
ash_sightings = filter(lambda x: x['fields']['taxon'] == 100004, sightings)
not_rejects = filter(lambda x: not x['fields']['rejected'], ash_sightings)
have_emails = filter(lambda x: x['fields']['email'], not_rejects)
only_pks = sorted(
filter(lambda x: x['pk'] >= int(pks__gte), have_emails),
key=lambda x: x['pk'])
print red("%s sightings don't have emails" % (
len(not_rejects) - len(have_emails)
))
unknown = Sighting.DISEASE_STATE.unknown
diseased = Sighting.DISEASE_STATE.diseased
print green("Processing %s sightings with PK >= %s" % (
len(only_pks), pks__gte))
for sighting in only_pks:
fields = sighting['fields']
tree = Tree.objects.create(
location=fields['location'],
creator_email=fields['email']
)
s = Sighting.objects.create(
tree=tree,
location=fields['location'],
notes="Imported from IAS-ESS: http://ias-ess.org/ias/sighting/%s" % sighting['pk'],
disease_state=diseased if fields['verified'] else unknown,
created=fields['datetime'],
creator_email=fields['email']
)
if not skip_images:
image = urllib.urlretrieve(
"http://ias-ess.org/static/media/%s" % fields['photo'])
s.image.save(
os.path.basename(fields['photo']),
File(open(image[0]))
)
s.save()
print green("Made %s Trees and Sightings, latest PK = %s" % (
len(pks__gte), only_pks[-1]['pk']))
def run_nose():
# use test settings, not actual settings
django.settings_module('pysis.settings.test_settings')
local('nosetests -v -d -x ' +
'--with-doctest ' +
'--with-django ' +
'--with-djangoliveserver ' +
'--with-selenium ' +
'apps tests')
def test(module='tests'):
"""
Runs the unit tests.
"""
sys.path.insert(0, '')
django.settings_module('tests.settings')
sys.argv.pop()
sys.argv.extend(['test', module])
from django.core import management
management.execute_from_command_line()
def erase_all_local():
"""Nuke the Tree/Sighting objects."""
django.settings_module(os.environ.get('DJANGO_SETTINGS_MODULE'))
from ashtag.apps.core.models import Sighting, Tree
if confirm(red("Really delete all local Trees and Sightings?"),
default=False):
Tree.objects.all().delete()
Sighting.objects.all().delete()
else:
abort("Not continuing with anything.")
def build_prod():
"""
Build the application for the production environment.
"""
require_secrets()
django.settings_module('texas_choropleth.settings.production')
build()
print(blue("\n [ Collecting Staticfiles ]"))
local('./src/manage.py collectstatic --noinput')
done()
def get_settings():
from fabric.contrib.django import settings_module
settings_module(env.django_settings)
# mimic Django's dual path :/
if env.project_root not in sys.path:
sys.path.append(os.path.join(env.project_root))
if env.project_django_root not in sys.path:
sys.path.append(os.path.join(env.project_django_root))
from django.conf import settings
return settings
def deploy(opt=False):
server_dict = {
'action': '',
}
env.user = '******'
if not opt:
env.hosts = server_dict.values()
else:
env.hosts = [server_dict[str(opt)]]
env.activate = 'source ~/readbox_env/bin/activate'
env.key_filename = '~/.ssh/server_rsa'
django.settings_module('manhattan.settings')
def _drop_database_mysql():
"""Destroy (DROP) MySQL database according to env's settings.py"""
# Unless explicitly provided, uses local Django settings to
# extract username/password to access remote database
database = env.project.get('database', None)
if not database:
django.settings_module(env.project['settings'])
database = django_settings.DATABASES['default']
# Drops database
with settings(hide('warnings'), warn_only=True):
result = run(MYSQL_PREFIX % "\"DROP DATABASE %(NAME)s;\"" % database)
def _drop_database_mysql():
"""Destroy (DROP) MySQL database according to env's settings.py"""
# Unless explicitly provided, uses local Django settings to
# extract username/password to access remote database
database = env.project.get('database', None)
if not database:
django.settings_module(env.project['settings'])
database = django_settings.DATABASES['default']
# Drops database
with settings(hide('warnings'), warn_only=True):
result = run(MYSQL_PREFIX % "\"DROP DATABASE %(NAME)s;\"" % database)
def deploy(opt=False):
server_dict = {
'action': '',
}
env.user = '******'
if not opt:
env.hosts = server_dict.values()
else:
env.hosts = [server_dict[str(opt)]]
env.activate = 'source ~/readbox_env/bin/activate'
env.key_filename = '~/.ssh/server_rsa'
django.settings_module('manhattan.settings')
def restore_project(filename):
"""
Restore server's database with .sql file contained in filename
"""
_require_environment()
# Confirms action
if not console.confirm(
'ATTENTION! This will destroy current database! Confirm?',
default=False):
return
# Unless explicitly provided, uses local Django settings to
# extract username/password to access remote database
database = env.project.get('database', None)
if not database:
django.settings_module(env.project['settings'])
database = django_settings.DATABASES['default']
# Remote side
with prefix(_django_prefix()):
with cd(_django_project_dir()):
# Uploads tar file
tarfile = os.path.basename(filename)
basename = tarfile[:tarfile.index('.tar.gz')]
if console.confirm('Upload backup?'):
put(filename, '../backup/%s' % tarfile)
# Drop and recreate current database
_drop_database_mysql()
_setup_project_mysql()
# Restore MySQL
# To avoid silly mistakes, instead of using project's user & password, uses root's
with cd('../'):
run('tar -xzvf backup/%s' % tarfile)
run('mysql -u root -p %s < backup/%s/%s.sql' % (
#database['USER'],
#database['PASSWORD'],
database['NAME'],
basename,
env.project['project'],
))
# Restore extra files
extra_backup_files = env.project.get('extra_backup_files', [])
for file in extra_backup_files:
run('cp -R ../backup/%s/%s ./%s' %
(basename, os.path.basename(file), os.path.dirname(file)))
# Removes uncompressed files, but leaves .tar.gz
run('rm -rf ../backup/%s' % basename)
def fabenv(environ, path='environ.json'):
with open(path) as configfile:
conf = json.load(configfile)[environ]
defaults = DEFAULTS.copy()
django.settings_module(conf['django_settings'])
# for now, also use env to store other global state
env.update(defaults)
env.update(conf)
# and finally update with the env dict
env.update(conf.pop('env', {}))
def backup_project():
"""
Backup server's database and copy tar.gz to local ../backup dir
"""
_require_environment()
# Unless explicitly provided, uses local Django settings to
# extract username/password to access remote database
database = env.project.get('database', None)
if not database:
django.settings_module(env.project['settings'])
database = django_settings.DATABASES['default']
# Remote side
with prefix(_django_prefix()):
with cd(_django_project_dir()):
# Creates dir to store backup, avoiding existing similar names
dirname = '../backup/%s_%s' % (
datetime.date.today().strftime('%Y%m%d'), env.environment)
path = dirname
index = 0
while files.exists(path) or files.exists('%s.tar.gz' % path):
index += 1
path = '%s.%s' % (dirname, index)
run('mkdir -p %s' % path)
# Backup MySQL
run('mysqldump %s -u %s -p%s %s > %s/%s.sql' % (
'-h %s' %
database['HOST'] if database.get('HOST', None) else '',
database['USER'],
database['PASSWORD'],
database['NAME'],
path,
env.project['project'],
))
# Backup extra files
extra_backup_files = env.project.get('extra_backup_files', [])
for file in extra_backup_files:
run('cp -R %s %s/' % (file, path))
# Create .tar.gz and removes uncompressed files
with hide('stdout'):
run('tar -czvf %s.tar.gz %s/' % (path, path))
run('rm -rf %s/' % path)
# Download backup?
if console.confirm('Download backup?'):
return get('%s.tar.gz' % path, '../backup')
def mysql_db(local_path='/backups/db/'):
"""
usage: `fab -r <server_name> backups.mysql_db /path/to/backup/folder/`
backups the target's database to local destination.
backup path defaults to /backups/db/[target]/.
1. creates the target folder if doesnt exists
2. by default keeps 30 days of backups
"""
with prefix(env.activate):
if env.host == "localhost":
_settings = '.'.join([env.project_name, 'settings'])
else:
_settings = '.'.join([env.project_name, 'settings', env.target])
print 1231231, _settings
django.settings_module(_settings)
from django.conf import settings
print 123123123
print settings
print settings
timestamp = datetime.datetime.now().strftime("%y-%m-%d_%h-%m-%s")
fname = '%(dump_path)s%(database)s-backup-%(date)s.xml.gz' % {
'dump_path': local_path,
'database': settings.databases['default']['name'],
'date': timestamp,
}
if not os.path.exists(local_path):
os.makedirs(local_path)
if exists(fname):
run('rm "%s"' % fname)
print("dumping %s to %s" % (settings.databases['default']['name'], fname))
run('mysqldump -u %(username)s -p%(password)s %(database)s --xml | '
'gzip > %(fname)s' % {'username': settings.databases['default']['user'],
'password': settings.databases['default']['password'],
'database': settings.databases['defaukt']['name'],
'fname': fname})
print("transferring...")
get(fname, local_path)
print("removing remote dump file...")
run("rm %(dump_file)s" % {'dump_file': fname})
print("finish")
def _setup_project_mysql():
"""Creates MySQL database according to env's settings.py"""
# Unless explicitly provided, uses local Django settings to
# extract username/password to access remote database
database = env.project.get('database', None)
if not database:
django.settings_module(env.project['settings'])
database = django_settings.DATABASES['default']
# Create database & user, if not already there
with settings(hide('warnings'), warn_only=True):
result = run(MYSQL_PREFIX % "\"CREATE DATABASE %(NAME)s DEFAULT CHARACTER SET utf8;\"" % database)
if result.succeeded:
run(MYSQL_PREFIX % "\"GRANT ALL ON %(NAME)s.* TO '%(USER)s'@'localhost' IDENTIFIED BY '%(PASSWORD)s';\"" % database)
def restore_local_from_file(path="db.dump", data_only=True):
""" Drops local database, and restores data from file """
data_only = prep_bool_arg(data_only)
# make it look like a local only command
env.hosts = []
django.settings_module('secondfunnel.settings.dev')
from django.conf import settings
# Dump our local database to backup, then flush it out
now = datetime.now()
local('fab database.dump:path=db_backup-%s' %
(now.strftime('%Y-%m-%dT%H:%M.dump')))
# clear all existing data from database
flush_local()
# load downloaded database
local_restore(path=path, data_only=data_only)
def gravatars():
from fabric.contrib import django
django.settings_module('settings')
from django.contrib.auth.models import User
import requests
_display_message('Updating all users\' Gravatar flags')
################
for user in User.objects.all().order_by('id'):
_display_message('{}. {}'.format(
user.pk, user.username.encode('ascii', 'ignore')))
################
email_md5 = hashlib.md5(user.email.lower()).hexdigest()
print 'Email MD5: {}'.format(email_md5)
greq = requests.get(
'https://secure.gravatar.com/avatar/{}?s=50&d=404'.format(
email_md5))
if greq.status_code == 404:
has_gravatar = False
else:
has_gravatar = True
profile = user.profile
profile.has_gravatar = has_gravatar
profile.save()
try:
from fabric.colors import green, red
if has_gravatar:
print 'Has Gravatar: {}'.format(green(has_gravatar))
else:
print 'Has Gravatar: {}'.format(red(has_gravatar))
except ImportError:
print 'Has Gravatar: {}'.format(has_gravatar)
def dev_db(assume_yes=False):
"""Recreate development database."""
django.settings_module('settings_debug')
conf = dj_settings.DATABASES.get('default')
if not conf:
abort(colors.red('No default database configured.'))
if conf.get('ENGINE', '').endswith('sqlite3'):
path = conf.get('NAME')
if os.path.isfile(path):
if not assume_yes and not confirm('Delete existing database?'):
abort('Aborting at user request.')
os.remove(path)
elif conf.get('ENGINE', '').endswith('psycopg2'):
with settings(warn_only=True):
local('dropdb -U %s %s' % (conf.get('USER'), conf.get('NAME')))
local('createdb -U %s %s' % (conf.get('USER'), conf.get('NAME')))
load_initial_data()
load_dev_data()
def _setup_project_mysql():
"""Creates MySQL database according to env's settings.py"""
# Unless explicitly provided, uses local Django settings to
# extract username/password to access remote database
database = env.project.get('database', None)
if not database:
django.settings_module(env.project['settings'])
database = django_settings.DATABASES['default']
# Create database & user, if not already there
with settings(hide('warnings'), warn_only=True):
result = run(
MYSQL_PREFIX %
"\"CREATE DATABASE %(NAME)s DEFAULT CHARACTER SET utf8;\"" %
database)
if result.succeeded:
run(MYSQL_PREFIX %
"\"GRANT ALL ON %(NAME)s.* TO '%(USER)s'@'localhost' IDENTIFIED BY '%(PASSWORD)s';\""
% database)
def load_base_env():
django.settings_module('bcpp_clinic.settings')
CONFIG_FILENAME = 'bcpp-clinic.conf'
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
ETC_CONFIG_PATH = os.path.join(BASE_DIR, 'fabfile', 'etc')
FABRIC_CONFIG_PATH = os.path.join(
BASE_DIR, 'fabfile', 'conf', 'fabric.conf')
timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
env.log_folder = os.path.expanduser('~/fabric/{}'.format(timestamp))
if not os.path.exists(env.log_folder):
os.makedirs(env.log_folder)
print('log_folder', env.log_folder)
update_env_secrets(path=ETC_CONFIG_PATH)
env.hosts, env.passwords = get_hosts(
path=ETC_CONFIG_PATH, gpg_filename='hosts.conf.gpg')
env.prompts = prompts
env.prompts.update({'Enter password: ': env.dbpasswd})
def get_arguments():
""" parse arguments passed to postgres command """
django.settings_module('secondfunnel.settings.{0}'.format(
env.environment.lower()))
from django.conf import settings
# workaround for running all from locally, to remote machines
# but still reading the os environment variables
# if there is hosts of course
env.remote_environ = read_remote_env()
if env.hosts and 'RDS_DB_NAME' in env.remote_environ:
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'NAME': env.remote_environ['RDS_DB_NAME'],
'USER': env.remote_environ['RDS_USERNAME'],
'PASSWORD': env.remote_environ['RDS_PASSWORD'],
'HOST': env.remote_environ['RDS_HOSTNAME'],
'PORT': env.remote_environ['RDS_PORT'],
}
}
else:
DATABASES = settings.DATABASES
password = '******'.format(
DATABASES['default']['PASSWORD'])
arguments = '--host=%s --port=%s --username=%s --dbname=%s' % (
DATABASES['default']['HOST'], DATABASES['default']['PORT'],
DATABASES['default']['USER'], DATABASES['default']['NAME'])
if is_windows():
password = ''
arguments = '-W ' + arguments
return {
'password': password,
'arguments': arguments,
}
def wrap_function(*args, **kwargs):
if len(args) > 0 and 'env' in kwargs:
raise TypeError(
'%s() got multiple values for keyword argument "env"' %
function.__name__)
environ = DEFAULT_ENV
if len(args) > 0:
environ = args[0]
else:
if 'env' in kwargs:
environ = kwargs['env']
django.settings_module('{}.config.settings.{}'.format(
DJANGO_PROJECT, environ))
try:
# logger.debug('args = %s' % str(args))
# logger.debug('kwargs = %s' % str(kwargs))
return function(*args, **kwargs)
except Exception as e:
raise e
from __future__ import with_statement
from fabric.api import *
from fabric.contrib import django
import os
SETTINGS_FILE = "app.settings_my"
django.settings_module(SETTINGS_FILE)
PORT = os.environ.get('PORT', 9000)
# run server locally
def start():
local("python manage.py runserver 127.0.0.1:%s --traceback --settings=%s" %
(PORT, SETTINGS_FILE))
import os
import sys
from lib.utils import log
from .other import log_success
from fabric.api import *
from fabric.contrib import django
django.settings_module("{{ project_name }}.settings")
from django.conf import settings
from {{ project_name }}.settings.production import (
AWS_BUCKET_NAME,
AWS_MEDIA_BUCKET_NAME,
AWS_STAGING_BUCKET_NAME,
VERBOSE_APP_NAME,
BUILD_DIR
)
"""
Deployment Tasks
================
"""
project_name = "{{ project_name }}"
pwd = os.path.dirname(__file__)
gzip_path = '{0}/{1}/gzip/static/'.format(pwd, project_name)
static_path = '{0}/{1}/static/'.format(pwd, project_name)
@task
from fabric.context_managers import cd, prefix
from fabric.contrib.files import exists
from fabric.api import run, env
from fabric.decorators import task, runs_once, parallel
from fabric.contrib import django
from fabric.operations import local
from fabric.tasks import execute
from django.conf import settings
from slackclient import SlackClient
# Load the django settings. This needs to read the env-variables and setup
# django-configurations, before the settings_module can be accessed.
envdir.read(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'envs'))
configurations.setup()
django.settings_module('qcat.settings')
ENVIRONMENTS = {
'develop': {
'branch': 'develop',
'label': 'dev',
'host_string': settings.HOST_STRING_DEV,
'touch_file': settings.TOUCH_FILE_DEV,
'use_deploy_announcement': False,
},
'demo': {
'branch': 'master',
'label': 'live',
'host_string': settings.HOST_STRING_DEMO,
'touch_file': settings.TOUCH_FILE_DEMO,
from boto.s3.key import Key
from boto.ec2.connection import EC2Connection
from fabric.contrib import django
from fabric.state import connections
from vendor import yaml
from pprint import pprint
import os
import time
import sys
import re
try:
import dop.client
except ImportError:
print "Digital Ocean's API not loaded"
django.settings_module('settings')
try:
from django.conf import settings as django_settings
except ImportError:
print " ---> Django not installed yet."
django_settings = None
# ============
# = DEFAULTS =
# ============
env.NEWSBLUR_PATH = "~/projects/newsblur"
env.SECRETS_PATH = "~/projects/secrets-newsblur"
env.VENDOR_PATH = "~/projects/code"
env.user = '******'
