def refresh_env(): # TODO: is there a way to get around having to put this at the top of every task? server = None for k, v in servers.itervalues(): if v['ip'] == env.host: server = v env.server_name = k break if server is None: error('Could not find the current attempted server in config: %s' % env.host) if 'stage' in server: env.stage = server['stage'] else: env.stage = 'prod' django.settings_module('conf.%(stage)s.settings' % env) from django.conf import settings env.django_settings = settings if 'os' in server: env.os = server['os'] else: env.os = 'ubuntu' if env.os == 'ubuntu': env.sudoers = 'wheel' if 'deploy_type' in server: env.deploy_type = server['deploy_type'] else: env.deploy_type = 'shell'
def build_dev(): """ Build the application for a development environment. """ require_secrets() django.settings_module('texas_choropleth.settings.local') build()
def django_syncdb(): """ Synchronize/create database on remote host """ _setup_env() sys.path.append(env.base_path) django.settings_module('%(project_name)s.settings_%(role)s' % env) settings = __import__('%(project_name)s.settings_%(role)s' % env) # Not sure what's going on here .. but it fixes a problem.. if getattr(settings, 'settings_%(role)s' % env, False): settings = getattr(settings, 'settings_%(role)s' % env, False) dbconf = settings.DATABASES['default'] #if dbconf['ENGINE'].endswith('mysql'): # _create_mysqldb(dbconf) # Give apache write permission to the project directory if dbconf['ENGINE'].endswith('sqlite3') and env.role != 'dev': user = '******' in env.stage and env.stage['user'] or 'www-data' sudo('chown -R %s %s' % (user, os.path.dirname(dbconf['NAME']))) sudo('chmod 777 %s' % dbconf['NAME']) if env.role in ['prod', 'demo']: path = env.stage['path'] do = run else: path = env.base_path do = local if (yes_no_prompt("Do you want to syncdb ?")): with (cd(os.path.join(path, env.project_name))): do(env.venv_activate + ' && %s manage.py syncdb --noinput --settings=settings_%s' % (env.venv_python, env.role))
def add_photos_to_db(): ### DJANGO CONFIGURATION SETUP ### import sys from fabric.contrib import django sys.path.append(PROJECT_ROOT) os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' django.settings_module('bostongreenmap.settings') from parks.models import Park, Parkimage ### END DJANGO SETUP ############# db = anydbm.open('has_been_downloaded_2', 'r') for url, park_id in curated_park_info(): if url in db: try: park = Park.objects.get(id=park_id) except Exception as e: print e continue ### Create the image row and relate it to the Park filename = db[url] new_image_path = CURATE_PATH + '/' + filename p_img, created = Parkimage.objects.get_or_create(image=new_image_path) if created: park.images.add(p_img) # new_image = Parkimage.objects.get_or_create() print new_image_path + " added to " + park.name + "with id " + str(park.id)
def settings(filepath, proj_name): sys.path.append(os.path.join( os.path.dirname(os.path.abspath(filepath)), proj_name)) from fabric.contrib.django import settings_module # NOQA settings_module('settings') from django.conf import settings as djsettings # NOQA return djsettings
def set_db_data(env): """ set database variables to env """ # if using django override db settings if env.settings_django_package: import_non_local('django','django_std') from django_std.conf import settings as django_settings from fabric.contrib import django django.settings_module(env.settings_django_package) env.db_user = django_settings.DB_USER env.db_pass = django_settings.DB_PASSWORD env.db_name = django_settings.DB_NAME env.site_url = django_settings.SITE_URL # SET BACKUP VARS from datetime import date day = str(date.today()).split('-')[2] month = str(date.today()).split('-')[1] year = str(date.today()).split('-')[0] env.year_backup_folder = env.backup_path + '/' + year env.month_backup_folder = env.year_backup_folder + '/' + month env.today_backup_folder = env.month_backup_folder + '/' + day env.today_backup_gzip = env.today_backup_folder + '.tgz'; # SET CRON VARS env.crontab_path_tmp = env.crontab_path + '.tmp'
def setup_jenkins_server(): """ Complete setup for the ci server """ install_jenkins() install_git() install_nginx() install_virtualenvwrapper() install_postgres() install_npm() install_jshint() install_csslint() install_sloccount() install('ttf-dejavu') install_additional_packages() upload_ci_nginx_settings() django.settings_module(fabconf.CI_SETTINGS_PATH) setup_database_from_settings() add_pub_key_as_jenkins() upload_env_keys() nginx('restart')
def add_photos_to_db(): ### DJANGO CONFIGURATION SETUP ### import sys from fabric.contrib import django sys.path.append(PROJECT_ROOT) os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' django.settings_module('bostongreenmap.settings') from parks.models import Park, Parkimage ### END DJANGO SETUP ############# db = anydbm.open('has_been_downloaded_2', 'r') for img_url, os_id in curated_park_info(): if img_url in db: try: park = Park.objects.get(os_id=os_id) except Exception as e: print e continue ### Create the image row and relate it to the Park filename = db[img_url] new_image_path = CURATE_PATH + '/' + filename p_img, created = Parkimage.objects.get_or_create(image=new_image_path, default=True) if created: park.images.add(p_img) # new_image = Parkimage.objects.get_or_create() print new_image_path + " added to " + park.name + "with id " + str(park.id)
def compress_static(): pgreen('*** Check compress static files...') # Add project dir to PYTHONPATH so "config.settings" object is found # TODO: Since this code is extracted from project it doesn't work anymore # trying to get project path relative to python package path. Refactor!! # sys.path.append(os.path.join( # os.path.dirname(os.path.realpath(__file__)), # os.pardir, # os.pardir)) current_path = '/home/vagrant/Development/projects/cirujanos' sys.path.append(current_path) django.settings_module('config.settings.%s' % env.environment) from django.conf import settings as django_settings # current_path = os.path.dirname(os.path.realpath(__file__)) pcyan('CURRENT PATH: %s\n' % current_path) pcyan('COMPRESS_ENABLED: %s\n' % django_settings.COMPRESS_ENABLED) pcyan('DEBUG: %s\n' % django_settings.DEBUG) if (django_settings.COMPRESS_ENABLED and not django_settings.DEBUG): pred('Compressing files..') command = './manage.py compress --settings="config.settings.%s"; ' command_params = (env.environment, ) sudo_command(command, command_params) else: pred('WARNING: Django settings NOT allow compression')
def django_syncdb(): """ Synchronize/create database on remote host """ _setup_env() sys.path.append(env.base_path) django.settings_module('%(project_name)s.settings_%(role)s' % env) settings = __import__('%(project_name)s.settings_%(role)s' % env) # Not sure what's going on here .. but it fixes a problem.. if getattr(settings, 'settings_%(role)s' % env, False): settings = getattr(settings, 'settings_%(role)s' % env, False) dbconf = settings.DATABASES['default'] if dbconf['ENGINE'].endswith('mysql'): _create_mysqldb(dbconf) # Give apache write permission to the project directory elif dbconf['ENGINE'].endswith('sqlite3') and env.role != 'dev': user = '******' in env.stage and env.stage['user'] or 'www-data' sudo('chown -R %s %s' % (user, os.path.dirname(dbconf['NAME']))) sudo('chmod 777 %s' % dbconf['NAME']) if env.role in ['prod', 'demo']: path = env.stage['path'] do = run else: path = env.base_path do = local if (yes_no_prompt("Do you want to syncdb ?")): with(cd(os.path.join(path, env.project_name))): do(env.venv_activate +' && %s manage.py syncdb --noinput --settings=settings_%s' % (env.venv_python, env.role))
def wrapper(*args, **kwargs): if env.host == "localhost": _env_set("localhost") for key, value in fabsettings.PROJECT_SITES.iteritems(): # key is the identifier, e.g. "dev", "prod", etc etc for k, v in value.iteritems(): if v == env.host: # iterate in the nested dictionary # if the nested dictionary's name is equivalent to env.host # we will set our global state env with _env_set(key) _env_set(key) with prefix(env.activate): from fabric.contrib import django if env.host == "localhost": _settings = '.'.join( [env.project_name, 'settings']) else: _settings = '.'.join( [env.project_name, 'settings', env.target]) django.settings_module(_settings) if not env.get('distro'): _env_set_distro() return f(*args, **kwargs)
def insertIntoGallery( gallery_title, gallery_slug, screenshot, title, slug, gallery_description="", gallery_tags="", caption="", tags="", fab_dir='%s/.fabric-bolt' % (os.path.expanduser('~/')) ): # Add custom fabric-bolt settings directory sys.path.insert(0, fab_dir) # Utilize django within fabfile # Load custom fabric-bolt settings file django.settings_module('settings') # Loads the django Models get_wsgi_application() # Once loaded we can reference them from photologue.models import Photo from photologue.models import Gallery file = open(screenshot, 'rb') data = file.read() # First Generate or Retrieve the Photo Model and save or update it try: photo = Photo.objects.get(slug=slug) photo.date_added = datetime.now() photo.date_taken = datetime.now() print("~~~ FOUND existing Screenshot ~~~") except Photo.DoesNotExist: photo = Photo(title=title, slug=slug, caption=caption, is_public=True, tags=tags,) print("~~~ CREATED new Screenshot ~~~") try: photo.image.save(os.path.basename(screenshot), ContentFile(data)) except FieldError: # For some reason a field, 'photo,' is being passed to model as a field. pass print("~~~ SAVED Screenshot ~~~") # Now Create or Retrieve the named Gallery and add the photo to it. gallery = None try: gallery = Gallery.objects.get(title=gallery_title) print("~~~ FOUND existing Screenshot Gallery ~~~") except Gallery.DoesNotExist: gallery = Gallery(title=gallery_title, slug=gallery_slug, description=gallery_description, is_public=True, tags=gallery_tags,) gallery.save() print("~~~ CREATED new Screenshot Gallery ~~~") if gallery: gallery.photos.add(photo) print("~~~ Added Screenshot to Gallery ~~~") print("<a target=\"_parent\" href=\"/photologue/gallery/%s\">View Screenshot Gallery %s</a>") % (gallery_title, gallery_title) # Reset the syspath sys.path.remove(fab_dir)
def import_remote_db(): print(green('IMPORTING REMOTE DB ...')) sys.path.append(os.path.dirname(__file__)) django.settings_module('{{ project_name }}.settings') from django.conf import settings engine = settings.DATABASES['default']['ENGINE'].split('.')[-1] if engine == 'mysql': with cd(env.tmp_dir): db_file = 'mysql-%s.gz' % env.db_name run('mysqldump -uadmin -p %s | gzip -9 > %s' % (env.db_name, db_file)) get(db_file, '../') run('rm %s' % db_file) local('gunzip ../%s' % db_file) local('mysql -uroot -p %s < %s' % (env.db_name, '../%s' % db_file.replace('.gz', ''))) elif engine == 'postgresql_psycopg2': with cd(env.tmp_dir): db_file = 'psql-%s.gz' % env.db_name run('pg_dump -O -F p -c -U postgres %s | gzip -9 > %s' % (env.db_name, db_file)) get(db_file, '../../') run('rm %s' % db_file) local('gunzip ../../%s' % db_file) local('psql -U postgres -d %s -f %s' % (env.db_name, '../../%s' % db_file.replace('.gz', '')))
def loc(): '''Set config to local (requires SSH server running on localhost) ''' env.dev_settings_file = "settings_local.py" django.settings_module(os.path.splitext(env.dev_settings_file)[0]) env.os = env.local_os
def local_clear_database(): """Recreate schema""" django.settings_module(env.django_project) from django.conf import settings env.role = settings.DATABASES['default']['USER'] env.db_password = settings.DATABASES['default']['PASSWORD'] local('echo "DROP SCHEMA public CASCADE;CREATE SCHEMA public AUTHORIZATION %(role)s;GRANT ALL ON SCHEMA public TO %(role)s;" | PGPASSWORD="******" bin/django dbshell' % env)
def load_base_env(): django.settings_module('bcpp.settings') CONFIG_FILENAME = 'bcpp.conf' DOWNLOADS_DIR = '~/Downloads' BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ETC_CONFIG_PATH = os.path.join(BASE_DIR, 'fabfile', 'etc') FABRIC_CONFIG_PATH = os.path.join(BASE_DIR, 'fabfile', 'conf', 'fabric.conf') timestamp = datetime.now().strftime('%Y%m%d%H%M%S') env.log_folder = os.path.expanduser('~/fabric/{}'.format(timestamp)) if not os.path.exists(env.log_folder): os.makedirs(env.log_folder) print('log_folder', env.log_folder) update_env_secrets(path=ETC_CONFIG_PATH) env.roledefs = roledefs env.hosts, env.passwords = get_hosts(path=ETC_CONFIG_PATH, gpg_filename='hosts.conf.gpg') env.hostname_pattern = hostname_pattern env.device_ids = get_device_ids() env.prompts = prompts env.prompts.update({'Enter password: '******'hosts.txt'), 'a') as f: f.write('{}\n'.format(',\n'.join([h for h in env.hosts])))
def _get_settings(conf=None): # do this here. django settings cannot be imported more than once...probably. # still dont really get the mess here. if not conf: conf = env.project_conf django.settings_module(conf) from django.conf import settings return settings
def installdev(): django.settings_module('django-heroku.settings.dev') local('pip install -r requirements.txt') local('python manage.py migrate') local('python manage.py installwatson') local('python manage.py buildwatson') # fixture_dev() local('bower install --config.interactive=false')
def batch(): django.settings_module('django-heroku.settings.dev') local('python manage.py batch batch_passage_en_presouscription') local('python manage.py batch batch_fin_presouscription') local('python manage.py batch batch_souscription') local('python manage.py batch batch_fin_campagne') local('python manage.py batch batch_task_mail_auteur_demandes_new_souscription') local('python manage.py batch batch_detecte_nouveaux_livres')
def _get_database_name(): """Gets database dictionary either from ENVS or form Django settings.py""" _require_environment() database = env.project.get('database', None) if not database: django.settings_module(_interpolate('%(project)s.%(settings)s')) database = django_settings.DATABASES['default'] return database
def recreate_initial_data(): django.settings_module('settings_debug') if os.path.isfile('inhouse/fixtures/_initial_data.json'): local('mv inhouse/fixtures/_initial_data.json inhouse/fixtures/_initial_data.json.old') local('rm -f dev.db') local('python manage.py syncdb --noinput') local('python manage.py loaddata %s' % ' '.join(INITIAL_FIXTURES)) local('python manage.py dumpdata -a -e inhouse.AuthUserGroup -e contenttypes.ContentType --indent=2> inhouse/fixtures/_initial_data.json')
def import_sightings(json_file='ias_ess_dump.json', skip_images=False, pks__gte=0, pks__lte=None): """Put sightings from a fixture file into db. :param settings: change to change settings file inside ashtag.settings... :param json_file: change to use locally dumped json :param skip_images: set True to ignore images :param pks__gte: set > 0 to limit import (or do an update, etc.) :param pks__lte: set not None to limit import (or do an update, etc.) """ django.settings_module(os.environ.get('DJANGO_SETTINGS_MODULE')) from ashtag.apps.core.models import Sighting, Tree dumped_json = "" with open(json_file, 'r') as fp: dumped_json = fp.read() sightings = json.loads(dumped_json) ash_sightings = filter(lambda x: x['fields']['taxon'] == 100004, sightings) not_rejects = filter(lambda x: not x['fields']['rejected'], ash_sightings) have_emails = filter(lambda x: x['fields']['email'], not_rejects) only_pks = sorted(filter(lambda x: x['pk'] >= int(pks__gte), have_emails), key=lambda x: x['pk']) if pks__lte is not None: only_pks = filter(lambda x: x['pk'] <= int(pks__lte), only_pks) print red("%s sightings don't have emails" % (len(not_rejects) - len(have_emails))) unknown = Sighting.DISEASE_STATE.unknown diseased = Sighting.DISEASE_STATE.diseased print green("Processing %s sightings with %s <= PK <= %s" % (len(only_pks), pks__gte, pks__lte)) for sighting in only_pks: fields = sighting['fields'] tree = Tree.objects.create(location=fields['location'], creator_email=fields['email']) s = Sighting.objects.create( tree=tree, location=fields['location'], notes="Imported from IAS-ESS: http://ias-ess.org/ias/sighting/%s" % sighting['pk'], disease_state=diseased if fields['verified'] else unknown, created=fields['datetime'], creator_email=fields['email']) if not skip_images: image = urllib.urlretrieve("http://ias-ess.org/static/media/%s" % fields['photo']) s.image.save(os.path.basename(fields['photo']), File(open(image[0]))) s.save() if len(only_pks): print green("Made %s Trees and Sightings, latest PK = %s" % (len(only_pks), only_pks[-1]['pk'])) else: print red("Nothing made!")
def erase_all_local(): """Nuke the Tree/Sighting objects.""" django.settings_module(os.environ.get('DJANGO_SETTINGS_MODULE')) from ashtag.apps.core.models import Sighting, Tree if confirm(red("Really delete all local Trees and Sightings?"), default=False): Tree.objects.all().delete() Sighting.objects.all().delete() else: abort("Not continuing with anything.")
def erase_all_local(settings=SETTINGS): """Nuke the Tree/Sighting objects.""" django.settings_module('ashtag.settings.%s' % settings) from ashtag.apps.core.models import Sighting, Tree if confirm(red("Really delete all local Trees and Sightings?"), default=False): Tree.objects.all().delete() Sighting.objects.all().delete() else: abort("Not continuing with anything.")
def test(module='tests'): """ Runs all unit tests for OpenSlides using coverage. The settings file in the tests directory is used, therefor the environment variable DJANGO_SETTINGS_MODULE is set to 'tests.settings'. """ django.settings_module('tests.settings') local('coverage run ./manage.py django test %s' % module)
def env_setup(config=env.CONFIG,settings_module='codalab.settings'): if config is None: sys.path.append('.') os.environ['DJANGO_CONFIGURATION'] = config os.environ["DJANGO_SETTINGS_MODULE"] = settings_module from fabric.contrib import django from configurations import importer importer.install() django.settings_module(settings_module) from django.conf import settings as django_settings env.roledefs = django_settings.DEPLOY_ROLES env.django_settings = django_settings @task def local(**kwargs): env.run = lrun env_setup(**kwargs) @task def remote(**kwargs): env.run = run env_setup(**kwargs) @task def clone_repo(repo_url='https://github.com/codalab/codalab.git',path=env.conf.DEPLOY_PATH): env.run('git clone %s %s' % (repo_url, path)) @task def provision(config='Dev'): clone_repo() with cd(env.conf.DEPLOY_PATH): sudo('/bin/bash codalab/scripts/provision %s' % env.conf.DEPLOY_USER) sudo('python manage.py config_gen --configuration=%s' % config, user=env.conf.DEPLOY_USER) @task def bootstrap(): make_virtualenv(path='venv', system_site_packages=False) with virtualenv('venv'): run('pip install --upgrade pip') run('pip install --upgrade distribute') run('rm -rf codalab_src') run("git clone %s codalab_src" % env.django_settings.SOURCE_GIT_URL) put(os.path.join(THIS_SETTINGS_DIR,'deploy.py'), 'codalab_src/codalab/codalab/settings') with virtualenv('venv'): run('cd codalab_src && bash ./requirements') @task def install(): env.run('deploymentcmd')
def download_media(dirname): remote_media_path = os.path.join(os.path.dirname(env.code_dir), 'media') remote_dir = os.path.join(remote_media_path, dirname) sys.path.append(os.path.dirname(__file__)) django.settings_module('bridges.settings') from django.conf import settings get(remote_dir, settings.MEDIA_ROOT)
def import_sightings(settings=SETTINGS, json_file='ias_ess_dump.json', skip_images=False, pks__gte=0): """Put sightings from a fixture file into db. :param settings: change to change settings file inside ashtag.settings... :param json_file: change to use locally dumped json :param skip_images: set True to ignore images :param pks__gte: set > 0 to limit import (or do an update, etc.) """ django.settings_module('ashtag.settings.%s' % settings) from ashtag.apps.core.models import Sighting, Tree dumped_json = "" with open(json_file, 'r') as fp: dumped_json = fp.read() sightings = json.loads(dumped_json) ash_sightings = filter(lambda x: x['fields']['taxon'] == 100004, sightings) not_rejects = filter(lambda x: not x['fields']['rejected'], ash_sightings) have_emails = filter(lambda x: x['fields']['email'], not_rejects) only_pks = sorted( filter(lambda x: x['pk'] >= int(pks__gte), have_emails), key=lambda x: x['pk']) print red("%s sightings don't have emails" % ( len(not_rejects) - len(have_emails) )) unknown = Sighting.DISEASE_STATE.unknown diseased = Sighting.DISEASE_STATE.diseased print green("Processing %s sightings with PK >= %s" % ( len(only_pks), pks__gte)) for sighting in only_pks: fields = sighting['fields'] tree = Tree.objects.create( location=fields['location'], creator_email=fields['email'] ) s = Sighting.objects.create( tree=tree, location=fields['location'], notes="Imported from IAS-ESS: http://ias-ess.org/ias/sighting/%s" % sighting['pk'], disease_state=diseased if fields['verified'] else unknown, created=fields['datetime'], creator_email=fields['email'] ) if not skip_images: image = urllib.urlretrieve( "http://ias-ess.org/static/media/%s" % fields['photo']) s.image.save( os.path.basename(fields['photo']), File(open(image[0])) ) s.save() print green("Made %s Trees and Sightings, latest PK = %s" % ( len(pks__gte), only_pks[-1]['pk']))
def run_nose(): # use test settings, not actual settings django.settings_module('pysis.settings.test_settings') local('nosetests -v -d -x ' + '--with-doctest ' + '--with-django ' + '--with-djangoliveserver ' + '--with-selenium ' + 'apps tests')
def test(module='tests'): """ Runs the unit tests. """ sys.path.insert(0, '') django.settings_module('tests.settings') sys.argv.pop() sys.argv.extend(['test', module]) from django.core import management management.execute_from_command_line()
def build_prod(): """ Build the application for the production environment. """ require_secrets() django.settings_module('texas_choropleth.settings.production') build() print(blue("\n [ Collecting Staticfiles ]")) local('./src/manage.py collectstatic --noinput') done()
def get_settings(): from fabric.contrib.django import settings_module settings_module(env.django_settings) # mimic Django's dual path :/ if env.project_root not in sys.path: sys.path.append(os.path.join(env.project_root)) if env.project_django_root not in sys.path: sys.path.append(os.path.join(env.project_django_root)) from django.conf import settings return settings
def deploy(opt=False): server_dict = { 'action': '', } env.user = '******' if not opt: env.hosts = server_dict.values() else: env.hosts = [server_dict[str(opt)]] env.activate = 'source ~/readbox_env/bin/activate' env.key_filename = '~/.ssh/server_rsa' django.settings_module('manhattan.settings')
def _drop_database_mysql(): """Destroy (DROP) MySQL database according to env's settings.py""" # Unless explicitly provided, uses local Django settings to # extract username/password to access remote database database = env.project.get('database', None) if not database: django.settings_module(env.project['settings']) database = django_settings.DATABASES['default'] # Drops database with settings(hide('warnings'), warn_only=True): result = run(MYSQL_PREFIX % "\"DROP DATABASE %(NAME)s;\"" % database)
def restore_project(filename): """ Restore server's database with .sql file contained in filename """ _require_environment() # Confirms action if not console.confirm( 'ATTENTION! This will destroy current database! Confirm?', default=False): return # Unless explicitly provided, uses local Django settings to # extract username/password to access remote database database = env.project.get('database', None) if not database: django.settings_module(env.project['settings']) database = django_settings.DATABASES['default'] # Remote side with prefix(_django_prefix()): with cd(_django_project_dir()): # Uploads tar file tarfile = os.path.basename(filename) basename = tarfile[:tarfile.index('.tar.gz')] if console.confirm('Upload backup?'): put(filename, '../backup/%s' % tarfile) # Drop and recreate current database _drop_database_mysql() _setup_project_mysql() # Restore MySQL # To avoid silly mistakes, instead of using project's user & password, uses root's with cd('../'): run('tar -xzvf backup/%s' % tarfile) run('mysql -u root -p %s < backup/%s/%s.sql' % ( #database['USER'], #database['PASSWORD'], database['NAME'], basename, env.project['project'], )) # Restore extra files extra_backup_files = env.project.get('extra_backup_files', []) for file in extra_backup_files: run('cp -R ../backup/%s/%s ./%s' % (basename, os.path.basename(file), os.path.dirname(file))) # Removes uncompressed files, but leaves .tar.gz run('rm -rf ../backup/%s' % basename)
def fabenv(environ, path='environ.json'): with open(path) as configfile: conf = json.load(configfile)[environ] defaults = DEFAULTS.copy() django.settings_module(conf['django_settings']) # for now, also use env to store other global state env.update(defaults) env.update(conf) # and finally update with the env dict env.update(conf.pop('env', {}))
def backup_project(): """ Backup server's database and copy tar.gz to local ../backup dir """ _require_environment() # Unless explicitly provided, uses local Django settings to # extract username/password to access remote database database = env.project.get('database', None) if not database: django.settings_module(env.project['settings']) database = django_settings.DATABASES['default'] # Remote side with prefix(_django_prefix()): with cd(_django_project_dir()): # Creates dir to store backup, avoiding existing similar names dirname = '../backup/%s_%s' % ( datetime.date.today().strftime('%Y%m%d'), env.environment) path = dirname index = 0 while files.exists(path) or files.exists('%s.tar.gz' % path): index += 1 path = '%s.%s' % (dirname, index) run('mkdir -p %s' % path) # Backup MySQL run('mysqldump %s -u %s -p%s %s > %s/%s.sql' % ( '-h %s' % database['HOST'] if database.get('HOST', None) else '', database['USER'], database['PASSWORD'], database['NAME'], path, env.project['project'], )) # Backup extra files extra_backup_files = env.project.get('extra_backup_files', []) for file in extra_backup_files: run('cp -R %s %s/' % (file, path)) # Create .tar.gz and removes uncompressed files with hide('stdout'): run('tar -czvf %s.tar.gz %s/' % (path, path)) run('rm -rf %s/' % path) # Download backup? if console.confirm('Download backup?'): return get('%s.tar.gz' % path, '../backup')
def mysql_db(local_path='/backups/db/'): """ usage: `fab -r <server_name> backups.mysql_db /path/to/backup/folder/` backups the target's database to local destination. backup path defaults to /backups/db/[target]/. 1. creates the target folder if doesnt exists 2. by default keeps 30 days of backups """ with prefix(env.activate): if env.host == "localhost": _settings = '.'.join([env.project_name, 'settings']) else: _settings = '.'.join([env.project_name, 'settings', env.target]) print 1231231, _settings django.settings_module(_settings) from django.conf import settings print 123123123 print settings print settings timestamp = datetime.datetime.now().strftime("%y-%m-%d_%h-%m-%s") fname = '%(dump_path)s%(database)s-backup-%(date)s.xml.gz' % { 'dump_path': local_path, 'database': settings.databases['default']['name'], 'date': timestamp, } if not os.path.exists(local_path): os.makedirs(local_path) if exists(fname): run('rm "%s"' % fname) print("dumping %s to %s" % (settings.databases['default']['name'], fname)) run('mysqldump -u %(username)s -p%(password)s %(database)s --xml | ' 'gzip > %(fname)s' % {'username': settings.databases['default']['user'], 'password': settings.databases['default']['password'], 'database': settings.databases['defaukt']['name'], 'fname': fname}) print("transferring...") get(fname, local_path) print("removing remote dump file...") run("rm %(dump_file)s" % {'dump_file': fname}) print("finish")
def _setup_project_mysql(): """Creates MySQL database according to env's settings.py""" # Unless explicitly provided, uses local Django settings to # extract username/password to access remote database database = env.project.get('database', None) if not database: django.settings_module(env.project['settings']) database = django_settings.DATABASES['default'] # Create database & user, if not already there with settings(hide('warnings'), warn_only=True): result = run(MYSQL_PREFIX % "\"CREATE DATABASE %(NAME)s DEFAULT CHARACTER SET utf8;\"" % database) if result.succeeded: run(MYSQL_PREFIX % "\"GRANT ALL ON %(NAME)s.* TO '%(USER)s'@'localhost' IDENTIFIED BY '%(PASSWORD)s';\"" % database)
def restore_local_from_file(path="db.dump", data_only=True): """ Drops local database, and restores data from file """ data_only = prep_bool_arg(data_only) # make it look like a local only command env.hosts = [] django.settings_module('secondfunnel.settings.dev') from django.conf import settings # Dump our local database to backup, then flush it out now = datetime.now() local('fab database.dump:path=db_backup-%s' % (now.strftime('%Y-%m-%dT%H:%M.dump'))) # clear all existing data from database flush_local() # load downloaded database local_restore(path=path, data_only=data_only)
def gravatars(): from fabric.contrib import django django.settings_module('settings') from django.contrib.auth.models import User import requests _display_message('Updating all users\' Gravatar flags') ################ for user in User.objects.all().order_by('id'): _display_message('{}. {}'.format( user.pk, user.username.encode('ascii', 'ignore'))) ################ email_md5 = hashlib.md5(user.email.lower()).hexdigest() print 'Email MD5: {}'.format(email_md5) greq = requests.get( 'https://secure.gravatar.com/avatar/{}?s=50&d=404'.format( email_md5)) if greq.status_code == 404: has_gravatar = False else: has_gravatar = True profile = user.profile profile.has_gravatar = has_gravatar profile.save() try: from fabric.colors import green, red if has_gravatar: print 'Has Gravatar: {}'.format(green(has_gravatar)) else: print 'Has Gravatar: {}'.format(red(has_gravatar)) except ImportError: print 'Has Gravatar: {}'.format(has_gravatar)
def dev_db(assume_yes=False): """Recreate development database.""" django.settings_module('settings_debug') conf = dj_settings.DATABASES.get('default') if not conf: abort(colors.red('No default database configured.')) if conf.get('ENGINE', '').endswith('sqlite3'): path = conf.get('NAME') if os.path.isfile(path): if not assume_yes and not confirm('Delete existing database?'): abort('Aborting at user request.') os.remove(path) elif conf.get('ENGINE', '').endswith('psycopg2'): with settings(warn_only=True): local('dropdb -U %s %s' % (conf.get('USER'), conf.get('NAME'))) local('createdb -U %s %s' % (conf.get('USER'), conf.get('NAME'))) load_initial_data() load_dev_data()
def _setup_project_mysql(): """Creates MySQL database according to env's settings.py""" # Unless explicitly provided, uses local Django settings to # extract username/password to access remote database database = env.project.get('database', None) if not database: django.settings_module(env.project['settings']) database = django_settings.DATABASES['default'] # Create database & user, if not already there with settings(hide('warnings'), warn_only=True): result = run( MYSQL_PREFIX % "\"CREATE DATABASE %(NAME)s DEFAULT CHARACTER SET utf8;\"" % database) if result.succeeded: run(MYSQL_PREFIX % "\"GRANT ALL ON %(NAME)s.* TO '%(USER)s'@'localhost' IDENTIFIED BY '%(PASSWORD)s';\"" % database)
def load_base_env(): django.settings_module('bcpp_clinic.settings') CONFIG_FILENAME = 'bcpp-clinic.conf' BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ETC_CONFIG_PATH = os.path.join(BASE_DIR, 'fabfile', 'etc') FABRIC_CONFIG_PATH = os.path.join( BASE_DIR, 'fabfile', 'conf', 'fabric.conf') timestamp = datetime.now().strftime('%Y%m%d%H%M%S') env.log_folder = os.path.expanduser('~/fabric/{}'.format(timestamp)) if not os.path.exists(env.log_folder): os.makedirs(env.log_folder) print('log_folder', env.log_folder) update_env_secrets(path=ETC_CONFIG_PATH) env.hosts, env.passwords = get_hosts( path=ETC_CONFIG_PATH, gpg_filename='hosts.conf.gpg') env.prompts = prompts env.prompts.update({'Enter password: ': env.dbpasswd})
def get_arguments(): """ parse arguments passed to postgres command """ django.settings_module('secondfunnel.settings.{0}'.format( env.environment.lower())) from django.conf import settings # workaround for running all from locally, to remote machines # but still reading the os environment variables # if there is hosts of course env.remote_environ = read_remote_env() if env.hosts and 'RDS_DB_NAME' in env.remote_environ: DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': env.remote_environ['RDS_DB_NAME'], 'USER': env.remote_environ['RDS_USERNAME'], 'PASSWORD': env.remote_environ['RDS_PASSWORD'], 'HOST': env.remote_environ['RDS_HOSTNAME'], 'PORT': env.remote_environ['RDS_PORT'], } } else: DATABASES = settings.DATABASES password = '******'.format( DATABASES['default']['PASSWORD']) arguments = '--host=%s --port=%s --username=%s --dbname=%s' % ( DATABASES['default']['HOST'], DATABASES['default']['PORT'], DATABASES['default']['USER'], DATABASES['default']['NAME']) if is_windows(): password = '' arguments = '-W ' + arguments return { 'password': password, 'arguments': arguments, }
def wrap_function(*args, **kwargs): if len(args) > 0 and 'env' in kwargs: raise TypeError( '%s() got multiple values for keyword argument "env"' % function.__name__) environ = DEFAULT_ENV if len(args) > 0: environ = args[0] else: if 'env' in kwargs: environ = kwargs['env'] django.settings_module('{}.config.settings.{}'.format( DJANGO_PROJECT, environ)) try: # logger.debug('args = %s' % str(args)) # logger.debug('kwargs = %s' % str(kwargs)) return function(*args, **kwargs) except Exception as e: raise e
from __future__ import with_statement from fabric.api import * from fabric.contrib import django import os SETTINGS_FILE = "app.settings_my" django.settings_module(SETTINGS_FILE) PORT = os.environ.get('PORT', 9000) # run server locally def start(): local("python manage.py runserver 127.0.0.1:%s --traceback --settings=%s" % (PORT, SETTINGS_FILE))
import os import sys from lib.utils import log from .other import log_success from fabric.api import * from fabric.contrib import django django.settings_module("{{ project_name }}.settings") from django.conf import settings from {{ project_name }}.settings.production import ( AWS_BUCKET_NAME, AWS_MEDIA_BUCKET_NAME, AWS_STAGING_BUCKET_NAME, VERBOSE_APP_NAME, BUILD_DIR ) """ Deployment Tasks ================ """ project_name = "{{ project_name }}" pwd = os.path.dirname(__file__) gzip_path = '{0}/{1}/gzip/static/'.format(pwd, project_name) static_path = '{0}/{1}/static/'.format(pwd, project_name) @task
from fabric.context_managers import cd, prefix from fabric.contrib.files import exists from fabric.api import run, env from fabric.decorators import task, runs_once, parallel from fabric.contrib import django from fabric.operations import local from fabric.tasks import execute from django.conf import settings from slackclient import SlackClient # Load the django settings. This needs to read the env-variables and setup # django-configurations, before the settings_module can be accessed. envdir.read(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'envs')) configurations.setup() django.settings_module('qcat.settings') ENVIRONMENTS = { 'develop': { 'branch': 'develop', 'label': 'dev', 'host_string': settings.HOST_STRING_DEV, 'touch_file': settings.TOUCH_FILE_DEV, 'use_deploy_announcement': False, }, 'demo': { 'branch': 'master', 'label': 'live', 'host_string': settings.HOST_STRING_DEMO, 'touch_file': settings.TOUCH_FILE_DEMO,
from boto.s3.key import Key from boto.ec2.connection import EC2Connection from fabric.contrib import django from fabric.state import connections from vendor import yaml from pprint import pprint import os import time import sys import re try: import dop.client except ImportError: print "Digital Ocean's API not loaded" django.settings_module('settings') try: from django.conf import settings as django_settings except ImportError: print " ---> Django not installed yet." django_settings = None # ============ # = DEFAULTS = # ============ env.NEWSBLUR_PATH = "~/projects/newsblur" env.SECRETS_PATH = "~/projects/secrets-newsblur" env.VENDOR_PATH = "~/projects/code" env.user = '******'