def load_config_files(conf_file,
default_conf=DEFAULT_CONF_FILE,
use_default=True):
"""Populates env['ctx'] with settings from the provided and default files.
Keyword arguments:
conf_file -- configuration file to use
default_conf -- default configuration file used as a base, global default
is 'target_defs/defaults.json'
use_defaults -- allows to avoid using the defaults file.
"""
# Try loading configuration from JSON file.
if conf_file:
with open(conf_file) as f:
fctx = json.load(f)
show(yellow("Using configuration from: %s"), conf_file)
else:
show(yellow("Using sane defaults"))
fctx = {}
# Load the default configuration if exists
if use_default:
try:
with open(default_conf) as f:
dctx = json.load(f)
msg = "Using default configuration from: %s"
show(yellow(msg), default_conf)
except Exception as e:
show(yellow("Default conf file error! %s"), e)
confirm_or_abort(red("\nDo you want to continue?"))
dctx = {}
dctx.update(fctx)
env["ctx"] = dctx
return dctx
def load_config_files(conf_file, default_conf=DEFAULT_CONF_FILE,
use_default=True):
"""Populates env['ctx'] with settings from the provided and default files.
Keyword arguments:
conf_file -- configuration file to use
default_conf -- default configuration file used as a base, global default
is 'target_defs/defaults.json'
use_defaults -- allows to avoid using the defaults file.
"""
# Try loading configuration from JSON file.
if conf_file:
with open(conf_file) as f:
fctx = json.load(f)
show(yellow("Using configuration from: %s"), conf_file)
else:
show(yellow("Using sane defaults"))
fctx = {}
# Load the default configuration if exists
if use_default:
try:
with open(default_conf) as f:
dctx = json.load(f)
msg = "Using default configuration from: %s"
show(yellow(msg), default_conf)
except Exception as e:
show(yellow("Default conf file error! %s"), e)
confirm_or_abort(red("\nDo you want to continue?"))
dctx = {}
dctx.update(fctx)
env["ctx"] = dctx
return dctx
def upload_settings_files():
"""Uploads target specific (templated) settings files.
If specified also uploads user supplied locals.py.
*Warning*: Settings are uploaded from your local template file.
Make sure to have proper branch/revision checked out.
"""
base_dir = cget("base_dir")
user = cget("user")
locals_path = cget("locals_path")
show(yellow("Uploading Django settings files."))
# This is Template context not the deployment context.
# A split should happen some time.
context = dict(env["ctx"])
context
# Upload main settings and ensure permissions.
source = pjoin(local_files_dir("django"), "settings_template.py")
destination = pjoin(base_dir, "settings", "%s.py" % cget("settings_name"))
upload_template_with_perms(source, destination, context, mode="644",
user=user, group=user)
# We could be deploying from different directory.
# Try our best to find correct path.
# First - direct, absolute match.
if not os.path.isfile(locals_path):
# Try relative to deployment directory.
this_dir = os.path.dirname(os.path.abspath(__file__))
locals_path = pjoin(this_dir, locals_path)
if not os.path.isfile(locals_path): # :((
msg = u"Warning: Specified local settings path is incorrect: {0}."
show(red(msg.format(locals_path)))
confirm_or_abort(red("\nDo you want to continue?"))
locals_path = None
# Upload user supplied locals if present.
if locals_path:
show(yellow("Uploading your custom local settings files."))
destination = pjoin(base_dir, "settings", "local.py")
put_file_with_perms(locals_path, destination, mode="644", user=user,
group=user)
def deploy(conf_file=None,
instance=None,
branch=None,
commit=None,
locals_path=None,
setup_environment=False,
prompt=True,
requirements=True):
u"""Does a full deployment of the project code.
You have to supply an ``instance`` name (the name of deployment
target in colocation environment).
``conf_file`` should be a path to a properly formatted JSON
configuration file that will override default values.
If ``locals_path`` is specified this file is used as
a local settings file.
Arguments ``commit`` and ``branch`` can be used to deploy
some specific state of the codebase.
"""
# Get file configuration and update with args
env['ctx'] = {}
ctx = load_config_files(conf_file)
ctx = update_args(ctx, instance, branch, commit, locals_path, requirements,
setup_environment)
# Fill instance context.
set_instance_conf()
print_context()
# Give user a chance to abort deployment.
if prompt:
confirm_or_abort(red("\nDo you want to continue?"))
# Prepare server environment for deployment.
show(yellow("Preparing project environment"))
if get_boolean(setup_environment):
prepare_global_env()
# create folders
prepare_target_env()
# Fetch source code.
fetch_project_code()
# Upload target specific Django settings.
upload_settings_files()
if get_boolean(requirements):
# Update Virtualenv packages.
update_virtualenv()
# Collect static files.
collect_staticfiles()
# Compile translation messages.
# Make sure this folder exists before using compile_messages
# compile_messages()
# Update database schema.
sync_db()
# Uploads settings and scripts for services.
configure_services()
# Reload services to load new config.
__reload_services(setup=setup_environment)
def upload_settings_files():
"""Uploads target specific (templated) settings files.
If specified also uploads user supplied locals.py.
*Warning*: Settings are uploaded from your local template file.
Make sure to have proper branch/revision checked out.
"""
base_dir = cget("base_dir")
user = cget("user")
locals_path = cget("locals_path")
show(yellow("Uploading Django settings files."))
# This is Template context not the deployment context.
# A split should happen some time.
context = dict(env["ctx"])
context
# Upload main settings and ensure permissions.
# source = pjoin(local_files_dir("django"), "settings_template.py")
# destination = pjoin(base_dir, "settings", "%s.py" % cget("settings_name"))
# upload_template_with_perms(source, destination, context, mode="644",
# user=user, group=user)
# We could be deploying from different directory.
# Try our best to find correct path.
# First - direct, absolute match.
if not locals_path:
return
if not os.path.isfile(locals_path):
# Try relative to deployment directory.
this_dir = os.path.dirname(os.path.abspath(__file__))
locals_path = pjoin(this_dir, locals_path)
if not os.path.isfile(locals_path): # :((
msg = u"Warning: Specified local settings path is incorrect: {0}."
show(red(msg.format(locals_path)))
confirm_or_abort(red("\nDo you want to continue?"))
locals_path = None
# Upload user supplied locals if present.
if locals_path:
show(yellow("Uploading your custom local settings files."))
destination = pjoin(base_dir, "settings", "local.py")
put_file_with_perms(locals_path,
destination,
mode="644",
user=user,
group=user)
# Upload Google Prediction credentials
this_dir = os.path.dirname(os.path.abspath(__file__))
cred_file = pjoin(this_dir, '..', 'prediction.dat')
base_dir = cget('project_dir')
if os.path.isfile(cred_file):
show(yellow("Uploading Google Prediction credentials storage."))
destination = pjoin(base_dir, 'code', 'prediction.dat')
# Main project directory
show(yellow("Uploading to %s." % destination))
put_file_with_perms(cred_file,
destination,
mode="644",
user=user,
group=user)
# Classification module
destination = pjoin(base_dir, 'code', 'urlannotator', 'classification',
'prediction.dat')
show(yellow("Uploading to %s." % destination))
put_file_with_perms(cred_file,
destination,
mode="644",
user=user,
group=user)
def deploy(conf_file=None, instance=None, branch=None, commit=None,
locals_path=None, setup_environment=False, requirements=True):
u"""Does a full deployment of the project code.
You have to supply an ``instance`` name (the name of deployment
target in colocation environment).
``conf_file`` should be a path to a properly formatted JSON
configuration file that will override default values.
If ``locals_path`` is specified this file is used as
a local settings file.
Arguments ``commit`` and ``branch`` can be used to deploy
some specific state of the codebase.
"""
# Get file configuration and update with args
env['ctx'] = {}
ctx = load_config_files(conf_file)
ctx = update_args(ctx, instance, branch, commit, locals_path,
requirements, setup_environment)
# Fill instance context.
set_instance_conf()
print_context()
# Give user a chance to abort deployment.
confirm_or_abort(red("\nDo you want to continue?"))
# Prepare server environment for deployment.
show(yellow("Preparing project environment"))
if get_boolean(setup_environment):
prepare_global_env()
# create folders
prepare_target_env()
# Fetch source code.
fetch_project_code()
# Upload target specific Django settings.
upload_settings_files()
# Setup database (this relies on settings)
setup_database()
if get_boolean(requirements):
# Update Virtualenv packages.
update_virtualenv()
# Collect static files.
collect_staticfiles()
# Compile translation messages.
# Make sure this folder exists before using compile_messages
# compile_messages()
# Update database schema.
sync_db()
# Uploads settings and scripts for services.
configure_services()
# Reload services to load new config.
__reload_services()
# Configure and build documentation
doc.configure()
doc.build()
