def _write_deployment_config(outfilename, bundle_name, dbinfo, num_workers=1):
utils.render_tpl_to_file(
'deploy/thisbundle.py.tmpl',
outfilename,
bundle_name=bundle_name,
dbinfo=dbinfo,
num_workers=num_workers)
os.chmod(outfilename, 0700)
def start_serving_bundle(app_id, bundle_name):
"""
Serve the given bundle under supervisor, and return the appserver info
for where the service is running.
If you are running locally as dev, you need to make sure the user
running Celery has permissions to write to the /etc/supervisor/conf.d dir.
$ sudo chgrp nateaune /etc/supervisor/conf.d/
$ sudo chmod g+w /etc/supervisor/conf.d/
:returns: (instance_id, node_name, host_ip, host_port)
"""
# check that this bundle isn't already being served here - otherwise
# supervisor will silently ignore the redundant config files!
for bun in get_active_bundles():
if bun["app_id"] == app_id and bun["bundle_name"] == bundle_name:
raise utils.InfrastructureException((
"Redundant bundle service request: server %s (hostname=%s)"
" is already serving app_id %s, bundle_name %s.") % (
utils.node_meta("name"),
socket.gethostname(),
app_id,
bundle_name))
port_to_use = _get_a_free_port()
config_filename = os.path.join(taskconfig.SUPERVISOR_APP_CONF_DIR,
"%s.%s.port.%d.conf" % (app_id,
bundle_name,
port_to_use))
app_dir, bundle_dir = utils.app_and_bundle_dirs(app_id, bundle_name)
utils.render_tpl_to_file(
'deploy/supervisor_entry.conf',
config_filename,
run_in_userenv=os.path.join(taskconfig.PRIVILEGED_PROGRAMS_PATH,
"run_in_userenv"),
custdir=taskconfig.NR_CUSTOMER_DIR,
bundle_name=bundle_name,
bundle_runner=os.path.join(bundle_dir, "thisbundle.py"),
bundle_dir=bundle_dir,
app_user=app_id,
port=port_to_use)
_kick_supervisor()
instance_id = utils.node_meta("instance_id")
node_name = utils.node_meta("name")
host_ip = utils.get_internal_ip()
return (instance_id, node_name, host_ip, port_to_use)
def test_render_tpl_to_file(self):
"""
Render a template to a file.
"""
tpl_path = path.join(self.dir, 'tpl')
content = utils.render_tpl_to_file('bundle/settings.py.tmpl',
tpl_path,
dz_settings='foo')
read = open(tpl_path, 'r').read()
self.assertEqual(content, read)
self.assertTrue('foo' in read)
def update_local_proxy_config(app_id, bundle_name,
appservers, virtual_hostnames, site_media_map,
bundle_storage_engine=None,
remove_other_bundles=True):
bundle_storage_engine = bundle.get_bundle_storage_engine(
bundle_storage_engine)
site_conf_filename = _get_nginx_conffile(app_id)
if len(appservers) == 0:
raise utils.InfrastructureException((
"No appserver URLs provided for nginx config update to %s. "
"At least one upstream is required.") % app_id)
app_dir, bundle_dir = utils.app_and_bundle_dirs(app_id, bundle_name)
# We need to make sure this bundle's static is installed locally, so we
# can serve its static media.
if bundle_storage_engine is not SKIP_BUNDLE_INSTALL:
deploy.install_app_bundle_static(
app_id, bundle_name,
bundle_storage_engine,
remove_other_bundles=remove_other_bundles)
file_path_vars = {
"{SITE_PACKAGES}": os.path.join(bundle_dir,
"lib/python2.6/site-packages"),
"{SRC_PACKAGES}": os.path.join(bundle_dir, "src"),
}
default_file_path_base = os.path.join(bundle_dir, "user-repo")
def _make_full_path(original_file_path):
for varname, pathbase in file_path_vars.items():
if original_file_path.startswith(varname):
rest = original_file_path[len(varname):]
rest = rest.lstrip("/")
return os.path.join(pathbase, rest)
return os.path.join(default_file_path_base,
original_file_path)
sme = [dict(url_path=url_path,
alias_dest=_make_full_path(file_path.strip('/')),
) for url_path, file_path in site_media_map.items()]
sme.append(dict(url_path=taskconfig.DZ_ADMIN_MEDIA["url_path"],
alias_dest=os.path.join(
bundle_dir,
taskconfig.DZ_ADMIN_MEDIA["bundle_file_path"])))
# NOTE THE SLASHES::::
# location /static/ {
# alias /tmp/tmpR_1dI5/test001/bundle_test001_2011-03-09-03.52.55/user-src/static/;
# }
utils.render_tpl_to_file("nginx/site.conf",
site_conf_filename,
app_id=app_id,
appservers=[dict(
# (instance_id, node_name, host_ip, host_port)
instance_id=a[0],
node_name=a[1],
host_ip=a[2],
host_port=a[3],
)
for a in appservers],
virtual_hostnames=virtual_hostnames,
site_media_entries=sme)
utils.local_privileged(["kick_nginx"])
buildconfig_info["extra_requirements"].splitlines()],
basedir=repo_link,
ignore_keys="django",
env=ue)
utils.install_requirements(reqs, bundle_dir, env=ue)
# Remove the python executable, we don't use it
ue.remove(os.path.join(bundle_dir, "bin", "python"))
#os.remove(os.path.join(bundle_dir, "bin", "python"))
# Add settings file
utils.render_tpl_to_file(
'bundle/settings.py.tmpl',
os.path.join(bundle_dir, 'dz_settings.py'),
env=ue,
dz_settings=buildconfig_info["django_settings_module"],
admin_media_prefix=taskconfig.DZ_ADMIN_MEDIA["url_path"],
database_type=buildconfig_info["database_type"])
if return_ue:
return bundle_name, code_revision, ue
else:
return bundle_name, code_revision
def get_bundle_storage_engine(default=None):
"""
Get the appropriate bundle storage engine. If ``default`` is set,
returns that; if it is None, get the engine set as the global default.
"""
def build_project_bundle(zoomdb, opts):
zoomdb.log("We're getting your project's dependencies and packaging "
"everything up. This might take a couple of minutes.")
bundle_name, code_revision, ue = bundle.bundle_app(
opts["APP_ID"],
src_repo_type=opts["SRC_REPO_TYPE"],
return_ue=True)
zoomdb.log("Built project into bundle: %s" % bundle_name)
opts["BUNDLE_NAME"] = bundle_name
# and log this bundle into zoomdb
opts["BUNDLE_INFO"] = zoomdb.add_bundle(bundle_name, code_revision)
post_build_hooks = opts["POST_BUILD_HOOKS"]
bundle_runner = os.path.join(opts["APP_DIR"],
bundle_name,
"thisbundle_build.py")
# write out a temporary build-time bundle runner
utils.render_tpl_to_file('deploy/thisbundle.py.tmpl',
bundle_runner,
dbinfo=None, # no database access yet
num_workers=0,
env=ue)
ue.call(["chmod", "700", bundle_runner])
def run_buildtime_managepy_cmd(cmdlist, nonzero_exit_ok=False):
stdout, stderr, p = ue.subproc([bundle_runner] + cmdlist,
nonzero_exit_ok=nonzero_exit_ok)
return stdout, stderr, p
if post_build_hooks is None:
post_build_hooks = []
_stdout, managepy_help, _p = run_buildtime_managepy_cmd(
["help"], nonzero_exit_ok=True)
try:
available_commands = [x.strip() for x in
managepy_help.rsplit(
"Available subcommands:",
1)[1].splitlines()]
if "collectstatic" in available_commands:
zoomdb.log("Found that your project offers a 'collectstatic' "
"management command; adding that to post-build "
"hooks.")
post_build_hooks.append(["collectstatic",
"--link",
"--noinput"])
except IndexError:
zoomdb.log(("Warning: Couldn't determine whether you have a "
"'collectstatic' command because 'manage.py help' didn't "
"provide a list of available subcommands as expected. "
"Full manage.py help output was:\n%s") % managepy_help)
if not post_build_hooks:
zoomdb.log("No post-build hooks found.")
else:
try:
for cmdlist in post_build_hooks:
cmdtext = " ".join(cmdlist)
zoomdb.log("Running post-build 'manage.py %s': " % cmdtext)
cmd_output, cmd_err, cmd_p = run_buildtime_managepy_cmd(
cmdlist)
zoomdb.log("Executed: " + cmd_output + cmd_err)
except RuntimeError, e:
zoomdb.log("Warning: there was an error running a post-build "
"command. Detail:\n" + e.message,
zoomdb.LOG_WARN)