def test_parse_site_media_map(self):
here = path.abspath(path.split(__file__)[0])
test_fixture_cfg = path.join(here, '../fixtures',
'app', 'zoombuild.cfg')
zcfg = utils.parse_zoombuild(test_fixture_cfg)
input_text = zcfg["site_media_map"]
self.assertEqual(len(input_text.splitlines()), 3)
self.assertTrue("static static" in input_text)
self.assertTrue("staticfiles staticfiles" in input_text)
self.assertTrue("foo {SITE_PACKAGES}/foo" in input_text)
smm = utils.parse_site_media_map(input_text)
self.assertTrue(isinstance(smm, dict))
# each url path should start and end with a single slash
for url_path in smm.keys():
self.assertTrue(url_path.startswith("/"))
self.assertTrue(url_path.endswith("/"))
self.assertFalse(url_path.startswith("//"))
self.assertFalse(url_path.endswith("//"))
self.assertEqual(smm["/foo/"], "{SITE_PACKAGES}/foo")
self.assertEqual(smm["/static/"], "static")
self.assertEqual(smm["/staticfiles/"], "staticfiles")
self.assertEqual(len(smm), 3)
def update_hostnames(zoomdb, app_id, zoombuild_cfg_content, use_subtasks=True):
zcfg = utils.parse_zoombuild_string(zoombuild_cfg_content)
site_media_map = utils.parse_site_media_map(zcfg.get("site_media_map", ""))
opts = {"APP_ID": app_id,
"SITE_MEDIA_MAP": site_media_map,
"USE_SUBTASKS": use_subtasks}
utils.run_steps(zoomdb, opts, (
find_deployments,
update_proxy_configuration,
))
def update_front_end_proxy(zoomdb, opts):
"""
Generate config template for site, note location of app servers as proxy
backend. Reload nginx.
Nginx examples for this are in the chef nginx recipe.
When running locally in dev, you must make sure whatever user runs celeryd
has write permission to /etc/nginx/sites-enabled
$ sudo chgrp nateaune /etc/nginx/sites-enabled/
$ sudo chmod g+w /etc/nginx/sites-enabled/
"""
# (instance_id, node_name, host_ip, host_port) format
appservers = opts["DEPLOYED_ADDRESSES"]
virtual_hostnames = zoomdb.get_project_virtual_hosts()
zcfg = utils.parse_zoombuild(os.path.join(opts["APP_DIR"],
"zoombuild.cfg"))
site_media_map = utils.parse_site_media_map(zcfg.get("site_media_map", ""))
args = [zoomdb._job_id, opts["APP_ID"], opts["BUNDLE_NAME"],
appservers, virtual_hostnames, site_media_map]
remove_other_bundles = taskconfig.NGINX_REMOVE_OLD_BUNDLES_ON_UPDATE
if opts["USE_SUBTASKS"]:
res = nginx.update_proxy_conf.apply_async(args=args,
kwargs={
"remove_other_bundles":
remove_other_bundles})
res.wait()
else:
nginx.update_proxy_conf(*args,
remove_other_bundles=remove_other_bundles)
zoomdb.log("Updated proxy server configuration. Your project is now "
"available from the following URLs: " +
", ".join(virtual_hostnames))
def undeploy(zoomdb, app_id, bundle_ids=None, use_subtasks=True,
also_update_proxies=True, dep_ids=None,
zero_undeploys_ok=False,
zoombuild_cfg_content=None,
log_step_events=True):
"""Given an app_id and list of bundle names, undeploy those bundles.
:param bundle_ids: Database IDs of bundles to undeploy. If None, all
bundles for this app will be undeployed.
:param dep_ids: AppServerDeployment IDs to undeploy. This may be optionally
specified instead of specifying bundle_ids.
:param also_update_proxies: Update proxy configuration to point at
remaining deployments. Requires the zoombuild_cfg_content parameter
as well.
:param zoombuild_cfg_content: Content of zoombuild.cfg, used only if
also_update_proxies is true.
"""
if also_update_proxies and (bundle_ids or dep_ids):
assert zoombuild_cfg_content, ("tasklib.deploy.undeploy requires "
"zoombuild_cfg_content parameter if "
"also_update_proxies is true and "
"any instances will remain up.")
step_title = "Deactivating instances"
if log_step_events:
zoomdb.log(step_title, zoomdb.LOG_STEP_BEGIN)
if not dep_ids:
matching_deployments = zoomdb.search_workers(bundle_ids, active=True)
else:
all_workers = zoomdb.get_project_workers()
matching_deployments = [w for w in all_workers if w.id in dep_ids]
if len(matching_deployments) == 0:
if not zero_undeploys_ok:
raise utils.InfrastructureException(
"No active deployments found for app_id=%s, bundle_ids=%r." %
(app_id, bundle_ids))
for dep in matching_deployments:
if dep.deactivation_date:
zoomdb.log("Deployment %s appears to be already deactivated."
% dep, zoomdb.LOG_WARN)
droptasks = []
import dz.tasks.deploy # do this non-globally due to dependencies
def save_undeployment(dep):
dep.deactivation_date = datetime.datetime.utcnow().replace(
tzinfo=pytz.utc)
zoomdb.flush()
zoomdb.log(("Dropped bundle #%d (worker #%d) from server %s (%s:%d). "
"Deactivated: %s.") % (
dep.bundle_id,
dep.id,
dep.server_instance_id,
dep.server_ip, dep.server_port,
dep.deactivation_date,
))
for dep in matching_deployments:
args = [app_id,
dep.bundle_id, ### DOH! Need to set dep.id here too!
dep.server_instance_id,
dep.server_port]
kwargs = {"zero_undeploys_ok": zero_undeploys_ok}
zoomdb.log("Dropping bundle #%d from server %s (%s:%d)..." % (
dep.bundle_id, dep.server_instance_id,
dep.server_ip, dep.server_port))
if use_subtasks:
droptasks.append(
dz.tasks.deploy.undeploy_from_appserver.apply_async(
args=[zoomdb.get_job_id()] + args,
kwargs=kwargs,
queue="appserver:" + dep.server_instance_id))
else:
result = undeploy_from_appserver(zoomdb, *args, **kwargs)
save_undeployment(dep)
# if using subtasks, wait for the async tasks to finish
if use_subtasks:
for dep, dt in zip(matching_deployments, droptasks):
result = dt.wait()
save_undeployment(dep)
# now update frontend proxies
if also_update_proxies:
active_workers = [w for w in zoomdb.get_project_workers()
if not(w.deactivation_date)]
remaining_appservers = [(w.server_instance_id,
w.server_instance_id, # should be node name
w.server_ip,
w.server_port)
for w in active_workers]
if len(remaining_appservers):
newest_worker = max(active_workers, key=lambda x: x.creation_date)
newest_bundle = zoomdb.get_bundle(newest_worker.bundle_id)
zoomdb.log("Updating front-end proxy to use remaining appservers "
"(%r)" % (remaining_appservers,))
if use_subtasks:
zcfg = utils.parse_zoombuild_string(zoombuild_cfg_content)
site_media_map = utils.parse_site_media_map(
zcfg.get("site_media_map", ""))
import dz.tasks.nginx
proxy_task = dz.tasks.nginx.update_proxy_conf.apply_async(args=[
zoomdb.get_job_id(),
app_id,
newest_bundle.bundle_name, # to serve static assets
remaining_appservers,
zoomdb.get_project_virtual_hosts(),
site_media_map])
proxy_task.wait()
else:
import dz.tasklib.nginx
dz.tasklib.nginx.update_local_proxy_config(
app_id,
remaining_appservers,
zoomdb.get_project_virtual_hosts())
else:
# there are no more appservers; remove from proxy
zoomdb.log(("This undeployment removes the last active appservers "
"for %r; stopping front-end proxy service for "
"associated virtual hostnames too.") % app_id)
if use_subtasks:
import dz.tasks.nginx
subtask = dz.tasks.nginx.remove_proxy_conf.apply_async(
args=[zoomdb.get_job_id(), app_id])
subtask.wait()
else:
import dz.tasklib.nginx
dz.tasklib.nginx.remove_local_proxy_config(app_id)
if log_step_events:
zoomdb.log(step_title, zoomdb.LOG_STEP_END)
def restore_worker(zoomdb, w):
from dz.tasklib import build_and_deploy as bd
from dz.tasklib import (taskconfig,
utils)
from dz.tasklib.database import DatabaseInfo
from dz.tasks import nginx
print "restoring: %s" % w
soup = zoomdb._soup
project = soup.dz2_project.filter(
soup.dz2_project.id == w.project_id).one()
bundle = soup.dz2_appbundle.filter(
soup.dz2_appbundle.id == w.bundle_id).one()
print w, project, bundle
# force zoomdb to have the right project id - HIDEOUS HACK
zoomdb.get_project_id = lambda: project.id
zoomdb._project = project
def fake_log(msg):
print "ZOOMDB:", msg
zoomdb.log = fake_log
app_id = taskconfig.PROJECT_SYSID_FORMAT % project.id
app_dir = os.path.join(taskconfig.NR_CUSTOMER_DIR, app_id)
opts = {
"BUNDLE_INFO": bundle,
"APP_ID": app_id,
"APP_DIR": app_dir,
"BUNDLE_NAME": bundle.bundle_name,
"DB": DatabaseInfo(project.db_host,
project.db_name,
project.db_username,
project.db_password),
"NUM_WORKERS": project.num_workers,
"USE_SUBTASKS": True,
# Fake ZOOMBUILD_CFG_CONTENT. bd.remove_previous_versions
# needs it in some cases, but doesn't actually use the value here.
"ZOOMBUILD_CFG_CONTENT": None,
}
bd.select_app_server_for_deployment(zoomdb, opts)
# undeploy all existing versions
opts["DEPLOYED_WORKERS"] = [] # trick the following into removing all
bd.remove_previous_versions(zoomdb, opts)
bd.deploy_project_to_appserver(zoomdb, opts)
# sets opts["DEPLOYED_ADDRESSES"] with
# (instance_id, node_name, host_ip, host_port)
# and opts["DEPLOYED_WORKERS"] with worker objects.
new_w = opts["DEPLOYED_WORKERS"][0]
#from celery.contrib import rdb; rdb.set_trace()
# # don't run_post_deploy_hooks
# replace bd.update_front_end_proxy
fake_job_id = 1676 # TODO: fixme. Must have jobid in both local db and remote db.
appservers = opts["DEPLOYED_ADDRESSES"]
virtual_hostnames = zoomdb.get_project_virtual_hosts()
site_media_map = utils.parse_site_media_map(project.site_media)
args = [fake_job_id, opts["APP_ID"], opts["BUNDLE_NAME"],
appservers, virtual_hostnames, site_media_map]
res = nginx.update_proxy_conf.apply_async(
args=args,
kwargs={"remove_other_bundles": True})
res.wait()
return new_w
