def test_parse_site_media_map(self): here = path.abspath(path.split(__file__)[0]) test_fixture_cfg = path.join(here, '../fixtures', 'app', 'zoombuild.cfg') zcfg = utils.parse_zoombuild(test_fixture_cfg) input_text = zcfg["site_media_map"] self.assertEqual(len(input_text.splitlines()), 3) self.assertTrue("static static" in input_text) self.assertTrue("staticfiles staticfiles" in input_text) self.assertTrue("foo {SITE_PACKAGES}/foo" in input_text) smm = utils.parse_site_media_map(input_text) self.assertTrue(isinstance(smm, dict)) # each url path should start and end with a single slash for url_path in smm.keys(): self.assertTrue(url_path.startswith("/")) self.assertTrue(url_path.endswith("/")) self.assertFalse(url_path.startswith("//")) self.assertFalse(url_path.endswith("//")) self.assertEqual(smm["/foo/"], "{SITE_PACKAGES}/foo") self.assertEqual(smm["/static/"], "static") self.assertEqual(smm["/staticfiles/"], "staticfiles") self.assertEqual(len(smm), 3)
def update_hostnames(zoomdb, app_id, zoombuild_cfg_content, use_subtasks=True): zcfg = utils.parse_zoombuild_string(zoombuild_cfg_content) site_media_map = utils.parse_site_media_map(zcfg.get("site_media_map", "")) opts = {"APP_ID": app_id, "SITE_MEDIA_MAP": site_media_map, "USE_SUBTASKS": use_subtasks} utils.run_steps(zoomdb, opts, ( find_deployments, update_proxy_configuration, ))
def update_front_end_proxy(zoomdb, opts): """ Generate config template for site, note location of app servers as proxy backend. Reload nginx. Nginx examples for this are in the chef nginx recipe. When running locally in dev, you must make sure whatever user runs celeryd has write permission to /etc/nginx/sites-enabled $ sudo chgrp nateaune /etc/nginx/sites-enabled/ $ sudo chmod g+w /etc/nginx/sites-enabled/ """ # (instance_id, node_name, host_ip, host_port) format appservers = opts["DEPLOYED_ADDRESSES"] virtual_hostnames = zoomdb.get_project_virtual_hosts() zcfg = utils.parse_zoombuild(os.path.join(opts["APP_DIR"], "zoombuild.cfg")) site_media_map = utils.parse_site_media_map(zcfg.get("site_media_map", "")) args = [zoomdb._job_id, opts["APP_ID"], opts["BUNDLE_NAME"], appservers, virtual_hostnames, site_media_map] remove_other_bundles = taskconfig.NGINX_REMOVE_OLD_BUNDLES_ON_UPDATE if opts["USE_SUBTASKS"]: res = nginx.update_proxy_conf.apply_async(args=args, kwargs={ "remove_other_bundles": remove_other_bundles}) res.wait() else: nginx.update_proxy_conf(*args, remove_other_bundles=remove_other_bundles) zoomdb.log("Updated proxy server configuration. Your project is now " "available from the following URLs: " + ", ".join(virtual_hostnames))
def undeploy(zoomdb, app_id, bundle_ids=None, use_subtasks=True, also_update_proxies=True, dep_ids=None, zero_undeploys_ok=False, zoombuild_cfg_content=None, log_step_events=True): """Given an app_id and list of bundle names, undeploy those bundles. :param bundle_ids: Database IDs of bundles to undeploy. If None, all bundles for this app will be undeployed. :param dep_ids: AppServerDeployment IDs to undeploy. This may be optionally specified instead of specifying bundle_ids. :param also_update_proxies: Update proxy configuration to point at remaining deployments. Requires the zoombuild_cfg_content parameter as well. :param zoombuild_cfg_content: Content of zoombuild.cfg, used only if also_update_proxies is true. """ if also_update_proxies and (bundle_ids or dep_ids): assert zoombuild_cfg_content, ("tasklib.deploy.undeploy requires " "zoombuild_cfg_content parameter if " "also_update_proxies is true and " "any instances will remain up.") step_title = "Deactivating instances" if log_step_events: zoomdb.log(step_title, zoomdb.LOG_STEP_BEGIN) if not dep_ids: matching_deployments = zoomdb.search_workers(bundle_ids, active=True) else: all_workers = zoomdb.get_project_workers() matching_deployments = [w for w in all_workers if w.id in dep_ids] if len(matching_deployments) == 0: if not zero_undeploys_ok: raise utils.InfrastructureException( "No active deployments found for app_id=%s, bundle_ids=%r." % (app_id, bundle_ids)) for dep in matching_deployments: if dep.deactivation_date: zoomdb.log("Deployment %s appears to be already deactivated." % dep, zoomdb.LOG_WARN) droptasks = [] import dz.tasks.deploy # do this non-globally due to dependencies def save_undeployment(dep): dep.deactivation_date = datetime.datetime.utcnow().replace( tzinfo=pytz.utc) zoomdb.flush() zoomdb.log(("Dropped bundle #%d (worker #%d) from server %s (%s:%d). " "Deactivated: %s.") % ( dep.bundle_id, dep.id, dep.server_instance_id, dep.server_ip, dep.server_port, dep.deactivation_date, )) for dep in matching_deployments: args = [app_id, dep.bundle_id, ### DOH! Need to set dep.id here too! dep.server_instance_id, dep.server_port] kwargs = {"zero_undeploys_ok": zero_undeploys_ok} zoomdb.log("Dropping bundle #%d from server %s (%s:%d)..." % ( dep.bundle_id, dep.server_instance_id, dep.server_ip, dep.server_port)) if use_subtasks: droptasks.append( dz.tasks.deploy.undeploy_from_appserver.apply_async( args=[zoomdb.get_job_id()] + args, kwargs=kwargs, queue="appserver:" + dep.server_instance_id)) else: result = undeploy_from_appserver(zoomdb, *args, **kwargs) save_undeployment(dep) # if using subtasks, wait for the async tasks to finish if use_subtasks: for dep, dt in zip(matching_deployments, droptasks): result = dt.wait() save_undeployment(dep) # now update frontend proxies if also_update_proxies: active_workers = [w for w in zoomdb.get_project_workers() if not(w.deactivation_date)] remaining_appservers = [(w.server_instance_id, w.server_instance_id, # should be node name w.server_ip, w.server_port) for w in active_workers] if len(remaining_appservers): newest_worker = max(active_workers, key=lambda x: x.creation_date) newest_bundle = zoomdb.get_bundle(newest_worker.bundle_id) zoomdb.log("Updating front-end proxy to use remaining appservers " "(%r)" % (remaining_appservers,)) if use_subtasks: zcfg = utils.parse_zoombuild_string(zoombuild_cfg_content) site_media_map = utils.parse_site_media_map( zcfg.get("site_media_map", "")) import dz.tasks.nginx proxy_task = dz.tasks.nginx.update_proxy_conf.apply_async(args=[ zoomdb.get_job_id(), app_id, newest_bundle.bundle_name, # to serve static assets remaining_appservers, zoomdb.get_project_virtual_hosts(), site_media_map]) proxy_task.wait() else: import dz.tasklib.nginx dz.tasklib.nginx.update_local_proxy_config( app_id, remaining_appservers, zoomdb.get_project_virtual_hosts()) else: # there are no more appservers; remove from proxy zoomdb.log(("This undeployment removes the last active appservers " "for %r; stopping front-end proxy service for " "associated virtual hostnames too.") % app_id) if use_subtasks: import dz.tasks.nginx subtask = dz.tasks.nginx.remove_proxy_conf.apply_async( args=[zoomdb.get_job_id(), app_id]) subtask.wait() else: import dz.tasklib.nginx dz.tasklib.nginx.remove_local_proxy_config(app_id) if log_step_events: zoomdb.log(step_title, zoomdb.LOG_STEP_END)
def restore_worker(zoomdb, w): from dz.tasklib import build_and_deploy as bd from dz.tasklib import (taskconfig, utils) from dz.tasklib.database import DatabaseInfo from dz.tasks import nginx print "restoring: %s" % w soup = zoomdb._soup project = soup.dz2_project.filter( soup.dz2_project.id == w.project_id).one() bundle = soup.dz2_appbundle.filter( soup.dz2_appbundle.id == w.bundle_id).one() print w, project, bundle # force zoomdb to have the right project id - HIDEOUS HACK zoomdb.get_project_id = lambda: project.id zoomdb._project = project def fake_log(msg): print "ZOOMDB:", msg zoomdb.log = fake_log app_id = taskconfig.PROJECT_SYSID_FORMAT % project.id app_dir = os.path.join(taskconfig.NR_CUSTOMER_DIR, app_id) opts = { "BUNDLE_INFO": bundle, "APP_ID": app_id, "APP_DIR": app_dir, "BUNDLE_NAME": bundle.bundle_name, "DB": DatabaseInfo(project.db_host, project.db_name, project.db_username, project.db_password), "NUM_WORKERS": project.num_workers, "USE_SUBTASKS": True, # Fake ZOOMBUILD_CFG_CONTENT. bd.remove_previous_versions # needs it in some cases, but doesn't actually use the value here. "ZOOMBUILD_CFG_CONTENT": None, } bd.select_app_server_for_deployment(zoomdb, opts) # undeploy all existing versions opts["DEPLOYED_WORKERS"] = [] # trick the following into removing all bd.remove_previous_versions(zoomdb, opts) bd.deploy_project_to_appserver(zoomdb, opts) # sets opts["DEPLOYED_ADDRESSES"] with # (instance_id, node_name, host_ip, host_port) # and opts["DEPLOYED_WORKERS"] with worker objects. new_w = opts["DEPLOYED_WORKERS"][0] #from celery.contrib import rdb; rdb.set_trace() # # don't run_post_deploy_hooks # replace bd.update_front_end_proxy fake_job_id = 1676 # TODO: fixme. Must have jobid in both local db and remote db. appservers = opts["DEPLOYED_ADDRESSES"] virtual_hostnames = zoomdb.get_project_virtual_hosts() site_media_map = utils.parse_site_media_map(project.site_media) args = [fake_job_id, opts["APP_ID"], opts["BUNDLE_NAME"], appservers, virtual_hostnames, site_media_map] res = nginx.update_proxy_conf.apply_async( args=args, kwargs={"remove_other_bundles": True}) res.wait() return new_w