def managepy_shell(app_id, bundle_name, some_python_code):
app_dir, bundle_dir = utils.app_and_bundle_dirs(app_id, bundle_name)
if not os.path.isdir(bundle_dir):
raise utils.InfrastructureException(
"This server doesn't seem to have the requested app bundle " +
"currently deployed: app=%s, bundle=%s." % (app_id, bundle_name))
procargs = [os.path.join(bundle_dir, "thisbundle.py"),
"shell", "--plain"]
ue = userenv.UserEnv(app_id)
stdout, stderr, proc = ue.subproc(procargs,
stdin_string=some_python_code)
result = stdout
if stderr:
result += "\n" + stderr
if proc.returncode != 0:
raise RuntimeError(
(("Nonzero return code %d from manage.py shell script:"
"\n%s\n\nOutput of script was:\n") % (
proc.returncode,
some_python_code)) +
result)
return result
def managepy_command(app_id, bundle_name, command, nonzero_exit_ok=False,
return_exit_code=False):
app_dir, bundle_dir = utils.app_and_bundle_dirs(app_id, bundle_name)
if not os.path.isdir(bundle_dir):
raise utils.InfrastructureException(
"This server doesn't seem to have the requested app bundle " +
"currently deployed: app=%s, bundle=%s." % (app_id, bundle_name))
procargs = [os.path.join(bundle_dir, "thisbundle.py")]
if isinstance(command, list):
procargs += command
else:
procargs.append(command)
ue = userenv.UserEnv(app_id)
stdout, stderr, proc = ue.subproc(procargs, nonzero_exit_ok=True)
ue.destroy()
result = stdout + "\n" + stderr
if proc.returncode != 0 and not(nonzero_exit_ok):
raise RuntimeError(
("Nonzero return code %d from manage.py %s:\n" % (proc.returncode,
command)) +
result)
if return_exit_code:
return proc.returncode, result
else:
return result
def undeploy_from_appserver(zoomdb, app_id, bundle_id,
appserver_instance_id, appserver_port,
zero_undeploys_ok=False):
my_hostname = utils.node_meta("name")
my_instance_id = utils.node_meta("instance_id")
if appserver_instance_id not in (my_hostname, my_instance_id, "localhost"):
raise utils.InfrastructureException(
"Incorrect appserver received undeploy_from_appserver task; " +
"I am %s but the undeploy is for %s." % (my_hostname,
appserver_instance_id))
bundle = zoomdb.get_bundle(bundle_id)
num_stopped = stop_serving_bundle(app_id, bundle.bundle_name)
if num_stopped == 0 and zero_undeploys_ok:
zoomdb.log("Note: no matching bundles were running on %s." %
my_hostname)
elif num_stopped != 1:
raise utils.InfrastructureException(
("Attempting to undeploy one bundle (app_id %s, bundle_id %s, "
"bundle_name %s) from appserver %s:%d, but %d bundles were "
"stopped.")
% (app_id, bundle_id, bundle.bundle_name,
appserver_instance_id, appserver_port, num_stopped))
app_dir, bundle_dir = utils.app_and_bundle_dirs(app_id,
bundle.bundle_name)
if os.path.isdir(bundle_dir):
zoomdb.log("Removing old bundle from %s." % bundle_dir)
utils.chown_to_me(bundle_dir)
shutil.rmtree(bundle_dir)
def test_undeploy_by_dep_ids(self):
"""
Test taking down a deployed bundle based on the appserverdeployment id.
"""
app_dir, bundle_dir = utils.app_and_bundle_dirs(self.app_id,
self.bundle_name)
self.install_my_bundle()
(instance_id, node_name, host_ip, host_port) = \
deploy.start_serving_bundle(self.app_id, self.bundle_name)
self.assertTrue(os.path.isdir(bundle_dir))
self.check_can_eventually_load(
"http://%s:%s" % (host_ip, host_port),
"Welcome to the Django tutorial polls app")
zoomdb = StubZoomDB()
mydep = zoomdb.add_worker(1, "localhost", "127.0.0.1", host_port)
self.assertFalse(deploy.is_port_open(host_port))
deploy.undeploy(zoomdb,
self.app_id,
dep_ids=[mydep.id],
use_subtasks=False,
also_update_proxies=False)
self.assertTrue(deploy.is_port_open(host_port))
#self.assertFalse(os.path.isdir(app_dir))
self.assertFalse(os.path.isdir(bundle_dir))
def start_serving_bundle(app_id, bundle_name):
"""
Serve the given bundle under supervisor, and return the appserver info
for where the service is running.
If you are running locally as dev, you need to make sure the user
running Celery has permissions to write to the /etc/supervisor/conf.d dir.
$ sudo chgrp nateaune /etc/supervisor/conf.d/
$ sudo chmod g+w /etc/supervisor/conf.d/
:returns: (instance_id, node_name, host_ip, host_port)
"""
# check that this bundle isn't already being served here - otherwise
# supervisor will silently ignore the redundant config files!
for bun in get_active_bundles():
if bun["app_id"] == app_id and bun["bundle_name"] == bundle_name:
raise utils.InfrastructureException((
"Redundant bundle service request: server %s (hostname=%s)"
" is already serving app_id %s, bundle_name %s.") % (
utils.node_meta("name"),
socket.gethostname(),
app_id,
bundle_name))
port_to_use = _get_a_free_port()
config_filename = os.path.join(taskconfig.SUPERVISOR_APP_CONF_DIR,
"%s.%s.port.%d.conf" % (app_id,
bundle_name,
port_to_use))
app_dir, bundle_dir = utils.app_and_bundle_dirs(app_id, bundle_name)
utils.render_tpl_to_file(
'deploy/supervisor_entry.conf',
config_filename,
run_in_userenv=os.path.join(taskconfig.PRIVILEGED_PROGRAMS_PATH,
"run_in_userenv"),
custdir=taskconfig.NR_CUSTOMER_DIR,
bundle_name=bundle_name,
bundle_runner=os.path.join(bundle_dir, "thisbundle.py"),
bundle_dir=bundle_dir,
app_user=app_id,
port=port_to_use)
_kick_supervisor()
instance_id = utils.node_meta("instance_id")
node_name = utils.node_meta("name")
host_ip = utils.get_internal_ip()
return (instance_id, node_name, host_ip, port_to_use)
def test_undeploy_with_proxy_update(self):
"""
Test taking down a deployed bundle and updating the proxy config too.
"""
app_dir, bundle_dir = utils.app_and_bundle_dirs(self.app_id,
self.bundle_name)
self.install_my_bundle()
(instance_id, node_name, host_ip, host_port) = \
deploy.start_serving_bundle(self.app_id, self.bundle_name)
# also add to nginx - fake it for this test
here = os.path.abspath(os.path.split(__file__)[0])
fixture_dir = os.path.join(here, '../fixtures')
nginx_site_file = os.path.join(taskconfig.NGINX_SITES_ENABLED_DIR,
self.app_id)
shutil.copyfile(os.path.join(fixture_dir, 'test_deploy_nginx_site'),
nginx_site_file)
self.check_can_eventually_load(
"http://%s:%s" % (host_ip, host_port),
"Welcome to the Django tutorial polls app")
zoomdb = StubZoomDB()
mydep = zoomdb.add_worker(1, "localhost", "127.0.0.1", host_port)
self.assertFalse(deploy.is_port_open(host_port))
zcfg_path = os.path.join(fixture_dir, "app", "zoombuild.cfg")
zcfg_content = open(zcfg_path).read()
# make sure we require proper parameters - skip zoombuild_cfg_content
with self.assertRaises(AssertionError):
deploy.undeploy(zoomdb,
self.app_id,
dep_ids=[mydep.id],
use_subtasks=False,
also_update_proxies=True)
deploy.undeploy(zoomdb,
self.app_id,
dep_ids=[mydep.id],
use_subtasks=False,
also_update_proxies=True,
zoombuild_cfg_content=zcfg_content)
self.assertTrue(deploy.is_port_open(host_port))
self.assertFalse(os.path.isdir(bundle_dir))
self.assertFalse(os.path.isfile(nginx_site_file),
"Expected nginx site file %s to be gone, but it isn't"
% nginx_site_file)
def remove_local_proxy_config(app_id):
site_conf_filename = _get_nginx_conffile(app_id)
if not os.path.isfile(site_conf_filename):
raise utils.InfrastructureException((
"Requested remove_local_proxy_config for app %s, but that "
"app is not currently proxied from this nginx instance (%s). "
"No site configuration file in %s.")
% (app_id,
utils.node_meta("name"),
site_conf_filename))
app_dir, _ = utils.app_and_bundle_dirs(app_id)
shutil.rmtree(app_dir, ignore_errors=True)
os.remove(site_conf_filename)
utils.local_privileged(["kick_nginx"])
def install_app_bundle(app_id, bundle_name, appserver_name, dbinfo,
bundle_storage_engine=bundle_storage,
static_only=False, num_workers=1,
remove_other_bundles=False):
app_dir, bundle_dir = utils.app_and_bundle_dirs(app_id, bundle_name)
if not os.path.exists(bundle_dir):
utils.get_and_extract_bundle(bundle_name, app_dir,
bundle_storage_engine)
if remove_other_bundles:
for fname in os.listdir(app_dir):
if (fname.startswith("bundle_")
and os.path.isdir(fname)
and fname != bundle_name):
shutil.rmtree(fname)
if not static_only:
utils.chown_to_me(bundle_dir)
_write_deployment_config(os.path.join(bundle_dir, "thisbundle.py"),
bundle_name, dbinfo, num_workers=num_workers)
utils.local_privileged(["project_chown", app_id, bundle_dir])
def update_local_proxy_config(app_id, bundle_name,
appservers, virtual_hostnames, site_media_map,
bundle_storage_engine=None,
remove_other_bundles=True):
bundle_storage_engine = bundle.get_bundle_storage_engine(
bundle_storage_engine)
site_conf_filename = _get_nginx_conffile(app_id)
if len(appservers) == 0:
raise utils.InfrastructureException((
"No appserver URLs provided for nginx config update to %s. "
"At least one upstream is required.") % app_id)
app_dir, bundle_dir = utils.app_and_bundle_dirs(app_id, bundle_name)
# We need to make sure this bundle's static is installed locally, so we
# can serve its static media.
if bundle_storage_engine is not SKIP_BUNDLE_INSTALL:
deploy.install_app_bundle_static(
app_id, bundle_name,
bundle_storage_engine,
remove_other_bundles=remove_other_bundles)
file_path_vars = {
"{SITE_PACKAGES}": os.path.join(bundle_dir,
"lib/python2.6/site-packages"),
"{SRC_PACKAGES}": os.path.join(bundle_dir, "src"),
}
default_file_path_base = os.path.join(bundle_dir, "user-repo")
def _make_full_path(original_file_path):
for varname, pathbase in file_path_vars.items():
if original_file_path.startswith(varname):
rest = original_file_path[len(varname):]
rest = rest.lstrip("/")
return os.path.join(pathbase, rest)
return os.path.join(default_file_path_base,
original_file_path)
sme = [dict(url_path=url_path,
alias_dest=_make_full_path(file_path.strip('/')),
) for url_path, file_path in site_media_map.items()]
sme.append(dict(url_path=taskconfig.DZ_ADMIN_MEDIA["url_path"],
alias_dest=os.path.join(
bundle_dir,
taskconfig.DZ_ADMIN_MEDIA["bundle_file_path"])))
# NOTE THE SLASHES::::
# location /static/ {
# alias /tmp/tmpR_1dI5/test001/bundle_test001_2011-03-09-03.52.55/user-src/static/;
# }
utils.render_tpl_to_file("nginx/site.conf",
site_conf_filename,
app_id=app_id,
appservers=[dict(
# (instance_id, node_name, host_ip, host_port)
instance_id=a[0],
node_name=a[1],
host_ip=a[2],
host_port=a[3],
)
for a in appservers],
virtual_hostnames=virtual_hostnames,
site_media_entries=sme)
utils.local_privileged(["kick_nginx"])
