def destroy(self):
# first, attempt to send a polite shutdown request
self.sock.send_pyobj(["shutdown"])
#print "CLOSING ZMQ WORKER SOCKET"
self.sock.close()
# give it a little time to exit
for wait in range(20):
if self.worker_proc.poll() is None:
eventlet.sleep(0.1 * wait)
else:
break
# didn't work? try SIGTERM and SIGKILL
pid = self.worker_proc.pid
for signum in (signal.SIGTERM, signal.SIGKILL):
if self.worker_proc.poll() is None:
utils.local_privileged(["kill_worker", signum, str(pid)])
for wait in range(20):
eventlet.sleep(0.1 * wait)
if self.worker_proc.poll() is not None:
break
# if it's still alive, CRAZY ERROR
if self.worker_proc.poll() is None:
print ("ERROR: cannot terminate worker %s, process %d" %
(self.worker_dir, self.worker_proc.pid))
return
utils.chown_to_me(self.worker_dir)
shutil.rmtree(self.worker_dir)
def test_local_privileged_nonexistent(self):
"""
Test trying to run a program that doesn't actually exist in the
privileged-bin directory.
"""
with self.assertRaises(AssertionError):
utils.local_privileged(["cat"])
def __init__(self, served_bundle):
self.served_bundle = served_bundle
fmt_info = {
"app_id": served_bundle.app_id,
"bundle_name": served_bundle.bundle_name,
"worker_id": str(id(self))}
self.worker_dir = wd = mafconfig.WORKER_DIR_FORMAT % fmt_info
self.addr = mafconfig.WORKER_ADDR_FORMAT % {"worker_dir": wd}
bundleworker_path = (mafconfig.WORKER_USERENV_BUNDLEWORKER_PATH_FORMAT
% {"worker_dir": wd})
print "LAUNCHING NM WORKER", self.addr, "...",
ue = self.served_bundle.userenv
if not os.path.isdir(wd):
os.makedirs(wd)
utils.local_privileged(["project_chown", served_bundle.app_id, wd])
ue.subproc(["chmod", "0755", wd])
ue.write_string_to_file(open(mafconfig.WORKER_EXE).read(),
bundleworker_path)
self.worker_proc = ue.popen(["/usr/bin/python", bundleworker_path,
self.addr])
#print "Worker proc launched: %r" % self.worker_proc
#self.subproc = subprocess.Popen([mafconfig.WORKER_EXE, self.addr])
self.sock = context.socket(zmq.REQ)
self.sock.connect(self.addr)
#print "connected to worker"
# make the first request to the worker before returning
self.sock.send_pyobj(["ready?"])
print "Waiting for worker to be ready ... ",
rep = self.sock.recv_pyobj()
print "OK: %r" % (rep,)
def test_local_privileged_stdin_string(self):
"""
Runs a privileged external program with a string passed in on stdin.
"""
test_string = "Hello, privileged world!"
result = utils.local_privileged(["test_cat"], stdin_string=test_string)
self.assertEqual(test_string, result, result)
def gunicorn_signal(gunicorn_master_pid, signal_name, appserver_name):
my_hostname = utils.node_meta("name")
if appserver_name not in (my_hostname, "localhost"):
raise utils.InfrastructureException(
"Incorrect appserver received gunicorn_signal task; " +
"I am %s but the task is requesting %s." % (my_hostname,
appserver_name))
if signal_name not in ("TTIN", "TTOU"):
raise utils.InfrastructureException(
"Unexpected gunicorn_signal %s: only TTIN & TTOU allowed."
% signal_name)
utils.local_privileged(["gunicorn_signal",
signal_name,
gunicorn_master_pid])
def test_chown_to_me(self):
"""
Test function to chown any file to the current user.
"""
test_file = path.join(self.dir, "test%d" % random.randint(9999, 19999))
utils.local("touch %s" % test_file)
me = utils.local("whoami").strip()
self.assertFileOwnedBy(test_file, me)
utils.local_privileged(["project_chown", "root", test_file])
self.assertFileOwnedBy(test_file, "root")
utils.chown_to_me(test_file)
self.assertFileOwnedBy(test_file, me)
utils.local("rm %s" % test_file)
def remove_local_proxy_config(app_id):
site_conf_filename = _get_nginx_conffile(app_id)
if not os.path.isfile(site_conf_filename):
raise utils.InfrastructureException((
"Requested remove_local_proxy_config for app %s, but that "
"app is not currently proxied from this nginx instance (%s). "
"No site configuration file in %s.")
% (app_id,
utils.node_meta("name"),
site_conf_filename))
app_dir, _ = utils.app_and_bundle_dirs(app_id)
shutil.rmtree(app_dir, ignore_errors=True)
os.remove(site_conf_filename)
utils.local_privileged(["kick_nginx"])
def test_env_includes_cust_dir(self):
"""
Test that env includes the current customer's directory.
"""
some_crap = str(random.randint(999999999999999999999999, 1999999999999999999999999))
user_dir = path.join(taskconfig.NR_CUSTOMER_DIR, self.project_sysid)
fake_bundle_dir = path.join(user_dir, "testbundle")
if not path.isdir(fake_bundle_dir):
os.mkdir(fake_bundle_dir)
utils.local_privileged(["project_chown", self.project_sysid, fake_bundle_dir])
some_crap_filename = path.join(fake_bundle_dir, "test_" + some_crap)
self.ue.write_string_to_file(some_crap, some_crap_filename)
self.assertTrue(path.isfile(some_crap_filename), "Couldn't find %s in container" % some_crap_filename)
self.ue.subproc(["rm", some_crap_filename])
def get_log_content(app_id, log_id, lines=100):
"""
Get the bottom ``lines`` of content from the given log.
:returns: A string containing no more than ``lines`` lines.
"""
is_allowed = any(x["log_id"] == log_id for x in get_available_logs(app_id))
if not is_allowed:
raise UnavailableLogError(
"Requested log %s is not available [app_id: %s]." %
(log_id, app_id))
return utils.local_privileged(["logtail", str(lines), log_id])
def install_app_bundle(app_id, bundle_name, appserver_name, dbinfo,
bundle_storage_engine=bundle_storage,
static_only=False, num_workers=1,
remove_other_bundles=False):
app_dir, bundle_dir = utils.app_and_bundle_dirs(app_id, bundle_name)
if not os.path.exists(bundle_dir):
utils.get_and_extract_bundle(bundle_name, app_dir,
bundle_storage_engine)
if remove_other_bundles:
for fname in os.listdir(app_dir):
if (fname.startswith("bundle_")
and os.path.isdir(fname)
and fname != bundle_name):
shutil.rmtree(fname)
if not static_only:
utils.chown_to_me(bundle_dir)
_write_deployment_config(os.path.join(bundle_dir, "thisbundle.py"),
bundle_name, dbinfo, num_workers=num_workers)
utils.local_privileged(["project_chown", app_id, bundle_dir])
def enable_postgis(database):
"""
Enable PostGIS on the given database.
:returns: Output of postgis install script (lots of SQL confirmations).
"""
(stdout, stderr, p) = utils.local_privileged(
["psql_enable_postgis", database],
return_details=True)
if p.returncode != 0:
raise utils.InfrastructureException(
("Error (return code %d) while attempting to run PostGIS "
"installation: \n%s\n%s") % (p.returncode, stdout, stderr))
else:
return "%s\n%s" % (stdout, stderr)
def test_local_privileged(self):
"""Runs a privileged (setuid) external program.
"""
result = utils.local_privileged(["whoami"])
self.assertEqual("root\n", result)
def update_local_proxy_config(app_id, bundle_name,
appservers, virtual_hostnames, site_media_map,
bundle_storage_engine=None,
remove_other_bundles=True):
bundle_storage_engine = bundle.get_bundle_storage_engine(
bundle_storage_engine)
site_conf_filename = _get_nginx_conffile(app_id)
if len(appservers) == 0:
raise utils.InfrastructureException((
"No appserver URLs provided for nginx config update to %s. "
"At least one upstream is required.") % app_id)
app_dir, bundle_dir = utils.app_and_bundle_dirs(app_id, bundle_name)
# We need to make sure this bundle's static is installed locally, so we
# can serve its static media.
if bundle_storage_engine is not SKIP_BUNDLE_INSTALL:
deploy.install_app_bundle_static(
app_id, bundle_name,
bundle_storage_engine,
remove_other_bundles=remove_other_bundles)
file_path_vars = {
"{SITE_PACKAGES}": os.path.join(bundle_dir,
"lib/python2.6/site-packages"),
"{SRC_PACKAGES}": os.path.join(bundle_dir, "src"),
}
default_file_path_base = os.path.join(bundle_dir, "user-repo")
def _make_full_path(original_file_path):
for varname, pathbase in file_path_vars.items():
if original_file_path.startswith(varname):
rest = original_file_path[len(varname):]
rest = rest.lstrip("/")
return os.path.join(pathbase, rest)
return os.path.join(default_file_path_base,
original_file_path)
sme = [dict(url_path=url_path,
alias_dest=_make_full_path(file_path.strip('/')),
) for url_path, file_path in site_media_map.items()]
sme.append(dict(url_path=taskconfig.DZ_ADMIN_MEDIA["url_path"],
alias_dest=os.path.join(
bundle_dir,
taskconfig.DZ_ADMIN_MEDIA["bundle_file_path"])))
# NOTE THE SLASHES::::
# location /static/ {
# alias /tmp/tmpR_1dI5/test001/bundle_test001_2011-03-09-03.52.55/user-src/static/;
# }
utils.render_tpl_to_file("nginx/site.conf",
site_conf_filename,
app_id=app_id,
appservers=[dict(
# (instance_id, node_name, host_ip, host_port)
instance_id=a[0],
node_name=a[1],
host_ip=a[2],
host_port=a[3],
)
for a in appservers],
virtual_hostnames=virtual_hostnames,
site_media_entries=sme)
utils.local_privileged(["kick_nginx"])
djver = taskconfig.DJANGO_VERSIONS[buildconfig_info['django_version']]
ver_tarball = os.path.join(taskconfig.DJANGO_TARBALLS_DIR,
djver["tarball"])
if os.path.isfile(ver_tarball):
django_req = ver_tarball
else:
django_req = djver["pip_line"]
utils.install_requirements([django_req], bundle_dir,
logsuffix="-django")
# This is where we've finished running code we trust (virtualenv, django,
# etc) and switch to running code provided or pointed to by the user. So
# let's chown everything to the app user.
utils.local_privileged(["project_chown",
app_id,
bundle_dir])
# and let's create the userenv!
ue = userenv.UserEnv(app_id)
# install user-provided requirements
reqs = utils.assemble_requirements(
files=[l.strip() for l in
buildconfig_info["requirements_files"].splitlines()],
lines=[l.strip() for l in
buildconfig_info["extra_requirements"].splitlines()],
basedir=repo_link,
ignore_keys="django",
env=ue)
