def setUp(self):
self.zoomdb = StubZoomDB()
self.work_dir = self.makeDir()
self.opts = {"CO_DIR": self.work_dir}
self.repo_dirs = {}
for vcs in VCSes:
self.repo_dirs[vcs] = self.makeDir()
repo_archive = self.get_fixture_path("test-repo-%s.tar.gz" % vcs)
utils.local("cd %s; tar xvzf %s" % (self.repo_dirs[vcs], repo_archive))
super(RepoCheckoutTestCase, self).setUp()
def tearDown(self):
# TODO: instead of just manually throwing away DB stuff, add a
# destroy_project_data function that could be user-accessible in
# case a user ever wants to throw away their DB and start over.
try:
database.drop_database(self.app_id)
database.drop_user(self.app_id)
except ProgrammingError: # probably indicates DB/user doesn't exist
pass
# chown cust dir to me so we can delete it
utils.chown_to_me(self.dir)
# delete any lingering supervisor conf files
utils.local("rm -f %s/%s*" % (taskconfig.SUPERVISOR_APP_CONF_DIR, self.app_id))
def test_run(self):
"""
Tests our small wrapper around fabric's `run`
"""
cmd = 'echo "foo"'
out = utils.local(cmd, capture=True)
self.assertEqual(out, 'foo')
def test_safe_build(self):
"""
Test that build actions run under the proper user.
"""
# use the app fixture and create a repo
here = path.abspath(path.split(__file__)[0])
src = path.join(here, '../fixtures', self.project_name)
dest = path.join(self.cust_dir, self.project_name)
shutil.copytree(src, dest)
utils.local('(cd %s; git init; git add -A; git commit -m test)' %
path.join(dest, 'src'))
(bundle_name, code_revision) = bundle.bundle_app(self.project_name)
bundle_dir = path.join(self.cust_dir, self.project_name, bundle_name)
a_bundle_file = path.join(bundle_dir, "user-repo", "settings.py")
self.assertFileOwnedBy(a_bundle_file, self.project_name)
def test_chown_to_me(self):
"""
Test function to chown any file to the current user.
"""
test_file = path.join(self.dir, "test%d" % random.randint(9999, 19999))
utils.local("touch %s" % test_file)
me = utils.local("whoami").strip()
self.assertFileOwnedBy(test_file, me)
utils.local_privileged(["project_chown", "root", test_file])
self.assertFileOwnedBy(test_file, "root")
utils.chown_to_me(test_file)
self.assertFileOwnedBy(test_file, me)
utils.local("rm %s" % test_file)
def _prep_build_test_bundle(self, **kwargs):
self.patch(taskconfig, "NR_CUSTOMER_DIR", self.dir)
# install_requirements = self.mocker.replace(
# "dz.tasklib.utils.install_requirements")
# install_requirements(["Django==1.2.5"], MATCH(os.path.isdir),
# logsuffix="-django")
# install_requirements(ANY, MATCH(os.path.isdir))
# self.mocker.replay()
here = path.abspath(path.split(__file__)[0])
src = path.join(here, '../fixtures', 'app')
dest = path.join(self.dir, 'app')
if not os.path.isdir(dest):
shutil.copytree(src, dest)
utils.local('(cd %s; git init; git add -A; git commit -m test)' %
path.join(dest, 'src'))
return bundle.bundle_app('app', **kwargs)
def get_df():
result = []
for cmd in ("df -h", "df -i"):
output = utils.local(cmd)
for i, line in enumerate(output.splitlines()):
if i == 0 or line.startswith("/dev"):
result.append(line)
result.append("")
return "\n".join(result)
def test_change_of_repo_url(self):
"""
Test checking out from a changed repo URL.
"""
repos = []
for repo_num in range(2):
repo_dir = self.makeDir()
utils.local("git init %s" % repo_dir)
utils.local("touch %s/file_%d" % (repo_dir, repo_num))
utils.local("cd %s; git add file_%d; git commit -m 'unittest'" % (repo_dir, repo_num))
repos.append(dict(num=repo_num, path=repo_dir))
co_dir = self.makeDir()
opts = {"CO_DIR": co_dir, "SRC_URL": repos[0]["path"]}
common_steps.checkout_code(self.zoomdb, opts)
self.assertTrue(os.path.isfile("%s/file_%d" % (co_dir, 0)))
self.assertFalse(os.path.isfile("%s/file_%d" % (co_dir, 1)))
# now change the repo source url
opts["SRC_URL"] = repos[1]["path"]
common_steps.checkout_code(self.zoomdb, opts)
self.assertTrue(os.path.isfile("%s/file_%d" % (co_dir, 1)), "New repo should be in place.")
self.assertFalse(os.path.isfile("%s/file_%d" % (co_dir, 0)), "Old repo shouldn't still be in place.")
# and run again to ensure we now do a pull
self.zoomdb.logs = []
self.assertFalse(("Running git pull...", "i") in self.zoomdb.logs)
common_steps.checkout_code(self.zoomdb, opts)
self.assertTrue(("Running git pull...", "i") in self.zoomdb.logs)
def test_check_repo(self):
"""
Check repo and make guesses!
"""
self.patch(taskconfig, "NR_CUSTOMER_DIR", self.dir)
zoomdb = StubZoomDB()
app_id = "p001"
here = path.abspath(path.split(__file__)[0])
tarball_path = path.join(here, '../fixtures', 'repo.tar.gz')
tmp_repo_dir = self.makeDir()
utils.local("tar xvzf %s -C %s" % (tarball_path, tmp_repo_dir))
src_url = path.join(tmp_repo_dir, "repo")
check_repo.check_repo(zoomdb, app_id, "git", src_url)
src_dir = path.join(self.dir, app_id, "src")
self.assertTrue(path.isdir(src_dir),
"src dir does not exist")
self.assertTrue(path.isdir(path.join(src_dir,
".git")),
"no .git dir inside src dir")
self.assertEqual(len(zoomdb.config_guesses), 5)
def has_one_guess(field, value):
matches = [g for g in zoomdb.config_guesses
if g["field"] == field and g["value"] == value]
return len(matches) == 1
self.assertTrue(has_one_guess("additional_python_path_dirs",
"version2\nversion2/voo2do"))
self.assertTrue(has_one_guess(
"requirements_file_names",
"\n".join(["requirements.txt",
"version2/voo2do/requirements/base.txt"])))
def test_node_metadata(self):
instance_id = utils.node_meta("instance_id")
node_name = utils.node_meta("name")
node_role = utils.node_meta("role")
self.assertEqual(instance_id, "localhost")
self.assertEqual(node_name, utils.local("hostname").strip())
self.assertEqual(node_role, "localhost")
here = path.abspath(path.split(__file__)[0])
test_fixture_meta = path.join(here, '../fixtures', 'node_meta')
self.patch(taskconfig, "NODE_META_DATA_DIR", test_fixture_meta)
self.assertEqual(utils.node_meta("instance_id"), "i-12345")
self.assertEqual(utils.node_meta("name"), "myname")
self.assertEqual(utils.node_meta("role"), "myrole")
def _maxdisk():
maxdisk = dict(pct=0)
# parse df output to find max disk use indicator
header_line = ""
for cmd, usetype in (("df -h", "space"),
("df -i", "inode")):
output = utils.local(cmd)
for i, line in enumerate(output.splitlines()):
if i == 0:
header_line = line
if line.startswith("/dev") or line.startswith(".host"): # added for VMWare compatibility
pct_use = line.split()[-2]
if pct_use.endswith("%"):
pct = int(pct_use[:-1])
if pct > maxdisk["pct"]:
maxdisk = dict(pct=pct,
type=usetype,
detail="\n".join((header_line,
line)))
return maxdisk
def create_test_bundle_in_local_storage():
"""
Creates a bundle for testing, and uploads it to the local storage.
Returns the bundle's tarball's name.
This function is used from test_nginx and perhaps other tests that
require a bundle -- modify with care.
"""
print "Making a bundle fixture for testing."
bundle_name = "bundle_test_deploy_app_2011-fixture"
here = os.path.abspath(os.path.split(__file__)[0])
fixture_dir = os.path.join(here, '../fixtures')
app_name = "test_deploy_app"
# force rename the bundle
app_dir = os.path.join(taskconfig.NR_CUSTOMER_DIR, app_name)
if os.path.isdir(app_dir):
utils.chown_to_me(app_dir)
shutil.rmtree(app_dir)
shutil.copytree(os.path.join(fixture_dir, "app"), app_dir)
# create a git repo in app_dir and do a commit so that we can get the
# most recent commit during bundle_app.
utils.local(";".join([
("cd %s/src" % app_dir),
"git init .",
"git add __init__.py",
"git commit -m 'initial test_deploy commit'"]))
zcfg_path = os.path.join(app_dir, "zoombuild.cfg")
zcfg_content = file(zcfg_path).read()
# django_tarball = os.path.join(fixture_dir, 'Django-1.2.5.tar.gz')
# we don't use pip_reqs any more
# zcfg_content = zcfg_content.replace(
# "pip_reqs: Django==1.2.5", "pip_reqs: %s" % django_tarball)
faster_zcfg = file(zcfg_path, "w")
faster_zcfg.write(zcfg_content)
faster_zcfg.close()
bundle_name, code_revision = bundle.bundle_app(
app_name,
force_bundle_name=bundle_name)
bundle_dir = os.path.join(taskconfig.NR_CUSTOMER_DIR,
app_name,
bundle_name)
tarball_name = bundle.zip_and_upload_bundle(app_name,
bundle_name,
bundle_storage_local)
print "Created bundle fixture in %s" % tarball_name
# after upload, delete the dir where bundle was created
shutil.rmtree(bundle_dir)
return tarball_name
def get_revision_info(self, checkout_path):
return utils.local("(cd %s; hg log -l 1)" % checkout_path)
def get_revision_info(self, checkout_path):
return utils.local("svn info %s" % checkout_path)
def test_build_and_deploy(self):
"""Invoke the build and deploy task."""
zoomdb = StubZoomDB()
src_repo_type = "git"
src_url = "git://github.com/shimon/djangotutorial.git"
here = path.abspath(path.split(__file__)[0])
app_fixture = path.join(here, "../fixtures", "app")
zcfg_fixture = path.join(app_fixture, "zoombuild.cfg")
zoombuild_cfg_content = file(zcfg_fixture).read()
self.assertFalse(zoomdb.is_flushed)
self.assertEqual(len(zoomdb.get_all_bundles()), 0)
self.assertEqual(len(zoomdb.get_project_workers()), 0)
self.assertEqual(len(zoomdb.get_project_virtual_hosts()), 1)
zoomdb.test_vhosts = ["awesomesite.com", "foo.co.br"]
self.assertEqual(len(zoomdb.get_project_virtual_hosts()), 3)
deployed_addresses = build_and_deploy.build_and_deploy(
zoomdb,
self.app_id,
src_repo_type,
src_url,
zoombuild_cfg_content,
use_subtasks=False,
bundle_storage_engine=bundle_storage_local,
)
print "build_and_deploy returned: %r" % deployed_addresses
# # call the tasks module directly instead, so we get that tested too.
# # actually this doesn't work because of the decorator; doh!
# deployed_addresses = builder.build_and_deploy(zoomdb._job_id, zoomdb,
# {
# "app_id": self.app_id,
# "src_url": src_url,
# "zoombuild_cfg_content": zoombuild_cfg_content,
# })
zoombuild_cfg_output_filename = path.join(self.dir, self.app_id, "zoombuild.cfg")
self.assertTrue(path.isfile(zoombuild_cfg_output_filename))
self.assertEqual(file(zoombuild_cfg_output_filename).read(), zoombuild_cfg_content)
p = zoomdb.get_project()
for attr in ("db_host", "db_name", "db_username", "db_password"):
print "project.%s = %s" % (attr, getattr(p, attr))
self.assertTrue(getattr(p, attr))
self.assertTrue(zoomdb.is_flushed)
self.assertEqual(len(zoomdb.get_all_bundles()), 1)
self.assertEqual(len(zoomdb.get_project_workers()), 1)
site_nginx_conf_file = os.path.join(taskconfig.NGINX_SITES_ENABLED_DIR, self.app_id)
self.assertTrue(
os.path.isfile(site_nginx_conf_file),
"expected to find a config file in nginx's " "sites-enabled directory, but didn't",
)
# check the deployed app!
self.assertEqual(len(deployed_addresses), 1)
for (instance_id, node_name, host_ip, host_port) in deployed_addresses:
polls_url = "http://%s:%d/polls/" % (host_ip, host_port)
print "Testing Polls URL: %s" % polls_url
polls_src = urllib.urlopen(polls_url).read()
self.assertTrue("No polls are available." in polls_src)
# now check the nginx service
hosts = zoomdb.get_project_virtual_hosts()
# ensure each hostname works!
for host in hosts:
# GZIP DEBUG# import ipdb; ipdb.set_trace()
self.check_can_eventually_load_custom("127.0.0.1", "/polls/", host, "No polls are available.")
# for nginx to serve static files, the cust dir has to be
# world-read/executable. This should be the default on the
# proxy server ONLY.
os.chmod(self.dir, 0755)
image_src = self.check_can_eventually_load_custom("127.0.0.1", "/static/img/polls.jpg", hosts[0])
local_image_file = os.path.join(app_fixture, "src", "static", "polls.jpg")
self.assertEqual(image_src, open(local_image_file).read())
# try a collectstatic-handled file
collectstatic_src = self.check_can_eventually_load_custom(
"127.0.0.1", "/staticfiles/polls/Lineup.jpg", hosts[0]
)
local_collectstatic_file = os.path.join(app_fixture, "src", "polls", "static", "polls", "Lineup.jpg")
self.assertEqual(collectstatic_src, open(local_collectstatic_file).read())
# OK, now undeploy.
deploy.undeploy(zoomdb, self.app_id, bundle_ids=None, use_subtasks=False)
# check that URLs are no longer accessible
for (instance_id, node_name, host_ip, host_port) in deployed_addresses:
polls_url = "http://%s:%d/polls/" % (host_ip, host_port)
with self.assertRaises(IOError):
urllib.urlopen(polls_url).read()
# check that Nginx conf file is gone
self.assertFalse(os.path.isfile(site_nginx_conf_file), "expected nginx config file gone")
# check that supervisor files are gone
for fname in os.listdir(taskconfig.SUPERVISOR_APP_CONF_DIR):
self.assertFalse(
fname.startswith("%s." % self.app_id), "There is a lingering supervisor config " "file: %s" % fname
)
# check that DB still exists though
dblist = utils.local("psql -l -U nrweb | awk '{print $1}'")
self.assertTrue(self.app_id in dblist.splitlines())
def get_managepy_output(cmd):
# manage.py sends syntax to stdout; list of subcommands to stderr
return utils.local((cmd + " 2>&1") % (main_runner,))
def _kick_supervisor():
"""
Have supervisor re-read its config files and start any services
that aren't started yet.
"""
utils.local('echo -e "reread\nupdate" | sudo /usr/bin/supervisorctl')