예제 #1
0
    def setUp(self):
        self.zoomdb = StubZoomDB()

        self.work_dir = self.makeDir()
        self.opts = {"CO_DIR": self.work_dir}
        self.repo_dirs = {}

        for vcs in VCSes:
            self.repo_dirs[vcs] = self.makeDir()
            repo_archive = self.get_fixture_path("test-repo-%s.tar.gz" % vcs)
            utils.local("cd %s; tar xvzf %s" % (self.repo_dirs[vcs], repo_archive))
        super(RepoCheckoutTestCase, self).setUp()
    def tearDown(self):
        # TODO: instead of just manually throwing away DB stuff, add a
        # destroy_project_data function that could be user-accessible in
        # case a user ever wants to throw away their DB and start over.
        try:
            database.drop_database(self.app_id)
            database.drop_user(self.app_id)
        except ProgrammingError:  # probably indicates DB/user doesn't exist
            pass

        # chown cust dir to me so we can delete it
        utils.chown_to_me(self.dir)

        # delete any lingering supervisor conf files
        utils.local("rm -f %s/%s*" % (taskconfig.SUPERVISOR_APP_CONF_DIR, self.app_id))
예제 #3
0
 def test_run(self):
     """
     Tests our small wrapper around fabric's `run`
     """
     cmd = 'echo "foo"'
     out = utils.local(cmd, capture=True)
     self.assertEqual(out, 'foo')
예제 #4
0
    def test_safe_build(self):
        """
        Test that build actions run under the proper user.
        """
        # use the app fixture and create a repo
        here = path.abspath(path.split(__file__)[0])
        src = path.join(here, '../fixtures', self.project_name)
        dest = path.join(self.cust_dir, self.project_name)
        shutil.copytree(src, dest)
        utils.local('(cd %s; git init; git add -A; git commit -m test)' %
                    path.join(dest, 'src'))

        (bundle_name, code_revision) = bundle.bundle_app(self.project_name)
        bundle_dir = path.join(self.cust_dir, self.project_name, bundle_name)

        a_bundle_file = path.join(bundle_dir, "user-repo", "settings.py")
        self.assertFileOwnedBy(a_bundle_file, self.project_name)
예제 #5
0
    def test_chown_to_me(self):
        """
        Test function to chown any file to the current user.
        """
        test_file = path.join(self.dir, "test%d" % random.randint(9999, 19999))
        utils.local("touch %s" % test_file)
        me = utils.local("whoami").strip()
        self.assertFileOwnedBy(test_file, me)

        utils.local_privileged(["project_chown", "root", test_file])

        self.assertFileOwnedBy(test_file, "root")
        utils.chown_to_me(test_file)

        self.assertFileOwnedBy(test_file, me)

        utils.local("rm %s" % test_file)
예제 #6
0
    def _prep_build_test_bundle(self, **kwargs):
        self.patch(taskconfig, "NR_CUSTOMER_DIR", self.dir)

        # install_requirements = self.mocker.replace(
        #     "dz.tasklib.utils.install_requirements")
        # install_requirements(["Django==1.2.5"], MATCH(os.path.isdir),
        #                      logsuffix="-django")
        # install_requirements(ANY, MATCH(os.path.isdir))
        # self.mocker.replay()

        here = path.abspath(path.split(__file__)[0])
        src = path.join(here, '../fixtures', 'app')
        dest = path.join(self.dir, 'app')

        if not os.path.isdir(dest):
            shutil.copytree(src, dest)
            utils.local('(cd %s; git init; git add -A; git commit -m test)' %
                        path.join(dest, 'src'))

        return bundle.bundle_app('app', **kwargs)
예제 #7
0
def get_df():
    result = []

    for cmd in ("df -h", "df -i"):
        output = utils.local(cmd)
        for i, line in enumerate(output.splitlines()):
            if i == 0 or line.startswith("/dev"):
                result.append(line)
        result.append("")

    return "\n".join(result)
예제 #8
0
    def test_change_of_repo_url(self):
        """
        Test checking out from a changed repo URL.
        """
        repos = []
        for repo_num in range(2):
            repo_dir = self.makeDir()
            utils.local("git init %s" % repo_dir)
            utils.local("touch %s/file_%d" % (repo_dir, repo_num))
            utils.local("cd %s; git add file_%d; git commit -m 'unittest'" % (repo_dir, repo_num))
            repos.append(dict(num=repo_num, path=repo_dir))

        co_dir = self.makeDir()
        opts = {"CO_DIR": co_dir, "SRC_URL": repos[0]["path"]}
        common_steps.checkout_code(self.zoomdb, opts)
        self.assertTrue(os.path.isfile("%s/file_%d" % (co_dir, 0)))
        self.assertFalse(os.path.isfile("%s/file_%d" % (co_dir, 1)))

        # now change the repo source url
        opts["SRC_URL"] = repos[1]["path"]
        common_steps.checkout_code(self.zoomdb, opts)
        self.assertTrue(os.path.isfile("%s/file_%d" % (co_dir, 1)), "New repo should be in place.")
        self.assertFalse(os.path.isfile("%s/file_%d" % (co_dir, 0)), "Old repo shouldn't still be in place.")

        # and run again to ensure we now do a pull
        self.zoomdb.logs = []
        self.assertFalse(("Running git pull...", "i") in self.zoomdb.logs)
        common_steps.checkout_code(self.zoomdb, opts)
        self.assertTrue(("Running git pull...", "i") in self.zoomdb.logs)
예제 #9
0
    def test_check_repo(self):
        """
        Check repo and make guesses!
        """
        self.patch(taskconfig, "NR_CUSTOMER_DIR", self.dir)

        zoomdb = StubZoomDB()
        app_id = "p001"
        here = path.abspath(path.split(__file__)[0])
        tarball_path = path.join(here, '../fixtures', 'repo.tar.gz')
        tmp_repo_dir = self.makeDir()
        utils.local("tar xvzf %s -C %s" % (tarball_path, tmp_repo_dir))
        src_url = path.join(tmp_repo_dir, "repo")
        check_repo.check_repo(zoomdb, app_id, "git", src_url)

        src_dir = path.join(self.dir, app_id, "src")

        self.assertTrue(path.isdir(src_dir),
                        "src dir does not exist")

        self.assertTrue(path.isdir(path.join(src_dir,
                                             ".git")),
                        "no .git dir inside src dir")

        self.assertEqual(len(zoomdb.config_guesses), 5)

        def has_one_guess(field, value):
            matches = [g for g in zoomdb.config_guesses
                       if g["field"] == field and g["value"] == value]

            return len(matches) == 1

        self.assertTrue(has_one_guess("additional_python_path_dirs",
                                      "version2\nversion2/voo2do"))

        self.assertTrue(has_one_guess(
            "requirements_file_names",
            "\n".join(["requirements.txt",
                       "version2/voo2do/requirements/base.txt"])))
예제 #10
0
    def test_node_metadata(self):
        instance_id = utils.node_meta("instance_id")
        node_name = utils.node_meta("name")
        node_role = utils.node_meta("role")

        self.assertEqual(instance_id, "localhost")
        self.assertEqual(node_name, utils.local("hostname").strip())
        self.assertEqual(node_role, "localhost")

        here = path.abspath(path.split(__file__)[0])
        test_fixture_meta = path.join(here, '../fixtures', 'node_meta')
        self.patch(taskconfig, "NODE_META_DATA_DIR", test_fixture_meta)

        self.assertEqual(utils.node_meta("instance_id"), "i-12345")
        self.assertEqual(utils.node_meta("name"), "myname")
        self.assertEqual(utils.node_meta("role"), "myrole")
예제 #11
0
    def _maxdisk():
        maxdisk = dict(pct=0)

        # parse df output to find max disk use indicator
        header_line = ""
        for cmd, usetype in (("df -h", "space"),
                             ("df -i", "inode")):
            output = utils.local(cmd)
            for i, line in enumerate(output.splitlines()):
                if i == 0:
                    header_line = line
                if line.startswith("/dev") or line.startswith(".host"): # added for VMWare compatibility
                    pct_use = line.split()[-2]
                    if pct_use.endswith("%"):
                        pct = int(pct_use[:-1])
                        if pct > maxdisk["pct"]:
                            maxdisk = dict(pct=pct,
                                           type=usetype,
                                           detail="\n".join((header_line,
                                                             line)))
        return maxdisk
예제 #12
0
def create_test_bundle_in_local_storage():
    """
    Creates a bundle for testing, and uploads it to the local storage.
    Returns the bundle's tarball's name.

    This function is used from test_nginx and perhaps other tests that
    require a bundle -- modify with care.
    """
    print "Making a bundle fixture for testing."
    bundle_name = "bundle_test_deploy_app_2011-fixture"

    here = os.path.abspath(os.path.split(__file__)[0])
    fixture_dir = os.path.join(here, '../fixtures')
    app_name = "test_deploy_app"

    # force rename the bundle
    app_dir = os.path.join(taskconfig.NR_CUSTOMER_DIR, app_name)

    if os.path.isdir(app_dir):
        utils.chown_to_me(app_dir)
        shutil.rmtree(app_dir)

    shutil.copytree(os.path.join(fixture_dir, "app"), app_dir)

    # create a git repo in app_dir and do a commit so that we can get the
    # most recent commit during bundle_app.
    utils.local(";".join([
        ("cd %s/src" % app_dir),
        "git init .",
        "git add __init__.py",
        "git commit -m 'initial test_deploy commit'"]))

    zcfg_path = os.path.join(app_dir, "zoombuild.cfg")
    zcfg_content = file(zcfg_path).read()
    # django_tarball = os.path.join(fixture_dir, 'Django-1.2.5.tar.gz')
    # we don't use pip_reqs any more
    # zcfg_content = zcfg_content.replace(
    #     "pip_reqs: Django==1.2.5", "pip_reqs: %s" % django_tarball)

    faster_zcfg = file(zcfg_path, "w")
    faster_zcfg.write(zcfg_content)
    faster_zcfg.close()

    bundle_name, code_revision = bundle.bundle_app(
        app_name,
        force_bundle_name=bundle_name)

    bundle_dir = os.path.join(taskconfig.NR_CUSTOMER_DIR,
                              app_name,
                              bundle_name)

    tarball_name = bundle.zip_and_upload_bundle(app_name,
                                                bundle_name,
                                                bundle_storage_local)

    print "Created bundle fixture in %s" % tarball_name

    # after upload, delete the dir where bundle was created
    shutil.rmtree(bundle_dir)

    return tarball_name
예제 #13
0
 def get_revision_info(self, checkout_path):
     return utils.local("(cd %s; hg log -l 1)" % checkout_path)
예제 #14
0
 def get_revision_info(self, checkout_path):
     return utils.local("svn info %s" % checkout_path)
예제 #15
0
    def test_build_and_deploy(self):
        """Invoke the build and deploy task."""
        zoomdb = StubZoomDB()

        src_repo_type = "git"
        src_url = "git://github.com/shimon/djangotutorial.git"

        here = path.abspath(path.split(__file__)[0])
        app_fixture = path.join(here, "../fixtures", "app")
        zcfg_fixture = path.join(app_fixture, "zoombuild.cfg")

        zoombuild_cfg_content = file(zcfg_fixture).read()

        self.assertFalse(zoomdb.is_flushed)
        self.assertEqual(len(zoomdb.get_all_bundles()), 0)
        self.assertEqual(len(zoomdb.get_project_workers()), 0)

        self.assertEqual(len(zoomdb.get_project_virtual_hosts()), 1)
        zoomdb.test_vhosts = ["awesomesite.com", "foo.co.br"]
        self.assertEqual(len(zoomdb.get_project_virtual_hosts()), 3)

        deployed_addresses = build_and_deploy.build_and_deploy(
            zoomdb,
            self.app_id,
            src_repo_type,
            src_url,
            zoombuild_cfg_content,
            use_subtasks=False,
            bundle_storage_engine=bundle_storage_local,
        )

        print "build_and_deploy returned: %r" % deployed_addresses

        # # call the tasks module directly instead, so we get that tested too.
        # # actually this doesn't work because of the decorator; doh!
        # deployed_addresses = builder.build_and_deploy(zoomdb._job_id, zoomdb,
        #         {
        #         "app_id": self.app_id,
        #         "src_url": src_url,
        #         "zoombuild_cfg_content": zoombuild_cfg_content,
        #         })

        zoombuild_cfg_output_filename = path.join(self.dir, self.app_id, "zoombuild.cfg")
        self.assertTrue(path.isfile(zoombuild_cfg_output_filename))
        self.assertEqual(file(zoombuild_cfg_output_filename).read(), zoombuild_cfg_content)

        p = zoomdb.get_project()

        for attr in ("db_host", "db_name", "db_username", "db_password"):
            print "project.%s = %s" % (attr, getattr(p, attr))
            self.assertTrue(getattr(p, attr))

        self.assertTrue(zoomdb.is_flushed)
        self.assertEqual(len(zoomdb.get_all_bundles()), 1)
        self.assertEqual(len(zoomdb.get_project_workers()), 1)
        site_nginx_conf_file = os.path.join(taskconfig.NGINX_SITES_ENABLED_DIR, self.app_id)
        self.assertTrue(
            os.path.isfile(site_nginx_conf_file),
            "expected to find a config file in nginx's " "sites-enabled directory, but didn't",
        )

        # check the deployed app!
        self.assertEqual(len(deployed_addresses), 1)

        for (instance_id, node_name, host_ip, host_port) in deployed_addresses:
            polls_url = "http://%s:%d/polls/" % (host_ip, host_port)
            print "Testing Polls URL: %s" % polls_url
            polls_src = urllib.urlopen(polls_url).read()
            self.assertTrue("No polls are available." in polls_src)

        # now check the nginx service
        hosts = zoomdb.get_project_virtual_hosts()

        # ensure each hostname works!
        for host in hosts:
            # GZIP DEBUG# import ipdb; ipdb.set_trace()

            self.check_can_eventually_load_custom("127.0.0.1", "/polls/", host, "No polls are available.")

        # for nginx to serve static files, the cust dir has to be
        # world-read/executable. This should be the default on the
        # proxy server ONLY.
        os.chmod(self.dir, 0755)

        image_src = self.check_can_eventually_load_custom("127.0.0.1", "/static/img/polls.jpg", hosts[0])
        local_image_file = os.path.join(app_fixture, "src", "static", "polls.jpg")
        self.assertEqual(image_src, open(local_image_file).read())

        # try a collectstatic-handled file
        collectstatic_src = self.check_can_eventually_load_custom(
            "127.0.0.1", "/staticfiles/polls/Lineup.jpg", hosts[0]
        )
        local_collectstatic_file = os.path.join(app_fixture, "src", "polls", "static", "polls", "Lineup.jpg")
        self.assertEqual(collectstatic_src, open(local_collectstatic_file).read())

        # OK, now undeploy.
        deploy.undeploy(zoomdb, self.app_id, bundle_ids=None, use_subtasks=False)

        # check that URLs are no longer accessible
        for (instance_id, node_name, host_ip, host_port) in deployed_addresses:
            polls_url = "http://%s:%d/polls/" % (host_ip, host_port)
            with self.assertRaises(IOError):
                urllib.urlopen(polls_url).read()

        # check that Nginx conf file is gone
        self.assertFalse(os.path.isfile(site_nginx_conf_file), "expected nginx config file gone")

        # check that supervisor files are gone
        for fname in os.listdir(taskconfig.SUPERVISOR_APP_CONF_DIR):
            self.assertFalse(
                fname.startswith("%s." % self.app_id), "There is a lingering supervisor config " "file: %s" % fname
            )

        # check that DB still exists though
        dblist = utils.local("psql -l -U nrweb | awk '{print $1}'")
        self.assertTrue(self.app_id in dblist.splitlines())
예제 #16
0
 def get_managepy_output(cmd):
     # manage.py sends syntax to stdout; list of subcommands to stderr
     return utils.local((cmd + " 2>&1") % (main_runner,))
예제 #17
0
def _kick_supervisor():
    """
    Have supervisor re-read its config files and start any services
    that aren't started yet.
    """
    utils.local('echo -e "reread\nupdate" | sudo /usr/bin/supervisorctl')