def undeploy_from_appserver(zoomdb, app_id, bundle_id, appserver_instance_id, appserver_port, zero_undeploys_ok=False): my_hostname = utils.node_meta("name") my_instance_id = utils.node_meta("instance_id") if appserver_instance_id not in (my_hostname, my_instance_id, "localhost"): raise utils.InfrastructureException( "Incorrect appserver received undeploy_from_appserver task; " + "I am %s but the undeploy is for %s." % (my_hostname, appserver_instance_id)) bundle = zoomdb.get_bundle(bundle_id) num_stopped = stop_serving_bundle(app_id, bundle.bundle_name) if num_stopped == 0 and zero_undeploys_ok: zoomdb.log("Note: no matching bundles were running on %s." % my_hostname) elif num_stopped != 1: raise utils.InfrastructureException( ("Attempting to undeploy one bundle (app_id %s, bundle_id %s, " "bundle_name %s) from appserver %s:%d, but %d bundles were " "stopped.") % (app_id, bundle_id, bundle.bundle_name, appserver_instance_id, appserver_port, num_stopped)) app_dir, bundle_dir = utils.app_and_bundle_dirs(app_id, bundle.bundle_name) if os.path.isdir(bundle_dir): zoomdb.log("Removing old bundle from %s." % bundle_dir) utils.chown_to_me(bundle_dir) shutil.rmtree(bundle_dir)
def destroy(self): # first, attempt to send a polite shutdown request self.sock.send_pyobj(["shutdown"]) #print "CLOSING ZMQ WORKER SOCKET" self.sock.close() # give it a little time to exit for wait in range(20): if self.worker_proc.poll() is None: eventlet.sleep(0.1 * wait) else: break # didn't work? try SIGTERM and SIGKILL pid = self.worker_proc.pid for signum in (signal.SIGTERM, signal.SIGKILL): if self.worker_proc.poll() is None: utils.local_privileged(["kill_worker", signum, str(pid)]) for wait in range(20): eventlet.sleep(0.1 * wait) if self.worker_proc.poll() is not None: break # if it's still alive, CRAZY ERROR if self.worker_proc.poll() is None: print ("ERROR: cannot terminate worker %s, process %d" % (self.worker_dir, self.worker_proc.pid)) return utils.chown_to_me(self.worker_dir) shutil.rmtree(self.worker_dir)
def tearDown(self): # TODO: instead of just manually throwing away DB stuff, add a # destroy_project_data function that could be user-accessible in # case a user ever wants to throw away their DB and start over. try: database.drop_database(self.app_id) database.drop_user(self.app_id) except ProgrammingError: # probably indicates DB/user doesn't exist pass # chown cust dir to me so we can delete it utils.chown_to_me(self.dir) # delete any lingering supervisor conf files utils.local("rm -f %s/%s*" % (taskconfig.SUPERVISOR_APP_CONF_DIR, self.app_id))
def zip_and_upload_bundle(app_id, bundle_name, bundle_storage_engine=None, delete_after_upload=False): """ Task: Zip up the bundle and upload it to S3 :param custdir: Absolute path to the base customer directory :param app_id: A path such that ``os.path.join(custdir, app_id)`` is a valid directory. :param delete_after_upload: If true, delete the bundle directory after it is uploaded. """ bundle_storage_engine = get_bundle_storage_engine(bundle_storage_engine) archive_file_path = tempfile.mktemp(suffix=".tgz") app_dir = os.path.join(taskconfig.NR_CUSTOMER_DIR, app_id) bundle_dir = os.path.join(app_dir, bundle_name) try: current_dir = os.getcwd() # change ownership in app_dir before upload # because it was built inside a container utils.chown_to_me(bundle_dir) os.chdir(app_dir) try: p = subprocess.Popen( ["tar", "czf", archive_file_path, bundle_name], env=dict(PWD=app_dir), close_fds=True) os.waitpid(p.pid, 0) finally: os.chdir(current_dir) bundle_storage_engine.put(bundle_name + ".tgz", archive_file_path) if delete_after_upload: shutil.rmtree(bundle_dir) finally: if os.path.exists(archive_file_path): os.remove(archive_file_path) return bundle_name + ".tgz"
def test_chown_to_me(self): """ Test function to chown any file to the current user. """ test_file = path.join(self.dir, "test%d" % random.randint(9999, 19999)) utils.local("touch %s" % test_file) me = utils.local("whoami").strip() self.assertFileOwnedBy(test_file, me) utils.local_privileged(["project_chown", "root", test_file]) self.assertFileOwnedBy(test_file, "root") utils.chown_to_me(test_file) self.assertFileOwnedBy(test_file, me) utils.local("rm %s" % test_file)
def test_install_num_workers(self): """ Test installing with varying numbers of workers. """ bundle_dir = os.path.join(self.customer_directory, self.app_id, self.bundle_name) thisbundle = os.path.join(bundle_dir, "thisbundle.py") for num_workers in (1, 3, 17, 104): self.install_my_bundle(num_workers=num_workers) # for testing purposes only, let me read this file: utils.chown_to_me(thisbundle) thisbundle_content = [x.strip() for x in open(thisbundle).readlines()] self.assertIn('"--workers=%d",' % num_workers, thisbundle_content)
def install_app_bundle(app_id, bundle_name, appserver_name, dbinfo, bundle_storage_engine=bundle_storage, static_only=False, num_workers=1, remove_other_bundles=False): app_dir, bundle_dir = utils.app_and_bundle_dirs(app_id, bundle_name) if not os.path.exists(bundle_dir): utils.get_and_extract_bundle(bundle_name, app_dir, bundle_storage_engine) if remove_other_bundles: for fname in os.listdir(app_dir): if (fname.startswith("bundle_") and os.path.isdir(fname) and fname != bundle_name): shutil.rmtree(fname) if not static_only: utils.chown_to_me(bundle_dir) _write_deployment_config(os.path.join(bundle_dir, "thisbundle.py"), bundle_name, dbinfo, num_workers=num_workers) utils.local_privileged(["project_chown", app_id, bundle_dir])
def chown_to_me(self, path): utils.chown_to_me(path)
def create_test_bundle_in_local_storage(): """ Creates a bundle for testing, and uploads it to the local storage. Returns the bundle's tarball's name. This function is used from test_nginx and perhaps other tests that require a bundle -- modify with care. """ print "Making a bundle fixture for testing." bundle_name = "bundle_test_deploy_app_2011-fixture" here = os.path.abspath(os.path.split(__file__)[0]) fixture_dir = os.path.join(here, '../fixtures') app_name = "test_deploy_app" # force rename the bundle app_dir = os.path.join(taskconfig.NR_CUSTOMER_DIR, app_name) if os.path.isdir(app_dir): utils.chown_to_me(app_dir) shutil.rmtree(app_dir) shutil.copytree(os.path.join(fixture_dir, "app"), app_dir) # create a git repo in app_dir and do a commit so that we can get the # most recent commit during bundle_app. utils.local(";".join([ ("cd %s/src" % app_dir), "git init .", "git add __init__.py", "git commit -m 'initial test_deploy commit'"])) zcfg_path = os.path.join(app_dir, "zoombuild.cfg") zcfg_content = file(zcfg_path).read() # django_tarball = os.path.join(fixture_dir, 'Django-1.2.5.tar.gz') # we don't use pip_reqs any more # zcfg_content = zcfg_content.replace( # "pip_reqs: Django==1.2.5", "pip_reqs: %s" % django_tarball) faster_zcfg = file(zcfg_path, "w") faster_zcfg.write(zcfg_content) faster_zcfg.close() bundle_name, code_revision = bundle.bundle_app( app_name, force_bundle_name=bundle_name) bundle_dir = os.path.join(taskconfig.NR_CUSTOMER_DIR, app_name, bundle_name) tarball_name = bundle.zip_and_upload_bundle(app_name, bundle_name, bundle_storage_local) print "Created bundle fixture in %s" % tarball_name # after upload, delete the dir where bundle was created shutil.rmtree(bundle_dir) return tarball_name