def register(mgt_info, sudo_password=None, upgrade=False): mgt_info = mgt_info.with_only_pidfile_services() # first copy the monitrc.template to deployment_home monit_substitutions = { 'monitinterval' : 120, 'monitstartdelay' : 60, 'monitlogfile' : os.path.join(mgt_info.deployment_home, 'log/monit.log'), 'monitemailalertsto' : '*****@*****.**', 'monithost' : 'localhost', 'monitadmin' : 'admin', 'monitadminpassword' : 'engage_monit' } monitrc_template_file = os.path.join(os.path.dirname(__file__), 'data/monitrc.template') monitrc_file = os.path.join(mgt_info.deployment_home, 'monitrc') iufile.instantiate_template_file(monitrc_template_file, monitrc_file, monit_substitutions) with open(monitrc_file, 'a') as f: for svc in mgt_info.services: monitor = generate_monitrc(mgt_info.svcctl_exe, svc) f.write(monitor) os.chmod(monitrc_file, 0600) # now start monit iuprocess.run_and_log_program( [monit_exe, '-c', monitrc_file], {}, logger) print 'monit started'
def install(self, package): # if this is a linux system, we need to make sure that uuid/uuid.h is present if self.config.input_ports.host.os_type == "linux": if not os.path.exists("/usr/include/uuid/uuid.h"): aptget.apt_get_install(["uuid-dev"], self._get_sudo_password()) base_name = os.path.basename(self.config.home_path) parent_dir = os.path.dirname(self.config.home_path) extracted_dir = package.extract(parent_dir) assert extracted_dir == base_name # configure with prefix self.confg.home_path, make and make install logger.info("Building gearman daemon") rc = iuprocess.run_and_log_program( ["/usr/bin/make", "all"], {"PATH": "/usr/bin:/bin:/usr/sbin"}, logger, cwd=self.config.home_path ) if rc != 0: raise UserError(errors[ERR_GEARMAN_BUILD_FAILED]) gearman_pyclient_dir = os.path.join(os.path.join(self.config.home_path, "src"), "gearman-1.5.0") logger.info("Installing gearman python client") # TODO: this should really be done using easy_install. Even better, # we should upgrade to the latest version of Gearman and install the # version of the client on PyPi. rc = iuprocess.run_and_log_program( [self.config.input_ports.python.home, "setup.py", "install"], {}, logger, cwd=gearman_pyclient_dir ) if rc != 0: raise UserError(errors[ERR_GEARMAN_PY_SETUP_FAILED]) # instantiate the startup/shutdown script for gearmand script_tmpl_path = iufile.get_data_file_path(__file__, "gearmand.sh.tmpl") iufile.instantiate_template_file( script_tmpl_path, self.config.gearman_admin_script, {"gearman_home": self.config.home_path, "log_dir": self.config.log_dir, "pid_file": self.config.pid_file}, logger=logger, ) # check that everything is now in place self.validate_post_install()
def install(self, package, upgrade=False): # verify that the django-admin.py utility was installed (as a part of the # Django package) if not os.path.exists(self.config.django_admin_script): raise UserError(errors[ERR_NO_DJANGO_ADMIN_SCRIPT], msg_args={"file":self.config.django_admin_script}) (file_layout, django_config) = self._extract_package(package) # if present, install any pip requirements requirements_file_path = os.path.join(file_layout.get_app_dir_path(), "requirements.txt") if os.path.exists(requirements_file_path): assert hasattr(self.config.input_ports, "pip") and hasattr(self.config.input_ports.pip, "pipbin"), \ "%s has a requirements file, but resource is not configured for installing dependencies via pip" % self.config.package_name self._install_pip_requirements(requirements_file_path) else: logger.debug("Requirements file '%s' not found, skipping requirements step" % requirements_file_path) if file_layout.has_static_url_mapping(): (static_url, static_root) = file_layout.get_static_url_mapping() else: static_url = static_root = None # Instantiate the settings file substitutions = { gs.INSTALL_PATH: self.config.home_path, gs.HOSTNAME: self.config.config_port.websvr_hostname, gs.PRIVATE_IP_ADDRESS: (lambda x: "'%s'" % x if x else "None")(self.config.input_ports.host.private_ip), gs.PORT: self.config.config_port.websvr_port, gs.SECRET_KEY: gen_password(50, chars=string.digits+string.letters+"!#%&()*+,-./:;<=>?@[]^_`{|}~"), gs.EMAIL_HOST:self.config.config_port.email_host, gs.EMAIL_HOST_USER:self.config.config_port.email_host_user, gs.EMAIL_FROM:self.config.config_port.email_from, gs.EMAIL_HOST_PASSWORD:self.config.config_port.email_password, gs.EMAIL_PORT:self.config.config_port.email_port, gs.ADMIN_NAME:self.config.config_port.admin_name, gs.ADMIN_EMAIL:self.config.config_port.admin_email, gs.TIME_ZONE:self.config.config_port.time_zone, gs.LOG_DIRECTORY:self.config.config_port.log_directory, gs.DATABASE_ENGINE:self.config.input_ports.django_db.ENGINE, gs.DATABASE_NAME:self.config.input_ports.django_db.NAME, gs.DATABASE_USER:self.config.input_ports.django_db.USER, gs.DATABASE_PASSWORD: self.install_context.password_repository.get_value(self.config.input_ports.django_db.PASSWORD) \ if self.config.input_ports.django_db.PASSWORD else '', gs.DATABASE_HOST:self.config.input_ports.django_db.HOST, gs.DATABASE_PORT:self.config.input_ports.django_db.PORT, gs.CACHE_BACKEND:'memcached', gs.CACHE_LOCATION:'localhost:11211', gs.CELERY_CONFIG_BROKER_HOST:'None', gs.CELERY_CONFIG_BROKER_PORT:0, gs.CELERY_CONFIG_BROKER_USER:'******', gs.CELERY_CONFIG_BROKER_PASSWORD:'******', gs.CELERY_CONFIG_BROKER_VHOST:'None', gs.CELERY_CONFIG_CELERY_RESULT_BACKEND:'amqp', gs.REDIS_HOST:"localhost", gs.REDIS_PORT:6379, gs.STATIC_ROOT:static_root } # we need to check that the cache input port is present (it isn't if this is # the gfwebsite resource). if hasattr(self.config.input_ports, "ocache") and self.config.input_ports.ocache.provider == 'memcached': substitutions[gs.CACHE_BACKEND] = 'memcached' substitutions[gs.CACHE_LOCATION] = '%s:%s' % (self.config.input_ports.ocache.host, self.config.input_ports.ocache.port) else: # Dummy substitutions[gs.CACHE_BACKEND] = 'django.core.cache.backends.dummy.DummyCache' substitutions[gs.CACHE_LOCATION] = '' # set up with django-celery if hasattr(self.config.input_ports, "celery") and self.config.input_ports.celery.provider == "celery": # celery configuration logger.info('Adding celery config!') substitutions[gs.CELERY_CONFIG_BROKER_HOST] = self.config.input_ports.celery.BROKER_HOST substitutions[gs.CELERY_CONFIG_BROKER_PORT] = self.config.input_ports.celery.BROKER_PORT substitutions[gs.CELERY_CONFIG_BROKER_USER] = self.config.input_ports.celery.BROKER_USER substitutions[gs.CELERY_CONFIG_BROKER_PASSWORD] = self.config.input_ports.celery.BROKER_PASSWORD substitutions[gs.CELERY_CONFIG_BROKER_VHOST] = self.config.input_ports.celery.BROKER_VHOST substitutions[gs.CELERY_CONFIG_CELERY_RESULT_BACKEND] = 'amqp' else: logger.info('No input port for celery') gs.generate_settings_file(file_layout.get_app_dir_path(), file_layout.get_app_settings_module(), django_config.components, properties=substitutions) # Validate the settings file. We run the validation script as an external program, have the results # written to an external file, and then parse that file to get detailed results. validate_results = \ engage_django_sdk.packager.run_installed_tests_as_subprocess(file_layout.get_app_dir_path(), file_layout.get_app_settings_module(), python_exe_path=self.config.input_ports.python.home, use_logger=logger) rc = validate_results.get_return_code() if validate_results.run_was_successful(): logger.info("Django settings file validation successful") else: raise UserError(errors[ERR_DJANGO_VALIDATE_FAILED], msg_args={"rc":validate_results.get_return_code_desc()}, developer_msg=validate_results.format_messages()) # instantiate the admin script (startup/shutdown/status) from a template substitutions = { "app_short_name": self.config.app_short_name, "install_dir": self.config.home_path, "settings_file_dir": file_layout.get_settings_file_directory(), "python_path": file_layout.get_python_path(), "django_settings_module": file_layout.get_deployed_settings_module(), "python_bin_dir": self.config.python_bin_dir, "websvr_hostname": self.config.input_ports.webserver_config.listen_host, "port": self.config.input_ports.webserver_config.listen_port, "log_directory": self.config.config_port.log_directory } admin_script_tmpl_path = iufile.get_data_file_path(__file__, "django.sh.tmpl") iufile.instantiate_template_file(admin_script_tmpl_path, self.config.app_admin_script, substitutions, logger=logger) os.chmod(self.config.app_admin_script, 0755) # modify manage.py to point at the our local python version and to have the right PYTHONPATH manage_py_file = os.path.join(file_layout.get_settings_file_directory(), "manage.py") if os.path.exists(manage_py_file): logger.debug("Updating manage.py file with correct python path and python executable") # default first line is "#!/usr/bin/env python", but user could # replace this with a different path shbang_pattern = '^\\#\\!.+$' # python.home is the python executable # home_path is where any local modules should go shbang_replacement = "#!%s\nimport sys\nsys.path.extend(%s)" % \ (self.config.input_ports.python.home, file_layout.get_python_path().split(":").__repr__()) import_pattern = re.escape("import settings") import_replacement = "import deployed_settings" call_pattern = re.escape("execute_manager(settings)") call_replacement = "execute_manager(deployed_settings)" cnt = iufile.subst_in_file(manage_py_file, [(shbang_pattern, shbang_replacement), (import_pattern, import_replacement), (call_pattern, call_replacement)]) if cnt != 3: logger.warning("Substitutions for %s did not match expected number (got %d, expecting 3). Your manage.py script might not be usable without hand-modifying it." % (manage_py_file, cnt)) else: logger.debug("No manage.py file, skipping shbang replacement") # make the log directory if not os.path.exists(self.config.config_port.log_directory): logger.action("mkdir -p %s" % self.config.config_port.log_directory) os.makedirs(self.config.config_port.log_directory) # setup the database if self.config.input_ports.django_db.ENGINE=='django.db.backends.sqlite3': database_dir = os.path.dirname(self.config.input_ports.django_db.NAME) if not os.path.exists(database_dir): logger.action("mkdir -p %s" % database_dir) os.makedirs(database_dir) file_layout.run_admin_command("syncdb", ["--noinput"]) if "south" in django_config.installed_apps: # We use the presence of South in INSTALLED_APPS to indicate that # we should run migrations. If South isn't there, migrate will fail. if upgrade: file_layout.run_admin_command("migrate", ["--no-initial-data"]) else: file_layout.run_admin_command("migrate", []) # if we have fixtures and this is the initial install, load them if len(django_config.fixtures)>0 and not upgrade: file_layout.run_admin_command("loaddata", django_config.fixtures) # gather static files, if requested if static_url and \ ('django.contrib.staticfiles' in django_config.installed_apps): if not os.path.exists(static_root): logger.debug("mkdir -p %s" % static_root) os.makedirs(static_root) file_layout.run_admin_command("collectstatic", ["--noinput"]) # See if this application includes the admin password parameter. # If so, we set the password by running a script. try: app_pw = self.config.config_port.app_admin_password except: app_pw = None if app_pw: self._run_python_script("django_set_password.py", ["-c", "-e", self.config.config_port.admin_email, "-s"], file_layout, input=self.config.config_port.app_admin_password) # setup the webserver hooks if self.config.input_ports.webserver_config.webserver_type == "apache": self._wsgi_setup(file_layout) self.ctx.r(apache_utils.restart_apache, self.ctx.props.input_ports.webserver_config)
def _wsgi_setup(self, file_layout): # setup the wsgi configuration files # TODO: eventually, this should go into a separate driver def get_url_for_mapping(full_url): path = urlparse.urlparse(full_url)[2] if not (path[len(path)-1]=='/'): path = path + "/" return path def get_path_for_mapping(path): if not path[len(path)-1]=='/': path = path + "/" return path def get_apache_directory_decl(path): return "<Directory %s>\nOrder deny,allow\nAllow from all\n</Directory>\n" % path wsgi_deploy_dir_path = os.path.join(self.config.home_path, "wsgi_deploy") if not os.path.exists(wsgi_deploy_dir_path): logger.action("mkdir -p %s" % wsgi_deploy_dir_path) os.makedirs(wsgi_deploy_dir_path) media_alias_directives = "" media_directory_directives = "" if file_layout.has_media_url_mapping(): (media_url, media_root) = file_layout.get_media_url_mapping() media_alias_directives += "Alias %s %s\n" % (get_url_for_mapping(media_url), get_path_for_mapping(media_root)) media_directory_directives += get_apache_directory_decl(media_root) if file_layout.has_static_url_mapping(): (static_url, static_root) = file_layout.get_static_url_mapping() media_alias_directives += "Alias %s %s\n" % (get_url_for_mapping(static_url), get_path_for_mapping(static_root)) media_directory_directives += get_apache_directory_decl(static_root) error_log = os.path.join(self.config.input_ports.webserver_config.log_dir, "error.log") access_log = os.path.join(self.config.input_ports.webserver_config.log_dir, "access.log") wsgi_substitutions = { "install_path": self.config.home_path, "settings_file_directory": file_layout.get_settings_file_directory(), "python_path": file_layout.get_python_path(), "python_exe": self.config.input_ports.python.home, "genforma_home": self.config.input_ports.host.genforma_home, "django_settings_module": file_layout.get_deployed_settings_module(), "app_short_name": self.config.app_short_name, "file_owner_group": self.config.input_ports.webserver_config.file_owner_group, "additional_config_dir":self.config.input_ports.webserver_config.additional_config_dir, "media_alias_directives":media_alias_directives, "media_directory_directives":media_directory_directives, "apache_error_log": error_log, "apache_access_log": access_log, "apache_log_dir":self.config.input_ports.webserver_config.log_dir } target_wsgi_file = os.path.join(wsgi_deploy_dir_path, "apache_wsgi.wsgi") wsgi_file_template = iufile.get_data_file_path(__file__,"apache_wsgi.wsgi") iufile.instantiate_template_file(wsgi_file_template, target_wsgi_file, wsgi_substitutions, logger=logger) # the app can override the apache config file by providing one at # <settings_file_dir>/wsgi_deploy/<app_short_name>_apache_wsgi.conf. app_specific_apache_conf_file = os.path.join(os.path.join(file_layout.get_settings_file_directory(), "wsgi_deploy"), "%s_apache_wsgi.conf" % self.config.app_short_name) if os.path.exists(app_specific_apache_conf_file): conf_file_template = app_specific_apache_conf_file else: conf_file_template = iufile.get_data_file_path(__file__,"django_apache_wsgi.conf") target_conf_file = os.path.join(wsgi_deploy_dir_path, "%s_apache_wsgi.conf" % self.config.app_short_name) iufile.instantiate_template_file(conf_file_template, target_conf_file, wsgi_substitutions, logger=logger) self.ctx.r(sudo_add_config_file_line, self.config.mod_wsgi_config_file, "WSGIPythonHome %s/python" % self.config.input_ports.host.genforma_home) self.ctx.r(sudo_ensure_shared_perms, self.config.input_ports.webserver_config.log_dir, self.config.input_ports.webserver_config.file_owner_group, writable_to_group=True) self.ctx.r(sudo_ensure_shared_perms, self.config.input_ports.host.genforma_home, self.config.input_ports.webserver_config.file_owner_group, writable_to_group=True) self.ctx.r(apache_utils.add_apache_config_file, target_conf_file, self.config.input_ports.webserver_config)