def register(mgt_info, sudo_password=None, upgrade=False):
mgt_info = mgt_info.with_only_pidfile_services()
# first copy the monitrc.template to deployment_home
monit_substitutions = {
'monitinterval' : 120,
'monitstartdelay' : 60,
'monitlogfile' : os.path.join(mgt_info.deployment_home, 'log/monit.log'),
'monitemailalertsto' : '*****@*****.**',
'monithost' : 'localhost',
'monitadmin' : 'admin',
'monitadminpassword' : 'engage_monit'
}
monitrc_template_file = os.path.join(os.path.dirname(__file__), 'data/monitrc.template')
monitrc_file = os.path.join(mgt_info.deployment_home, 'monitrc')
iufile.instantiate_template_file(monitrc_template_file, monitrc_file, monit_substitutions)
with open(monitrc_file, 'a') as f:
for svc in mgt_info.services:
monitor = generate_monitrc(mgt_info.svcctl_exe, svc)
f.write(monitor)
os.chmod(monitrc_file, 0600)
# now start monit
iuprocess.run_and_log_program(
[monit_exe,
'-c', monitrc_file],
{}, logger)
print 'monit started'
def install(self, package):
# if this is a linux system, we need to make sure that uuid/uuid.h is present
if self.config.input_ports.host.os_type == "linux":
if not os.path.exists("/usr/include/uuid/uuid.h"):
aptget.apt_get_install(["uuid-dev"], self._get_sudo_password())
base_name = os.path.basename(self.config.home_path)
parent_dir = os.path.dirname(self.config.home_path)
extracted_dir = package.extract(parent_dir)
assert extracted_dir == base_name
# configure with prefix self.confg.home_path, make and make install
logger.info("Building gearman daemon")
rc = iuprocess.run_and_log_program(
["/usr/bin/make", "all"], {"PATH": "/usr/bin:/bin:/usr/sbin"}, logger, cwd=self.config.home_path
)
if rc != 0:
raise UserError(errors[ERR_GEARMAN_BUILD_FAILED])
gearman_pyclient_dir = os.path.join(os.path.join(self.config.home_path, "src"), "gearman-1.5.0")
logger.info("Installing gearman python client")
# TODO: this should really be done using easy_install. Even better,
# we should upgrade to the latest version of Gearman and install the
# version of the client on PyPi.
rc = iuprocess.run_and_log_program(
[self.config.input_ports.python.home, "setup.py", "install"], {}, logger, cwd=gearman_pyclient_dir
)
if rc != 0:
raise UserError(errors[ERR_GEARMAN_PY_SETUP_FAILED])
# instantiate the startup/shutdown script for gearmand
script_tmpl_path = iufile.get_data_file_path(__file__, "gearmand.sh.tmpl")
iufile.instantiate_template_file(
script_tmpl_path,
self.config.gearman_admin_script,
{"gearman_home": self.config.home_path, "log_dir": self.config.log_dir, "pid_file": self.config.pid_file},
logger=logger,
)
# check that everything is now in place
self.validate_post_install()
def install(self, package, upgrade=False):
# verify that the django-admin.py utility was installed (as a part of the
# Django package)
if not os.path.exists(self.config.django_admin_script):
raise UserError(errors[ERR_NO_DJANGO_ADMIN_SCRIPT],
msg_args={"file":self.config.django_admin_script})
(file_layout, django_config) = self._extract_package(package)
# if present, install any pip requirements
requirements_file_path = os.path.join(file_layout.get_app_dir_path(), "requirements.txt")
if os.path.exists(requirements_file_path):
assert hasattr(self.config.input_ports, "pip") and hasattr(self.config.input_ports.pip, "pipbin"), \
"%s has a requirements file, but resource is not configured for installing dependencies via pip" % self.config.package_name
self._install_pip_requirements(requirements_file_path)
else:
logger.debug("Requirements file '%s' not found, skipping requirements step" % requirements_file_path)
if file_layout.has_static_url_mapping():
(static_url, static_root) = file_layout.get_static_url_mapping()
else:
static_url = static_root = None
# Instantiate the settings file
substitutions = {
gs.INSTALL_PATH: self.config.home_path,
gs.HOSTNAME: self.config.config_port.websvr_hostname,
gs.PRIVATE_IP_ADDRESS: (lambda x: "'%s'" % x if x else "None")(self.config.input_ports.host.private_ip),
gs.PORT: self.config.config_port.websvr_port,
gs.SECRET_KEY: gen_password(50, chars=string.digits+string.letters+"!#%&()*+,-./:;<=>?@[]^_`{|}~"),
gs.EMAIL_HOST:self.config.config_port.email_host,
gs.EMAIL_HOST_USER:self.config.config_port.email_host_user,
gs.EMAIL_FROM:self.config.config_port.email_from,
gs.EMAIL_HOST_PASSWORD:self.config.config_port.email_password,
gs.EMAIL_PORT:self.config.config_port.email_port,
gs.ADMIN_NAME:self.config.config_port.admin_name,
gs.ADMIN_EMAIL:self.config.config_port.admin_email,
gs.TIME_ZONE:self.config.config_port.time_zone,
gs.LOG_DIRECTORY:self.config.config_port.log_directory,
gs.DATABASE_ENGINE:self.config.input_ports.django_db.ENGINE,
gs.DATABASE_NAME:self.config.input_ports.django_db.NAME,
gs.DATABASE_USER:self.config.input_ports.django_db.USER,
gs.DATABASE_PASSWORD:
self.install_context.password_repository.get_value(self.config.input_ports.django_db.PASSWORD) \
if self.config.input_ports.django_db.PASSWORD else '',
gs.DATABASE_HOST:self.config.input_ports.django_db.HOST,
gs.DATABASE_PORT:self.config.input_ports.django_db.PORT,
gs.CACHE_BACKEND:'memcached',
gs.CACHE_LOCATION:'localhost:11211',
gs.CELERY_CONFIG_BROKER_HOST:'None',
gs.CELERY_CONFIG_BROKER_PORT:0,
gs.CELERY_CONFIG_BROKER_USER:'******',
gs.CELERY_CONFIG_BROKER_PASSWORD:'******',
gs.CELERY_CONFIG_BROKER_VHOST:'None',
gs.CELERY_CONFIG_CELERY_RESULT_BACKEND:'amqp',
gs.REDIS_HOST:"localhost",
gs.REDIS_PORT:6379,
gs.STATIC_ROOT:static_root
}
# we need to check that the cache input port is present (it isn't if this is
# the gfwebsite resource).
if hasattr(self.config.input_ports, "ocache") and self.config.input_ports.ocache.provider == 'memcached':
substitutions[gs.CACHE_BACKEND] = 'memcached'
substitutions[gs.CACHE_LOCATION] = '%s:%s' % (self.config.input_ports.ocache.host, self.config.input_ports.ocache.port)
else: # Dummy
substitutions[gs.CACHE_BACKEND] = 'django.core.cache.backends.dummy.DummyCache'
substitutions[gs.CACHE_LOCATION] = ''
# set up with django-celery
if hasattr(self.config.input_ports, "celery") and self.config.input_ports.celery.provider == "celery":
# celery configuration
logger.info('Adding celery config!')
substitutions[gs.CELERY_CONFIG_BROKER_HOST] = self.config.input_ports.celery.BROKER_HOST
substitutions[gs.CELERY_CONFIG_BROKER_PORT] = self.config.input_ports.celery.BROKER_PORT
substitutions[gs.CELERY_CONFIG_BROKER_USER] = self.config.input_ports.celery.BROKER_USER
substitutions[gs.CELERY_CONFIG_BROKER_PASSWORD] = self.config.input_ports.celery.BROKER_PASSWORD
substitutions[gs.CELERY_CONFIG_BROKER_VHOST] = self.config.input_ports.celery.BROKER_VHOST
substitutions[gs.CELERY_CONFIG_CELERY_RESULT_BACKEND] = 'amqp'
else:
logger.info('No input port for celery')
gs.generate_settings_file(file_layout.get_app_dir_path(),
file_layout.get_app_settings_module(),
django_config.components,
properties=substitutions)
# Validate the settings file. We run the validation script as an external program, have the results
# written to an external file, and then parse that file to get detailed results.
validate_results = \
engage_django_sdk.packager.run_installed_tests_as_subprocess(file_layout.get_app_dir_path(),
file_layout.get_app_settings_module(),
python_exe_path=self.config.input_ports.python.home,
use_logger=logger)
rc = validate_results.get_return_code()
if validate_results.run_was_successful():
logger.info("Django settings file validation successful")
else:
raise UserError(errors[ERR_DJANGO_VALIDATE_FAILED],
msg_args={"rc":validate_results.get_return_code_desc()},
developer_msg=validate_results.format_messages())
# instantiate the admin script (startup/shutdown/status) from a template
substitutions = {
"app_short_name": self.config.app_short_name,
"install_dir": self.config.home_path,
"settings_file_dir": file_layout.get_settings_file_directory(),
"python_path": file_layout.get_python_path(),
"django_settings_module": file_layout.get_deployed_settings_module(),
"python_bin_dir": self.config.python_bin_dir,
"websvr_hostname": self.config.input_ports.webserver_config.listen_host,
"port": self.config.input_ports.webserver_config.listen_port,
"log_directory": self.config.config_port.log_directory
}
admin_script_tmpl_path = iufile.get_data_file_path(__file__, "django.sh.tmpl")
iufile.instantiate_template_file(admin_script_tmpl_path, self.config.app_admin_script, substitutions,
logger=logger)
os.chmod(self.config.app_admin_script, 0755)
# modify manage.py to point at the our local python version and to have the right PYTHONPATH
manage_py_file = os.path.join(file_layout.get_settings_file_directory(), "manage.py")
if os.path.exists(manage_py_file):
logger.debug("Updating manage.py file with correct python path and python executable")
# default first line is "#!/usr/bin/env python", but user could
# replace this with a different path
shbang_pattern = '^\\#\\!.+$'
# python.home is the python executable
# home_path is where any local modules should go
shbang_replacement = "#!%s\nimport sys\nsys.path.extend(%s)" % \
(self.config.input_ports.python.home, file_layout.get_python_path().split(":").__repr__())
import_pattern = re.escape("import settings")
import_replacement = "import deployed_settings"
call_pattern = re.escape("execute_manager(settings)")
call_replacement = "execute_manager(deployed_settings)"
cnt = iufile.subst_in_file(manage_py_file,
[(shbang_pattern, shbang_replacement),
(import_pattern, import_replacement),
(call_pattern, call_replacement)])
if cnt != 3:
logger.warning("Substitutions for %s did not match expected number (got %d, expecting 3). Your manage.py script might not be usable without hand-modifying it." % (manage_py_file, cnt))
else:
logger.debug("No manage.py file, skipping shbang replacement")
# make the log directory
if not os.path.exists(self.config.config_port.log_directory):
logger.action("mkdir -p %s" % self.config.config_port.log_directory)
os.makedirs(self.config.config_port.log_directory)
# setup the database
if self.config.input_ports.django_db.ENGINE=='django.db.backends.sqlite3':
database_dir = os.path.dirname(self.config.input_ports.django_db.NAME)
if not os.path.exists(database_dir):
logger.action("mkdir -p %s" % database_dir)
os.makedirs(database_dir)
file_layout.run_admin_command("syncdb", ["--noinput"])
if "south" in django_config.installed_apps:
# We use the presence of South in INSTALLED_APPS to indicate that
# we should run migrations. If South isn't there, migrate will fail.
if upgrade:
file_layout.run_admin_command("migrate", ["--no-initial-data"])
else:
file_layout.run_admin_command("migrate", [])
# if we have fixtures and this is the initial install, load them
if len(django_config.fixtures)>0 and not upgrade:
file_layout.run_admin_command("loaddata", django_config.fixtures)
# gather static files, if requested
if static_url and \
('django.contrib.staticfiles' in django_config.installed_apps):
if not os.path.exists(static_root):
logger.debug("mkdir -p %s" % static_root)
os.makedirs(static_root)
file_layout.run_admin_command("collectstatic", ["--noinput"])
# See if this application includes the admin password parameter.
# If so, we set the password by running a script.
try:
app_pw = self.config.config_port.app_admin_password
except:
app_pw = None
if app_pw:
self._run_python_script("django_set_password.py", ["-c", "-e", self.config.config_port.admin_email, "-s"],
file_layout, input=self.config.config_port.app_admin_password)
# setup the webserver hooks
if self.config.input_ports.webserver_config.webserver_type == "apache":
self._wsgi_setup(file_layout)
self.ctx.r(apache_utils.restart_apache, self.ctx.props.input_ports.webserver_config)
def _wsgi_setup(self, file_layout):
# setup the wsgi configuration files
# TODO: eventually, this should go into a separate driver
def get_url_for_mapping(full_url):
path = urlparse.urlparse(full_url)[2]
if not (path[len(path)-1]=='/'):
path = path + "/"
return path
def get_path_for_mapping(path):
if not path[len(path)-1]=='/':
path = path + "/"
return path
def get_apache_directory_decl(path):
return "<Directory %s>\nOrder deny,allow\nAllow from all\n</Directory>\n" % path
wsgi_deploy_dir_path = os.path.join(self.config.home_path, "wsgi_deploy")
if not os.path.exists(wsgi_deploy_dir_path):
logger.action("mkdir -p %s" % wsgi_deploy_dir_path)
os.makedirs(wsgi_deploy_dir_path)
media_alias_directives = ""
media_directory_directives = ""
if file_layout.has_media_url_mapping():
(media_url, media_root) = file_layout.get_media_url_mapping()
media_alias_directives += "Alias %s %s\n" % (get_url_for_mapping(media_url),
get_path_for_mapping(media_root))
media_directory_directives += get_apache_directory_decl(media_root)
if file_layout.has_static_url_mapping():
(static_url, static_root) = file_layout.get_static_url_mapping()
media_alias_directives += "Alias %s %s\n" % (get_url_for_mapping(static_url),
get_path_for_mapping(static_root))
media_directory_directives += get_apache_directory_decl(static_root)
error_log = os.path.join(self.config.input_ports.webserver_config.log_dir,
"error.log")
access_log = os.path.join(self.config.input_ports.webserver_config.log_dir,
"access.log")
wsgi_substitutions = {
"install_path": self.config.home_path,
"settings_file_directory": file_layout.get_settings_file_directory(),
"python_path": file_layout.get_python_path(),
"python_exe": self.config.input_ports.python.home,
"genforma_home": self.config.input_ports.host.genforma_home,
"django_settings_module": file_layout.get_deployed_settings_module(),
"app_short_name": self.config.app_short_name,
"file_owner_group": self.config.input_ports.webserver_config.file_owner_group,
"additional_config_dir":self.config.input_ports.webserver_config.additional_config_dir,
"media_alias_directives":media_alias_directives,
"media_directory_directives":media_directory_directives,
"apache_error_log": error_log,
"apache_access_log": access_log,
"apache_log_dir":self.config.input_ports.webserver_config.log_dir
}
target_wsgi_file = os.path.join(wsgi_deploy_dir_path, "apache_wsgi.wsgi")
wsgi_file_template = iufile.get_data_file_path(__file__,"apache_wsgi.wsgi")
iufile.instantiate_template_file(wsgi_file_template, target_wsgi_file, wsgi_substitutions, logger=logger)
# the app can override the apache config file by providing one at
# <settings_file_dir>/wsgi_deploy/<app_short_name>_apache_wsgi.conf.
app_specific_apache_conf_file = os.path.join(os.path.join(file_layout.get_settings_file_directory(), "wsgi_deploy"),
"%s_apache_wsgi.conf" % self.config.app_short_name)
if os.path.exists(app_specific_apache_conf_file):
conf_file_template = app_specific_apache_conf_file
else:
conf_file_template = iufile.get_data_file_path(__file__,"django_apache_wsgi.conf")
target_conf_file = os.path.join(wsgi_deploy_dir_path, "%s_apache_wsgi.conf" % self.config.app_short_name)
iufile.instantiate_template_file(conf_file_template, target_conf_file, wsgi_substitutions, logger=logger)
self.ctx.r(sudo_add_config_file_line, self.config.mod_wsgi_config_file,
"WSGIPythonHome %s/python" % self.config.input_ports.host.genforma_home)
self.ctx.r(sudo_ensure_shared_perms,
self.config.input_ports.webserver_config.log_dir,
self.config.input_ports.webserver_config.file_owner_group,
writable_to_group=True)
self.ctx.r(sudo_ensure_shared_perms,
self.config.input_ports.host.genforma_home,
self.config.input_ports.webserver_config.file_owner_group,
writable_to_group=True)
self.ctx.r(apache_utils.add_apache_config_file,
target_conf_file,
self.config.input_ports.webserver_config)
