def _setup_xvfb(env): _setup_conf_file(env, "/etc/init.d/xvfb", "xvfb_init", default_source="xvfb_init") _setup_conf_file(env, "/etc/default/xvfb", "xvfb_default", default_source="xvfb_default") _setup_simple_service("xvfb") env.safe_sudo("mkdir /var/lib/xvfb; chown root:root /var/lib/xvfb; chmod 0755 /var/lib/xvfb") display_export = "-v DIPSLAY=:42" _add_to_profiles(display_export, profiles=["/home/%s/.sge_request" % env.get("galaxy_user", "galaxy")])
def _configure_ec2_autorun(env, use_repo_autorun=False): """ ec2autorun.py is a script that launches CloudMan on instance boot and is thus required on an instance. See the script itself for the details of what it does. This script also adds a cloudman service to ``/etc/init``, which actually runs ec2autorun.py as a system-level service at system boot. """ script = "ec2autorun.py" remote = os.path.join(env.install_dir, "bin", script) if not exists(os.path.dirname(remote)): sudo('mkdir -p {0}'.format(os.path.dirname(remote))) if use_repo_autorun: url = os.path.join(MI_REPO_ROOT_URL, script) sudo("wget --output-document=%s %s" % (remote, url)) else: install_file_dir = os.path.join(env.config_dir, os.pardir, "installed_files") put(os.path.join(install_file_dir, script), remote, mode=0777, use_sudo=True) # Create upstart configuration file for boot-time script cloudman_boot_file = 'cloudman.conf' remote_file = '/etc/init/%s' % cloudman_boot_file _write_to_file(cm_upstart % (remote, os.path.splitext(remote)[0]), remote_file, mode=0644) # Setup default image user data (if configured by image_user_data_path or # image_user_data_template_path). This specifies defaults for CloudMan when # used with resulting image, normal userdata supplied by user will override # these defaults. _setup_conf_file(env, os.path.join(env.install_dir, "bin", "IMAGE_USER_DATA"), "image_user_data", default_source="image_user_data") env.logger.info("Done configuring CloudMan's ec2_autorun")
def _install_nginx(env): """Nginx open source web server. http://www.nginx.org/ """ version = "1.2.0" url = "http://nginx.org/download/nginx-%s.tar.gz" % version install_dir = os.path.join(env.install_dir, "nginx") remote_conf_dir = os.path.join(install_dir, "conf") # Skip install if already present if env.safe_exists(remote_conf_dir) and env.safe_contains( os.path.join(remote_conf_dir, "nginx.conf"), "/cloud"): env.logger.debug("Nginx already installed; not installing it again.") return with _make_tmp_dir() as work_dir: with contextlib.nested(cd(work_dir), settings(hide('stdout'))): modules = _get_nginx_modules(env) module_flags = " ".join( ["--add-module=../%s" % x for x in modules]) env.safe_run("wget %s" % url) env.safe_run("tar xvzf %s" % os.path.split(url)[1]) with cd("nginx-%s" % version): env.safe_run( "./configure --prefix=%s --with-ipv6 %s " "--user=galaxy --group=galaxy --with-debug " "--with-http_ssl_module --with-http_gzip_static_module" % (install_dir, module_flags)) env.safe_sed("objs/Makefile", "-Werror", "") env.safe_run("make") env.safe_sudo("make install") env.safe_sudo("cd %s; stow nginx" % env.install_dir) defaults = {"galaxy_home": "/mnt/galaxy/galaxy-app"} _setup_conf_file(env, os.path.join(remote_conf_dir, "nginx.conf"), "nginx.conf", defaults=defaults) nginx_errdoc_file = 'nginx_errdoc.tar.gz' url = os.path.join(REPO_ROOT_URL, nginx_errdoc_file) remote_errdoc_dir = os.path.join(install_dir, "html") with cd(remote_errdoc_dir): env.safe_sudo("wget --output-document=%s/%s %s" % (remote_errdoc_dir, nginx_errdoc_file, url)) env.safe_sudo('tar xvzf %s' % nginx_errdoc_file) env.safe_sudo("mkdir -p %s" % env.install_dir) if not env.safe_exists("%s/nginx" % env.install_dir): env.safe_sudo("ln -s %s/sbin/nginx %s/nginx" % (install_dir, env.install_dir)) # If the guessed symlinking did not work, force it now cloudman_default_dir = "/opt/galaxy/sbin" if not env.safe_exists(cloudman_default_dir): env.safe_sudo("mkdir -p %s" % cloudman_default_dir) if not env.safe_exists(os.path.join(cloudman_default_dir, "nginx")): env.safe_sudo("ln -s %s/sbin/nginx %s/nginx" % (install_dir, cloudman_default_dir)) env.logger.debug("Nginx {0} installed to {1}".format(version, install_dir))
def _setup_service(env): _setup_conf_file(env, "/etc/init.d/galaxy", "galaxy_init", default_source="galaxy_init") _setup_conf_file(env, "/etc/default/galaxy", "galaxy_default") _setup_simple_service("galaxy")
def _setup_protvis_service(env): _setup_conf_file(env, os.path.join("/etc/init.d/protvis"), "protvis_init", default_source="protvis_init") _setup_conf_file(env, os.path.join("/etc/default/protvis"), "protvis_default") _setup_simple_service("protvis")
def _setup_nginx_service(env): # Setup system service for nginx, not needed for CloudMan but it is # useful if CloudMan is not being used (such as galaxy-vm-launcher work). _setup_conf_file(env, "/etc/init.d/nginx", "nginx_init", default_source="nginx_init") _setup_simple_service("nginx")
def install_proftpd(env): """Highly configurable GPL-licensed FTP server software. http://proftpd.org/ """ version = "1.3.4c" postgres_ver = "9.1" url = "ftp://ftp.tpnet.pl/pub/linux/proftpd/distrib/source/proftpd-%s.tar.gz" % version modules = "mod_sql:mod_sql_postgres:mod_sql_passwd" extra_modules = env.get("extra_proftp_modules", "") # Comma separated list of extra modules if extra_modules: modules = "%s:%s" % (modules, extra_modules.replace(",", ":")) install_dir = os.path.join(env.install_dir, 'proftpd') remote_conf_dir = os.path.join(install_dir, "etc") # Skip install if already available if env.safe_exists(remote_conf_dir): env.logger.debug("ProFTPd seems to already be installed in {0}".format(install_dir)) return with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget %s" % url) with settings(hide('stdout')): env.safe_run("tar xvzf %s" % os.path.split(url)[1]) with cd("proftpd-%s" % version): env.safe_run("CFLAGS='-I/usr/include/postgresql' ./configure --prefix=%s " "--disable-auth-file --disable-ncurses --disable-ident --disable-shadow " "--enable-openssl --with-modules=%s " "--with-libraries=/usr/lib/postgresql/%s/lib" % (install_dir, modules, postgres_ver)) env.safe_sudo("make") env.safe_sudo("make install") env.safe_sudo("make clean") # Get the init.d startup script initd_script = 'proftpd.initd' initd_url = os.path.join(REPO_ROOT_URL, 'conf_files', initd_script) remote_file = "/etc/init.d/proftpd" env.safe_sudo("wget --output-document=%s %s" % (remote_file, initd_url)) env.safe_sed(remote_file, 'REPLACE_THIS_WITH_CUSTOM_INSTALL_DIR', install_dir, use_sudo=True) env.safe_sudo("chmod 755 %s" % remote_file) # Set the configuration file conf_file = 'proftpd.conf' remote_file = os.path.join(remote_conf_dir, conf_file) if "postgres_port" not in env: env.postgres_port = '5910' if "galaxy_ftp_user_password" not in env: env.galaxy_ftp_user_password = '******' proftpd_conf = {'galaxy_uid': env.safe_run('id -u galaxy'), 'galaxy_fs': '/mnt/galaxy', # Should be a var but uncertain how to get it 'install_dir': install_dir} _setup_conf_file(env, remote_file, conf_file, overrides=proftpd_conf, default_source="proftpd.conf.template") # Get the custom welcome msg file welcome_msg_file = 'welcome_msg.txt' welcome_url = os.path.join(REPO_ROOT_URL, 'conf_files', welcome_msg_file) env.safe_sudo("wget --output-document=%s %s" % (os.path.join(remote_conf_dir, welcome_msg_file), welcome_url)) # Stow env.safe_sudo("cd %s; stow proftpd" % env.install_dir) env.logger.debug("----- ProFTPd %s installed to %s -----" % (version, install_dir))
def _install_nginx(env): """Nginx open source web server. http://www.nginx.org/ """ if "use_nginx_package" in env and env.use_nginx_package.upper() in ["TRUE", "YES"]: _install_nginx_package(env) return # Install nginx from directly version = "1.3.8" url = "http://nginx.org/download/nginx-%s.tar.gz" % version install_dir = os.path.join(env.install_dir, "nginx") remote_conf_dir = os.path.join(install_dir, "conf") # Skip install if already present if env.safe_exists(remote_conf_dir) and env.safe_contains(os.path.join(remote_conf_dir, "nginx.conf"), "/cloud"): env.logger.debug("Nginx already installed; not installing it again.") return with _make_tmp_dir() as work_dir: with contextlib.nested(cd(work_dir), settings(hide('stdout'))): modules = _get_nginx_modules(env) module_flags = " ".join(["--add-module=../%s" % x for x in modules]) env.safe_run("wget %s" % url) env.safe_run("tar xvzf %s" % os.path.split(url)[1]) with cd("nginx-%s" % version): env.safe_run("./configure --prefix=%s --with-ipv6 %s " "--user=galaxy --group=galaxy --with-debug " "--with-http_ssl_module --with-http_gzip_static_module " % (install_dir, module_flags)) env.safe_sed("objs/Makefile", "-Werror", "") env.safe_run("make") env.safe_sudo("make install") env.safe_sudo("cd %s; stow nginx" % env.install_dir) defaults = {"galaxy_home": "/mnt/galaxy/galaxy-app"} _setup_conf_file(env, os.path.join(remote_conf_dir, "nginx.conf"), "nginx.conf", defaults=defaults) nginx_errdoc_file = 'nginx_errdoc.tar.gz' url = os.path.join(REPO_ROOT_URL, nginx_errdoc_file) remote_errdoc_dir = os.path.join(install_dir, "html") with cd(remote_errdoc_dir): env.safe_sudo("wget --output-document=%s/%s %s" % (remote_errdoc_dir, nginx_errdoc_file, url)) env.safe_sudo('tar xvzf %s' % nginx_errdoc_file) env.safe_sudo("mkdir -p %s" % env.install_dir) if not env.safe_exists("%s/nginx" % env.install_dir): env.safe_sudo("ln -s %s/sbin/nginx %s/nginx" % (install_dir, env.install_dir)) # If the guessed symlinking did not work, force it now cloudman_default_dir = "/opt/galaxy/sbin" if not env.safe_exists(cloudman_default_dir): env.safe_sudo("mkdir -p %s" % cloudman_default_dir) if not env.safe_exists(os.path.join(cloudman_default_dir, "nginx")): env.safe_sudo("ln -s %s/sbin/nginx %s/nginx" % (install_dir, cloudman_default_dir)) env.logger.debug("Nginx {0} installed to {1}".format(version, install_dir))
def _configure_desktop(env): """ Configure a desktop manager to work with VNC. At the moment, this is tailored to `JWM` (and `jwm` and `vnc4server` packages need to be installed). """ if not _read_boolean(env, "configure_desktop", False): return # Create a start script for X _setup_conf_file(env, "/home/ubuntu/.vnc/xstartup", "xstartup", default_source="xstartup") # Create jwmrc config file _setup_conf_file(env, "/home/ubuntu/.jwmrc", "jwmrc.xml", default_source="jwmrc.xml", mode="0644") env.logger.info("----- Done configuring desktop -----")
def _configure_novnc(env): if not _read_boolean(env, "configure_novnc", False): # Longer term would like this enabled by default. -John return if not "novnc_install_dir" in env: env.novnc_install_dir = "/opt/novnc" if not "vnc_password" in env: env.vnc_password = "******" if not "vnc_user" in env: env.vnc_user = env.user if not "vnc_display" in env: env.vnc_display = "1" if not "vnc_depth" in env: env.vnc_depth = "16" if not "vnc_geometry" in env: env.vnc_geometry = "1024x768" _configure_vncpasswd(env) novnc_dir = env.novnc_install_dir env.safe_sudo("mkdir -p '%s'" % novnc_dir) env.safe_sudo("chown %s '%s'" % (env.user, novnc_dir)) clone_cmd = "NOVNC_DIR='%s'; rm -rf $NOVNC_DIR; git clone https://github.com/kanaka/noVNC.git $NOVNC_DIR" % novnc_dir run(clone_cmd) ## Move vnc_auto.html which takes vnc_password as query argument ## to index.html and rewrite it so that password is autoset, no ## need to specify via query parameter. run("sed s/password\\ =\\ /password\\ =\\ \\\'%s\\\'\\;\\\\\\\\/\\\\\\\\// '%s/vnc_auto.html' > '%s/index.html'" % (env.vnc_password, novnc_dir, novnc_dir)) _setup_conf_file(env, "/etc/init.d/novnc", "novnc_init", default_source="novnc_init") _setup_conf_file(env, "/etc/default/novnc", "novnc_default", default_source="novnc_default.template") _setup_conf_file(env, "/etc/init.d/vncserver", "vncserver_init", default_source="vncserver_init") _setup_conf_file(env, "/etc/default/vncserver", "vncserver_default", default_source="vncserver_default.template") _setup_simple_service("novnc") _setup_simple_service("vncserver")
def _configure_ec2_autorun(env, use_repo_autorun=False): """ ec2autorun.py is a script that launches CloudMan on instance boot and is thus required on an instance. See the script itself for the details of what it does. This script also adds a cloudman service to ``/etc/init``, which actually runs ec2autorun.py as a system-level service at system boot. """ script = "ec2autorun.py" remote = os.path.join(env.install_dir, "bin", script) if not exists(os.path.dirname(remote)): sudo('mkdir -p {0}'.format(os.path.dirname(remote))) if use_repo_autorun: # Is this used, can we eliminate use_repo_autorun? url = os.path.join(MI_REPO_ROOT_URL, script) sudo("wget --output-document=%s %s" % (remote, url)) else: install_file_dir = os.path.join(env.config_dir, os.pardir, "installed_files") put(os.path.join(install_file_dir, script), remote, mode=0777, use_sudo=True) # Create upstart configuration file for boot-time script cloudman_boot_file = 'cloudman.conf' remote_file = '/etc/init/%s' % cloudman_boot_file _write_to_file(cm_upstart % (remote, os.path.splitext(remote)[0]), remote_file, mode=0644) # Setup default image user data (if configured by image_user_data_path or # image_user_data_template_path). This specifies defaults for CloudMan when # used with resulting image, normal userdata supplied by user will override # these defaults. image_user_data_path = os.path.join(env.install_dir, "bin", "IMAGE_USER_DATA") if "image_user_data_dict" in env: # Explicit YAML contents defined in env, just dump them as is. import yaml _write_to_file(yaml.dump(env.get("image_user_data_dict")), image_user_data_path, mode=0644) else: # Else use file or template file. _setup_conf_file(env, image_user_data_path, "image_user_data", default_source="image_user_data") env.logger.info("Done configuring CloudMan's ec2_autorun")
def _configure_desktop(env): """ Configure a desktop manager to work with VNC. Note that `xfce4` (or `jwm`) and `vnc4server` packages need to be installed for this to have effect. """ if not _read_boolean(env, "configure_desktop", False): return # Set nginx PAM module to allow logins for any system user if env.safe_exists("/etc/pam.d"): env.safe_sudo('echo "@include common-auth" > /etc/pam.d/nginx') env.safe_sudo('usermod -a -G shadow galaxy') # Create a start script for X _setup_conf_file(env, "/home/ubuntu/.vnc/xstartup", "xstartup", default_source="xstartup") # Create jwmrc config file (uncomment this if using jwm window manager) # _setup_conf_file(env, "/home/ubuntu/.jwmrc", "jwmrc.xml", # default_source="jwmrc.xml", mode="0644") env.logger.info("----- Done configuring desktop -----")
def _install_nginx_package(env): """ Install nginx from a custom package; also see https://github.com/afgane/gvl_flavor/tree/master/playbook """ version = "1.4.7-gvl20140507b01-precise" package_url = "http://cloudman-dev.s3.amazonaws.com/gvl-nginx-%s-amd64.deb" % version with _make_tmp_dir() as work_dir: with contextlib.nested(cd(work_dir), settings(hide('stdout'))): env.safe_run("wget %s" % package_url) env.safe_sudo("dpkg -i %s" % os.path.split(package_url)[1]) # The path for `nginx.conf` depends on how the package was built and # what's set in `nginx_remote_conf_path` must match that! defaults = {"galaxy_home": "/mnt/galaxy/galaxy-app"} _setup_conf_file(env, env.nginx_remote_conf_path, "nginx.conf", defaults=defaults) env.logger.debug("Nginx v{0} installed from package {1}".format(version, package_url))
def _install_nginx_package(env): """ Install nginx from a custom package; also see https://github.com/afgane/gvl_flavor/tree/master/playbook """ version = "1.4.7-gvl20140407b03" package_url = "http://cloudman-dev.s3.amazonaws.com/gvl-nginx-%s-precise-amd64.deb" % version with _make_tmp_dir() as work_dir: with contextlib.nested(cd(work_dir), settings(hide('stdout'))): env.safe_run("wget %s" % package_url) env.safe_sudo("dpkg -i %s" % os.path.split(package_url)[1]) # The path for `nginx.conf` depends on how the package was built and # what's set in `nginx_remote_conf_path` must match that! defaults = {"galaxy_home": "/mnt/galaxy/galaxy-app"} _setup_conf_file(env, env.nginx_remote_conf_path, "nginx.conf", defaults=defaults) env.logger.debug("Nginx v{0} installed from package {1}".format(version, package_url))
def _configure_ec2_autorun(env, use_repo_autorun=False): """ ec2autorun.py is a script that launches CloudMan on instance boot and is thus required on an instance. See the script itself for the details of what it does. This script also adds a cloudman service to ``/etc/init``, which actually runs ec2autorun.py as a system-level service at system boot. """ script = "ec2autorun.py" remote = os.path.join(env.install_dir, "bin", script) if not env.safe_exists(os.path.dirname(remote)): env.safe_sudo('mkdir -p {0}'.format(os.path.dirname(remote))) if use_repo_autorun: # Is this used, can we eliminate use_repo_autorun? url = os.path.join(MI_REPO_ROOT_URL, script) env.safe_sudo("wget --output-document=%s %s" % (remote, url)) else: install_file_dir = os.path.join(env.config_dir, os.pardir, "installed_files") tmp_remote = os.path.join("/tmp", os.path.basename(remote)) env.safe_put(os.path.join(install_file_dir, script), tmp_remote) env.safe_sudo("mv %s %s" % (tmp_remote, remote)) env.safe_sudo("chmod 0777 %s" % remote) # Create upstart configuration file for boot-time script cloudman_boot_file = 'cloudman.conf' remote_file = '/etc/init/%s' % cloudman_boot_file _write_to_file(cm_upstart % (remote, os.path.splitext(remote)[0]), remote_file, mode="0644") # Setup default image user data (if configured by image_user_data_path or # image_user_data_template_path). This specifies defaults for CloudMan when # used with resulting image, normal userdata supplied by user will override # these defaults. image_user_data_path = os.path.join(env.install_dir, "bin", "IMAGE_USER_DATA") if "image_user_data_dict" in env: # Explicit YAML contents defined in env, just dump them as is. import yaml _write_to_file(yaml.dump(env.get("image_user_data_dict")), image_user_data_path, mode="0644") else: # Else use file or template file. _setup_conf_file(env, image_user_data_path, "image_user_data", default_source="image_user_data") env.logger.info("Done configuring CloudMan's ec2_autorun")
def _setup_xvfb(env): _setup_conf_file(env, "/etc/init.d/xvfb", "xvfb_init", default_source="xvfb_init") _setup_conf_file(env, "/etc/default/xvfb", "xvfb_default", default_source="xvfb_default") _setup_simple_service("xvfb") env.safe_sudo("mkdir /var/lib/xvfb; chown root:root /var/lib/xvfb; chmod 0755 /var/lib/xvfb")