def _configure(self): """ Configure the Pulsar application. """ self.state = service_states.CONFIGURING misc.make_dir(self.pulsar_home) # Write out app.yml token = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.lowercase + string.digits) for _ in range(25)) app_template = Template(app_yml) app_yml_file = os.path.join(self.pulsar_home, 'app.yml') misc.write_template_file(app_template, {'token': token}, app_yml_file) # Write out server.ini srvr_template = Template(server_ini) server_ini_file = os.path.join(self.pulsar_home, 'server.ini') misc.write_template_file(srvr_template, {'pulsar_port': self.pulsar_port}, server_ini_file) # Write out local_env.sh lcl_template = Template(local_env_sh) lcl_file = os.path.join(self.pulsar_home, 'local_env.sh') misc.write_template_file(lcl_template, {'galaxy_home': '/mnt/galaxy/galaxy-app'}, lcl_file) # Set the owner to 'galaxy' system user attempt_chown_galaxy(self.pulsar_home, recursive=True) if self.supervisor: # Create a supervisor conf file supervisor_conf_file = os.path.join(self.supervisor.conf_dir, '{0}.conf'.format(self.supervisor_prog_name)) template = Template(supervisor_conf) misc.write_template_file(template, None, supervisor_conf_file) else: log.warning("No supervisor service?")
def configure_proftpd(self): """ Configure environment for running ProFTPd service. """ log.debug("Configuring ProFTPd") # Because we're rewriting the proftpd config file below, update the # password for PostgreSQL galaxyftp user gftp_pwd = self.app.path_resolver.proftpd_galaxyftp_user_pwd log.debug("Setting psql password for galaxyftp role to {0}".format(gftp_pwd)) # Check if galaxtftp role already exists cmd = ('{0} - postgres -c"{1} -p {2} -tAc\\\"SELECT 1 FROM pg_roles WHERE rolname=\'galaxyftp\'\\\""' .format(paths.P_SU, self.app.path_resolver.psql_cmd, self.app.path_resolver.psql_db_port)) role = 'ALTER' if misc.getoutput(cmd) == '1' else 'CREATE' # Now either CREATE or ALTER the galaxyftp role to set the permissions cmd = ('{0} - postgres -c"{1} -p {4} -c\\\"{2} ROLE galaxyftp LOGIN PASSWORD \'{3}\'\\\""' .format(paths.P_SU, self.app.path_resolver.psql_cmd, role, gftp_pwd, self.app.path_resolver.psql_db_port)) misc.run(cmd) # Update the config to match the current environment proftpd_tmplt = conf_manager.PROFTPD_CONF_TEMPLATE proftpd_conf_template = conf_manager.load_conf_template(proftpd_tmplt) params = { 'galaxy_user_name': paths.GALAXY_USER_NAME, 'galaxyftp_user_name': 'galaxyftp', 'psql_galaxyftp_password': gftp_pwd, 'galaxy_db_port': self.app.path_resolver.psql_db_port, 'galaxyFS_base_path': self.app.path_resolver.galaxy_data, 'public_ip_address': self.app.cloud_interface.get_public_ip() } template = proftpd_conf_template.substitute(params) # Write out the config file with open(self.app.path_resolver.proftpd_conf_file, 'w') as f: print >> f, template log.debug("Updated ProFTPd conf file {0}".format( self.app.path_resolver.proftpd_conf_file)) # Place the FTP welcome message file urllib.urlretrieve("https://s3.amazonaws.com/cloudman/files/proftpd_welcome.txt", "/usr/proftpd/etc/welcome_msg.txt") # Setup the Galaxy data dir for FTP ftp_data_dir = '%s/tmp/ftp' % self.app.path_resolver.galaxy_data if not os.path.exists(ftp_data_dir): os.makedirs(ftp_data_dir) attempt_chown_galaxy(ftp_data_dir) # Some images have vsFTPd server included so stop it first vsFTPd_exists = misc.run('status vsftpd', quiet=True) if vsFTPd_exists and 'start/running' in vsFTPd_exists: log.debug("Stopping vsFTPd") misc.run('stop vsftpd') # Start the server now if misc.run('/etc/init.d/proftpd start'): self.state = service_states.RUNNING return True else: log.error("Trouble starting ProFTPd") self.state = service_states.ERROR return False
def _setup(self): log.debug("Running GalaxyReportsService _setup") # WORKAROUND: The run_reports.sh command refers to a parameter # named --safe-pidfile which is not supported by the uwsgi binary. # Replace it with --pidfile instead. patch_start_command = ("sudo sed -i \"s/--safe-pidfile/--pidfile/g" "\" %s/scripts/common_startup_functions.sh" % self.galaxy_home) misc.run(patch_start_command) # Create default output dir for files file_path = os.path.join(self.app.path_resolver.galaxy_home, "database/files") misc.make_dir(file_path, owner='galaxy') tmp_file_path = os.path.join(self.app.path_resolver.galaxy_home, "database/tmp") misc.make_dir(tmp_file_path, owner='galaxy') # Create the new reports config params = { 'galaxy_db_port': self.app.path_resolver.psql_db_port } template = conf_manager.load_conf_template(conf_manager.GALAXY_REPORTS_TEMPLATE) misc.write_template_file(template, params, self.conf_file) attempt_chown_galaxy(self.conf_file)
def _configure(self): """ Configure the Pulsar application. """ self.state = service_states.CONFIGURING misc.make_dir(self.pulsar_home) # Write out app.yml token = ''.join( random.SystemRandom().choice(string.ascii_uppercase + string.lowercase + string.digits) for _ in range(25)) app_template = Template(app_yml) app_yml_file = os.path.join(self.pulsar_home, 'app.yml') misc.write_template_file(app_template, {'token': token}, app_yml_file) # Write out server.ini srvr_template = Template(server_ini) server_ini_file = os.path.join(self.pulsar_home, 'server.ini') misc.write_template_file(srvr_template, {'pulsar_port': self.pulsar_port}, server_ini_file) # Write out local_env.sh lcl_template = Template(local_env_sh) lcl_file = os.path.join(self.pulsar_home, 'local_env.sh') misc.write_template_file(lcl_template, {'galaxy_home': '/mnt/galaxy/galaxy-app'}, lcl_file) # Set the owner to 'galaxy' system user attempt_chown_galaxy(self.pulsar_home, recursive=True) if self.supervisor: # Create a supervisor conf file supervisor_conf_file = os.path.join( self.supervisor.conf_dir, '{0}.conf'.format(self.supervisor_prog_name)) template = Template(supervisor_conf) misc.write_template_file(template, None, supervisor_conf_file) else: log.warning("No supervisor service?")
def configure_proftpd(self): """ Configure environment for running ProFTPd service. """ # In the config, set the port on which postgres is running log.debug("Configuring ProFTPd") # This is a bit dodgy but ports are hardcoded in CBL so shoudl be a pretty # safe bet for the time being misc.replace_string('/usr/proftpd/etc/proftpd.conf', 'galaxy@localhost:5840', 'galaxy@localhost:{0}'.format(paths.C_PSQL_PORT)) misc.replace_string('/usr/proftpd/etc/proftpd.conf', '/mnt/galaxyData', self.app.path_resolver.galaxy_data) # Verify that the expected config path exists. This is expected to be # /mnt/galaxy/tools/proftpd # TODO just use /usr/proftpd. Should be a simple update to the init.d expected_config_path = os.path.join(self.app.path_resolver.galaxy_data, 'tools', 'proftpd') if not os.path.exists(expected_config_path): misc.run('ln -s /usr/profpd %s' % expected_config_path) # Setup the data dir for FTP ftp_data_dir = '%s/tmp/ftp' % self.app.path_resolver.galaxy_data if not os.path.exists(ftp_data_dir): os.makedirs(ftp_data_dir) attempt_chown_galaxy(ftp_data_dir) # Some images have vsFTPd server included so stop it first vsFTPd_exists = misc.run('status vsftpd', quiet=True) if vsFTPd_exists and 'start/running' in vsFTPd_exists: log.debug("Stopping vsFTPd") misc.run('stop vsftpd') # Start the server now if misc.run('/etc/init.d/proftpd start'): self.state = service_states.RUNNING return True else: log.debug("Trouble starting ProFTPd") return False
def manage_galaxy(self, to_be_started=True): if self.app.TESTFLAG is True and self.app.LOCALFLAG is False: log.debug("Attempted to manage Galaxy, but TESTFLAG is set.") return log.debug("Using Galaxy from '{0}'".format(self.galaxy_home)) os.putenv("GALAXY_HOME", self.galaxy_home) os.putenv("TEMP", self.app.path_resolver.galaxy_temp) os.putenv("TMPDIR", self.app.path_resolver.galaxy_temp) self.env_vars["GALAXY_HOME"] = self.galaxy_home self.env_vars["TEMP"] = self.app.path_resolver.galaxy_temp self.env_vars["TMPDIR"] = self.app.path_resolver.galaxy_temp conf_dir = self.option_manager.setup() if conf_dir: self.env_vars["GALAXY_UNIVERSE_CONFIG_DIR"] = conf_dir if self._multiple_processes(): self.env_vars["GALAXY_RUN_ALL"] = "TRUE" # HACK: Galaxy has a known problem when starting from a fresh configuration # in multiple process mode. Each process attempts to create the same directories # and one or more processes can fail to start because it "failed" to create # said directories (because another process created them first). This hack staggers # the process starts in an attempt to circumvent this problem. patch_run_sh_command = "sudo sed -i -e \"s/server.log \\$\\@$/\\0; sleep 4/\" %s/run.sh" % self.galaxy_home misc.run(patch_run_sh_command) self.extra_daemon_args = "" else: # Instead of sticking with default paster.pid and paster.log, explicitly # set pid and log file to ``main.pid`` and ``main.log`` to bring single # process case inline with defaults for for multiple process case (i.e. # when GALAXY_RUN_ALL is set and multiple servers are defined). self.extra_daemon_args = "--pid-file=main.pid --log-file=main.log" if to_be_started and self.remaining_start_attempts > 0: self.status() # If not provided as part of user data, update nginx conf with # current paths if self.app.ud.get('nginx_conf_contents', None) is None: self.configure_nginx() if not self.configured: log.debug("Setting up Galaxy application") s3_conn = self.app.cloud_interface.get_s3_connection() if not os.path.exists(self.galaxy_home): log.error("Galaxy application directory '%s' does not exist! Aborting." % self.galaxy_home) log.debug("ls /mnt/: %s" % os.listdir('/mnt/')) self.state = service_states.ERROR self.last_state_change_time = datetime.utcnow() return False # If a configuration file is not already in Galaxy's dir, # retrieve it from a persistent data repository (i.e., S3) if s3_conn: for f_name in ['universe_wsgi.ini', 'tool_conf.xml', 'tool_data_table_conf.xml', 'shed_tool_conf.xml', 'datatypes_conf.xml', 'shed_tool_data_table_conf.xml']: f_path = os.path.join(self.galaxy_home, f_name) if not os.path.exists(f_path): if not misc.get_file_from_bucket(s3_conn, self.app.ud['bucket_cluster'], '{0}.cloud'.format(f_name), f_path): # We did not get the config file from cluster's # bucket so get it from the default bucket log.debug("Did not get Galaxy configuration file " + "'{0}' from cluster bucket '{1}'" .format(f_name, self.app.ud['bucket_cluster'])) log.debug("Trying to retrieve one ({0}.cloud) " "from the default '{1}' bucket." .format(f_name, self.app.ud['bucket_default'])) local_file = os.path.join(self.galaxy_home, f_name) misc.get_file_from_bucket(s3_conn, self.app.ud['bucket_default'], '{0}.cloud'.format(f_name), local_file) attempt_chown_galaxy_if_exists(local_file) # Make sure the temporary job_working_directory exists on user # data volume (defined in universe_wsgi.ini.cloud) if not os.path.exists('%s/tmp/job_working_directory' % self.app.path_resolver.galaxy_data): os.makedirs('%s/tmp/job_working_directory/' % self.app.path_resolver.galaxy_data) attempt_chown_galaxy('%s/tmp/job_working_directory/' % self.app.path_resolver.galaxy_data) # Make sure the default shed_tools directory exists if not os.path.exists('%s/../shed_tools' % self.app.path_resolver.galaxy_data): os.makedirs('%s/../shed_tools/' % self.app.path_resolver.galaxy_data) attempt_chown_galaxy('%s/../shed_tools/' % self.app.path_resolver.galaxy_data) # TEMPORARY ONLY - UNTIL SAMTOOLS WRAPPER IS CONVERTED TO USE # DATA TABLES if os.path.exists('/mnt/galaxyIndices/locfiles/sam_fa_indices.loc'): shutil.copy( '/mnt/galaxyIndices/locfiles/sam_fa_indices.loc', '%s/tool-data/sam_fa_indices.loc' % self.galaxy_home) # Ensure the environment is setup for running Galaxy # This can also be setup on the tools snapshot and thus avoid these patches # try: # subprocess.call( "sed 's/cd `dirname $0`/cd `dirname $0`; export TEMP=\/mnt\/galaxyData\/tmp/; export DRMAA_LIBRARY_PATH=/opt/sge/lib/lx24-amd64/libdrmaa.so.1.0' %s/run.sh > %s/run.sh.custom" % (self.galaxy_home, self.galaxy_home), shell=True ) # misc.run("cd %s; sed 's/python/python -ES/g' run.sh.custom > run.sh" % self.galaxy_home, "Failed to adjust run.sh", "Successfully adjusted run.sh") # shutil.copy( self.galaxy_home + '/run.sh.custom', self.galaxy_home + '/run.sh' ) # os.chown( self.galaxy_home + '/run.sh', pwd.getpwnam( "galaxy" )[2], grp.getgrnam( "galaxy" )[2] ) # except Exception, e: # log.debug("Problem customizing Galaxy's run.sh: %s" % e) # try: # misc.run("cd %s; sed 's/pyhton/python -ES/g' setup.sh > setup.sh.custom" % self.galaxy_home, "Failed to edit setup.sh", "Successfully adjusted setup.sh") # shutil.copy( self.galaxy_home + '/setup.sh.custom', self.galaxy_home + '/setup.sh' ) # os.chown( self.galaxy_home + '/setup.sh', pwd.getpwnam( "galaxy" )[2], grp.getgrnam( "galaxy" )[2] ) # except Exception, e: # log.error("Error adjusting setup.sh: %s" % e) # subprocess.call( 'sed "s/#start_job_runners = pbs/start_job_runners = sge/" $GALAXY_HOME/universe_wsgi.ini > $GALAXY_HOME/universe_wsgi.ini.custom', shell=True ) # shutil.move( self.galaxy_home + '/universe_wsgi.ini.custom', self.galaxy_home + '/universe_wsgi.ini' ) # subprocess.call( 'sed "s/#default_cluster_job_runner = pbs:\/\/\//default_cluster_job_runner = sge:\/\/\//" $GALAXY_HOME/universe_wsgi.ini > $GALAXY_HOME/universe_wsgi.ini.custom', shell=True ) # shutil.move( self.galaxy_home + '/universe_wsgi.ini.custom', self.galaxy_home + '/universe_wsgi.ini' ) # Configure PATH in /etc/profile because otherwise some tools do not work # with open('/etc/profile', 'a') as f: # f.write('export PATH=/mnt/galaxyTools/tools/bin:/mnt/galaxyTools/tools/pkg/fastx_toolkit_0.0.13:/mnt/galaxyTools/tools/pkg/bowtie-0.12.5:/mnt/galaxyTools/tools/pkg/samtools-0.1.7_x86_64-linux:/mnt/galaxyTools/tools/pkg/gnuplot-4.4.0/bin:/opt/PostgreSQL/8.4/bin:$PATH\n') # os.chown(self.galaxy_home + '/universe_wsgi.ini', # pwd.getpwnam("galaxy")[2], grp.getgrnam("galaxy")[2]) self.remaining_start_attempts -= 1 self.configured = True if self.state != service_states.RUNNING: log.debug("Starting Galaxy...") # Make sure admin users get added self.update_galaxy_config() start_command = self.galaxy_run_command( "%s --daemon" % self.extra_daemon_args) log.debug(start_command) if not misc.run(start_command, "Error invoking Galaxy", "Successfully initiated Galaxy start from {0}.".format(self.galaxy_home)): if self.remaining_start_attempts > 0: self.state = service_states.UNSTARTED self.last_state_change_time = datetime.utcnow() else: self.state = service_states.ERROR self.last_state_change_time = datetime.utcnow() else: log.debug("Galaxy already running.") else: log.info("Shutting down Galaxy...") self.state = service_states.SHUTTING_DOWN stop_command = self.galaxy_run_command("%s --stop-daemon" % self.extra_daemon_args) if misc.run(stop_command): self.state = service_states.SHUT_DOWN self.last_state_change_time = datetime.utcnow() # Move all log files subprocess.call("bash -c 'for f in $GALAXY_HOME/{main,handler,manager,web}*.log; do mv \"$f\" \"$f.%s\"; done'" % datetime.utcnow().strftime('%H_%M'), shell=True)
def configure_proftpd(self): """ Configure environment for running ProFTPd service. """ log.debug("Configuring ProFTPd") # Because we're rewriting the proftpd config file below, update the # password for PostgreSQL galaxyftp user gftp_pwd = self.app.path_resolver.proftpd_galaxyftp_user_pwd log.debug( "Setting psql password for galaxyftp role to {0}".format(gftp_pwd)) # Check if galaxtftp role already exists cmd = ( '{0} - postgres -c"{1} -p {2} -tAc\\\"SELECT 1 FROM pg_roles WHERE rolname=\'galaxyftp\'\\\""' .format(paths.P_SU, self.app.path_resolver.psql_cmd, self.app.path_resolver.psql_db_port)) role = 'ALTER' if misc.getoutput(cmd) == '1' else 'CREATE' # Now either CREATE or ALTER the galaxyftp role to set the permissions cmd = ( '{0} - postgres -c"{1} -p {4} -c\\\"{2} ROLE galaxyftp LOGIN PASSWORD \'{3}\'\\\""' .format(paths.P_SU, self.app.path_resolver.psql_cmd, role, gftp_pwd, self.app.path_resolver.psql_db_port)) misc.run(cmd) # Update the config to match the current environment proftpd_tmplt = conf_manager.PROFTPD_CONF_TEMPLATE proftpd_conf_template = conf_manager.load_conf_template(proftpd_tmplt) params = { 'galaxy_user_name': paths.GALAXY_USER_NAME, 'galaxyftp_user_name': 'galaxyftp', 'psql_galaxyftp_password': gftp_pwd, 'galaxy_db_port': self.app.path_resolver.psql_db_port, 'galaxyFS_base_path': self.app.path_resolver.galaxy_data, 'public_ip_address': self.app.cloud_interface.get_public_ip() } template = proftpd_conf_template.substitute(params) # Write out the config file with open(self.app.path_resolver.proftpd_conf_file, 'w') as f: print >> f, template log.debug("Updated ProFTPd conf file {0}".format( self.app.path_resolver.proftpd_conf_file)) # Place the FTP welcome message file urllib.urlretrieve( "https://s3.amazonaws.com/cloudman/files/proftpd_welcome.txt", "/usr/proftpd/etc/welcome_msg.txt") # Setup the Galaxy data dir for FTP ftp_data_dir = '%s/tmp/ftp' % self.app.path_resolver.galaxy_data if not os.path.exists(ftp_data_dir): os.makedirs(ftp_data_dir) attempt_chown_galaxy(ftp_data_dir) # Some images have vsFTPd server included so stop it first vsFTPd_exists = misc.run('status vsftpd', quiet=True) if vsFTPd_exists and 'start/running' in vsFTPd_exists: log.debug("Stopping vsFTPd") misc.run('stop vsftpd') # Start the server now if misc.run('/etc/init.d/proftpd start'): self.state = service_states.RUNNING return True else: log.error("Trouble starting ProFTPd") self.state = service_states.ERROR return False
def manage_galaxy(self, to_be_started=True): """ Use this method to start and stop Galaxy application. :type to_be_started: bool :param to_be_started: If set, this method will attempt to start the Galaxy application process. If not set, the method will attempt to shut down the application process. """ log.debug("Using Galaxy from '{0}'".format(self.galaxy_home)) os.putenv("GALAXY_HOME", self.galaxy_home) os.putenv("TEMP", self.app.path_resolver.galaxy_temp) os.putenv("TMPDIR", self.app.path_resolver.galaxy_temp) self.env_vars["GALAXY_HOME"] = self.galaxy_home self.env_vars["TEMP"] = self.app.path_resolver.galaxy_temp self.env_vars["TMPDIR"] = self.app.path_resolver.galaxy_temp conf_dir = self.option_manager.setup() if conf_dir: self.env_vars["GALAXY_UNIVERSE_CONFIG_DIR"] = conf_dir if self.multiple_processes(): self.env_vars["GALAXY_RUN_ALL"] = "TRUE" # HACK: Galaxy has a known problem when starting from a fresh # configuration in multiple process mode. Each process attempts to # create the same directories and one or more processes can fail to # start because it "failed" to create said directories (because # another process created them first). This hack staggers # the process starts in an attempt to circumvent this problem. patch_run_sh_command = ( "sudo sed -i -e \"s/server.log \\$\\@$/\\0; " "sleep 4/\" %s/run.sh" % self.galaxy_home) misc.run(patch_run_sh_command) self.extra_daemon_args = "" else: # Instead of sticking with default paster.pid and paster.log, # explicitly set pid and log file to ``main.pid`` and ``main.log`` # to bring single process case inline with defaults for for multiple # process case (i.e. when GALAXY_RUN_ALL is set and multiple servers # are defined). # self.extra_daemon_args = "--pid-file=main.pid --log-file=main.log" # No longer required pass if to_be_started and self.remaining_start_attempts > 0: self.status() if not self.configured: log.debug("Setting up Galaxy application") # Set job manager configs if necessary for job_manager_svc in self.app.manager.service_registry.active( service_role=ServiceRole.JOB_MANAGER): if ServiceRole.SGE in job_manager_svc.svc_roles: log.debug("Running on SGE; setting env_vars") self.env_vars[ "SGE_ROOT"] = self.app.path_resolver.sge_root, self.env_vars[ "DRMAA_LIBRARY_PATH"] = self.app.path_resolver.drmaa_library_path # Make sure Galaxy home dir exists if not os.path.exists(self.galaxy_home): log.error("Galaxy application directory '%s' does not " "exist! Aborting." % self.galaxy_home) log.debug("ls /mnt/: %s" % os.listdir('/mnt/')) self.state = service_states.ERROR self.last_state_change_time = datetime.utcnow() return False # Ensure the necessary directories exist for dir_name in [ paths.P_GALAXY_INDICES, ('%s/tmp/job_working_directory' % self.app.path_resolver.galaxy_data) ]: misc.make_dir(dir_name, 'galaxy') self.configured = True if not self._is_galaxy_running(): log.debug("Starting Galaxy...") self.update_galaxy_config() start_command = self.galaxy_run_command("%s --daemon" % self.extra_daemon_args) attempt_chown_galaxy(self.galaxy_home) if misc.run(start_command): self.remaining_start_attempts -= 1 elif self.remaining_start_attempts > 0: log.debug( "It seems Galaxy failed to start; will atempt to " "auto-restart (up to {0} more time(s)).".format( self.remaining_start_attempts)) self.state = service_states.UNSTARTED self.last_state_change_time = datetime.utcnow() else: log.debug( "It seems Galaxy failed to start; setting service " "state to {0}.".format(service_states.ERROR)) self.state = service_states.ERROR self.last_state_change_time = datetime.utcnow() else: log.debug("Galaxy already running.") else: log.info("Shutting down Galaxy...") self.state = service_states.SHUTTING_DOWN stop_command = self.galaxy_run_command("%s --stop-daemon" % self.extra_daemon_args) if self._is_galaxy_running(): misc.run(stop_command) if not self._is_galaxy_running(): log.debug( "Galaxy not running; setting service state to SHUT_DOWN.") self.state = service_states.SHUT_DOWN self.last_state_change_time = datetime.utcnow() # Move all log files subprocess.call( "bash -c 'for f in $GALAXY_HOME/{main,handler,manager,web}*.log; " "do mv \"$f\" \"$f.%s\"; done'" % datetime.utcnow().strftime('%H_%M'), shell=True)