def get_path(name, default_path):
path = None
try:
path = misc.load_yaml_file(USER_DATA_FILE).get(name, None)
if path is None:
downloaded_pd_file = 'pd.yaml'
if os.path.exists(downloaded_pd_file):
path = misc.load_yaml_file(downloaded_pd_file).get(name, default_path)
except:
pass
if not path:
path = default_path
return path
def get_user_data(self, force=False):
if self.user_data is None or force:
self.user_data = misc.load_yaml_file(paths.USER_DATA_FILE)
self.aws_access_key = self.user_data.get("access_key", None)
self.aws_secret_key = self.user_data.get("secret_key", None)
self.s3_url = self.user_data.get("s3_url", None)
self.ec2_url = self.user_data.get("ec2_url", None)
return self.user_data
def get_user_data(self, force=False):
if self.user_data is None or force:
self.user_data = misc.load_yaml_file(paths.USER_DATA_FILE)
self.aws_access_key = self.user_data.get('access_key', None)
self.aws_secret_key = self.user_data.get('secret_key', None)
self.s3_url = self.user_data.get('s3_url', None)
self.ec2_url = self.user_data.get('ec2_url', None)
return self.user_data
def get_cloud_type(self):
""" Determine the type of cloud currently being running on.
If needed, this method should be extended to include other
(or alternative) methods for detecting the type of cloud.
The current assumption is that the type of cloud is included as
part of user data and stored in paths.USER_DATA_FILE file
under 'cloud_type' key.
"""
return misc.load_yaml_file(paths.USER_DATA_FILE).get('cloud_type', 'ec2').lower()
def get_cloud_type(self):
""" Determine the type of cloud currently being running on.
If needed, this method should be extended to include other
(or alternative) methods for detecting the type of cloud.
The current assumption is that the type of cloud is included as
part of user data and stored in paths.USER_DATA_FILE file
under 'cloud_type' key.
"""
return misc.load_yaml_file(paths.USER_DATA_FILE).get("cloud_type", "ec2").lower()
def get_path(name, default_path):
"""
Get a file system path where a service with the given ``name`` resides/runs
as defined in the user data. For example, to use a custom path for Galaxy,
set ``galaxy_home: /my/custom/path/to/galaxy`` in the user data. If the custom
path is not provided, just return the ``default_path``.
"""
path = None
try:
path = misc.load_yaml_file(USER_DATA_FILE).get(name, None)
if path is None:
downloaded_pd_file = 'pd.yaml'
if os.path.exists(downloaded_pd_file):
path = misc.load_yaml_file(downloaded_pd_file).get(
name, default_path)
except:
pass
if not path:
path = default_path
return path
def get_path(name, default_path):
"""
Get a file system path where a service with the given ``name`` resides/runs
as defined in the user data. For example, to use a custom path for Galaxy,
set ``galaxy_home: /my/custom/path/to/galaxy`` in the user data. If the custom
path is not provided, just return the ``default_path``.
"""
path = None
try:
path = misc.load_yaml_file(USER_DATA_FILE).get(name, None)
if path is None:
downloaded_pd_file = 'pd.yaml'
if os.path.exists(downloaded_pd_file):
path = misc.load_yaml_file(
downloaded_pd_file).get(name, default_path)
except:
pass
if not path:
path = default_path
return path
def get_user_data(self, force=False):
""" Override this method in a cloud-specific interface if the
default approach does not apply.
NOTE that this method should call the set_configuration method!
:type force: boolean
:param force: If set to True, reload the user data regardless if
the data is already stored in the class field
:rtype: dict
:return: A key-value dictionary containing the user data.
"""
if self.user_data is None or force:
self.user_data = misc.load_yaml_file(paths.USER_DATA_FILE)
self.set_configuration()
return self.user_data
def get_user_data(self, force=False):
""" Override this method in a cloud-specific interface if the
default approach does not apply.
NOTE that this method should call the set_configuration method!
:type force: boolean
:param force: If set to True, reload the user data regardless if
the data is already stored in the class field
:rtype: dict
:return: A key-value dictionary containing the user data.
"""
if self.user_data is None or force:
self.user_data = misc.load_yaml_file(paths.USER_DATA_FILE)
self.set_configuration()
return self.user_data
def __init__(self, **kwargs):
self.config_dict = kwargs
self.root = kwargs.get('root_dir', '.')
# Where dataset files are stored
self.id_secret = kwargs.get(
"id_secret", "USING THE DEFAULT IS NOT SECURE!")
self.use_remote_user = string_as_bool(
kwargs.get("use_remote_user", "False"))
self.require_login = string_as_bool(
kwargs.get("require_login", "False"))
self.template_path = resolve_path(
kwargs.get("template_path", "templates"), self.root)
self.cloudman_source_file_name = kwargs.get(
"cloudman_file_name", "cm.tar.gz")
# self.template_cache = resolve_path( kwargs.get(
# "template_cache_path", "database/reports/compiled_templates" ),
# self.root )
self.sendmail_path = kwargs.get('sendmail_path', "/usr/sbin/sendmail")
self.brand = kwargs.get('brand', None)
self.wiki_url = kwargs.get('wiki_url', "http://g2.trac.bx.psu.edu/")
self.GC_url = kwargs.get(
'GC_url', 'http://bitbucket.org/afgane/galaxy-central-gc2/')
self.CM_url = kwargs.get(
'GC_url', 'http://bitbucket.org/galaxy/cloudman/')
self.bugs_email = kwargs.get(
'bugs_email', "mailto:[email protected]")
self.blog_url = kwargs.get(
'blog_url', "http://g2.trac.bx.psu.edu/blog")
self.screencasts_url = kwargs.get(
'screencasts_url', "http://g2.trac.bx.psu.edu/wiki/ScreenCasts")
# Parse global_conf
global_conf = kwargs.get('global_conf', None)
global_conf_parser = ConfigParser.ConfigParser()
if global_conf and "__file__" in global_conf:
global_conf_parser.read(global_conf['__file__'])
# Load supported image configuration from a file on the image.
# This is a dict containing a list of configurations supported by
# the by system/image. Primarily, this contains a list
# of apps (available under key 'apps'). This config file allows CloudMan
# to integrate native and custom support for additional applications.
# The dict contains default values for backward combatibility.
self.ic = {'apps': ['cloudman', 'galaxy']}
if os.path.exists(paths.IMAGE_CONF_SUPPORT_FILE):
self.ic = misc.load_yaml_file(paths.IMAGE_CONF_SUPPORT_FILE)
# Logger is not configured yet so print
print "Image configuration suports: %s" % self.ic
self.instance_types = []
def __init__(self, **kwargs):
print "Python version: ", sys.version_info[:2]
self.PERSISTENT_DATA_VERSION = 3 # Current expected and generated PD version
self.DEPLOYMENT_VERSION = 2
# Instance persistent data file. This file gets created for
# test/transient cluster types and stores the cluster config. In case
# of a reboot, read the file to automatically recreate the services.
self.INSTANCE_PD_FILE = '/mnt/persistent_data-current.yaml'
cc = CloudConfig(app=self)
# Get the type of cloud currently running on
self.cloud_type = cc.get_cloud_type()
# Create an appropriate cloud connection
self.cloud_interface = cc.get_cloud_interface(self.cloud_type)
# Read config file and check for errors
self.config = config.Configuration(self, kwargs, self.cloud_interface.get_user_data())
# From user data determine if object store (S3) should be used.
self.use_object_store = self.config.get("use_object_store", True)
# From user data determine if block storage (EBS/nova-volume) should be used.
# (OpenNebula and dummy clouds do not support volumes yet so skip those)
self.use_volumes = self.config.get(
"use_volumes", self.cloud_type not in ['opennebula', 'dummy'])
# self.config.init_with_user_data(self.ud)
self.config.validate()
# Setup logging
self.logger = CMLogHandler()
if "testflag" in self.config:
self.TESTFLAG = bool(self.config['testflag'])
self.logger.setLevel(logging.DEBUG)
else:
self.TESTFLAG = False
self.logger.setLevel(logging.INFO)
if "localflag" in self.config:
self.LOCALFLAG = bool(self.config['localflag'])
self.logger.setLevel(logging.DEBUG)
else:
self.LOCALFLAG = False
self.logger.setLevel(logging.INFO)
log.addHandler(self.logger)
config.configure_logging(self.config)
log.debug("Initializing app")
log.debug("Running on '{0}' type of cloud in zone '{1}' using image '{2}'."
.format(self.cloud_type, self.cloud_interface.get_zone(),
self.cloud_interface.get_ami()))
# App-wide object to store messages that need to travel between the back-end
# and the UI.
# TODO: Ideally, this should be stored some form of more persistent
# medium (eg, database, file, session) and used as a simple module (vs. object)
# but that's hopefully still forthcoming.
self.msgs = messages.Messages()
# App-wide consecutive number generator. Starting at 1, each time `next`
# is called, get the next integer.
self.number_generator = misc.get_a_number()
# Check that we actually got user creds in user data and inform user
if not ('access_key' in self.config or 'secret_key' in self.config):
self.msgs.error("No access credentials provided in user data. "
"You will not be able to add any services.")
# Update user data to include persistent data stored in cluster's bucket, if it exists
# This enables cluster configuration to be recovered on cluster re-
# instantiation
self.manager = None
pd = None
if self.use_object_store and 'bucket_cluster' in self.config:
log.debug("Looking for existing cluster persistent data (PD).")
validate = True if self.cloud_type == 'ec2' else False
if not self.TESTFLAG and misc.get_file_from_bucket(
self.cloud_interface.get_s3_connection(),
self.config['bucket_cluster'],
'persistent_data.yaml', 'pd.yaml',
validate=validate):
log.debug("Loading bucket PD file pd.yaml")
pd = misc.load_yaml_file('pd.yaml')
# Have not found the file in the cluster bucket, look on the instance
if not pd:
if os.path.exists(self.INSTANCE_PD_FILE):
log.debug("Loading instance PD file {0}".format(self.INSTANCE_PD_FILE))
pd = misc.load_yaml_file(self.INSTANCE_PD_FILE)
if pd:
self.config.user_data = misc.merge_yaml_objects(self.config.user_data, pd)
self.config.user_data = misc.normalize_user_data(self, self.config.user_data)
else:
log.debug("No PD to go by. Setting deployment_version to {0}."
.format(self.DEPLOYMENT_VERSION))
# This is a new cluster so default to the current deployment version
self.config.user_data['deployment_version'] = self.DEPLOYMENT_VERSION
def __init__( self, **kwargs ):
print "Python version: ", sys.version_info[:2]
cc = CloudConfig(app=self)
# Get the type of cloud currently running on
self.cloud_type = cc.get_cloud_type()
# Create an approprite cloud connection
self.cloud_interface = cc.get_cloud_interface(self.cloud_type)
# Load user data into a local field through a cloud interface
self.ud = self.cloud_interface.get_user_data()
# From user data determine if object store (S3) should be used.
self.use_object_store = self.ud.get("use_object_store", True)
# Read config file and check for errors
self.config = config.Configuration( **kwargs )
self.config.check()
# Setup logging
self.logger = CMLogHandler(self)
if self.ud.has_key("testflag"):
self.TESTFLAG = bool(self.ud['testflag'])
self.logger.setLevel(logging.DEBUG)
else:
self.TESTFLAG = False
self.logger.setLevel(logging.INFO)
if self.ud.has_key("localflag"):
self.LOCALFLAG = bool(self.ud['localflag'])
self.logger.setLevel(logging.DEBUG)
else:
self.LOCALFLAG = False
self.logger.setLevel(logging.INFO)
log.addHandler(self.logger)
config.configure_logging(self.config)
log.debug( "Initializing app" )
log.debug("Running on '{0}' type of cloud.".format(self.cloud_type))
# App-wide object to store messages that need to travel between the back-end
# and the UI.
# TODO: Ideally, this should be stored some form of more persistent
# medium (eg, database, file, session) and used as a simple module (vs. object)
# but that's hopefully still forthcoming.
self.msgs = messages.Messages()
# Check that we actually got user creds in user data and inform user
if not ('access_key' in self.ud or 'secret_key' in self.ud):
self.msgs.error("No access credentials provided in user data. "
"You will not be able to add any services.")
# Update user data to include persistent data stored in cluster's bucket, if it exists
# This enables cluster configuration to be recovered on cluster re-instantiation
self.manager = None
if self.use_object_store and self.ud.has_key('bucket_cluster'):
log.debug("Getting pd.yaml")
if misc.get_file_from_bucket(self.cloud_interface.get_s3_connection(), self.ud['bucket_cluster'], 'persistent_data.yaml', 'pd.yaml'):
pd = misc.load_yaml_file('pd.yaml')
self.ud = misc.merge_yaml_objects(self.ud, pd)
if self.ud.has_key('role'):
if self.ud['role'] == 'master':
log.info( "Master starting" )
from cm.util import master
self.manager = master.ConsoleManager(self)
elif self.ud['role'] == 'worker':
log.info( "Worker starting" )
from cm.util import worker
self.manager = worker.ConsoleManager(self)
self.manager.console_monitor.start()
else:
log.error("************ No ROLE in %s - this is a fatal error. ************" % paths.USER_DATA_FILE)
def __call__(self, environ, username, password):
if not self._pass:
ud = misc.load_yaml_file(paths.USER_DATA_FILE)
self._pass = ud['password']
return self._pass == password
def wrap_in_middleware(app, global_conf, **local_conf):
"""Based on the configuration wrap `app` in a set of common and useful middleware."""
# Merge the global and local configurations
conf = global_conf.copy()
conf.update(local_conf)
debug = asbool(conf.get('debug', False))
# First put into place httpexceptions, which must be most closely
# wrapped around the application (it can interact poorly with
# other middleware):
app = httpexceptions.make_middleware(app, conf)
log.debug("Enabling 'httpexceptions' middleware")
# The recursive middleware allows for including requests in other
# requests or forwarding of requests, all on the server side.
if asbool(conf.get('use_recursive', True)):
from paste import recursive
app = recursive.RecursiveMiddleware(app, conf)
log.debug("Enabling 'recursive' middleware")
# Various debug middleware that can only be turned on if the debug
# flag is set, either because they are insecure or greatly hurt
# performance
if debug:
# Middleware to check for WSGI compliance
if asbool(conf.get('use_lint', True)):
from paste import lint
app = lint.make_middleware(app, conf)
log.debug("Enabling 'lint' middleware")
# Middleware to run the python profiler on each request
if asbool(conf.get('use_profile', False)):
import profile
app = profile.ProfileMiddleware(app, conf)
log.debug("Enabling 'profile' middleware")
# Middleware that intercepts print statements and shows them on the
# returned page
if asbool(conf.get('use_printdebug', True)):
from paste.debug import prints
app = prints.PrintDebugMiddleware(app, conf)
log.debug("Enabling 'print debug' middleware")
if debug and asbool(conf.get('use_interactive', False)):
# Interactive exception debugging, scary dangerous if publicly
# accessible, if not enabled we'll use the regular error printing
# middleware.
pkg_resources.require("WebError")
from weberror import evalexception
app = evalexception.EvalException(
app, conf, templating_formatters=build_template_error_formatters())
log.debug("Enabling 'eval exceptions' middleware")
else:
# Not in interactive debug mode, just use the regular error middleware
from paste.exceptions import errormiddleware
app = errormiddleware.ErrorMiddleware(
app, conf, show_exceptions_in_wsgi_errors=True, error_log=log)
log.debug("Enabling 'error' middleware")
# Transaction logging (apache access.log style)
if asbool(conf.get('use_translogger', True)):
from paste.translogger import TransLogger
app = TransLogger(app)
log.debug("Enabling 'trans logger' middleware")
# Config middleware just stores the paste config along with the request,
# not sure we need this but useful
from paste.deploy.config import ConfigMiddleware
app = ConfigMiddleware(app, conf)
log.debug("Enabling 'config' middleware")
# X-Forwarded-Host handling
from cm.framework.middleware.xforwardedhost import XForwardedHostMiddleware
app = XForwardedHostMiddleware(app)
log.debug("Enabling 'x-forwarded-host' middleware")
# Paste digest authentication
ud = misc.load_yaml_file(paths.USER_DATA_FILE)
if ud.get('password', ''):
from paste.auth.basic import AuthBasicHandler
app = AuthBasicHandler(app, 'CM Administration', cm_authfunc)
return app
def wrap_in_middleware(app, global_conf, **local_conf):
"""Based on the configuration wrap `app` in a set of common and useful middleware."""
# Merge the global and local configurations
conf = global_conf.copy()
conf.update(local_conf)
debug = asbool(conf.get('debug', False))
# First put into place httpexceptions, which must be most closely
# wrapped around the application (it can interact poorly with
# other middleware):
app = httpexceptions.make_middleware(app, conf)
log.debug("Enabling 'httpexceptions' middleware")
# The recursive middleware allows for including requests in other
# requests or forwarding of requests, all on the server side.
if asbool(conf.get('use_recursive', True)):
from paste import recursive
app = recursive.RecursiveMiddleware(app, conf)
log.debug("Enabling 'recursive' middleware")
# Various debug middleware that can only be turned on if the debug
# flag is set, either because they are insecure or greatly hurt
# performance
if debug:
# Middleware to check for WSGI compliance
if asbool(conf.get('use_lint', True)):
from paste import lint
app = lint.make_middleware(app, conf)
log.debug("Enabling 'lint' middleware")
# Middleware to run the python profiler on each request
if asbool(conf.get('use_profile', False)):
import profile
app = profile.ProfileMiddleware(app, conf)
log.debug("Enabling 'profile' middleware")
# Middleware that intercepts print statements and shows them on the
# returned page
if asbool(conf.get('use_printdebug', True)):
from paste.debug import prints
app = prints.PrintDebugMiddleware(app, conf)
log.debug("Enabling 'print debug' middleware")
if debug and asbool(conf.get('use_interactive', False)):
# Interactive exception debugging, scary dangerous if publicly
# accessible, if not enabled we'll use the regular error printing
# middleware.
pkg_resources.require("WebError")
from weberror import evalexception
app = evalexception.EvalException(app, conf,
templating_formatters=build_template_error_formatters())
log.debug("Enabling 'eval exceptions' middleware")
else:
# Not in interactive debug mode, just use the regular error middleware
from paste.exceptions import errormiddleware
app = errormiddleware.ErrorMiddleware(app, conf, show_exceptions_in_wsgi_errors=True, error_log=log)
log.debug("Enabling 'error' middleware")
# Transaction logging (apache access.log style)
if asbool(conf.get('use_translogger', True)):
from paste.translogger import TransLogger
app = TransLogger(app)
log.debug("Enabling 'trans logger' middleware")
# Config middleware just stores the paste config along with the request,
# not sure we need this but useful
from paste.deploy.config import ConfigMiddleware
app = ConfigMiddleware(app, conf)
log.debug("Enabling 'config' middleware")
# X-Forwarded-Host handling
from cm.framework.middleware.xforwardedhost import XForwardedHostMiddleware
app = XForwardedHostMiddleware(app)
log.debug("Enabling 'x-forwarded-host' middleware")
# Paste digest authentication
ud = misc.load_yaml_file(paths.USER_DATA_FILE)
if ud.get('password', ''):
from paste.auth.basic import AuthBasicHandler
app = AuthBasicHandler(app, 'CM Administration', cm_authfunc)
return app
def __call__(self, environ, username, password):
if not self._pass:
ud = misc.load_yaml_file(paths.USER_DATA_FILE)
self._pass = ud['password']
return self._pass == password
def cm_authfunc(environ, username, password):
ud = misc.load_yaml_file(paths.USER_DATA_FILE)
if 'password' in ud:
if password == str(ud['password']):
return True
return False
def __init__(self, **kwargs):
print "Python version: ", sys.version_info[:2]
self.PERSISTENT_DATA_VERSION = 3 # Current expected and generated PD version
self.DEPLOYMENT_VERSION = 2
cc = CloudConfig(app=self)
# Get the type of cloud currently running on
self.cloud_type = cc.get_cloud_type()
# Create an appropriate cloud connection
self.cloud_interface = cc.get_cloud_interface(self.cloud_type)
# Load user data into a local field through a cloud interface
self.ud = self.cloud_interface.get_user_data()
# From user data determine if object store (S3) should be used.
self.use_object_store = self.ud.get("use_object_store", True)
# From user data determine if block storage (EBS/nova-volume) should be used.
# (OpenNebula and dummy clouds do not support volumes yet so skip those)
self.use_volumes = self.ud.get(
"use_volumes", self.cloud_type not in ['opennebula', 'dummy'])
# Read config file and check for errors
self.config = config.Configuration(**kwargs)
self.config.init_with_user_data(self.ud)
self.config.check()
# Setup logging
self.logger = CMLogHandler(self)
if "testflag" in self.ud:
self.TESTFLAG = bool(self.ud['testflag'])
self.logger.setLevel(logging.DEBUG)
else:
self.TESTFLAG = False
self.logger.setLevel(logging.INFO)
if "localflag" in self.ud:
self.LOCALFLAG = bool(self.ud['localflag'])
self.logger.setLevel(logging.DEBUG)
else:
self.LOCALFLAG = False
self.logger.setLevel(logging.INFO)
log.addHandler(self.logger)
config.configure_logging(self.config, self.ud)
log.debug("Initializing app")
log.debug("Running on '{0}' type of cloud in zone '{1}' using image '{2}'."
.format(self.cloud_type, self.cloud_interface.get_zone(),
self.cloud_interface.get_ami()))
# App-wide object to store messages that need to travel between the back-end
# and the UI.
# TODO: Ideally, this should be stored some form of more persistent
# medium (eg, database, file, session) and used as a simple module (vs. object)
# but that's hopefully still forthcoming.
self.msgs = messages.Messages()
# Check that we actually got user creds in user data and inform user
if not ('access_key' in self.ud or 'secret_key' in self.ud):
self.msgs.error("No access credentials provided in user data. "
"You will not be able to add any services.")
# Update user data to include persistent data stored in cluster's bucket, if it exists
# This enables cluster configuration to be recovered on cluster re-
# instantiation
self.manager = None
if self.use_object_store and 'bucket_cluster' in self.ud:
log.debug("Getting pd.yaml")
if misc.get_file_from_bucket(self.cloud_interface.get_s3_connection(),
self.ud['bucket_cluster'], 'persistent_data.yaml', 'pd.yaml'):
pd = misc.load_yaml_file('pd.yaml')
self.ud = misc.merge_yaml_objects(self.ud, pd)
self.ud = misc.normalize_user_data(self, self.ud)
else:
log.debug("Setting deployment_version to {0}".format(self.DEPLOYMENT_VERSION))
# This is a new cluster so default to the current version
self.ud['deployment_version'] = self.DEPLOYMENT_VERSION
