def get_path(name, default_path): path = None try: path = misc.load_yaml_file(USER_DATA_FILE).get(name, None) if path is None: downloaded_pd_file = 'pd.yaml' if os.path.exists(downloaded_pd_file): path = misc.load_yaml_file(downloaded_pd_file).get(name, default_path) except: pass if not path: path = default_path return path
def get_user_data(self, force=False): if self.user_data is None or force: self.user_data = misc.load_yaml_file(paths.USER_DATA_FILE) self.aws_access_key = self.user_data.get("access_key", None) self.aws_secret_key = self.user_data.get("secret_key", None) self.s3_url = self.user_data.get("s3_url", None) self.ec2_url = self.user_data.get("ec2_url", None) return self.user_data
def get_user_data(self, force=False): if self.user_data is None or force: self.user_data = misc.load_yaml_file(paths.USER_DATA_FILE) self.aws_access_key = self.user_data.get('access_key', None) self.aws_secret_key = self.user_data.get('secret_key', None) self.s3_url = self.user_data.get('s3_url', None) self.ec2_url = self.user_data.get('ec2_url', None) return self.user_data
def get_cloud_type(self): """ Determine the type of cloud currently being running on. If needed, this method should be extended to include other (or alternative) methods for detecting the type of cloud. The current assumption is that the type of cloud is included as part of user data and stored in paths.USER_DATA_FILE file under 'cloud_type' key. """ return misc.load_yaml_file(paths.USER_DATA_FILE).get('cloud_type', 'ec2').lower()
def get_cloud_type(self): """ Determine the type of cloud currently being running on. If needed, this method should be extended to include other (or alternative) methods for detecting the type of cloud. The current assumption is that the type of cloud is included as part of user data and stored in paths.USER_DATA_FILE file under 'cloud_type' key. """ return misc.load_yaml_file(paths.USER_DATA_FILE).get("cloud_type", "ec2").lower()
def get_path(name, default_path): """ Get a file system path where a service with the given ``name`` resides/runs as defined in the user data. For example, to use a custom path for Galaxy, set ``galaxy_home: /my/custom/path/to/galaxy`` in the user data. If the custom path is not provided, just return the ``default_path``. """ path = None try: path = misc.load_yaml_file(USER_DATA_FILE).get(name, None) if path is None: downloaded_pd_file = 'pd.yaml' if os.path.exists(downloaded_pd_file): path = misc.load_yaml_file(downloaded_pd_file).get( name, default_path) except: pass if not path: path = default_path return path
def get_path(name, default_path): """ Get a file system path where a service with the given ``name`` resides/runs as defined in the user data. For example, to use a custom path for Galaxy, set ``galaxy_home: /my/custom/path/to/galaxy`` in the user data. If the custom path is not provided, just return the ``default_path``. """ path = None try: path = misc.load_yaml_file(USER_DATA_FILE).get(name, None) if path is None: downloaded_pd_file = 'pd.yaml' if os.path.exists(downloaded_pd_file): path = misc.load_yaml_file( downloaded_pd_file).get(name, default_path) except: pass if not path: path = default_path return path
def get_user_data(self, force=False): """ Override this method in a cloud-specific interface if the default approach does not apply. NOTE that this method should call the set_configuration method! :type force: boolean :param force: If set to True, reload the user data regardless if the data is already stored in the class field :rtype: dict :return: A key-value dictionary containing the user data. """ if self.user_data is None or force: self.user_data = misc.load_yaml_file(paths.USER_DATA_FILE) self.set_configuration() return self.user_data
def __init__(self, **kwargs): self.config_dict = kwargs self.root = kwargs.get('root_dir', '.') # Where dataset files are stored self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!") self.use_remote_user = string_as_bool( kwargs.get("use_remote_user", "False")) self.require_login = string_as_bool( kwargs.get("require_login", "False")) self.template_path = resolve_path( kwargs.get("template_path", "templates"), self.root) self.cloudman_source_file_name = kwargs.get( "cloudman_file_name", "cm.tar.gz") # self.template_cache = resolve_path( kwargs.get( # "template_cache_path", "database/reports/compiled_templates" ), # self.root ) self.sendmail_path = kwargs.get('sendmail_path', "/usr/sbin/sendmail") self.brand = kwargs.get('brand', None) self.wiki_url = kwargs.get('wiki_url', "http://g2.trac.bx.psu.edu/") self.GC_url = kwargs.get( 'GC_url', 'http://bitbucket.org/afgane/galaxy-central-gc2/') self.CM_url = kwargs.get( 'GC_url', 'http://bitbucket.org/galaxy/cloudman/') self.bugs_email = kwargs.get( 'bugs_email', "mailto:[email protected]") self.blog_url = kwargs.get( 'blog_url', "http://g2.trac.bx.psu.edu/blog") self.screencasts_url = kwargs.get( 'screencasts_url', "http://g2.trac.bx.psu.edu/wiki/ScreenCasts") # Parse global_conf global_conf = kwargs.get('global_conf', None) global_conf_parser = ConfigParser.ConfigParser() if global_conf and "__file__" in global_conf: global_conf_parser.read(global_conf['__file__']) # Load supported image configuration from a file on the image. # This is a dict containing a list of configurations supported by # the by system/image. Primarily, this contains a list # of apps (available under key 'apps'). This config file allows CloudMan # to integrate native and custom support for additional applications. # The dict contains default values for backward combatibility. self.ic = {'apps': ['cloudman', 'galaxy']} if os.path.exists(paths.IMAGE_CONF_SUPPORT_FILE): self.ic = misc.load_yaml_file(paths.IMAGE_CONF_SUPPORT_FILE) # Logger is not configured yet so print print "Image configuration suports: %s" % self.ic self.instance_types = []
def __init__(self, **kwargs): print "Python version: ", sys.version_info[:2] self.PERSISTENT_DATA_VERSION = 3 # Current expected and generated PD version self.DEPLOYMENT_VERSION = 2 # Instance persistent data file. This file gets created for # test/transient cluster types and stores the cluster config. In case # of a reboot, read the file to automatically recreate the services. self.INSTANCE_PD_FILE = '/mnt/persistent_data-current.yaml' cc = CloudConfig(app=self) # Get the type of cloud currently running on self.cloud_type = cc.get_cloud_type() # Create an appropriate cloud connection self.cloud_interface = cc.get_cloud_interface(self.cloud_type) # Read config file and check for errors self.config = config.Configuration(self, kwargs, self.cloud_interface.get_user_data()) # From user data determine if object store (S3) should be used. self.use_object_store = self.config.get("use_object_store", True) # From user data determine if block storage (EBS/nova-volume) should be used. # (OpenNebula and dummy clouds do not support volumes yet so skip those) self.use_volumes = self.config.get( "use_volumes", self.cloud_type not in ['opennebula', 'dummy']) # self.config.init_with_user_data(self.ud) self.config.validate() # Setup logging self.logger = CMLogHandler() if "testflag" in self.config: self.TESTFLAG = bool(self.config['testflag']) self.logger.setLevel(logging.DEBUG) else: self.TESTFLAG = False self.logger.setLevel(logging.INFO) if "localflag" in self.config: self.LOCALFLAG = bool(self.config['localflag']) self.logger.setLevel(logging.DEBUG) else: self.LOCALFLAG = False self.logger.setLevel(logging.INFO) log.addHandler(self.logger) config.configure_logging(self.config) log.debug("Initializing app") log.debug("Running on '{0}' type of cloud in zone '{1}' using image '{2}'." .format(self.cloud_type, self.cloud_interface.get_zone(), self.cloud_interface.get_ami())) # App-wide object to store messages that need to travel between the back-end # and the UI. # TODO: Ideally, this should be stored some form of more persistent # medium (eg, database, file, session) and used as a simple module (vs. object) # but that's hopefully still forthcoming. self.msgs = messages.Messages() # App-wide consecutive number generator. Starting at 1, each time `next` # is called, get the next integer. self.number_generator = misc.get_a_number() # Check that we actually got user creds in user data and inform user if not ('access_key' in self.config or 'secret_key' in self.config): self.msgs.error("No access credentials provided in user data. " "You will not be able to add any services.") # Update user data to include persistent data stored in cluster's bucket, if it exists # This enables cluster configuration to be recovered on cluster re- # instantiation self.manager = None pd = None if self.use_object_store and 'bucket_cluster' in self.config: log.debug("Looking for existing cluster persistent data (PD).") validate = True if self.cloud_type == 'ec2' else False if not self.TESTFLAG and misc.get_file_from_bucket( self.cloud_interface.get_s3_connection(), self.config['bucket_cluster'], 'persistent_data.yaml', 'pd.yaml', validate=validate): log.debug("Loading bucket PD file pd.yaml") pd = misc.load_yaml_file('pd.yaml') # Have not found the file in the cluster bucket, look on the instance if not pd: if os.path.exists(self.INSTANCE_PD_FILE): log.debug("Loading instance PD file {0}".format(self.INSTANCE_PD_FILE)) pd = misc.load_yaml_file(self.INSTANCE_PD_FILE) if pd: self.config.user_data = misc.merge_yaml_objects(self.config.user_data, pd) self.config.user_data = misc.normalize_user_data(self, self.config.user_data) else: log.debug("No PD to go by. Setting deployment_version to {0}." .format(self.DEPLOYMENT_VERSION)) # This is a new cluster so default to the current deployment version self.config.user_data['deployment_version'] = self.DEPLOYMENT_VERSION
def __init__( self, **kwargs ): print "Python version: ", sys.version_info[:2] cc = CloudConfig(app=self) # Get the type of cloud currently running on self.cloud_type = cc.get_cloud_type() # Create an approprite cloud connection self.cloud_interface = cc.get_cloud_interface(self.cloud_type) # Load user data into a local field through a cloud interface self.ud = self.cloud_interface.get_user_data() # From user data determine if object store (S3) should be used. self.use_object_store = self.ud.get("use_object_store", True) # Read config file and check for errors self.config = config.Configuration( **kwargs ) self.config.check() # Setup logging self.logger = CMLogHandler(self) if self.ud.has_key("testflag"): self.TESTFLAG = bool(self.ud['testflag']) self.logger.setLevel(logging.DEBUG) else: self.TESTFLAG = False self.logger.setLevel(logging.INFO) if self.ud.has_key("localflag"): self.LOCALFLAG = bool(self.ud['localflag']) self.logger.setLevel(logging.DEBUG) else: self.LOCALFLAG = False self.logger.setLevel(logging.INFO) log.addHandler(self.logger) config.configure_logging(self.config) log.debug( "Initializing app" ) log.debug("Running on '{0}' type of cloud.".format(self.cloud_type)) # App-wide object to store messages that need to travel between the back-end # and the UI. # TODO: Ideally, this should be stored some form of more persistent # medium (eg, database, file, session) and used as a simple module (vs. object) # but that's hopefully still forthcoming. self.msgs = messages.Messages() # Check that we actually got user creds in user data and inform user if not ('access_key' in self.ud or 'secret_key' in self.ud): self.msgs.error("No access credentials provided in user data. " "You will not be able to add any services.") # Update user data to include persistent data stored in cluster's bucket, if it exists # This enables cluster configuration to be recovered on cluster re-instantiation self.manager = None if self.use_object_store and self.ud.has_key('bucket_cluster'): log.debug("Getting pd.yaml") if misc.get_file_from_bucket(self.cloud_interface.get_s3_connection(), self.ud['bucket_cluster'], 'persistent_data.yaml', 'pd.yaml'): pd = misc.load_yaml_file('pd.yaml') self.ud = misc.merge_yaml_objects(self.ud, pd) if self.ud.has_key('role'): if self.ud['role'] == 'master': log.info( "Master starting" ) from cm.util import master self.manager = master.ConsoleManager(self) elif self.ud['role'] == 'worker': log.info( "Worker starting" ) from cm.util import worker self.manager = worker.ConsoleManager(self) self.manager.console_monitor.start() else: log.error("************ No ROLE in %s - this is a fatal error. ************" % paths.USER_DATA_FILE)
def __call__(self, environ, username, password): if not self._pass: ud = misc.load_yaml_file(paths.USER_DATA_FILE) self._pass = ud['password'] return self._pass == password
def wrap_in_middleware(app, global_conf, **local_conf): """Based on the configuration wrap `app` in a set of common and useful middleware.""" # Merge the global and local configurations conf = global_conf.copy() conf.update(local_conf) debug = asbool(conf.get('debug', False)) # First put into place httpexceptions, which must be most closely # wrapped around the application (it can interact poorly with # other middleware): app = httpexceptions.make_middleware(app, conf) log.debug("Enabling 'httpexceptions' middleware") # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. if asbool(conf.get('use_recursive', True)): from paste import recursive app = recursive.RecursiveMiddleware(app, conf) log.debug("Enabling 'recursive' middleware") # Various debug middleware that can only be turned on if the debug # flag is set, either because they are insecure or greatly hurt # performance if debug: # Middleware to check for WSGI compliance if asbool(conf.get('use_lint', True)): from paste import lint app = lint.make_middleware(app, conf) log.debug("Enabling 'lint' middleware") # Middleware to run the python profiler on each request if asbool(conf.get('use_profile', False)): import profile app = profile.ProfileMiddleware(app, conf) log.debug("Enabling 'profile' middleware") # Middleware that intercepts print statements and shows them on the # returned page if asbool(conf.get('use_printdebug', True)): from paste.debug import prints app = prints.PrintDebugMiddleware(app, conf) log.debug("Enabling 'print debug' middleware") if debug and asbool(conf.get('use_interactive', False)): # Interactive exception debugging, scary dangerous if publicly # accessible, if not enabled we'll use the regular error printing # middleware. pkg_resources.require("WebError") from weberror import evalexception app = evalexception.EvalException( app, conf, templating_formatters=build_template_error_formatters()) log.debug("Enabling 'eval exceptions' middleware") else: # Not in interactive debug mode, just use the regular error middleware from paste.exceptions import errormiddleware app = errormiddleware.ErrorMiddleware( app, conf, show_exceptions_in_wsgi_errors=True, error_log=log) log.debug("Enabling 'error' middleware") # Transaction logging (apache access.log style) if asbool(conf.get('use_translogger', True)): from paste.translogger import TransLogger app = TransLogger(app) log.debug("Enabling 'trans logger' middleware") # Config middleware just stores the paste config along with the request, # not sure we need this but useful from paste.deploy.config import ConfigMiddleware app = ConfigMiddleware(app, conf) log.debug("Enabling 'config' middleware") # X-Forwarded-Host handling from cm.framework.middleware.xforwardedhost import XForwardedHostMiddleware app = XForwardedHostMiddleware(app) log.debug("Enabling 'x-forwarded-host' middleware") # Paste digest authentication ud = misc.load_yaml_file(paths.USER_DATA_FILE) if ud.get('password', ''): from paste.auth.basic import AuthBasicHandler app = AuthBasicHandler(app, 'CM Administration', cm_authfunc) return app
def wrap_in_middleware(app, global_conf, **local_conf): """Based on the configuration wrap `app` in a set of common and useful middleware.""" # Merge the global and local configurations conf = global_conf.copy() conf.update(local_conf) debug = asbool(conf.get('debug', False)) # First put into place httpexceptions, which must be most closely # wrapped around the application (it can interact poorly with # other middleware): app = httpexceptions.make_middleware(app, conf) log.debug("Enabling 'httpexceptions' middleware") # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. if asbool(conf.get('use_recursive', True)): from paste import recursive app = recursive.RecursiveMiddleware(app, conf) log.debug("Enabling 'recursive' middleware") # Various debug middleware that can only be turned on if the debug # flag is set, either because they are insecure or greatly hurt # performance if debug: # Middleware to check for WSGI compliance if asbool(conf.get('use_lint', True)): from paste import lint app = lint.make_middleware(app, conf) log.debug("Enabling 'lint' middleware") # Middleware to run the python profiler on each request if asbool(conf.get('use_profile', False)): import profile app = profile.ProfileMiddleware(app, conf) log.debug("Enabling 'profile' middleware") # Middleware that intercepts print statements and shows them on the # returned page if asbool(conf.get('use_printdebug', True)): from paste.debug import prints app = prints.PrintDebugMiddleware(app, conf) log.debug("Enabling 'print debug' middleware") if debug and asbool(conf.get('use_interactive', False)): # Interactive exception debugging, scary dangerous if publicly # accessible, if not enabled we'll use the regular error printing # middleware. pkg_resources.require("WebError") from weberror import evalexception app = evalexception.EvalException(app, conf, templating_formatters=build_template_error_formatters()) log.debug("Enabling 'eval exceptions' middleware") else: # Not in interactive debug mode, just use the regular error middleware from paste.exceptions import errormiddleware app = errormiddleware.ErrorMiddleware(app, conf, show_exceptions_in_wsgi_errors=True, error_log=log) log.debug("Enabling 'error' middleware") # Transaction logging (apache access.log style) if asbool(conf.get('use_translogger', True)): from paste.translogger import TransLogger app = TransLogger(app) log.debug("Enabling 'trans logger' middleware") # Config middleware just stores the paste config along with the request, # not sure we need this but useful from paste.deploy.config import ConfigMiddleware app = ConfigMiddleware(app, conf) log.debug("Enabling 'config' middleware") # X-Forwarded-Host handling from cm.framework.middleware.xforwardedhost import XForwardedHostMiddleware app = XForwardedHostMiddleware(app) log.debug("Enabling 'x-forwarded-host' middleware") # Paste digest authentication ud = misc.load_yaml_file(paths.USER_DATA_FILE) if ud.get('password', ''): from paste.auth.basic import AuthBasicHandler app = AuthBasicHandler(app, 'CM Administration', cm_authfunc) return app
def cm_authfunc(environ, username, password): ud = misc.load_yaml_file(paths.USER_DATA_FILE) if 'password' in ud: if password == str(ud['password']): return True return False
def __init__(self, **kwargs): print "Python version: ", sys.version_info[:2] self.PERSISTENT_DATA_VERSION = 3 # Current expected and generated PD version self.DEPLOYMENT_VERSION = 2 cc = CloudConfig(app=self) # Get the type of cloud currently running on self.cloud_type = cc.get_cloud_type() # Create an appropriate cloud connection self.cloud_interface = cc.get_cloud_interface(self.cloud_type) # Load user data into a local field through a cloud interface self.ud = self.cloud_interface.get_user_data() # From user data determine if object store (S3) should be used. self.use_object_store = self.ud.get("use_object_store", True) # From user data determine if block storage (EBS/nova-volume) should be used. # (OpenNebula and dummy clouds do not support volumes yet so skip those) self.use_volumes = self.ud.get( "use_volumes", self.cloud_type not in ['opennebula', 'dummy']) # Read config file and check for errors self.config = config.Configuration(**kwargs) self.config.init_with_user_data(self.ud) self.config.check() # Setup logging self.logger = CMLogHandler(self) if "testflag" in self.ud: self.TESTFLAG = bool(self.ud['testflag']) self.logger.setLevel(logging.DEBUG) else: self.TESTFLAG = False self.logger.setLevel(logging.INFO) if "localflag" in self.ud: self.LOCALFLAG = bool(self.ud['localflag']) self.logger.setLevel(logging.DEBUG) else: self.LOCALFLAG = False self.logger.setLevel(logging.INFO) log.addHandler(self.logger) config.configure_logging(self.config, self.ud) log.debug("Initializing app") log.debug("Running on '{0}' type of cloud in zone '{1}' using image '{2}'." .format(self.cloud_type, self.cloud_interface.get_zone(), self.cloud_interface.get_ami())) # App-wide object to store messages that need to travel between the back-end # and the UI. # TODO: Ideally, this should be stored some form of more persistent # medium (eg, database, file, session) and used as a simple module (vs. object) # but that's hopefully still forthcoming. self.msgs = messages.Messages() # Check that we actually got user creds in user data and inform user if not ('access_key' in self.ud or 'secret_key' in self.ud): self.msgs.error("No access credentials provided in user data. " "You will not be able to add any services.") # Update user data to include persistent data stored in cluster's bucket, if it exists # This enables cluster configuration to be recovered on cluster re- # instantiation self.manager = None if self.use_object_store and 'bucket_cluster' in self.ud: log.debug("Getting pd.yaml") if misc.get_file_from_bucket(self.cloud_interface.get_s3_connection(), self.ud['bucket_cluster'], 'persistent_data.yaml', 'pd.yaml'): pd = misc.load_yaml_file('pd.yaml') self.ud = misc.merge_yaml_objects(self.ud, pd) self.ud = misc.normalize_user_data(self, self.ud) else: log.debug("Setting deployment_version to {0}".format(self.DEPLOYMENT_VERSION)) # This is a new cluster so default to the current version self.ud['deployment_version'] = self.DEPLOYMENT_VERSION