Beispiel #1
0
 def _test_path_option(self, option):
     self.assertRaises(ValueError, setattr, self.cnf, option, '/not/exists')
     setattr(self.cnf, option, '/tmp')
     c = Configuration('mongodb')
     c.read(self.cnf_path)
     self.assertEqual('/tmp', c.get(option))
     self.assertEqual('/tmp', getattr(self.cnf, option))
Beispiel #2
0
class Cassandra(object):

	def __init__(self):
		self._logger = logging.getLogger(__name__)
		self._initd = initdv2.lookup(SERVICE_NAME)		
		
		bus.on(reload=self.on_reload)
		self.on_reload()
		
	def on_reload(self):
		self.queryenv = bus.queryenv_service
		self.platform = bus.platform
		self.private_ip = self.platform.get_private_ip()
		self.zone = self.platform.get_avail_zone()
		
		cnf = bus.cnf 
		self.ini = cnf.rawini
		
		self.role_name = self.ini.get(config.SECT_GENERAL, config.OPT_ROLE_NAME)

		self.storage_path = self.ini.get(CNF_SECTION, OPT_STORAGE_PATH)
		self.storage_conf_path = self.ini.get(CNF_SECTION, OPT_STORAGE_CNF_PATH)

		self.data_file_directory = self.storage_path + "/datafile"
		self.commit_log_directory = self.storage_path + "/commitlog"
		
		self.cassandra_conf = Configuration('xml')
		try:
			self.cassandra_conf.read(self.storage_conf_path)
		except (OSError, MetaconfError, ParseError), e:
			self._logger.error('Cassandra storage-conf.xml is broken. %s' % e)
Beispiel #3
0
 def _test_path_option(self, option):
     self.assertRaises(ValueError, setattr, self.cnf, option, '/not/exists')
     setattr(self.cnf, option, '/tmp')
     c = Configuration('mongodb')
     c.read(self.cnf_path)
     self.assertEqual('/tmp', c.get(option))
     self.assertEqual('/tmp', getattr(self.cnf, option))
Beispiel #4
0
class Cassandra(object):

    def __init__(self):
        self._logger = logging.getLogger(__name__)
        self._initd = initdv2.lookup(SERVICE_NAME)

        bus.on(reload=self.on_reload)
        self.on_reload()

    def on_reload(self):
        self.queryenv = bus.queryenv_service
        self.platform = bus.platform
        self.private_ip = self.platform.get_private_ip()
        self.zone = self.platform.get_avail_zone()

        cnf = bus.cnf
        self.ini = cnf.rawini

        self.role_name = self.ini.get(config.SECT_GENERAL, config.OPT_ROLE_NAME)

        self.storage_path = self.ini.get(CNF_SECTION, OPT_STORAGE_PATH)
        self.storage_conf_path = self.ini.get(CNF_SECTION, OPT_STORAGE_CNF_PATH)

        self.data_file_directory = self.storage_path + "/datafile"
        self.commit_log_directory = self.storage_path + "/commitlog"

        self.cassandra_conf = Configuration('xml')
        try:
            self.cassandra_conf.read(self.storage_conf_path)
        except (OSError, MetaconfError, ParseError), e:
            self._logger.error('Cassandra storage-conf.xml is broken. %s' % e)
Beispiel #5
0
	def get_system_variables(self):
		#TESTING REQUIRED!
		conf = Configuration(self._config_format)
		conf.read(self._config_path)
		vars = {}
		for section in conf.sections('./'):
			vars[section] = conf.get(section)
		return vars
Beispiel #6
0
 def __init__(self, body):
     config = Configuration("apache")
     try:
         config.reads(str(body))
     except ParseError, e:
         LOG.error("MetaConf failed to parse Apache VirtualHost body: \n%s" % body)
         e._err = body + "\n" + e._err
         raise
Beispiel #7
0
    def _https_config_exists(self):
        config_dir = os.path.dirname(self.api.app_inc_path)
        conf_path = os.path.join(config_dir, 'https.include')

        config = None
        try:
            config = Configuration('nginx')
            config.read(conf_path)
        except (Exception, BaseException), e:
            raise HandlerError('Cannot read/parse nginx main configuration file: %s' % str(e))
Beispiel #8
0
 def __init__(self, body):
     config = Configuration("apache")
     try:
         config.reads(str(body))
     except ParseError, e:
         LOG.error(
             "MetaConf failed to parse Apache VirtualHost body: \n%s" %
             body)
         e._err = body + "\n" + e._err
         raise
Beispiel #9
0
    def _https_config_exists(self):
        config_dir = os.path.dirname(self.api.app_inc_path)
        conf_path = os.path.join(config_dir, 'https.include')

        config = None
        try:
            config = Configuration('nginx')
            config.read(conf_path)
        except (Exception, BaseException), e:
            raise HandlerError('Cannot read/parse nginx main configuration file: %s' % str(e))
Beispiel #10
0
    def _update_main_config(self, remove_server_section=True, reload_service=True):
        config_dir = os.path.dirname(self.api.app_inc_path)
        nginx_conf_path = os.path.join(config_dir, 'nginx.conf')

        config = None
        try:
            config = Configuration('nginx')
            config.read(nginx_conf_path)
        except (Exception, BaseException), e:
            raise HandlerError('Cannot read/parse nginx main configuration file: %s' % str(e))
Beispiel #11
0
    def _reload_backends(self):
        self._logger.info('Updating mysql-proxy backends list')
        self.config = Configuration('mysql')
        if os.path.exists(CONFIG_FILE_PATH):
            self.config.read(CONFIG_FILE_PATH)
            self.config.remove('./mysql-proxy/proxy-backend-addresses')
            self.config.remove(
                './mysql-proxy/proxy-read-only-backend-addresses')

        try:
            self.config.get('./mysql-proxy')
        except NoPathError:
            self.config.add('./mysql-proxy')

        queryenv = bus.queryenv_service
        roles = queryenv.list_roles()
        master = None
        slaves = []

        for role in roles:
            if not is_mysql_role(role.behaviour):
                continue

            for host in role.hosts:
                ip = host.internal_ip or host.external_ip
                if host.replication_master:
                    master = ip
                else:
                    slaves.append(ip)

        if master:
            self._logger.debug(
                'Adding mysql master %s to  mysql-proxy defaults file', master)
            self.config.add('./mysql-proxy/proxy-backend-addresses',
                            '%s:3306' % master)
        if slaves:
            self._logger.debug(
                'Adding mysql slaves to  mysql-proxy defaults file: %s',
                ', '.join(slaves))
            for slave in slaves:
                self.config.add(
                    './mysql-proxy/proxy-read-only-backend-addresses',
                    '%s:3306' % slave)

        self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True)
        self.config.set('./mysql-proxy/daemon', 'true', force=True)
        self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True)
        if self.service.version > (0, 8, 0):
            self.config.set('./mysql-proxy/plugins', 'proxy', force=True)

        self._logger.debug('Saving new mysql-proxy defaults file')
        self.config.write(CONFIG_FILE_PATH)
        os.chmod(CONFIG_FILE_PATH, 0660)

        self.service.restart()
Beispiel #12
0
	def load(self, preset_type):
		'''
		@rtype: Preset
		@raise OSError: When cannot read preset file
		@raise MetaconfError: When experience problems with preset file parsing
		'''
		self._logger.debug('Loading %s %s preset' % (preset_type, self.service_name))
		ini = Configuration('ini')
		ini.read(self._filename(preset_type))
		
		return CnfPreset(ini.get('general/name'), dict(ini.items('settings/'))) 
Beispiel #13
0
    def _main_config_contains_server(self):
        config_dir = os.path.dirname(self.api.app_inc_path)
        nginx_conf_path = os.path.join(config_dir, 'nginx.conf')

        config = None
        result = False
        try:
            config = Configuration('nginx')
            config.read(nginx_conf_path)
        except (Exception, BaseException), e:
            raise HandlerError('Cannot read/parse nginx main configuration file: %s' % str(e))
Beispiel #14
0
	def __init__(self, manifest_path):
		self._options = []
		ini = Configuration('ini')
		ini.read(manifest_path)
		try:
			self._defaults = dict(ini.items('__defaults__'))
		except NoPathError:
			self._defaults = dict()
		
		for name in ini.sections("./"):
			if name == '__defaults__':
				continue
			self._options.append(_OptionSpec.from_ini(ini, name, self._defaults))
Beispiel #15
0
    def _test_option(self, option):
        setattr(self.cnf, option, 'value')
        c = Configuration('mongodb')
        c.read(self.cnf_path)
        self.assertEqual('value', c.get(option))
 
        setattr(self.cnf, option, None)
        c = Configuration('mongodb')
        c.read(self.cnf_path)
        self.assertRaises(NoPathError, c.get, option)
Beispiel #16
0
    def download_and_restore(self, volume, snapshot, tranzit_path):
        # Load manifest
        clear_queue(self._writer_queue)
        clear_queue(self._download_queue)
        self._download_finished.clear()
        transfer = self._transfer_cls()
        mnf_path = transfer.download(snapshot.path, tranzit_path)
        mnf = Configuration('ini')
        mnf.read(mnf_path)

        volume.fs_created = False
        volume.mkfs(snapshot.fstype)

        remote_path = os.path.dirname(snapshot.path)
        # Get links with md5 sums
        links = [(os.path.join(remote_path, chunk[0]), chunk[1])
                 for chunk in mnf.items('chunks')]
        links.sort()

        # Download 2 first chunks
        for link in links[:2]:
            transfer.download(link[0], tranzit_path)
            chunk_path = os.path.join(tranzit_path, os.path.basename(link[0]))
            if self._md5sum(chunk_path) != link[1]:
                raise Exception("md5sum of chunk %s is not correct." %
                                chunk_path)
            self._writer_queue.put(chunk_path)

        if hasattr(snapshot,
                   'snap_strategy') and snapshot.snap_strategy == 'data':
            restore_strategy = DataRestoreStrategy(self._logger)
        else:
            restore_strategy = DeviceRestoreStrategy(self._logger)

        writer = threading.Thread(target=restore_strategy.restore,
                                  name='writer',
                                  args=(self._writer_queue, volume,
                                        self._download_finished))
        writer.start()

        # Add remaining files to download queue
        for link in links[2:]:
            self._download_queue.put(link)

        downloader = threading.Thread(name="Downloader",
                                      target=self._downloader,
                                      args=(tranzit_path, ))
        downloader.start()
        downloader.join()
        writer.join()
Beispiel #17
0
class ApacheConfigManager(object):

    _cnf = None
    path = None

    def __init__(self, path):
        self._cnf = Configuration("apache")
        self.path = path

    def __enter__(self):
        self._cnf.read(self.path)
        return self._cnf

    def __exit__(self, type, value, traceback):
        self._cnf.write(self.path)
Beispiel #18
0
class ApacheConfigManager(object):

    _cnf = None
    path = None

    def __init__(self, path):
        self._cnf = Configuration("apache")
        self.path = path

    def __enter__(self):
        self._cnf.read(self.path)
        return self._cnf

    def __exit__(self, type, value, traceback):
        self._cnf.write(self.path)
Beispiel #19
0
    def _manifest(self):
        f_manifest = CnfController._manifest
        base_manifest = f_manifest.fget(self)
        path = self._manifest_path

        s = {}
        out = None

        if not self._merged_manifest:
            cmd = '%s --no-defaults --verbose SU_EXEC' % mysql_svc.MYSQLD_PATH
            out = system2('%s - mysql -s %s -c "%s"' % (SU_EXEC, BASH, cmd),
                                    shell=True, raise_exc=False, silent=True)[0]

        if out:
            raw = out.split(49*'-'+' '+24*'-')
            if raw:
                a = raw[-1].split('\n')
                if len(a) > 5:
                    b = a[1:-5]
                    for item in b:
                        c = item.split()
                        if len(c) > 1:
                            key = c[0]
                            val = ' '.join(c[1:])
                            s[key.strip()] = val.strip()

        if s:
            m_config = Configuration('ini')
            if os.path.exists(path):
                m_config.read(path)

            for variable in base_manifest:
                name = variable.name
                dv_path = './%s/default-value' % name

                try:
                    old_value =  m_config.get(dv_path)
                    if name in s:
                        new_value = s[name]
                    else:
                        name = name.replace('_','-')
                        if name in s:
                            new_value = self.definitions[s[name]] if s[name] in self.definitions else s[name]
                            if old_value != new_value and new_value != '(No default value)':
                                LOG.debug('Replacing %s default value %s with precompiled value %s',
                                                name, old_value, new_value)
                                m_config.set(path=dv_path, value=new_value, force=True)
                except NoPathError:
                    pass
            m_config.write(path)

        self._merged_manifest = _CnfManifest(path)
        return self._merged_manifest
Beispiel #20
0
    def _test_numeric_option(self, option):
        self.assertRaises(ValueError, setattr, self.cnf, option, 'NotNumericValue')
 
        setattr(self.cnf, option, 113)
        c = Configuration('mongodb')
        c.read(self.cnf_path)
        self.assertEqual('113', c.get(option))
        self.assertEqual(113, getattr(self.cnf, option))
 
        setattr(self.cnf, option, None)
        c = Configuration('mongodb')
        c.read(self.cnf_path)
        self.assertRaises(NoPathError, c.get, option)
Beispiel #21
0
    def download_and_restore(self, volume, snapshot, tranzit_path):
        # Load manifest
        clear_queue(self._writer_queue)
        clear_queue(self._download_queue)
        self._download_finished.clear()
        transfer = self._transfer_cls()
        mnf_path = transfer.download(snapshot.path, tranzit_path)
        mnf = Configuration('ini')
        mnf.read(mnf_path)

        volume.fs_created = False
        volume.mkfs(snapshot.fstype)

        remote_path = os.path.dirname(snapshot.path)
        # Get links with md5 sums
        links = [(os.path.join(remote_path, chunk[0]), chunk[1]) for chunk in mnf.items('chunks')]
        links.sort()

        # Download 2 first chunks
        for link in links[:2]:
            transfer.download(link[0], tranzit_path)
            chunk_path = os.path.join(tranzit_path, os.path.basename(link[0]))
            if self._md5sum(chunk_path) != link[1]:
                raise Exception("md5sum of chunk %s is not correct." % chunk_path)
            self._writer_queue.put(chunk_path)

        if hasattr(snapshot, 'snap_strategy') and snapshot.snap_strategy == 'data':
            restore_strategy = DataRestoreStrategy(self._logger)
        else:
            restore_strategy = DeviceRestoreStrategy(self._logger)

        writer = threading.Thread(target=restore_strategy.restore, name='writer',
                                                        args=(self._writer_queue, volume, self._download_finished))
        writer.start()

        # Add remaining files to download queue
        for link in links[2:]:
            self._download_queue.put(link)

        downloader = threading.Thread(name="Downloader", target=self._downloader,
                                                                  args=(tranzit_path,))
        downloader.start()
        downloader.join()
        writer.join()
Beispiel #22
0
	def set(self, option, value):
		if not self.data:
			self.data = Configuration(self.config_type)
			if os.path.exists(self.path):
				self.data.read(self.path)
		if value:
			self.data.set(option,str(value), force=True)
		elif self.comment_empty: 
			self.data.comment(option)
		if self.autosave:
			self.save_data()
			self.data = None
Beispiel #23
0
    def _test_option(self, option):
        setattr(self.cnf, option, 'value')
        c = Configuration('mongodb')
        c.read(self.cnf_path)
        self.assertEqual('value', c.get(option))

        setattr(self.cnf, option, None)
        c = Configuration('mongodb')
        c.read(self.cnf_path)
        self.assertRaises(NoPathError, c.get, option)
Beispiel #24
0
 def set(self, option, value):
     if not self.data:
         self.data = Configuration(self.config_type)
         if os.path.exists(self.path):
             self.data.read(self.path)
     if value:
         self.data.set(option, value, force=True)
     else:
         self.data.remove(option)
     if self.autosave:
         self.save_data()
         self.data = None
Beispiel #25
0
 def get_system_variables(self):
     #TESTING REQUIRED!
     conf = Configuration(self._config_format)
     conf.read(self._config_path)
     vars = {}
     for section in conf.sections('./'):
         vars[section] = conf.get(section)
     return vars
Beispiel #26
0
    def _test_numeric_option(self, option):
        self.assertRaises(ValueError, setattr, self.cnf, option, 'NotNumericValue')

        setattr(self.cnf, option, 113)
        c = Configuration('mongodb')
        c.read(self.cnf_path)
        self.assertEqual('113', c.get(option))
        self.assertEqual(113, getattr(self.cnf, option))

        setattr(self.cnf, option, None)
        c = Configuration('mongodb')
        c.read(self.cnf_path)
        self.assertRaises(NoPathError, c.get, option)
Beispiel #27
0
	def get_list(self, option):
		if not self.data:
			self.data =  Configuration(self.config_type)
			if os.path.exists(self.path):
				self.data.read(self.path)
		try:
			value = self.data.get_list(option)
		except NoPathError:
			try:
				value = getattr(self, option+'_default')
			except AttributeError:
				value = ()
		if self.autosave:
			self.data = None
		return value
Beispiel #28
0
	def set(self, option, value, append=False):
		if not self.data:
			self.data = Configuration(self.config_type)
			if os.path.exists(self.path):
				self.data.read(self.path)
		if value:
			if append:
				self.data.add(option, str(value))
			else:
				self.data.set(option,str(value), force=True)
		else:
			self.data.comment(option)
		if self.autosave:
			self.save_data()
			self.data = None
Beispiel #29
0
    def _reload_backends(self):
        self._logger.info('Updating mysql-proxy backends list')
        self.config = Configuration('mysql')
        if os.path.exists(CONFIG_FILE_PATH):
            self.config.read(CONFIG_FILE_PATH)
            self.config.remove('./mysql-proxy/proxy-backend-addresses')
            self.config.remove('./mysql-proxy/proxy-read-only-backend-addresses')

        try:
            self.config.get('./mysql-proxy')
        except NoPathError:
            self.config.add('./mysql-proxy')

        queryenv = bus.queryenv_service
        roles = queryenv.list_roles()
        master = None
        slaves = []

        for role in roles:
            if not is_mysql_role(role.behaviour):
                continue

            for host in role.hosts:
                ip = host.internal_ip or host.external_ip
                if host.replication_master:
                    master = ip
                else:
                    slaves.append(ip)

        if master:
            self._logger.debug('Adding mysql master %s to  mysql-proxy defaults file', master)
            self.config.add('./mysql-proxy/proxy-backend-addresses', '%s:3306' % master)
        if slaves:
            self._logger.debug('Adding mysql slaves to  mysql-proxy defaults file: %s', ', '.join(slaves))
            for slave in slaves:
                self.config.add('./mysql-proxy/proxy-read-only-backend-addresses', '%s:3306' % slave)

        self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True)
        self.config.set('./mysql-proxy/daemon', 'true', force=True)
        self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True)
        if self.service.version > (0,8,0):
            self.config.set('./mysql-proxy/plugins', 'proxy', force=True)

        self._logger.debug('Saving new mysql-proxy defaults file')
        self.config.write(CONFIG_FILE_PATH)
        os.chmod(CONFIG_FILE_PATH, 0660)

        self.service.restart()
Beispiel #30
0
    def load(self, preset_type):
        '''
        @rtype: Preset
        @raise OSError: When cannot read preset file
        @raise MetaconfError: When experience problems with preset file parsing
        '''
        self._logger.debug('Loading %s %s preset' %
                           (preset_type, self.service_name))
        ini = Configuration('ini')
        ini.read(self._filename(preset_type))

        return CnfPreset(ini.get('general/name'), dict(ini.items('settings/')))
Beispiel #31
0
    def __init__(self, manifest_path):
        self._options = []
        ini = Configuration('ini')
        ini.read(manifest_path)
        try:
            self._defaults = dict(ini.items('__defaults__'))
        except NoPathError:
            self._defaults = dict()

        for name in ini.sections("./"):
            if name == '__defaults__':
                continue
            self._options.append(
                _OptionSpec.from_ini(ini, name, self._defaults))
Beispiel #32
0
            self._logger.debug('Do not removing http/server section')
            if not config.get_list('http/server'):
                config.read(os.path.join(bus.share_path, "nginx/server.tpl"))

        if disttool.is_debian_based():
        # Comment /etc/nginx/sites-enabled/*
            try:
                i = config.get_list('http/include').index('/etc/nginx/sites-enabled/*')
                config.comment('http/include[%d]' % (i+1))
                self._logger.debug('comment site-enabled include')
            except (ValueError, IndexError):
                self._logger.debug('site-enabled include already commented')
        elif disttool.is_redhat_based():
            def_host_path = '/etc/nginx/conf.d/default.conf'
            if os.path.exists(def_host_path):
                default_host = Configuration('nginx')
                default_host.read(def_host_path)
                default_host.comment('server')
                default_host.write(def_host_path)

        if dump == self._dump_config(config):
            self._logger.debug("Main nginx config wasn`t changed")
        else:
            # Write new nginx.conf
            shutil.copy(nginx_conf_path, nginx_conf_path + '.bak')
            config.write(nginx_conf_path)
            if reload_service:
                self.api._reload_service()

    def _insert_iptables_rules(self, *args, **kwargs):
        if iptables.enabled():
Beispiel #33
0
    def save(self, preset, preset_type):
        '''
        @type preset: CnfPreset
        @type preset_type: CnfPresetStore.PresetType
        @raise ValueError: When `preset` is not an instance of CnfPreset
        @raise OSError: When cannot save preset file
        '''
        if not isinstance(preset, CnfPreset):
            raise ValueError(
                'argument `preset` should be a CnfPreset instance, %s is given',
                type(preset))

        self._logger.debug('Saving preset as %s' % preset_type)
        ini = Configuration('ini')
        ini.add('general')
        ini.add(
            'general/name', preset.name if
            (hasattr(preset, 'name') and preset.name) else 'Noname')
        ini.add('settings')

        for k, v in preset.settings.items():
            ini.add('settings/%s' % k, v)
        ini.write(self._filename(preset_type))
Beispiel #34
0
 def __init__(self, path):
     self._cnf = Configuration("apache")
     self.path = path
Beispiel #35
0
    def _write_manifest(self, snapshot, tranzit_path):
        ''' Make snapshot manifest '''
        manifest_path = os.path.join(tranzit_path, '%s.%s' % (snapshot.id, self.MANIFEST_NAME))
        self._logger.info('Writing snapshot manifest file in %s', manifest_path)
        config = Configuration('ini')
        config.add('snapshot/description', snapshot.description, force=True)
        config.add('snapshot/created_at', time.strftime("%Y-%m-%d %H:%M:%S"))
        config.add('snapshot/pack_method', 'pigz') # Not used yet
        for chunk, md5 in self._chunks_md5.iteritems():
            config.add('chunks/%s' % chunk, md5, force=True)

        config.write(manifest_path)

        return manifest_path
Beispiel #36
0
class BaseConfig(object):

    '''
    Parent class for object representations of postgresql.conf and recovery.conf which fortunately both have similar syntax
    '''

    autosave = None
    path = None
    data = None
    config_name = None
    config_type = None
    comment_empty = False


    def __init__(self, path, autosave=True):
        self._logger = logging.getLogger(__name__)
        self.autosave = autosave
        self.path = path


    @classmethod
    def find(cls, config_dir):
        return cls(os.path.join(config_dir.path, cls.config_name))


    def set(self, option, value):
        self.apply_dict({option:value})


    def set_path_type_option(self, option, path):
        if not os.path.exists(path):
            raise ValueError('%s %s does not exist' % (option, path))
        self.set(option, path)


    def set_numeric_option(self, option, number):
        try:
            assert number is None or type(number) is int
        except AssertionError:
            raise ValueError('%s must be a number (got %s instead)' % (option, number))

        is_numeric = type(number) is int
        self.set(option, str(number) if is_numeric else None)


    def get(self, option):
        self._init_configuration()
        try:
            value = self.data.get(option)
        except NoPathError:
            try:
                value = getattr(self, option+'_default')
            except AttributeError:
                value = None
        self._cleanup()
        return value


    def get_numeric_option(self, option):
        value = self.get(option)
        return value if value is None else int(value)


    def to_dict(self):
        self._init_configuration()

        result = {}

        for section in self.data.sections('./'):
            try:
                kv = dict(self.data.items(section))
            except NoPathError:
                kv = {}
            for variable, value in kv.items():
                path = '%s/%s' % (section,variable)
                result[path] = value
        '''
        variables in root section
        '''
        for variable,value in self.data.items('.'):
            if value and value.strip():
                result[variable] = value

        self._cleanup()
        return result


    def apply_dict(self, kv):
        self._init_configuration()
        for path, value in kv.items():
            if not value and self.comment_empty:
                self.data.comment(path)
            else:
                self.data.set(path,str(value), force=True)
        self._cleanup(True)


    def delete_options(self, options):
        self._init_configuration()
        for path  in options:
            self.data.remove(path)
        self._cleanup(True)


    def _init_configuration(self):
        if not self.data:
            self.data = Configuration(self.config_type)
            if os.path.exists(self.path):
                self.data.read(self.path)


    def _cleanup(self, save_data=False):
        if self.autosave:
            if save_data and self.data:
                self.data.write(self.path)
            self.data = None
Beispiel #37
0
    def apply_preset(self, preset):
        conf = Configuration(self._config_format)
        conf.read(self._config_path)

        self._before_apply_preset()

        ver = self._software_version
        for opt in self._manifest:
            path = opt.name if not opt.section else '%s/%s' % (opt.section,
                                                               opt.name)

            try:
                value = conf.get(path)
            except NoPathError:
                value = ''

            if opt.name in preset.settings:
                new_value = preset.settings[opt.name]

                # Skip unsupported
                if ver and opt.supported_from and opt.supported_from > ver:
                    self._logger.debug(
                        "Skipping option '%s' supported from %s; installed %s"
                        % (opt.name, opt.supported_from, ver))
                    continue

                if not opt.default_value:
                    self._logger.debug("Option '%s' has no default value" %
                                       opt.name)
                    pass
                elif new_value == opt.default_value:
                    if value:
                        self._logger.debug(
                            "Option '%s' equal to default. Removing." %
                            opt.name)
                        conf.remove(path)
                    self._after_remove_option(opt)
                    continue

                if self.definitions and new_value in self.definitions:
                    manifest = Configuration('ini')
                    if os.path.exists(self._manifest_path):
                        manifest.read(self._manifest_path)
                    try:
                        if manifest.get('%s/type' % opt.name) == 'boolean':
                            new_value = self.definitions[new_value]
                    except NoPathError, e:
                        pass

                self._logger.debug("Check that '%s' value changed:'%s'='%s'" %
                                   (opt.name, value, new_value))

                if new_value == value:
                    self._logger.debug("Skip option '%s'. Not changed" %
                                       opt.name)
                    pass
                else:
                    self._logger.debug("Set option '%s' = '%s'" %
                                       (opt.name, new_value))
                    self._logger.debug('Set path %s = %s', path, new_value)
                    conf.set(path, new_value, force=True)
                    self._after_set_option(opt, new_value)
            else:
                if value:
                    self._logger.debug(
                        "Removing option '%s'. Not found in preset" % opt.name)
                    conf.remove(path)
                self._after_remove_option(opt)
Beispiel #38
0
    def _manifest(self):
        class HeadRequest(urllib2.Request):
            def get_method(self):
                return "HEAD"

        manifest_url = bus.scalr_url + '/storage/service-configuration-manifests/%s.ini' % self.behaviour
        path = self._manifest_path

        url_handle = urllib2.urlopen(HeadRequest(manifest_url))
        headers = url_handle.info()
        url_last_modified = headers.getdate("Last-Modified")

        file_modified = tuple(time.localtime(
            os.path.getmtime(path))) if os.path.exists(path) else None

        if not file_modified or url_last_modified > file_modified:
            self._logger.debug('Fetching %s', manifest_url)
            response = urllib2.urlopen(manifest_url)
            data = response.read()
            if data:
                old_manifest = Configuration('ini')
                if os.path.exists(path):
                    old_manifest.read(path)

                new_manifest = Configuration('ini')
                o = StringIO()
                o.write(data)
                o.seek(0)
                new_manifest.readfp(o)

                new_sections = new_manifest.sections('./')
                old_sections = old_manifest.sections('./')

                diff_path = os.path.join(os.path.dirname(path),
                                         self.behaviour + '.incdiff')
                diff = Configuration('ini')

                if old_sections and old_sections != new_sections:
                    #skipping diff if no previous manifest found or it is equal to the new one
                    if os.path.exists(diff_path):
                        diff.read(diff_path)

                    sys_vars = self.get_system_variables()

                    for section in new_sections:
                        if section not in old_sections and sys_vars.has_key(
                                section):
                            sys_var = sys_vars[section]
                            if self.definitions:
                                if self.definitions.has_key(sys_var):
                                    sys_var = self.definitions[sys_var]
                            diff.add('./%s/default-value' % section,
                                     sys_var,
                                     force=True)
                            diff.write(diff_path)

                if os.path.exists(diff_path):
                    diff.read(diff_path)

                for variable in diff.sections('./'):
                    sys_value = diff.get('./%s/default-value' % variable)
                    if sys_value and variable in new_manifest.sections('./'):
                        new_manifest.set('./%s/default-value' % variable,
                                         sys_value,
                                         force=True)
                new_manifest.write(path)

        return _CnfManifest(path)
Beispiel #39
0
    def _test_bool_option(self, option):
        self.assertRaises(ValueError, setattr, self.cnf, option, 'NotBoolValue')

        setattr(self.cnf, option, True)
        c = Configuration('mongodb')
        c.read(self.cnf_path)
        self.assertEqual('true', c.get(option))
        self.assertEqual(True, getattr(self.cnf, option))

        setattr(self.cnf, option, False)
        c = Configuration('mongodb')
        c.read(self.cnf_path)
        self.assertEqual('false', c.get(option))
        self.assertEqual(False, getattr(self.cnf, option))

        setattr(self.cnf, option, None)
        c = Configuration('mongodb')
        c.read(self.cnf_path)
        self.assertRaises(NoPathError, c.get, option)

        c.set(option, 'NotBool', force=True)
        c.write(self.cnf_path)
        self.assertRaises(ValueError, getattr, self.cnf, option)
Beispiel #40
0
 def __init__(self, path):
     self._cnf = Configuration("apache")
     self.path = path
Beispiel #41
0
    def _test_bool_option(self, option):
        self.assertRaises(ValueError, setattr, self.cnf, option, 'NotBoolValue')
 
        setattr(self.cnf, option, True)
        c = Configuration('mongodb')
        c.read(self.cnf_path)
        self.assertEqual('true', c.get(option))
        self.assertEqual(True, getattr(self.cnf, option))
 
        setattr(self.cnf, option, False)
        c = Configuration('mongodb')
        c.read(self.cnf_path)
        self.assertEqual('false', c.get(option))
        self.assertEqual(False, getattr(self.cnf, option))
 
        setattr(self.cnf, option, None)
        c = Configuration('mongodb')
        c.read(self.cnf_path)
        self.assertRaises(NoPathError, c.get, option)
 
        c.set(option, 'NotBool', force=True)
        c.write(self.cnf_path)
        self.assertRaises(ValueError, getattr, self.cnf, option)
Beispiel #42
0
class MysqlProxyHandler(ServiceCtlHandler):
    def __init__(self):
        self._logger = logging.getLogger(__name__)
        self.service = initdv2.lookup(BEHAVIOUR)
        self._service_name = BEHAVIOUR
        bus.on(init=self.on_init)

    def accept(self,
               message,
               queue,
               behaviour=None,
               platform=None,
               os=None,
               dist=None):
        return message.behaviour and is_mysql_role(
            message.behaviour) and message.name in (
                Messages.HOST_UP, Messages.HOST_DOWN, NEW_MASTER_UP,
                DbMsrMessages.DBMSR_NEW_MASTER_UP)

    def on_init(self):
        bus.on(start=self.on_start,
               before_host_up=self.on_before_host_up,
               reload=self.on_reload)

    def on_reload(self):
        self._reload_backends()

    def on_start(self):
        cnf = bus.cnf
        if cnf.state == config.ScalarizrState.RUNNING:
            self._reload_backends()

    def on_before_host_up(self, msg):
        self._reload_backends()

    def _reload_backends(self):
        self._logger.info('Updating mysql-proxy backends list')
        self.config = Configuration('mysql')
        if os.path.exists(CONFIG_FILE_PATH):
            self.config.read(CONFIG_FILE_PATH)
            self.config.remove('./mysql-proxy/proxy-backend-addresses')
            self.config.remove(
                './mysql-proxy/proxy-read-only-backend-addresses')

        try:
            self.config.get('./mysql-proxy')
        except NoPathError:
            self.config.add('./mysql-proxy')

        queryenv = bus.queryenv_service
        roles = queryenv.list_roles()
        master = None
        slaves = []

        for role in roles:
            if not is_mysql_role(role.behaviour):
                continue

            for host in role.hosts:
                ip = host.internal_ip or host.external_ip
                if host.replication_master:
                    master = ip
                else:
                    slaves.append(ip)

        if master:
            self._logger.debug(
                'Adding mysql master %s to  mysql-proxy defaults file', master)
            self.config.add('./mysql-proxy/proxy-backend-addresses',
                            '%s:3306' % master)
        if slaves:
            self._logger.debug(
                'Adding mysql slaves to  mysql-proxy defaults file: %s',
                ', '.join(slaves))
            for slave in slaves:
                self.config.add(
                    './mysql-proxy/proxy-read-only-backend-addresses',
                    '%s:3306' % slave)

        self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True)
        self.config.set('./mysql-proxy/daemon', 'true', force=True)
        self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True)
        if self.service.version > (0, 8, 0):
            self.config.set('./mysql-proxy/plugins', 'proxy', force=True)

        self._logger.debug('Saving new mysql-proxy defaults file')
        self.config.write(CONFIG_FILE_PATH)
        os.chmod(CONFIG_FILE_PATH, 0660)

        self.service.restart()

    def on_HostUp(self, message):
        self._reload_backends()

    on_DbMsr_NewMasterUp = on_Mysql_NewMasterUp = on_HostDown = on_HostUp
Beispiel #43
0
	def _manifest(self):		
		
		class HeadRequest(urllib2.Request):
			def get_method(self):
				return "HEAD"
		
		manifest_url = bus.scalr_url + '/storage/service-configuration-manifests/%s.ini' % self.behaviour	
		path = self._manifest_path

			
		url_handle = urllib2.urlopen(HeadRequest(manifest_url))
		headers = url_handle.info()
		url_last_modified = headers.getdate("Last-Modified")
		
		file_modified = tuple(time.localtime(os.path.getmtime(path))) if os.path.exists(path) else None
		
		if not file_modified or url_last_modified > file_modified:
			self._logger.debug('Fetching %s', manifest_url)
			response = urllib2.urlopen(manifest_url)
			data = response.read()
			if data:
				old_manifest = Configuration('ini')
				if os.path.exists(path):
					old_manifest.read(path)

				new_manifest = Configuration('ini')
				o = StringIO()
				o.write(data)
				o.seek(0)
				new_manifest.readfp(o)
				
				new_sections = new_manifest.sections('./')  
				old_sections = old_manifest.sections('./')

				diff_path = os.path.join(os.path.dirname(path), self.behaviour + '.incdiff')
				diff = Configuration('ini')
									
				if old_sections and old_sections != new_sections:
					#skipping diff if no previous manifest found or it is equal to the new one
					if os.path.exists(diff_path):
						diff.read(diff_path)

					sys_vars = self.get_system_variables()

					for section in new_sections:
						if section not in old_sections and sys_vars.has_key(section):
							sys_var = sys_vars[section]
							if self.definitions:
								if self.definitions.has_key(sys_var):
									sys_var = self.definitions[sys_var]
							diff.add('./%s/default-value' % section, sys_var, force=True)
							diff.write(diff_path)
				
				if os.path.exists(diff_path):
					diff.read(diff_path)
				
				for variable in diff.sections('./'):
					sys_value = diff.get('./%s/default-value' % variable)
					if sys_value and variable in new_manifest.sections('./'):
						new_manifest.set('./%s/default-value' % variable, sys_value, force=True)
				new_manifest.write(path)

		return _CnfManifest(path)
Beispiel #44
0
class BaseRedisConfig(BaseConfig):

	config_type = 'redis'


	def set(self, option, value, append=False):
		if not self.data:
			self.data = Configuration(self.config_type)
			if os.path.exists(self.path):
				self.data.read(self.path)
		if value:
			if append:
				self.data.add(option, str(value))
			else:
				self.data.set(option,str(value), force=True)
		else:
			self.data.comment(option)
		if self.autosave:
			self.save_data()
			self.data = None


	def set_sequential_option(self, option, seq):
		is_typle = type(seq) is tuple
		try:
			assert seq is None or is_typle
		except AssertionError:
			raise ValueError('%s must be a sequence (got %s instead)' % (option, seq))
		self.set(option, ' '.join(map(str,seq)) if is_typle else None)


	def get_sequential_option(self, option):
		raw = self.get(option)
		return raw.split() if raw else ()


	def get_list(self, option):
		if not self.data:
			self.data =  Configuration(self.config_type)
			if os.path.exists(self.path):
				self.data.read(self.path)
		try:
			value = self.data.get_list(option)
		except NoPathError:
			try:
				value = getattr(self, option+'_default')
			except AttributeError:
				value = ()
		if self.autosave:
			self.data = None
		return value


	def get_dict_option(self, option):
		raw = self.get_list(option)
		d = {}
		for raw_value in raw:
			k,v = raw_value.split()
			if k and v:
				d[k] = v
		return d


	def set_dict_option(self, option, d):
		try:
			assert d is None or type(d)==dict
			#cleaning up
			#TODO: make clean process smarter using indexes
			for i in self.get_list(option):
				self.set(option+'[0]', None)

			#adding multiple entries
			for k,v in d.items():
				val = ' '.join(map(str,'%s %s'%(k,v)))
				self.set(option, val, append=True)
		except ValueError:
			raise ValueError('%s must be a sequence (got %s instead)' % (option, d))
Beispiel #45
0
class BaseConfig(object):
    '''
    Parent class for object representations of postgresql.conf and recovery.conf which fortunately both have similar syntax
    '''

    autosave = None
    path = None
    data = None
    config_name = None
    config_type = None
    comment_empty = False

    def __init__(self, path, autosave=True):
        self._logger = logging.getLogger(__name__)
        self.autosave = autosave
        self.path = path

    @classmethod
    def find(cls, config_dir):
        return cls(os.path.join(config_dir.path, cls.config_name))

    def set(self, option, value):
        self.apply_dict({option: value})

    def set_path_type_option(self, option, path):
        if not os.path.exists(path):
            raise ValueError('%s %s does not exist' % (option, path))
        self.set(option, path)

    def set_numeric_option(self, option, number):
        try:
            assert number is None or type(number) is int
        except AssertionError:
            raise ValueError('%s must be a number (got %s instead)' %
                             (option, number))

        is_numeric = type(number) is int
        self.set(option, str(number) if is_numeric else None)

    def get(self, option):
        self._init_configuration()
        try:
            value = self.data.get(option)
        except NoPathError:
            try:
                value = getattr(self, option + '_default')
            except AttributeError:
                value = None
        self._cleanup()
        return value

    def get_numeric_option(self, option):
        value = self.get(option)
        return value if value is None else int(value)

    def to_dict(self):
        self._init_configuration()

        result = {}

        for section in self.data.sections('./'):
            try:
                kv = dict(self.data.items(section))
            except NoPathError:
                kv = {}
            for variable, value in kv.items():
                path = '%s/%s' % (section, variable)
                result[path] = value
        '''
        variables in root section
        '''
        for variable, value in self.data.items('.'):
            if value and value.strip():
                result[variable] = value

        self._cleanup()
        return result

    def apply_dict(self, kv):
        self._init_configuration()
        for path, value in kv.items():
            if not value and self.comment_empty:
                self.data.comment(path)
            else:
                self.data.set(path, str(value), force=True)
        self._cleanup(True)

    def delete_options(self, options):
        self._init_configuration()
        for path in options:
            self.data.remove(path)
        self._cleanup(True)

    def _init_configuration(self):
        if not self.data:
            self.data = Configuration(self.config_type)
            if os.path.exists(self.path):
                self.data.read(self.path)

    def _cleanup(self, save_data=False):
        if self.autosave:
            if save_data and self.data:
                self.data.write(self.path)
            self.data = None
Beispiel #46
0
	def save(self, preset, preset_type):
		'''
		@type preset: CnfPreset
		@type preset_type: CnfPresetStore.PresetType
		@raise ValueError: When `preset` is not an instance of CnfPreset
		@raise OSError: When cannot save preset file
		'''
		if not isinstance(preset, CnfPreset):
			raise ValueError('argument `preset` should be a CnfPreset instance, %s is given', type(preset))
		
		self._logger.debug('Saving preset as %s' % preset_type)
		ini = Configuration('ini')
		ini.add('general')
		ini.add('general/name', preset.name if (hasattr(preset, 'name') and preset.name) else 'Noname')
		ini.add('settings')

		for k, v in preset.settings.items():
			ini.add('settings/%s' % k, v)
		ini.write(self._filename(preset_type))
Beispiel #47
0
class MongoDBConfig(BaseConfig):

    config_type = "mongodb"
    config_name = os.path.basename(CONFIG_PATH_DEFAULT)

    @classmethod
    def find(cls, config_dir=None):
        # conf_path = UBUNTU_CONFIG_PATH if disttool.is_ubuntu() else CENTOS_CONFIG_PATH
        return cls(os.path.join(config_dir, cls.config_name) if config_dir else CONFIG_PATH_DEFAULT)

    def set(self, option, value):
        if not self.data:
            self.data = Configuration(self.config_type)
            if os.path.exists(self.path):
                self.data.read(self.path)
        if value:
            self.data.set(option, value, force=True)
        else:
            self.data.remove(option)
        if self.autosave:
            self.save_data()
            self.data = None

    def set_bool_option(self, option, value):
        try:
            assert value in (None, True, False)
        except AssertionError:
            raise ValueError("%s must be a boolean (got %s instead)" % (option, type(value)))
        value_to_set = None if value is None else str(value).lower()
        self.set(option, value_to_set)

    def get_bool_option(self, option):
        value = self.get(option)
        try:
            assert not value or value == "true" or value == "false"
        except AssertionError:
            raise ValueError("%s must be true or false (got %s instead)" % (option, type(value)))
        return True if value == "true" else False

    def _get_logpath(self):
        return self.get("logpath")

    def _set_logpath(self, path):
        self.set("logpath", path)

    def _get_replSet(self):
        return self.get("replSet")

    def _set_replSet(self, name):
        self.set("replSet", name)

    def _get_port(self):
        return self.get_numeric_option("port")

    def _set_port(self, number):
        self.set_numeric_option("port", number)

    def _get_logappend(self):
        return self.get_bool_option("logappend")

    def _set_logappend(self, on=True):
        self.set_bool_option("logappend", on)

    def _get_dbpath(self):
        return self.get("dbpath")

    def _set_dbpath(self, path):
        self.set_path_type_option("dbpath", path)

    def _get_nojournal(self):
        return self.get_bool_option("nojournal")

    def _set_nojournal(self, on=False):
        self.set_bool_option("nojournal", on)

    def _get_nohttpinterface(self):
        return self.get_bool_option("nohttpinterface")

    def _set_nohttpinterface(self, on=False):
        self.set_bool_option("nohttpinterface", on)

    def _get_rest(self):
        return self.get_bool_option("rest")

    def _set_rest(self, on=False):
        self.set_bool_option("rest", on)

    def _set_shardsvr(self, value):
        self.set_bool_option("shardsvr", value)

    def _get_shardsvr(self):
        return self.get_bool_option("shardsvr")

    shardsvr = property(_get_shardsvr, _set_shardsvr)
    rest = property(_get_rest, _set_rest)
    nohttpinterface = property(_get_nohttpinterface, _set_nohttpinterface)
    nojournal = property(_get_nojournal, _set_nojournal)
    dbpath = property(_get_dbpath, _set_dbpath)
    logappend = property(_get_logappend, _set_logappend)
    port = property(_get_port, _set_port)
    replSet = property(_get_replSet, _set_replSet)
    logpath = property(_get_logpath, _set_logpath)
Beispiel #48
0
    def _write_manifest(self, snapshot, tranzit_path):
        ''' Make snapshot manifest '''
        manifest_path = os.path.join(
            tranzit_path, '%s.%s' % (snapshot.id, self.MANIFEST_NAME))
        self._logger.info('Writing snapshot manifest file in %s',
                          manifest_path)
        config = Configuration('ini')
        config.add('snapshot/description', snapshot.description, force=True)
        config.add('snapshot/created_at', time.strftime("%Y-%m-%d %H:%M:%S"))
        config.add('snapshot/pack_method', 'pigz')  # Not used yet
        for chunk, md5 in self._chunks_md5.iteritems():
            config.add('chunks/%s' % chunk, md5, force=True)

        config.write(manifest_path)

        return manifest_path
Beispiel #49
0
 def _init_configuration(self):
     if not self.data:
         self.data = Configuration(self.config_type)
         if os.path.exists(self.path):
             self.data.read(self.path)
Beispiel #50
0
 def _init_configuration(self):
     if not self.data:
         self.data = Configuration(self.config_type)
         if os.path.exists(self.path):
             self.data.read(self.path)
Beispiel #51
0
class MysqlProxyHandler(ServiceCtlHandler):


    def __init__(self):
        self._logger = logging.getLogger(__name__)
        self.service = initdv2.lookup(BEHAVIOUR)
        self._service_name = BEHAVIOUR
        bus.on(init=self.on_init)


    def accept(self, message, queue, behaviour=None, platform=None, os=None, dist=None):
        return message.behaviour and is_mysql_role(message.behaviour) and message.name in (
                    Messages.HOST_UP,
                    Messages.HOST_DOWN,
                    NEW_MASTER_UP,
                    DbMsrMessages.DBMSR_NEW_MASTER_UP
        )


    def on_init(self):
        bus.on(
            start=self.on_start,
            before_host_up=self.on_before_host_up,
            reload=self.on_reload
        )


    def on_reload(self):
        self._reload_backends()


    def on_start(self):
        cnf = bus.cnf
        if cnf.state == config.ScalarizrState.RUNNING:
            self._reload_backends()


    def on_before_host_up(self, msg):
        self._reload_backends()


    def _reload_backends(self):
        self._logger.info('Updating mysql-proxy backends list')
        self.config = Configuration('mysql')
        if os.path.exists(CONFIG_FILE_PATH):
            self.config.read(CONFIG_FILE_PATH)
            self.config.remove('./mysql-proxy/proxy-backend-addresses')
            self.config.remove('./mysql-proxy/proxy-read-only-backend-addresses')

        try:
            self.config.get('./mysql-proxy')
        except NoPathError:
            self.config.add('./mysql-proxy')

        queryenv = bus.queryenv_service
        roles = queryenv.list_roles()
        master = None
        slaves = []

        for role in roles:
            if not is_mysql_role(role.behaviour):
                continue

            for host in role.hosts:
                ip = host.internal_ip or host.external_ip
                if host.replication_master:
                    master = ip
                else:
                    slaves.append(ip)

        if master:
            self._logger.debug('Adding mysql master %s to  mysql-proxy defaults file', master)
            self.config.add('./mysql-proxy/proxy-backend-addresses', '%s:3306' % master)
        if slaves:
            self._logger.debug('Adding mysql slaves to  mysql-proxy defaults file: %s', ', '.join(slaves))
            for slave in slaves:
                self.config.add('./mysql-proxy/proxy-read-only-backend-addresses', '%s:3306' % slave)

        self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True)
        self.config.set('./mysql-proxy/daemon', 'true', force=True)
        self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True)
        if self.service.version > (0,8,0):
            self.config.set('./mysql-proxy/plugins', 'proxy', force=True)

        self._logger.debug('Saving new mysql-proxy defaults file')
        self.config.write(CONFIG_FILE_PATH)
        os.chmod(CONFIG_FILE_PATH, 0660)

        self.service.restart()


    def on_HostUp(self, message):
        self._reload_backends()

    on_DbMsr_NewMasterUp = on_Mysql_NewMasterUp = on_HostDown = on_HostUp