def build_log_file_name(self, log_name, owner, datastore_dir=None):
"""Build a log file name based on the log_name and make sure the
directories exist and are accessible by owner.
"""
if datastore_dir is None:
base_dir = self.GUEST_LOG_BASE_DIR
if not operating_system.exists(base_dir, is_directory=True):
operating_system.ensure_directory(base_dir,
user=owner,
group=owner,
force=True,
as_root=True)
datastore_dir = guestagent_utils.build_file_path(
base_dir, self.GUEST_LOG_DATASTORE_DIRNAME)
if not operating_system.exists(datastore_dir, is_directory=True):
operating_system.ensure_directory(datastore_dir,
user=owner,
group=owner,
force=True,
as_root=True)
log_file_name = guestagent_utils.build_file_path(
datastore_dir, '%s-%s.log' % (self.manager, log_name))
return self.validate_log_file(log_file_name, owner)
def test_build_file_path(self):
self.assertEqual(
'base_dir/base_name',
guestagent_utils.build_file_path('base_dir', 'base_name'))
self.assertEqual(
'base_dir/base_name.ext1',
guestagent_utils.build_file_path('base_dir', 'base_name', 'ext1'))
self.assertEqual(
'base_dir/base_name.ext1.ext2',
guestagent_utils.build_file_path('base_dir', 'base_name', 'ext1',
'ext2'))
def test_build_file_path(self):
self.assertEqual(
'base_dir/base_name',
guestagent_utils.build_file_path('base_dir', 'base_name'))
self.assertEqual(
'base_dir/base_name.ext1',
guestagent_utils.build_file_path('base_dir', 'base_name', 'ext1'))
self.assertEqual(
'base_dir/base_name.ext1.ext2',
guestagent_utils.build_file_path(
'base_dir', 'base_name', 'ext1', 'ext2'))
def apply(self, group_name, change_id, options):
self._initialize_import_directory()
revision_file = self._find_revision_file(group_name, change_id)
if revision_file is None:
# Create a new file.
last_revision_index = self._get_last_file_index(group_name)
revision_file = guestagent_utils.build_file_path(
self._revision_dir,
'%s-%03d-%s' % (group_name, last_revision_index + 1,
change_id),
self._revision_ext)
else:
# Update the existing file.
current = operating_system.read_file(
revision_file, codec=self._codec, as_root=self._requires_root)
options = guestagent_utils.update_dict(options, current)
operating_system.write_file(
revision_file, options, codec=self._codec,
as_root=self._requires_root)
operating_system.chown(
revision_file, self._owner, self._group,
as_root=self._requires_root)
operating_system.chmod(
revision_file, FileMode.ADD_READ_ALL, as_root=self._requires_root)
def configure(self, base_config_path, owner, group, codec, requires_root):
"""
:param base_config_path Path to the configuration file.
:type base_config_path string
:param owner Owner of the configuration and
revision files.
:type owner string
:param group Group of the configuration and
revision files.
:type group string
:param codec Codec for reading/writing of the particular
configuration format.
:type codec StreamCodec
:param requires_root Whether the strategy requires superuser
privileges.
:type requires_root boolean
"""
self._base_config_path = base_config_path
self._owner = owner
self._group = group
self._codec = codec
self._requires_root = requires_root
self._base_revision_file = guestagent_utils.build_file_path(
self._revision_dir, self.BASE_REVISION_NAME, self.REVISION_EXT)
self._import_strategy.configure(base_config_path, owner, group, codec,
requires_root)
def __refresh_prepare_completed(self):
# Set the value of __prepared_completed based on the existence of
# the file. This is required as the state is cached so this method
# must be called any time the existence of the file changes.
self.__prepare_completed = os.path.isfile(
guestagent_utils.build_file_path(
self.GUESTAGENT_DIR, self.PREPARE_END_FILENAME))
def build_module_dir(cls, module_type, module_id):
sub_dir = os.path.join(module_type, module_id)
module_dir = guestagent_utils.build_file_path(cls.MODULE_BASE_DIR,
sub_dir)
if not operating_system.exists(module_dir, is_directory=True):
operating_system.create_directory(module_dir, force=True)
return module_dir
def post_restore(self):
try:
# Root enabled for the backup
pwd_file = guestagent_utils.build_file_path(
system.COUCHBASE_DUMP_DIR, self.app.SECRET_KEY_FILE)
if os.path.exists(pwd_file):
with open(pwd_file, "r") as f:
pw = f.read().rstrip("\n")
self.app.reset_admin_credentials(password=pw)
buckets_json = system.COUCHBASE_DUMP_DIR + system.BUCKETS_JSON
buckets = self._parse_buckets(buckets_json)
admin = self.app.build_admin()
max_num_replicas = admin.get_num_cluster_nodes() - 1
for bucket in buckets:
bucket.bucket_replica_count = min(bucket.bucket_replica_count,
max_num_replicas)
admin.create_buckets(buckets)
for bucket in buckets:
self.run_cbrestore(bucket.name)
except exception.ProcessExecutionError as p:
LOG.error(p)
raise base.RestoreError("Couchbase restore failed.")
def prepare_completed(self, value):
# Set the value based on the existence of the file; 'value' is ignored
# This is required as the value of prepare_completed is cached, so
# this must be referenced any time the existence of the file changes
self._prepare_completed = os.path.isfile(
guestagent_utils.build_file_path(
self.GUESTAGENT_DIR, self.PREPARE_END_FILENAME))
def configure(self, base_config_path, owner, group, codec, requires_root):
"""
:param base_config_path Path to the configuration file.
:type base_config_path string
:param owner Owner of the configuration and
revision files.
:type owner string
:param group Group of the configuration and
revision files.
:type group string
:param codec Codec for reading/writing of the particular
configuration format.
:type codec StreamCodec
:param requires_root Whether the strategy requires superuser
privileges.
:type requires_root boolean
"""
self._base_config_path = base_config_path
self._owner = owner
self._group = group
self._codec = codec
self._requires_root = requires_root
self._base_revision_file = guestagent_utils.build_file_path(
self._revision_dir, self.BASE_REVISION_NAME, self.REVISION_EXT)
self._import_strategy.configure(
base_config_path, owner, group, codec, requires_root)
def prepare_completed(self, value):
# Set the value based on the existence of the file; 'value' is ignored
# This is required as the value of prepare_completed is cached, so
# this must be referenced any time the existence of the file changes
self._prepare_completed = os.path.isfile(
guestagent_utils.build_file_path(self.GUESTAGENT_DIR,
self.PREPARE_END_FILENAME))
def datastore_log_defs(self):
if not self.appStatus.is_running:
# do nothing if Oracle is not running
return {}
owner = system.ORACLE_INSTANCE_OWNER
group = system.ORACLE_GROUP_OWNER
sid = self.admin.database_name
diag_dest = self.admin.get_parameter('diagnostic_dest')
dbname = sid.lower()
# alert log path:
# <diagnostic_dest>/diag/rdbms/<dbname>/<instname>/alert/log.xml
alert_log_file = self.validate_log_file(
guestagent_utils.build_file_path(
path.join(diag_dest, 'diag', 'rdbms', dbname, sid, 'alert'),
'log', 'xml'),
owner,
group=group)
return {
'alert': {
self.GUEST_LOG_TYPE_LABEL: guest_log.LogType.SYS,
self.GUEST_LOG_USER_LABEL: owner,
self.GUEST_LOG_FILE_LABEL: alert_log_file,
},
}
def assert_module_retrieve(self, client, instance_id, expected_count,
expected_http_code=200, expected_results=None):
try:
temp_dir = tempfile.mkdtemp()
prefix = 'contents'
modretrieve_list = client.instances.module_retrieve(
instance_id, directory=temp_dir, prefix=prefix)
self.assert_client_code(expected_http_code, client)
count = len(modretrieve_list)
self.assert_equal(expected_count, count,
"Wrong number of modules from retrieve")
expected_results = expected_results or {}
for module_name, filename in modretrieve_list.items():
if module_name in expected_results:
expected = expected_results[module_name]
contents_name = '%s_%s_%s_%s' % (
prefix, module_name,
expected['datastore'], expected['datastore_version'])
expected_filename = guestagent_utils.build_file_path(
temp_dir, contents_name, 'dat')
self.assert_equal(expected_filename, filename,
'Unexpected retrieve filename')
if 'contents' in expected and expected['contents']:
with open(filename, 'rb') as fh:
contents = fh.read()
# convert contents into bytearray to work with py27
# and py34
contents = bytes([ord(item) for item in contents])
expected_contents = bytes(
[ord(item) for item in expected['contents']])
self.assert_equal(expected_contents, contents,
"Unexpected contents for %s" %
module_name)
finally:
operating_system.remove(temp_dir)
def __refresh_prepare_completed(self):
# Set the value of __prepared_completed based on the existence of
# the file. This is required as the state is cached so this method
# must be called any time the existence of the file changes.
self.__prepare_completed = os.path.isfile(
guestagent_utils.build_file_path(self.GUESTAGENT_DIR,
self.PREPARE_END_FILENAME))
def __init__(self, state_change_wait_time=None):
"""
Sets default status and state_change_wait_time
"""
if state_change_wait_time:
self.state_change_wait_time = state_change_wait_time
else:
self.state_change_wait_time = CONF.state_change_wait_time
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(system.REDIS_CONFIG),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
config_value_mappings = {'yes': True, 'no': False, "''": None}
self._value_converter = StringConverter(config_value_mappings)
self.configuration_manager = ConfigurationManager(
system.REDIS_CONFIG,
system.REDIS_OWNER,
system.REDIS_OWNER,
PropertiesCodec(unpack_singletons=False,
string_mappings=config_value_mappings),
requires_root=True,
override_strategy=OneFileOverrideStrategy(revision_dir))
self.admin = self._build_admin_client()
self.status = RedisAppStatus(self.admin)
def __init__(self, status):
self.state_change_wait_time = CONF.state_change_wait_time
self.status = status
revision_dir = \
guestagent_utils.build_file_path(
os.path.join(MOUNT_POINT,
os.path.dirname(system.VERTICA_ADMIN)),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
if not operating_system.exists(FAKE_CFG):
operating_system.write_file(FAKE_CFG, '', as_root=True)
operating_system.chown(FAKE_CFG,
system.VERTICA_ADMIN,
system.VERTICA_ADMIN_GRP,
as_root=True)
operating_system.chmod(FAKE_CFG,
FileMode.ADD_GRP_RX_OTH_RX(),
as_root=True)
self.configuration_manager = \
ConfigurationManager(FAKE_CFG, system.VERTICA_ADMIN,
system.VERTICA_ADMIN_GRP,
PropertiesCodec(delimiter='='),
requires_root=True,
override_strategy=ImportOverrideStrategy(
revision_dir, "cnf"))
def __init__(self, state_change_wait_time=None):
"""
Sets default status and state_change_wait_time
"""
if state_change_wait_time:
self.state_change_wait_time = state_change_wait_time
else:
self.state_change_wait_time = CONF.state_change_wait_time
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(system.REDIS_CONFIG),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
config_value_mappings = {'yes': True, 'no': False, "''": None}
self._value_converter = StringConverter(config_value_mappings)
self.configuration_manager = ConfigurationManager(
system.REDIS_CONFIG,
system.REDIS_OWNER, system.REDIS_OWNER,
PropertiesCodec(
unpack_singletons=False,
string_mappings=config_value_mappings
), requires_root=True,
override_strategy=OneFileOverrideStrategy(revision_dir))
self.admin = self._build_admin_client()
self.status = RedisAppStatus(self.admin)
def apply(self, group_name, change_id, options):
revision_file = self._find_revision_file(group_name, change_id)
if revision_file is None:
# Create a new file.
last_revision_index = self._get_last_file_index(group_name)
revision_file = guestagent_utils.build_file_path(
self._revision_dir, '%s-%03d-%s' %
(group_name, last_revision_index + 1, change_id),
self._revision_ext)
else:
# Update the existing file.
current = operating_system.read_file(revision_file,
codec=self._codec)
options = guestagent_utils.update_dict(options, current)
operating_system.write_file(revision_file,
options,
codec=self._codec,
as_root=self._requires_root)
operating_system.chown(revision_file,
self._owner,
self._group,
as_root=self._requires_root)
operating_system.chmod(revision_file,
FileMode.ADD_READ_ALL,
as_root=self._requires_root)
def datastore_log_defs(self):
if not self.appStatus.is_running:
# do nothing if Oracle is not running
return {}
owner = system.ORACLE_INSTANCE_OWNER
group = system.ORACLE_GROUP_OWNER
sid = self.admin.database_name
diag_dest = self.admin.get_parameter('diagnostic_dest')
dbname = sid.lower()
# alert log path:
# <diagnostic_dest>/diag/rdbms/<dbname>/<instname>/alert/log.xml
alert_log_file = self.validate_log_file(
guestagent_utils.build_file_path(
path.join(diag_dest, 'diag', 'rdbms', dbname, sid, 'alert'),
'log', 'xml'
), owner, group=group
)
return {
'alert': {
self.GUEST_LOG_TYPE_LABEL: guest_log.LogType.SYS,
self.GUEST_LOG_USER_LABEL: owner,
self.GUEST_LOG_FILE_LABEL: alert_log_file,
},
}
def assert_module_retrieve(
self, client, instance_id, expected_count, expected_http_code=200, expected_results=None
):
try:
temp_dir = tempfile.mkdtemp()
prefix = "contents"
modretrieve_list = client.instances.module_retrieve(instance_id, directory=temp_dir, prefix=prefix)
self.assert_client_code(expected_http_code, client)
count = len(modretrieve_list)
self.assert_equal(expected_count, count, "Wrong number of modules from retrieve")
expected_results = expected_results or {}
for module_name, filename in modretrieve_list.items():
if module_name in expected_results:
expected = expected_results[module_name]
contents_name = "%s_%s_%s_%s" % (
prefix,
module_name,
expected["datastore"],
expected["datastore_version"],
)
expected_filename = guestagent_utils.build_file_path(temp_dir, contents_name, "dat")
self.assert_equal(expected_filename, filename, "Unexpected retrieve filename")
if "contents" in expected and expected["contents"]:
with open(filename, "rb") as fh:
contents = fh.read()
# convert contents into bytearray to work with py27
# and py34
contents = bytes([ord(item) for item in contents])
expected_contents = bytes([ord(item) for item in expected["contents"]])
self.assert_equal(expected_contents, contents, "Unexpected contents for %s" % module_name)
finally:
operating_system.remove(temp_dir)
def __init__(self,
base_config_path,
owner,
group,
codec,
requires_root=False,
override_strategy=None):
"""
:param base_config_path Path to the configuration file.
:type base_config_path string
:param owner Owner of the configuration files.
:type owner string
:param group Group of the configuration files.
:type group string
:param codec Codec for reading/writing of the particular
configuration format.
:type codec StreamCodec
:param requires_root Whether the manager requires superuser
privileges.
:type requires_root boolean
:param override_strategy Strategy used to manage configuration
overrides (e.g. ImportOverrideStrategy).
Defaults to OneFileOverrideStrategy
if None. This strategy should be
compatible with very much any datastore.
It is recommended each datastore defines
its strategy explicitly to avoid upgrade
compatibility issues in case the default
implementation changes in the future.
:type override_strategy ConfigurationOverrideStrategy
"""
self._base_config_path = base_config_path
self._owner = owner
self._group = group
self._codec = codec
self._requires_root = requires_root
self._value_cache = None
if not override_strategy:
# Use OneFile strategy by default. Store the revisions in a
# sub-directory at the location of the configuration file.
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(base_config_path),
self.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
operating_system.create_directory(revision_dir,
user=owner,
group=group,
force=True,
as_root=requires_root)
self._override_strategy = OneFileOverrideStrategy(revision_dir)
else:
self._override_strategy = override_strategy
self._override_strategy.configure(base_config_path, owner, group,
codec, requires_root)
def build_module_dir(cls, module_type, module_id):
sub_dir = os.path.join(module_type, module_id)
module_dir = guestagent_utils.build_file_path(
cls.MODULE_BASE_DIR, sub_dir)
if not operating_system.exists(module_dir, is_directory=True):
operating_system.create_directory(module_dir, force=True)
return module_dir
def begin_install(self):
"""Called right before DB is prepared."""
prepare_start_file = guestagent_utils.build_file_path(
self.GUESTAGENT_DIR, self.PREPARE_START_FILENAME)
operating_system.write_file(prepare_start_file, '')
self.prepare_completed = False
self.set_status(instance.ServiceStatuses.BUILDING, True)
def begin_install(self):
"""First call of the DB prepare."""
prepare_start_file = guestagent_utils.build_file_path(
self.GUESTAGENT_DIR, self.PREPARE_START_FILENAME)
operating_system.write_file(prepare_start_file, '')
self.__refresh_prepare_completed()
self.set_status(instance.ServiceStatuses.BUILDING, True)
def begin_install(self):
"""Called right before DB is prepared."""
prepare_start_file = guestagent_utils.build_file_path(
self.GUESTAGENT_DIR, self.PREPARE_START_FILENAME)
operating_system.write_file(prepare_start_file, '')
self.prepare_completed = False
self.set_status(instance.ServiceStatuses.BUILDING, True)
def begin_install(self):
"""First call of the DB prepare."""
prepare_start_file = guestagent_utils.build_file_path(
self.GUESTAGENT_DIR, self.PREPARE_START_FILENAME)
operating_system.write_file(prepare_start_file, '')
self.__refresh_prepare_completed()
self.set_status(instance.ServiceStatuses.BUILDING, True)
def apply_next(self, options):
revision_num = self.count_revisions() + 1
revision_file_path = guestagent_utils.build_file_path(
self._revision_dir, self._base_config_name, str(revision_num), self._revision_ext
)
operating_system.write_file(revision_file_path, options, codec=self._codec, as_root=self._requires_root)
operating_system.chown(revision_file_path, self._owner, self._group, as_root=self._requires_root)
operating_system.chmod(revision_file_path, FileMode.ADD_READ_ALL, as_root=self._requires_root)
def _remove_system_tables(self):
"""
Clean up the system keyspace.
System tables are initialized on the first boot.
They store certain properties, such as 'cluster_name',
that cannot be easily changed once afterwards.
The system keyspace needs to be cleaned up first. The
tables will be regenerated on the next startup.
Make sure to also cleanup the commitlog and caches to avoid
startup errors due to inconsistencies.
The service should not be running at this point.
"""
if self.status.is_running:
raise RuntimeError(_("Cannot remove system tables. "
"The service is still running."))
LOG.info(_('Removing existing system tables.'))
system_keyspace_dir = guestagent_utils.build_file_path(
self.cassandra_data_dir, 'system')
commitlog_file = guestagent_utils.build_file_path(
self.cassandra_working_dir, 'commitlog')
chaches_dir = guestagent_utils.build_file_path(
self.cassandra_working_dir, 'saved_caches')
operating_system.remove(system_keyspace_dir,
force=True, recursive=True, as_root=True)
operating_system.remove(commitlog_file,
force=True, recursive=True, as_root=True)
operating_system.remove(chaches_dir,
force=True, recursive=True, as_root=True)
operating_system.create_directory(
system_keyspace_dir,
user=self.cassandra_owner, group=self.cassandra_owner,
force=True, as_root=True)
operating_system.create_directory(
commitlog_file,
user=self.cassandra_owner, group=self.cassandra_owner,
force=True, as_root=True)
operating_system.create_directory(
chaches_dir,
user=self.cassandra_owner, group=self.cassandra_owner,
force=True, as_root=True)
def init_config(self):
if not operating_system.exists(MOUNT_POINT, True):
operating_system.create_directory(MOUNT_POINT,
system.DB2_INSTANCE_OWNER,
system.DB2_INSTANCE_OWNER,
as_root=True)
"""
The database manager configuration file - db2systm is stored under the
/home/db2inst1/sqllib directory. To update the configuration
parameters, DB2 recommends using the command - UPDATE DBM CONFIGURATION
commands instead of directly updating the config file.
The existing PropertiesCodec implementation has been reused to handle
text-file operations. Configuration overrides are implemented using
the ImportOverrideStrategy of the guestagent configuration manager.
"""
LOG.debug("Initialize DB2 configuration")
revision_dir = (
guestagent_utils.build_file_path(
os.path.join(MOUNT_POINT,
os.path.dirname(system.DB2_INSTANCE_OWNER)),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
)
if not operating_system.exists(FAKE_CFG):
operating_system.write_file(FAKE_CFG, '', as_root=True)
operating_system.chown(FAKE_CFG, system.DB2_INSTANCE_OWNER,
system.DB2_INSTANCE_OWNER, as_root=True)
self.configuration_manager = (
ConfigurationManager(FAKE_CFG, system.DB2_INSTANCE_OWNER,
system.DB2_INSTANCE_OWNER,
PropertiesCodec(delimiter='='),
requires_root=True,
override_strategy=ImportOverrideStrategy(
revision_dir, "cnf"))
)
'''
Below we are getting the database manager default configuration and
saving it to the DB2_DEFAULT_CFG file. This is done to help with
correctly resetting the configurations to the original values when
user wants to detach a user-defined configuration group from an
instance. DB2 provides a command to reset the database manager
configuration parameters (RESET DBM CONFIGURATION) but this command
resets all the configuration parameters to the system defaults. When
we build a DB2 guest image there are certain configurations
parameters like SVCENAME which we set so that the instance can start
correctly. Hence resetting this value to the system default will
render the instance in an unstable state. Instead, the recommended
way for resetting a subset of configuration parameters is to save
the output of GET DBM CONFIGURATION of the original configuration
and then call UPDATE DBM CONFIGURATION to reset the value.
http://www.ibm.com/support/knowledgecenter/SSEPGG_10.5.0/
com.ibm.db2.luw.admin.cmd.doc/doc/r0001970.html
'''
if not operating_system.exists(DB2_DEFAULT_CFG):
run_command(system.GET_DBM_CONFIGURATION % {
"dbm_config": DB2_DEFAULT_CFG})
self.process_default_dbm_config()
def build_log_file_name(self, log_name, owner, datastore_dir=None):
"""Build a log file name based on the log_name and make sure the
directories exist and are accessible by owner.
"""
if datastore_dir is None:
base_dir = self.GUEST_LOG_BASE_DIR
if not operating_system.exists(base_dir, is_directory=True):
operating_system.create_directory(
base_dir, user=owner, group=owner, force=True,
as_root=True)
datastore_dir = guestagent_utils.build_file_path(
base_dir, self.GUEST_LOG_DATASTORE_DIRNAME)
if not operating_system.exists(datastore_dir, is_directory=True):
operating_system.create_directory(
datastore_dir, user=owner, group=owner, force=True,
as_root=True)
log_file_name = guestagent_utils.build_file_path(
datastore_dir, '%s-%s.log' % (self.manager, log_name))
return self.validate_log_file(log_file_name, owner)
def __init__(self, base_config_path, owner, group, codec,
requires_root=False, override_strategy=None):
"""
:param base_config_path Path to the configuration file.
:type base_config_path string
:param owner Owner of the configuration files.
:type owner string
:param group Group of the configuration files.
:type group string
:param codec Codec for reading/writing of the particular
configuration format.
:type codec StreamCodec
:param requires_root Whether the manager requires superuser
privileges.
:type requires_root boolean
:param override_strategy Strategy used to manage configuration
overrides (e.g. ImportOverrideStrategy).
Defaults to OneFileOverrideStrategy
if None. This strategy should be
compatible with very much any datastore.
It is recommended each datastore defines
its strategy explicitly to avoid upgrade
compatibility issues in case the default
implementation changes in the future.
:type override_strategy ConfigurationOverrideStrategy
"""
self._base_config_path = base_config_path
self._owner = owner
self._group = group
self._codec = codec
self._requires_root = requires_root
self._value_cache = None
if not override_strategy:
# Use OneFile strategy by default. Store the revisions in a
# sub-directory at the location of the configuration file.
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(base_config_path),
self.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
operating_system.create_directory(
revision_dir, user=owner, group=group, force=True,
as_root=requires_root)
self._override_strategy = OneFileOverrideStrategy(revision_dir)
else:
self._override_strategy = override_strategy
self._override_strategy.configure(
base_config_path, owner, group, codec, requires_root)
def __init__(self):
self.state_change_wait_time = CONF.state_change_wait_time
self.status = CassandraAppStatus(self.get_current_superuser())
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(self.cassandra_conf),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
self.configuration_manager = ConfigurationManager(
self.cassandra_conf,
self.cassandra_owner, self.cassandra_owner,
SafeYamlCodec(default_flow_style=False), requires_root=True,
override_strategy=OneFileOverrideStrategy(revision_dir))
def _init_overrides_dir(cls):
"""Initialize a directory for configuration overrides.
"""
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(CONFIG_FILE), ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR
)
if not os.path.exists(revision_dir):
operating_system.create_directory(
revision_dir, user=system.MONGO_USER, group=system.MONGO_USER, force=True, as_root=True
)
return revision_dir
def __init__(self, *args, **kwargs):
super(PgSqlConfig, self).__init__(*args, **kwargs)
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(self.pgsql_config),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
self._configuration_manager = ConfigurationManager(
self.pgsql_config, self.PGSQL_OWNER, self.PGSQL_OWNER,
PropertiesCodec(
delimiter='=',
string_mappings={'on': True, 'off': False, "''": None}),
requires_root=True,
override_strategy=OneFileOverrideStrategy(revision_dir))
def apply_next(self, options):
revision_num = self.count_revisions() + 1
old_revision_backup = guestagent_utils.build_file_path(
self._revision_backup_dir, self._base_config_name, str(revision_num), self._BACKUP_EXT
)
operating_system.copy(
self._base_config_path, old_revision_backup, force=True, preserve=True, as_root=self._requires_root
)
current = operating_system.read_file(self._base_config_path, codec=self._codec)
guestagent_utils.update_dict(options, current)
operating_system.write_file(self._base_config_path, current, codec=self._codec, as_root=self._requires_root)
operating_system.chown(self._base_config_path, self._owner, self._group, as_root=self._requires_root)
operating_system.chmod(self._base_config_path, FileMode.ADD_READ_ALL, as_root=self._requires_root)
def guestagent_log_defs(self):
"""These are log files that should be available on every Trove
instance. By definition, these should be of type LogType.SYS
"""
log_dir = CONF.get('log_dir', '/var/log/trove/')
log_file = CONF.get('log_file', 'trove-guestagent.log')
guestagent_log = guestagent_utils.build_file_path(log_dir, log_file)
return {
self.GUEST_LOG_DEFS_GUEST_LABEL: {
self.GUEST_LOG_TYPE_LABEL: guest_log.LogType.SYS,
self.GUEST_LOG_USER_LABEL: None,
self.GUEST_LOG_FILE_LABEL: guestagent_log,
},
}
def guestagent_log_defs(self):
"""These are log files that should be available on every Trove
instance. By definition, these should be of type LogType.SYS
"""
log_dir = CONF.log_dir or '/var/log/trove/'
log_file = CONF.log_file or 'trove-guestagent.log'
guestagent_log = guestagent_utils.build_file_path(log_dir, log_file)
return {
self.GUEST_LOG_DEFS_GUEST_LABEL: {
self.GUEST_LOG_TYPE_LABEL: guest_log.LogType.SYS,
self.GUEST_LOG_USER_LABEL: None,
self.GUEST_LOG_FILE_LABEL: guestagent_log,
},
}
def _init_overrides_dir(cls):
"""Initialize a directory for configuration overrides.
"""
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(system.REDIS_CONFIG),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
if not os.path.exists(revision_dir):
operating_system.create_directory(
revision_dir,
user=system.REDIS_OWNER, group=system.REDIS_OWNER,
force=True, as_root=True)
return revision_dir
def __init__(self):
"""By default login with root no password for initial setup."""
self.state_change_wait_time = CONF.state_change_wait_time
self.status = CassandraAppStatus(self.get_current_superuser())
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(self.cassandra_conf),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
self.configuration_manager = ConfigurationManager(
self.cassandra_conf,
self.cassandra_owner,
self.cassandra_owner,
SafeYamlCodec(default_flow_style=False),
requires_root=True,
override_strategy=OneFileOverrideStrategy(revision_dir))
def __init__(self):
self.state_change_wait_time = CONF.state_change_wait_time
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(CONFIG_FILE),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
self.configuration_manager = ConfigurationManager(
CONFIG_FILE, system.MONGO_USER, system.MONGO_USER,
SafeYamlCodec(default_flow_style=False),
requires_root=True,
override_strategy=OneFileOverrideStrategy(revision_dir))
self.is_query_router = False
self.is_cluster_member = False
self.status = MongoDBAppStatus()
def __init__(self):
super(PgSqlApp, self).__init__()
self._current_admin_user = None
self.status = PgSqlAppStatus(self.pgsql_extra_bin_dir)
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(self.pgsql_config),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
self.configuration_manager = ConfigurationManager(
self.pgsql_config, self.pgsql_owner, self.pgsql_owner,
PropertiesCodec(
delimiter='=',
string_mappings={'on': True, 'off': False, "''": None}),
requires_root=True,
override_strategy=OneFileOverrideStrategy(revision_dir))
def apply_next(self, options):
revision_num = self.count_revisions() + 1
revision_file_path = guestagent_utils.build_file_path(
self._revision_dir, self._base_config_name, str(revision_num),
self._revision_ext)
operating_system.write_file(revision_file_path,
options,
codec=self._codec,
as_root=self._requires_root)
operating_system.chown(revision_file_path,
self._owner,
self._group,
as_root=self._requires_root)
operating_system.chmod(revision_file_path,
FileMode.ADD_READ_ALL,
as_root=self._requires_root)
def __init__(self):
self.state_change_wait_time = CONF.state_change_wait_time
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(CONFIG_FILE),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
self.configuration_manager = ConfigurationManager(
CONFIG_FILE,
system.MONGO_USER,
system.MONGO_USER,
SafeYamlCodec(default_flow_style=False),
requires_root=True,
override_strategy=OneFileOverrideStrategy(revision_dir))
self.is_query_router = False
self.is_cluster_member = False
self.status = MongoDBAppStatus()
def post_restore(self):
try:
# Root enabled for the backup
pwd_file = guestagent_utils.build_file_path(
system.COUCHBASE_DUMP_DIR, self.app.SECRET_KEY_FILE)
if os.path.exists(pwd_file):
with open(pwd_file, "r") as f:
pw = f.read().rstrip("\n")
self.app.reset_admin_credentials(password=pw)
# Iterate through each bucket config
buckets_json = system.COUCHBASE_DUMP_DIR + system.BUCKETS_JSON
with open(buckets_json, "r") as f:
out = f.read()
if out == "[]":
# No buckets or data to restore. Done.
return
d = json.loads(out)
for i in range(len(d)):
bucket_name = d[i]["name"]
bucket_type = d[i]["bucketType"]
if bucket_type == "membase":
bucket_type = "couchbase"
if d[i]["authType"] != "none":
bucket_password = d[i]["saslPassword"]
# SASL buckets can be only on this port.
bucket_port = "11211"
else:
bucket_password = None
bucket_port = d[i]["proxyPort"]
replica_count = d[i]["replicaNumber"]
enable_index_replica = 1 if d[i]["replicaIndex"] else 0
self._create_restore_bucket(bucket_name, bucket_password,
bucket_port, bucket_type,
enable_index_replica,
CONF.couchbase.eviction_policy,
replica_count)
self.run_cbrestore(bucket_name)
except exception.ProcessExecutionError as p:
LOG.error(p)
raise base.RestoreError("Couchbase restore failed.")
def __init__(self):
super(PgSqlApp, self).__init__()
self._current_admin_user = None
self.status = PgSqlAppStatus(self.pgsql_extra_bin_dir)
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(self.pgsql_config),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
self.configuration_manager = ConfigurationManager(
self.pgsql_config,
self.pgsql_owner,
self.pgsql_owner,
PropertiesCodec(delimiter='=',
string_mappings={
'on': True,
'off': False,
"''": None
}),
requires_root=True,
override_strategy=OneFileOverrideStrategy(revision_dir))
def __init__(self, status):
self.state_change_wait_time = CONF.state_change_wait_time
self.status = status
revision_dir = \
guestagent_utils.build_file_path(
os.path.join(MOUNT_POINT,
os.path.dirname(system.VERTICA_ADMIN)),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
if not operating_system.exists(FAKE_CFG):
operating_system.write_file(FAKE_CFG, '', as_root=True)
operating_system.chown(FAKE_CFG, system.VERTICA_ADMIN,
system.VERTICA_ADMIN_GRP, as_root=True)
operating_system.chmod(FAKE_CFG, FileMode.ADD_GRP_RX_OTH_RX(),
as_root=True)
self.configuration_manager = \
ConfigurationManager(FAKE_CFG, system.VERTICA_ADMIN,
system.VERTICA_ADMIN_GRP,
PropertiesCodec(delimiter='='),
requires_root=True,
override_strategy=ImportOverrideStrategy(
revision_dir, "cnf"))
def end_install(self, error_occurred=False, post_processing=False):
"""Called after prepare completes."""
# Set the "we're done" flag if there's no error and
# no post_processing is necessary
if not (error_occurred or post_processing):
prepare_end_file = guestagent_utils.build_file_path(
self.GUESTAGENT_DIR, self.PREPARE_END_FILENAME)
operating_system.write_file(prepare_end_file, '')
self.prepare_completed = True
final_status = None
if error_occurred:
final_status = instance.ServiceStatuses.FAILED
elif post_processing:
final_status = instance.ServiceStatuses.INSTANCE_READY
if final_status:
LOG.info(_("Set final status to %s.") % final_status)
self.set_status(final_status, force=True)
else:
self._end_install_or_restart(True)
def end_install(self, error_occurred=False, post_processing=False):
"""Called after prepare has ended."""
# Set the "we're done" flag if there's no error and
# no post_processing is necessary
if not (error_occurred or post_processing):
prepare_end_file = guestagent_utils.build_file_path(
self.GUESTAGENT_DIR, self.PREPARE_END_FILENAME)
operating_system.write_file(prepare_end_file, '')
self.__refresh_prepare_completed()
final_status = None
if error_occurred:
final_status = instance.ServiceStatuses.FAILED
elif post_processing:
final_status = instance.ServiceStatuses.INSTANCE_READY
if final_status:
LOG.info(_("Set final status to %s.") % final_status)
self.set_status(final_status, force=True)
else:
self._end_install_or_restart(True)
def reset_root_password(self, new_password):
host_and_port = 'localhost:%d' % self._http_client_port
cmd = guestagent_utils.build_file_path(self.couchbase_bin_dir,
'cbreset_password')
cmd_tokens = ['sudo', cmd, host_and_port]
child = pexpect.spawn(' '.join(cmd_tokens))
try:
child.expect('.*password.*')
child.sendline(new_password)
child.expect('.*(yes/no).*')
child.sendline('yes')
child.expect('.*successfully.*')
except pexpect.TIMEOUT:
child.delayafterclose = 1
child.delayafterterminate = 1
try:
child.close(force=True)
except pexpect.ExceptionPexpect:
# Close fails to terminate a sudo process on some OSes.
utils.execute_with_timeout(
'kill', str(child.pid),
run_as_root=True, root_helper='sudo')
def apply_next(self, options):
revision_num = self.count_revisions() + 1
old_revision_backup = guestagent_utils.build_file_path(
self._revision_backup_dir, self._base_config_name,
str(revision_num), self._BACKUP_EXT)
operating_system.copy(self._base_config_path,
old_revision_backup,
force=True,
preserve=True,
as_root=self._requires_root)
current = operating_system.read_file(self._base_config_path,
codec=self._codec)
guestagent_utils.update_dict(options, current)
operating_system.write_file(self._base_config_path,
current,
codec=self._codec,
as_root=self._requires_root)
operating_system.chown(self._base_config_path,
self._owner,
self._group,
as_root=self._requires_root)
operating_system.chmod(self._base_config_path,
FileMode.ADD_READ_ALL,
as_root=self._requires_root)
def reset_root_password(self, new_password):
host_and_port = 'localhost:%d' % self._http_client_port
cmd = guestagent_utils.build_file_path(self.couchbase_bin_dir,
'cbreset_password')
cmd_tokens = ['sudo', cmd, host_and_port]
child = pexpect.spawn(' '.join(cmd_tokens))
try:
child.expect('.*password.*')
child.sendline(new_password)
child.expect('.*(yes/no).*')
child.sendline('yes')
child.expect('.*successfully.*')
except pexpect.TIMEOUT:
child.delayafterclose = 1
child.delayafterterminate = 1
try:
child.close(force=True)
except pexpect.ExceptionPexpect:
# Close fails to terminate a sudo process on some OSes.
utils.execute_with_timeout('kill',
str(child.pid),
run_as_root=True,
root_helper='sudo')
def _find_config_file(self, name_pattern):
version_base = guestagent_utils.build_file_path(self.CONFIG_BASE,
self.pg_version[1])
return sorted(operating_system.list_files_in_directory(
version_base, recursive=True, pattern=name_pattern,
as_root=True), key=len)[0]
def get_client_auth_file(self):
return guestagent_utils.build_file_path("~", ".my.cnf")
def get_persistence_filepath(self):
"""Returns the full path to the persistence file."""
return guestagent_utils.build_file_path(
self.get_working_dir(), self.get_db_filename())
def get_client_auth_file(self):
return guestagent_utils.build_file_path("~", ".my.cnf")
class ModuleManager():
"""This is a Manager utility class (mixin) for managing module-related
tasks.
"""
MODULE_APPLY_TO_ALL = 'all'
MODULE_BASE_DIR = guestagent_utils.build_file_path('~', 'modules')
MODULE_CONTENTS_FILENAME = 'contents.dat'
MODULE_RESULT_FILENAME = 'result.json'
@classmethod
def get_current_timestamp(cls):
return datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@classmethod
def apply_module(cls, driver, module_type, name, tenant, datastore,
ds_version, contents, module_id, md5, auto_apply,
visible):
tenant = tenant or cls.MODULE_APPLY_TO_ALL
datastore = datastore or cls.MODULE_APPLY_TO_ALL
ds_version = ds_version or cls.MODULE_APPLY_TO_ALL
module_dir = cls.build_module_dir(module_type, module_id)
data_file = cls.write_module_contents(module_dir, contents, md5)
applied = True
message = None
now = cls.get_current_timestamp()
default_result = cls.build_default_result(module_type, name, tenant,
datastore, ds_version,
module_id, md5, auto_apply,
visible, now)
result = cls.read_module_result(module_dir, default_result)
try:
applied, message = driver.apply(name, datastore, ds_version,
data_file)
except Exception as ex:
LOG.exception(_("Could not apply module '%s'") % name)
applied = False
message = ex.message
finally:
status = 'OK' if applied else 'ERROR'
admin_only = (not visible or tenant == cls.MODULE_APPLY_TO_ALL
or auto_apply)
result['status'] = status
result['message'] = message
result['updated'] = now
result['id'] = module_id
result['md5'] = md5
result['tenant'] = tenant
result['auto_apply'] = auto_apply
result['visible'] = visible
result['admin_only'] = admin_only
cls.write_module_result(module_dir, result)
return result
@classmethod
def build_module_dir(cls, module_type, module_id):
sub_dir = os.path.join(module_type, module_id)
module_dir = guestagent_utils.build_file_path(cls.MODULE_BASE_DIR,
sub_dir)
if not operating_system.exists(module_dir, is_directory=True):
operating_system.create_directory(module_dir, force=True)
return module_dir
@classmethod
def write_module_contents(cls, module_dir, contents, md5):
contents_file = cls.build_contents_filename(module_dir)
operating_system.write_file(contents_file,
contents,
codec=stream_codecs.Base64Codec(),
encode=False)
return contents_file
@classmethod
def build_contents_filename(cls, module_dir):
contents_file = guestagent_utils.build_file_path(
module_dir, cls.MODULE_CONTENTS_FILENAME)
return contents_file
@classmethod
def build_default_result(cls, module_type, name, tenant, datastore,
ds_version, module_id, md5, auto_apply, visible,
now):
admin_only = (not visible or tenant == cls.MODULE_APPLY_TO_ALL
or auto_apply)
result = {
'type': module_type,
'name': name,
'datastore': datastore,
'datastore_version': ds_version,
'tenant': tenant,
'id': module_id,
'md5': md5,
'status': None,
'message': None,
'created': now,
'updated': now,
'removed': None,
'auto_apply': auto_apply,
'visible': visible,
'admin_only': admin_only,
'contents': None,
}
return result
@classmethod
def read_module_result(cls, result_file, default=None):
result_file = cls.get_result_filename(result_file)
result = default
try:
result = operating_system.read_file(
result_file, codec=stream_codecs.JsonCodec())
except Exception:
if not result:
LOG.exception(
_("Could not find module result in %s") % result_file)
raise
return result
@classmethod
def get_result_filename(cls, file_or_dir):
result_file = file_or_dir
if operating_system.exists(file_or_dir, is_directory=True):
result_file = guestagent_utils.build_file_path(
file_or_dir, cls.MODULE_RESULT_FILENAME)
return result_file
@classmethod
def write_module_result(cls, result_file, result):
result_file = cls.get_result_filename(result_file)
operating_system.write_file(result_file,
result,
codec=stream_codecs.JsonCodec())
@classmethod
def read_module_results(cls, is_admin=False, include_contents=False):
"""Read all the module results on the guest and return a list
of them.
"""
results = []
pattern = cls.MODULE_RESULT_FILENAME
result_files = operating_system.list_files_in_directory(
cls.MODULE_BASE_DIR, recursive=True, pattern=pattern)
for result_file in result_files:
result = cls.read_module_result(result_file)
if (not result.get('removed')
and (is_admin or result.get('visible'))):
if include_contents:
codec = stream_codecs.Base64Codec()
if not is_admin and result.get('admin_only'):
contents = (
"Must be admin to retrieve contents for module %s"
% result.get('name', 'Unknown'))
result['contents'] = codec.serialize(contents)
else:
contents_dir = os.path.dirname(result_file)
contents_file = cls.build_contents_filename(
contents_dir)
result['contents'] = operating_system.read_file(
contents_file, codec=codec, decode=False)
results.append(result)
return results
@classmethod
def remove_module(cls, driver, module_type, module_id, name, datastore,
ds_version):
datastore = datastore or cls.MODULE_APPLY_TO_ALL
ds_version = ds_version or cls.MODULE_APPLY_TO_ALL
module_dir = cls.build_module_dir(module_type, module_id)
contents_file = cls.build_contents_filename(module_dir)
if not operating_system.exists(cls.get_result_filename(module_dir)):
raise exception.NotFound(
_("Module '%s' has not been applied") % name)
try:
removed, message = driver.remove(name, datastore, ds_version,
contents_file)
cls.remove_module_result(module_dir)
except Exception:
LOG.exception(_("Could not remove module '%s'") % name)
raise
return removed, message
@classmethod
def remove_module_result(cls, result_file):
now = cls.get_current_timestamp()
result = cls.read_module_result(result_file, None)
result['removed'] = now
cls.write_module_result(result_file, result)
def get_result_filename(cls, file_or_dir):
result_file = file_or_dir
if operating_system.exists(file_or_dir, is_directory=True):
result_file = guestagent_utils.build_file_path(
file_or_dir, cls.MODULE_RESULT_FILENAME)
return result_file
def build_contents_filename(cls, module_dir):
contents_file = guestagent_utils.build_file_path(
module_dir, cls.MODULE_CONTENTS_FILENAME)
return contents_file
