def test_build_restore_command_default(self):
"""Ensure that a restore uses sudo by default"""
cmd = self.default_restore_job._build_restore_cmd()
# sudo is enabled by default
assert evaluate_boolean(self.medusa_config.cassandra.use_sudo)
# Ensure that Kubernetes mode is not enabled in default test config
assert not evaluate_boolean(self.medusa_config.kubernetes.enabled)
assert 'sudo' in cmd
def parse_config(args, config_file):
"""Parse a medusa.ini file and allow to override settings from command line
:param dict args: settings override. Higher priority than settings defined in medusa.ini
:param pathlib.Path config_file: path to medusa.ini file
:return: None
"""
config = _build_default_config()
if config_file is None and not DEFAULT_CONFIGURATION_PATH.exists():
logging.error(
'No configuration file provided via CLI, nor no default file found in {}'
.format(DEFAULT_CONFIGURATION_PATH))
sys.exit(1)
actual_config_file = DEFAULT_CONFIGURATION_PATH if config_file is None else config_file
logging.debug('Loading configuration from {}'.format(actual_config_file))
with actual_config_file.open() as f:
config.read_file(f)
# Override config file settings with command line options
for config_section in config.keys():
# Default section is not used in medusa.ini
if config_section == 'DEFAULT':
continue
settings = CONFIG_SECTIONS[config_section]._fields
config.read_dict({
config_section: {
key: value
for key, value in _zip_fields_with_arg_values(settings, args)
if value is not None
}
})
if evaluate_boolean(config['kubernetes']['enabled']):
if evaluate_boolean(config['cassandra']['use_sudo']):
logging.warning(
'Forcing use_sudo to False because Kubernetes mode is enabled')
config['cassandra']['use_sudo'] = 'False'
resolve_ip_addresses = evaluate_boolean(
config['cassandra']['resolve_ip_addresses'])
config.set('cassandra', 'resolve_ip_addresses',
'True' if resolve_ip_addresses else 'False')
if config['storage']['fqdn'] == socket.getfqdn(
) and not resolve_ip_addresses:
# Use the ip address instead of the fqdn when DNS resolving is turned off
config['storage']['fqdn'] = socket.gethostbyname(socket.getfqdn())
if "CQL_USERNAME" in os.environ:
config['cassandra']['cql_username'] = os.environ["CQL_USERNAME"]
if "CQL_PASSWORD" in os.environ:
config['cassandra']['cql_password'] = os.environ["CQL_PASSWORD"]
return config
def __enter__(self):
self._env = os.environ.copy()
if self._config.api_profile:
self._env['AWS_PROFILE'] = self._config.api_profile
if self._config.key_file:
self._env['AWS_SHARED_CREDENTIALS_FILE'] = self._config.key_file
if self._config.region and self._config.region != "default":
self._env['AWS_REGION'] = self._config.region
elif self._config.storage_provider not in [Provider.S3, "s3_compatible"] and self._config.region == "default":
# Legacy libcloud S3 providers that were tied to a specific region such as s3_us_west_oregon
self._env['AWS_REGION'] = get_driver(self._config.storage_provider).region_name
if self._config.aws_cli_path == 'dynamic':
self._aws_cli_cmd = self.cmd()
else:
self._aws_cli_cmd = [self._config.aws_cli_path]
self.endpoint_url = None
if self._config.host is not None:
self.endpoint_url = '{}:{}'.format(self._config.host, self._config.port) \
if self._config.port is not None else self._config.host
if utils.evaluate_boolean(self._config.secure):
self.endpoint_url = 'https://{}'.format(self.endpoint_url)
else:
self.endpoint_url = 'http://{}'.format(self.endpoint_url)
return self
def delete_snapshot(self, tag):
cmd = self.delete_snapshot_command(tag)
if self.snapshot_exists(tag):
if evaluate_boolean(self.kubernetes_config.enabled):
data = {
"type": "exec",
"mbean": "org.apache.cassandra.db:type=StorageService",
"operation": "clearSnapshot",
"arguments": [tag, []]
}
response = self.__do_post(data)
if response["status"] != 200:
raise Exception("failed to delete snapshot: {}".format(
response["error"]))
else:
if self._is_ccm == 1:
os.popen(cmd).read()
else:
logging.debug('Executing: {}'.format(' '.join(cmd)))
try:
output = subprocess.check_output(
cmd, universal_newlines=True)
logging.debug('nodetool output: {}'.format(output))
except subprocess.CalledProcessError as e:
logging.debug('nodetool resulted in error: {}'.format(
e.output))
logging.warning(
'Medusa may have failed at cleaning up snapshot {}. '
'Check if the snapshot exists and clear it manually '
'by running: {}'.format(tag, ' '.join(cmd)))
def create_snapshot(self, backup_name):
cmd = self.create_snapshot_command(backup_name)
tag = "{}{}".format(self.SNAPSHOT_PREFIX, backup_name)
if not self.snapshot_exists(tag):
# TODO introduce abstraction layer/interface for invoking Cassandra
# Eventually I think we will want to introduce an abstraction layer for Cassandra's
# API that Medusa requires. There should be an implementation for using nodetool,
# one for Jolokia, and a 3rd for the management sidecard used by Cass Operator.
if evaluate_boolean(self.kubernetes_config.enabled):
data = {
"type": "exec",
"mbean": "org.apache.cassandra.db:type=StorageService",
"operation":
"takeSnapshot(java.lang.String,java.util.Map,[Ljava.lang.String;)",
"arguments": [tag, {}, []]
}
response = self.__do_post(data)
if response["status"] != 200:
raise Exception("failed to create snapshot: {}".format(
response["error"]))
else:
if self._is_ccm == 1:
os.popen(cmd).read()
else:
logging.debug('Executing: {}'.format(' '.join(cmd)))
subprocess.check_call(cmd,
stdout=subprocess.DEVNULL,
universal_newlines=True)
return Cassandra.Snapshot(self, tag)
def test_build_restore_command_kubernetes(self):
"""Ensure Kubernetes mode does not generate a command line with sudo"""
medusa_config_file = pathlib.Path(
__file__).parent / "resources/config/medusa-kubernetes.ini"
cassandra_yaml = pathlib.Path(
__file__).parent / "resources/config/cassandra.yaml"
args = {'config_file': str(cassandra_yaml)}
config = parse_config(args, medusa_config_file)
medusa_config = MedusaConfig(
file_path=medusa_config_file,
storage=_namedtuple_from_dict(StorageConfig, config['storage']),
monitoring={},
cassandra=_namedtuple_from_dict(CassandraConfig,
config['cassandra']),
ssh=None,
checks=None,
logging=None,
grpc=_namedtuple_from_dict(GrpcConfig, config['grpc']),
kubernetes=_namedtuple_from_dict(KubernetesConfig,
config['kubernetes']),
)
restore_job = RestoreJob(Mock(), medusa_config, self.tmp_dir, None,
None, False, False, None)
cmd = restore_job._build_restore_cmd()
assert evaluate_boolean(medusa_config.kubernetes.enabled)
assert 'sudo' not in cmd, 'Kubernetes mode should not generate command line with sudo'
assert str(medusa_config_file) in cmd
def test_build_restore_command_without_sudo(self):
"""Ensure that a restore can be done without using sudo"""
config = self._build_config_parser()
config['cassandra']['use_sudo'] = 'False'
medusa_config = MedusaConfig(
file_path=None,
storage=_namedtuple_from_dict(StorageConfig, config['storage']),
monitoring={},
cassandra=_namedtuple_from_dict(CassandraConfig, config['cassandra']),
ssh=None,
checks=None,
logging=None,
grpc=_namedtuple_from_dict(GrpcConfig, config['grpc']),
kubernetes=_namedtuple_from_dict(KubernetesConfig, config['kubernetes']),
)
restore_job = RestoreJob(Mock(), medusa_config, self.tmp_dir, None, None, False, False, None)
cmd = restore_job._build_restore_cmd()
assert not evaluate_boolean(medusa_config.cassandra.use_sudo)
assert 'sudo' not in cmd, 'command line should not contain sudo when use_sudo is explicitly set to False'
def __enter__(self):
if self._config.key_file:
self._env = dict(os.environ, AWS_SHARED_CREDENTIALS_FILE=self._config.key_file)
else:
self._env = dict(os.environ)
if self._config.aws_cli_path == 'dynamic':
self._aws_cli_path = self.find_aws_cli()
else:
self._aws_cli_path = self._config.aws_cli_path
self.endpoint_url = None
if self._config.host is not None:
self.endpoint_url = '{}:{}'.format(self._config.host, self._config.port) \
if self._config.port is not None else self._config.host
if utils.evaluate_boolean(self._config.secure):
self.endpoint_url = 'https://{}'.format(self.endpoint_url)
else:
self.endpoint_url = 'http://{}'.format(self.endpoint_url)
return self
def load_config(args, config_file):
config = configparser.ConfigParser(interpolation=None)
# Set defaults
config['storage'] = {
'host_file_separator': ',',
'max_backup_age': 0,
'max_backup_count': 0,
'api_profile': 'default',
'transfer_max_bandwidth': '50MB/s',
'concurrent_transfers': 1,
'multi_part_upload_threshold': 100 * 1024 * 1024,
'secure': True,
'aws_cli_path': 'aws',
'fqdn': socket.getfqdn(),
'region': 'default',
}
config['logging'] = {
'enabled': 'false',
'file': 'medusa.log',
'level': 'INFO',
'format': '[%(asctime)s] %(levelname)s: %(message)s',
'maxBytes': 20000000,
'backupCount': 50,
}
config['cassandra'] = {
'config_file':
medusa.cassandra_utils.CassandraConfigReader.DEFAULT_CASSANDRA_CONFIG,
'start_cmd': 'sudo service cassandra start',
'stop_cmd': 'sudo service cassandra stop',
'check_running': 'nodetool version',
'is_ccm': 0,
'sstableloader_bin': 'sstableloader',
'resolve_ip_addresses': True
}
config['ssh'] = {
'username': os.environ.get('USER') or '',
'key_file': '',
'port': 22
}
config['checks'] = {
'health_check': 'cql',
'query': '',
'expected_rows': '0',
'expected_result': ''
}
config['monitoring'] = {'monitoring_provider': 'None'}
config['grpc'] = {
'enabled': False,
}
config['kubernetes'] = {
'enabled': False,
'cassandra_url': 'None',
'use_mgmt_api': False
}
if config_file:
logging.debug('Loading configuration from {}'.format(config_file))
config.read_file(config_file.open())
elif DEFAULT_CONFIGURATION_PATH.exists():
logging.debug(
'Loading configuration from {}'.format(DEFAULT_CONFIGURATION_PATH))
config.read_file(DEFAULT_CONFIGURATION_PATH.open())
else:
logging.error(
'No configuration file provided via CLI, nor no default file found in {}'
.format(DEFAULT_CONFIGURATION_PATH))
sys.exit(1)
config.read_dict({
'storage': {
key: value
for key, value in _zip_fields_with_arg_values(
StorageConfig._fields, args) if value is not None
}
})
config.read_dict({
'logging': {
key: value
for key, value in _zip_fields_with_arg_values(
LoggingConfig._fields, args) if value is not None
}
})
config.read_dict({
'ssh': {
key: value
for key, value in _zip_fields_with_arg_values(
SSHConfig._fields, args) if value is not None
}
})
config.read_dict({
'checks': {
key: value
for key, value in _zip_fields_with_arg_values(
ChecksConfig._fields, args) if value is not None
}
})
config.read_dict({
'monitoring': {
key: value
for key, value in _zip_fields_with_arg_values(
MonitoringConfig._fields, args) if value is not None
}
})
config.read_dict({
'grpc': {
key: value
for key, value in _zip_fields_with_arg_values(
GrpcConfig._fields, args) if value is not None
}
})
config.read_dict({
'kubernetes': {
key: value
for key, value in _zip_fields_with_arg_values(
KubernetesConfig._fields, args) if value is not None
}
})
resolve_ip_addresses = evaluate_boolean(
config['cassandra']['resolve_ip_addresses'])
config.set('cassandra', 'resolve_ip_addresses',
'True' if resolve_ip_addresses else 'False')
if config['storage']['fqdn'] == socket.getfqdn(
) and not resolve_ip_addresses:
# Use the ip address instead of the fqdn when DNS resolving is turned off
config['storage']['fqdn'] = socket.gethostbyname(socket.getfqdn())
if "CQL_USERNAME" in os.environ:
config['cassandra']['cql_username'] = os.environ["CQL_USERNAME"]
if "CQL_PASSWORD" in os.environ:
config['cassandra']['cql_password'] = os.environ["CQL_PASSWORD"]
medusa_config = MedusaConfig(
storage=_namedtuple_from_dict(StorageConfig, config['storage']),
cassandra=_namedtuple_from_dict(CassandraConfig, config['cassandra']),
ssh=_namedtuple_from_dict(SSHConfig, config['ssh']),
checks=_namedtuple_from_dict(ChecksConfig, config['checks']),
monitoring=_namedtuple_from_dict(MonitoringConfig,
config['monitoring']),
logging=_namedtuple_from_dict(LoggingConfig, config['logging']),
grpc=_namedtuple_from_dict(GrpcConfig, config['grpc']),
kubernetes=_namedtuple_from_dict(KubernetesConfig,
config['kubernetes']))
for field in ['bucket_name', 'storage_provider']:
if getattr(medusa_config.storage, field) is None:
logging.error(
'Required configuration "{}" is missing in [storage] section.'.
format(field))
sys.exit(2)
for field in ['start_cmd', 'stop_cmd']:
if getattr(medusa_config.cassandra, field) is None:
logging.error(
'Required configuration "{}" is missing in [cassandra] section.'
.format(field))
sys.exit(2)
for field in ['username', 'key_file']:
if getattr(medusa_config.ssh, field) is None:
logging.error(
'Required configuration "{}" is missing in [ssh] section.'.
format(field))
sys.exit(2)
return medusa_config
def parse_config(args, config_file):
"""Parse a medusa.ini file and allow to override settings from command line
:param dict args: settings override. Higher priority than settings defined in medusa.ini
:param pathlib.Path config_file: path to medusa.ini file
:return: None
"""
config = _build_default_config()
if config_file is None and not DEFAULT_CONFIGURATION_PATH.exists():
logging.error(
'No configuration file provided via CLI, nor no default file found in {}'.format(DEFAULT_CONFIGURATION_PATH)
)
sys.exit(1)
actual_config_file = DEFAULT_CONFIGURATION_PATH if config_file is None else config_file
logging.debug('Loading configuration from {}'.format(actual_config_file))
with actual_config_file.open() as f:
config.read_file(f)
# Override config file settings with command line options
for config_section in config.keys():
# Default section is not used in medusa.ini
if config_section == 'DEFAULT':
continue
settings = CONFIG_SECTIONS[config_section]._fields
config.read_dict({config_section: {
key: value
for key, value in _zip_fields_with_arg_values(settings, args)
if value is not None
}})
if evaluate_boolean(config['kubernetes']['enabled']):
if evaluate_boolean(config['cassandra']['use_sudo']):
logging.warning('Forcing use_sudo to False because Kubernetes mode is enabled')
config['cassandra']['use_sudo'] = 'False'
resolve_ip_addresses = evaluate_boolean(config['cassandra']['resolve_ip_addresses'])
config.set('cassandra', 'resolve_ip_addresses', 'True' if resolve_ip_addresses else 'False')
if config['storage']['fqdn'] == socket.getfqdn() and not resolve_ip_addresses:
# Use the ip address instead of the fqdn when DNS resolving is turned off
config['storage']['fqdn'] = socket.gethostbyname(socket.getfqdn())
for config_property in ['cql_username', 'cql_password']:
config_property_upper_old = config_property.upper()
config_property_upper_new = "MEDUSA_{}".format(config_property.upper())
if config_property_upper_old in os.environ:
config['cassandra'][config_property] = os.environ[config_property_upper_old]
logging.warning('The {} environment variable is deprecated and has been replaced by the {} variable'
.format(config_property_upper_old, config_property_upper_new))
if config_property_upper_new in os.environ:
config['cassandra'][config_property] = os.environ[config_property_upper_new]
logging.warning('Both {0} and {1} are defined in the environment; using the value set in {1}'
.format(config_property_upper_old, config_property_upper_new))
for config_property in [
'nodetool_username',
'nodetool_password',
'sstableloader_tspw',
'sstableloader_kspw'
]:
config_property_upper = "MEDUSA_{}".format(config_property.upper())
if config_property_upper in os.environ:
config['cassandra'][config_property] = os.environ[config_property_upper]
return config