def test_build_hosts_file(self):
expected = [
'10.2.1.1\tmd md0 mongod0 mongod0.dsitest.dev',
'10.2.1.2\tmd1 mongod1 mongod1.dsitest.dev',
'10.2.1.3\tmd2 mongod2 mongod2.dsitest.dev',
'10.2.1.4\tmd3 mongod3 mongod3.dsitest.dev',
'10.2.1.5\tmd4 mongod4 mongod4.dsitest.dev',
'10.2.1.6\tmd5 mongod5 mongod5.dsitest.dev',
'10.2.1.7\tmd6 mongod6 mongod6.dsitest.dev',
'10.2.1.8\tmd7 mongod7 mongod7.dsitest.dev',
'10.2.1.9\tmd8 mongod8 mongod8.dsitest.dev',
'10.2.1.100\tms ms0 mongos0 mongos0.dsitest.dev',
'10.2.1.101\tms1 mongos1 mongos1.dsitest.dev',
'10.2.1.102\tms2 mongos2 mongos2.dsitest.dev',
'10.2.1.51\tcs cs0 configsvr0 configsvr0.dsitest.dev',
'10.2.1.52\tcs1 configsvr1 configsvr1.dsitest.dev',
'10.2.1.53\tcs2 configsvr2 configsvr2.dsitest.dev',
'10.2.1.10\twc wc0 workload_client0 workload_client0.dsitest.dev'
]
config_files = os.path.dirname(os.path.abspath(__file__)) + '/../../docs/config-specs/'
with test_config.in_dir(config_files):
real_config_dict = ConfigDict('infrastructure_provisioning')
real_config_dict.load()
real_config_dict.save = MagicMock(name='save')
provisioner = ip.Provisioner(real_config_dict,
provisioning_file=self.provision_log_path)
hosts_contents = provisioner._build_hosts_file()
self.assertEqual(expected, hosts_contents)
def load_bootstrap(config, directory):
"""
Move specified bootstrap.yml file to correct location for read_runtime_values
"""
# Create directory if it doesn't exist
if not os.path.exists(directory):
os.makedirs(directory)
if 'bootstrap_file' in config:
bootstrap_path = os.path.abspath(
os.path.expanduser(config['bootstrap_file']))
if os.path.isfile(bootstrap_path):
if not bootstrap_path == os.path.abspath(
os.path.join(directory, 'bootstrap.yml')):
if os.path.isfile(
os.path.abspath(
os.path.join(directory, 'bootstrap.yml'))):
LOGGER.critical(
'Attempting to overwrite existing bootstrap.yml file. Aborting.',
directory=directory)
assert False
shutil.copyfile(bootstrap_path,
os.path.join(directory, 'bootstrap.yml'))
else:
LOGGER.critical("Location specified for bootstrap.yml is invalid.")
assert False
else:
bootstrap_path = os.path.abspath(
os.path.expanduser(os.path.join(os.getcwd(), 'bootstrap.yml')))
if os.path.isfile(bootstrap_path):
if not bootstrap_path == os.path.abspath(
os.path.join(directory, 'bootstrap.yml')):
if os.path.isfile(
os.path.abspath(
os.path.join(directory, 'bootstrap.yml'))):
LOGGER.critical(
'Attempting to overwrite existing bootstrap.yml file in %s. '
'Aborting.', directory)
assert False
shutil.copyfile(bootstrap_path,
os.path.join(directory, 'bootstrap.yml'))
current_path = os.getcwd()
os.chdir(directory)
config_dict = ConfigDict('bootstrap')
config_dict.load()
for key in config_dict['bootstrap'].keys():
config[key] = config_dict['bootstrap'][key]
# terraform required_version must be specified, we fail hard if user has tried to unset
config['terraform_version_check'] = \
config_dict['infrastructure_provisioning']['terraform']['required_version']
config['terraform_linux_download'] = \
config_dict['infrastructure_provisioning']['terraform']['linux_download']
config['terraform_mac_download'] = \
config_dict['infrastructure_provisioning']['terraform']['mac_download']
os.chdir(current_path)
return config_dict
def test_setup_overrides_default_username(self):
"""
Testing setup_overrides fails if user doesn't change username
from bootstrap.example.yml default
"""
real_configdict = ConfigDict('bootstrap')
real_configdict.load()
real_configdict.raw['bootstrap']['overrides'] = {
'infrastructure_provisioning': {
'tfvars': {
'tags': {
'owner': 'your.username'
}
}
}
}
test_override_path = os.path.dirname(os.path.abspath(__file__))
with self.assertRaises(AssertionError):
bootstrap.setup_overrides(real_configdict, test_override_path)
# Removing created file
try:
os.remove(os.path.join(test_override_path, 'overrides.yml'))
except OSError:
pass
def test_setup_overrides_file_exists_empty_config(self):
"""
Testing setup_overrides, path = True and config vals not given
"""
real_configdict = ConfigDict('bootstrap')
real_configdict.load()
test_override_path = os.path.dirname(os.path.abspath(__file__))
test_override_str = yaml.dump({}, default_flow_style=False)
# Creating 'overrides.yml' in current dir
with open(os.path.join(test_override_path, 'overrides.yml'), 'w+') as test_override_file:
test_override_file.write(test_override_str)
# Call to setup_overrides updates 'overrides.yml' in current dir
bootstrap.setup_overrides(real_configdict, test_override_path)
test_override_dict = {}
with open(os.path.join(test_override_path, 'overrides.yml'), 'r') as test_override_file:
test_override_dict = yaml.load(test_override_file)
self.assertEqual(test_override_dict, {})
# Removing created file
os.remove(os.path.join(test_override_path, 'overrides.yml'))
def test_find_nested_config_dicts(self):
"""
We check for duplicate ids in lists of lists correctly.
"""
with in_dir(FIXTURE_FILES.fixture_file_path('invalid-ids-in-lists')):
with self.assertRaises(config.InvalidConfigurationException):
conf = ConfigDict('mongodb_setup')
conf.load()
def test_variable_reference_contains_invalid_id(self):
"""
Variable references cannot evaluate to blocks containing duplicate ids.
"""
with in_dir(FIXTURE_FILES.fixture_file_path('nested-invalid-ids')):
with self.assertRaises(config.InvalidConfigurationException):
conf = ConfigDict('mongodb_setup')
conf.load()
def main():
""" Main function """
args = parse_command_line()
setup_logging(args.debug, args.log_file)
config = ConfigDict('infrastructure_provisioning')
config.load()
provisioner = Provisioner(config, verbose=args.debug)
provisioner.provision_resources()
def setUp(self):
"""Init a ConfigDict object and load the configuration files from docs/config-specs/"""
self.restore = dirmarker(
'./../../docs/config-specs/') # Save the old path to restore Note
# that this chdir only works without breaking relative imports
# because it's at the same directory depth
self.conf = ConfigDict('mongodb_setup')
self.conf.load()
self.assertEqual(self.conf.module, 'mongodb_setup')
def test_none_valued_keys(self):
config_dict = ConfigDict('test_control')
config_dict.load()
self.assertEqual(config_dict['runtime']['overridden_none'],
'hey there')
self.assertEqual(config_dict['runtime']['override_with_none'], None)
self.assertEqual(config_dict['runtime']['overridden_dict'], None)
self.assertEqual(config_dict['runtime']['overridden_list'], None)
with self.assertRaises(KeyError):
config_dict['runtime']['nonexistant'] # pylint: disable=pointless-statement
def load_config_dict(module):
"""
Load ConfigDict for the given module with id checks mocked out.
:param str module: Name of module for ConfigDict.
"""
with patch('common.config.ConfigDict.assert_valid_ids'
) as mock_assert_valid_ids:
conf = ConfigDict(module)
conf.load()
mock_assert_valid_ids.assert_called_once()
return conf
def test_setup_overrides_file_exists_config_vals(self):
"""
Testing setup_overrides where path = True and config vals given
"""
real_configdict = ConfigDict('bootstrap')
real_configdict.load()
test_override_path = os.path.dirname(os.path.abspath(__file__))
master_overrides = {}
master_overrides.update({
'infrastructure_provisioning': {
'tfvars': {
'ssh_key_file': 'test_ssh_key_file1.pem',
'ssh_key_name': 'test_ssh_key_name1',
'tags': {
'owner': 'testuser1',
'expire-on-delta': 24
}
}
}
})
real_configdict.raw['bootstrap']['overrides'] = master_overrides
test_override_str = yaml.dump(
{
'infrastructure_provisioning': {
'tfvars': {
'ssh_key_file': 'test_ssh_key_file2.pem',
'ssh_key_name': 'test_ssh_key_name2',
'tags': {
'owner': 'testuser2',
'expire-on-delta': 48
}
}
}
},
default_flow_style=False)
# Creating 'overrides.yml' in current dir
with open(os.path.join(test_override_path, 'overrides.yml'), 'w') as test_override_file:
test_override_file.write(test_override_str)
# Call to setup_overrides updates 'overrides.yml' in current dir
bootstrap.setup_overrides(real_configdict, test_override_path)
test_override_dict = {}
with open(os.path.join(test_override_path, 'overrides.yml'), 'r') as test_override_file:
test_override_dict = yaml.load(test_override_file)
self.assertEqual(test_override_dict, master_overrides)
# Removing created file
os.remove(os.path.join(test_override_path, 'overrides.yml'))
def setUp(self):
""" Init a ConfigDict object and load the configuration files from docs/config-specs/ """
self.old_dir = os.getcwd() # Save the old path to restore
# Note that this chdir only works without breaking relative imports
# because it's at the same directory depth
os.chdir(
os.path.dirname(os.path.abspath(__file__)) +
'/../../docs/config-specs/')
self.config = ConfigDict('mongodb_setup')
self.config.load()
self.parent_dir = os.path.join(os.path.expanduser('~'),
'checkout_repos_test')
self._delete_fixtures()
def test_setup_overrides_no_file_empty_config(self, mock_path_exists):
"""
Testing setup_overrides, path = False and config vals not given
"""
mock_path_exists.return_value = False
real_configdict = ConfigDict('bootstrap')
real_configdict.load()
test_override_path = os.path.dirname(os.path.abspath(__file__))
# Call to setup_overrides creates 'overrides.yml' in current dir
bootstrap.setup_overrides(real_configdict, test_override_path)
self.assertFalse(os.path.exists(os.path.join(test_override_path, 'overrides.yml')))
def setUp(self):
"""
Setup basic environment
"""
# Mocking `ConfigDict.assert_valid_ids` because it enforces structural constraints on yaml
# files that aren't necessary here.
with patch('common.config.ConfigDict.assert_valid_ids'
) as mock_assert_valid_ids:
prev_dir = os.getcwd()
os.chdir(FIXTURE_FILES.fixture_dir_path)
self.config = ConfigDict('test_control')
self.config.load()
mock_assert_valid_ids.assert_called_once()
os.chdir(prev_dir)
def test_setup_hostnames(self, mock_exec_command, mock_create_file, mock_ssh):
_ = mock_ssh
config_files = os.path.dirname(os.path.abspath(__file__)) + '/../../docs/config-specs/'
with test_config.in_dir(config_files):
real_config_dict = ConfigDict('infrastructure_provisioning')
real_config_dict.load()
real_config_dict.save = MagicMock(name='save')
provisioner = ip.Provisioner(real_config_dict,
provisioning_file=self.provision_log_path)
provisioner.setup_hostnames()
out = provisioner.config['infrastructure_provisioning']['out']
self.assertEqual(out['mongod'][0]['private_hostname'], 'mongod0.dsitest.dev')
self.assertEqual(out['configsvr'][2]['private_hostname'], 'configsvr2.dsitest.dev')
self.assertEqual(mock_create_file.call_count, 16)
self.assertEqual(mock_exec_command.call_count, 16)
def test_pre_post_commands_ordering(self, mock_cedar, mock_copy_perf,
mock_check_call, mock_prep_rep,
mock_parse_results, mock_run_test,
mock_pre_post, mock_delays):
"""Test that pre and post commands are called in the right order"""
real_config_dict = ConfigDict('test_control')
real_config_dict.raw = self.config
run_tests(real_config_dict)
# We will check that the calls to run_pre_post_commands() happened in expected order
expected_args = [
'pre_task', 'pre_test', 'post_test', 'between_tests', 'pre_test',
'post_test', 'between_tests', 'pre_test', 'post_test', 'post_task'
]
observed_args = [args[0][0] for args in mock_pre_post.call_args_list]
self.assertEqual(expected_args, observed_args)
def test_load_new(self, mock_path_join):
"""Test loading ConfigDict with old naming convention .yml files"""
os.chdir(
os.path.dirname(os.path.abspath(__file__)) +
'/../tests/test_config_files/new_format')
mock_path_join.return_value = './configurations/defaults.yml'
test_conf = ConfigDict('bootstrap')
test_conf.load()
self.assertFalse('cluster_type' in test_conf.raw['bootstrap'])
self.assertTrue(
'infrastructure_provisioning' in test_conf.raw['bootstrap'])
self.assertFalse('cluster_type' in test_conf.defaults['bootstrap'])
self.assertTrue(
'infrastructure_provisioning' in test_conf.defaults['bootstrap'])
os.chdir(
os.path.dirname(os.path.abspath(__file__)) +
'/../../docs/config-specs/')
def main():
""" Handle the main functionality (parse args /setup logging ) and then start the mongodb
cluster."""
args = parse_command_line()
setup_logging(args.debug, args.log_file)
config = ConfigDict('mongodb_setup')
config.load()
# Delays should be unset at the end of each test_control.py run, but if it didn't complete...
safe_reset_all_delays(config)
# Start MongoDB cluster(s) using config given in mongodb_setup.topology (if any).
# Note: This also installs mongo client binary onto workload client.
mongo = MongodbSetup(config=config)
start_cluster(mongo, config)
def test_cedar_report(self, mock_cedar_report, mock_copy_perf,
mock_check_call, mock_prep_rep, mock_parse_results,
mock_run_test, mock_pre_post, mock_delays):
"""Test that cedar report is called the correct number of times"""
real_config_dict = ConfigDict('test_control')
real_config_dict.raw = self.config
run_tests(real_config_dict)
mock_cedar_test = MagicMock()
mock_parse_results.return_value = (True, [mock_cedar_test])
run_tests(real_config_dict)
mock_cedar_report.assert_called()
mock_cedar_report().add_test.assert_has_calls([
call(mock_cedar_test),
call(mock_cedar_test),
call(mock_cedar_test),
])
mock_cedar_report().write_report.assert_called()
def test_setup_overrides_no_file_config_vals(self, mock_path_exists):
"""
Testing setup_overrides where path = False and config vals given
"""
mock_path_exists.return_value = False
real_configdict = ConfigDict('bootstrap')
real_configdict.load()
master_overrides = {}
master_overrides.update({
'infrastructure_provisioning': {
'tfvars': {
'ssh_key_file': 'test_ssh_key_file.pem',
'ssh_key_name': 'test_ssh_key_name',
'tags': {
'owner': 'testuser',
'expire-on-delta': 24
}
}
}
})
real_configdict.raw['bootstrap'] = {}
real_configdict.raw['bootstrap']['overrides'] = master_overrides
test_override_path = os.path.dirname(os.path.abspath(__file__))
test_override_dict = {}
# Call to setup_overrides creates 'overrides.yml' in current dir
bootstrap.setup_overrides(real_configdict, test_override_path)
with open(os.path.join(test_override_path, 'overrides.yml'), 'r') as test_override_file:
test_override_dict = yaml.load(test_override_file)
self.assertEqual(test_override_dict, master_overrides)
# Removing created file
os.remove(os.path.join(test_override_path, 'overrides.yml'))
def test_run_test_exception(self, mock_cedar, mock_copy_perf,
mock_check_call, mock_prep_rep,
mock_parse_results, mock_pre_post,
mock_delays):
"""
Test CalledProcessErrors with cause run_tests return false but other errors will
cause it to return true
"""
real_config_dict = ConfigDict('test_control')
real_config_dict.raw = self.config
# pylint: disable=bad-continuation
with patch('test_control.run_test',
side_effect=[
subprocess.CalledProcessError(99, 'failed-cmd'), 0, 0
]):
utter_failure = run_tests(real_config_dict)
self.assertFalse(utter_failure)
with patch('test_control.run_test', side_effect=[ValueError(), 0, 0]):
utter_failure = run_tests(real_config_dict)
self.assertTrue(utter_failure)
def test_setup_overrides_type_error(self):
"""
Testing setup_overrides doesn't throw TypeError.
"""
real_configdict = ConfigDict('bootstrap')
real_configdict.load()
real_configdict.raw['bootstrap']['overrides'] = {
'infrastructure_provisioning': {
'tfvars': {
'tags': None
}
}
}
test_override_path = os.path.dirname(os.path.abspath(__file__))
bootstrap.setup_overrides(real_configdict, test_override_path)
# Removing created file
try:
os.remove(os.path.join(test_override_path, 'overrides.yml'))
except OSError:
pass
def main(argv):
""" Main function. Parse command line options, and run tests.
:returns: int the exit status to return to the caller (0 for OK)
"""
parser = argparse.ArgumentParser(description='DSI Test runner')
parser.add_argument('-d',
'--debug',
action='store_true',
help='enable debug output')
parser.add_argument('--log-file', help='path to log file')
args = parser.parse_args(argv)
common.log.setup_logging(args.debug, args.log_file)
config = ConfigDict('test_control')
config.load()
# Delays should be unset at the end of each test_control.py run, but if it didn't complete...
safe_reset_all_delays(config)
error = run_tests(config)
return 1 if error else 0
def main(argv):
"""
Parse args and call workload_setup.yml operations
"""
parser = argparse.ArgumentParser(description='Workload Setup')
parser.add_argument('-d',
'--debug',
action='store_true',
help='enable debug output')
parser.add_argument('--log-file', help='path to log file')
args = parser.parse_args(argv)
setup_logging(args.debug, args.log_file)
config = ConfigDict('workload_setup')
config.load()
# Delays should be unset at the end of each test_control.py run, but if it didn't complete...
safe_reset_all_delays(config)
setup = WorkloadSetupRunner(config)
setup.setup_workloads()
def setUp(self):
''' Load self.config (ConfigDict) and set some other common values '''
self.old_dir = os.getcwd() # Save the old path to restore Note
# that this chdir only works without breaking relative imports
# because it's at the same directory depth
os.chdir(
os.path.dirname(os.path.abspath(__file__)) +
'/../../docs/config-specs/')
self.config = ConfigDict('infrastructure_provisioning')
self.config.load()
cookiejar = requests.cookies.RequestsCookieJar() # pylint: disable=abstract-class-instantiated, line-too-long
request = requests.Request('GET', 'http://ip.42.pl/raw')
request.prepare()
self.response_state = {
'cookies': cookiejar,
'_content': 'ip.42.hostname',
'encoding': 'UTF-8',
'url': u'http://ip.42.pl/raw',
'status_code': 200,
'request': request,
'elapsed': datetime.timedelta(0, 0, 615501),
'headers': {
'Content-Length': '14',
'X-Powered-By': 'PHP/5.6.27',
'Keep-Alive': 'timeout=5, max=100',
'Server':
'Apache/2.4.23 (FreeBSD) OpenSSL/1.0.1l-freebsd PHP/5.6.27',
'Connection': 'Keep-Alive',
'Date': 'Tue, 25 Jul 2017 14:20:06 GMT',
'Content-Type': 'text/html; charset=UTF-8'
},
'reason': 'OK',
'history': []
}
def main(argv):
""" Main function. Parse command line options, and run analysis.
Note that the return value here determines whether Evergreen considers the entire task passed
or failed. Non-zero return value means failure.
:returns: int the exit status to return to the caller (0 for OK)
"""
parser = argparse.ArgumentParser(description='Analyze DSI test results.')
parser.add_argument('-d',
'--debug',
action='store_true',
help='enable debug output')
parser.add_argument('--log-file', help='path to log file')
args = parser.parse_args(argv)
setup_logging(args.debug, args.log_file)
config = ConfigDict('analysis')
config.load()
analyzer = ResultsAnalyzer(config)
analyzer.analyze_all()
return 1 if analyzer.failures > 0 else 0
def main(argv):
""" Main function. parse args and execute
:param argv list the command line arguments excluding the program name. Default is
sys.argv[1:]
"""
args = parse_args(argv)
config = ConfigDict('infrastructure_provisioning').load()
host = args.host
expanded = expand(host)
unaliased = unalias(expanded)
ip_address = lookup_host(unaliased, config)
template = "{ip_address}"
if args.export:
template = "export {host}={ip_address}"
print(template.format(host=host, ip_address=ip_address))
def main(argv):
""" Main function. parse args and execute
:param argv list the command line arguments excluding the program name. Default is
sys.argv[1:]
"""
parser, args = parse_args(argv)
config = ConfigDict('infrastructure_provisioning').load()
if len(args.host) == 1:
host = alias.expand(args.host[0])
host = alias.unalias(host)
cmd = ';'.join(args.command)
remote_cmd(host, cmd, config, args)
else:
if not args.command:
print("You must provide a command with more than one host\n\n")
parser.print_help()
sys.exit(1)
cmd = ';'.join(args.command)
threads = []
for host in args.host:
host = alias.expand(host)
host = alias.unalias(host)
thread = threading.Thread(target=remote_cmd, args=(
host,
cmd,
config,
args,
))
threads.append(thread)
thread.start()
for thread in threads:
thread.join()
class CommandRunnerTestCase(unittest.TestCase):
""" Unit Tests for Host Utils library """
def _delete_fixtures(self):
""" delete fixture path and set filename attribute """
local_host_path = FIXTURE_FILES.fixture_file_path('fixtures')
self.filename = os.path.join(local_host_path, 'file')
shutil.rmtree(os.path.dirname(self.filename), ignore_errors=True)
def setUp(self):
""" Init a ConfigDict object and load the configuration files from docs/config-specs/ """
self.old_dir = os.getcwd() # Save the old path to restore
# Note that this chdir only works without breaking relative imports
# because it's at the same directory depth
os.chdir(os.path.dirname(os.path.abspath(__file__)) + '/../../docs/config-specs/')
self.config = ConfigDict('mongodb_setup')
self.config.load()
self.parent_dir = os.path.join(os.path.expanduser('~'), 'checkout_repos_test')
self._delete_fixtures()
self.reports_container = FIXTURE_FILES.fixture_file_path('container')
self.reports_path = os.path.join(self.reports_container, 'reports_tests')
mkdir_p(self.reports_path)
def tearDown(self):
""" Restore working directory """
shutil.rmtree(self.reports_container)
os.chdir(self.old_dir)
self._delete_fixtures()
@patch("common.command_runner._run_host_command_map")
def test_make_host_runner_str(self, mock_run_host_command_map):
""" Test run RemoteHost.make_host_runner with str"""
with patch('common.host_factory.make_host') as mock_make_host:
mock_target_host = Mock()
mock_make_host.return_value = mock_target_host
dummy_host_info = HostInfo("host_info")
common.command_runner.make_host_runner(dummy_host_info, "command", "test_id",
self.config)
mock_make_host.assert_called_once_with(dummy_host_info, ANY, False)
mock_target_host.run.assert_called_once_with('command')
mock_target_host.close.assert_called_once()
@patch("common.command_runner._run_host_command_map")
def test_make_host_runner_map(self, mock_run_host_command_map):
""" Test run Remotecommon.command_runner.make_host_runner with map"""
with patch('common.host_factory.make_host') as mock_make_host:
command = {}
mock_target_host = Mock()
mock_make_host.return_value = mock_target_host
dummy_host_info = HostInfo("host_name")
common.command_runner.make_host_runner(dummy_host_info, command, "test_id", self.config)
mock_make_host.assert_called_once_with(dummy_host_info, ANY, False)
mock_run_host_command_map.assert_called_once_with(mock_target_host, command, 'test_id',
self.config)
mock_target_host.close.assert_called_once()
def test_run_host_commands(self):
"""Test 2-commands common.command_runner.run_host_commands invocation"""
with patch('common.host_factory.RemoteSSHHost') as mongod:
commands = [
{
'on_workload_client': {
'upload_files': [{
'source': 'src1',
'target': 'dest1'
}]
}
},
{
'on_workload_client': {
'upload_files': [{
'source': 'src2',
'target': 'dest2'
}]
}
},
]
common.command_runner.run_host_commands(commands, self.config, "test_id")
self.assertEqual(mongod.call_count, 2)
def test_run_host_command_map(self):
""" Test run command map not known """
with self.assertRaises(UserWarning):
with patch('common.remote_host.RemoteHost') as mongod:
command = {"garbage": {"remote_path": "mongos.log"}}
common.command_runner._run_host_command_map(mongod, command, "test_id", {})
def __run_host_command_map_ex(self, command, run_return_value=False, exec_return_value=None):
with patch('common.remote_host.RemoteHost') as mongod:
if run_return_value is not None:
mongod.run.return_value = run_return_value
else:
mongod.exec_mongo_command.return_value = exec_return_value
common.command_runner._run_host_command_map(mongod, command, "test_id", {})
def test__exec_ex(self):
""" Test run command map excpetion """
# test upload_files
with self.assertRaisesRegex(common.host_utils.HostException, r'^\(1, .*cowsay moo'):
command = {"exec": 'cowsay moo'}
self.__run_host_command_map_ex(command)
def test__exec_mongo_shell_ex(self):
""" Test run command map excpetion """
with self.assertRaisesRegex(common.host_utils.HostException, r'^\(1, .*this is a script'):
command = {
"exec_mongo_shell": {
"script": "this is a script",
"connection_string": "connection string"
}
}
self.__run_host_command_map_ex(command, run_return_value=None, exec_return_value=1)
def test_upload_repo_files(self):
""" Test run command map upload_repo_files """
root = common.utils.get_dsi_path() + os.sep
# test upload_repo_files
with patch('common.remote_host.RemoteHost') as mongod:
command = {"upload_repo_files": [{"target": "remote_path", "source": "mongos.log"}]}
common.command_runner._run_host_command_map(mongod, command, "test_id", {})
mongod.upload_file.assert_called_once_with(root + "mongos.log", "remote_path")
with patch('common.remote_host.RemoteHost') as mongod:
command = {
"upload_repo_files": [{
"target": "remote_path",
"source": "mongos.log"
}, {
"target": "to",
"source": "from"
}]
}
common.command_runner._run_host_command_map(mongod, command, "test_id", {})
calls = [
mock.call(root + "mongos.log", "remote_path"),
mock.call(root + "from", "to"),
]
mongod.upload_file.assert_has_calls(calls, any_order=True)
def test_upload_files(self):
""" Test run command map upload_files """
# test upload_files
with patch('common.remote_ssh_host.RemoteSSHHost') as mongod:
command = {"upload_files": [{"target": "remote_path", "source": "mongos.log"}]}
common.command_runner._run_host_command_map(mongod, command, "test_id", {})
mongod.upload_file.assert_called_once_with("mongos.log", "remote_path")
with patch('common.remote_ssh_host.RemoteSSHHost') as mongod:
command = {
"upload_files": [{
"source": "mongos.log",
"target": "remote_path"
}, {
"source": "to",
"target": "from"
}]
}
common.command_runner._run_host_command_map(mongod, command, "test_id", {})
calls = [mock.call("mongos.log", "remote_path"), mock.call("to", "from")]
mongod.upload_file.assert_has_calls(calls, any_order=True)
def test_retrieve_files(self):
""" Test run command map retrieve_files """
# retrieve_files tests
with patch('common.remote_host.RemoteHost') as mongod:
mock_retrieve_file = Mock()
mongod.retrieve_path = mock_retrieve_file
command = {"retrieve_files": [{"source": "remote_path", "target": "mongos.log"}]}
mongod.alias = 'host'
common.command_runner._run_host_command_map(mongod, command, "test_id", {})
mock_retrieve_file.assert_any_call("remote_path", "reports/test_id/host/mongos.log")
# retrieve_files tests
with patch('common.remote_host.RemoteHost') as mongod:
mock_retrieve_file = Mock()
mongod.retrieve_path = mock_retrieve_file
command = {"retrieve_files": [{"source": "remote_path", "target": "mongos.log"}]}
mongod.alias = 'host'
common.command_runner._run_host_command_map(mongod, command, "test_id", {})
mock_retrieve_file.assert_any_call("remote_path", "reports/test_id/host/mongos.log")
with patch('common.remote_host.RemoteHost') as mongod:
mock_retrieve_file = Mock()
mongod.retrieve_path = mock_retrieve_file
command = {"retrieve_files": [{"source": "remote_path", "target": "local_path"}]}
mongod.alias = 'host'
common.command_runner._run_host_command_map(mongod, command, "test_id", {})
mock_retrieve_file.assert_any_call("remote_path", "reports/test_id/host/local_path")
with patch('common.remote_host.RemoteHost') as mongod:
mock_retrieve_file = Mock()
mongod.retrieve_path = mock_retrieve_file
mongod.alias = "mongod.0"
common.command_runner._run_host_command_map(mongod, command, "test_id", {})
mock_retrieve_file.assert_any_call("remote_path", "reports/test_id/mongod.0/local_path")
with patch('common.remote_host.RemoteHost') as mongod:
mock_retrieve_file = Mock()
mongod.retrieve_path = mock_retrieve_file
command = {"retrieve_files": [{"source": "remote_path", "target": "./local_path"}]}
mongod.alias = "mongos.0"
common.command_runner._run_host_command_map(mongod, command, "test_id", {})
mock_retrieve_file.assert_any_call("remote_path", "reports/test_id/mongos.0/local_path")
def test_exec(self):
""" Test run command map exec """
# test exec
with patch('common.remote_host.RemoteHost') as mongod:
command = {"exec": "this is a command"}
mongod.run.return_value = True
common.command_runner._run_host_command_map(mongod, command, "test_id", {})
mongod.run.assert_called_once_with("this is a command")
def test_exec_mongo_shell(self):
""" Test run command map exec mongo shell """
# test exec_mongo_shell
with patch('common.remote_host.RemoteHost') as mongod:
command = {
"exec_mongo_shell": {
"script": "this is a script",
"connection_string": "connection string"
}
}
mongod.exec_mongo_command.return_value = 0
common.command_runner._run_host_command_map(mongod, command, "test_id", {})
mongod.exec_mongo_command.assert_called_once_with("this is a script",
connection_string="connection string")
with patch('common.remote_host.RemoteHost') as mongod:
command = {"exec_mongo_shell": {"script": "this is a script"}}
mongod.exec_mongo_command.return_value = 0
common.command_runner._run_host_command_map(mongod, command, "test_id", {})
mongod.exec_mongo_command.assert_called_once_with("this is a script",
connection_string="")
def test_run_upon_error(self):
""" test run_upon_error """
@mock.patch('common.command_runner.prepare_reports_dir')
@mock.patch('common.command_runner.run_pre_post_commands')
def _test_run_upon_error(behave, mock_run_pre_post_commands, mock_prepare_reports_dir):
config = {'mongodb_setup': 'setup'}
if behave:
run_upon_error('mongodb_setup', [], config)
expected = EXCEPTION_BEHAVIOR.EXIT
else:
run_upon_error('mongodb_setup', [], config, EXCEPTION_BEHAVIOR.CONTINUE)
expected = EXCEPTION_BEHAVIOR.CONTINUE
mock_prepare_reports_dir.assert_called_once()
mock_run_pre_post_commands.assert_called_with('upon_error', [], config, expected,
'upon_error/mongodb_setup')
_test_run_upon_error(True)
_test_run_upon_error(False)
@unittest.skip("TODO: Disabled during python3 port. Please fix the mock check.")
@patch('common.command_runner.run_host_command')
def test_run_pre_post(self, mock_run_host_command):
"""Test test_control.run_pre_post_commands()"""
command_dicts = [self.config['test_control'], self.config['mongodb_setup']]
run_pre_post_commands('post_test', command_dicts, self.config, EXCEPTION_BEHAVIOR.EXIT)
# expected_args = ['on_workload_client', 'on_all_servers', 'on_mongod', 'on_configsvr']
expected_args = ['on_mongod', 'on_all_hosts', 'on_all_servers', 'on_mongod', 'on_configsvr']
observed_args = []
for args in mock_run_host_command.call_args_list:
observed_args.append(args[0][0])
self.assertEqual(observed_args, expected_args)
def test_prepare_reports_dir(self):
"""Test test_control.run_test where the exec command returns non-zero"""
previous_directory = os.getcwd()
reports_dir = os.path.join(self.reports_path, 'reports')
reports_tarball = os.path.join(self.reports_container, 'reports.tgz')
def _test_prepare_reports_dir():
try:
os.chdir(self.reports_path)
prepare_reports_dir(reports_dir=reports_dir)
finally:
os.chdir(previous_directory)
self.assertFalse(os.path.exists(reports_tarball))
self.assertTrue(os.path.exists(reports_dir))
self.assertTrue(os.path.islink(reports_dir))
_test_prepare_reports_dir()
touch(reports_tarball)
_test_prepare_reports_dir()
os.remove(reports_dir)
mkdir_p(reports_dir)
self.assertRaises(OSError, _test_prepare_reports_dir)
class HostUtilsTestCase(unittest.TestCase):
""" Unit Tests for Host Utils library """
def _delete_fixtures(self):
""" delete FIXTURE_FILES path and set filename attribute """
local_host_path = FIXTURE_FILES.fixture_file_path('fixtures')
self.filename = os.path.join(local_host_path, 'file')
shutil.rmtree(os.path.dirname(self.filename), ignore_errors=True)
def setUp(self):
""" Init a ConfigDict object and load the configuration files from docs/config-specs/ """
self.old_dir = os.getcwd() # Save the old path to restore
# Note that this chdir only works without breaking relative imports
# because it's at the same directory depth
os.chdir(
os.path.dirname(os.path.abspath(__file__)) +
'/../../docs/config-specs/')
self.config = ConfigDict('mongodb_setup')
self.config.load()
self.parent_dir = os.path.join(os.path.expanduser('~'),
'checkout_repos_test')
self._delete_fixtures()
def tearDown(self):
""" Restore working directory """
os.chdir(self.old_dir)
self._delete_fixtures()
def test_never_timeout(self):
""" test never_timeout"""
self.assertFalse(common.host_utils.never_timeout())
self.assertFalse(common.host_utils.never_timeout())
def test_check_timed_out(self):
""" test check_timed_out"""
start = datetime.now()
self.assertFalse(common.host_utils.check_timed_out(start, 50))
time.sleep(51 / 1000.0)
self.assertTrue(common.host_utils.check_timed_out(start, 50))
def test_create_timer(self):
""" test create_timer """
start = datetime.now()
self.assertEqual(common.host_utils.create_timer(start, None),
common.host_utils.never_timeout)
with patch('common.host_utils.partial') as mock_partial:
self.assertTrue(common.host_utils.create_timer(start, 50))
mock_partial.assert_called_once_with(
common.host_utils.check_timed_out, start, 50)
def test_extract_hosts(self):
""" Test extract hosts using config info """
default_host_info = common.host_utils.HostInfo(
public_ip=None,
# These are the user and key files used by this test.
ssh_user='******',
ssh_key_file=os.path.join(os.path.expanduser('~'), '.ssh',
'linustorvalds.pem'),
category=None,
offset=-1)
def customize_host_info(new_ip, new_category, offset):
new_host_info = copy.copy(default_host_info)
new_host_info.public_ip = new_ip
new_host_info.category = new_category
new_host_info.offset = offset
return new_host_info
mongods = [
customize_host_info('53.1.1.{}'.format(i + 1), 'mongod', i)
for i in range(0, 9)
]
configsvrs = [
customize_host_info('53.1.1.{}'.format(i + 51), "configsvr", i)
for i in range(0, 3)
]
mongos = [
customize_host_info('53.1.1.{}'.format(i + 100), "mongos", i)
for i in range(0, 3)
]
workload_clients = [
customize_host_info('53.1.1.101', 'workload_client', 0)
]
localhost = [
common.host_utils.HostInfo(public_ip='localhost',
category='localhost',
offset=0)
]
self.assertEqual(
common.host_utils.extract_hosts('localhost', self.config),
localhost)
self.assertEqual(
common.host_utils.extract_hosts('workload_client', self.config),
workload_clients)
self.assertEqual(
common.host_utils.extract_hosts('mongod', self.config), mongods)
self.assertEqual(
common.host_utils.extract_hosts('mongos', self.config), mongos)
self.assertEqual(
common.host_utils.extract_hosts('configsvr', self.config),
configsvrs)
self.assertEqual(
common.host_utils.extract_hosts('all_servers', self.config),
mongods + mongos + configsvrs)
self.assertEqual(
common.host_utils.extract_hosts('all_hosts', self.config),
mongods + mongos + configsvrs + workload_clients)
def test_stream_lines(self):
""" Test stream_lines """
source = ['first', 'second', 'third', 'fourth']
destination = MagicMock(name="destination")
destination.write.side_effect = socket.timeout('args')
any_lines = common.host_utils.stream_lines(source, destination)
self.assertEqual(False, any_lines)
destination.write.assert_called_once()
destination = MagicMock(name="destination")
destination.write.side_effect = [
'first', 'second',
socket.timeout('args'), 'third'
]
any_lines = common.host_utils.stream_lines(source, destination)
self.assertEqual(True, any_lines)
calls = [
call('first'),
call('second'),
]
destination.write.assert_has_calls(calls)
