def test_run_and_check_errors(self):
scripts_dir = self.useFixture(TempDirectory()).path
scripts = make_scripts(scripts_dir=scripts_dir, with_output=False)
script = scripts[0]
self.assertFalse(run_and_check(
['/bin/bash', '-c', 'echo %s;echo %s >&2;false' % (
script['stdout'], script['stderr'])],
scripts))
self.assertEquals(
'%s\n' % script['stdout'], open(script['stdout_path'], 'r').read())
self.assertEquals(
'%s\n' % script['stderr'], open(script['stderr_path'], 'r').read())
self.assertEquals(
'%s\n%s\n' % (script['stdout'], script['stderr']),
open(script['combined_path'], 'r').read())
for script in scripts:
self.assertThat(
self.mock_output_and_send, MockAnyCall(
'Failed installing package(s) for %s' % script['msg_name'],
exit_status=1, status='INSTALLING', files={
scripts[0]['combined_name']: ('%s\n%s\n' % (
scripts[0]['stdout'],
scripts[0]['stderr'])).encode(),
scripts[0]['stdout_name']: (
'%s\n' % scripts[0]['stdout']).encode(),
scripts[0]['stderr_name']: (
'%s\n' % scripts[0]['stderr']).encode(),
}))
def test_run_scripts_from_metadata_redownloads_after_commiss(self):
scripts_dir = self.useFixture(TempDirectory()).path
self.mock_run_scripts.return_value = 0
testing_scripts = make_scripts()
testing_scripts[0]['parameters'] = {'storage': {'type': 'storage'}}
mock_download_and_extract_tar = self.patch(
maas_run_remote_scripts, 'download_and_extract_tar')
simple_dl_and_extract = lambda *args, **kwargs: self.make_index_json(
scripts_dir, with_commissioning=False,
testing_scripts=testing_scripts)
mock_download_and_extract_tar.side_effect = simple_dl_and_extract
index_json = self.make_index_json(
scripts_dir, testing_scripts=testing_scripts)
# Don't need to give the url, creds, or out_dir as we're not running
# the scripts and sending the results.
run_scripts_from_metadata(None, None, scripts_dir, None)
self.assertThat(
self.mock_run_scripts,
MockAnyCall(
None, None, scripts_dir, None,
index_json['commissioning_scripts'], True))
self.assertThat(self.mock_signal, MockAnyCall(None, None, 'TESTING'))
self.assertThat(
mock_download_and_extract_tar,
MockCalledOnceWith('None/maas-scripts/', None, scripts_dir))
self.assertThat(
self.mock_run_scripts,
MockAnyCall(
None, None, scripts_dir, None,
index_json['testing_scripts'], True))
def setUp(self):
super().setUp()
self.ipc_path = os.path.join(
self.useFixture(TempDirectory()).path, "maas-regiond.sock")
self.patch(ipaddr, "get_ip_addr").return_value = {}
self.patch(ipc, "get_all_interface_addresses")
self.patch(ipc, "get_all_interface_source_addresses")
def test_run_script_timed_out_script(self):
scripts_dir = self.useFixture(TempDirectory()).path
script = make_script(scripts_dir=scripts_dir)
self.mock_capture_script_output.side_effect = TimeoutExpired(
[factory.make_name('arg') for _ in range(3)],
script['timeout_seconds'])
self.args.pop('status')
self.assertFalse(run_script(script, scripts_dir))
self.assertThat(self.mock_output_and_send, MockCallsMatch(
call(
'Starting %s' % script['msg_name'], status='WORKING',
**self.args),
call(
'Timeout(%s) expired on %s' % (
str(timedelta(seconds=script['timeout_seconds'])),
script['msg_name']),
files={
script['combined_name']: script['combined'].encode(),
script['stdout_name']: script['stdout'].encode(),
script['stderr_name']: script['stderr'].encode(),
script['result_name']: script['result'].encode(),
}, status='TIMEDOUT', **self.args),
))
def test_update_metrics(self):
self.patch(metrics, "GLOBAL_LABELS", {"service_type": "rack"})
tempdir = self.useFixture(TempDirectory())
meminfo = Path(tempdir.path) / "meminfo"
meminfo.write_text(
dedent(
"""\
MemTotal: 123 Kb
SwapCached: 456 Kb
VmallocUsed: 789 Kb
HugePages_Total: 321
"""
)
)
prometheus_metrics = create_metrics(
node_metrics_definitions(),
registry=prometheus_client.CollectorRegistry(),
)
update_memory_metrics(prometheus_metrics, path=meminfo)
output = prometheus_metrics.generate_latest().decode("ascii")
self.assertIn(
'maas_node_mem_MemTotal{service_type="rack"} 123.0', output
)
self.assertIn(
'maas_node_mem_SwapCached{service_type="rack"} 456.0', output
)
self.assertIn(
'maas_node_mem_VmallocUsed{service_type="rack"} 789.0', output
)
self.assertIn(
'maas_node_mem_HugePages_Total{service_type="rack"} 321.0', output
)
def test_run_script_uses_timeout_from_parameter(self):
scripts_dir = self.useFixture(TempDirectory()).path
script = make_script(scripts_dir=scripts_dir)
script['parameters'] = {
'runtime': {
'type': 'runtime',
'value': random.randint(0, 1000),
}
}
run_script(script, scripts_dir)
self.assertThat(self.mock_output_and_send, MockCallsMatch(
call('Starting %s' % script['msg_name'], **self.args),
call(
'Finished %s: None' % script['msg_name'], exit_status=None,
files={
script['combined_name']: script['combined'].encode(),
script['stdout_name']: script['stdout'].encode(),
script['stderr_name']: script['stderr'].encode(),
script['result_name']: script['result'].encode(),
}, **self.args),
))
self.assertThat(self.mock_capture_script_output, MockCalledOnceWith(
ANY, script['combined_path'], script['stdout_path'],
script['stderr_path'], script['parameters']['runtime']['value']))
def test_ensure(self):
base_path = Path(self.useFixture(TempDirectory()).path)
scripts_dir = ScriptsDir(base_path=base_path)
scripts_dir.ensure()
self.assertTrue(scripts_dir.scripts.exists())
self.assertTrue(scripts_dir.out.exists())
self.assertTrue(scripts_dir.downloads.exists())
def test_override_config(self):
conf = {
"reporting": {
"maas": {
"token_key": "token-key",
"token_secret": "token-secret",
"consumer_key": "consumer-key",
"consumer_secret": "consumer-secret",
"endpoint": "http://example.com",
}
}
}
tempdir = Path(self.useFixture(TempDirectory()).path)
config_file = tempdir / "conf.yaml"
with config_file.open("w") as fd:
yaml.dump(conf, fd)
ns = parse_args([
"report-results",
"--config",
str(config_file),
"--machine-token",
"new-consumer-key:new-token-key:new-token-secret:new-consumer-secret",
"--metadata-url",
"http://new.example.com",
])
config = get_config(ns)
self.assertEqual(config.credentials.token_key, "new-token-key")
self.assertEqual(config.credentials.token_secret, "new-token-secret")
self.assertEqual(config.credentials.consumer_key, "new-consumer-key")
self.assertEqual(config.credentials.consumer_secret,
"new-consumer-secret")
self.assertEqual(config.metadata_url, "http://new.example.com")
def test_run_scripts_from_metadata(self):
scripts_dir = self.useFixture(TempDirectory()).path
self.mock_run_scripts.return_value = 0
index_json = self.make_index_json(scripts_dir)
mock_download_and_extract_tar = self.patch(
maas_run_remote_scripts, 'download_and_extract_tar')
mock_download_and_extract_tar.side_effect = (
self.mock_download_and_extract_tar)
# Don't need to give the url, creds, or out_dir as we're not running
# the scripts and sending the results.
run_scripts_from_metadata(None, None, scripts_dir, None)
self.assertThat(
self.mock_run_scripts,
MockAnyCall(
None, None, scripts_dir, None,
index_json['commissioning_scripts'], True))
self.assertThat(
self.mock_run_scripts,
MockAnyCall(
None, None, scripts_dir, None,
index_json['testing_scripts'], True))
self.assertThat(self.mock_signal, MockAnyCall(None, None, 'TESTING'))
self.assertThat(mock_download_and_extract_tar, MockCalledOnceWith(
'None/maas-scripts/', None, scripts_dir))
def test_path_is_decoded_using_filesystem_encoding(self):
sys = self.patch(fixtures, "sys")
sys.getfilesystemencoding.return_value = "rot13"
with TempDirectory() as fixture:
self.assertIsInstance(fixture.path, unicode)
self.assertThat(fixture.path, Not(DirExists()))
self.assertThat(fixture.path.decode("rot13"), DirExists())
def make_dir(self):
"""Create a temporary directory.
This is a convenience wrapper around a fixture incantation. That's
the only reason why it's on the test case and not in a factory.
"""
return self.useFixture(TempDirectory()).path
def test_install_dependencies_snap_str_dict(self):
mock_run_and_check = self.patch(
maas_run_remote_scripts, 'run_and_check')
scripts_dir = self.useFixture(TempDirectory()).path
scripts = make_scripts(scripts_dir=scripts_dir, with_output=False)
packages = [
{'name': factory.make_name('pkg')},
{
'name': factory.make_name('pkg'),
'channel': random.choice([
'edge', 'beta', 'candidate', 'stable']),
},
{
'name': factory.make_name('pkg'),
'channel': random.choice([
'edge', 'beta', 'candidate', 'stable']),
'mode': random.choice(['dev', 'jail']),
},
{
'name': factory.make_name('pkg'),
'channel': random.choice([
'edge', 'beta', 'candidate', 'stable']),
'mode': random.choice(['dev', 'jail']),
},
]
for script in scripts:
script['packages'] = {'snap': packages}
self.assertTrue(install_dependencies(scripts))
for script in scripts:
self.assertThat(self.mock_output_and_send, MockAnyCall(
'Installing snap packages for %s' % script['msg_name'],
True, status='INSTALLING'))
# Verify cleanup
self.assertFalse(os.path.exists(script['combined_path']))
self.assertFalse(os.path.exists(script['stdout_path']))
self.assertFalse(os.path.exists(script['stderr_path']))
self.assertThat(mock_run_and_check, MockAnyCall(
['snap', 'install', packages[0]['name']], scripts, True, True))
self.assertThat(mock_run_and_check, MockAnyCall(
[
'snap', 'install', packages[1]['name'],
'--%s' % packages[1]['channel']
],
scripts, True, True))
self.assertThat(mock_run_and_check, MockAnyCall(
[
'snap', 'install', packages[2]['name'],
'--%s' % packages[2]['channel'],
'--%smode' % packages[2]['mode'],
],
scripts, True, True))
self.assertThat(mock_run_and_check, MockAnyCall(
[
'snap', 'install', packages[3]['name'],
'--%s' % packages[3]['channel'],
'--%smode' % packages[3]['mode'],
],
scripts, True, True))
def test_ensure_clears_existing_content(self):
base_path = Path(self.useFixture(TempDirectory()).path)
scripts_dir = ScriptsDir(base_path=base_path)
scripts_dir.scripts.mkdir()
a_file = scripts_dir.scripts / "a-file"
a_file.touch()
scripts_dir.ensure()
self.assertFalse(a_file.exists())
def setUp(self):
"""Setup a special database cluster to perform the tests."""
super(TestDBUpgrade, self).setUp()
self.datadir = self.useFixture(TempDirectory()).path
self.cluster = self.useFixture(ClusterFixture(self.datadir))
self.useFixture(RegionConfigurationFixture(
database_name=self.dbname, database_user=None,
database_pass=None, database_host=self.datadir))
def test_update_metrics(self):
self.patch(metrics, "GLOBAL_LABELS", {"service_type": "rack"})
tempdir = self.useFixture(TempDirectory())
stat = Path(tempdir.path) / "stat"
stat.write_text(
dedent("""\
cpu 111 222 333 444 555 666 7 888 9 11
cpu0 222 333 444 555 666 777 8 999 1 22
cpu1 222 333 444 555 666 777 8 999 1 22
other line
other line
"""))
prometheus_metrics = create_metrics(
node_metrics_definitions(),
registry=prometheus_client.CollectorRegistry(),
)
update_cpu_metrics(prometheus_metrics, path=stat)
output = prometheus_metrics.generate_latest().decode("ascii")
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="user"} 1.11',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="nice"} 2.22',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="system"} 3.33',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="idle"} 4.44',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="iowait"} 5.55',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="irq"} 6.66',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="softirq"} 0.07',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="steal"} 8.88',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="guest"} 0.09',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="guest_nice"} 0.11',
output,
)
def setUp(self):
super().setUp()
self.certificates_dir = self.useFixture(TempDirectory()).path
maas_certificates.MAAS_PRIVATE_KEY = os.path.join(
self.certificates_dir, "maas.key")
maas_certificates.MAAS_PUBLIC_KEY = os.path.join(
self.certificates_dir, "maas.pub")
maas_certificates.MAAS_CERTIFICATE = os.path.join(
self.certificates_dir, "maas.crt")
def test__quietly_exits_in_container(self):
script_dir = self.useFixture(TempDirectory()).path
script_path = os.path.join(script_dir, '00-maas-07-block-devices')
virtuality_result_path = os.path.join(script_dir, 'out')
os.makedirs(virtuality_result_path)
virtuality_result_path = os.path.join(virtuality_result_path,
'00-maas-02-virtuality')
open(virtuality_result_path, 'w').write('lxc\n')
self.assertItemsEqual(
[], self.call_gather_physical_block_devices(file_path=script_path))
def test_download_and_extract_tar_returns_false_on_no_content(self):
self.patch(maas_run_remote_scripts.sys.stdout, 'write')
scripts_dir = self.useFixture(TempDirectory()).path
mock_geturl = self.patch(maas_run_remote_scripts, 'geturl')
mm = MagicMock()
mm.status = int(http.client.NO_CONTENT)
mm.read.return_value = b'No content'
mock_geturl.return_value = mm
# geturl is mocked out so we don't need to give a url or creds.
self.assertFalse(download_and_extract_tar(None, None, scripts_dir))
def make_content(self, size=None):
tmpdir = self.useFixture(TempDirectory()).path
if size is None:
size = randint(1024, 2048)
data = factory.make_bytes(size)
content_path = os.path.join(tmpdir, "content")
with open(content_path, "wb") as stream:
stream.write(data)
sha256 = hashlib.sha256()
sha256.update(data)
return size, sha256.hexdigest(), partial(open, content_path, "rb")
def set_up_config(self):
if self.homedir is None:
self.homedir = self.useFixture(TempDirectory()).path
if self.access_log_file is None:
self.access_log_file = os.path.join(self.homedir,
"nginx.access.log")
if self.error_log_file is None:
self.error_log_file = os.path.join(self.homedir, "nginx.error.log")
self.nginx_file = os.path.join(self.homedir,
os.path.basename(self.NGINX_PATH))
self.conf_file = os.path.join(self.homedir, "nginx.conf")
self.pid_file = os.path.join(self.homedir, "nginx.pid")
def set_up_config(self):
if self.port is None:
[self.port] = allocate_ports("localhost")
if self.rndc_port is None:
[self.rndc_port] = allocate_ports("localhost")
if self.homedir is None:
self.homedir = self.useFixture(TempDirectory()).path
if self.log_file is None:
self.log_file = os.path.join(self.homedir, "named.log")
self.named_file = os.path.join(self.homedir,
os.path.basename(self.NAMED_PATH))
self.conf_file = os.path.join(self.homedir, "named.conf")
self.rndcconf_file = os.path.join(self.homedir, "rndc.conf")
def test_delete_file_disappeared(self):
real_os_remove = os.remove
def mock_os_remove(path):
# remove it twice, so that FileNotFoundError is raised
real_os_remove(path)
real_os_remove(path)
self.patch(os, 'remove', mock_os_remove)
tmpdir = Path(self.useFixture(TempDirectory()).path)
file1 = tmpdir / 'histogram_{}.db'.format(self.get_unused_pid())
file1.touch()
self.assertIsNone(clean_prometheus_dir(str(tmpdir)))
def test_run_scripts_from_metadata_does_nothing_on_empty(self):
scripts_dir = self.useFixture(TempDirectory()).path
self.patch(maas_run_remote_scripts, 'run_scripts')
mock_signal = self.patch(maas_run_remote_scripts, 'signal')
self.make_index_json(scripts_dir, False, False)
# Don't need to give the url, creds, or out_dir as we're not running
# the scripts and sending the results.
run_scripts_from_metadata(None, None, scripts_dir, None)
self.assertThat(
mock_signal,
MockCalledOnceWith(None, None, 'OK',
'All scripts successfully ran'))
def setUp(self):
super(ConfigFixture, self).setUp()
# Create a real configuration file, and populate it.
self.dir = self.useFixture(TempDirectory()).path
self.filename = path.join(self.dir, "config.yaml")
with open(self.filename, "wb") as stream:
yaml.safe_dump(self.config, stream=stream)
# Export this filename to the environment, so that subprocesses will
# pick up this configuration. Define the new environment as an
# instance variable so that users of this fixture can use this to
# extend custom subprocess environments.
self.environ = {"MAAS_PROVISIONING_SETTINGS": self.filename}
for name, value in self.environ.items():
self.useFixture(EnvironmentVariableFixture(name, value))
def test_makeService_cleanup_prometheus_dir(self):
tmpdir = Path(self.useFixture(TempDirectory()).path)
self.useFixture(
EnvironmentVariable("prometheus_multiproc_dir", str(tmpdir)))
pid = os.getpid()
file1 = tmpdir / "histogram_{}.db".format(pid)
file1.touch()
file2 = tmpdir / "histogram_{}.db".format(self.get_unused_pid())
file2.touch()
service_maker = ProvisioningServiceMaker("Harry", "Hill")
service_maker.makeService(Options(), clock=None)
self.assertTrue(file1.exists())
self.assertFalse(file2.exists())
def test_update_targets_only_runs_when_conf_exists(self):
# Regression test for LP:1655721
temp_dir = self.useFixture(TempDirectory()).path
self.useFixture(ClusterConfigurationFixture(tftp_root=temp_dir))
mock_ensureService = self.patch(boot_resources.service_monitor,
"ensureService")
mock_call_and_check = self.patch(boot_resources, "call_and_check")
mock_path_exists = self.patch(boot_resources.os.path, 'exists')
mock_path_exists.return_value = False
boot_resources.update_targets_conf(temp_dir)
self.assertThat(mock_ensureService, MockCalledOnceWith("tgt"))
self.assertThat(mock_path_exists,
MockCalledOnceWith(os.path.join(temp_dir, 'maas.tgt')))
self.assertThat(mock_call_and_check, MockNotCalled())
def writes_file(self):
tempdir = Path(self.useFixture(TempDirectory()).path)
token_info = {
"token_key": "tk",
"token_secret": "ts",
"consumer_key": "ck",
}
path = write_token("myhost", token_info, basedir=tempdir)
self.assertEqual(
yaml.load(path),
{"reporting": {
"maas": token_info
}},
)
def setUp(self):
super().setUp()
# Create a real configuration file, and populate it.
self.dir = self.useFixture(TempDirectory()).path
self.filename = path.join(self.dir, self.name)
with open(self.filename, "wb") as stream:
yaml.safe_dump(self.config, stream=stream, encoding="utf-8")
# Export this filename to the environment, so that subprocesses will
# pick up this configuration. Define the new environment as an
# instance variable so that users of this fixture can use this to
# extend custom subprocess environments.
self.environ = {self.schema.envvar: self.filename}
for name, value in self.environ.items():
self.useFixture(EnvironmentVariableFixture(name, value))
def test_install_dependencies_url_snap(self):
mock_run_and_check = self.patch(
maas_run_remote_scripts, 'run_and_check')
scripts_dir = self.useFixture(TempDirectory()).path
scripts = make_scripts(scripts_dir=scripts_dir, with_output=False)
snap_file = os.path.join(scripts[0]['download_path'], 'file.snap')
open(snap_file, 'w').close()
with open(scripts[0]['combined_path'], 'w') as output:
output.write("Saving to: '%s'" % snap_file)
for script in scripts:
script['packages'] = {'url': [snap_file]}
self.assertTrue(install_dependencies(scripts))
self.assertThat(mock_run_and_check, MockAnyCall(
['snap', snap_file], scripts, True, True))
def test_install_dependencies_url_errors(self):
mock_run_and_check = self.patch(
maas_run_remote_scripts, 'run_and_check')
mock_run_and_check.return_value = False
scripts_dir = self.useFixture(TempDirectory()).path
scripts = make_scripts(scripts_dir=scripts_dir)
packages = [factory.make_name('url') for _ in range(3)]
for script in scripts:
script['packages'] = {'url': packages}
self.assertFalse(install_dependencies(scripts))
for script in scripts:
self.assertThat(self.mock_output_and_send, MockAnyCall(
'Downloading and extracting URLs for %s' % script['msg_name'],
True, status='INSTALLING'))
