def test_read(self):
data = factory.make_string(size=10).encode("ascii")
reader = BytesReader(data)
self.addCleanup(reader.finish)
self.assertEqual(data[:7], reader.read(7))
self.assertEqual(data[7:], reader.read(7))
self.assertEqual(b"", reader.read(7))
def compose_bcd(self, kernel_params, local_host):
"""Composes the Windows boot configuration data.
:param kernel_params: An instance of `KernelParameters`.
:return: Binary data
"""
preseed_url = self.compose_preseed_url(kernel_params.preseed_url)
release_path = "%s\\source" % kernel_params.release
remote_path = "\\\\%s\\reminst" % local_host
loadoptions = "%s;%s;%s" % \
(remote_path, release_path, preseed_url)
# Generate the bcd file.
bcd_template = self.get_resource_path(kernel_params, "bcd")
if not os.path.isfile(bcd_template):
raise BootMethodError("Failed to find bcd template: %s" %
bcd_template)
with tempdir() as tmp:
bcd_tmp = os.path.join(tmp, "bcd")
shutil.copyfile(bcd_template, bcd_tmp)
bcd = Bcd(bcd_tmp)
bcd.set_load_options(loadoptions)
with open(bcd_tmp, 'rb') as stream:
return BytesReader(stream.read())
def test_render_GET_produces_from_reader(self):
path = factory.make_name("path")
ip = factory.make_ip_address()
request = DummyRequest([path.encode("utf-8")])
request.requestHeaders = Headers({
"X-Server-Addr": ["192.168.1.1"],
"X-Server-Port": ["5248"],
"X-Forwarded-For": [ip],
"X-Forwarded-Port": ["%s" % factory.pick_port()],
})
self.patch(http.log, "info")
mock_deferLater = self.patch(http, "deferLater")
mock_deferLater.side_effect = always_succeed_with(None)
content = factory.make_string(size=100).encode("utf-8")
reader = BytesReader(content)
self.tftp.backend.get_reader.return_value = succeed(reader)
resource = http.HTTPBootResource()
yield self.render_GET(resource, request)
self.assertEqual(
[100], request.responseHeaders.getRawHeaders(b"Content-Length"))
self.assertEqual(content, b"".join(request.written))
def test_render_GET_produces_from_reader(self):
path = factory.make_name('path')
ip = factory.make_ip_address()
request = DummyRequest([path.encode('utf-8')])
request.requestHeaders = Headers({
'X-Server-Addr': ['192.168.1.1'],
'X-Server-Port': ['5248'],
'X-Forwarded-For': [ip],
'X-Forwarded-Port': ['%s' % factory.pick_port()],
})
self.patch(http.log, 'info')
mock_deferLater = self.patch(http, 'deferLater')
mock_deferLater.side_effect = always_succeed_with(None)
content = factory.make_string(size=100).encode('utf-8')
reader = BytesReader(content)
self.tftp.backend.get_reader.return_value = succeed(reader)
resource = http.HTTPBootResource()
yield self.render_GET(resource, request)
self.assertEquals(
[100], request.responseHeaders.getRawHeaders(b'Content-Length'))
self.assertEquals(content, b''.join(request.written))
def get_reader(self, backend, kernel_params, mac=None, **extra):
"""Render a configuration file as a unicode string.
:param backend: requesting backend
:param kernel_params: An instance of `KernelParameters`.
:param mac: Optional MAC address discovered by `match_path`.
:param extra: Allow for other arguments. This is a safety valve;
parameters generated in another component (for example, see
`TFTPBackend.get_boot_method_reader`) won't cause this to break.
"""
template = self.get_template(
kernel_params.purpose, kernel_params.arch, kernel_params.subarch
)
namespace = self.compose_template_namespace(kernel_params)
def kernel_command(params):
cmd_line = compose_kernel_command_line(params)
# Modify the kernel_command to inject the BOOTIF. S390X doesn't
# support the IPAPPEND pxelinux flag.
if mac is not None:
return "%s BOOTIF=%s" % (cmd_line, format_bootif(mac))
return cmd_line
namespace["kernel_command"] = kernel_command
# We are going to do 2 passes of tempita substitution because there
# may be things like kernel params which include variables that can
# only be populated at run time and thus contain variables themselves.
# For example, an OS may need a kernel parameter that points back to
# fs_host and the kernel parameter comes through as part of the simple
# stream.
step1 = template.substitute(namespace)
return BytesReader(
tempita.Template(step1).substitute(namespace).encode("utf-8")
)
def get_reader(self, backend, kernel_params, **extra):
"""Render a configuration file as a unicode string.
:param backend: requesting backend
:param kernel_params: An instance of `KernelParameters`.
:param extra: Allow for other arguments. This is a safety valve;
parameters generated in another component (for example, see
`TFTPBackend.get_boot_method_reader`) won't cause this to break.
"""
def kernel_command(params):
"""Return the kernel command, adjusted for UEFI to work.
See the similar function in BootMethod, and the callsite below.
The issue here is that grub throws a fit when the braces on
cc:{...}end_cc are hit, for whatever reason. Escape _JUST_ those.
"""
return re.sub(
r"cc:{(?P<inner>[^}]*)}end_cc",
r"cc:\{\g<inner>\}end_cc",
compose_kernel_command_line(params),
)
template = self.get_template(kernel_params.purpose, kernel_params.arch,
kernel_params.subarch)
namespace = self.compose_template_namespace(kernel_params)
# Bug#1651452 - kernel command needs some extra escapes, but ONLY for
# UEFI. And so we fix it here, instead of in the common code. See
# also src/provisioningserver/kernel_opts.py.
namespace["kernel_command"] = kernel_command
return BytesReader(template.substitute(namespace).encode("utf-8"))
def get_reader(self, backend, kernel_params, **extra):
"""Render a configuration file as a unicode string.
:param backend: requesting backend
:param kernel_params: An instance of `KernelParameters`.
:param extra: Allow for other arguments. This is a safety valve;
parameters generated in another component (for example, see
`TFTPBackend.get_boot_method_reader`) won't cause this to break.
"""
template = self.get_template(
kernel_params.purpose, kernel_params.arch, kernel_params.subarch
)
kernel_params.mac = extra.get("mac", "")
namespace = self.compose_template_namespace(kernel_params)
# We are going to do 2 passes of tempita substitution because there
# may be things like kernel params which include variables that can
# only be populated at run time and thus contain variables themselves.
# For example, an OS may need a kernel parameter that points back to
# fs_host and the kernel parameter comes through as part of
# the simplestream.
step1 = template.substitute(namespace)
return BytesReader(
tempita.Template(step1).substitute(namespace).encode("utf-8")
)
def test_get_boot_method_reader_returns_rendered_params(self):
# Fake configuration parameters, as discovered from the file path.
fake_params = {"mac": factory.make_mac_address("-")}
# Fake kernel configuration parameters, as returned from the RPC call.
fake_kernel_params = make_kernel_parameters()
fake_params = fake_kernel_params._asdict()
# Stub the output of list_boot_images so the label is set in the
# kernel parameters.
boot_image = {
"osystem": fake_params["osystem"],
"release": fake_params["release"],
"architecture": fake_params["arch"],
"subarchitecture": fake_params["subarch"],
"purpose": fake_params["purpose"],
"supported_subarches": "",
"label": fake_params["label"],
}
self.patch(tftp_module, "list_boot_images").return_value = [boot_image]
del fake_params["label"]
# Stub RPC call to return the fake configuration parameters.
client = Mock()
client.localIdent = factory.make_name("system_id")
client.return_value = succeed(fake_params)
client_service = Mock()
client_service.getClientNow.return_value = succeed(client)
# get_boot_method_reader() takes a dict() of parameters and returns an
# `IReader` of a PXE configuration, rendered by
# `PXEBootMethod.get_reader`.
backend = TFTPBackend(self.make_dir(), client_service)
# Stub get_reader to return the render parameters.
method = PXEBootMethod()
fake_render_result = factory.make_name("render").encode("utf-8")
render_patch = self.patch(method, "get_reader")
render_patch.return_value = BytesReader(fake_render_result)
# Get the rendered configuration, which will actually be a JSON dump
# of the render-time parameters.
params_with_ip = dict(fake_params)
params_with_ip['remote_ip'] = factory.make_ipv4_address()
reader = yield backend.get_boot_method_reader(method, params_with_ip)
self.addCleanup(reader.finish)
self.assertIsInstance(reader, BytesReader)
output = reader.read(10000)
# The result has been rendered by `method.get_reader`.
self.assertEqual(fake_render_result, output)
self.assertThat(
method.get_reader,
MockCalledOnceWith(backend,
kernel_params=fake_kernel_params,
**params_with_ip))
def test_track_time(self):
prometheus_metrics = create_metrics(METRICS_DEFINITIONS)
session = TransferTimeTrackingSession(
'file.txt', BytesReader(b'some data'), _clock=Clock(),
prometheus_metrics=prometheus_metrics)
session.transport = Mock()
session.startProtocol()
session.cancel()
metrics = prometheus_metrics.generate_latest().decode('ascii')
self.assertIn(
'maas_tftp_file_transfer_latency_count{filename="file.txt"} 1.0',
metrics)
def get_reader(self, backend, kernel_params, mac=None, path=None, **extra):
"""Render a configuration file as a unicode string.
:param backend: requesting backend
:param kernel_params: An instance of `KernelParameters`.
:param path: Optional MAC address discovered by `match_path`.
:param path: Optional path discovered by `match_path`.
:param extra: Allow for other arguments. This is a safety valve;
parameters generated in another component (for example, see
`TFTPBackend.get_config_reader`) won't cause this to break.
"""
if path is not None:
# This is a request for a static file, not a configuration file.
# The prefix was already trimmed by `match_path` so we need only
# return a FilesystemReader for `path` beneath the backend's base.
target_path = backend.base.descendant(path.split("/"))
return FilesystemReader(target_path)
# Return empty config for S390x local. S390x fails to
# support the LOCALBOOT flag. Empty config will allow it
# to select the first device.
if kernel_params.purpose == "local":
return BytesReader("".encode("utf-8"))
template = self.get_template(
kernel_params.purpose, kernel_params.arch, kernel_params.subarch
)
namespace = self.compose_template_namespace(kernel_params)
# Modify the kernel_command to inject the BOOTIF. S390x fails to
# support the IPAPPEND pxelinux flag.
def kernel_command(params):
cmd_line = compose_kernel_command_line(params)
if mac is not None:
return "%s BOOTIF=%s" % (cmd_line, format_bootif(mac))
return cmd_line
namespace["kernel_command"] = kernel_command
return BytesReader(template.substitute(namespace).encode("utf-8"))
def test_get_boot_method_reader_rendered_parms_for_depoyed_ephemeral(self):
# Fake kernel configuration parameters, as returned from the RPC call.
fake_kernel_params = make_kernel_parameters(
purpose="local", label="local", osystem="caringo"
)
fake_params = fake_kernel_params._asdict()
# Stub the output of list_boot_images so the label is set in the
# kernel parameters.
boot_image = {
"osystem": fake_params["osystem"],
"release": fake_params["release"],
"architecture": fake_params["arch"],
"subarchitecture": fake_params["subarch"],
"purpose": "ephemeral",
"supported_subarches": "",
"label": fake_params["label"],
}
self.patch(tftp_module, "list_boot_images").return_value = [boot_image]
del fake_params["label"]
# Stub RPC call to return the fake configuration parameters.
client = Mock()
client.localIdent = factory.make_name("system_id")
client.return_value = succeed(fake_params)
client_service = Mock()
client_service.getClient.return_value = client
# get_boot_method_reader() takes a dict() of parameters and returns an
# `IReader` of a PXE configuration, rendered by
# `PXEBootMethod.get_reader`.
backend = TFTPBackend(self.make_dir(), client_service)
# Stub get_reader to return the render parameters.
method = PXEBootMethod()
fake_render_result = factory.make_name("render").encode("utf-8")
render_patch = self.patch(method, "get_reader")
render_patch.return_value = BytesReader(fake_render_result)
# Get the rendered configuration, which will actually be a JSON dump
# of the render-time parameters.
reader = yield backend.get_boot_method_reader(method, fake_params)
self.addCleanup(reader.finish)
self.assertIsInstance(reader, BytesReader)
output = reader.read(10000)
# The result has been rendered by `method.get_reader`.
self.assertEqual(fake_render_result, output)
def test_get_boot_method_reader_returns_rendered_params_local_device(self):
# Fake kernel configuration parameters, as returned from the RPC call.
fake_kernel_params = make_kernel_parameters(
purpose="local", label="local"
)
fake_params = fake_kernel_params._asdict()
del fake_params["label"]
# Set purpose to `local-device` as this is what will be passed on.
fake_params["purpose"] = "local-device"
# Stub RPC call to return the fake configuration parameters.
client = Mock()
client.localIdent = factory.make_name("system_id")
client.return_value = succeed(fake_params)
client_service = Mock()
client_service.getClientNow.return_value = succeed(client)
# get_boot_method_reader() takes a dict() of parameters and returns an
# `IReader` of a PXE configuration, rendered by
# `PXEBootMethod.get_reader`.
backend = TFTPBackend(self.make_dir(), client_service)
# Stub get_reader to return the render parameters.
method = PXEBootMethod()
fake_render_result = factory.make_name("render").encode("utf-8")
render_patch = self.patch(method, "get_reader")
render_patch.return_value = BytesReader(fake_render_result)
# Get the rendered configuration, which will actually be a JSON dump
# of the render-time parameters.
params_with_ip = dict(fake_params)
params_with_ip["remote_ip"] = factory.make_ipv4_address()
reader = yield backend.get_boot_method_reader(method, params_with_ip)
self.addCleanup(reader.finish)
self.assertIsInstance(reader, BytesReader)
output = reader.read(10000)
# The result has been rendered by `method.get_reader`.
self.assertEqual(fake_render_result, output)
self.assertThat(
method.get_reader,
MockCalledOnceWith(
backend, kernel_params=fake_kernel_params, **params_with_ip
),
)
def get_reader(self, backend, kernel_params, protocol, **extra):
"""Render a configuration file as a unicode string.
:param backend: requesting backend
:param kernel_params: An instance of `KernelParameters`.
:param protocol: The protocol the transfer is happening over.
:param extra: Allow for other arguments. This is a safety valve;
parameters generated in another component (for example, see
`TFTPBackend.get_boot_method_reader`) won't cause this to break.
"""
def kernel_command(params):
"""Return the kernel command, adjusted for UEFI to work.
See the similar function in BootMethod, and the callsite below.
The issue here is that grub throws a fit when the braces on
cc:{...}end_cc are hit, for whatever reason. Escape _JUST_ those.
"""
return re.sub(
r"cc:{(?P<inner>[^}]*)}end_cc",
r"cc:\{\g<inner>\}end_cc",
compose_kernel_command_line(params),
)
template = self.get_template(kernel_params.purpose, kernel_params.arch,
kernel_params.subarch)
namespace = self.compose_template_namespace(kernel_params)
# TFTP is much slower than HTTP. If GRUB was transfered over TFTP use
# GRUBs internal HTTP implementation to download the kernel and initrd.
# If HTTP or HTTPS was used don't specify host to continue to use the
# UEFI firmware's internal HTTP implementation.
if protocol == "tftp":
namespace["fs_efihost"] = "(http,%s:5248)/images/" % (
convert_host_to_uri_str(kernel_params.fs_host))
else:
namespace["fs_efihost"] = "/images/"
# Bug#1651452 - kernel command needs some extra escapes, but ONLY for
# UEFI. And so we fix it here, instead of in the common code. See
# also src/provisioningserver/kernel_opts.py.
namespace["kernel_command"] = kernel_command
return BytesReader(
template.substitute(namespace).strip().encode("utf-8"))
def get_boot_method_reader(boot_method, params):
params_json = json.dumps(params).encode("ascii")
params_json_reader = BytesReader(params_json)
return succeed(params_json_reader)
def test_get_boot_method_reader_grabs_new_client_on_lost_conn(self):
# Fake kernel configuration parameters, as returned from the RPC call.
fake_kernel_params = make_kernel_parameters()
fake_params = fake_kernel_params._asdict()
# Stub the output of list_boot_images so the label is set in the
# kernel parameters.
boot_image = {
"osystem": fake_params["osystem"],
"release": fake_params["release"],
"architecture": fake_params["arch"],
"subarchitecture": fake_params["subarch"],
"purpose": fake_params["purpose"],
"supported_subarches": "",
"label": fake_params["label"],
}
self.patch(tftp_module, "list_boot_images").return_value = [boot_image]
del fake_params["label"]
# Stub RPC call to return the fake configuration parameters.
clients = []
for _ in range(10):
client = Mock()
client.localIdent = factory.make_name("system_id")
client.side_effect = lambda *args, **kwargs: (
succeed(dict(fake_params))
)
clients.append(client)
client_service = Mock()
client_service.getClientNow.side_effect = [
succeed(client) for client in clients
]
client_service.getAllClients.side_effect = [clients[1:], clients[2:]]
# get_boot_method_reader() takes a dict() of parameters and returns an
# `IReader` of a PXE configuration, rendered by
# `PXEBootMethod.get_reader`.
backend = TFTPBackend(self.make_dir(), client_service)
# Stub get_reader to return the render parameters.
method = PXEBootMethod()
fake_render_result = factory.make_name("render").encode("utf-8")
render_patch = self.patch(method, "get_reader")
render_patch.return_value = BytesReader(fake_render_result)
# Get the reader once.
remote_ip = factory.make_ipv4_address()
params_with_ip = dict(fake_params)
params_with_ip["remote_ip"] = remote_ip
reader = yield backend.get_boot_method_reader(method, params_with_ip)
self.addCleanup(reader.finish)
# The first client is now saved.
self.assertEqual(clients[0], backend.client_to_remote[remote_ip])
# Get the reader twice.
params_with_ip = dict(fake_params)
params_with_ip["remote_ip"] = remote_ip
reader = yield backend.get_boot_method_reader(method, params_with_ip)
self.addCleanup(reader.finish)
# The second client is now saved.
self.assertEqual(clients[1], backend.client_to_remote[remote_ip])
# Only the first and second client should have been called once, and
# all the other clients should not have been called.
self.assertEqual(1, clients[0].call_count)
self.assertEqual(1, clients[1].call_count)
for idx in range(2, 10):
self.assertThat(clients[idx], MockNotCalled())
def test_interfaces(self):
reader = BytesReader(b"")
self.addCleanup(reader.finish)
verifyObject(IReader, reader)
def get_reader(backend, kernel_params, **extra):
return BytesReader("")
def test_finish(self):
reader = BytesReader(b"1234")
reader.finish()
self.assertRaises(ValueError, reader.read, 1)
