async def test_output(self):
existing = FakeCustomResource("existing-dependency")
res = FakeCustomResource("some-dependency")
fut = asyncio.Future()
fut.set_result(42)
known_fut = asyncio.Future()
known_fut.set_result(True)
out = Output({res}, fut, known_fut)
deps = [existing]
prop = await rpc.serialize_property(out, deps)
self.assertListEqual(deps, [existing, res])
self.assertEqual(42, prop)
known_fut = asyncio.Future()
known_fut.set_result(False)
out = Output({}, fut, known_fut)
# For compatibility, future() should still return 42 even if the value is unknown.
prop = await out.future()
self.assertEqual(42, prop)
fut = asyncio.Future()
fut.set_result(UNKNOWN)
known_fut = asyncio.Future()
known_fut.set_result(True)
out = Output({}, fut, known_fut)
# For compatibility, is_known() should return False and future() should return None when the value contains
# first-class unknowns.
self.assertEqual(False, await out.is_known())
self.assertEqual(None, await out.future())
# If the caller of future() explicitly accepts first-class unknowns, they should be present in the result.
self.assertEqual(UNKNOWN, await out.future(with_unknowns=True))
async def test_lifted_unknown(self):
fut = asyncio.Future()
fut.set_result(UNKNOWN)
out = Output.from_input({ "foo": "foo", "bar": UNKNOWN, "baz": fut})
self.assertFalse(await out.is_known())
r1 = out["foo"]
self.assertTrue(await r1.is_known())
self.assertEqual(await r1.future(), "foo")
r2 = out["bar"]
self.assertFalse(await r2.is_known())
self.assertEqual(await r2.future(), UNKNOWN)
r3 = out["baz"]
self.assertFalse(await r3.is_known())
self.assertEqual(await r3.future(), UNKNOWN)
r4 = out["baz"]["qux"]
self.assertFalse(await r4.is_known())
self.assertEqual(await r4.future(), UNKNOWN)
out = Output.from_input([ "foo", UNKNOWN ])
r5 = out[0]
self.assertTrue(await r5.is_known())
self.assertEqual(await r5.future(), "foo")
r6 = out[1]
self.assertFalse(await r6.is_known())
self.assertEqual(await r6.future(), UNKNOWN)
def _provision_network(self, protect=False):
deploy_network = networking.Network(
self.props.deploy_network["name"],
name=self.props.deploy_network["name"],
opts=ResourceOptions(delete_before_replace=True, protect=protect),
)
deploy_subnet = networking.Subnet(
self.props.deploy_network["subnet_name"],
name=self.props.deploy_network["subnet_name"],
network_id=deploy_network.id,
cidr=self.props.deploy_network["cidr"],
ip_version=4,
opts=ResourceOptions(delete_before_replace=True, protect=protect),
)
public_router = networking.Router(
self.props.public_router_name,
name=self.props.public_router_name,
external_network_id=self.props.external_network["id"],
opts=ResourceOptions(delete_before_replace=True, protect=protect),
)
networking.RouterInterface(
"router-interface-management",
router_id=public_router.id,
subnet_id=self.resources.mgmt_subnet.id,
opts=ResourceOptions(
# provider=self.provider_cloud_admin,
delete_before_replace=True,
protect=protect,
),
)
networking.RouterInterface(
"router-interface-deployment",
router_id=public_router.id,
subnet_id=deploy_subnet.id,
opts=ResourceOptions(
# provider=self.provider_cloud_admin,
delete_before_replace=True,
protect=protect,
),
)
pulumi.export(
"DeploymentNetwork",
Output.all(deploy_network.name, deploy_network.id).apply(
lambda args: f"{args[0]} ({args[1]})"
),
)
pulumi.export(
"PublicRouter",
Output.all(public_router.name, public_router.id).apply(
lambda args: f"{args[0]} ({args[1]})"
),
)
def create_output(self,
val: Any,
is_known: bool,
is_secret: Optional[bool] = None):
fut = asyncio.Future()
fut.set_result(val)
known_fut = asyncio.Future()
known_fut.set_result(is_known)
if is_secret is not None:
is_secret_fut = asyncio.Future()
is_secret_fut.set_result(True)
return Output(set(), fut, known_fut, is_secret_fut)
return Output(set(), fut, known_fut)
async def test_output_all(self):
res = FakeCustomResource("some-resource")
fut = asyncio.Future()
fut.set_result(42)
known_fut = asyncio.Future()
known_fut.set_result(True)
out = Output({res}, fut, known_fut)
other = Output.from_input(99)
combined = Output.all(out, other)
deps = []
prop = await rpc.serialize_property(combined, deps)
self.assertListEqual(deps, [res])
self.assertEqual([42, 99], prop)
def _provision_router(self, protect=False):
public_router = networking.Router(
self.props.public_router_name,
name=self.props.public_router_name,
external_network_id=self.props.external_network["id"],
opts=ResourceOptions(delete_before_replace=True, protect=protect),
)
networking.RouterInterface(
"router-interface-management",
router_id=public_router.id,
subnet_id=self.resources.mgmt_subnet.id,
opts=ResourceOptions(
delete_before_replace=True,
protect=protect,
),
)
networking.RouterInterface(
"router-interface-deployment",
router_id=public_router.id,
subnet_id=self.resources.deploy_subnet.id,
opts=ResourceOptions(
delete_before_replace=True,
protect=protect,
),
)
pulumi.export(
"PublicRouter",
Output.all(public_router.name, public_router.id).apply(
lambda args: f"{args[0]} ({args[1]})"
),
)
async def test_distinguished_unknown_output(self):
fut = asyncio.Future()
fut.set_result(UNKNOWN)
known_fut = asyncio.Future()
known_fut.set_result(True)
out = Output({}, fut, known_fut)
self.assertFalse(await out.is_known())
fut = asyncio.Future()
fut.set_result(["foo", UNKNOWN])
out = Output({}, fut, known_fut)
self.assertFalse(await out.is_known())
fut = asyncio.Future()
fut.set_result({"foo": "foo", "bar": UNKNOWN})
out = Output({}, fut, known_fut)
self.assertFalse(await out.is_known())
def _provision_helper_vm(self):
init_script = r"""#!/bin/bash
echo 'net.ipv4.conf.default.rp_filter = 2' >> /etc/sysctl.conf
echo 'net.ipv4.conf.all.rp_filter = 2' >> /etc/sysctl.conf
/usr/sbin/sysctl -p /etc/sysctl.conf
"""
sg = compute.SecGroup(
"helper-vm-sg",
description="allow ssh",
rules=[
compute.SecGroupRuleArgs(
cidr="0.0.0.0/0", from_port=22, to_port=22, ip_protocol="tcp"
)
],
)
external_port = networking.Port(
"helper-vm-external-port",
network_id=self.resources.mgmt_network.id,
fixed_ips=[
networking.PortFixedIpArgs(
subnet_id=self.resources.mgmt_subnet.id,
ip_address=self.props.helper_vm["ip"],
)
],
security_group_ids=[sg.id],
)
helper_vm = compute.Instance(
"helper-vm",
name="helper-vm",
flavor_name=self.props.helper_vm["flavor_name"],
image_name=self.props.helper_vm["image_name"],
networks=[
compute.InstanceNetworkArgs(name=self.resources.deploy_network.name),
],
key_pair=self.resources.keypair.name,
user_data=init_script,
opts=ResourceOptions(
delete_before_replace=True,
ignore_changes=["image_name", "key_pair"],
),
)
attach_external_ip = compute.InterfaceAttach(
"helper-vm-attatch",
instance_id=helper_vm.id,
port_id=external_port.id,
opts=ResourceOptions(delete_before_replace=True, depends_on=[helper_vm]),
)
pulumi.export(
"HelperVM",
Output.all(
helper_vm.name, helper_vm.id, external_port.all_fixed_ips[0]
).apply(lambda args: f"{args[0]} ({args[1]}, {args[2]})"),
)
return helper_vm, attach_external_ip
async def test_unknown_output(self):
res = FakeCustomResource("some-dependency")
fut = asyncio.Future()
fut.set_result(None)
known_fut = asyncio.Future()
known_fut.set_result(False)
out = Output({res}, fut, known_fut)
deps = []
prop = await rpc.serialize_property(out, deps)
self.assertListEqual(deps, [res])
self.assertEqual(rpc.UNKNOWN, prop)
async def test_output_all_composes_dependencies(self):
res = FakeCustomResource("some-resource")
fut = asyncio.Future()
fut.set_result(42)
known_fut = asyncio.Future()
known_fut.set_result(True)
out = Output({res}, fut, known_fut)
other = FakeCustomResource("some-other-resource")
other_fut = asyncio.Future()
other_fut.set_result(99)
other_known_fut = asyncio.Future()
other_known_fut.set_result(True)
other_out = Output({other}, other_fut, other_known_fut)
combined = Output.all(out, other_out)
deps = []
prop = await rpc.serialize_property(combined, deps)
self.assertSetEqual(set(deps), {res, other})
self.assertEqual([42, 99], prop)
def _provision_deployment_network(self, protect=False):
deploy_network = networking.Network(
self.props.deploy_network["name"],
opts=ResourceOptions(delete_before_replace=True, protect=protect),
)
pulumi.export(
"DeploymentNetwork",
Output.all(deploy_network.name, deploy_network.id).apply(
lambda args: f"{args[0]} ({args[1]})"
),
)
return deploy_network
def apply_pinpoint_stream_role_policy_document_outputs(delivery_stream_name,
pinpoint_application_id,
func):
""" Applies the Pulumi outputs `delivery_stream_name` and `pinpoint_application_id`
to the given function.
delivery_stream_name
pinpoint_application_id
func -- A lambda function with inputs (delivery_stream_name: str,
pinpoint_application_id: str)
"""
return Output.all(delivery_stream_name, pinpoint_application_id).apply(
lambda outputs: func(outputs[0], outputs[1]))
async def test_output(self):
existing = FakeCustomResource("existing-dependency")
res = FakeCustomResource("some-dependency")
fut = asyncio.Future()
fut.set_result(42)
known_fut = asyncio.Future()
known_fut.set_result(True)
out = Output({res}, fut, known_fut)
deps = [existing]
prop = await rpc.serialize_property(out, deps)
self.assertListEqual(deps, [existing, res])
self.assertEqual(42, prop)
async def test_output_all_known_if_all_are_known(self):
res = FakeCustomResource("some-resource")
fut = asyncio.Future()
fut.set_result(42)
known_fut = asyncio.Future()
known_fut.set_result(True)
out = Output({res}, fut, known_fut)
other = FakeCustomResource("some-other-resource")
other_fut = asyncio.Future()
other_fut.set_result(99)
other_known_fut = asyncio.Future()
other_known_fut.set_result(False) # <- not known
other_out = Output({other}, other_fut, other_known_fut)
combined = Output.all(out, other_out)
deps = []
prop = await rpc.serialize_property(combined, deps)
self.assertSetEqual(set(deps), {res, other})
# The contents of the list are unknown if any of the Outputs used to
# create it were unknown.
self.assertEqual(rpc.UNKNOWN, prop)
async def test_output_coros(self):
# Ensure that Outputs function properly when the input value and is_known are coroutines. If the implementation
# is not careful to wrap these coroutines in Futures, they will be awaited more than once and the runtime will
# throw.
async def value():
await asyncio.sleep(0)
return 42
async def is_known():
await asyncio.sleep(0)
return True
out = Output({}, value(), is_known())
self.assertTrue(await out.is_known())
self.assertEqual(42, await out.future())
self.assertEqual(42, await out.apply(lambda v: v).future())
async def test_apply_unknown_output(self):
fut = asyncio.Future()
fut.set_result("foo")
known_fut = asyncio.Future()
known_fut.set_result(True)
out = Output({}, fut, known_fut)
r1 = out.apply(lambda v: UNKNOWN)
r2 = out.apply(lambda v: [v, UNKNOWN])
r3 = out.apply(lambda v: {"v": v, "unknown": UNKNOWN})
r4 = out.apply(lambda v: UNKNOWN).apply(lambda v: v, True)
r5 = out.apply(lambda v: [v, UNKNOWN]).apply(lambda v: v, True)
r6 = out.apply(lambda v: {"v": v, "unknown": UNKNOWN}).apply(lambda v: v, True)
self.assertFalse(await r1.is_known())
self.assertFalse(await r2.is_known())
self.assertFalse(await r3.is_known())
self.assertFalse(await r4.is_known())
self.assertFalse(await r5.is_known())
self.assertFalse(await r6.is_known())
def __init__(self, provider_cloud_admin, provider_ccadmin_master) -> None:
self.config = pulumi.Config()
self.openstack_config = pulumi.Config("openstack")
self.stack_name = pulumi.get_stack()
self.provider_cloud_admin = provider_cloud_admin
self.provider_ccadmin_master = provider_ccadmin_master
public_key_file = (
self.config.get("publicKeyFile") or "/pulumi/automation/etc/.ssh/id_rsa.pub"
)
private_key_file = (
self.config.get("privateKeyFile") or "/pulumi/automation/etc/.ssh/id_rsa"
)
try:
private_networks = json.loads(self.config.require("privateNetworks"))
except ConfigMissingError:
private_networks = []
try:
esxi_nodes = json.loads(self.config.require("esxiNodes"))
esxi_image_name = self.config.require("esxiServerImage")
esxi_flavor_name = self.config.require("esxiServerFlavor")
except ConfigMissingError:
esxi_nodes = []
esxi_image_name = ""
esxi_flavor_name = ""
try:
shares = json.loads(self.config.require("shares"))
except ConfigMissingError:
shares = []
try:
reserved_ips = json.loads(self.config.require("reservedIPs"))
except ConfigMissingError:
reserved_ips = []
try:
sddc_manager = json.loads(self.config.require("sddcManager"))
except ConfigMissingError:
sddc_manager = {}
try:
vcenter = json.loads(self.config.require("vcenter"))
except ConfigMissingError:
vcenter = {}
try:
nsxt = json.loads(self.config.require("nsxt"))
except ConfigMissingError:
nsxt = {}
try:
nsxt_managers = json.loads(self.config.require("nsxtManagers"))
except ConfigMissingError:
nsxt_managers = []
try:
helper_vsanwitness = json.loads(self.config.require("helperVsanWiteness"))
except ConfigMissingError:
helper_vsanwitness = None
self.props = SimpleNamespace(
helper_vm=json.loads(self.config.require("helperVM")),
helper_vsanwitness=helper_vsanwitness,
public_key_file=public_key_file,
private_key_file=private_key_file,
nsxt=nsxt,
nsxt_managers=nsxt_managers,
sddc_manager=sddc_manager,
vcenter=vcenter,
vmware_password=self.config.require("vmwarePassword"),
# networks
external_network=json.loads(self.config.require("externalNetwork")),
mgmt_network=json.loads(self.config.require("managementNetwork")),
deploy_network=json.loads(self.config.require("deploymentNetwork")),
public_router_name=self.config.require("publicRouter"),
private_networks=private_networks,
# ips and dns names
dns_zone_name=self.config.require("dnsZoneName"),
reverse_dns_zone_name=self.config.require("reverseDnsZoneName"),
reserved_ips=reserved_ips,
# esxi servers
esxi_image=esxi_image_name,
esxi_flavor_name=esxi_flavor_name,
esxi_nodes=esxi_nodes,
shares=shares,
)
mgmt_network = networking.get_network(name=self.props.mgmt_network["name"])
mgmt_subnet = networking.get_subnet(name=self.props.mgmt_network["subnet_name"])
self.resources = SimpleNamespace(
mgmt_network=mgmt_network,
mgmt_subnet=mgmt_subnet,
)
pulumi.export(
"ManagementNetwork",
Output.all(mgmt_network.name, mgmt_network.id).apply(
lambda args: "{name} ({_id})".format(name=args[0], _id=args[1])
),
)
pulumi.export(
"ManagementSubnet",
Output.all(mgmt_subnet.name, mgmt_subnet.id).apply(
lambda args: "{name} ({_id})".format(name=args[0], _id=args[1])
),
)
async def test_lifted_unknown(self):
settings.SETTINGS.dry_run = True
fut = asyncio.Future()
fut.set_result(UNKNOWN)
out = Output.from_input({"foo": "foo", "bar": UNKNOWN, "baz": fut})
self.assertFalse(await out.is_known())
r1 = out["foo"]
self.assertTrue(await r1.is_known())
self.assertEqual(await r1.future(with_unknowns=True), "foo")
r2 = out["bar"]
self.assertFalse(await r2.is_known())
self.assertEqual(await r2.future(with_unknowns=True), UNKNOWN)
r3 = out["baz"]
self.assertFalse(await r3.is_known())
self.assertEqual(await r3.future(with_unknowns=True), UNKNOWN)
r4 = out["baz"]["qux"]
self.assertFalse(await r4.is_known())
self.assertEqual(await r4.future(with_unknowns=True), UNKNOWN)
out = Output.from_input(["foo", UNKNOWN])
r5 = out[0]
self.assertTrue(await r5.is_known())
self.assertEqual(await r5.future(with_unknowns=True), "foo")
r6 = out[1]
self.assertFalse(await r6.is_known())
self.assertEqual(await r6.future(with_unknowns=True), UNKNOWN)
out = Output.all(
Output.from_input("foo"), Output.from_input(UNKNOWN),
Output.from_input(
[Output.from_input(UNKNOWN),
Output.from_input("bar")]))
self.assertFalse(await out.is_known())
r7 = out[0]
self.assertTrue(await r7.is_known())
self.assertEqual(await r7.future(with_unknowns=True), "foo")
r8 = out[1]
self.assertFalse(await r8.is_known())
self.assertEqual(await r8.future(with_unknowns=True), UNKNOWN)
r9 = out[2]
self.assertFalse(await r9.is_known())
r10 = r9[0]
self.assertFalse(await r10.is_known())
self.assertEqual(await r10.future(with_unknowns=True), UNKNOWN)
r11 = r9[1]
self.assertTrue(await r11.is_known())
self.assertEqual(await r11.future(with_unknowns=True), "bar")
def _provision_helper_vm(self):
init_script = r"""#!/bin/bash
echo 'net.ipv4.conf.default.rp_filter = 2' >> /etc/sysctl.conf
echo 'net.ipv4.conf.all.rp_filter = 2' >> /etc/sysctl.conf
/usr/sbin/sysctl -p /etc/sysctl.conf
"""
sg = compute.SecGroup(
"helper-vm-sg",
description="allow ssh",
rules=[
compute.SecGroupRuleArgs(
cidr="0.0.0.0/0", from_port=22, to_port=22, ip_protocol="tcp"
)
],
)
external_port = networking.Port(
"helper-vm-external-port",
network_id=self.resources.mgmt_network.id,
fixed_ips=[
networking.PortFixedIpArgs(
subnet_id=self.resources.mgmt_subnet.id,
ip_address=self.props.helper_vm["ip"],
)
],
security_group_ids=[sg.id],
)
helper_vm = compute.Instance(
"helper-vm",
name="helper-vm",
flavor_id=self.props.helper_vm["flavor_id"],
image_name=self.props.helper_vm["image_name"],
networks=[
compute.InstanceNetworkArgs(name=self.props.deploy_network["name"]),
],
key_pair=self.props.keypair_name,
user_data=init_script,
opts=ResourceOptions(
delete_before_replace=True,
ignore_changes=["image_name"],
),
)
attach_external_ip = compute.InterfaceAttach(
"helper-vm-attatch",
instance_id=helper_vm.id,
port_id=external_port.id,
opts=ResourceOptions(delete_before_replace=True, depends_on=[helper_vm]),
)
# configure helper vm
conn_args = ConnectionArgs(
host=self.props.helper_vm["ip"],
username="******",
private_key_file=self.props.private_key_file,
)
exec_install_pwsh = RemoteExec(
"install-powershell",
host_id=helper_vm.id,
conn=conn_args,
commands=[
"[ ! -f packages-microsoft-prod.deb ] && wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb || true",
"sudo dpkg -i packages-microsoft-prod.deb",
"sudo apt-get update",
"echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections",
"sudo apt-get install -y -q powershell",
"pwsh -Command Set-PSRepository -Name 'PSGallery' -InstallationPolicy Trusted",
"pwsh -Command Install-Module VMware.PowerCLI",
"pwsh -Command Set-PowerCLIConfiguration -InvalidCertificateAction Ignore -Confirm:0",
"pwsh -Command Set-PowerCLIConfiguration -Scope User -ParticipateInCEIP 0 -Confirm:0",
],
opts=ResourceOptions(depends_on=[attach_external_ip]),
)
# copy rsa key
CopyFile(
"copy-rsa-key",
host_id=helper_vm.id,
conn=conn_args,
src=self.props.private_key_file,
dest="/home/ccloud/esxi_rsa",
mode="600",
opts=ResourceOptions(depends_on=[attach_external_ip]),
)
# copy from path relative to the project root
CopyFile(
"copy-cleanup",
host_id=helper_vm.id,
conn=conn_args,
src="./scripts/cleanup.sh",
dest="/home/ccloud/cleanup.sh",
opts=ResourceOptions(depends_on=[attach_external_ip]),
)
with open("./scripts/config.sh") as f:
template = jinja2.Template(f.read())
config_script = template.render(
management_network=self.props.mgmt_network,
)
CopyFileFromString(
"copy-config-sh",
host_id=helper_vm.id,
conn=conn_args,
from_str=config_script,
dest="/home/ccloud/config.sh",
opts=ResourceOptions(depends_on=[attach_external_ip]),
)
pulumi.export(
"HelperVM",
Output.all(
helper_vm.name, helper_vm.id, external_port.all_fixed_ips[0]
).apply(lambda args: f"{args[0]} ({args[1]}, {args[2]})"),
)
