def on_catalog(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_catalog")
tree = ET.fromstring(response.text)
obj = Catalog().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}")
path = path._replace(file="catalog.yaml")
cache(path, obj)
items = find_xpath(
".//*[@type='application/vnd.vmware.vcloud.catalogItem+xml']",
ET.fromstring(response.text)
)
ops = [session.get(
item.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_catalogitem, path,
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, item in enumerate(items)]
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
def on_org(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_org")
tree = ET.fromstring(response.text)
obj = Org().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}")
path = path._replace(file="org.yaml")
fP = cache(path, obj)
ctlgs = find_xpath(
"./*/[@type='application/vnd.vmware.vcloud.catalog+xml']",
ET.fromstring(response.text)
)
vdcs = find_xpath(
"./*/[@type='application/vnd.vmware.vcloud.vdc+xml']",
ET.fromstring(response.text)
)
ops = [session.get(
vdc.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_vdc,
path._replace(service=vdc.attrib.get("name")),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, vdc in enumerate(vdcs)] + [session.get(
ctlg.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_catalog,
path._replace(
service="catalogs",
category=ctlg.attrib.get("name")
),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, ctlg in enumerate(ctlgs)]
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
def get_tasks(response):
if response is None:
return iter(tuple())
tree = ET.fromstring(response.text)
return (
Task().feed_xml(elem)
for elem in find_xpath(
"./*/*/[@type='application/vnd.vmware.vcloud.task+xml']",
tree
)
)
def on_vapp(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_vapp")
tree = ET.fromstring(response.text)
obj = VApp().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}")
path = path._replace(file="vapp.yaml")
fP = cache(path, obj)
url = urlparse(response.url)
query = "/".join((
url.scheme + "://" + url.netloc,
"api/vms/query?filter=(container=={0})".format(urlquote(response.url))
))
vms = list(find_xpath(
"./*/*/[@type='application/vnd.vmware.vcloud.vm+xml']",
ET.fromstring(response.text)
))
try:
ops = [session.get(
vm.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_vm,
path._replace(node=vm.attrib.get("name")),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, vm in enumerate(vms)] + [session.get(
query,
background_callback=functools.partial(
Surveyor.on_vmrecords,
path,
results=results,
status=status._replace(job=status.job + len(vms) + 1) if status else None
)
)]
except Exception as e:
log.error(e)
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
def on_org_list(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_org_list")
tree = ET.fromstring(response.text)
orgs = find_xpath(
"./*/[@type='application/vnd.vmware.vcloud.org+xml']", tree)
ops = [session.get(
org.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_org,
path._replace(org=org.attrib.get("name")),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, org in enumerate(orgs)]
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
def on_catalogitem(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_catalogitem")
templates = find_xpath(
".//*[@type='application/vnd.vmware.vcloud.vAppTemplate+xml']",
ET.fromstring(response.text)
)
templates = list(templates)
ops = [session.get(
tmplt.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_template,
path._replace(container=tmplt.attrib.get("name")),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, tmplt in enumerate(templates)]
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
def on_vdc(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_vdc")
tree = ET.fromstring(response.text)
obj = Vdc().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}")
path = path._replace(file="vdc.yaml")
cache(path, obj)
edgeGWs = find_xpath(
"./*/[@type='application/vnd.vmware.vcloud.query.records+xml']",
tree,
rel="edgeGateways"
)
orgVdcNets = find_xpath(
"./*/[@type='application/vnd.vmware.vcloud.query.records+xml']",
tree,
rel="orgVdcNetworks"
)
vapps = find_xpath(
"./*/*/[@type='application/vnd.vmware.vcloud.vApp+xml']",
tree
)
ops = [session.get(
edgeGW.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_edgeGateway,
path,
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, edgeGW in enumerate(edgeGWs)] + [session.get(
orgVdcNet.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_orgVdcNetwork,
path._replace(
category="networks",
container=orgVdcNet.attrib.get("name")
),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, orgVdcNet in enumerate(orgVdcNets)] + [session.get(
vapp.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_vapp,
path._replace(
category="vapps",
container=vapp.attrib.get("name")
),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, vapp in enumerate(vapps)]
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
def feed_xml(self, tree, ns="{http://www.vmware.com/vcloud/v1.5}"):
"""
Updates the object by feeding it XML
from the VMware API.
"""
if tree.tag in (ns + "Owner", ns + "VAppTemplate", ns + "Vm", ns + "VMRecord"):
super().feed_xml(tree, ns=ns)
if tree.tag == ns + "VMRecord":
try:
val = int(tree.attrib.get("hardwareVersion"))
if val not in self.hardwareVersion:
self.hardwareVersion.append(val)
except TypeError as e:
# No version supplied
pass
if tree.tag in (ns + "VAppTemplate", ns + "Vm"):
system = next(find_xpath(
"./*/{}System".format("{http://schemas.dmtf.org/ovf/envelope/1}"), tree
), None)
if system is not None:
try:
versions = set(self.hardwareVersion).union({
int(i.lstrip("vmx-")) for i in system.find((
"{http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2"
"/CIM_VirtualSystemSettingData}VirtualSystemType")).text.split()})
self.hardwareVersion = list(versions)
except (AttributeError, TypeError, ValueError):
pass
hardware = find_xpath(
"./*/{}Item".format("{http://schemas.dmtf.org/ovf/envelope/1}"), tree
)
rasdNs = (
"{http://schemas.dmtf.org/wbem/wscim/1/cim-schema/"
"2/CIM_ResourceAllocationSettingData}"
)
for item in hardware:
key = int(getattr(item.find(rasdNs + "ResourceType"), "text", "0"))
typ = Vm.rasd[key]
fields = [(rasdNs + i).lower() for i in typ._fields]
obj = typ(*(i for i in list(item) if i.tag.lower() in fields))
if key == 3:
self.cpu = int(obj.virtualQuantity.text)
elif key == 4:
self.memoryMB = int(obj.virtualQuantity.text)
elif key == 6:
entry = Vm.SCSIController(
obj.elementName.text,
obj.resourceSubType.text
)
if entry not in self.scsi:
self.scsi.append(entry)
elif key == 10:
entry = Vm.NetworkCard(
obj.elementName.text,
obj.address.text,
obj.resourceSubType.text
)
if entry not in self.networkcards:
self.networkcards.append(entry)
elif key == 17:
entry = Vm.HardDisk(
obj.description.text,
int(obj.hostResource.attrib.get(ns + "capacity"))
)
if entry not in self.harddisks:
self.harddisks.append(entry)
self.networkconnections = [
Vm.NetworkConnection(
i.attrib["network"],
getattr(i.find(ns + "IpAddress"), "text", None),
i.find(ns + "IsConnected").text == "true",
i.find(ns + "MACAddress").text,
getattr(i.find(ns + "IpAddressAllocationMode"), "text", None))
for i in tree.iter(ns + "NetworkConnection")]
section = tree.find(ns + "GuestCustomizationSection")
# TODO: Define customization storage
return self
def configure_gateway(self, session, token, callback=None, status=None, **kwargs):
log = logging.getLogger("maloja.builder.configure_gateway")
ns = "{http://www.vmware.com/vcloud/v1.5}"
natMacro = PageTemplateFile(
pkg_resources.resource_filename(
"maloja.workflow", "NatRule.pt"
)
)
fwMacro = PageTemplateFile(
pkg_resources.resource_filename(
"maloja.workflow", "FirewallRule.pt"
)
)
gw = self.plans[Gateway][0]
try:
response = self.check_response(
*self.wait_for(
session.get(gw.href)
)
)
tree = ET.fromstring(response.text)
endpoint = next(find_xpath((
".//*[@type='application/vnd.vmware.admin"
".edgeGatewayServiceConfiguration+xml']"),
tree,
rel="edgeGateway:configureServices"
))
except (StopIteration, TypeError):
self.send_status(status, stop=True)
return
# Prepare EdgeGateway services to receive NAT rules
try:
elem = next(tree.iter(ns + "EdgeGatewayServiceConfiguration"))
natService = next(elem.iter(ns + "NatService"), None)
if natService is None:
natService = ET.XML(
"""<NatService><IsEnabled>true</IsEnabled></NatService>""")
elem.append(natService)
fwService = next(elem.iter(ns + "FirewallService"))
except StopIteration:
self.send_status(status, stop=True)
return
for rule in gw.dnat:
for ips in itertools.zip_longest(
rule.ext_addr, rule.int_addr, fillvalue=rule.int_addr[-1]
):
data = {
"network": self.plans[Network][0],
"ips": ips,
"ports": (rule.ext_port, rule.int_port),
"rule": rule
}
natService.append(ET.XML(natMacro(**data)))
for rule in gw.snat:
for ips in itertools.zip_longest(
rule.int_addr, rule.ext_addr, fillvalue=rule.int_addr[-1]
):
data = {
"network": self.plans[Network][0],
"ips": ips,
"ports": (rule.int_port, rule.ext_port),
"rule": rule
}
natService.append(ET.XML(natMacro(**data)))
for rule in gw.fw:
log.debug(rule)
for ips in itertools.zip_longest(
rule.ext_addr or [], rule.int_addr, fillvalue=rule.int_addr[-1]
):
data = {
"network": self.plans[Network][0],
"ips": ips,
"ports": (rule.ext_port, rule.int_port),
"rule": rule
}
fwService.append(ET.XML(fwMacro(**data)))
log.info("Configuring...")
url = endpoint.attrib.get("href")
xml = ET.tostring(elem, encoding="unicode")
log.debug(xml)
session.headers.update(
{
"Content-Type": (
"application/vnd.vmware.admin"
".edgeGatewayServiceConfiguration+xml"
)
}
)
try:
response = self.check_response(
*self.wait_for(
session.post(url, data=xml),
timeout=None
)
)
tree = ET.fromstring(response.text)
task = next(
self.get_tasks(response),
Task().feed_xml(
tree, ns="{http://www.vmware.com/vcloud/v1.5}"
)
)
log.info("Waiting...")
self.monitor(task, session, status=status)
log.info("Task complete.")
except (StopIteration, TypeError) as e:
log.error(e)
self.send_status(status, stop=True)
def recompose_vapp(self, session, token, callback=None, status=None, **kwargs):
log = logging.getLogger("maloja.builder.recompose_vapp")
macro = PageTemplateFile(
pkg_resources.resource_filename(
"maloja.workflow", "RecomposeVAppParams.pt"
)
)
vapp = self.built[VApp][0]
template = self.plans[Template][0]
# TODO: Decide on naming system
for vm in self.built[Vm]:
vm.name = uuid.uuid4().hex
# TODO: think harder about Networks
data = {
"appliance": {
"name": vapp.name,
"description": "Created by Maloja builder",
"vms": self.built[Vm],
},
"networks": self.built[Network][-1:],
"template": {
"name": template.name,
"href": template.href
}
}
xml = macro(**data)
url = "{vapp.href}/{endpoint}".format(
vapp=vapp,
endpoint="action/recomposeVApp"
)
session.headers.update(
{"Content-Type": "application/vnd.vmware.vcloud.recomposeVAppParams+xml"}
)
log.info("Recomposing...")
try:
response = self.check_response(
*self.wait_for(
session.post(url, data=xml),
timeout=None
)
)
tree = ET.fromstring(response.text)
task = next(
self.get_tasks(response),
Task().feed_xml(
tree, ns="{http://www.vmware.com/vcloud/v1.5}"
)
)
log.info("Waiting...")
self.monitor(task, session, status=status)
log.info("Task complete.")
except (StopIteration, TypeError) as e:
log.error(e)
self.send_status(status, stop=True)
try:
response = self.check_response(*self.wait_for(session.get(vapp.href)))
except (StopIteration, TypeError):
self.send_status(status, stop=True)
return
elems = list(find_xpath(
"./*/*/[@type='application/vnd.vmware.vcloud.vm+xml']",
ET.fromstring(response.text)
))
refs = {elem.attrib.get("name"): elem.attrib.get("href") for elem in elems}
for vm in self.built[Vm]:
if vm.name in refs:
vm.href = refs[vm.name]
