def on_catalog(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_catalog")
tree = ET.fromstring(response.text)
obj = Catalog().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}")
path = path._replace(file="catalog.yaml")
cache(path, obj)
items = find_xpath(
".//*[@type='application/vnd.vmware.vcloud.catalogItem+xml']",
ET.fromstring(response.text)
)
ops = [session.get(
item.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_catalogitem, path,
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, item in enumerate(items)]
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
def test_ypath_with_keywords(self):
proj = self.fixture[0][1]
for obj, path in self.fixture:
cache(path, obj)
results = list(find_ypath(proj, Vm(), name="server"))
self.assertEqual(2, len(results))
self.assertTrue(all(len(i) == 2 for i in results), results)
self.assertTrue(all(isinstance(i[0], Path) for i in results), results)
self.assertTrue(all(isinstance(i[1], Vm) for i in results), results)
self.assertIn(self.fixture[-1][1], [i[0] for i in results], results)
def test_ypath_by_type(self):
proj = self.fixture[0][1]
for obj, path in self.fixture:
cache(path, obj)
results = list(find_ypath(proj, Vm()))
self.assertEqual(4, len(results))
self.assertTrue(all(len(i) == 2 for i in results), results)
self.assertTrue(all(isinstance(i[0], Path) for i in results), results)
self.assertTrue(all(isinstance(i[1], Vm) for i in results), results)
self.assertIn(self.fixture[-1][1], [i[0] for i in results], results)
def on_vm(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_vm")
ns = "{http://www.vmware.com/vcloud/v1.5}"
tree = ET.fromstring(response.text)
path = path._replace(file="vm.yaml")
obj = Vm()
found, obj = next(find_ypath(path, obj), (None, obj))
if found is not None:
log.debug("Loaded existing object: {0}".format(vars(obj)))
# Update the existing object with attributes from the VM
obj.feed_xml(tree, ns=ns)
cache(path, obj)
if results and status:
results.put((status._replace(path=path), None))
def test_cache_path(self):
self.maxDiff = 1200
for obj, path in self.fixture:
with self.subTest(path=path):
rv = cache(path, obj)
check = split_to_path(rv, root=self.drcty.name)
self.assertEqual(path, check, check)
self.assertTrue(os.path.isfile(rv))
def test_ypath_with_attributes(self):
proj = self.fixture[0][1]
for obj, path in self.fixture:
cache(path, obj)
results = list(find_ypath(proj, Vm(name="server")))
self.assertEqual(2, len(results))
self.assertTrue(all(len(i) == 2 for i in results), results)
self.assertTrue(all(isinstance(i[0], Path) for i in results), results)
self.assertTrue(all(isinstance(i[1], Vm) for i in results), results)
self.assertIn(self.fixture[-1][1], [i[0] for i in results], results)
results = list(find_ypath(proj, Gateway(name="0-123-4-567890-edge")))
self.assertTrue(results)
results = list(find_ypath(proj, Network(name="USER_NET")))
self.assertTrue(results)
def test_object_cache(self):
self.maxDiff = 1200
for obj, path in self.fixture:
with self.subTest(path=path):
fP = cache(path, obj)
with open(fP, 'r') as data:
text = data.read()
rv = type(obj)(**yaml_loads(text))
self.assertEqual(vars(obj), vars(rv))
def on_org(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_org")
tree = ET.fromstring(response.text)
obj = Org().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}")
path = path._replace(file="org.yaml")
fP = cache(path, obj)
ctlgs = find_xpath(
"./*/[@type='application/vnd.vmware.vcloud.catalog+xml']",
ET.fromstring(response.text)
)
vdcs = find_xpath(
"./*/[@type='application/vnd.vmware.vcloud.vdc+xml']",
ET.fromstring(response.text)
)
ops = [session.get(
vdc.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_vdc,
path._replace(service=vdc.attrib.get("name")),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, vdc in enumerate(vdcs)] + [session.get(
ctlg.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_catalog,
path._replace(
service="catalogs",
category=ctlg.attrib.get("name")
),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, ctlg in enumerate(ctlgs)]
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
def on_vapp(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_vapp")
tree = ET.fromstring(response.text)
obj = VApp().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}")
path = path._replace(file="vapp.yaml")
fP = cache(path, obj)
url = urlparse(response.url)
query = "/".join((
url.scheme + "://" + url.netloc,
"api/vms/query?filter=(container=={0})".format(urlquote(response.url))
))
vms = list(find_xpath(
"./*/*/[@type='application/vnd.vmware.vcloud.vm+xml']",
ET.fromstring(response.text)
))
try:
ops = [session.get(
vm.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_vm,
path._replace(node=vm.attrib.get("name")),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, vm in enumerate(vms)] + [session.get(
query,
background_callback=functools.partial(
Surveyor.on_vmrecords,
path,
results=results,
status=status._replace(job=status.job + len(vms) + 1) if status else None
)
)]
except Exception as e:
log.error(e)
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
def on_edgeGateway(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_edgeGateway")
log.debug(path)
obj = None
ns = "{http://www.vmware.com/vcloud/v1.5}"
tree = ET.fromstring(response.text)
backoff = 5
try:
elem = next(tree.iter(ns + "EdgeGatewayRecord"))
while True:
op = session.get(elem.attrib.get("href"))
done, not_done = concurrent.futures.wait(
[op], timeout=10,
return_when=concurrent.futures.FIRST_EXCEPTION
)
try:
response = done.pop().result()
if response.status_code != 200:
raise HTTPError(response.status_code)
except (HTTPError, KeyError):
time.sleep(backoff)
backoff += 5
else:
tree = ET.fromstring(response.text)
log.debug(response.text)
obj = Gateway().feed_xml(tree, ns=ns)
break
except Exception as e:
log.error(e)
if obj is None:
log.warning("Found no Edge Gateway.")
else:
path = path._replace(file="edge.yaml")
fP = cache(path, obj)
if results and status:
results.put((status._replace(path=path), None))
def on_orgVdcNetwork(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_orgVdcNetwork")
ns = "{http://www.vmware.com/vcloud/v1.5}"
tree = ET.fromstring(response.text)
backoff = 5
try:
for elem in tree.iter(ns + "OrgVdcNetworkRecord"):
obj = None
while True:
op = session.get(elem.attrib.get("href"))
done, not_done = concurrent.futures.wait(
[op], timeout=10,
return_when=concurrent.futures.FIRST_EXCEPTION
)
try:
response = done.pop().result()
if response.status_code != 200:
raise HTTPError(response.status_code)
except (HTTPError, KeyError):
time.sleep(backoff)
backoff += 5
else:
tree = ET.fromstring(response.text)
obj = Network().feed_xml(tree, ns=ns)
break
if obj is not None:
path = path._replace(container=obj.name, file="network.yaml")
fP = cache(path, obj)
except Exception as e:
log.error(e)
if results and status:
results.put((status._replace(path=path), None))
def on_vdc(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_vdc")
tree = ET.fromstring(response.text)
obj = Vdc().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}")
path = path._replace(file="vdc.yaml")
cache(path, obj)
edgeGWs = find_xpath(
"./*/[@type='application/vnd.vmware.vcloud.query.records+xml']",
tree,
rel="edgeGateways"
)
orgVdcNets = find_xpath(
"./*/[@type='application/vnd.vmware.vcloud.query.records+xml']",
tree,
rel="orgVdcNetworks"
)
vapps = find_xpath(
"./*/*/[@type='application/vnd.vmware.vcloud.vApp+xml']",
tree
)
ops = [session.get(
edgeGW.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_edgeGateway,
path,
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, edgeGW in enumerate(edgeGWs)] + [session.get(
orgVdcNet.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_orgVdcNetwork,
path._replace(
category="networks",
container=orgVdcNet.attrib.get("name")
),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, orgVdcNet in enumerate(orgVdcNets)] + [session.get(
vapp.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_vapp,
path._replace(
category="vapps",
container=vapp.attrib.get("name")
),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, vapp in enumerate(vapps)]
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
