def on_catalog(path, session, response, results=None, status=None): log = logging.getLogger("maloja.surveyor.on_catalog") tree = ET.fromstring(response.text) obj = Catalog().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}") path = path._replace(file="catalog.yaml") cache(path, obj) items = find_xpath( ".//*[@type='application/vnd.vmware.vcloud.catalogItem+xml']", ET.fromstring(response.text) ) ops = [session.get( item.attrib.get("href"), background_callback=functools.partial( Surveyor.on_catalogitem, path, results=results, status=status._replace(job=status.job + n) if status else None ) ) for n, item in enumerate(items)] tasks = concurrent.futures.wait( ops, timeout=3 * len(ops), return_when=concurrent.futures.FIRST_EXCEPTION ) if results and status: results.put((status._replace(path=path), None))
def test_ypath_with_keywords(self): proj = self.fixture[0][1] for obj, path in self.fixture: cache(path, obj) results = list(find_ypath(proj, Vm(), name="server")) self.assertEqual(2, len(results)) self.assertTrue(all(len(i) == 2 for i in results), results) self.assertTrue(all(isinstance(i[0], Path) for i in results), results) self.assertTrue(all(isinstance(i[1], Vm) for i in results), results) self.assertIn(self.fixture[-1][1], [i[0] for i in results], results)
def test_ypath_by_type(self): proj = self.fixture[0][1] for obj, path in self.fixture: cache(path, obj) results = list(find_ypath(proj, Vm())) self.assertEqual(4, len(results)) self.assertTrue(all(len(i) == 2 for i in results), results) self.assertTrue(all(isinstance(i[0], Path) for i in results), results) self.assertTrue(all(isinstance(i[1], Vm) for i in results), results) self.assertIn(self.fixture[-1][1], [i[0] for i in results], results)
def on_vm(path, session, response, results=None, status=None): log = logging.getLogger("maloja.surveyor.on_vm") ns = "{http://www.vmware.com/vcloud/v1.5}" tree = ET.fromstring(response.text) path = path._replace(file="vm.yaml") obj = Vm() found, obj = next(find_ypath(path, obj), (None, obj)) if found is not None: log.debug("Loaded existing object: {0}".format(vars(obj))) # Update the existing object with attributes from the VM obj.feed_xml(tree, ns=ns) cache(path, obj) if results and status: results.put((status._replace(path=path), None))
def test_cache_path(self): self.maxDiff = 1200 for obj, path in self.fixture: with self.subTest(path=path): rv = cache(path, obj) check = split_to_path(rv, root=self.drcty.name) self.assertEqual(path, check, check) self.assertTrue(os.path.isfile(rv))
def test_ypath_with_attributes(self): proj = self.fixture[0][1] for obj, path in self.fixture: cache(path, obj) results = list(find_ypath(proj, Vm(name="server"))) self.assertEqual(2, len(results)) self.assertTrue(all(len(i) == 2 for i in results), results) self.assertTrue(all(isinstance(i[0], Path) for i in results), results) self.assertTrue(all(isinstance(i[1], Vm) for i in results), results) self.assertIn(self.fixture[-1][1], [i[0] for i in results], results) results = list(find_ypath(proj, Gateway(name="0-123-4-567890-edge"))) self.assertTrue(results) results = list(find_ypath(proj, Network(name="USER_NET"))) self.assertTrue(results)
def test_object_cache(self): self.maxDiff = 1200 for obj, path in self.fixture: with self.subTest(path=path): fP = cache(path, obj) with open(fP, 'r') as data: text = data.read() rv = type(obj)(**yaml_loads(text)) self.assertEqual(vars(obj), vars(rv))
def on_org(path, session, response, results=None, status=None): log = logging.getLogger("maloja.surveyor.on_org") tree = ET.fromstring(response.text) obj = Org().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}") path = path._replace(file="org.yaml") fP = cache(path, obj) ctlgs = find_xpath( "./*/[@type='application/vnd.vmware.vcloud.catalog+xml']", ET.fromstring(response.text) ) vdcs = find_xpath( "./*/[@type='application/vnd.vmware.vcloud.vdc+xml']", ET.fromstring(response.text) ) ops = [session.get( vdc.attrib.get("href"), background_callback=functools.partial( Surveyor.on_vdc, path._replace(service=vdc.attrib.get("name")), results=results, status=status._replace(job=status.job + n) if status else None ) ) for n, vdc in enumerate(vdcs)] + [session.get( ctlg.attrib.get("href"), background_callback=functools.partial( Surveyor.on_catalog, path._replace( service="catalogs", category=ctlg.attrib.get("name") ), results=results, status=status._replace(job=status.job + n) if status else None ) ) for n, ctlg in enumerate(ctlgs)] tasks = concurrent.futures.wait( ops, timeout=3 * len(ops), return_when=concurrent.futures.FIRST_EXCEPTION ) if results and status: results.put((status._replace(path=path), None))
def on_vapp(path, session, response, results=None, status=None): log = logging.getLogger("maloja.surveyor.on_vapp") tree = ET.fromstring(response.text) obj = VApp().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}") path = path._replace(file="vapp.yaml") fP = cache(path, obj) url = urlparse(response.url) query = "/".join(( url.scheme + "://" + url.netloc, "api/vms/query?filter=(container=={0})".format(urlquote(response.url)) )) vms = list(find_xpath( "./*/*/[@type='application/vnd.vmware.vcloud.vm+xml']", ET.fromstring(response.text) )) try: ops = [session.get( vm.attrib.get("href"), background_callback=functools.partial( Surveyor.on_vm, path._replace(node=vm.attrib.get("name")), results=results, status=status._replace(job=status.job + n) if status else None ) ) for n, vm in enumerate(vms)] + [session.get( query, background_callback=functools.partial( Surveyor.on_vmrecords, path, results=results, status=status._replace(job=status.job + len(vms) + 1) if status else None ) )] except Exception as e: log.error(e) tasks = concurrent.futures.wait( ops, timeout=3 * len(ops), return_when=concurrent.futures.FIRST_EXCEPTION ) if results and status: results.put((status._replace(path=path), None))
def on_edgeGateway(path, session, response, results=None, status=None): log = logging.getLogger("maloja.surveyor.on_edgeGateway") log.debug(path) obj = None ns = "{http://www.vmware.com/vcloud/v1.5}" tree = ET.fromstring(response.text) backoff = 5 try: elem = next(tree.iter(ns + "EdgeGatewayRecord")) while True: op = session.get(elem.attrib.get("href")) done, not_done = concurrent.futures.wait( [op], timeout=10, return_when=concurrent.futures.FIRST_EXCEPTION ) try: response = done.pop().result() if response.status_code != 200: raise HTTPError(response.status_code) except (HTTPError, KeyError): time.sleep(backoff) backoff += 5 else: tree = ET.fromstring(response.text) log.debug(response.text) obj = Gateway().feed_xml(tree, ns=ns) break except Exception as e: log.error(e) if obj is None: log.warning("Found no Edge Gateway.") else: path = path._replace(file="edge.yaml") fP = cache(path, obj) if results and status: results.put((status._replace(path=path), None))
def on_orgVdcNetwork(path, session, response, results=None, status=None): log = logging.getLogger("maloja.surveyor.on_orgVdcNetwork") ns = "{http://www.vmware.com/vcloud/v1.5}" tree = ET.fromstring(response.text) backoff = 5 try: for elem in tree.iter(ns + "OrgVdcNetworkRecord"): obj = None while True: op = session.get(elem.attrib.get("href")) done, not_done = concurrent.futures.wait( [op], timeout=10, return_when=concurrent.futures.FIRST_EXCEPTION ) try: response = done.pop().result() if response.status_code != 200: raise HTTPError(response.status_code) except (HTTPError, KeyError): time.sleep(backoff) backoff += 5 else: tree = ET.fromstring(response.text) obj = Network().feed_xml(tree, ns=ns) break if obj is not None: path = path._replace(container=obj.name, file="network.yaml") fP = cache(path, obj) except Exception as e: log.error(e) if results and status: results.put((status._replace(path=path), None))
def on_vdc(path, session, response, results=None, status=None): log = logging.getLogger("maloja.surveyor.on_vdc") tree = ET.fromstring(response.text) obj = Vdc().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}") path = path._replace(file="vdc.yaml") cache(path, obj) edgeGWs = find_xpath( "./*/[@type='application/vnd.vmware.vcloud.query.records+xml']", tree, rel="edgeGateways" ) orgVdcNets = find_xpath( "./*/[@type='application/vnd.vmware.vcloud.query.records+xml']", tree, rel="orgVdcNetworks" ) vapps = find_xpath( "./*/*/[@type='application/vnd.vmware.vcloud.vApp+xml']", tree ) ops = [session.get( edgeGW.attrib.get("href"), background_callback=functools.partial( Surveyor.on_edgeGateway, path, results=results, status=status._replace(job=status.job + n) if status else None ) ) for n, edgeGW in enumerate(edgeGWs)] + [session.get( orgVdcNet.attrib.get("href"), background_callback=functools.partial( Surveyor.on_orgVdcNetwork, path._replace( category="networks", container=orgVdcNet.attrib.get("name") ), results=results, status=status._replace(job=status.job + n) if status else None ) ) for n, orgVdcNet in enumerate(orgVdcNets)] + [session.get( vapp.attrib.get("href"), background_callback=functools.partial( Surveyor.on_vapp, path._replace( category="vapps", container=vapp.attrib.get("name") ), results=results, status=status._replace(job=status.job + n) if status else None ) ) for n, vapp in enumerate(vapps)] tasks = concurrent.futures.wait( ops, timeout=3 * len(ops), return_when=concurrent.futures.FIRST_EXCEPTION ) if results and status: results.put((status._replace(path=path), None))