def on_template(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_template")
tree = ET.fromstring(response.text)
obj = Template().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}")
path = path._replace(file="template.yaml")
cache(path, obj)
vms = find_xpath(
"./*/*/[@type='application/vnd.vmware.vcloud.vm+xml']",
ET.fromstring(response.text)
)
ops = [session.get(
vm.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_vm,
path._replace(node=vm.attrib.get("name")),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, vm in enumerate(vms)]
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
def make_path(path:Path, prefix="tmp", suffix=""):
if not path.home:
path = path._replace(home=getpass.getuser())
dctry = os.path.join(path.root, path.home)
os.makedirs(dctry, exist_ok=True)
if path.slot is None and path.file is not None:
slot = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dctry)
return path._replace(slot=os.path.basename(slot))
else:
return path
def create_from_resource(path:Resource, poa:list, role:list, routing:list, prefix="flow_", suffix=""):
if all(path[:5]) and not any(path[5:]):
parent = os.path.join(*path[:5])
drctry = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=parent)
flow = path._replace(flow=os.path.basename(drctry))
# MRO important here.
for registry, choices in [
("turberfield.ipc.routing", routing),
("turberfield.ipc.poa", poa),
("turberfield.ipc.role", role)
]:
policies = dict(gather_installed(registry))
for option in choices:
try:
typ = policies[option]
if issubclass(typ, Pooled):
others = [Flow.inspect(i) for i in Flow.find(path, application="*", policy=option)]
obj = typ.allocate(others=others)
else:
obj = typ()
flow = flow._replace(policy=option, suffix=".json")
with open(os.path.join(*flow[:-1]) + flow.suffix, 'w') as record:
record.write(obj.__json__())
record.flush()
except KeyError:
warnings.warn("No policy found for '{}'.".format(option))
yield None
except Exception as e:
warnings.warn("Create error: {}".format(e))
yield None
else:
yield flow
def load(self):
name, pickler = next(
((k, v) for k, v in self._services.items()
if isinstance(v, Persistent.Pickled)
and v.name == "businesses"),
None
)
if pickler is not None:
path = Persistent.make_path(
Persistent.recent_slot(pickler.path)._replace(file=pickler.path.file)
)
self._services[name] = self._services[name]._replace(path=path)
fP = os.path.join(*path)
if not os.path.isfile(fP):
proprietor = Player(id=uuid.uuid4().hex, name=self.player.name)
locns = [Location("Addison Arches 18a", 100)]
self.businesses.insert(
0, CashBusiness(proprietor, None, locns, tally=1000))
else:
with open(fP, "rb") as fObj:
self.businesses = pickle.load(fObj)
self.path = path._replace(file=None)
self.location = locations[-1].name
return self
def create_from_resource(path: Resource, poa: list, role: list, routing: list, prefix="flow_", suffix=""):
if all(path[:5]) and not any(path[5:]):
parent = os.path.join(*path[:5])
drctry = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=parent)
flow = path._replace(flow=os.path.basename(drctry))
# MRO important here.
for registry, choices in [
("turberfield.ipc.routing", routing),
("turberfield.ipc.poa", poa),
("turberfield.ipc.role", role),
]:
policies = dict(gather_installed(registry))
for option in choices:
try:
typ = policies[option]
if issubclass(typ, Pooled):
others = [Flow.inspect(i) for i in Flow.find(path, application="*", policy=option)]
obj = typ.allocate(others=others)
else:
obj = typ()
flow = flow._replace(policy=option, suffix=".json")
with open(os.path.join(*flow[:-1]) + flow.suffix, "w") as record:
record.write(obj.__json__())
record.flush()
except KeyError:
warnings.warn("No policy found for '{}'.".format(option))
yield None
except Exception as e:
warnings.warn("Create error: {}".format(e))
yield None
else:
yield flow
def on_org(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_org")
tree = ET.fromstring(response.text)
obj = Org().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}")
path = path._replace(file="org.yaml")
fP = cache(path, obj)
ctlgs = find_xpath(
"./*/[@type='application/vnd.vmware.vcloud.catalog+xml']",
ET.fromstring(response.text)
)
vdcs = find_xpath(
"./*/[@type='application/vnd.vmware.vcloud.vdc+xml']",
ET.fromstring(response.text)
)
ops = [session.get(
vdc.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_vdc,
path._replace(service=vdc.attrib.get("name")),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, vdc in enumerate(vdcs)] + [session.get(
ctlg.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_catalog,
path._replace(
service="catalogs",
category=ctlg.attrib.get("name")
),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, ctlg in enumerate(ctlgs)]
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
def on_vapp(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_vapp")
tree = ET.fromstring(response.text)
obj = VApp().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}")
path = path._replace(file="vapp.yaml")
fP = cache(path, obj)
url = urlparse(response.url)
query = "/".join((
url.scheme + "://" + url.netloc,
"api/vms/query?filter=(container=={0})".format(urlquote(response.url))
))
vms = list(find_xpath(
"./*/*/[@type='application/vnd.vmware.vcloud.vm+xml']",
ET.fromstring(response.text)
))
try:
ops = [session.get(
vm.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_vm,
path._replace(node=vm.attrib.get("name")),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, vm in enumerate(vms)] + [session.get(
query,
background_callback=functools.partial(
Surveyor.on_vmrecords,
path,
results=results,
status=status._replace(job=status.job + len(vms) + 1) if status else None
)
)]
except Exception as e:
log.error(e)
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
def on_vm(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_vm")
ns = "{http://www.vmware.com/vcloud/v1.5}"
tree = ET.fromstring(response.text)
path = path._replace(file="vm.yaml")
obj = Vm()
found, obj = next(find_ypath(path, obj), (None, obj))
if found is not None:
log.debug("Loaded existing object: {0}".format(vars(obj)))
# Update the existing object with attributes from the VM
obj.feed_xml(tree, ns=ns)
cache(path, obj)
if results and status:
results.put((status._replace(path=path), None))
def on_org_list(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_org_list")
tree = ET.fromstring(response.text)
orgs = find_xpath(
"./*/[@type='application/vnd.vmware.vcloud.org+xml']", tree)
ops = [session.get(
org.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_org,
path._replace(org=org.attrib.get("name")),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, org in enumerate(orgs)]
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
def on_edgeGateway(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_edgeGateway")
log.debug(path)
obj = None
ns = "{http://www.vmware.com/vcloud/v1.5}"
tree = ET.fromstring(response.text)
backoff = 5
try:
elem = next(tree.iter(ns + "EdgeGatewayRecord"))
while True:
op = session.get(elem.attrib.get("href"))
done, not_done = concurrent.futures.wait(
[op], timeout=10,
return_when=concurrent.futures.FIRST_EXCEPTION
)
try:
response = done.pop().result()
if response.status_code != 200:
raise HTTPError(response.status_code)
except (HTTPError, KeyError):
time.sleep(backoff)
backoff += 5
else:
tree = ET.fromstring(response.text)
log.debug(response.text)
obj = Gateway().feed_xml(tree, ns=ns)
break
except Exception as e:
log.error(e)
if obj is None:
log.warning("Found no Edge Gateway.")
else:
path = path._replace(file="edge.yaml")
fP = cache(path, obj)
if results and status:
results.put((status._replace(path=path), None))
def on_catalogitem(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_catalogitem")
templates = find_xpath(
".//*[@type='application/vnd.vmware.vcloud.vAppTemplate+xml']",
ET.fromstring(response.text)
)
templates = list(templates)
ops = [session.get(
tmplt.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_template,
path._replace(container=tmplt.attrib.get("name")),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, tmplt in enumerate(templates)]
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
def on_orgVdcNetwork(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_orgVdcNetwork")
ns = "{http://www.vmware.com/vcloud/v1.5}"
tree = ET.fromstring(response.text)
backoff = 5
try:
for elem in tree.iter(ns + "OrgVdcNetworkRecord"):
obj = None
while True:
op = session.get(elem.attrib.get("href"))
done, not_done = concurrent.futures.wait(
[op], timeout=10,
return_when=concurrent.futures.FIRST_EXCEPTION
)
try:
response = done.pop().result()
if response.status_code != 200:
raise HTTPError(response.status_code)
except (HTTPError, KeyError):
time.sleep(backoff)
backoff += 5
else:
tree = ET.fromstring(response.text)
obj = Network().feed_xml(tree, ns=ns)
break
if obj is not None:
path = path._replace(container=obj.name, file="network.yaml")
fP = cache(path, obj)
except Exception as e:
log.error(e)
if results and status:
results.put((status._replace(path=path), None))
def on_vdc(path, session, response, results=None, status=None):
log = logging.getLogger("maloja.surveyor.on_vdc")
tree = ET.fromstring(response.text)
obj = Vdc().feed_xml(tree, ns="{http://www.vmware.com/vcloud/v1.5}")
path = path._replace(file="vdc.yaml")
cache(path, obj)
edgeGWs = find_xpath(
"./*/[@type='application/vnd.vmware.vcloud.query.records+xml']",
tree,
rel="edgeGateways"
)
orgVdcNets = find_xpath(
"./*/[@type='application/vnd.vmware.vcloud.query.records+xml']",
tree,
rel="orgVdcNetworks"
)
vapps = find_xpath(
"./*/*/[@type='application/vnd.vmware.vcloud.vApp+xml']",
tree
)
ops = [session.get(
edgeGW.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_edgeGateway,
path,
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, edgeGW in enumerate(edgeGWs)] + [session.get(
orgVdcNet.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_orgVdcNetwork,
path._replace(
category="networks",
container=orgVdcNet.attrib.get("name")
),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, orgVdcNet in enumerate(orgVdcNets)] + [session.get(
vapp.attrib.get("href"),
background_callback=functools.partial(
Surveyor.on_vapp,
path._replace(
category="vapps",
container=vapp.attrib.get("name")
),
results=results,
status=status._replace(job=status.job + n) if status else None
)
) for n, vapp in enumerate(vapps)]
tasks = concurrent.futures.wait(
ops, timeout=3 * len(ops),
return_when=concurrent.futures.FIRST_EXCEPTION
)
if results and status:
results.put((status._replace(path=path), None))
def test_no_recent_slot(self):
path = self.make_home()
path = Persistent.recent_slot(path)
self.assertIs(None, path.slot)
path = path._replace(file="objects.pkl")
path = Persistent.make_path(path)