def desired_status(self):
"""Get the desired status. Will download and compile status if necessary."""
if self._desired_status is None:
services = context.machination_worker_elt.xpath("services/service")
# TODO download from all services and merge. For now just
# do the first one.
# hurl = services[0].xpath('hierarchy/@id')
service_id = services[0].get("id")
l.lmsg('Connecting to service "{}"'.format(service_id))
# find the machination id for this service
mid = context.get_id(service_id)
wc = WebClient.from_service_id(service_id, "os_instance")
# channel = wc.call("ProfChannel", 'os_instance')
try:
data = wc.call("GetAssertionList", "os_instance", mid)
except:
# couldn't dowload assertions - go with last desireed
# status. Should already be canonicalized.
l.wmsg("Failed to download assertions - using desired status from context.")
self._desired_status = copy.deepcopy(context.desired_status.getroot())
else:
# we do have some assertions - compile them
# pprint.pprint(data)
ac = AssertionCompiler(wc)
self._desired_status, res = ac.compile(data)
mc14n(self._desired_status)
# Save as desired-status.xml
with open(os.path.join(context.status_dir(), "desired-status.xml"), "w") as ds:
ds.write(etree.tostring(self._desired_status, pretty_print=True).decode())
return self._desired_status
def load_previous_status(self):
"""Load previous_status.xml"""
fname = os.path.join(context.status_dir(), "previous-status.xml")
try:
self._previous_status = etree.parse(fname).getroot()
except IOError:
# couldn't read file may be lack of permission or not exists
# if not exists (first run?) we should make a new status
if not os.path.isfile(fname):
self._previous_status = E.status()
else:
raise
mc14n(self._previous_status)
return self._previous_status
def spawn_work(parcels):
"""Take a work parcels dictionary and do the work"""
for workername in parcels:
workerdesc = xmltools.WorkerDescription(os.path.join(context.status_dir(), "workers", workername, "description.xml"))
def __init__(self, hierarchy_url, authen_type, obj_type,
credentials=None, service_id=None):
'''Create a new WebClient
hierarchy_url: URL on which to contact hierarchy
authen_type: Type of authentication to use (cosign, cert, public)
obj_type: Type of entity making the request (os_instance, person, ...)
credentials (=None): A dictionary of credential information or
a callable that will return such.
service_id (=None): Used by the 'cert' method to look up
certificate locations if they are not specified.
'''
self.hierarchy_url = hierarchy_url
self.authen_type = authen_type
self.obj_type = obj_type
self.url = '{}/{}'.format(
self.hierarchy_url,
self.authen_type
)
self.encoding = 'utf-8'
self.l = context.logger
self.cookie_file = os.path.join(context.status_dir(), 'cookies.txt')
self.cookie_jar = None
handlers = []
if self.authen_type == 'cosign':
self.l.lmsg('building cosign handlers')
self.cookie_jar = http.cookiejar.MozillaCookieJar(
self.cookie_file
)
handlers.append(
urllib_request.HTTPCookieProcessor(
self.cookie_jar
)
)
if (not hasattr(credentials, '__call__')) and (credentials is not None):
values = credentials
credentials = lambda x: values
handlers.append(
CosignHandler(
self.authen_elt.get('cosignLoginPage'),
self.cookie_jar,
CosignPasswordMgr(callback = credentials),
save_cookies = True
)
)
elif self.authen_type == 'cert':
# Get the cert and key locations from credentials
try:
# See if credentials is callable
cred = credentials()
except TypeError:
# It should be a dictionary
if credentials is None:
cred = {}
else:
cred = credentials
keyfile = cred.get('key')
if keyfile is None and service_id is not None:
keyfile = os.path.join(context.conf_dir(),
'services',
service_id,
'myself.key')
certfile = cred.get('cert')
if certfile is None and service_id is not None:
certfile = os.path.join(context.conf_dir(),
'services',
service_id,
'myself.crt')
handlers.append(
HTTPSClientAuthHandler(keyfile,certfile)
)
elif self.authen_type == 'debug':
try:
# See if credentials is callable
cred = credentials()
except TypeError:
# It should be a dictionary
cred = credentials
if cred is None:
# Still not set: raise exception
raise ValueError('"name" not set for debug authentication')
self.url = '{}/{}:{}'.format(self.url,
self.obj_type,
cred.get('name'))
elif self.authen_type == 'public':
# Nothing to be done - just need it to be in the list of
# auth types.
pass
else:
raise ValueError(
'Invalid authentication type "{}"'.format(self.authen_type)
)
self.opener = urllib_request.build_opener(*handlers)
def do_update(self):
"""Perform an update cycle"""
self.results = None
if context.desired_status.getroot().get("autoconstructed"):
raise ValueError("Refusing to use autoconstructed status.")
l.dmsg("desired:\n%s" % pstring(self.desired_status()), 10)
l.dmsg("initial:\n%s" % pstring(self.initial_status()), 10)
comp = XMLCompare(copy.deepcopy(self.initial_status()), self.desired_status())
l.dmsg("xpaths by state:\n" + pprint.pformat(comp.bystate), 10)
# See if we have to do a self update
iv_mrx = MRXpath('/status/worker[@id="__machination__"]/installedVersion')
selfupdate = False
selfupdate_bundles = set()
if iv_mrx.to_xpath() in comp.find_work():
# installedVersion has changed somehow
wus, working = generate_wus({iv_mrx.to_xpath()}, comp)
wu = wus[0] if wus else etree.Element("wu", op="nothing")
if wu.get("op") == "add" or wu.get("op") == "deepmod":
# Definitely updating
l.lmsg("{} on {}: need to self update".format(wu.get("op"), iv_mrx.to_xpath()), 3)
selfupdate = True
# Check for bundles and add to selfupdate_bundles
for ivb_elt in wu[0].xpath("machinationFetcherBundle"):
bid = MRXpath.quote_id(MRXpath, ivb_elt.get("id"))
bundle_xp = MRXpath("/status/worker[@id='fetcher']/bundle['{}']".format(bid)).to_xpath()
selfupdate_bundles.add(bundle_xp)
# only interested in bundles with a work unit to do
selfupdate_bundles = selfupdate_bundles & comp.find_work()
# fetcher_wus, working = generate_wus(todo, comp)
# # use the fetcher worker to get bundles
# worker = self.worker('fetcher')
# for wu in fetcher_wus:
try:
deps = self.desired_status().xpath("/status/deps")[0]
except IndexError:
deps = etree.fromstring("<status><deps/></status>")[0]
wudeps = comp.wudeps(deps.iterchildren(tag=etree.Element))
# Track success/failure of work units.
#
# Before a work unit is attempted work_status[wu] should not
# exist.
#
# Afterward, work_status[wu] should contain an array with a
# status (True = succeeded, False = failed) and either the wu
# element or an error message as appropriate:
#
# {
# wu1: [True, wu_elt],
# wu2: [False, "Worker 'splat' not available"]
# wu3: [False, "Dependency 'wu2' failed"]
# }
work_status = {}
# set up a dictionary:
# { work_unit: [list, of, units, work_unit, depends, on] }
work_depends = {}
if selfupdate:
# installedVersion depends on all bundles
wudeps.extend([[x, iv_mrx.to_xpath()] for x in selfupdate_bundles])
# Everything else apart from selfupdate bundles depends on
# installedVersion
wudeps.extend(
[[iv_mrx.to_xpath(), x] for x in (comp.find_work() - selfupdate_bundles - {iv_mrx.to_xpath()})]
)
for dep in wudeps:
if work_depends.get(dep[1]):
# entry for dep[1] already exists, add to it
work_depends.get(dep[1]).append(dep[0])
else:
# entry for dep[1] does not exist, create it
work_depends[dep[1]] = [dep[0]]
# l.dmsg('work_depends = {}'.format(pprint.pformat(work_depends)))
# we need to make all workunits depend on something for
# topsort to work
if selfupdate:
# selfupdate_bundles should be done first
wudeps.extend([["", x] for x in selfupdate_bundles])
else:
wudeps.extend([["", x] for x in comp.find_work()])
# l.dmsg('wudeps = {}'.format(pprint.pformat(wudeps)))
wu_updated_status = copy.deepcopy(self.initial_status())
i = 0
# failures = []
for lev in iter(topsort.topsort_levels(wudeps)):
i += 1
if i == 1:
# this is the fake workunit '' we put in above
continue
l.dmsg("xpaths for level {}:\n".format(i) + pprint.pformat(lev), 10)
wus, working_elt = generate_wus(set(lev), comp)
# l.dmsg(pstring(self.initial_status(),10))
# l.dmsg(pstring(self.desired_status(),10))
for wu in wus:
l.dmsg(pstring(wu), 10)
add_map = {}
for wu in wus:
# If it's an add for a worker, add the worker element
wu_mrx = MRXpath(wu.get("id"))
if wu_mrx.to_noid_path() == "/status/worker" and wu.get("op") == "add":
l.lmsg("Adding worker element " + wu_mrx.to_xpath())
wu_updated_status.xpath("/status")[0].append(etree.Element("worker", id=wu_mrx.id()))
continue
# If it's an add, we need to add it to the add_map so
# that adds still function properly if they get out of
# order or previous adds have failed.
if wu.get("op") == "add":
add_map[wu.get("id")] = get_fullpos(wu.get("pos"), MRXpath(wu.get("id")).parent())
# check to make sure any dependencies have been done
check = self.check_deps(wu, work_depends, work_status)
if not check[0]:
l.wmsg("Failing {}: dep {} failed".format(wu.get("id"), check[1]))
work_status[wu.get("id")] = [False, "Dependency '{}' failed".format(check[1])]
# don't include this wu in work to be done
continue
wname = MRXpath(wu.get("id")).workername(prefix="/status")
worker = self.worker(wname)
l.lmsg("dispatching to " + wname)
l.dmsg("work:\n" + pstring(wu))
if worker:
# need to wrap the wu in a wus element
workelt = etree.Element("wus", worker=wname)
workelt.append(copy.deepcopy(wu))
try:
results = worker.do_work(workelt)
except Exception as e:
exc_type, exc_value, exc_tb = sys.exc_info()
# There's only one, but in future there might
# be more - loop over them.
for curwu in workelt:
work_status[curwu.get("id")] = [False, "Exception in worker {}\n{}".format(wname, str(e))]
l.emsg(
"Exception during {} - failing it\n{}".format(
curwu.get("id"), "".join(traceback.format_tb(exc_tb)) + repr(e)
)
)
else:
self.process_results(results, workelt, work_status)
wid = wu.get("id")
completed = work_status.get(wid)
if completed[0]:
# Apply successes to wu_updated_status
l.dmsg("Marking {} succeeded.".format(wid))
wu_updated_status = apply_wu(completed[1], wu_updated_status, add_map=add_map)
else:
l.dmsg("Marking {} failed.".format(wid))
# failures.append([wid, completed[1]])
else:
# No worker: fail this set of work
work_status[wu.get("id")] = [False, "No worker '{}'".format(wname)]
# TODO(colin): parallelise downloads and other work
# Report successes
l.lmsg(
"The following work units reported success:\n{}".format(
pprint.pformat([k for k, v in work_status.items() if v[0]])
)
)
# Report failures.
l.wmsg(
"The following work units reported failure:\n{}".format(
pprint.pformat([[k, v[1]] for k, v in work_status.items() if not v[0]])
)
)
# write calculated status to file
fname = os.path.join(context.status_dir(), "previous-status.xml")
with open(fname, "w") as prev:
prev.write(etree.tostring(wu_updated_status, pretty_print=True).decode())
# see how the status has changed including calls to generate_status()
new_status = self.gather_status()
# write this status out as previous_status.xml
with open(fname, "w") as prev:
prev.write(etree.tostring(new_status, pretty_print=True).decode())
