def desired_status(self): """Get the desired status. Will download and compile status if necessary.""" if self._desired_status is None: services = context.machination_worker_elt.xpath("services/service") # TODO download from all services and merge. For now just # do the first one. # hurl = services[0].xpath('hierarchy/@id') service_id = services[0].get("id") l.lmsg('Connecting to service "{}"'.format(service_id)) # find the machination id for this service mid = context.get_id(service_id) wc = WebClient.from_service_id(service_id, "os_instance") # channel = wc.call("ProfChannel", 'os_instance') try: data = wc.call("GetAssertionList", "os_instance", mid) except: # couldn't dowload assertions - go with last desireed # status. Should already be canonicalized. l.wmsg("Failed to download assertions - using desired status from context.") self._desired_status = copy.deepcopy(context.desired_status.getroot()) else: # we do have some assertions - compile them # pprint.pprint(data) ac = AssertionCompiler(wc) self._desired_status, res = ac.compile(data) mc14n(self._desired_status) # Save as desired-status.xml with open(os.path.join(context.status_dir(), "desired-status.xml"), "w") as ds: ds.write(etree.tostring(self._desired_status, pretty_print=True).decode()) return self._desired_status
def load_previous_status(self): """Load previous_status.xml""" fname = os.path.join(context.status_dir(), "previous-status.xml") try: self._previous_status = etree.parse(fname).getroot() except IOError: # couldn't read file may be lack of permission or not exists # if not exists (first run?) we should make a new status if not os.path.isfile(fname): self._previous_status = E.status() else: raise mc14n(self._previous_status) return self._previous_status
def spawn_work(parcels): """Take a work parcels dictionary and do the work""" for workername in parcels: workerdesc = xmltools.WorkerDescription(os.path.join(context.status_dir(), "workers", workername, "description.xml"))
def __init__(self, hierarchy_url, authen_type, obj_type, credentials=None, service_id=None): '''Create a new WebClient hierarchy_url: URL on which to contact hierarchy authen_type: Type of authentication to use (cosign, cert, public) obj_type: Type of entity making the request (os_instance, person, ...) credentials (=None): A dictionary of credential information or a callable that will return such. service_id (=None): Used by the 'cert' method to look up certificate locations if they are not specified. ''' self.hierarchy_url = hierarchy_url self.authen_type = authen_type self.obj_type = obj_type self.url = '{}/{}'.format( self.hierarchy_url, self.authen_type ) self.encoding = 'utf-8' self.l = context.logger self.cookie_file = os.path.join(context.status_dir(), 'cookies.txt') self.cookie_jar = None handlers = [] if self.authen_type == 'cosign': self.l.lmsg('building cosign handlers') self.cookie_jar = http.cookiejar.MozillaCookieJar( self.cookie_file ) handlers.append( urllib_request.HTTPCookieProcessor( self.cookie_jar ) ) if (not hasattr(credentials, '__call__')) and (credentials is not None): values = credentials credentials = lambda x: values handlers.append( CosignHandler( self.authen_elt.get('cosignLoginPage'), self.cookie_jar, CosignPasswordMgr(callback = credentials), save_cookies = True ) ) elif self.authen_type == 'cert': # Get the cert and key locations from credentials try: # See if credentials is callable cred = credentials() except TypeError: # It should be a dictionary if credentials is None: cred = {} else: cred = credentials keyfile = cred.get('key') if keyfile is None and service_id is not None: keyfile = os.path.join(context.conf_dir(), 'services', service_id, 'myself.key') certfile = cred.get('cert') if certfile is None and service_id is not None: certfile = os.path.join(context.conf_dir(), 'services', service_id, 'myself.crt') handlers.append( HTTPSClientAuthHandler(keyfile,certfile) ) elif self.authen_type == 'debug': try: # See if credentials is callable cred = credentials() except TypeError: # It should be a dictionary cred = credentials if cred is None: # Still not set: raise exception raise ValueError('"name" not set for debug authentication') self.url = '{}/{}:{}'.format(self.url, self.obj_type, cred.get('name')) elif self.authen_type == 'public': # Nothing to be done - just need it to be in the list of # auth types. pass else: raise ValueError( 'Invalid authentication type "{}"'.format(self.authen_type) ) self.opener = urllib_request.build_opener(*handlers)
def do_update(self): """Perform an update cycle""" self.results = None if context.desired_status.getroot().get("autoconstructed"): raise ValueError("Refusing to use autoconstructed status.") l.dmsg("desired:\n%s" % pstring(self.desired_status()), 10) l.dmsg("initial:\n%s" % pstring(self.initial_status()), 10) comp = XMLCompare(copy.deepcopy(self.initial_status()), self.desired_status()) l.dmsg("xpaths by state:\n" + pprint.pformat(comp.bystate), 10) # See if we have to do a self update iv_mrx = MRXpath('/status/worker[@id="__machination__"]/installedVersion') selfupdate = False selfupdate_bundles = set() if iv_mrx.to_xpath() in comp.find_work(): # installedVersion has changed somehow wus, working = generate_wus({iv_mrx.to_xpath()}, comp) wu = wus[0] if wus else etree.Element("wu", op="nothing") if wu.get("op") == "add" or wu.get("op") == "deepmod": # Definitely updating l.lmsg("{} on {}: need to self update".format(wu.get("op"), iv_mrx.to_xpath()), 3) selfupdate = True # Check for bundles and add to selfupdate_bundles for ivb_elt in wu[0].xpath("machinationFetcherBundle"): bid = MRXpath.quote_id(MRXpath, ivb_elt.get("id")) bundle_xp = MRXpath("/status/worker[@id='fetcher']/bundle['{}']".format(bid)).to_xpath() selfupdate_bundles.add(bundle_xp) # only interested in bundles with a work unit to do selfupdate_bundles = selfupdate_bundles & comp.find_work() # fetcher_wus, working = generate_wus(todo, comp) # # use the fetcher worker to get bundles # worker = self.worker('fetcher') # for wu in fetcher_wus: try: deps = self.desired_status().xpath("/status/deps")[0] except IndexError: deps = etree.fromstring("<status><deps/></status>")[0] wudeps = comp.wudeps(deps.iterchildren(tag=etree.Element)) # Track success/failure of work units. # # Before a work unit is attempted work_status[wu] should not # exist. # # Afterward, work_status[wu] should contain an array with a # status (True = succeeded, False = failed) and either the wu # element or an error message as appropriate: # # { # wu1: [True, wu_elt], # wu2: [False, "Worker 'splat' not available"] # wu3: [False, "Dependency 'wu2' failed"] # } work_status = {} # set up a dictionary: # { work_unit: [list, of, units, work_unit, depends, on] } work_depends = {} if selfupdate: # installedVersion depends on all bundles wudeps.extend([[x, iv_mrx.to_xpath()] for x in selfupdate_bundles]) # Everything else apart from selfupdate bundles depends on # installedVersion wudeps.extend( [[iv_mrx.to_xpath(), x] for x in (comp.find_work() - selfupdate_bundles - {iv_mrx.to_xpath()})] ) for dep in wudeps: if work_depends.get(dep[1]): # entry for dep[1] already exists, add to it work_depends.get(dep[1]).append(dep[0]) else: # entry for dep[1] does not exist, create it work_depends[dep[1]] = [dep[0]] # l.dmsg('work_depends = {}'.format(pprint.pformat(work_depends))) # we need to make all workunits depend on something for # topsort to work if selfupdate: # selfupdate_bundles should be done first wudeps.extend([["", x] for x in selfupdate_bundles]) else: wudeps.extend([["", x] for x in comp.find_work()]) # l.dmsg('wudeps = {}'.format(pprint.pformat(wudeps))) wu_updated_status = copy.deepcopy(self.initial_status()) i = 0 # failures = [] for lev in iter(topsort.topsort_levels(wudeps)): i += 1 if i == 1: # this is the fake workunit '' we put in above continue l.dmsg("xpaths for level {}:\n".format(i) + pprint.pformat(lev), 10) wus, working_elt = generate_wus(set(lev), comp) # l.dmsg(pstring(self.initial_status(),10)) # l.dmsg(pstring(self.desired_status(),10)) for wu in wus: l.dmsg(pstring(wu), 10) add_map = {} for wu in wus: # If it's an add for a worker, add the worker element wu_mrx = MRXpath(wu.get("id")) if wu_mrx.to_noid_path() == "/status/worker" and wu.get("op") == "add": l.lmsg("Adding worker element " + wu_mrx.to_xpath()) wu_updated_status.xpath("/status")[0].append(etree.Element("worker", id=wu_mrx.id())) continue # If it's an add, we need to add it to the add_map so # that adds still function properly if they get out of # order or previous adds have failed. if wu.get("op") == "add": add_map[wu.get("id")] = get_fullpos(wu.get("pos"), MRXpath(wu.get("id")).parent()) # check to make sure any dependencies have been done check = self.check_deps(wu, work_depends, work_status) if not check[0]: l.wmsg("Failing {}: dep {} failed".format(wu.get("id"), check[1])) work_status[wu.get("id")] = [False, "Dependency '{}' failed".format(check[1])] # don't include this wu in work to be done continue wname = MRXpath(wu.get("id")).workername(prefix="/status") worker = self.worker(wname) l.lmsg("dispatching to " + wname) l.dmsg("work:\n" + pstring(wu)) if worker: # need to wrap the wu in a wus element workelt = etree.Element("wus", worker=wname) workelt.append(copy.deepcopy(wu)) try: results = worker.do_work(workelt) except Exception as e: exc_type, exc_value, exc_tb = sys.exc_info() # There's only one, but in future there might # be more - loop over them. for curwu in workelt: work_status[curwu.get("id")] = [False, "Exception in worker {}\n{}".format(wname, str(e))] l.emsg( "Exception during {} - failing it\n{}".format( curwu.get("id"), "".join(traceback.format_tb(exc_tb)) + repr(e) ) ) else: self.process_results(results, workelt, work_status) wid = wu.get("id") completed = work_status.get(wid) if completed[0]: # Apply successes to wu_updated_status l.dmsg("Marking {} succeeded.".format(wid)) wu_updated_status = apply_wu(completed[1], wu_updated_status, add_map=add_map) else: l.dmsg("Marking {} failed.".format(wid)) # failures.append([wid, completed[1]]) else: # No worker: fail this set of work work_status[wu.get("id")] = [False, "No worker '{}'".format(wname)] # TODO(colin): parallelise downloads and other work # Report successes l.lmsg( "The following work units reported success:\n{}".format( pprint.pformat([k for k, v in work_status.items() if v[0]]) ) ) # Report failures. l.wmsg( "The following work units reported failure:\n{}".format( pprint.pformat([[k, v[1]] for k, v in work_status.items() if not v[0]]) ) ) # write calculated status to file fname = os.path.join(context.status_dir(), "previous-status.xml") with open(fname, "w") as prev: prev.write(etree.tostring(wu_updated_status, pretty_print=True).decode()) # see how the status has changed including calls to generate_status() new_status = self.gather_status() # write this status out as previous_status.xml with open(fname, "w") as prev: prev.write(etree.tostring(new_status, pretty_print=True).decode())