def load_files(self, rootdir): entity_path = os.path.join(rootdir, self.entity) _log.info("Loading data for entity %s from %s", self.entity, rootdir) if not os.path.isdir(entity_path): _log.error( "Data dir %s does not exist, will NOT remove missing entities", entity_path ) self.skip_removal = True for filename in glob.glob(os.path.join(entity_path, '*.yaml')): _log.info("Parsing file %s", filename) filedata = yaml_safe_load(filename, default={}) if not filedata: _log.error( "The file %s returned empty content, will NOT remove missing entities", filename ) self.skip_removal = True try: exp_data = self.cls.from_yaml(filedata) self.data.update(exp_data) except Exception: _log.critical( "Data in file %s could not be loaded", self.data, exc_info=True) self.skip_removal = True
def load(self): """ Load all the entities from file and sync """ if self.schema.has_errors: raise ValueError("Schema is broken, NOT loading data.") syncers = {} # load the files and Create the load order for name, cls in self.schema.entities.items(): cls = self.schema.entities[name] syncers[name] = EntitySyncer(name, cls) syncers[name].load_files(self.base_path) self.add(name, cls, []) for name in self.load_order: _log.info("Adding objects for %s", name) sync = syncers[name] try: sync.load() except Exception as e: _log.error("Loading of data for entity %s failed: %s", name, e) sync.skip_removal = True # Now let's cleanup in reverse order self.load_order.reverse() for name in self.load_order: _log.info("Removing stale objects for %s", name) syncers[name].cleanup()
def load_files(self, rootdir): entity_path = os.path.join(rootdir, self.entity) _log.info("Loading data for entity %s from %s", self.entity, rootdir) if not os.path.isdir(entity_path): _log.error( "Data dir %s does not exist, will NOT remove missing entities", entity_path) self.skip_removal = True for filename in glob.glob(os.path.join(entity_path, '*.yaml')): _log.info("Parsing file %s", filename) filedata = yaml_safe_load(filename, default={}) if not filedata: _log.error( "The file %s returned empty content, will NOT remove missing entities", filename) self.skip_removal = True try: exp_data = self.cls.from_yaml(filedata) self.data.update(exp_data) except Exception: _log.critical("Data in file %s could not be loaded", self.data, exc_info=True) self.skip_removal = True
def _catch(*args, **kwdargs): try: return fn(*args, **kwdargs) except BackendError as e: _log.error("%s Backend %s: %s", fn.__name__, log_msg, e) except Exception as e: _log.critical("%s generic %s: %s", fn.__name__, log_msg, e) raise
def __init__(self, args): if args.object_type != 'node': _log.error('%s can only act on node objects', args.mode) sys.exit(1) args.selector = 'name={}'.format(args.hostname) if 'service' in args and args.service is not None: args.selector += ',service={}'.format(args.service) args.action = [self.simple_actions[args.mode]] args.mode = 'select' super().__init__(args)
def __init__(self, args): if args.object_type != 'node': _log.error('%s can only act on node objects', args.mode) sys.exit(1) args.selector = 'name={}'.format(args.hostname) if 'service' in args and args.service is not None: args.selector += ',service={}'.format(args.service) args.action = [self.simple_actions[args.mode]] args.mode = 'select' super(ToolCliSimpleAction, self).__init__(args)
def fetch(self): self.exists = False try: values = self.backend.driver.read(self.key) if values: self.exists = True self.from_net(values) except drivers.NotFoundError: self.from_net(None) except drivers.BackendError as e: _log.error("Backend error while fetching %s: %s", self.key, e)
def fetch(self): self.exists = False try: values = self.backend.driver.read(self.key) if values: self.exists = True except drivers.NotFoundError: return self._from_net(None) except drivers.BackendError as e: _log.error("Backend error while fetching %s: %s", self.key, e) # TODO: maybe catch the backend errors separately return None self._from_net(values)
def get(configfile): """ Loads the config from file """ try: with open(configfile, 'rb') as fh: config = yaml.load(fh.read()) except Exception as e: _log.error('Could not load file %s: %s', configfile, e) config = {} return Config(**config)
def _run_action(self): fail = False for obj in self.host_list(): try: a = action.get_action(obj, self._action) msg = a.run() except action.ActionError as e: fail = True _log.error("Invalid action, reason: %s", str(e)) except BackendError as e: fail = True _log.error("Error when trying to %s on %s", self._action, self._namedef) _log.error("Failure writing to the kvstore: %s", str(e)) except Exception as e: fail = True _log.error("Error when trying to %s on %s", self._action, self._namedef) _log.exception("Generic action failure: %s", str(e)) else: print(msg) if not fail: self.announce() return True else: return False
def _run_action(self): fail = False for obj in self.host_list(): try: a = action.get_action(obj, self._action) msg = a.run() except action.ActionError as e: fail = True _log.error("Invalid action, reason: %s", str(e)) except BackendError as e: fail = True _log.error("Error when trying to %s on %s", self._action, self._namedef) _log.error("Failure writing to the kvstore: %s", str(e)) except Exception as e: fail = True _log.error("Error when trying to %s on %s", self._action, self._namedef) _log.exception("Generic action failure: %s", str(e)) else: if sys.version_info[0] == 2: # Python 2 msg = msg.decode('utf-8') print(msg) if not fail: self.announce() return True else: return False
def from_file(cls, filename): """ Load a yaml file """ instance = cls() if not os.path.isfile(filename): return instance data = yaml_safe_load(filename, default={}) if not data: instance.has_errors = True return instance for objname, defs in data.items(): try: _log.debug('Loading entity %s', objname) entity_name = re.sub(r'\W', '_', objname.capitalize()) entity = factory(entity_name, defs) instance.entities[objname] = entity except Exception as e: _log.error('Could not load entity %s: %s', objname, e, exc_info=True) instance.has_errors = True return instance
def main(arguments=None): if arguments is None: arguments = list(sys.argv) arguments.pop(0) args = get_args(arguments) if args.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.WARN) try: c = configuration.get(args.config) KVObject.setup(c) except Exception as e: raise _log.critical("Invalid configuration: %s", e) sys.exit(1) files = tag_files(args.directory) # Load services data. servdata = {} if files['services']: for service_file in files['services']: with open(service_file, 'rb') as fh: try: d = yaml.load(fh) except: d = {} servdata.update(d) if not servdata: _log.critical("We found no services, so we can't import" " nodes either. Bailing out") sys.exit(1) # Refresh services: rem = {} for cluster, data in servdata.items(): if not type(data) == dict: continue load, rem[cluster] = get_service_actions(cluster, data) if args.lock: servlocker = service.Service.lock else: servlocker = dummy_lock load_services(cluster, load, data, servlocker) # sync nodes for filename in files['nodes']: dc = os.path.basename(filename).rstrip('.yaml') try: with open(filename, 'rb') as fh: dc_data = yaml.load(fh) except: _log.error("Malformed yaml data in %s", filename) _log.error("Skipping loading/removing nodes, please correct!") else: if args.lock: locker = service.Service.lock else: locker = dummy_lock load_nodes(dc, dc_data, locker) # Now delete services for cluster, servnames in rem.items(): remove_services(cluster, servnames, servlocker)