def load_files(self, rootdir): entity_path = os.path.join(rootdir, self.entity) _log.info("Loading data for entity %s from %s", self.entity, rootdir) if not os.path.isdir(entity_path): _log.error( "Data dir %s does not exist, will NOT remove missing entities", entity_path) self.skip_removal = True for filename in glob.glob(os.path.join(entity_path, '*.yaml')): _log.info("Parsing file %s", filename) filedata = yaml_safe_load(filename, default={}) if not filedata: _log.error( "The file %s returned empty content, will NOT remove missing entities", filename) self.skip_removal = True try: exp_data = self.cls.from_yaml(filedata) self.data.update(exp_data) except Exception: _log.critical("Data in file %s could not be loaded", self.data, exc_info=True) self.skip_removal = True
def load_files(self, rootdir): entity_path = os.path.join(rootdir, self.entity) _log.info("Loading data for entity %s from %s", self.entity, rootdir) if not os.path.isdir(entity_path): _log.error( "Data dir %s does not exist, will NOT remove missing entities", entity_path ) self.skip_removal = True for filename in glob.glob(os.path.join(entity_path, '*.yaml')): _log.info("Parsing file %s", filename) filedata = yaml_safe_load(filename, default={}) if not filedata: _log.error( "The file %s returned empty content, will NOT remove missing entities", filename ) self.skip_removal = True try: exp_data = self.cls.from_yaml(filedata) self.data.update(exp_data) except Exception: _log.critical( "Data in file %s could not be loaded", self.data, exc_info=True) self.skip_removal = True
def main(arguments=None): if arguments is None: arguments = sys.argv[1:] args = get_args(arguments) if args.debug: log_level = logging.DEBUG else: log_level = logging.INFO logging.basicConfig( level=log_level, format='%(asctime)s [%(levelname)s] %(name)s::%(funcName)s: %(message)s', datefmt='%F %T' ) try: c = configuration.get(args.config) KVObject.setup(c) except Exception as e: _log.critical("Invalid configuration: %s", e) sys.exit(1) if not os.path.isdir(args.directory): _log.critical("Could not find directory %s", args.directory) sys.exit(2) sync = Syncer(args.schema, args.directory) sync.load()
def _tagged_host_list(self): cur_dir = self.entity.dir(*self.tags) warn = False if self._namedef == "all": all_objects = KVObject.backend.driver.ls(cur_dir) objlist = [k for (k, v) in all_objects] if self._action == "get": if self.args.yaml: print(yaml.dump(dict(all_objects), default_flow_style=False)) else: print(json.dumps(dict(all_objects))) return [] else: retval = objlist warn = True elif not self._namedef.startswith('re:'): return [self._namedef] else: regex = self._namedef.replace('re:', '', 1) try: r = re.compile(regex) except Exception: _log.critical("Invalid regexp: %s", regex) sys.exit(1) objlist = [k for (k, v) in KVObject.backend.driver.ls(cur_dir)] retval = [objname for objname in objlist if r.match(objname)] warn = (len(objlist) <= 2 * len(retval)) if warn and self._action[0:3] in ['set', 'del']: ToolCli.raise_warning() return retval
def _tagged_host_list(self): cur_dir = self.entity.dir(*self.tags) warn = False if self._namedef == "all": all_objects = KVObject.backend.driver.ls(cur_dir) objlist = [k for (k, v) in all_objects] if self._action == "get": if self.args.yaml: print( yaml.dump(dict(all_objects), default_flow_style=False)) else: print(json.dumps(dict(all_objects))) return [] else: retval = objlist warn = True elif not self._namedef.startswith('re:'): return [self._namedef] else: regex = self._namedef.replace('re:', '', 1) try: r = re.compile(regex) except Exception: _log.critical("Invalid regexp: %s", regex) sys.exit(1) objlist = [k for (k, v) in KVObject.backend.driver.ls(cur_dir)] retval = [objname for objname in objlist if r.match(objname)] warn = (len(objlist) <= 2 * len(retval)) if warn and self._action[0:3] in ['set', 'del']: ToolCli.raise_warning() return retval
def main(cmdline=None): if cmdline is None: cmdline = list(sys.argv) cmdline.pop(0) cmdline = mangle_argv(cmdline) args = parse_args(cmdline) if args.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.WARN) if args.mode == 'select': cli = ToolCliByLabel(args) elif args.mode == 'find': cli = ToolCliFind(args) else: cli = ToolCli(args) try: cli.setup() except Exception as e: _log.critical("Invalid configuration: %s", e) sys.exit(1) exit_status = 0 for unit in args.action: # TODO: fix base class if not cli.run_action(unit): exit_status = 1 sys.exit(exit_status)
def main(arguments=None): if arguments is None: arguments = sys.argv[1:] args = get_args(arguments) if args.debug: log_level = logging.DEBUG else: log_level = logging.INFO logging.basicConfig( level=log_level, format= '%(asctime)s [%(levelname)s] %(name)s::%(funcName)s: %(message)s', datefmt='%F %T') try: c = configuration.get(args.config) KVObject.setup(c) except Exception as e: _log.critical("Invalid configuration: %s", e) sys.exit(1) if not os.path.isdir(args.directory): _log.critical("Could not find directory %s", args.directory) sys.exit(2) sync = Syncer(args.schema, args.directory) sync.load()
def _catch(*args, **kwdargs): try: return fn(*args, **kwdargs) except BackendError as e: _log.error("%s Backend %s: %s", fn.__name__, log_msg, e) except Exception as e: _log.critical("%s generic %s: %s", fn.__name__, log_msg, e) raise
def _load_schema(self): self._schema = loader.Schema.from_file(self.args.schema) try: self.entity = self._schema.entities[self.args.object_type] except KeyError: _log.critical( "Object type %s is not available in the current schema", self.args.object_type) raise ObjectTypeError(self.args.object_type)
def _load_schema(self): self._schema = loader.Schema.from_file(self.args.schema) try: self.entity = self._schema.entities[self.args.object_type] except KeyError: _log.critical( "Object type %s is not available in the current schema", self.args.object_type ) raise ObjectTypeError(self.args.object_type)
def tags(self): try: return self.entity.parse_tags(self._tags) except KeyError as e: _log.critical("Invalid tag list %s - we're missing tag: %s", self.args.taglist, e) sys.exit(1) except ValueError: _log.critical("Invalid tag list %s", self.args.taglist) sys.exit(1)
def tags(self): try: return self.entity.parse_tags(self._tags) except KeyError as e: _log.critical( "Invalid tag list %s - we're missing tag: %s", self.args.taglist, e) sys.exit(1) except ValueError: _log.critical("Invalid tag list %s", self.args.taglist) sys.exit(1)
def lock(cls, path): try: l = cls.backend.driver.get_lock(path) yield l cls.backend.driver.release_lock(path) except Exception as e: _log.critical("Problems inside lock for %s: %s", path, e) cls.backend.driver.release_lock(path) raise except (SystemExit, KeyboardInterrupt) as e: _log.critical("Aborted.") cls.backend.driver.release_lock(path)
def __init__(self, config): self.config = config dir = os.path.dirname(__file__) driver_file = os.path.join( dir, "drivers/{}.py".format(self.config.driver)) ctx = {} try: exec(compile(open(driver_file).read(), driver_file, 'exec'), ctx) cls = ctx['Driver'] self.driver = cls(config) except Exception as e: _log.critical("Could not load driver %s: %s", self.config.driver, e, exc_info=True) sys.exit(3)
def __init__(self, config): self.config = config curr_dir = os.path.dirname(__file__) driver_file = os.path.join(curr_dir, "drivers/{}.py".format(self.config.driver)) ctx = {} try: exec(compile(open(driver_file).read(), driver_file, 'exec'), ctx) cls = ctx['Driver'] self.driver = cls(config) except Exception as e: _log.critical("Could not load driver %s: %s", self.config.driver, e, exc_info=True) sys.exit(3)
def main(cmdline=None): if cmdline is None: cmdline = sys.argv[1:] cmdline = mangle_argv(cmdline) args = parse_args(cmdline) if args.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.WARN) try: if args.mode == 'select': cli = ToolCliByLabel(args) elif args.mode == 'tags': cli = ToolCli(args) elif args.mode in ToolCliSimpleAction.simple_actions.keys(): cli = ToolCliSimpleAction(args) else: raise ValueError(args.mode) except ObjectTypeError: sys.exit(1) try: cli.setup() except Exception as e: _log.critical("Invalid configuration: %s", e) sys.exit(1) exit_status = 0 for unit in args.action: # TODO: fix base class if not cli.run_action(unit): exit_status = 1 sys.exit(exit_status)
def main(arguments=None): if arguments is None: arguments = list(sys.argv) arguments.pop(0) args = get_args(arguments) if args.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.WARN) try: c = configuration.get(args.config) KVObject.setup(c) except Exception as e: raise _log.critical("Invalid configuration: %s", e) sys.exit(1) files = tag_files(args.directory) # Load services data. servdata = {} if files['services']: for service_file in files['services']: with open(service_file, 'rb') as fh: try: d = yaml.load(fh) except: d = {} servdata.update(d) if not servdata: _log.critical("We found no services, so we can't import" " nodes either. Bailing out") sys.exit(1) # Refresh services: rem = {} for cluster, data in servdata.items(): if not type(data) == dict: continue load, rem[cluster] = get_service_actions(cluster, data) if args.lock: servlocker = service.Service.lock else: servlocker = dummy_lock load_services(cluster, load, data, servlocker) # sync nodes for filename in files['nodes']: dc = os.path.basename(filename).rstrip('.yaml') try: with open(filename, 'rb') as fh: dc_data = yaml.load(fh) except: _log.error("Malformed yaml data in %s", filename) _log.error("Skipping loading/removing nodes, please correct!") else: if args.lock: locker = service.Service.lock else: locker = dummy_lock load_nodes(dc, dc_data, locker) # Now delete services for cluster, servnames in rem.items(): remove_services(cluster, servnames, servlocker)