def load_files(self, rootdir):
entity_path = os.path.join(rootdir, self.entity)
_log.info("Loading data for entity %s from %s", self.entity, rootdir)
if not os.path.isdir(entity_path):
_log.error(
"Data dir %s does not exist, will NOT remove missing entities",
entity_path)
self.skip_removal = True
for filename in glob.glob(os.path.join(entity_path, '*.yaml')):
_log.info("Parsing file %s", filename)
filedata = yaml_safe_load(filename, default={})
if not filedata:
_log.error(
"The file %s returned empty content, will NOT remove missing entities",
filename)
self.skip_removal = True
try:
exp_data = self.cls.from_yaml(filedata)
self.data.update(exp_data)
except Exception:
_log.critical("Data in file %s could not be loaded",
self.data,
exc_info=True)
self.skip_removal = True
def load_files(self, rootdir):
entity_path = os.path.join(rootdir, self.entity)
_log.info("Loading data for entity %s from %s", self.entity, rootdir)
if not os.path.isdir(entity_path):
_log.error(
"Data dir %s does not exist, will NOT remove missing entities",
entity_path
)
self.skip_removal = True
for filename in glob.glob(os.path.join(entity_path, '*.yaml')):
_log.info("Parsing file %s", filename)
filedata = yaml_safe_load(filename, default={})
if not filedata:
_log.error(
"The file %s returned empty content, will NOT remove missing entities",
filename
)
self.skip_removal = True
try:
exp_data = self.cls.from_yaml(filedata)
self.data.update(exp_data)
except Exception:
_log.critical(
"Data in file %s could not be loaded",
self.data, exc_info=True)
self.skip_removal = True
def main(arguments=None):
if arguments is None:
arguments = sys.argv[1:]
args = get_args(arguments)
if args.debug:
log_level = logging.DEBUG
else:
log_level = logging.INFO
logging.basicConfig(
level=log_level,
format='%(asctime)s [%(levelname)s] %(name)s::%(funcName)s: %(message)s',
datefmt='%F %T'
)
try:
c = configuration.get(args.config)
KVObject.setup(c)
except Exception as e:
_log.critical("Invalid configuration: %s", e)
sys.exit(1)
if not os.path.isdir(args.directory):
_log.critical("Could not find directory %s", args.directory)
sys.exit(2)
sync = Syncer(args.schema, args.directory)
sync.load()
def _tagged_host_list(self):
cur_dir = self.entity.dir(*self.tags)
warn = False
if self._namedef == "all":
all_objects = KVObject.backend.driver.ls(cur_dir)
objlist = [k for (k, v) in all_objects]
if self._action == "get":
if self.args.yaml:
print(yaml.dump(dict(all_objects), default_flow_style=False))
else:
print(json.dumps(dict(all_objects)))
return []
else:
retval = objlist
warn = True
elif not self._namedef.startswith('re:'):
return [self._namedef]
else:
regex = self._namedef.replace('re:', '', 1)
try:
r = re.compile(regex)
except Exception:
_log.critical("Invalid regexp: %s", regex)
sys.exit(1)
objlist = [k for (k, v) in KVObject.backend.driver.ls(cur_dir)]
retval = [objname for objname in objlist if r.match(objname)]
warn = (len(objlist) <= 2 * len(retval))
if warn and self._action[0:3] in ['set', 'del']:
ToolCli.raise_warning()
return retval
def _tagged_host_list(self):
cur_dir = self.entity.dir(*self.tags)
warn = False
if self._namedef == "all":
all_objects = KVObject.backend.driver.ls(cur_dir)
objlist = [k for (k, v) in all_objects]
if self._action == "get":
if self.args.yaml:
print(
yaml.dump(dict(all_objects), default_flow_style=False))
else:
print(json.dumps(dict(all_objects)))
return []
else:
retval = objlist
warn = True
elif not self._namedef.startswith('re:'):
return [self._namedef]
else:
regex = self._namedef.replace('re:', '', 1)
try:
r = re.compile(regex)
except Exception:
_log.critical("Invalid regexp: %s", regex)
sys.exit(1)
objlist = [k for (k, v) in KVObject.backend.driver.ls(cur_dir)]
retval = [objname for objname in objlist if r.match(objname)]
warn = (len(objlist) <= 2 * len(retval))
if warn and self._action[0:3] in ['set', 'del']:
ToolCli.raise_warning()
return retval
def main(cmdline=None):
if cmdline is None:
cmdline = list(sys.argv)
cmdline.pop(0)
cmdline = mangle_argv(cmdline)
args = parse_args(cmdline)
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARN)
if args.mode == 'select':
cli = ToolCliByLabel(args)
elif args.mode == 'find':
cli = ToolCliFind(args)
else:
cli = ToolCli(args)
try:
cli.setup()
except Exception as e:
_log.critical("Invalid configuration: %s", e)
sys.exit(1)
exit_status = 0
for unit in args.action:
# TODO: fix base class
if not cli.run_action(unit):
exit_status = 1
sys.exit(exit_status)
def main(arguments=None):
if arguments is None:
arguments = sys.argv[1:]
args = get_args(arguments)
if args.debug:
log_level = logging.DEBUG
else:
log_level = logging.INFO
logging.basicConfig(
level=log_level,
format=
'%(asctime)s [%(levelname)s] %(name)s::%(funcName)s: %(message)s',
datefmt='%F %T')
try:
c = configuration.get(args.config)
KVObject.setup(c)
except Exception as e:
_log.critical("Invalid configuration: %s", e)
sys.exit(1)
if not os.path.isdir(args.directory):
_log.critical("Could not find directory %s", args.directory)
sys.exit(2)
sync = Syncer(args.schema, args.directory)
sync.load()
def _catch(*args, **kwdargs):
try:
return fn(*args, **kwdargs)
except BackendError as e:
_log.error("%s Backend %s: %s", fn.__name__, log_msg, e)
except Exception as e:
_log.critical("%s generic %s: %s", fn.__name__, log_msg, e)
raise
def _load_schema(self):
self._schema = loader.Schema.from_file(self.args.schema)
try:
self.entity = self._schema.entities[self.args.object_type]
except KeyError:
_log.critical(
"Object type %s is not available in the current schema",
self.args.object_type)
raise ObjectTypeError(self.args.object_type)
def _load_schema(self):
self._schema = loader.Schema.from_file(self.args.schema)
try:
self.entity = self._schema.entities[self.args.object_type]
except KeyError:
_log.critical(
"Object type %s is not available in the current schema",
self.args.object_type
)
raise ObjectTypeError(self.args.object_type)
def tags(self):
try:
return self.entity.parse_tags(self._tags)
except KeyError as e:
_log.critical("Invalid tag list %s - we're missing tag: %s",
self.args.taglist, e)
sys.exit(1)
except ValueError:
_log.critical("Invalid tag list %s", self.args.taglist)
sys.exit(1)
def _catch(*args, **kwdargs):
try:
return fn(*args, **kwdargs)
except BackendError as e:
_log.error("%s Backend %s: %s", fn.__name__,
log_msg, e)
except Exception as e:
_log.critical("%s generic %s: %s", fn.__name__,
log_msg, e)
raise
def tags(self):
try:
return self.entity.parse_tags(self._tags)
except KeyError as e:
_log.critical(
"Invalid tag list %s - we're missing tag: %s",
self.args.taglist, e)
sys.exit(1)
except ValueError:
_log.critical("Invalid tag list %s", self.args.taglist)
sys.exit(1)
def lock(cls, path):
try:
l = cls.backend.driver.get_lock(path)
yield l
cls.backend.driver.release_lock(path)
except Exception as e:
_log.critical("Problems inside lock for %s: %s", path, e)
cls.backend.driver.release_lock(path)
raise
except (SystemExit, KeyboardInterrupt) as e:
_log.critical("Aborted.")
cls.backend.driver.release_lock(path)
def __init__(self, config):
self.config = config
dir = os.path.dirname(__file__)
driver_file = os.path.join(
dir, "drivers/{}.py".format(self.config.driver))
ctx = {}
try:
exec(compile(open(driver_file).read(), driver_file, 'exec'), ctx)
cls = ctx['Driver']
self.driver = cls(config)
except Exception as e:
_log.critical("Could not load driver %s: %s",
self.config.driver, e, exc_info=True)
sys.exit(3)
def __init__(self, config):
self.config = config
curr_dir = os.path.dirname(__file__)
driver_file = os.path.join(curr_dir,
"drivers/{}.py".format(self.config.driver))
ctx = {}
try:
exec(compile(open(driver_file).read(), driver_file, 'exec'), ctx)
cls = ctx['Driver']
self.driver = cls(config)
except Exception as e:
_log.critical("Could not load driver %s: %s",
self.config.driver,
e,
exc_info=True)
sys.exit(3)
def main(cmdline=None):
if cmdline is None:
cmdline = sys.argv[1:]
cmdline = mangle_argv(cmdline)
args = parse_args(cmdline)
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARN)
try:
if args.mode == 'select':
cli = ToolCliByLabel(args)
elif args.mode == 'tags':
cli = ToolCli(args)
elif args.mode in ToolCliSimpleAction.simple_actions.keys():
cli = ToolCliSimpleAction(args)
else:
raise ValueError(args.mode)
except ObjectTypeError:
sys.exit(1)
try:
cli.setup()
except Exception as e:
_log.critical("Invalid configuration: %s", e)
sys.exit(1)
exit_status = 0
for unit in args.action:
# TODO: fix base class
if not cli.run_action(unit):
exit_status = 1
sys.exit(exit_status)
def main(cmdline=None):
if cmdline is None:
cmdline = sys.argv[1:]
cmdline = mangle_argv(cmdline)
args = parse_args(cmdline)
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARN)
try:
if args.mode == 'select':
cli = ToolCliByLabel(args)
elif args.mode == 'tags':
cli = ToolCli(args)
elif args.mode in ToolCliSimpleAction.simple_actions.keys():
cli = ToolCliSimpleAction(args)
else:
raise ValueError(args.mode)
except ObjectTypeError:
sys.exit(1)
try:
cli.setup()
except Exception as e:
_log.critical("Invalid configuration: %s", e)
sys.exit(1)
exit_status = 0
for unit in args.action:
# TODO: fix base class
if not cli.run_action(unit):
exit_status = 1
sys.exit(exit_status)
def main(arguments=None):
if arguments is None:
arguments = list(sys.argv)
arguments.pop(0)
args = get_args(arguments)
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARN)
try:
c = configuration.get(args.config)
KVObject.setup(c)
except Exception as e:
raise
_log.critical("Invalid configuration: %s", e)
sys.exit(1)
files = tag_files(args.directory)
# Load services data.
servdata = {}
if files['services']:
for service_file in files['services']:
with open(service_file, 'rb') as fh:
try:
d = yaml.load(fh)
except:
d = {}
servdata.update(d)
if not servdata:
_log.critical("We found no services, so we can't import"
" nodes either. Bailing out")
sys.exit(1)
# Refresh services:
rem = {}
for cluster, data in servdata.items():
if not type(data) == dict:
continue
load, rem[cluster] = get_service_actions(cluster, data)
if args.lock:
servlocker = service.Service.lock
else:
servlocker = dummy_lock
load_services(cluster, load, data, servlocker)
# sync nodes
for filename in files['nodes']:
dc = os.path.basename(filename).rstrip('.yaml')
try:
with open(filename, 'rb') as fh:
dc_data = yaml.load(fh)
except:
_log.error("Malformed yaml data in %s", filename)
_log.error("Skipping loading/removing nodes, please correct!")
else:
if args.lock:
locker = service.Service.lock
else:
locker = dummy_lock
load_nodes(dc, dc_data, locker)
# Now delete services
for cluster, servnames in rem.items():
remove_services(cluster, servnames, servlocker)
