def load_files(self, rootdir):
entity_path = os.path.join(rootdir, self.entity)
_log.info("Loading data for entity %s from %s", self.entity, rootdir)
if not os.path.isdir(entity_path):
_log.error(
"Data dir %s does not exist, will NOT remove missing entities",
entity_path
)
self.skip_removal = True
for filename in glob.glob(os.path.join(entity_path, '*.yaml')):
_log.info("Parsing file %s", filename)
filedata = yaml_safe_load(filename, default={})
if not filedata:
_log.error(
"The file %s returned empty content, will NOT remove missing entities",
filename
)
self.skip_removal = True
try:
exp_data = self.cls.from_yaml(filedata)
self.data.update(exp_data)
except Exception:
_log.critical(
"Data in file %s could not be loaded",
self.data, exc_info=True)
self.skip_removal = True
def load(self):
"""
Load all the entities from file and sync
"""
if self.schema.has_errors:
raise ValueError("Schema is broken, NOT loading data.")
syncers = {}
# load the files and Create the load order
for name, cls in self.schema.entities.items():
cls = self.schema.entities[name]
syncers[name] = EntitySyncer(name, cls)
syncers[name].load_files(self.base_path)
self.add(name, cls, [])
for name in self.load_order:
_log.info("Adding objects for %s", name)
sync = syncers[name]
try:
sync.load()
except Exception as e:
_log.error("Loading of data for entity %s failed: %s", name, e)
sync.skip_removal = True
# Now let's cleanup in reverse order
self.load_order.reverse()
for name in self.load_order:
_log.info("Removing stale objects for %s", name)
syncers[name].cleanup()
def load_files(self, rootdir):
entity_path = os.path.join(rootdir, self.entity)
_log.info("Loading data for entity %s from %s", self.entity, rootdir)
if not os.path.isdir(entity_path):
_log.error(
"Data dir %s does not exist, will NOT remove missing entities",
entity_path)
self.skip_removal = True
for filename in glob.glob(os.path.join(entity_path, '*.yaml')):
_log.info("Parsing file %s", filename)
filedata = yaml_safe_load(filename, default={})
if not filedata:
_log.error(
"The file %s returned empty content, will NOT remove missing entities",
filename)
self.skip_removal = True
try:
exp_data = self.cls.from_yaml(filedata)
self.data.update(exp_data)
except Exception:
_log.critical("Data in file %s could not be loaded",
self.data,
exc_info=True)
self.skip_removal = True
def load(self):
"""
Load all the entities from file and sync
"""
if self.schema.has_errors:
raise ValueError("Schema is broken, NOT loading data.")
syncers = {}
# load the files and Create the load order
for name, cls in self.schema.entities.items():
cls = self.schema.entities[name]
syncers[name] = EntitySyncer(name, cls)
syncers[name].load_files(self.base_path)
self.add(name, cls, [])
for name in self.load_order:
_log.info("Adding objects for %s", name)
sync = syncers[name]
try:
sync.load()
except Exception as e:
_log.error("Loading of data for entity %s failed: %s", name, e)
sync.skip_removal = True
# Now let's cleanup in reverse order
self.load_order.reverse()
for name in self.load_order:
_log.info("Removing stale objects for %s", name)
syncers[name].cleanup()
def _catch(*args, **kwdargs):
try:
return fn(*args, **kwdargs)
except BackendError as e:
_log.error("%s Backend %s: %s", fn.__name__, log_msg, e)
except Exception as e:
_log.critical("%s generic %s: %s", fn.__name__, log_msg, e)
raise
def __init__(self, args):
if args.object_type != 'node':
_log.error('%s can only act on node objects', args.mode)
sys.exit(1)
args.selector = 'name={}'.format(args.hostname)
if 'service' in args and args.service is not None:
args.selector += ',service={}'.format(args.service)
args.action = [self.simple_actions[args.mode]]
args.mode = 'select'
super().__init__(args)
def _catch(*args, **kwdargs):
try:
return fn(*args, **kwdargs)
except BackendError as e:
_log.error("%s Backend %s: %s", fn.__name__,
log_msg, e)
except Exception as e:
_log.critical("%s generic %s: %s", fn.__name__,
log_msg, e)
raise
def __init__(self, args):
if args.object_type != 'node':
_log.error('%s can only act on node objects', args.mode)
sys.exit(1)
args.selector = 'name={}'.format(args.hostname)
if 'service' in args and args.service is not None:
args.selector += ',service={}'.format(args.service)
args.action = [self.simple_actions[args.mode]]
args.mode = 'select'
super(ToolCliSimpleAction, self).__init__(args)
def fetch(self):
self.exists = False
try:
values = self.backend.driver.read(self.key)
if values:
self.exists = True
self.from_net(values)
except drivers.NotFoundError:
self.from_net(None)
except drivers.BackendError as e:
_log.error("Backend error while fetching %s: %s", self.key, e)
def fetch(self):
self.exists = False
try:
values = self.backend.driver.read(self.key)
if values:
self.exists = True
self.from_net(values)
except drivers.NotFoundError:
self.from_net(None)
except drivers.BackendError as e:
_log.error("Backend error while fetching %s: %s", self.key, e)
def fetch(self):
self.exists = False
try:
values = self.backend.driver.read(self.key)
if values:
self.exists = True
except drivers.NotFoundError:
return self._from_net(None)
except drivers.BackendError as e:
_log.error("Backend error while fetching %s: %s", self.key, e)
# TODO: maybe catch the backend errors separately
return None
self._from_net(values)
def get(configfile):
"""
Loads the config from file
"""
try:
with open(configfile, 'rb') as fh:
config = yaml.load(fh.read())
except Exception as e:
_log.error('Could not load file %s: %s',
configfile, e)
config = {}
return Config(**config)
def _run_action(self):
fail = False
for obj in self.host_list():
try:
a = action.get_action(obj, self._action)
msg = a.run()
except action.ActionError as e:
fail = True
_log.error("Invalid action, reason: %s", str(e))
except BackendError as e:
fail = True
_log.error("Error when trying to %s on %s", self._action,
self._namedef)
_log.error("Failure writing to the kvstore: %s", str(e))
except Exception as e:
fail = True
_log.error("Error when trying to %s on %s", self._action,
self._namedef)
_log.exception("Generic action failure: %s", str(e))
else:
print(msg)
if not fail:
self.announce()
return True
else:
return False
def _run_action(self):
fail = False
for obj in self.host_list():
try:
a = action.get_action(obj, self._action)
msg = a.run()
except action.ActionError as e:
fail = True
_log.error("Invalid action, reason: %s", str(e))
except BackendError as e:
fail = True
_log.error("Error when trying to %s on %s", self._action,
self._namedef)
_log.error("Failure writing to the kvstore: %s", str(e))
except Exception as e:
fail = True
_log.error("Error when trying to %s on %s", self._action,
self._namedef)
_log.exception("Generic action failure: %s", str(e))
else:
if sys.version_info[0] == 2: # Python 2
msg = msg.decode('utf-8')
print(msg)
if not fail:
self.announce()
return True
else:
return False
def from_file(cls, filename):
"""
Load a yaml file
"""
instance = cls()
if not os.path.isfile(filename):
return instance
data = yaml_safe_load(filename, default={})
if not data:
instance.has_errors = True
return instance
for objname, defs in data.items():
try:
_log.debug('Loading entity %s', objname)
entity_name = re.sub(r'\W', '_', objname.capitalize())
entity = factory(entity_name, defs)
instance.entities[objname] = entity
except Exception as e:
_log.error('Could not load entity %s: %s', objname,
e, exc_info=True)
instance.has_errors = True
return instance
def from_file(cls, filename):
"""
Load a yaml file
"""
instance = cls()
if not os.path.isfile(filename):
return instance
data = yaml_safe_load(filename, default={})
if not data:
instance.has_errors = True
return instance
for objname, defs in data.items():
try:
_log.debug('Loading entity %s', objname)
entity_name = re.sub(r'\W', '_', objname.capitalize())
entity = factory(entity_name, defs)
instance.entities[objname] = entity
except Exception as e:
_log.error('Could not load entity %s: %s', objname,
e, exc_info=True)
instance.has_errors = True
return instance
def main(arguments=None):
if arguments is None:
arguments = list(sys.argv)
arguments.pop(0)
args = get_args(arguments)
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARN)
try:
c = configuration.get(args.config)
KVObject.setup(c)
except Exception as e:
raise
_log.critical("Invalid configuration: %s", e)
sys.exit(1)
files = tag_files(args.directory)
# Load services data.
servdata = {}
if files['services']:
for service_file in files['services']:
with open(service_file, 'rb') as fh:
try:
d = yaml.load(fh)
except:
d = {}
servdata.update(d)
if not servdata:
_log.critical("We found no services, so we can't import"
" nodes either. Bailing out")
sys.exit(1)
# Refresh services:
rem = {}
for cluster, data in servdata.items():
if not type(data) == dict:
continue
load, rem[cluster] = get_service_actions(cluster, data)
if args.lock:
servlocker = service.Service.lock
else:
servlocker = dummy_lock
load_services(cluster, load, data, servlocker)
# sync nodes
for filename in files['nodes']:
dc = os.path.basename(filename).rstrip('.yaml')
try:
with open(filename, 'rb') as fh:
dc_data = yaml.load(fh)
except:
_log.error("Malformed yaml data in %s", filename)
_log.error("Skipping loading/removing nodes, please correct!")
else:
if args.lock:
locker = service.Service.lock
else:
locker = dummy_lock
load_nodes(dc, dc_data, locker)
# Now delete services
for cluster, servnames in rem.items():
remove_services(cluster, servnames, servlocker)
