def validate_definitions():
data_root = config.runtime.default.data_root
print '\nValidating definitions...'
for definition in all_files(os.path.join(data_root,
'definitions')):
print 'Validating %s...' % definition,
try:
load(definition, CONTENT_TYPE_YAML,
'validator')
print 'Ok!'
except Exception as exc: #pylint: disable=W0703
print '\nERROR: Failed to validate %s\n%s' % \
(definition, exc)
def validate_resources(raiseException=False):
data_root = config.runtime.default.data_root
print '\nValidating resources...'
for resource in all_files(os.path.join(data_root, 'resources')):
print 'Validating %s...' % resource,
try:
load(resource, CONTENT_TYPE_YAML, 'validator')
print 'Ok!'
except Exception as exc: #pylint: disable=W0703
print '\nERROR: Failed to validate %s\n%s' % \
(resource, exc)
if raiseException:
raise exc
def validate_nodes():
data_root = config.runtime.default.data_root
print '\nValidating nodes...'
for filename in [
x for x in all_files(os.path.join(data_root, 'nodes'))
if x.split('/')[-1] in ['definition', 'pattern']
]:
print 'Validating %s...' % filename,
try:
load(filename, CONTENT_TYPE_YAML, 'validator')
print 'Ok!'
except Exception as exc: #pylint: disable=W0703
print '\nERROR: Failed to validate %s\n%s' % \
(filename, exc)
def validate_resources(raiseException=False):
data_root = config.runtime.default.data_root
print '\nValidating resources...'
for resource in all_files(os.path.join(data_root,
'resources')):
print 'Validating %s...' % resource,
try:
load(resource, CONTENT_TYPE_YAML,
'validator')
print 'Ok!'
except Exception as exc: #pylint: disable=W0703
print '\nERROR: Failed to validate %s\n%s' % \
(resource, exc)
if raiseException:
raise exc
def clear_resources(debug):
start_logging(debug)
try:
validate_resources(raiseException=True)
except Exception: #pylint: disable=W0703
sys.exit('ERROR: Unable to clear resources because of validation error')
data_root = config.runtime.default.data_root
print '\nClearing resources...'
for resource in all_files(os.path.join(data_root,
'resources')):
print 'Clearing %s...' % resource,
try:
contents = load(resource, CONTENT_TYPE_YAML,
'clear_resource')
for key in contents:
contents[key] = 'None'
dump(contents, resource, CONTENT_TYPE_YAML,
'clear_resource')
print 'Ok!'
except Exception as exc: #pylint: disable=W0703
print '\nERROR: Failed to clear %s\n%s' % \
(resource, exc)
def validate_nodes():
data_root = config.runtime.default.data_root
print '\nValidating nodes...'
for filename in [x for x in all_files(os.path.join(data_root,
'nodes'))
if x.split('/')[-1] in ['definition',
'pattern']]:
print 'Validating %s...' % filename,
try:
load(filename, CONTENT_TYPE_YAML,
'validator')
print 'Ok!'
except Exception as exc: #pylint: disable=W0703
print '\nERROR: Failed to validate %s\n%s' % \
(filename, exc)
def test_stress(cls):
# stress test writing and loading the same file over and
# over again and make sure the file does not get corrupted
# in the process
index = 0
while index < 100:
index += 1
data = get_data()
serializers.dump(data, TMP_FILE, CONTENT_TYPE_JSON)
assert serializers.load(TMP_FILE, CONTENT_TYPE_JSON) == data
def test_stress(cls):
# stress test writing and loading the same file over and
# over again and make sure the file does not get corrupted
# in the process
index = 0
while index < 100:
index += 1
data = get_data()
serializers.dump(data, TMP_FILE,
CONTENT_TYPE_JSON)
assert serializers.load(TMP_FILE,
CONTENT_TYPE_JSON) == data
def load_file(filename, content_type, node_id):
''' Returns the contents of a file specified by filename.
The requred content_type argument is required and indicates the
text serialization format the contents are stored in.
If the serializer load function encounters errors, None is returned
'''
try:
return load(filename, content_type, node_id)
except SerializerError:
log.error('%s: failed to load file: %s' % (node_id, filename))
raise
def load_file(filename, content_type, node_id):
''' Returns the contents of a file specified by filename.
The requred content_type argument is required and indicates the
text serialization format the contents are stored in.
If the serializer load function encounters errors, None is returned
'''
try:
return load(filename, content_type, node_id)
except SerializerError:
log.error('%s: failed to load file: %s' % (node_id, filename))
raise
def validate_definitions():
data_root = config.runtime.default.data_root
print '\nValidating definitions...'
for definition in all_files(os.path.join(data_root,
'definitions')):
print 'Validating %s...' % definition,
try:
def_data = load(definition, CONTENT_TYPE_YAML,
'validator')
resources = re.findall(FUNC_RE,
str(def_data))
# Validating plugins
plugins = resource_plugins()
missing_plugins = set([x for (x, _) in resources
if x not in plugins])
if missing_plugins:
plugins_path = os.path.join(data_root,
'plugins')
print ''
for plugin in missing_plugins:
print 'ERROR: Plugin \'%s\' configured in \'%s\' ' \
'is missing from \'%s\'!' % \
(plugin, definition, plugins_path)
# Special validation for 'allocate' plugin
resources_path = os.path.join(data_root,
'resources')
resource_files = [x.split('/')[-1]
for x in resources_path]
missing_resources = [x for (y, x) in resources
if x not in resource_files and
y == 'allocate']
if missing_resources:
if not missing_plugins:
print ''
for res in missing_resources:
print 'ERROR: Resource file \'%s\' configured in \'%s\' ' \
'is missing from \'%s\'!' % \
(res, definition, resources_path)
else:
print 'Ok!'
except Exception as exc: #pylint: disable=W0703
print '\nERROR: Failed to validate %s\n%s' % \
(definition, exc)
def load(self, pool):
self.data = dict()
filename = os.path.join(self.file_path, pool)
contents = load(filename, CONTENT_TYPE_YAML, self.node_id,
lock=True)
if contents and isinstance(contents, dict):
for key, value in contents.iteritems():
self.data[key] = str(value) if value else None
else:
if not contents:
contents = 'empty pool'
msg = '%s: unable to load resource pool %s: %s' % \
(self.node_id, pool, contents)
log.error(msg)
raise ResourcePoolError(msg)
log.debug('%s: loaded resource pool \'%s\': %s' %
(self.node_id, pool, self.data))
def run_validator(filename=None):
try:
print 'Validating file \'%s\'\n' % filename
validator = NeighbordbValidator()
filename = filename or default_filename()
validator.validate(load(filename, CONTENT_TYPE_YAML,
'validator'))
print 'Valid Patterns (count: %d)' % len(validator.valid_patterns)
print '--------------------------'
for index, pattern in enumerate(sorted(validator.valid_patterns)):
print '[%d] %s' % (index, pattern[1])
print
print 'Failed Patterns (count: %d)' % len(validator.invalid_patterns)
print '---------------------------'
for index, pattern in enumerate(sorted(validator.invalid_patterns)):
print '[%d] %s' % (index, pattern[1])
print
except Exception as exc: #pylint: disable=W0703
log.exception(exc)
print 'An unexpected error occurred trying to run the validator'
def validate_neighbordb():
# Validating neighbordb
validator = NeighbordbValidator('N/A')
neighbordb = neighbordb_path()
print 'Validating neighbordb (\'%s\')...' % neighbordb
try:
validator.validate(load(neighbordb, CONTENT_TYPE_YAML, 'validator'))
total_patterns = len(validator.valid_patterns) + \
len(validator.invalid_patterns)
if validator.invalid_patterns:
print '\nERROR: Failed to validate neighbordb patterns'
print ' Invalid Patterns (count: %d/%d)' % \
(len(validator.invalid_patterns),
total_patterns)
print ' ---------------------------'
for index, pattern in enumerate(sorted(
validator.invalid_patterns)):
print ' [%d] %s' % (index, pattern[1])
else:
print 'Ok!'
except Exception as exc: #pylint: disable=W0703
print 'ERROR: Failed to validate neighbordb\n%s' % exc
def validate_neighbordb():
# Validating neighbordb
validator = NeighbordbValidator('N/A')
neighbordb = neighbordb_path()
print 'Validating neighbordb (\'%s\')...' % neighbordb
try:
validator.validate(load(neighbordb, CONTENT_TYPE_YAML,
'validator'))
total_patterns = len(validator.valid_patterns) + \
len(validator.invalid_patterns)
if validator.invalid_patterns:
print '\nERROR: Failed to validate neighbordb patterns'
print ' Invalid Patterns (count: %d/%d)' % \
(len(validator.invalid_patterns),
total_patterns)
print ' ---------------------------'
for index, pattern in enumerate(
sorted(validator.invalid_patterns)):
print ' [%d] %s' % (index, pattern[1])
else:
print 'Ok!'
except Exception as exc: #pylint: disable=W0703
print 'ERROR: Failed to validate neighbordb\n%s' % exc
def load(self, pool):
self.data = dict()
filename = os.path.join(self.file_path, pool)
contents = load(filename, CONTENT_TYPE_YAML, self.node_id)
for key, value in contents.items():
self.data[key] = str(value) if value is not None else None
def load(self, pool):
self.data = dict()
filename = os.path.join(self.file_path, pool)
contents = load(filename, CONTENT_TYPE_YAML, self.node_id)
for key, value in contents.items():
self.data[key] = str(value) if value is not None else None