def discover(self):
self.history = OrderedDict()
try:
order = self.device.clean['order']
except KeyError:
raise Exception("Key 'order' is missing for device "
"'{d}'".format(d=self.device.name))
# Insert the 'images' value into necessary clean sections
if self.device.clean['images']:
initialize_clean_sections(self.image_handler, order)
# self.device.clean['change_boot_variable'] = {'images': []}
all_data = {}
all_schema = {}
sections = []
common_data = {}
self.parameters['common_data'] = common_data
for section in order:
try:
data = self.device.clean[section] or {}
except KeyError:
# Cannot find section - raise exception
raise Exception("Cannot find '{section}' in the provided "
"sections even though it was provided in "
"the order list '{order}'".\
format(section=section, order=order))
# Load it up
# If source isnt provided then check if it is inside the clean json
if 'source' not in data:
# Check if that one exists in the json
task = _get_clean(section, clean_data, self.device)
else:
task = load_class(data, self.device)
# Verify if schema exists for this section
if hasattr(task, 'schema'):
# if the stage has schema defined then build the bigger schema
all_schema[task.__name__] = task.schema
all_data[task.__name__] = data
# unwrap to get original method, tmp fix need to handle in genie core infra
task = unwrap(task)
func = copy_func(task)
func.uid = task.__name__
func.parameters = ParameterDict()
func.parameters['device'] = self.device
func.parameters['common_data'] = common_data
func.source = Source(self, objcls=func.__class__)
for parameter, value in data.items():
func.parameters[parameter] = value
# Bind it and append to the section list
new_section = func.__get__(self, func.__testcls__)
self.history[new_section.uid] = new_section
# Add processor, add parameters to it if any
if self.device.clean.get('device_recovery'):
processor = partial(recovery_processor,
**self.device.clean.get('device_recovery'))
processors.add(new_section,
pre=[block_section],
post=[processor],
exception=[])
sections.append(new_section)
recovery_data = self.device.clean.get('device_recovery')
# if recovery info not provided, don't need to check schema
if recovery_data:
recovery_schema = recovery_processor.schema
all_schema['device_recovery'] = recovery_schema
all_data['device_recovery'] = recovery_data
try:
Schema(all_schema).validate(all_data)
except SchemaMissingKeyError as e:
# proto type
raise ValueError(
"Clean schema check failed. The following keys are missing from clean yaml file:\n\n{}"
.format(self._format_missing_key_msg(
e.missing_list))) from None
except SchemaTypeError as e:
raise TypeError(
"Clean schema check failed. Incorrect value type was provided for the "
"following key:\n\n{}\n\nExpected type {} but got type {}".
format(self._format_missing_key_msg([e.path]), str(e.type),
type(e.data))) from None
except SchemaUnsupportedKeyError as e:
raise ValueError(
"Clean schema check failed. The following keys are not supported:\n\n{}"
.format(self._format_missing_key_msg(
e.unsupported_keys))) from None
return sections
def discover(self):
""" Discovers and loads all clean stages defined within the clean
yaml file. During loading validate that each stage is a TestSection.
Once loading is complete a schema validation is executed to ensure
the data provided is valid.
"""
clean_schema = {}
clean_to_validate = {}
# order is mandatory so verify schema
clean_schema['order'] = list
if self.device.clean.get('order'):
clean_to_validate['order'] = self.device.clean['order']
# device_recovery is optional so verify schema only if provided
device_recovery_data = self.device.clean.get('device_recovery')
if device_recovery_data:
clean_schema['device_recovery'] = recovery_processor.schema
clean_to_validate['device_recovery'] = device_recovery_data
# Setup and save the device recovery processor for later
# use in __iter__()
self.device_recovery_processor = partial(
recovery_processor, **self.device.clean['device_recovery'])
clean_json = load_clean_json()
# Verify schema and load each stage
for stage in self.device.clean:
if stage in NOT_A_STAGE:
continue
if self.image_handler:
self.image_handler.update_section(stage)
stage_data = self.device.clean[stage] or {}
if 'source' not in stage_data:
# Attempt to load from clean json
stage_func = get_clean_function(stage, clean_json, self.device)
else:
# Attempt to load from the provided source
stage_func = load_class(stage_data, self.device)
if hasattr(stage_func, 'schema'):
clean_schema[stage_func.__name__] = stage_func.schema
# Add schema that is common to all stages
clean_schema[stage_func.__name__].update(
{Optional('source'): dict})
clean_to_validate[stage_func.__name__] = stage_data
# Save for use later in __iter__()
self.stages[stage] = {
'func':
stage_func,
'change_order_if_pass':
stage_data.pop('change_order_if_pass', None),
'change_order_if_fail':
stage_data.pop('change_order_if_fail', None),
'stage_reuse_limit':
stage_data.pop('stage_reuse_limit', None),
'args':
stage_data
}
try:
Schema(clean_schema).validate(clean_to_validate)
except Exception as e:
raise pretty_schema_exception(e)
def validate_clean(clean_file, testbed_file, lint=True):
""" Validates the clean yaml using device abstraction to collect
the proper schemas
Args:
clean_file (str/dict): clean datafile
testbed_file (str/dict): testbed datafile
lint (bool, optional): Do yaml linting on the clean_file
Returns:
{
'warnings' ['Warning example', ...],
'exceptions: [ValueError, ...]
}
"""
warnings = []
exceptions = []
validation_results = {'warnings': warnings, 'exceptions': exceptions}
if lint:
lint_messages = do_lint(clean_file)
for message in lint_messages:
# we want to use the str representation not the object
warnings.append(str(message))
# these sections are not true stages and therefore cant be loaded
sections_to_ignore = [
'images',
'order'
]
base_schema = {
Optional('clean_devices'): list,
'cleaners': {
Any(): {
'module': str,
Optional('devices'): list,
Optional('platforms'): list,
Optional('groups'): list,
Any(): Any()
}
},
'devices': {
}
}
try:
# Load yaml without parsing markup
# Mock the use validate to prevent calling functions like
# translate_host or import_from_name
with patch.object(PyatsUse, 'validate') as mockvalid:
# return data on Use.validate
mockvalid.side_effect = lambda *x, **y: x[1]
loaded_tb = testbed_loader(testbed_file,
locations={},
markupprocessor=TestbedMarkupProcessor(
reference=True,
callable=False,
env_var=False,
include_file=False,
ask=False,
encode=False))
except Exception:
exceptions.append(
Exception("Could not load the testbed file. Use "
"'pyats validate testbed <file>' to validate "
"the testbed file.")
)
loaded_tb = testbed_loader({})
loader = Loader(enable_extensions=True,
markupprocessor=MarkupProcessor(reference=True,
callable=False,
env_var=False,
include_file=False,
ask=False,
encode=False))
try:
clean_dict = loader.load(clean_file, locations={})
except Exception as e:
exceptions.append(e)
return validation_results
loader = Loader(enable_extensions=True,
markupprocessor=MarkupProcessor(reference=True,
callable=False,
env_var=False,
include_file=False,
ask=False,
encode=False))
try:
clean_dict = loader.load(clean_file, locations={})
except Exception as e:
exceptions.append(e)
return validation_results
try:
clean_json = load_clean_json()
except Exception as e:
exceptions.append(e)
from genie.libs.clean.recovery import recovery_processor
for dev in clean_dict.get('devices', {}):
schema = base_schema.setdefault('devices', {}).setdefault(dev, {})
schema.update({Optional('order'): list})
schema.update({Optional('device_recovery'): dict})
schema.update({Optional('images'): Or(list, dict)})
clean_data = clean_dict["devices"][dev]
try:
dev = loaded_tb.devices[dev]
except KeyError as e:
warnings.append(
"The device {dev} specified in the clean yaml does "
"not exist in the testbed.".format(dev=e))
# cant validate schema so allow anything under dev
schema.update({Any(): Any()})
continue
except Exception as e:
exceptions.append(e)
schema.update({Any(): Any()})
continue
# update stages with image
if clean_data.get('images'):
setattr(dev, 'clean', clean_data)
try:
# Get abstracted ImageHandler class
abstract = Lookup.from_device(dev, packages={'clean': clean})
ImageHandler = abstract.clean.stages.image_handler.ImageHandler
image_handler = ImageHandler(dev, dev.clean['images'])
initialize_clean_sections(image_handler, clean_data['order'])
except Exception as e:
# If the device does not have custom.abstraction defined
# then we cannot load the correct stages to test the
# correct schema. Skip this device.
exceptions.append(Exception(dev.name+': '+str(e)))
schema.update({Any(): Any()})
continue
for section in clean_data:
# ignore sections that aren't true stages
if section in sections_to_ignore:
continue
if section == 'device_recovery':
schema.update({'device_recovery': recovery_processor.schema})
continue
clean_data[section].pop('change_order_if_fail', None)
clean_data[section].pop('change_order_if_pass', None)
# when no data is provided under stage, change None to dict
# this is needed for schema validation
if clean_data[section] is None:
clean_data[section] = {}
# Load it up so we can grab the schema from the stage
# If source isnt provided then check if it is inside the clean json
try:
if 'source' not in clean_data:
task = get_clean_function(section, clean_json, dev)
else:
task = load_class(clean_data, dev)
except Exception as e:
# Stage cannot be found. Allow any schema to prevent schema error
# and skip this stage
exceptions.append(str(e))
schema.update({section: Any()})
continue
# Add the stage schema to the base schema
if hasattr(task, 'schema'):
schema.update({task.__name__: task.schema})
try:
Schema(base_schema).validate(clean_dict)
except Exception as e:
exceptions.append(pretty_schema_exception(e))
return validation_results
def validate_clean(clean_dict, testbed_dict):
""" Validates the clean yaml using device abstraction to collect
the proper schemas
Args:
clean_dict (dict): clean datafile
testbed_dict (dict): testbed datafile
Returns:
{
'warnings' ['Warning example', ...],
'exceptions: [ValueError, ...]
}
"""
warnings = []
exceptions = []
validation_results = {'warnings': warnings, 'exceptions': exceptions}
# these sections are not true stages and therefore cant be loaded
sections_to_ignore = [
'images',
'order'
]
base_schema = {
'cleaners': {
Any(): {
'module': str,
Optional('devices'): list,
Optional('platforms'): list,
Optional('groups'): list,
Any(): Any()
}
},
'devices': {
}
}
try:
loaded_tb = testbed_loader(testbed_dict)
except Exception:
exceptions.append(
Exception("Could not load the testbed file. Use "
"'pyats validate testbed <file>' to validate "
"the testbed file.")
)
clean_json = load_clean_json()
from genie.libs.clean.stages.recovery import recovery_processor
for dev in clean_dict.get('devices', {}):
schema = base_schema.setdefault('devices', {}).setdefault(dev, {})
schema.update({Optional('order'): list})
schema.update({Optional('device_recovery'): dict})
schema.update({Optional('images'): Or(list, dict)})
clean_data = clean_dict["devices"][dev]
try:
dev = loaded_tb.devices[dev]
except KeyError as e:
warnings.append(
"The device {dev} specified in the clean yaml does "
"not exist in the testbed.".format(dev=e))
# cant validate schema so allow anything under dev
schema.update({Any(): Any()})
continue
# update stages with image
if clean_data.get('images'):
setattr(dev, 'clean', clean_data)
# Get abstracted ImageHandler class
abstract = Lookup.from_device(dev, packages={'clean': clean})
ImageHandler = abstract.clean.stages.image_handler.ImageHandler
# Image handler
image_handler = ImageHandler(dev, dev.clean['images'])
initialize_clean_sections(image_handler, clean_data['order'])
for section in clean_data:
# ignore sections that aren't true stages
if section in sections_to_ignore:
continue
if section == 'device_recovery':
schema.update({'device_recovery': recovery_processor.schema})
continue
# when no data is provided under stage, change None to dict
# this is needed for schema validation
if clean_data[section] is None:
clean_data[section] = {}
# Load it up so we can grab the schema from the stage
# If source isnt provided then check if it is inside the clean json
if 'source' not in clean_data:
task = get_clean_function(section, clean_json, dev)
else:
task = load_class(clean_data, dev)
# Add the stage schema to the base schema
if hasattr(task, 'schema'):
schema.update({task.__name__: task.schema})
try:
Schema(base_schema).validate(clean_dict)
except Exception as e:
exceptions.append(pretty_schema_exception(e))
return validation_results