def test_vlan_interface(self): # pylint: disable=too-many-locals
device_dict = DeviceDictionary.get('bbb-01')
chk = {
'hostname': 'bbb-01',
'parameters': {
'map': {'eth1': {'192.168.0.2': 7}, 'eth0': {'192.168.0.2': 5}},
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1',
'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0'
},
'mac_addr': {'eth1': '00:24:d7:9b:c0:8c', 'eth0': 'f0:de:f1:46:8c:21'},
'tags': {'eth1': ['1G'], 'eth0': ['1G', '10G']}}
}
self.assertEqual(chk, device_dict.to_dict())
submission = yaml.load(open(self.filename, 'r'))
self.assertIn('protocols', submission)
self.assertIn('lava-vland', submission['protocols'])
roles = [role for role, _ in submission['protocols']['lava-vland'].iteritems()]
params = submission['protocols']['lava-vland']
vlans = {}
for role in roles:
for name, tags in params[role].iteritems():
vlans[name] = tags
self.assertIn('vlan_one', vlans)
self.assertIn('vlan_two', vlans)
jobs = split_multinode_yaml(submission, 'abcdefghijkl')
job_roles = {}
for role in roles:
self.assertEqual(len(jobs[role]), 1)
job_roles[role] = jobs[role][0]
for role in roles:
self.assertIn('device_type', job_roles[role])
self.assertIn('protocols', job_roles[role])
self.assertIn('lava-vland', job_roles[role]['protocols'])
client_job = job_roles['client']
server_job = job_roles['server']
self.assertIn('vlan_one', client_job['protocols']['lava-vland'])
self.assertIn('10G', client_job['protocols']['lava-vland']['vlan_one']['tags'])
self.assertIn('vlan_two', server_job['protocols']['lava-vland'])
self.assertIn('1G', server_job['protocols']['lava-vland']['vlan_two']['tags'])
client_tags = client_job['protocols']['lava-vland']['vlan_one']
client_dict = DeviceDictionary.get('bbb-01').to_dict()
for interface, tags in client_dict['parameters']['tags'].iteritems():
if any(set(tags).intersection(client_tags)):
self.assertEqual(interface, 'eth0')
self.assertEqual(
client_dict['parameters']['map'][interface],
{'192.168.0.2': 5}
)
# find_device_for_job would have a call to match_vlan_interface(device, job.definition) added
bbb1 = Device.objects.get(hostname='bbb-01')
self.assertTrue(match_vlan_interface(bbb1, client_job))
cubie1 = Device.objects.get(hostname='ct-01')
self.assertTrue(match_vlan_interface(cubie1, server_job))
def handle(self, *args, **options):
"""
Accept options via lava-server manage which provides access
to the database.
"""
hostname = options['hostname']
if hostname is None:
self.stderr.write("Please specify a hostname")
sys.exit(2)
if options['import'] is not None:
data = parse_template(options['import'])
element = DeviceDictionary.get(hostname)
if element is None:
self.stdout.write("Adding new device dictionary for %s" %
hostname)
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = data
element.save()
self.stdout.write("Device dictionary updated for %s" % hostname)
elif options['export'] is not None or options['review'] is not None:
element = DeviceDictionary.get(hostname)
data = None
if element is None:
self.stderr.write(
"Unable to export - no dictionary found for '%s'" %
hostname)
sys.exit(2)
else:
data = devicedictionary_to_jinja2(
element.parameters, element.parameters['extends'])
if options['review'] is None:
self.stdout.write(data)
else:
string_loader = jinja2.DictLoader({'%s.yaml' % hostname: data})
type_loader = jinja2.FileSystemLoader(
[os.path.join(options['path'], 'device-types')])
env = jinja2.Environment(loader=jinja2.ChoiceLoader(
[string_loader, type_loader]),
trim_blocks=True)
template = env.get_template("%s.yaml" % hostname)
device_configuration = template.render()
# validate against the device schema
try:
validate_device(yaml.load(device_configuration))
except (yaml.YAMLError, SubmissionException) as exc:
self.stderr.write("Invalid template: %s" % exc)
self.stdout.write(device_configuration)
else:
self.stderr.write(
"Please specify one of --import, --export or --review")
sys.exit(1)
def handle(self, *args, **options):
"""
Accept options via lava-server manage which provides access
to the database.
"""
hostname = options['hostname']
if hostname is None:
self.stderr.write("Please specify a hostname")
sys.exit(2)
if options['import']:
data = parse_template(options['import'])
element = DeviceDictionary.get(hostname)
if element is None:
self.stdout.write("Adding new device dictionary for %s" %
hostname)
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = data
element.save()
self.stdout.write("Device dictionary updated for %s" % hostname)
elif options['export'] or options['review']:
element = DeviceDictionary.get(hostname)
if element is None:
self.stderr.write(
"Unable to export - no dictionary found for '%s'" %
hostname)
sys.exit(2)
else:
data = devicedictionary_to_jinja2(
element.parameters, element.parameters['extends'])
if not options['review']:
self.stdout.write(data)
else:
template = prepare_jinja_template(hostname,
data,
system_path=False,
path=options['path'])
device_configuration = template.render()
# validate against the device schema
try:
validate_device(yaml.load(device_configuration))
except (yaml.YAMLError, SubmissionException) as exc:
self.stderr.write("Invalid template: %s" % exc)
self.stdout.write(device_configuration)
else:
self.stderr.write(
"Please specify one of --import, --export or --review")
sys.exit(1)
def handle(self, *args, **options):
"""
Accept options via lava-server manage which provides access
to the database.
"""
hostname = options['hostname']
if hostname is None:
self.stderr.write("Please specify a hostname")
sys.exit(2)
if options['import'] is not None:
data = parse_template(options['import'])
element = DeviceDictionary.get(hostname)
if element is None:
self.stdout.write("Adding new device dictionary for %s" %
hostname)
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = data
element.save()
self.stdout.write("Device dictionary updated for %s" % hostname)
elif options['export'] is not None or options['review'] is not None:
element = DeviceDictionary.get(hostname)
data = None
if element is None:
self.stderr.write("Unable to export - no dictionary found for '%s'" %
hostname)
sys.exit(2)
else:
data = devicedictionary_to_jinja2(
element.parameters,
element.parameters['extends']
)
if options['review'] is None:
self.stdout.write(data)
else:
template = prepare_jinja_template(hostname, data, system_path=False, path=options['path'])
device_configuration = template.render()
# validate against the device schema
try:
validate_device(yaml.load(device_configuration))
except (yaml.YAMLError, SubmissionException) as exc:
self.stderr.write("Invalid template: %s" % exc)
self.stdout.write(device_configuration)
else:
self.stderr.write("Please specify one of --import, --export or --review")
sys.exit(1)
def test_dictionary_remove(self):
foo = DeviceDictionary(hostname='foo')
foo.parameters = {
'bootz': {
'kernel': '0x4700000',
'ramdisk': '0x4800000',
'dtb': '0x4300000'
},
}
foo.save()
baz = DeviceDictionary.get('foo')
self.assertEqual(baz.parameters, foo.parameters)
baz.delete()
self.assertIsInstance(baz, DeviceDictionary)
baz = DeviceDictionary.get('foo')
self.assertIsNone(baz)
def test_exclusivity(self):
device = Device.objects.get(hostname="fakeqemu1")
self.assertTrue(device.is_pipeline)
self.assertFalse(device.is_exclusive)
self.assertIsNotNone(DeviceDictionary.get(device.hostname))
device_dict = DeviceDictionary(hostname=device.hostname)
device_dict.save()
device_dict = DeviceDictionary.get(device.hostname)
self.assertTrue(device.is_pipeline)
self.assertFalse(device.is_exclusive)
update = device_dict.to_dict()
update.update({'exclusive': 'True'})
device_dict.parameters = update
device_dict.save()
self.assertTrue(device.is_pipeline)
self.assertTrue(device.is_exclusive)
def test_menu_context(self):
job_ctx = {
'menu_early_printk': '',
'menu_interrupt_prompt': 'Default boot will start in'
}
hostname = self.factory.make_fake_mustang_device()
device_dict = DeviceDictionary.get(hostname)
device_data = devicedictionary_to_jinja2(
device_dict.parameters,
device_dict.parameters['extends']
)
template = prepare_jinja_template(hostname, device_data, system_path=False, path=self.jinja_path)
config_str = template.render(**job_ctx)
self.assertIsNotNone(config_str)
config = yaml.load(config_str)
self.assertIsNotNone(config['actions']['boot']['methods']['uefi-menu']['nfs'])
menu_data = config['actions']['boot']['methods']['uefi-menu']
# assert that menu_interrupt_prompt replaces the default 'The default boot selection will start in'
self.assertEqual(
menu_data['parameters']['interrupt_prompt'],
job_ctx['menu_interrupt_prompt']
)
# assert that menu_early_printk replaces the default earlyprintk default
self.assertEqual(
[e for e in menu_data['nfs'] if 'enter' in e['select'] and 'new Entry' in e['select']['wait']][0]['select']['enter'],
'console=ttyS0,115200 debug root=/dev/nfs rw nfsroot={NFS_SERVER_IP}:{NFSROOTFS},tcp,hard,intr ip=dhcp'
)
def match_vlan_interface(device, job_def):
if not isinstance(job_def, dict):
raise RuntimeError("Invalid vlan interface data")
if 'protocols' not in job_def or 'lava-vland' not in job_def['protocols']:
return False
interfaces = []
logger = logging.getLogger('dispatcher-master')
device_dict = DeviceDictionary.get(device.hostname).to_dict()
if 'tags' not in device_dict['parameters']:
logger.error("%s has no tags in the device dictionary parameters",
device.hostname)
return False
for vlan_name in job_def['protocols']['lava-vland']:
tag_list = job_def['protocols']['lava-vland'][vlan_name]['tags']
for interface, tags in device_dict['parameters']['tags'].iteritems():
logger.info("Job requests %s for %s, device %s provides %s for %s",
tag_list, vlan_name, device.hostname, tags, interface)
# tags & job tags must equal job tags
# device therefore must support all job tags, not all job tags available on the device need to be specified
if set(tags) & set(tag_list) == set(
tag_list) and interface not in interfaces:
logger.info("Matched vlan %s to interface %s on %s", vlan_name,
interface, device)
interfaces.append(interface)
# matched, do not check any further interfaces of this device for this vlan
break
logger.info(
"Matched: %s" %
(len(interfaces) == len(job_def['protocols']['lava-vland'].keys())))
return len(interfaces) == len(job_def['protocols']['lava-vland'].keys())
def test_exclusivity(self):
device = Device.objects.get(hostname="fakeqemu1")
self.assertTrue(device.is_pipeline)
self.assertFalse(device.is_exclusive)
self.assertIsNotNone(DeviceDictionary.get(device.hostname))
device_dict = DeviceDictionary(hostname=device.hostname)
device_dict.save()
device_dict = DeviceDictionary.get(device.hostname)
self.assertTrue(device.is_pipeline)
self.assertFalse(device.is_exclusive)
update = device_dict.to_dict()
update.update({'exclusive': 'True'})
device_dict.parameters = update
device_dict.save()
self.assertTrue(device.is_pipeline)
self.assertTrue(device.is_exclusive)
def test_menu_context(self):
job_ctx = {
'menu_early_printk': '',
'menu_interrupt_prompt': 'Default boot will start in',
'base_ip_args': 'ip=dhcp'
}
hostname = self.factory.make_fake_mustang_device()
device_dict = DeviceDictionary.get(hostname)
device_data = devicedictionary_to_jinja2(
device_dict.parameters, device_dict.parameters['extends'])
template = prepare_jinja_template(hostname,
device_data,
system_path=False,
path=self.jinja_path)
config_str = template.render(**job_ctx)
self.assertIsNotNone(config_str)
config = yaml.load(config_str)
self.assertIsNotNone(
config['actions']['boot']['methods']['uefi-menu']['nfs'])
menu_data = config['actions']['boot']['methods']['uefi-menu']
# assert that menu_interrupt_prompt replaces the default 'The default boot selection will start in'
self.assertEqual(menu_data['parameters']['interrupt_prompt'],
job_ctx['menu_interrupt_prompt'])
# assert that menu_early_printk replaces the default earlyprintk default
self.assertEqual(
[
e for e in menu_data['nfs'] if 'enter' in e['select']
and 'new Entry' in e['select']['wait']
][0]['select']['enter'],
'console=ttyS0,115200 debug root=/dev/nfs rw nfsroot={NFS_SERVER_IP}:{NFSROOTFS},tcp,hard,intr ip=dhcp'
)
def test_dictionary_remove(self):
foo = DeviceDictionary(hostname='foo')
foo.parameters = {
'bootz': {
'kernel': '0x4700000',
'ramdisk': '0x4800000',
'dtb': '0x4300000'
},
}
foo.save()
baz = DeviceDictionary.get('foo')
self.assertEqual(baz.parameters, foo.parameters)
baz.delete()
self.assertIsInstance(baz, DeviceDictionary)
baz = DeviceDictionary.get('foo')
self.assertIsNone(baz)
def match_vlan_interface(device, job_def):
if not isinstance(job_def, dict):
raise RuntimeError("Invalid vlan interface data")
if 'protocols' not in job_def or 'lava-vland' not in job_def['protocols']:
return False
interfaces = []
logger = logging.getLogger('dispatcher-master')
device_dict = DeviceDictionary.get(device.hostname).to_dict()
if 'tags' not in device_dict['parameters']:
logger.error("%s has no tags in the device dictionary parameters", device.hostname)
return False
for vlan_name in job_def['protocols']['lava-vland']:
tag_list = job_def['protocols']['lava-vland'][vlan_name]['tags']
for interface, tags in device_dict['parameters']['tags'].iteritems():
logger.info(
"Job requests %s for %s, device %s provides %s for %s",
tag_list, vlan_name, device.hostname, tags, interface)
# tags & job tags must equal job tags
# device therefore must support all job tags, not all job tags available on the device need to be specified
if set(tags) & set(tag_list) == set(tag_list) and interface not in interfaces:
logger.info("Matched vlan %s to interface %s on %s", vlan_name, interface, device)
interfaces.append(interface)
# matched, do not check any further interfaces of this device for this vlan
break
logger.info("Matched: %s" % (len(interfaces) == len(job_def['protocols']['lava-vland'].keys())))
return len(interfaces) == len(job_def['protocols']['lava-vland'].keys())
def handle(self, *args, **options):
"""
Accept options via lava-server manage which provides access
to the database.
"""
hostname = options['hostname']
if hostname is None:
self.stderr.write("Please specify a hostname")
sys.exit(2)
if options['import'] is not None:
data = parse_template(options['import'])
element = DeviceDictionary.get(hostname)
if element is None:
self.stdout.write("Adding new device dictionary for %s" %
hostname)
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = data
element.save()
self.stdout.write("Device dictionary updated for %s" % hostname)
elif options['export'] is not None or options['review'] is not None:
element = DeviceDictionary.get(hostname)
data = None
if element is None:
self.stderr.write("Unable to export - no dictionary found for '%s'" %
hostname)
sys.exit(2)
else:
data = devicedictionary_to_jinja2(
element.parameters,
element.parameters['extends']
)
if options['review'] is None:
self.stdout.write(data)
else:
string_loader = jinja2.DictLoader({'%s.yaml' % hostname: data})
type_loader = jinja2.FileSystemLoader([
os.path.join(options['path'], 'device-types')])
env = jinja2.Environment(
loader=jinja2.ChoiceLoader([string_loader, type_loader]),
trim_blocks=True)
template = env.get_template("%s.yaml" % hostname)
device_configuration = template.render()
self.stdout.write(device_configuration)
else:
self.stderr.write("Please specify one of --import, --export or --review")
sys.exit(1)
def test_new_dictionary(self):
foo = JobPipeline.get('foo')
self.assertIsNone(foo)
foo = DeviceDictionary(hostname='foo')
foo.save()
self.assertEqual(foo.hostname, 'foo')
self.assertIsInstance(foo, DeviceDictionary)
foo = DeviceDictionary.get('foo')
self.assertIsNotNone(foo)
def test_new_dictionary(self):
foo = JobPipeline.get('foo')
self.assertIsNone(foo)
foo = DeviceDictionary(hostname='foo')
foo.save()
self.assertEqual(foo.hostname, 'foo')
self.assertIsInstance(foo, DeviceDictionary)
foo = DeviceDictionary.get('foo')
self.assertIsNotNone(foo)
def test_vland_jinja2(self):
"""
Test complex device dictionary values
The reference data can cross lines but cannot be indented as the pprint
object in utils uses indent=0, width=80 for YAML compatibility.
The strings read in from config files can have indenting spaces, these
are removed in the pprint.
"""
data = """{% extends 'vland.jinja2' %}
{% set interfaces = ['eth0', 'eth1'] %}
{% set sysfs = {'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0',
'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1'} %}
{% set mac_addr = {'eth0': 'f0:de:f1:46:8c:21', 'eth1': '00:24:d7:9b:c0:8c'} %}
{% set tags = {'eth0': ['1G', '10G'], 'eth1': ['1G']} %}
{% set map = {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} %}
"""
result = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1',
'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0'
},
'extends': 'vland.jinja2',
'mac_addr': {
'eth1': '00:24:d7:9b:c0:8c',
'eth0': 'f0:de:f1:46:8c:21'
},
'tags': {
'eth1': ['1G'],
'eth0': ['1G', '10G']
},
'map': {
'eth0': {
'192.168.0.2': 5
},
'eth1': {
'192.168.0.2': 7
}
}
}
dictionary = jinja2_to_devicedictionary(data_dict=data)
self.assertEqual(result, dictionary)
jinja2_str = devicedictionary_to_jinja2(data_dict=dictionary, extends='vland.jinja2')
# ordering within the dict can change but each line needs to still appear
for line in str(data).split('\n'):
self.assertIn(line, str(jinja2_str))
# create a DeviceDictionary for this test
vlan = DeviceDictionary(hostname='vlanned1')
vlan.parameters = dictionary
vlan.save()
del vlan
vlan = DeviceDictionary.get('vlanned1')
cmp_str = str(devicedictionary_to_jinja2(vlan.parameters, 'vland.jinja2'))
for line in str(data).split('\n'):
self.assertIn(line, cmp_str)
def test_vland_jinja2(self):
"""
Test complex device dictionary values
The reference data can cross lines but cannot be indented as the pprint
object in utils uses indent=0, width=80 for YAML compatibility.
The strings read in from config files can have indenting spaces, these
are removed in the pprint.
"""
data = """{% extends 'vland.jinja2' %}
{% set interfaces = ['eth0', 'eth1'] %}
{% set sysfs = {'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0',
'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1'} %}
{% set mac_addr = {'eth0': 'f0:de:f1:46:8c:21', 'eth1': '00:24:d7:9b:c0:8c'} %}
{% set tags = {'eth0': ['1G', '10G'], 'eth1': ['1G']} %}
{% set map = {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} %}
"""
result = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1',
'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0'
},
'extends': 'vland.jinja2',
'mac_addr': {
'eth1': '00:24:d7:9b:c0:8c',
'eth0': 'f0:de:f1:46:8c:21'
},
'tags': {
'eth1': ['1G'],
'eth0': ['1G', '10G']
},
'map': {
'eth0': {
'192.168.0.2': 5
},
'eth1': {
'192.168.0.2': 7
}
}
}
dictionary = jinja2_to_devicedictionary(data_dict=data)
self.assertEqual(result, dictionary)
jinja2_str = devicedictionary_to_jinja2(data_dict=dictionary, extends='vland.jinja2')
# ordering within the dict can change but each line needs to still appear
for line in str(data).split('\n'):
self.assertIn(line, str(jinja2_str))
# create a DeviceDictionary for this test
vlan = DeviceDictionary(hostname='vlanned1')
vlan.parameters = dictionary
vlan.save()
del vlan
vlan = DeviceDictionary.get('vlanned1')
cmp_str = str(devicedictionary_to_jinja2(vlan.parameters, 'vland.jinja2'))
for line in str(data).split('\n'):
self.assertIn(line, cmp_str)
def setUp(self):
super(VlanInterfaces, self).setUp()
# YAML, pipeline only
user = User.objects.create_user('test', '*****@*****.**', 'test')
user.user_permissions.add(
Permission.objects.get(codename='add_testjob'))
user.save()
bbb_type = self.factory.make_device_type('beaglebone-black')
bbb_1 = self.factory.make_device(hostname='bbb-01', device_type=bbb_type)
device_dict = DeviceDictionary.get(bbb_1.hostname)
self.assertIsNone(device_dict)
device_dict = DeviceDictionary(hostname=bbb_1.hostname)
device_dict.parameters = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"},
'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"},
'tags': {'eth0': ['1G', '10G'], 'eth1': ['1G']},
'map': {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}}
}
device_dict.save()
ct_type = self.factory.make_device_type('cubietruck')
cubie = self.factory.make_device(hostname='ct-01', device_type=ct_type)
device_dict = DeviceDictionary.get(cubie.hostname)
self.assertIsNone(device_dict)
device_dict = DeviceDictionary(hostname=cubie.hostname)
device_dict.parameters = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"},
'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"},
'tags': {'eth0': ['1G', '10G'], 'eth1': ['1G']},
'map': {'eth0': {'192.168.0.2': 4}, 'eth1': {'192.168.0.2': 6}}
}
device_dict.save()
self.filename = os.path.join(os.path.dirname(__file__), 'bbb-cubie-vlan-group.yaml')
def handle_set(self, options):
""" Set device properties """
hostname = options["hostname"]
try:
device = Device.objects.get(hostname=hostname)
except Device.DoesNotExist:
self.stderr.write("Unable to find device '%s'" % hostname)
sys.exit(1)
status = options["status"]
if status is not None:
device.status = self.device_status[status]
health = options["health"]
if health is not None:
device.health_status = self.health_status[health]
description = options["description"]
if description is not None:
device.description = description
worker_name = options["worker"]
if worker_name is not None:
try:
worker = Worker.objects.get(hostname=worker_name)
device.worker_host = worker
except Worker.DoesNotExist:
self.stderr.write("Unable to find worker '%s'" % worker_name)
sys.exit(1)
public = options["public"]
if public is not None:
device.is_public = public
dictionary = options["dictionary"]
if dictionary is not None:
data = jinja2_to_devicedictionary(dictionary.read())
if data is None:
self.stderr.write("Invalid device dictionary")
sys.exit(1)
element = DeviceDictionary.get(hostname)
if element is None:
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = data
element.save()
# Save the modifications
device.save()
def test_from_json_rejects_exclusive(self):
panda_type = self.factory.ensure_device_type(name='panda')
panda_board = self.factory.make_device(device_type=panda_type, hostname='panda01')
self.assertFalse(panda_board.is_exclusive)
job = TestJob.from_json_and_user(
self.factory.make_job_json(device_type='panda'),
self.factory.make_user())
self.assertEqual(panda_type, job.requested_device_type)
device_dict = DeviceDictionary.get(panda_board.hostname)
self.assertIsNone(device_dict)
device_dict = DeviceDictionary(hostname=panda_board.hostname)
device_dict.parameters = {'exclusive': 'True'}
device_dict.save()
self.assertTrue(panda_board.is_exclusive)
self.assertRaises(
DevicesUnavailableException, _check_exclusivity, [panda_board], pipeline=False
)
def export_device_dictionary(self, hostname):
"""
Name
----
`export_device_dictionary` (`device_hostname`)
Description
-----------
[superuser only]
Export the device dictionary key value store for a
pipeline device.
See also get_pipeline_device_config
Arguments
---------
`device_hostname`: string
Device hostname to update.
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
self._authenticate()
if not self.user.is_superuser:
raise xmlrpclib.Fault(
403, "User '%s' is not superuser." % self.user.username
)
try:
device = Device.objects.get(hostname=hostname)
except DeviceType.DoesNotExist:
raise xmlrpclib.Fault(
404, "Device '%s' was not found." % hostname
)
if not device.is_pipeline:
raise xmlrpclib.Fault(
400, "Device '%s' is not a pipeline device" % hostname
)
device_dict = DeviceDictionary.get(hostname)
if not device_dict:
raise xmlrpclib.Fault(
404, "Device '%s' does not have a device dictionary" % hostname
)
device_dict = device_dict.to_dict()
jinja_str = devicedictionary_to_jinja2(device_dict['parameters'], device_dict['parameters']['extends'])
return xmlrpclib.Binary(jinja_str.encode('UTF-8'))
def export_device_dictionary(self, hostname):
"""
Name
----
`export_device_dictionary` (`device_hostname`)
Description
-----------
[superuser only]
Export the device dictionary key value store for a
pipeline device.
See also get_pipeline_device_config
Arguments
---------
`device_hostname`: string
Device hostname to update.
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
self._authenticate()
if not self.user.is_superuser:
raise xmlrpclib.Fault(
403, "User '%s' is not superuser." % self.user.username
)
try:
device = Device.objects.get(hostname=hostname)
except DeviceType.DoesNotExist:
raise xmlrpclib.Fault(
404, "Device '%s' was not found." % hostname
)
if not device.is_pipeline:
raise xmlrpclib.Fault(
400, "Device '%s' is not a pipeline device" % hostname
)
device_dict = DeviceDictionary.get(hostname)
if not device_dict:
raise xmlrpclib.Fault(
404, "Device '%s' does not have a device dictionary" % hostname
)
device_dict = device_dict.to_dict()
jinja_str = devicedictionary_to_jinja2(device_dict['parameters'], device_dict['parameters']['extends'])
return xmlrpclib.Binary(jinja_str.encode('UTF-8'))
def test_from_json_rejects_exclusive(self):
panda_type = self.factory.ensure_device_type(name='panda')
panda_board = self.factory.make_device(device_type=panda_type,
hostname='panda01')
self.assertFalse(panda_board.is_exclusive)
job = TestJob.from_json_and_user(
self.factory.make_job_json(device_type='panda'),
self.factory.make_user())
self.assertEqual(panda_type, job.requested_device_type)
device_dict = DeviceDictionary.get(panda_board.hostname)
self.assertIsNone(device_dict)
device_dict = DeviceDictionary(hostname=panda_board.hostname)
device_dict.parameters = {'exclusive': 'True'}
device_dict.save()
self.assertTrue(panda_board.is_exclusive)
self.assertRaises(DevicesUnavailableException,
_check_exclusivity, [panda_board],
pipeline=False)
def get_pipeline_device_config(self, device_hostname):
"""
Name
----
`get_pipeline_device_config` (`device_hostname`)
Description
-----------
Get the pipeline device configuration for given device hostname.
Arguments
---------
`device_hostname`: string
Device hostname for which the configuration is required.
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
if not device_hostname:
raise xmlrpclib.Fault(
400, "Bad request: Device hostname was not "
"specified.")
element = DeviceDictionary.get(device_hostname)
if element is None:
raise xmlrpclib.Fault(404, "Specified device not found.")
data = devicedictionary_to_jinja2(element.parameters,
element.parameters['extends'])
string_loader = jinja2.DictLoader({'%s.yaml' % device_hostname: data})
type_loader = jinja2.FileSystemLoader(
[os.path.join(jinja_template_path(), 'device-types')])
env = jinja2.Environment(loader=jinja2.ChoiceLoader(
[string_loader, type_loader]),
trim_blocks=True)
template = env.get_template("%s.yaml" % device_hostname)
device_configuration = template.render()
# validate against the device schema
validate_device(device_configuration)
return xmlrpclib.Binary(device_configuration.encode('UTF-8'))
def match_vlan_interface(device, job_def):
if not isinstance(job_def, dict):
raise RuntimeError("Invalid vlan interface data")
if 'protocols' not in job_def or 'lava-vland' not in job_def['protocols']:
return False
interfaces = []
logger = logging.getLogger('lava_scheduler_app')
for vlan_name in job_def['protocols']['lava-vland']:
tag_list = job_def['protocols']['lava-vland'][vlan_name]['tags']
device_dict = DeviceDictionary.get(device.hostname).to_dict()
if 'tags' not in device_dict['parameters']:
return False
for interface, tags in device_dict['parameters']['tags'].iteritems():
if any(set(tags).intersection(tag_list)) and interface not in interfaces:
logger.debug("Matched vlan %s to interface %s on %s", vlan_name, interface, device)
interfaces.append(interface)
# matched, do not check any further interfaces of this device for this vlan
break
return len(interfaces) == len(job_def['protocols']['lava-vland'].keys())
def test_find_nonexclusive_device(self):
"""
test that exclusive devices are not assigned JSON jobs
"""
self.assertFalse(self.panda01.is_exclusive)
device_dict = DeviceDictionary.get(self.panda01.hostname)
self.assertIsNone(device_dict)
device_dict = DeviceDictionary(hostname=self.panda01.hostname)
device_dict.parameters = {'exclusive': 'True'}
device_dict.save()
self.assertTrue(self.panda01.is_exclusive)
self.assertRaises(DevicesUnavailableException,
self.submit_job,
target='panda01',
device_type='panda')
job = self.submit_job(device_type='panda')
devices = [self.panda02, self.panda01]
self.assertEqual(find_device_for_job(job, devices), self.panda02)
device_dict.delete()
self.assertFalse(self.panda01.is_exclusive)
def handle_details(self, hostname):
""" Print device details """
try:
device = Device.objects.get(hostname=hostname)
except Device.DoesNotExist:
self.stderr.write("Unable to find device '%s'" % hostname)
sys.exit(1)
self.stdout.write("hostname : %s" % hostname)
self.stdout.write("device_type: %s" % device.device_type.name)
self.stdout.write("status : %s" % device.get_status_display())
self.stdout.write("health : %s" % device.get_health_status_display())
self.stdout.write("health job : %s" % bool(device.get_health_check()))
self.stdout.write("description: %s" % device.description)
self.stdout.write("public : %s" % device.is_public)
self.stdout.write("pipeline : %s" % device.is_pipeline)
element = DeviceDictionary.get(hostname)
self.stdout.write("device-dict: %s" % bool(element))
self.stdout.write("worker : %s" % device.worker_host.hostname)
self.stdout.write("current_job: %s" % device.current_job)
def get_pipeline_device_config(self, device_hostname):
"""
Name
----
`get_pipeline_device_config` (`device_hostname`)
Description
-----------
Get the pipeline device configuration for given device hostname.
Arguments
---------
`device_hostname`: string
Device hostname for which the configuration is required.
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
if not device_hostname:
raise xmlrpclib.Fault(400, "Bad request: Device hostname was not "
"specified.")
element = DeviceDictionary.get(device_hostname)
if element is None:
raise xmlrpclib.Fault(404, "Specified device not found.")
data = devicedictionary_to_jinja2(element.parameters,
element.parameters['extends'])
string_loader = jinja2.DictLoader({'%s.yaml' % device_hostname: data})
type_loader = jinja2.FileSystemLoader(
[os.path.join(jinja_template_path(), 'device-types')])
env = jinja2.Environment(loader=jinja2.ChoiceLoader([string_loader,
type_loader]),
trim_blocks=True)
template = env.get_template("%s.yaml" % device_hostname)
device_configuration = template.render()
return xmlrpclib.Binary(device_configuration.encode('UTF-8'))
def test_find_nonexclusive_device(self):
"""
test that exclusive devices are not assigned JSON jobs
"""
self.assertFalse(self.panda01.is_exclusive)
device_dict = DeviceDictionary.get(self.panda01.hostname)
self.assertIsNone(device_dict)
device_dict = DeviceDictionary(hostname=self.panda01.hostname)
device_dict.parameters = {'exclusive': 'True'}
device_dict.save()
self.assertTrue(self.panda01.is_exclusive)
self.assertRaises(
DevicesUnavailableException,
self.submit_job,
target='panda01', device_type='panda')
job = self.submit_job(device_type='panda')
devices = [self.panda02, self.panda01]
self.assertEqual(
find_device_for_job(job, devices),
self.panda02
)
device_dict.delete()
self.assertFalse(self.panda01.is_exclusive)
def get_pipeline_device_config(self, device_hostname):
"""
Name
----
`get_pipeline_device_config` (`device_hostname`)
Description
-----------
Get the pipeline device configuration for given device hostname.
Arguments
---------
`device_hostname`: string
Device hostname for which the configuration is required.
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
if not device_hostname:
raise xmlrpclib.Fault(400, "Bad request: Device hostname was not "
"specified.")
element = DeviceDictionary.get(device_hostname)
if element is None:
raise xmlrpclib.Fault(404, "Specified device not found.")
data = devicedictionary_to_jinja2(element.parameters,
element.parameters['extends'])
template = prepare_jinja_template(device_hostname, data, system_path=True)
device_configuration = template.render()
# validate against the device schema
validate_device(yaml.load(device_configuration))
return xmlrpclib.Binary(device_configuration.encode('UTF-8'))
def test_dictionary_parameters(self):
foo = DeviceDictionary(hostname='foo')
foo.parameters = {
'bootz': {
'kernel': '0x4700000',
'ramdisk': '0x4800000',
'dtb': '0x4300000'
},
'media': {
'usb': {
'UUID-required': True,
'SanDisk_Ultra': {
'uuid': 'usb-SanDisk_Ultra_20060775320F43006019-0:0',
'device_id': 0
},
'sata': {
'UUID-required': False
}
}
}
}
foo.save()
bar = DeviceDictionary.get('foo')
self.assertEqual(bar.parameters, foo.parameters)
def test_dictionary_parameters(self):
foo = DeviceDictionary(hostname='foo')
foo.parameters = {
'bootz': {
'kernel': '0x4700000',
'ramdisk': '0x4800000',
'dtb': '0x4300000'
},
'media': {
'usb': {
'UUID-required': True,
'SanDisk_Ultra': {
'uuid': 'usb-SanDisk_Ultra_20060775320F43006019-0:0',
'device_id': 0
},
'sata': {
'UUID-required': False
}
}
}
}
foo.save()
bar = DeviceDictionary.get('foo')
self.assertEqual(bar.parameters, foo.parameters)
def get_pipeline_device_config(self, device_hostname):
"""
Name
----
`get_pipeline_device_config` (`device_hostname`)
Description
-----------
Get the pipeline device configuration for given device hostname.
Arguments
---------
`device_hostname`: string
Device hostname for which the configuration is required.
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
if not device_hostname:
raise xmlrpclib.Fault(400, "Bad request: Device hostname was not "
"specified.")
element = DeviceDictionary.get(device_hostname)
if element is None:
raise xmlrpclib.Fault(404, "Specified device not found.")
data = devicedictionary_to_jinja2(element.parameters,
element.parameters['extends'])
template = prepare_jinja_template(device_hostname, data, system_path=True)
device_configuration = template.render()
# validate against the device schema
validate_device(yaml.load(device_configuration))
return xmlrpclib.Binary(device_configuration.encode('UTF-8'))
def pipeline_network_map(self, switch=None):
"""
Name
----
pipeline_network_map(switch=None):
Description
-----------
Collate all the vland information from pipeline devices to create a complete map,
then return YAML data for all switches or a specified switch.
This function requires authentication with a username and token.
Arguments
---------
switch: string
hostname or IP of the switch to map. If None, data is returned for all switches.
Return value
------------
Returns a YAML file of the form:
switches:
'192.168.0.2':
- port: 5
device:
interface: eth0
sysfs: "/sys/devices/pci0000:00/0000:00:19.0/net/eth0"
mac: "f0:de:f1:46:8c:21"
hostname: bbb1
"""
self._authenticate()
# get all device dictionaries, build the entire map.
dictionaries = [
DeviceDictionary.get(device.hostname).to_dict()
for device in Device.objects.filter(is_pipeline=True)
]
network_map = {'switches': {}}
for device_dict in dictionaries:
params = device_dict['parameters']
hostname = device_dict['hostname']
if 'interfaces' not in params:
continue
for interface in params['interfaces']:
for map_switch, port in params['map'][interface].iteritems():
port_list = []
device = {
'interface': interface,
'mac': params['mac_addr'][interface],
'sysfs': params['sysfs'][interface],
'hostname': hostname
}
port_list.append({'port': port, 'device': device})
switch_port = network_map['switches'].setdefault(
map_switch, [])
# Any switch can only have one entry for one port
if port not in switch_port:
switch_port.extend(port_list)
if switch:
if switch in network_map['switches']:
return yaml.dump(network_map['switches'][switch])
else:
return xmlrpclib.Fault(
404,
"No switch '%s' was found in the network map of supported devices."
% switch)
return yaml.dump(network_map)
def import_device_dictionary(self, hostname, jinja_str):
"""
Name
----
`import_device_dictionary` (`device_hostname`, `jinja_string`)
Description
-----------
[superuser only]
Import or update the device dictionary key value store for a
pipeline device.
Arguments
---------
`device_hostname`: string
Device hostname to update.
`jinja_str`: string
Jinja2 settings to store in the DeviceDictionary
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
self._authenticate()
if not self.user.is_superuser:
raise xmlrpclib.Fault(
403,
"User '%s' is not superuser." % self.user.username
)
try:
Device.objects.get(hostname=hostname)
except DeviceType.DoesNotExist:
raise xmlrpclib.Fault(
404, "Device '%s' was not found." % hostname
)
try:
device_data = jinja2_to_devicedictionary(jinja_str)
except (ValueError, KeyError, TypeError):
raise xmlrpclib.Fault(
400, "Unable to parse specified jinja string"
)
if not device_data or 'extends' not in device_data:
raise xmlrpclib.Fault(
400, "Invalid device dictionary content - %s - not updating." % jinja_str
)
try:
template = prepare_jinja_template(hostname, jinja_str, system_path=True)
except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc:
raise xmlrpclib.Fault(
400, "Template error: %s" % exc
)
if not template:
raise xmlrpclib.Fault(400, "Empty template")
element = DeviceDictionary.get(hostname)
msg = ''
if element is None:
msg = "Adding new device dictionary for %s\n" % hostname
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = device_data
element.save()
msg += "Device dictionary updated for %s\n" % hostname
return msg
def validate_pipeline_devices(self, hostname=None):
"""
Name
----
`validate_pipeline_device` [`device_hostname`]
Description
-----------
Validate that the device dictionary and device-type template
together create a valid YAML file which matches the pipeline
device schema.
See also get_pipeline_device_config
Arguments
---------
`device_hostname`: string
Device hostname to validate.
If a device hostname is not specified, all pipeline devices
are checked.
Return value
------------
This function returns an XML-RPC structure of results with the
following fields.
`device_hostname`: {'Valid': null}
or
`device_hostname`: {'Invalid': message}
`
"""
if not hostname:
devices = Device.objects.filter(is_pipeline=True)
else:
devices = Device.objects.filter(is_pipeline=True, hostname=hostname)
if not devices and hostname:
raise xmlrpclib.Fault(
404, "No pipeline device found with hostname %s" % hostname
)
if not devices and not hostname:
raise xmlrpclib.Fault(
404, "No pipeline device found on this instance."
)
results = {}
for device in devices:
key = str(device.hostname)
element = DeviceDictionary.get(device.hostname)
if element is None:
results[key] = {'Invalid': "Missing device dictionary"}
continue
data = devicedictionary_to_jinja2(element.parameters,
element.parameters['extends'])
if data is None:
results[key] = {'Invalid': 'Unable to convert device dictionary into jinja2'}
continue
try:
template = prepare_jinja_template(device.hostname, data, system_path=True)
device_configuration = template.render()
except jinja2.TemplateError as exc:
results[key] = {'Invalid': exc}
continue
try:
# validate against the device schema
validate_device(yaml.load(device_configuration))
except SubmissionException as exc:
results[key] = {'Invalid': exc}
continue
results[key] = {'Valid': None}
return xmlrpclib.Binary(yaml.dump(results))
def validate_pipeline_devices(self, name=None):
"""
Name
----
`validate_pipeline_device` [`name`]
Description
-----------
Validate that the device dictionary and device-type template
together create a valid YAML file which matches the pipeline
device schema.
Retired devices are ignored.
See also get_pipeline_device_config
Arguments
---------
`name`: string
Can be device hostname or device type name.
If name is specified, method will search for either a matching device
hostname or matching device type name in which case it will only
validate that(those) device(s).
If not specified, this method will validate all non-retired devices
in the system.
Return value
------------
This function returns an XML-RPC structure of results with the
following fields.
`device_hostname`: {'Valid': null}
or
`device_hostname`: {'Invalid': message}
`
"""
if not name:
devices = Device.objects.filter(
Q(is_pipeline=True) & ~Q(status=Device.RETIRED))
else:
devices = Device.objects.filter(
Q(is_pipeline=True) & ~Q(status=Device.RETIRED) & Q(
device_type__name=name))
if not devices:
devices = Device.objects.filter(
Q(is_pipeline=True) & ~Q(status=Device.RETIRED) & Q(hostname=name))
if not devices and name:
raise xmlrpclib.Fault(
404,
"No devices found with hostname or device type name %s" % name
)
if not devices and not name:
raise xmlrpclib.Fault(
404, "No pipeline device found on this instance."
)
results = {}
for device in devices:
key = str(device.hostname)
element = DeviceDictionary.get(device.hostname)
if element is None:
results[key] = {'Invalid': "Missing device dictionary"}
continue
data = devicedictionary_to_jinja2(element.parameters,
element.parameters['extends'])
if data is None:
results[key] = {'Invalid': 'Unable to convert device dictionary into jinja2'}
continue
try:
template = prepare_jinja_template(device.hostname, data, system_path=True)
device_configuration = template.render()
except jinja2.TemplateError as exc:
results[key] = {'Invalid': exc}
continue
try:
# validate against the device schema
validate_device(yaml.load(device_configuration))
except SubmissionException as exc:
results[key] = {'Invalid': exc}
continue
results[key] = {'Valid': None}
return xmlrpclib.Binary(yaml.dump(results))
def import_device_dictionary(self, hostname, jinja_str):
"""
Name
----
`import_device_dictionary` (`device_hostname`, `jinja_string`)
Description
-----------
[superuser only]
Import or update the device dictionary key value store for a
pipeline device.
This action will be logged.
Arguments
---------
`device_hostname`: string
Device hostname to update.
`jinja_str`: string
Jinja2 settings to store in the DeviceDictionary
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
self._authenticate()
if not self.user.is_superuser:
raise xmlrpclib.Fault(
403,
"User '%s' is not superuser." % self.user.username
)
try:
device = Device.objects.get(hostname=hostname)
except DeviceType.DoesNotExist:
raise xmlrpclib.Fault(
404, "Device '%s' was not found." % hostname
)
try:
device_data = jinja2_to_devicedictionary(jinja_str)
except (ValueError, KeyError, TypeError):
raise xmlrpclib.Fault(
400, "Unable to parse specified jinja string"
)
if not device_data or 'extends' not in device_data:
raise xmlrpclib.Fault(
400, "Invalid device dictionary content - %s - not updating." % jinja_str
)
try:
template = prepare_jinja_template(hostname, jinja_str, system_path=True)
except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc:
raise xmlrpclib.Fault(
400, "Template error: %s" % exc
)
if not template:
raise xmlrpclib.Fault(400, "Empty template")
element = DeviceDictionary.get(hostname)
msg = ''
if element is None:
msg = "Adding new device dictionary for %s\n" % hostname
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = device_data
element.save()
msg += "Device dictionary updated for %s\n" % hostname
device.log_admin_entry(self.user, msg)
return msg
def validate_pipeline_devices(self, name=None):
"""
Name
----
`validate_pipeline_device` [`name`]
Description
-----------
Validate that the device dictionary and device-type template
together create a valid YAML file which matches the pipeline
device schema.
Retired devices are ignored.
See also get_pipeline_device_config
Arguments
---------
`name`: string
Can be device hostname or device type name.
If name is specified, method will search for either a matching device
hostname or matching device type name in which case it will only
validate that(those) device(s).
If not specified, this method will validate all non-retired devices
in the system.
Return value
------------
This function returns an XML-RPC structure of results with the
following fields.
`device_hostname`: {'Valid': null}
or
`device_hostname`: {'Invalid': message}
`
"""
if not name:
devices = Device.objects.filter(
Q(is_pipeline=True) & ~Q(status=Device.RETIRED))
else:
devices = Device.objects.filter(
Q(is_pipeline=True) & ~Q(status=Device.RETIRED) & Q(
device_type__name=name))
if not devices:
devices = Device.objects.filter(
Q(is_pipeline=True) & ~Q(status=Device.RETIRED) & Q(hostname=name))
if not devices and name:
raise xmlrpclib.Fault(
404,
"No devices found with hostname or device type name %s" % name
)
if not devices and not name:
raise xmlrpclib.Fault(
404, "No pipeline device found on this instance."
)
results = {}
for device in devices:
key = str(device.hostname)
element = DeviceDictionary.get(device.hostname)
if element is None:
results[key] = {'Invalid': "Missing device dictionary"}
continue
data = devicedictionary_to_jinja2(element.parameters,
element.parameters['extends'])
if data is None:
results[key] = {'Invalid': 'Unable to convert device dictionary into jinja2'}
continue
try:
template = prepare_jinja_template(device.hostname, data, system_path=True)
device_configuration = template.render()
except jinja2.TemplateError as exc:
results[key] = {'Invalid': exc}
continue
try:
# validate against the device schema
validate_device(yaml.load(device_configuration))
except SubmissionException as exc:
results[key] = {'Invalid': exc}
continue
results[key] = {'Valid': None}
return xmlrpclib.Binary(yaml.dump(results))
def pipeline_network_map(self, switch=None):
"""
Name
----
pipeline_network_map(switch=None):
Description
-----------
Collate all the vland information from pipeline devices to create a complete map,
then return YAML data for all switches or a specified switch.
This function requires authentication with a username and token.
Arguments
---------
switch: string
hostname or IP of the switch to map. If None, data is returned for all switches.
Return value
------------
Returns a YAML file of the form:
switches:
'192.168.0.2':
- port: 5
device:
interface: eth0
sysfs: "/sys/devices/pci0000:00/0000:00:19.0/net/eth0"
mac: "f0:de:f1:46:8c:21"
hostname: bbb1
"""
self._authenticate()
# get all device dictionaries, build the entire map.
dictionaries = [
DeviceDictionary.get(device.hostname).to_dict() for device in Device.objects.filter(is_pipeline=True)
]
network_map = {'switches': {}}
for device_dict in dictionaries:
params = device_dict['parameters']
hostname = device_dict['hostname']
if 'interfaces' not in params:
continue
for interface in params['interfaces']:
for map_switch, port in params['map'][interface].iteritems():
port_list = []
device = {
'interface': interface,
'mac': params['mac_addr'][interface],
'sysfs': params['sysfs'][interface],
'hostname': hostname
}
port_list.append({'port': port, 'device': device})
switch_port = network_map['switches'].setdefault(map_switch, [])
# Any switch can only have one entry for one port
if port not in switch_port:
switch_port.extend(port_list)
if switch:
if switch in network_map['switches']:
return yaml.dump(network_map['switches'][switch])
else:
return xmlrpclib.Fault(
404, "No switch '%s' was found in the network map of supported devices." % switch)
return yaml.dump(network_map)
def handle(self, *_, **options):
health_dir = "/etc/lava-server/dispatcher-config/health-checks"
self.stdout.write("Moving health checks to %s:" % health_dir)
# Create the directory
try:
os.mkdir(health_dir, 0o755)
except OSError as exc:
if exc.errno != errno.EEXIST:
self.stderr.write("Unable to create the directory: %s" %
str(exc))
return
dt_skipped = []
for dt in DeviceType.objects.order_by('name'):
if not dt.health_check_job or not dt.display or is_device_type_retired(
dt):
dt_skipped.append((dt.name, False))
continue
# Check that the health-check is a v2 job
if is_deprecated_json(dt.health_check_job):
dt_skipped.append((dt.name, True))
continue
# Dump to the filesystem
self.stdout.write("* %s" % dt.name)
filename = os.path.join(health_dir, dt.name + '.yaml')
with open(filename, 'w') as f_out:
f_out.write(dt.health_check_job)
# Remove the health check from the data base (if needed)
if options["clean"]:
if is_device_type_exclusive(dt) or is_device_type_retired(
dt) or not dt.display:
dt.health_check_job = None
dt.save(update_fields=["health_check_job"])
else:
self.stderr.write(
"-> Not cleaning %s, some devices still support V1" %
dt.name)
self.stdout.write("Device types skipped:")
for (dt, has_health_check) in dt_skipped:
if has_health_check:
self.stdout.write("* %s (v1 health check)" % dt)
else:
self.stdout.write("* %s" % dt)
self.stdout.write("Checking devices:")
for device in Device.objects.exclude(
status=Device.RETIRED).order_by('hostname'):
device_dict = DeviceDictionary.get(device.hostname)
if not device_dict:
self.stderr.write("* %s => no device dictionary" %
device.hostname)
continue
device_dict = device_dict.to_dict()
extends = device_dict['parameters']['extends']
extends = os.path.splitext(extends)[0]
filename = os.path.join(
"/etc/lava-server/dispatcher-config/health-checks",
"%s.yaml" % extends)
if os.path.exists(filename):
self.stdout.write("* %s => %s.yaml" %
(device.hostname, extends))
else:
self.stderr.write("* %s => no health check found for %s.yaml" %
(device.hostname, extends))
def test_match_devices_with_map(self):
devices = Device.objects.filter(status=Device.IDLE).order_by('is_public')
device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname)
device_dict.parameters = { # client, RJ45 10M only
'extends': 'beaglebone-black.jinja2',
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"},
'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"},
'tags': {'eth0': [], 'eth1': ['RJ45', '10M']},
'map': {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}}
}
device_dict.save()
device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname)
device_dict.parameters = { # server includes 100M
'extends': 'cubietruck.jinja2',
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"},
'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"},
'tags': {'eth0': [], 'eth1': ['RJ45', '10M', '100M']},
'map': {'eth0': {'192.168.0.2': 4}, 'eth1': {'192.168.0.2': 6}}
}
device_dict.save()
user = self.factory.make_user()
sample_job_file = os.path.join(os.path.dirname(__file__), 'bbb-cubie-vlan-group.yaml')
with open(sample_job_file, 'r') as test_support:
data = yaml.load(test_support)
del(data['protocols']['lava-multinode']['roles']['client']['tags'])
del(data['protocols']['lava-multinode']['roles']['server']['tags'])
interfaces = []
job_dict = split_multinode_yaml(data, 'abcdefg123456789')
client_job = job_dict['client'][0]
device_dict = DeviceDictionary.get(self.factory.bbb1.hostname).to_dict()
self.assertIsNotNone(device_dict)
tag_list = client_job['protocols']['lava-vland']['vlan_one']['tags']
for interface, tags in device_dict['parameters']['tags'].iteritems():
if set(tags) & set(tag_list) == set(tag_list) and interface not in interfaces:
interfaces.append(interface)
break
self.assertEqual(['eth1'], interfaces)
self.assertEqual(len(interfaces), len(client_job['protocols']['lava-vland'].keys()))
interfaces = []
server_job = job_dict['server'][0]
device_dict = DeviceDictionary.get(self.factory.cubie1.hostname).to_dict()
self.assertIsNotNone(device_dict)
tag_list = server_job['protocols']['lava-vland']['vlan_two']['tags']
for interface, tags in device_dict['parameters']['tags'].iteritems():
if set(tags) & set(tag_list) == set(tag_list) and interface not in interfaces:
interfaces.append(interface)
break
self.assertEqual(['eth1'], interfaces)
self.assertEqual(len(interfaces), len(client_job['protocols']['lava-vland'].keys()))
vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user)
# vlan_one: client role. RJ45 10M. bbb device type
# vlan_two: server role. RJ45 100M. cubie device type.
assignments = {}
for job in vlan_job:
device = find_device_for_job(job, devices)
self.assertEqual(device.device_type, job.requested_device_type)
# map has been defined
self.assertTrue(match_vlan_interface(device, yaml.load(job.definition)))
assignments[job.device_role] = device
self.assertEqual(assignments['client'].hostname, self.factory.bbb1.hostname)
self.assertEqual(assignments['server'].hostname, self.factory.cubie1.hostname)
def test_match_devices_with_map(self):
devices = Device.objects.filter(
status=Device.IDLE).order_by('is_public')
device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname)
device_dict.parameters = { # client, RJ45 10M only
'extends': 'beaglebone-black.jinja2',
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0':
"/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1':
"/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"
},
'mac_addr': {
'eth0': "f0:de:f1:46:8c:21",
'eth1': "00:24:d7:9b:c0:8c"
},
'tags': {
'eth0': [],
'eth1': ['RJ45', '10M']
},
'map': {
'eth0': {
'192.168.0.2': 5
},
'eth1': {
'192.168.0.2': 7
}
}
}
device_dict.save()
device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname)
device_dict.parameters = { # server includes 100M
'extends': 'cubietruck.jinja2',
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0':
"/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1':
"/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"
},
'mac_addr': {
'eth0': "f0:de:f1:46:8c:21",
'eth1': "00:24:d7:9b:c0:8c"
},
'tags': {
'eth0': [],
'eth1': ['RJ45', '10M', '100M']
},
'map': {
'eth0': {
'192.168.0.2': 4
},
'eth1': {
'192.168.0.2': 6
}
}
}
device_dict.save()
user = self.factory.make_user()
sample_job_file = os.path.join(os.path.dirname(__file__),
'bbb-cubie-vlan-group.yaml')
with open(sample_job_file, 'r') as test_support:
data = yaml.load(test_support)
del (data['protocols']['lava-multinode']['roles']['client']['tags'])
del (data['protocols']['lava-multinode']['roles']['server']['tags'])
interfaces = []
job_dict = split_multinode_yaml(data, 'abcdefg123456789')
client_job = job_dict['client'][0]
device_dict = DeviceDictionary.get(
self.factory.bbb1.hostname).to_dict()
self.assertIsNotNone(device_dict)
tag_list = client_job['protocols']['lava-vland']['vlan_one']['tags']
for interface, tags in device_dict['parameters']['tags'].iteritems():
if set(tags) & set(tag_list) == set(
tag_list) and interface not in interfaces:
interfaces.append(interface)
break
self.assertEqual(['eth1'], interfaces)
self.assertEqual(len(interfaces),
len(client_job['protocols']['lava-vland'].keys()))
interfaces = []
server_job = job_dict['server'][0]
device_dict = DeviceDictionary.get(
self.factory.cubie1.hostname).to_dict()
self.assertIsNotNone(device_dict)
tag_list = server_job['protocols']['lava-vland']['vlan_two']['tags']
for interface, tags in device_dict['parameters']['tags'].iteritems():
if set(tags) & set(tag_list) == set(
tag_list) and interface not in interfaces:
interfaces.append(interface)
break
self.assertEqual(['eth1'], interfaces)
self.assertEqual(len(interfaces),
len(client_job['protocols']['lava-vland'].keys()))
vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user)
# vlan_one: client role. RJ45 10M. bbb device type
# vlan_two: server role. RJ45 100M. cubie device type.
assignments = {}
for job in vlan_job:
device = find_device_for_job(job, devices)
self.assertEqual(device.device_type, job.requested_device_type)
# map has been defined
self.assertTrue(
match_vlan_interface(device, yaml.load(job.definition)))
assignments[job.device_role] = device
self.assertEqual(assignments['client'].hostname,
self.factory.bbb1.hostname)
self.assertEqual(assignments['server'].hostname,
self.factory.cubie1.hostname)
