def test_same_type_devices_with_map(self):
device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname)
device_dict.parameters = { # client, RJ45 10M 100M
'extends': 'beaglebone-black.jinja2',
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"},
'mac_addr': {'eth0': "f0:de:f1:46:8c:22", 'eth1': "00:24:d7:9b:c0:8b"},
'tags': {'eth0': [], 'eth1': ['RJ45', '10M', '100M']},
'map': {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}}
}
device_dict.save()
bbb2 = self.factory.make_device(self.factory.bbb_type, hostname='bbb2')
device_dict = DeviceDictionary(hostname=bbb2.hostname)
device_dict.parameters = { # server, RJ45 10M 100M
'extends': 'beaglebone-black.jinja2',
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"},
'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"},
'tags': {'eth0': [], 'eth1': ['RJ45', '10M', '100M']},
'map': {'eth0': {'192.168.0.2': 7}, 'eth1': {'192.168.0.2': 9}}
}
device_dict.save()
devices = list(Device.objects.filter(status=Device.IDLE).order_by('is_public'))
user = self.factory.make_user()
sample_job_file = os.path.join(os.path.dirname(__file__), 'bbb-bbb-vland-group.yaml')
with open(sample_job_file, 'r') as test_support:
data = yaml.load(test_support)
vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user)
assignments = {}
for job in vlan_job:
device = find_device_for_job(job, devices)
self.assertEqual(device.device_type, job.requested_device_type)
# map has been defined
self.assertTrue(match_vlan_interface(device, yaml.load(job.definition)))
assignments[job.device_role] = device
if device in devices:
devices.remove(device)
assign_jobs()
self.factory.bbb1.refresh_from_db()
bbb2.refresh_from_db()
self.assertIsNotNone(self.factory.bbb1.current_job)
self.assertIsNotNone(bbb2.current_job)
self.assertIsNotNone(self.factory.bbb1.current_job.actual_device)
self.assertIsNotNone(bbb2.current_job.actual_device)
self.assertNotEqual(self.factory.bbb1.current_job, bbb2.current_job)
self.assertNotEqual(self.factory.bbb1.current_job.actual_device, bbb2.current_job.actual_device)
def test_menu_device(self):
job_ctx = {}
hostname = 'mustang01'
device_dict = DeviceDictionary(hostname=hostname)
device_dict.parameters = self.conf
device_dict.save()
device = self.factory.make_device(self.device_type, hostname)
self.assertEqual(device.device_type.name, 'mustang-uefi')
device_data = devicedictionary_to_jinja2(
device_dict.parameters,
device_dict.parameters['extends']
)
template = prepare_jinja_template(hostname, device_data, system_path=False, path=self.jinja_path)
config_str = template.render(**job_ctx)
self.assertIsNotNone(config_str)
config = yaml.load(config_str)
self.assertIsNotNone(config)
self.assertEqual(config['device_type'], self.device_type.name)
self.assertIsNotNone(config['parameters'])
self.assertIsNotNone(config['actions']['boot']['methods']['uefi-menu']['nfs'])
menu_data = config['actions']['boot']['methods']['uefi-menu']['nfs']
tftp_menu = [item for item in menu_data if 'items' in item['select'] and 'TFTP' in item['select']['items'][0]][0]
tftp_mac = self.conf['tftp_mac']
# value from device dictionary correctly replaces device type default
self.assertIn(tftp_mac, tftp_menu['select']['items'][0])
def test_menu_device(self):
job_ctx = {}
hostname = 'mustang01'
device_dict = DeviceDictionary(hostname=hostname)
device_dict.parameters = self.conf
device_dict.save()
device = self.factory.make_device(self.device_type, hostname)
self.assertEqual(device.device_type.name, 'mustang-uefi')
device_data = devicedictionary_to_jinja2(
device_dict.parameters, device_dict.parameters['extends'])
template = prepare_jinja_template(hostname,
device_data,
system_path=False,
path=self.jinja_path)
config_str = template.render(**job_ctx)
self.assertIsNotNone(config_str)
config = yaml.load(config_str)
self.assertIsNotNone(config)
self.assertEqual(config['device_type'], self.device_type.name)
self.assertNotIn('parameters', config)
self.assertIsNotNone(
config['actions']['boot']['methods']['uefi-menu']['nfs'])
menu_data = config['actions']['boot']['methods']['uefi-menu']['nfs']
tftp_menu = [
item for item in menu_data if 'items' in item['select']
and 'TFTP' in item['select']['items'][0]
][0]
tftp_mac = self.conf['tftp_mac']
# value from device dictionary correctly replaces device type default
self.assertIn(tftp_mac, tftp_menu['select']['items'][0])
def handle_add(self, hostname, device_type, worker_name,
description, dictionary, pipeline, public, online):
try:
dt = DeviceType.objects.get(name=device_type)
except DeviceType.DoesNotExist:
self.stderr.write("Unable to find device-type '%s'" % device_type)
sys.exit(1)
try:
worker = Worker.objects.get(hostname=worker_name)
except Worker.DoesNotExist:
self.stderr.write("Unable to find worker '%s'" % worker_name)
sys.exit(1)
status = Device.IDLE if online else Device.OFFLINE
Device.objects.create(hostname=hostname, device_type=dt,
description=description, worker_host=worker,
is_pipeline=pipeline, status=status,
is_public=public)
if dictionary is not None:
data = jinja2_to_devicedictionary(dictionary.read())
if data is None:
self.stderr.write("Invalid device dictionary")
sys.exit(1)
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = data
element.save()
def test_select_device(self):
self.restart()
hostname = 'fakeqemu3'
device_dict = DeviceDictionary(hostname=hostname)
device_dict.parameters = self.conf
device_dict.save()
device = self.factory.make_device(self.device_type, hostname)
job = TestJob.from_yaml_and_user(
self.factory.make_job_yaml(),
self.factory.make_user())
# this uses the system jinja2 path - local changes to the qemu.jinja2
# will not be available.
selected = select_device(job, self.dispatchers)
self.assertIsNone(selected)
job.actual_device = device
selected = select_device(job, self.dispatchers)
self.assertIsNone(selected)
device.worker_host = self.worker
selected = select_device(job, self.dispatchers)
self.assertIsNone(selected)
# device needs to be in reserved state
# fake up the assignment which needs a separate test
job.actual_device = device
job.save()
device.current_job = job
device.status = Device.RESERVED
device.save()
selected = select_device(job, self.dispatchers)
self.assertEqual(selected, device)
def test_pipeline_device(self):
foo = DeviceDictionary(hostname='foo')
foo.parameters = {
'bootz': {
'kernel': '0x4700000',
'ramdisk': '0x4800000',
'dtb': '0x4300000'
},
'media': {
'usb': {
'UUID-required': True,
'SanDisk_Ultra': {
'uuid': 'usb-SanDisk_Ultra_20060775320F43006019-0:0',
'device_id': 0
},
'sata': {
'UUID-required': False
}
}
}
}
device = PipelineDevice(foo.parameters, 'foo')
self.assertEqual(device.target, 'foo')
self.assertIn('power_state', device)
self.assertEqual(device.power_state, '') # there is no power_on_command for this device, so the property is ''
self.assertTrue(hasattr(device, 'power_state'))
self.assertFalse(hasattr(device, 'hostname'))
self.assertIn('hostname', device)
def test_pipeline_device(self):
foo = DeviceDictionary(hostname='foo')
foo.parameters = {
'bootz': {
'kernel': '0x4700000',
'ramdisk': '0x4800000',
'dtb': '0x4300000'
},
'media': {
'usb': {
'UUID-required': True,
'SanDisk_Ultra': {
'uuid': 'usb-SanDisk_Ultra_20060775320F43006019-0:0',
'device_id': 0
},
'sata': {
'UUID-required': False
}
}
}
}
device = PipelineDevice(foo.parameters, 'foo')
self.assertEqual(device.target, 'foo')
self.assertIn('power_state', device)
self.assertEqual(device.power_state, '') # there is no power_on_command for this device, so the property is ''
self.assertTrue(hasattr(device, 'power_state'))
self.assertFalse(hasattr(device, 'hostname'))
self.assertIn('hostname', device)
def test_jinja_template(self):
jinja2_path = os.path.realpath(
os.path.join(__file__, '..', '..', '..', 'etc',
'dispatcher-config'))
self.assertTrue(os.path.exists(jinja2_path))
device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname)
device_dict.parameters = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0':
"/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1':
"/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"
},
'mac_addr': {
'eth0': "f0:de:f1:46:8c:21",
'eth1': "00:24:d7:9b:c0:8c"
},
'tags': {
'eth0': ['1G', '10G'],
'eth1': ['1G']
},
'map': {
'eth0': {
'192.168.0.2': 5
},
'eth1': {
'192.168.0.2': 7
}
}
}
# {% map = '{'eth1': {'3': 8}, 'eth0': {'3': 19}}' %}
device_dict.save()
data = devicedictionary_to_jinja2(device_dict.parameters,
'beaglebone-black.jinja2')
check_str = """{% extends 'beaglebone-black.jinja2' %}
{% set map = {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} %}
{% set interfaces = ['eth0', 'eth1'] %}
{% set sysfs = {'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0',
'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1'} %}
{% set mac_addr = {'eth0': 'f0:de:f1:46:8c:21', 'eth1': '00:24:d7:9b:c0:8c'} %}
{% set tags = {'eth0': ['1G', '10G'], 'eth1': ['1G']} %}
"""
self.assertEqual(check_str, data)
template = prepare_jinja_template(self.factory.bbb1.hostname,
data,
system_path=False)
device_configuration = template.render()
yaml_data = yaml.load(device_configuration)
self.assertIn('parameters', yaml_data)
self.assertIn('interfaces', yaml_data['parameters'])
self.assertIn('bootm', yaml_data['parameters'])
self.assertIn('bootz', yaml_data['parameters'])
self.assertIn('actions', yaml_data)
self.assertIn('eth0', yaml_data['parameters']['interfaces'])
self.assertIn('eth1', yaml_data['parameters']['interfaces'])
self.assertIn('sysfs', yaml_data['parameters']['interfaces']['eth0'])
self.assertEqual(
'/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1',
yaml_data['parameters']['interfaces']['eth1']['sysfs'])
def test_job_protocols(self):
self.factory.ensure_tag('usb-eth')
self.factory.ensure_tag('sata')
self.factory.bbb1.tags = Tag.objects.filter(name='usb-eth')
self.factory.bbb1.save()
self.factory.cubie1.tags = Tag.objects.filter(name='sata')
self.factory.cubie1.save()
device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname)
device_dict.parameters = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"},
'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"},
'tags': {'eth0': ['1G', '10G'], 'eth1': ['1G']},
'map': {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}}
}
device_dict.save()
device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname)
device_dict.parameters = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"},
'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"},
'tags': {'eth0': ['1G', '10G'], 'eth1': ['1G']},
'map': {'eth0': {'192.168.0.2': 4}, 'eth1': {'192.168.0.2': 6}}
}
device_dict.save()
target_group = "unit-test-only"
job_dict = split_multinode_yaml(self.factory.make_vland_job(), target_group)
client_job = job_dict['client'][0]
client_handle, client_file_name = tempfile.mkstemp()
yaml.dump(client_job, open(client_file_name, 'w'))
# YAML device file, as required by lava-dispatch --target
device_yaml_file = os.path.realpath(os.path.join(os.path.dirname(__file__), 'bbb-01.yaml'))
print device_yaml_file
self.assertTrue(os.path.exists(device_yaml_file))
parser = JobParser()
bbb_device = NewDevice(device_yaml_file)
with open(client_file_name) as sample_job_data:
bbb_job = parser.parse(sample_job_data, bbb_device, 4212, None, output_dir='/tmp/')
os.close(client_handle)
os.unlink(client_file_name)
self.assertIn('protocols', bbb_job.parameters)
self.assertIn(VlandProtocol.name, bbb_job.parameters['protocols'])
self.assertIn(MultinodeProtocol.name, bbb_job.parameters['protocols'])
def test_make_device(self):
hostname = 'fakeqemu3'
device_dict = DeviceDictionary(hostname=hostname)
device_dict.parameters = self.conf
device_dict.save()
device = self.factory.make_device(self.device_type, hostname)
self.assertEqual(device.device_type.name, 'qemu')
job = self.factory.make_job_yaml()
self.assertIsNotNone(job)
def test_make_device(self):
hostname = 'fakeqemu3'
device_dict = DeviceDictionary(hostname=hostname)
device_dict.parameters = self.conf
device_dict.save()
device = self.factory.make_device(self.device_type, hostname)
self.assertEqual(device.device_type.name, 'qemu')
job = self.factory.make_job_yaml()
self.assertIsNotNone(job)
def test_vland_jinja2(self):
"""
Test complex device dictionary values
The reference data can cross lines but cannot be indented as the pprint
object in utils uses indent=0, width=80 for YAML compatibility.
The strings read in from config files can have indenting spaces, these
are removed in the pprint.
"""
data = """{% extends 'vland.jinja2' %}
{% set interfaces = ['eth0', 'eth1'] %}
{% set sysfs = {'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0',
'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1'} %}
{% set mac_addr = {'eth0': 'f0:de:f1:46:8c:21', 'eth1': '00:24:d7:9b:c0:8c'} %}
{% set tags = {'eth0': ['1G', '10G'], 'eth1': ['1G']} %}
{% set map = {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} %}
"""
result = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1',
'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0'
},
'extends': 'vland.jinja2',
'mac_addr': {
'eth1': '00:24:d7:9b:c0:8c',
'eth0': 'f0:de:f1:46:8c:21'
},
'tags': {
'eth1': ['1G'],
'eth0': ['1G', '10G']
},
'map': {
'eth0': {
'192.168.0.2': 5
},
'eth1': {
'192.168.0.2': 7
}
}
}
dictionary = jinja2_to_devicedictionary(data_dict=data)
self.assertEqual(result, dictionary)
jinja2_str = devicedictionary_to_jinja2(data_dict=dictionary, extends='vland.jinja2')
# ordering within the dict can change but each line needs to still appear
for line in str(data).split('\n'):
self.assertIn(line, str(jinja2_str))
# create a DeviceDictionary for this test
vlan = DeviceDictionary(hostname='vlanned1')
vlan.parameters = dictionary
vlan.save()
del vlan
vlan = DeviceDictionary.get('vlanned1')
cmp_str = str(devicedictionary_to_jinja2(vlan.parameters, 'vland.jinja2'))
for line in str(data).split('\n'):
self.assertIn(line, cmp_str)
def test_vland_jinja2(self):
"""
Test complex device dictionary values
The reference data can cross lines but cannot be indented as the pprint
object in utils uses indent=0, width=80 for YAML compatibility.
The strings read in from config files can have indenting spaces, these
are removed in the pprint.
"""
data = """{% extends 'vland.jinja2' %}
{% set interfaces = ['eth0', 'eth1'] %}
{% set sysfs = {'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0',
'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1'} %}
{% set mac_addr = {'eth0': 'f0:de:f1:46:8c:21', 'eth1': '00:24:d7:9b:c0:8c'} %}
{% set tags = {'eth0': ['1G', '10G'], 'eth1': ['1G']} %}
{% set map = {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} %}
"""
result = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1',
'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0'
},
'extends': 'vland.jinja2',
'mac_addr': {
'eth1': '00:24:d7:9b:c0:8c',
'eth0': 'f0:de:f1:46:8c:21'
},
'tags': {
'eth1': ['1G'],
'eth0': ['1G', '10G']
},
'map': {
'eth0': {
'192.168.0.2': 5
},
'eth1': {
'192.168.0.2': 7
}
}
}
dictionary = jinja2_to_devicedictionary(data_dict=data)
self.assertEqual(result, dictionary)
jinja2_str = devicedictionary_to_jinja2(data_dict=dictionary, extends='vland.jinja2')
# ordering within the dict can change but each line needs to still appear
for line in str(data).split('\n'):
self.assertIn(line, str(jinja2_str))
# create a DeviceDictionary for this test
vlan = DeviceDictionary(hostname='vlanned1')
vlan.parameters = dictionary
vlan.save()
del vlan
vlan = DeviceDictionary.get('vlanned1')
cmp_str = str(devicedictionary_to_jinja2(vlan.parameters, 'vland.jinja2'))
for line in str(data).split('\n'):
self.assertIn(line, cmp_str)
def test_match_devices_with_map(self):
devices = Device.objects.filter(status=Device.IDLE).order_by('is_public')
self.factory.ensure_tag('usb-eth')
self.factory.ensure_tag('sata')
self.factory.bbb1.tags = Tag.objects.filter(name='usb-eth')
self.factory.bbb1.save()
self.factory.cubie1.tags = Tag.objects.filter(name='sata')
self.factory.cubie1.save()
device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname)
device_dict.parameters = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"},
'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"},
'tags': {'eth0': ['1G', '10G'], 'eth1': ['1G']},
'map': {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}}
}
device_dict.save()
device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname)
device_dict.parameters = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"},
'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"},
'tags': {'eth0': ['1G', '10G'], 'eth1': ['1G']},
'map': {'eth0': {'192.168.0.2': 4}, 'eth1': {'192.168.0.2': 6}}
}
device_dict.save()
user = self.factory.make_user()
sample_job_file = os.path.join(os.path.dirname(__file__), 'bbb-cubie-vlan-group.yaml')
with open(sample_job_file, 'r') as test_support:
data = yaml.load(test_support)
vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user)
assignments = {}
for job in vlan_job:
device = find_device_for_job(job, devices)
self.assertEqual(device.device_type, job.requested_device_type)
# map has been defined
self.assertTrue(match_vlan_interface(device, yaml.load(job.definition)))
assignments[job.device_role] = device
self.assertEqual(assignments['client'].hostname, self.factory.bbb1.hostname)
self.assertEqual(assignments['server'].hostname, self.factory.cubie1.hostname)
def test_jinja_postgres_loader(self):
# path used for the device_type template
jinja2_path = os.path.realpath(os.path.join(__file__, '..', '..', '..', 'etc', 'dispatcher-config'))
self.assertTrue(os.path.exists(jinja2_path))
device_type = 'cubietruck'
# pretend this was already imported into the database and use for comparison later.
device_dictionary = {
'usb_label': 'SanDisk_Ultra',
'sata_label': 'ST160LM003',
'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0",
'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184",
'connection_command': 'telnet localhost 6002'
}
# create a DeviceDictionary for this test
cubie = DeviceDictionary(hostname='cubie')
cubie.parameters = device_dictionary
cubie.save()
dict_loader = jinja2.DictLoader(
{
'cubie.yaml':
devicedictionary_to_jinja2(cubie.parameters, '%s.yaml' % device_type)
}
)
type_loader = jinja2.FileSystemLoader([os.path.join(jinja2_path, 'device-types')])
env = jinja2.Environment(
loader=jinja2.ChoiceLoader([dict_loader, type_loader]),
trim_blocks=True)
template = env.get_template("%s.yaml" % 'cubie')
device_configuration = template.render()
yaml_data = yaml.load(device_configuration)
self.assertIn('timeouts', yaml_data)
self.assertIn('parameters', yaml_data)
self.assertIn('bootz', yaml_data['parameters'])
self.assertIn('media', yaml_data['parameters'])
self.assertIn('usb', yaml_data['parameters']['media'])
self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb'])
self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']])
self.assertEqual(
yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'],
device_dictionary['usb_uuid']
)
self.assertIn('commands', yaml_data)
self.assertIn('connect', yaml_data['commands'])
self.assertEqual(
device_dictionary['connection_command'],
yaml_data['commands']['connect'])
device = PipelineDevice(yaml_data, 'cubie')
self.assertIn('power_state', device)
# cubie1 has no power_on_command defined
self.assertEqual(device.power_state, '')
self.assertTrue(hasattr(device, 'power_state'))
self.assertFalse(hasattr(device, 'hostname'))
self.assertIn('hostname', device)
def test_jinja_postgres_loader(self):
# path used for the device_type template
jinja2_path = jinja_template_path(system=False)
self.assertTrue(os.path.exists(jinja2_path))
device_type = 'cubietruck'
# pretend this was already imported into the database and use for comparison later.
device_dictionary = {
'usb_label': 'SanDisk_Ultra',
'sata_label': 'ST160LM003',
'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0",
'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184",
'connection_command': 'telnet localhost 6002'
}
# create a DeviceDictionary for this test
cubie = DeviceDictionary(hostname='cubie')
cubie.parameters = device_dictionary
cubie.save()
jinja_data = devicedictionary_to_jinja2(cubie.parameters, '%s.jinja2' % device_type)
dict_loader = jinja2.DictLoader({'cubie.jinja2': jinja_data})
type_loader = jinja2.FileSystemLoader([os.path.join(jinja2_path, 'device-types')])
env = jinja2.Environment(
loader=jinja2.ChoiceLoader([dict_loader, type_loader]),
trim_blocks=True)
template = env.get_template("%s.jinja2" % 'cubie')
# pylint gets this wrong from jinja
device_configuration = template.render() # pylint: disable=no-member
chk_template = prepare_jinja_template('cubie', jinja_data, system_path=False, path=jinja2_path)
self.assertEqual(template.render(), chk_template.render()) # pylint: disable=no-member
yaml_data = yaml.load(device_configuration)
self.assertTrue(validate_device(yaml_data))
self.assertIn('timeouts', yaml_data)
self.assertIn('parameters', yaml_data)
self.assertIn('bootz', yaml_data['parameters'])
self.assertIn('media', yaml_data['parameters'])
self.assertIn('usb', yaml_data['parameters']['media'])
self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb'])
self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']])
self.assertEqual(
yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'],
device_dictionary['usb_uuid']
)
self.assertIn('commands', yaml_data)
self.assertIn('connect', yaml_data['commands'])
self.assertEqual(
device_dictionary['connection_command'],
yaml_data['commands']['connect'])
device = PipelineDevice(yaml_data, 'cubie')
self.assertIn('power_state', device)
# cubie1 has no power_on_command defined
self.assertEqual(device.power_state, '')
self.assertTrue(hasattr(device, 'power_state'))
self.assertFalse(hasattr(device, 'hostname'))
self.assertIn('hostname', device)
def handle(self, *args, **options):
"""
Accept options via lava-server manage which provides access
to the database.
"""
hostname = options['hostname']
if hostname is None:
self.stderr.write("Please specify a hostname")
sys.exit(2)
if options['import'] is not None:
data = parse_template(options['import'])
element = DeviceDictionary.get(hostname)
if element is None:
self.stdout.write("Adding new device dictionary for %s" %
hostname)
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = data
element.save()
self.stdout.write("Device dictionary updated for %s" % hostname)
elif options['export'] is not None or options['review'] is not None:
element = DeviceDictionary.get(hostname)
data = None
if element is None:
self.stderr.write(
"Unable to export - no dictionary found for '%s'" %
hostname)
sys.exit(2)
else:
data = devicedictionary_to_jinja2(
element.parameters, element.parameters['extends'])
if options['review'] is None:
self.stdout.write(data)
else:
string_loader = jinja2.DictLoader({'%s.yaml' % hostname: data})
type_loader = jinja2.FileSystemLoader(
[os.path.join(options['path'], 'device-types')])
env = jinja2.Environment(loader=jinja2.ChoiceLoader(
[string_loader, type_loader]),
trim_blocks=True)
template = env.get_template("%s.yaml" % hostname)
device_configuration = template.render()
# validate against the device schema
try:
validate_device(yaml.load(device_configuration))
except (yaml.YAMLError, SubmissionException) as exc:
self.stderr.write("Invalid template: %s" % exc)
self.stdout.write(device_configuration)
else:
self.stderr.write(
"Please specify one of --import, --export or --review")
sys.exit(1)
def setUp(self):
super(VlanInterfaces, self).setUp()
# YAML, pipeline only
user = User.objects.create_user('test', '*****@*****.**', 'test')
user.user_permissions.add(
Permission.objects.get(codename='add_testjob'))
user.save()
bbb_type = self.factory.make_device_type('beaglebone-black')
bbb_1 = self.factory.make_device(hostname='bbb-01', device_type=bbb_type)
device_dict = DeviceDictionary.get(bbb_1.hostname)
self.assertIsNone(device_dict)
device_dict = DeviceDictionary(hostname=bbb_1.hostname)
device_dict.parameters = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"},
'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"},
'tags': {'eth0': ['1G', '10G'], 'eth1': ['1G']},
'map': {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}}
}
device_dict.save()
ct_type = self.factory.make_device_type('cubietruck')
cubie = self.factory.make_device(hostname='ct-01', device_type=ct_type)
device_dict = DeviceDictionary.get(cubie.hostname)
self.assertIsNone(device_dict)
device_dict = DeviceDictionary(hostname=cubie.hostname)
device_dict.parameters = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"},
'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"},
'tags': {'eth0': ['1G', '10G'], 'eth1': ['1G']},
'map': {'eth0': {'192.168.0.2': 4}, 'eth1': {'192.168.0.2': 6}}
}
device_dict.save()
self.filename = os.path.join(os.path.dirname(__file__), 'bbb-cubie-vlan-group.yaml')
def test_host_role(self):
# need a full job to properly test the multinode YAML split
hostname = 'fakeqemu3'
self.factory.make_device(self.device_type, hostname)
device_dict = DeviceDictionary(hostname=hostname)
device_dict.parameters = self.conf
device_dict.save()
# create a new device to allow the submission to reach the multinode YAML test.
hostname = 'fakeqemu4'
self.factory.make_device(self.device_type, hostname)
data = yaml.load(self.factory.make_job_json())
data['protocols']['lava-multinode']['roles']['host']['count'] = 2
self.assertRaises(SubmissionException, TestJob.from_yaml_and_user,
yaml.dump(data), self.factory.make_user())
def handle(self, *args, **options):
"""
Accept options via lava-server manage which provides access
to the database.
"""
hostname = options['hostname']
if hostname is None:
self.stderr.write("Please specify a hostname")
sys.exit(2)
if options['import']:
data = parse_template(options['import'])
element = DeviceDictionary.get(hostname)
if element is None:
self.stdout.write("Adding new device dictionary for %s" %
hostname)
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = data
element.save()
self.stdout.write("Device dictionary updated for %s" % hostname)
elif options['export'] or options['review']:
element = DeviceDictionary.get(hostname)
if element is None:
self.stderr.write(
"Unable to export - no dictionary found for '%s'" %
hostname)
sys.exit(2)
else:
data = devicedictionary_to_jinja2(
element.parameters, element.parameters['extends'])
if not options['review']:
self.stdout.write(data)
else:
template = prepare_jinja_template(hostname,
data,
system_path=False,
path=options['path'])
device_configuration = template.render()
# validate against the device schema
try:
validate_device(yaml.load(device_configuration))
except (yaml.YAMLError, SubmissionException) as exc:
self.stderr.write("Invalid template: %s" % exc)
self.stdout.write(device_configuration)
else:
self.stderr.write(
"Please specify one of --import, --export or --review")
sys.exit(1)
def handle_set(self, options):
""" Set device properties """
hostname = options["hostname"]
try:
device = Device.objects.get(hostname=hostname)
except Device.DoesNotExist:
self.stderr.write("Unable to find device '%s'" % hostname)
sys.exit(1)
status = options["status"]
if status is not None:
device.status = self.device_status[status]
health = options["health"]
if health is not None:
device.health_status = self.health_status[health]
description = options["description"]
if description is not None:
device.description = description
worker_name = options["worker"]
if worker_name is not None:
try:
worker = Worker.objects.get(hostname=worker_name)
device.worker_host = worker
except Worker.DoesNotExist:
self.stderr.write("Unable to find worker '%s'" % worker_name)
sys.exit(1)
public = options["public"]
if public is not None:
device.is_public = public
dictionary = options["dictionary"]
if dictionary is not None:
data = jinja2_to_devicedictionary(dictionary.read())
if data is None:
self.stderr.write("Invalid device dictionary")
sys.exit(1)
element = DeviceDictionary.get(hostname)
if element is None:
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = data
element.save()
# Save the modifications
device.save()
def test_host_role(self):
# need a full job to properly test the multinode YAML split
hostname = 'fakeqemu3'
self.factory.make_device(self.device_type, hostname)
device_dict = DeviceDictionary(hostname=hostname)
device_dict.parameters = self.conf
device_dict.save()
# create a new device to allow the submission to reach the multinode YAML test.
hostname = 'fakeqemu4'
self.factory.make_device(self.device_type, hostname)
data = yaml.load(self.factory.make_job_json())
data['protocols']['lava-multinode']['roles']['host']['count'] = 2
self.assertRaises(
SubmissionException, TestJob.from_yaml_and_user,
yaml.dump(data), self.factory.make_user())
def handle(self, *args, **options):
"""
Accept options via lava-server manage which provides access
to the database.
"""
hostname = options['hostname']
if hostname is None:
self.stderr.write("Please specify a hostname")
sys.exit(2)
if options['import'] is not None:
data = parse_template(options['import'])
element = DeviceDictionary.get(hostname)
if element is None:
self.stdout.write("Adding new device dictionary for %s" %
hostname)
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = data
element.save()
self.stdout.write("Device dictionary updated for %s" % hostname)
elif options['export'] is not None or options['review'] is not None:
element = DeviceDictionary.get(hostname)
data = None
if element is None:
self.stderr.write("Unable to export - no dictionary found for '%s'" %
hostname)
sys.exit(2)
else:
data = devicedictionary_to_jinja2(
element.parameters,
element.parameters['extends']
)
if options['review'] is None:
self.stdout.write(data)
else:
template = prepare_jinja_template(hostname, data, system_path=False, path=options['path'])
device_configuration = template.render()
# validate against the device schema
try:
validate_device(yaml.load(device_configuration))
except (yaml.YAMLError, SubmissionException) as exc:
self.stderr.write("Invalid template: %s" % exc)
self.stdout.write(device_configuration)
else:
self.stderr.write("Please specify one of --import, --export or --review")
sys.exit(1)
def test_dictionary_remove(self):
foo = DeviceDictionary(hostname='foo')
foo.parameters = {
'bootz': {
'kernel': '0x4700000',
'ramdisk': '0x4800000',
'dtb': '0x4300000'
},
}
foo.save()
baz = DeviceDictionary.get('foo')
self.assertEqual(baz.parameters, foo.parameters)
baz.delete()
self.assertIsInstance(baz, DeviceDictionary)
baz = DeviceDictionary.get('foo')
self.assertIsNone(baz)
def test_exclusivity(self):
device = Device.objects.get(hostname="fakeqemu1")
self.assertTrue(device.is_pipeline)
self.assertFalse(device.is_exclusive)
self.assertIsNotNone(DeviceDictionary.get(device.hostname))
device_dict = DeviceDictionary(hostname=device.hostname)
device_dict.save()
device_dict = DeviceDictionary.get(device.hostname)
self.assertTrue(device.is_pipeline)
self.assertFalse(device.is_exclusive)
update = device_dict.to_dict()
update.update({'exclusive': 'True'})
device_dict.parameters = update
device_dict.save()
self.assertTrue(device.is_pipeline)
self.assertTrue(device.is_exclusive)
def test_exclusivity(self):
device = Device.objects.get(hostname="fakeqemu1")
self.assertTrue(device.is_pipeline)
self.assertFalse(device.is_exclusive)
self.assertIsNotNone(DeviceDictionary.get(device.hostname))
device_dict = DeviceDictionary(hostname=device.hostname)
device_dict.save()
device_dict = DeviceDictionary.get(device.hostname)
self.assertTrue(device.is_pipeline)
self.assertFalse(device.is_exclusive)
update = device_dict.to_dict()
update.update({'exclusive': 'True'})
device_dict.parameters = update
device_dict.save()
self.assertTrue(device.is_pipeline)
self.assertTrue(device.is_exclusive)
def test_dictionary_remove(self):
foo = DeviceDictionary(hostname='foo')
foo.parameters = {
'bootz': {
'kernel': '0x4700000',
'ramdisk': '0x4800000',
'dtb': '0x4300000'
},
}
foo.save()
baz = DeviceDictionary.get('foo')
self.assertEqual(baz.parameters, foo.parameters)
baz.delete()
self.assertIsInstance(baz, DeviceDictionary)
baz = DeviceDictionary.get('foo')
self.assertIsNone(baz)
def handle(self, *args, **options):
"""
Accept options via lava-server manage which provides access
to the database.
"""
hostname = options['hostname']
if hostname is None:
self.stderr.write("Please specify a hostname")
sys.exit(2)
if options['import'] is not None:
data = parse_template(options['import'])
element = DeviceDictionary.get(hostname)
if element is None:
self.stdout.write("Adding new device dictionary for %s" %
hostname)
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = data
element.save()
self.stdout.write("Device dictionary updated for %s" % hostname)
elif options['export'] is not None or options['review'] is not None:
element = DeviceDictionary.get(hostname)
data = None
if element is None:
self.stderr.write("Unable to export - no dictionary found for '%s'" %
hostname)
sys.exit(2)
else:
data = devicedictionary_to_jinja2(
element.parameters,
element.parameters['extends']
)
if options['review'] is None:
self.stdout.write(data)
else:
string_loader = jinja2.DictLoader({'%s.yaml' % hostname: data})
type_loader = jinja2.FileSystemLoader([
os.path.join(options['path'], 'device-types')])
env = jinja2.Environment(
loader=jinja2.ChoiceLoader([string_loader, type_loader]),
trim_blocks=True)
template = env.get_template("%s.yaml" % hostname)
device_configuration = template.render()
self.stdout.write(device_configuration)
else:
self.stderr.write("Please specify one of --import, --export or --review")
sys.exit(1)
def test_from_json_rejects_exclusive(self):
panda_type = self.factory.ensure_device_type(name='panda')
panda_board = self.factory.make_device(device_type=panda_type, hostname='panda01')
self.assertFalse(panda_board.is_exclusive)
job = TestJob.from_json_and_user(
self.factory.make_job_json(device_type='panda'),
self.factory.make_user())
self.assertEqual(panda_type, job.requested_device_type)
device_dict = DeviceDictionary.get(panda_board.hostname)
self.assertIsNone(device_dict)
device_dict = DeviceDictionary(hostname=panda_board.hostname)
device_dict.parameters = {'exclusive': 'True'}
device_dict.save()
self.assertTrue(panda_board.is_exclusive)
self.assertRaises(
DevicesUnavailableException, _check_exclusivity, [panda_board], pipeline=False
)
def test_from_json_rejects_exclusive(self):
panda_type = self.factory.ensure_device_type(name='panda')
panda_board = self.factory.make_device(device_type=panda_type,
hostname='panda01')
self.assertFalse(panda_board.is_exclusive)
job = TestJob.from_json_and_user(
self.factory.make_job_json(device_type='panda'),
self.factory.make_user())
self.assertEqual(panda_type, job.requested_device_type)
device_dict = DeviceDictionary.get(panda_board.hostname)
self.assertIsNone(device_dict)
device_dict = DeviceDictionary(hostname=panda_board.hostname)
device_dict.parameters = {'exclusive': 'True'}
device_dict.save()
self.assertTrue(panda_board.is_exclusive)
self.assertRaises(DevicesUnavailableException,
_check_exclusivity, [panda_board],
pipeline=False)
def test_identify_context(self):
hostname = "fakebbb"
mustang_type = self.factory.make_device_type('beaglebone-black')
# this sets a qemu device dictionary, so replace it
self.factory.make_device(device_type=mustang_type, hostname=hostname)
mustang = DeviceDictionary(hostname=hostname)
mustang.parameters = {
'extends': 'beaglebone-black.jinja2',
'base_nfsroot_args': '10.16.56.2:/home/lava/debian/nfs/,tcp,hard,intr',
'console_device': 'ttyO0', # takes precedence over the job context as the same var name is used.
}
mustang.save()
mustang_dict = mustang.to_dict()
device = Device.objects.get(hostname="fakebbb")
self.assertEqual('beaglebone-black', device.device_type.name)
self.assertTrue(device.is_pipeline)
context_overrides = map_context_overrides('base.jinja2', 'beaglebone-black.jinja2', system=False)
job_ctx = {
'base_uboot_commands': 'dummy commands',
'usb_uuid': 'dummy usb uuid',
'console_device': 'ttyAMA0',
'usb_device_id': 1111111111111111
}
device_config = device.load_device_configuration(job_ctx, system=False) # raw dict
self.assertIsNotNone(device_config)
devicetype_blocks = []
devicedict_blocks = []
allowed = []
for key, _ in job_ctx.items():
if key in context_overrides:
if key is not 'extends' and key not in mustang_dict['parameters'].keys():
allowed.append(key)
else:
devicedict_blocks.append(key)
else:
devicetype_blocks.append(key)
# only values set in job_ctx are checked
self.assertEqual(set(allowed), {'usb_device_id', 'usb_uuid'})
self.assertEqual(set(devicedict_blocks), {'console_device'})
self.assertEqual(set(devicetype_blocks), {'base_uboot_commands'})
full_list = allowed_overrides(mustang_dict, system=False)
for key in allowed:
self.assertIn(key, full_list)
def test_find_nonexclusive_device(self):
"""
test that exclusive devices are not assigned JSON jobs
"""
self.assertFalse(self.panda01.is_exclusive)
device_dict = DeviceDictionary.get(self.panda01.hostname)
self.assertIsNone(device_dict)
device_dict = DeviceDictionary(hostname=self.panda01.hostname)
device_dict.parameters = {'exclusive': 'True'}
device_dict.save()
self.assertTrue(self.panda01.is_exclusive)
self.assertRaises(DevicesUnavailableException,
self.submit_job,
target='panda01',
device_type='panda')
job = self.submit_job(device_type='panda')
devices = [self.panda02, self.panda01]
self.assertEqual(find_device_for_job(job, devices), self.panda02)
device_dict.delete()
self.assertFalse(self.panda01.is_exclusive)
def test_jinja_template(self):
jinja2_path = os.path.realpath(os.path.join(
__file__, '..', '..', '..', 'etc', 'dispatcher-config'))
self.assertTrue(os.path.exists(jinja2_path))
device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname)
device_dict.parameters = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"},
'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"},
'tags': {'eth0': ['1G', '10G'], 'eth1': ['1G']},
'map': {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}}
}
# {% map = '{'eth1': {'3': 8}, 'eth0': {'3': 19}}' %}
device_dict.save()
data = devicedictionary_to_jinja2(device_dict.parameters, 'beaglebone-black.jinja2')
check_str = """{% extends 'beaglebone-black.jinja2' %}
{% set map = {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} %}
{% set interfaces = ['eth0', 'eth1'] %}
{% set sysfs = {'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0',
'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1'} %}
{% set mac_addr = {'eth0': 'f0:de:f1:46:8c:21', 'eth1': '00:24:d7:9b:c0:8c'} %}
{% set tags = {'eth0': ['1G', '10G'], 'eth1': ['1G']} %}
"""
self.assertEqual(check_str, data)
template = prepare_jinja_template(self.factory.bbb1.hostname, data, system_path=False)
device_configuration = template.render()
yaml_data = yaml.load(device_configuration)
self.assertIn('parameters', yaml_data)
self.assertIn('interfaces', yaml_data['parameters'])
self.assertIn('bootm', yaml_data['parameters'])
self.assertIn('bootz', yaml_data['parameters'])
self.assertIn('actions', yaml_data)
self.assertIn('eth0', yaml_data['parameters']['interfaces'])
self.assertIn('eth1', yaml_data['parameters']['interfaces'])
self.assertIn('sysfs', yaml_data['parameters']['interfaces']['eth0'])
self.assertEqual(
'/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1',
yaml_data['parameters']['interfaces']['eth1']['sysfs']
)
def test_job_handlers(self):
self.restart()
hostname = 'fakeqemu3'
device_dict = DeviceDictionary(hostname=hostname)
device_dict.parameters = self.conf
device_dict.save()
device = self.factory.make_device(self.device_type, hostname)
job = TestJob.from_yaml_and_user(
self.factory.make_job_yaml(),
self.factory.make_user())
selected = select_device(job, self.dispatchers)
self.assertIsNone(selected)
job.actual_device = device
selected = select_device(job, self.dispatchers)
self.assertIsNone(selected)
device.worker_host = self.worker
selected = select_device(job, self.dispatchers)
self.assertIsNone(selected)
create_job(job, device)
self.assertEqual(job.actual_device, device)
self.assertEqual(device.status, Device.RESERVED)
def test_find_nonexclusive_device(self):
"""
test that exclusive devices are not assigned JSON jobs
"""
self.assertFalse(self.panda01.is_exclusive)
device_dict = DeviceDictionary.get(self.panda01.hostname)
self.assertIsNone(device_dict)
device_dict = DeviceDictionary(hostname=self.panda01.hostname)
device_dict.parameters = {'exclusive': 'True'}
device_dict.save()
self.assertTrue(self.panda01.is_exclusive)
self.assertRaises(
DevicesUnavailableException,
self.submit_job,
target='panda01', device_type='panda')
job = self.submit_job(device_type='panda')
devices = [self.panda02, self.panda01]
self.assertEqual(
find_device_for_job(job, devices),
self.panda02
)
device_dict.delete()
self.assertFalse(self.panda01.is_exclusive)
def test_broken_link_yaml(self):
hostname = 'fakeqemu3'
self.factory.make_device(self.device_type, hostname)
device_dict = DeviceDictionary(hostname=hostname)
device_dict.parameters = self.conf
device_dict.save()
# create a new device to allow the submission to reach the multinode YAML test.
hostname = 'fakeqemu4'
self.factory.make_device(self.device_type, hostname)
data = yaml.load(self.factory.make_job_json())
deploy = [action['deploy'] for action in data['actions'] if 'deploy' in action]
# replace working image with a broken URL
for block in deploy:
block['image'] = 'http://localhost/unknown/invalid.gz'
try:
jobs = TestJob.from_yaml_and_user(
self.factory.make_job_json(),
self.factory.make_user())
except DevicesUnavailableException as exc:
self.fail(exc)
self.assertEqual(
jobs[0].sub_id,
"%d.%d" % (int(jobs[0].id), 0))
def test_broken_link_yaml(self):
hostname = 'fakeqemu3'
self.factory.make_device(self.device_type, hostname)
device_dict = DeviceDictionary(hostname=hostname)
device_dict.parameters = self.conf
device_dict.save()
# create a new device to allow the submission to reach the multinode YAML test.
hostname = 'fakeqemu4'
self.factory.make_device(self.device_type, hostname)
data = yaml.load(self.factory.make_job_json())
deploy = [action['deploy'] for action in data['actions'] if 'deploy' in action]
# replace working image with a broken URL
for block in deploy:
block['image'] = 'http://localhost/unknown/invalid.gz'
try:
jobs = TestJob.from_yaml_and_user(
self.factory.make_job_json(),
self.factory.make_user())
except DevicesUnavailableException as exc:
self.fail(exc)
self.assertEqual(
jobs[0].sub_id,
"%d.%d" % (int(jobs[0].id), 0))
def test_dictionary_parameters(self):
foo = DeviceDictionary(hostname='foo')
foo.parameters = {
'bootz': {
'kernel': '0x4700000',
'ramdisk': '0x4800000',
'dtb': '0x4300000'
},
'media': {
'usb': {
'UUID-required': True,
'SanDisk_Ultra': {
'uuid': 'usb-SanDisk_Ultra_20060775320F43006019-0:0',
'device_id': 0
},
'sata': {
'UUID-required': False
}
}
}
}
foo.save()
bar = DeviceDictionary.get('foo')
self.assertEqual(bar.parameters, foo.parameters)
def test_dictionary_parameters(self):
foo = DeviceDictionary(hostname='foo')
foo.parameters = {
'bootz': {
'kernel': '0x4700000',
'ramdisk': '0x4800000',
'dtb': '0x4300000'
},
'media': {
'usb': {
'UUID-required': True,
'SanDisk_Ultra': {
'uuid': 'usb-SanDisk_Ultra_20060775320F43006019-0:0',
'device_id': 0
},
'sata': {
'UUID-required': False
}
}
}
}
foo.save()
bar = DeviceDictionary.get('foo')
self.assertEqual(bar.parameters, foo.parameters)
def test_match_devices_with_map(self):
devices = Device.objects.filter(status=Device.IDLE).order_by('is_public')
device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname)
device_dict.parameters = { # client, RJ45 10M only
'extends': 'beaglebone-black.jinja2',
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"},
'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"},
'tags': {'eth0': [], 'eth1': ['RJ45', '10M']},
'map': {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}}
}
device_dict.save()
device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname)
device_dict.parameters = { # server includes 100M
'extends': 'cubietruck.jinja2',
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"},
'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"},
'tags': {'eth0': [], 'eth1': ['RJ45', '10M', '100M']},
'map': {'eth0': {'192.168.0.2': 4}, 'eth1': {'192.168.0.2': 6}}
}
device_dict.save()
user = self.factory.make_user()
sample_job_file = os.path.join(os.path.dirname(__file__), 'bbb-cubie-vlan-group.yaml')
with open(sample_job_file, 'r') as test_support:
data = yaml.load(test_support)
del(data['protocols']['lava-multinode']['roles']['client']['tags'])
del(data['protocols']['lava-multinode']['roles']['server']['tags'])
interfaces = []
job_dict = split_multinode_yaml(data, 'abcdefg123456789')
client_job = job_dict['client'][0]
device_dict = DeviceDictionary.get(self.factory.bbb1.hostname).to_dict()
self.assertIsNotNone(device_dict)
tag_list = client_job['protocols']['lava-vland']['vlan_one']['tags']
for interface, tags in device_dict['parameters']['tags'].iteritems():
if set(tags) & set(tag_list) == set(tag_list) and interface not in interfaces:
interfaces.append(interface)
break
self.assertEqual(['eth1'], interfaces)
self.assertEqual(len(interfaces), len(client_job['protocols']['lava-vland'].keys()))
interfaces = []
server_job = job_dict['server'][0]
device_dict = DeviceDictionary.get(self.factory.cubie1.hostname).to_dict()
self.assertIsNotNone(device_dict)
tag_list = server_job['protocols']['lava-vland']['vlan_two']['tags']
for interface, tags in device_dict['parameters']['tags'].iteritems():
if set(tags) & set(tag_list) == set(tag_list) and interface not in interfaces:
interfaces.append(interface)
break
self.assertEqual(['eth1'], interfaces)
self.assertEqual(len(interfaces), len(client_job['protocols']['lava-vland'].keys()))
vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user)
# vlan_one: client role. RJ45 10M. bbb device type
# vlan_two: server role. RJ45 100M. cubie device type.
assignments = {}
for job in vlan_job:
device = find_device_for_job(job, devices)
self.assertEqual(device.device_type, job.requested_device_type)
# map has been defined
self.assertTrue(match_vlan_interface(device, yaml.load(job.definition)))
assignments[job.device_role] = device
self.assertEqual(assignments['client'].hostname, self.factory.bbb1.hostname)
self.assertEqual(assignments['server'].hostname, self.factory.cubie1.hostname)
def make_fake_mustang_device(self, hostname='fakemustang1'): # pylint: disable=no-self-use
mustang = DeviceDictionary(hostname=hostname)
mustang.parameters = {'extends': 'mustang-uefi.yaml'}
mustang.save()
return hostname
def make_fake_mustang_device(self, hostname='fakemustang1'): # pylint: disable=no-self-use
mustang = DeviceDictionary(hostname=hostname)
mustang.parameters = {'extends': 'mustang-uefi.jinja2'}
mustang.save()
return hostname
def test_make_ssh_guest_yaml(self):
hostname = 'fakeqemu3'
device_dict = DeviceDictionary(hostname=hostname)
device_dict.parameters = self.conf
device_dict.save()
device = self.factory.make_device(self.device_type, hostname)
try:
jobs = TestJob.from_yaml_and_user(
self.factory.make_job_json(),
self.factory.make_user())
except DevicesUnavailableException as exc:
self.fail(exc)
sub_id = []
group_size = 0
path = os.path.dirname(os.path.join(__file__))
host_role = []
for job in jobs:
data = yaml.load(job.definition)
params = data['protocols']['lava-multinode']
params['target_group'] = 'replaced'
if not group_size:
group_size = params['group_size']
if job.sub_id.endswith('.0'):
self.assertFalse(job.dynamic_connection)
self.assertEqual(job.requested_device_type.name, device.device_type.name)
self.assertEqual(params['sub_id'], 0)
sub_id.append(params['sub_id'])
self.assertEqual(
data,
yaml.load(open(os.path.join(path, 'qemu-ssh-parent.yaml'), 'r').read())
)
self.assertEqual(job.device_role, 'host')
host_role.append(job.device_role)
else:
self.assertTrue(job.dynamic_connection)
self.assertNotIn(sub_id, params['sub_id'])
sub_id.append(params['sub_id'])
self.assertIsNone(job.requested_device_type)
deploy = [action for action in data['actions'] if 'deploy' in action][0]
self.assertEqual(deploy['deploy']['connection'], 'ssh')
# validate each job
if params['sub_id'] == 1:
self.assertEqual(
data,
yaml.load(open(os.path.join(path, 'qemu-ssh-guest-1.yaml'), 'r').read())
)
elif params['sub_id'] == 2:
self.assertEqual(
data,
yaml.load(open(os.path.join(path, 'qemu-ssh-guest-2.yaml'), 'r').read())
)
else:
self.fail("Unexpected sub_id parameter")
self.assertIsNone(job.requested_device_type)
self.assertIsNone(job.actual_device)
self.assertIsNone(job.requested_device)
host_role.append(data['host_role'])
self.assertFalse(any(role for role in host_role if role != 'host'))
self.assertEqual(len(sub_id), group_size)
self.assertEqual(sub_id, range(0, group_size))
def make_fake_qemu_device(self, hostname='fakeqemu1'): # pylint: disable=no-self-use
qemu = DeviceDictionary(hostname=hostname)
qemu.parameters = {'extends': 'qemu.yaml', 'arch': 'amd64'}
qemu.save()
def import_device_dictionary(self, hostname, jinja_str):
"""
Name
----
`import_device_dictionary` (`device_hostname`, `jinja_string`)
Description
-----------
[superuser only]
Import or update the device dictionary key value store for a
pipeline device.
Arguments
---------
`device_hostname`: string
Device hostname to update.
`jinja_str`: string
Jinja2 settings to store in the DeviceDictionary
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
self._authenticate()
if not self.user.is_superuser:
raise xmlrpclib.Fault(
403,
"User '%s' is not superuser." % self.user.username
)
try:
Device.objects.get(hostname=hostname)
except DeviceType.DoesNotExist:
raise xmlrpclib.Fault(
404, "Device '%s' was not found." % hostname
)
try:
device_data = jinja2_to_devicedictionary(jinja_str)
except (ValueError, KeyError, TypeError):
raise xmlrpclib.Fault(
400, "Unable to parse specified jinja string"
)
if not device_data or 'extends' not in device_data:
raise xmlrpclib.Fault(
400, "Invalid device dictionary content - %s - not updating." % jinja_str
)
try:
template = prepare_jinja_template(hostname, jinja_str, system_path=True)
except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc:
raise xmlrpclib.Fault(
400, "Template error: %s" % exc
)
if not template:
raise xmlrpclib.Fault(400, "Empty template")
element = DeviceDictionary.get(hostname)
msg = ''
if element is None:
msg = "Adding new device dictionary for %s\n" % hostname
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = device_data
element.save()
msg += "Device dictionary updated for %s\n" % hostname
return msg
def import_device_dictionary(self, hostname, jinja_str):
"""
Name
----
`import_device_dictionary` (`device_hostname`, `jinja_string`)
Description
-----------
[superuser only]
Import or update the device dictionary key value store for a
pipeline device.
This action will be logged.
Arguments
---------
`device_hostname`: string
Device hostname to update.
`jinja_str`: string
Jinja2 settings to store in the DeviceDictionary
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
self._authenticate()
if not self.user.is_superuser:
raise xmlrpclib.Fault(
403,
"User '%s' is not superuser." % self.user.username
)
try:
device = Device.objects.get(hostname=hostname)
except DeviceType.DoesNotExist:
raise xmlrpclib.Fault(
404, "Device '%s' was not found." % hostname
)
try:
device_data = jinja2_to_devicedictionary(jinja_str)
except (ValueError, KeyError, TypeError):
raise xmlrpclib.Fault(
400, "Unable to parse specified jinja string"
)
if not device_data or 'extends' not in device_data:
raise xmlrpclib.Fault(
400, "Invalid device dictionary content - %s - not updating." % jinja_str
)
try:
template = prepare_jinja_template(hostname, jinja_str, system_path=True)
except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc:
raise xmlrpclib.Fault(
400, "Template error: %s" % exc
)
if not template:
raise xmlrpclib.Fault(400, "Empty template")
element = DeviceDictionary.get(hostname)
msg = ''
if element is None:
msg = "Adding new device dictionary for %s\n" % hostname
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = device_data
element.save()
msg += "Device dictionary updated for %s\n" % hostname
device.log_admin_entry(self.user, msg)
return msg
def test_context(self):
"""
Test overrides using the job context
Defaults in the device-type can be overridden by the device dictionary.
If not overridden by the device dictionary, can be overridden by the job context.
If the
"""
device = Device.objects.get(hostname="fakeqemu1")
user = self.factory.make_user()
job = TestJob.from_yaml_and_user(
self.factory.make_job_json(), user)
job_def = yaml.load(job.definition)
job_ctx = job_def.get('context', {})
device_config = device.load_device_configuration(job_ctx, system=False) # raw dict
self.assertEqual(
device_config['actions']['boot']['methods']['qemu']['parameters']['command'],
'qemu-system-x86_64'
)
job_data = yaml.load(self.factory.make_job_yaml())
job_data['context'].update({'netdevice': 'tap'})
job_ctx = job_data.get('context', {})
device_config = device.load_device_configuration(job_ctx, system=False) # raw dict
opts = ' '.join(device_config['actions']['boot']['methods']['qemu']['parameters']['options'])
self.assertIn('-net tap', opts)
hostname = "fakemustang"
mustang_type = self.factory.make_device_type('mustang-uefi')
# this sets a qemu device dictionary, so replace it
self.factory.make_device(device_type=mustang_type, hostname=hostname)
mustang = DeviceDictionary(hostname=hostname)
mustang.parameters = {
'extends': 'mustang-uefi.jinja2',
'base_nfsroot_args': '10.16.56.2:/home/lava/debian/nfs/,tcp,hard,intr',
'console_device': 'ttyO0', # takes precedence over the job context as the same var name is used.
}
mustang.save()
device = Device.objects.get(hostname="fakemustang")
self.assertEqual('mustang-uefi', device.device_type.name)
self.assertTrue(device.is_pipeline)
job_ctx = {
'tftp_mac': 'FF:01:00:69:AA:CC',
'nfsroot_args': '172.164.56.2:/home/user/nfs/,tcp,hard,intr',
'console_device': 'ttyAMX0',
}
device_config = device.load_device_configuration(job_ctx, system=False) # raw dict
self.assertIn('uefi-menu', device_config['actions']['boot']['methods'])
self.assertIn('nfs', device_config['actions']['boot']['methods']['uefi-menu'])
menu_data = device_config['actions']['boot']['methods']['uefi-menu']['nfs']
self.assertIn(
job_ctx['nfsroot_args'],
[
item['select']['enter'] for item in menu_data if 'enter' in item['select'] and
'new Entry' in item['select']['wait']][0]
)
self.assertEqual(
[
item['select']['items'][0] for item in menu_data if 'select' in item and
'items' in item['select'] and 'TFTP' in item['select']['items'][0]][0],
'TFTP on MAC Address: FF:01:00:69:AA:CC' # matches the job_ctx
)
# note: the console_device and tftp_mac in the job_ctx has been *ignored* because the device type template
# has not allowed the job_ctx to use a different name for the variable and the variable is
# already defined in the device dictionary using the specified name. If the device dictionary lacked
# the variable, the job could set it to override the device type template default, as shown by the
# override of the base_nfsroot_args by allowing nfsroot_args in the device type template..
self.assertEqual(
[
item['select']['enter'] for item in menu_data if 'select' in item and
'wait' in item['select'] and 'Description' in item['select']['wait']][0],
'console=ttyO0,115200 earlyprintk=uart8250-32bit,0x1c020000 debug '
'root=/dev/nfs rw 172.164.56.2:/home/user/nfs/,tcp,hard,intr ip=dhcp'
)
def test_match_devices_with_map(self):
devices = Device.objects.filter(
status=Device.IDLE).order_by('is_public')
device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname)
device_dict.parameters = { # client, RJ45 10M only
'extends': 'beaglebone-black.jinja2',
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0':
"/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1':
"/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"
},
'mac_addr': {
'eth0': "f0:de:f1:46:8c:21",
'eth1': "00:24:d7:9b:c0:8c"
},
'tags': {
'eth0': [],
'eth1': ['RJ45', '10M']
},
'map': {
'eth0': {
'192.168.0.2': 5
},
'eth1': {
'192.168.0.2': 7
}
}
}
device_dict.save()
device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname)
device_dict.parameters = { # server includes 100M
'extends': 'cubietruck.jinja2',
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0':
"/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1':
"/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"
},
'mac_addr': {
'eth0': "f0:de:f1:46:8c:21",
'eth1': "00:24:d7:9b:c0:8c"
},
'tags': {
'eth0': [],
'eth1': ['RJ45', '10M', '100M']
},
'map': {
'eth0': {
'192.168.0.2': 4
},
'eth1': {
'192.168.0.2': 6
}
}
}
device_dict.save()
user = self.factory.make_user()
sample_job_file = os.path.join(os.path.dirname(__file__),
'bbb-cubie-vlan-group.yaml')
with open(sample_job_file, 'r') as test_support:
data = yaml.load(test_support)
del (data['protocols']['lava-multinode']['roles']['client']['tags'])
del (data['protocols']['lava-multinode']['roles']['server']['tags'])
interfaces = []
job_dict = split_multinode_yaml(data, 'abcdefg123456789')
client_job = job_dict['client'][0]
device_dict = DeviceDictionary.get(
self.factory.bbb1.hostname).to_dict()
self.assertIsNotNone(device_dict)
tag_list = client_job['protocols']['lava-vland']['vlan_one']['tags']
for interface, tags in device_dict['parameters']['tags'].iteritems():
if set(tags) & set(tag_list) == set(
tag_list) and interface not in interfaces:
interfaces.append(interface)
break
self.assertEqual(['eth1'], interfaces)
self.assertEqual(len(interfaces),
len(client_job['protocols']['lava-vland'].keys()))
interfaces = []
server_job = job_dict['server'][0]
device_dict = DeviceDictionary.get(
self.factory.cubie1.hostname).to_dict()
self.assertIsNotNone(device_dict)
tag_list = server_job['protocols']['lava-vland']['vlan_two']['tags']
for interface, tags in device_dict['parameters']['tags'].iteritems():
if set(tags) & set(tag_list) == set(
tag_list) and interface not in interfaces:
interfaces.append(interface)
break
self.assertEqual(['eth1'], interfaces)
self.assertEqual(len(interfaces),
len(client_job['protocols']['lava-vland'].keys()))
vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user)
# vlan_one: client role. RJ45 10M. bbb device type
# vlan_two: server role. RJ45 100M. cubie device type.
assignments = {}
for job in vlan_job:
device = find_device_for_job(job, devices)
self.assertEqual(device.device_type, job.requested_device_type)
# map has been defined
self.assertTrue(
match_vlan_interface(device, yaml.load(job.definition)))
assignments[job.device_role] = device
self.assertEqual(assignments['client'].hostname,
self.factory.bbb1.hostname)
self.assertEqual(assignments['server'].hostname,
self.factory.cubie1.hostname)
def make_fake_qemu_device(self, hostname='fakeqemu1'): # pylint: disable=no-self-use
qemu = DeviceDictionary(hostname=hostname)
qemu.parameters = {'extends': 'qemu.jinja2', 'arch': 'amd64'}
qemu.save()
def test_same_type_devices_with_map(self):
device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname)
device_dict.parameters = { # client, RJ45 10M 100M
'extends': 'beaglebone-black.jinja2',
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0':
"/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1':
"/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"
},
'mac_addr': {
'eth0': "f0:de:f1:46:8c:22",
'eth1': "00:24:d7:9b:c0:8b"
},
'tags': {
'eth0': [],
'eth1': ['RJ45', '10M', '100M']
},
'map': {
'eth0': {
'192.168.0.2': 5
},
'eth1': {
'192.168.0.2': 7
}
}
}
device_dict.save()
bbb2 = self.factory.make_device(self.factory.bbb_type, hostname='bbb2')
device_dict = DeviceDictionary(hostname=bbb2.hostname)
device_dict.parameters = { # server, RJ45 10M 100M
'extends': 'beaglebone-black.jinja2',
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0':
"/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1':
"/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"
},
'mac_addr': {
'eth0': "f0:de:f1:46:8c:21",
'eth1': "00:24:d7:9b:c0:8c"
},
'tags': {
'eth0': [],
'eth1': ['RJ45', '10M', '100M']
},
'map': {
'eth0': {
'192.168.0.2': 7
},
'eth1': {
'192.168.0.2': 9
}
}
}
device_dict.save()
devices = list(
Device.objects.filter(status=Device.IDLE).order_by('is_public'))
user = self.factory.make_user()
sample_job_file = os.path.join(os.path.dirname(__file__),
'bbb-bbb-vland-group.yaml')
with open(sample_job_file, 'r') as test_support:
data = yaml.load(test_support)
vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user)
assignments = {}
for job in vlan_job:
device = find_device_for_job(job, devices)
self.assertEqual(device.device_type, job.requested_device_type)
# map has been defined
self.assertTrue(
match_vlan_interface(device, yaml.load(job.definition)))
assignments[job.device_role] = device
if device in devices:
devices.remove(device)
assign_jobs()
self.factory.bbb1.refresh_from_db()
bbb2.refresh_from_db()
self.assertIsNotNone(self.factory.bbb1.current_job)
self.assertIsNotNone(bbb2.current_job)
self.assertIsNotNone(self.factory.bbb1.current_job.actual_device)
self.assertIsNotNone(bbb2.current_job.actual_device)
self.assertNotEqual(self.factory.bbb1.current_job, bbb2.current_job)
self.assertNotEqual(self.factory.bbb1.current_job.actual_device,
bbb2.current_job.actual_device)
def test_match_devices_with_map_and_tags(self):
devices = Device.objects.filter(
status=Device.IDLE).order_by('is_public')
self.factory.ensure_tag('usb-eth')
self.factory.ensure_tag('sata')
self.factory.bbb1.tags = Tag.objects.filter(name='usb-eth')
self.factory.bbb1.save()
self.factory.cubie1.tags = Tag.objects.filter(name='sata')
self.factory.cubie1.save()
device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname)
device_dict.parameters = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0':
"/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1':
"/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"
},
'mac_addr': {
'eth0': "f0:de:f1:46:8c:21",
'eth1': "00:24:d7:9b:c0:8c"
},
'tags': {
'eth0': [],
'eth1': ['RJ45', '10M']
},
'map': {
'eth0': {
'192.168.0.2': 5
},
'eth1': {
'192.168.0.2': 7
}
}
}
device_dict.save()
device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname)
device_dict.parameters = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0':
"/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1':
"/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"
},
'mac_addr': {
'eth0': "f0:de:f1:46:8c:21",
'eth1': "00:24:d7:9b:c0:8c"
},
'tags': {
'eth0': [],
'eth1': ['RJ45', '100M']
},
'map': {
'eth0': {
'192.168.0.2': 4
},
'eth1': {
'192.168.0.2': 6
}
}
}
device_dict.save()
user = self.factory.make_user()
sample_job_file = os.path.join(os.path.dirname(__file__),
'bbb-cubie-vlan-group.yaml')
with open(sample_job_file, 'r') as test_support:
data = yaml.load(test_support)
vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user)
assignments = {}
for job in vlan_job:
device = find_device_for_job(job, devices)
self.assertEqual(device.device_type, job.requested_device_type)
# map has been defined
self.assertTrue(
match_vlan_interface(device, yaml.load(job.definition)))
assignments[job.device_role] = device
self.assertEqual(assignments['client'].hostname,
self.factory.bbb1.hostname)
self.assertEqual(assignments['server'].hostname,
self.factory.cubie1.hostname)
def test_job_protocols(self):
self.factory.ensure_tag('usb-eth')
self.factory.ensure_tag('sata')
self.factory.bbb1.tags = Tag.objects.filter(name='usb-eth')
self.factory.bbb1.save()
self.factory.cubie1.tags = Tag.objects.filter(name='sata')
self.factory.cubie1.save()
device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname)
device_dict.parameters = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0':
"/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1':
"/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"
},
'mac_addr': {
'eth0': "f0:de:f1:46:8c:21",
'eth1': "00:24:d7:9b:c0:8c"
},
'tags': {
'eth0': ['1G', '10G'],
'eth1': ['1G']
},
'map': {
'eth0': {
'192.168.0.2': 5
},
'eth1': {
'192.168.0.2': 7
}
}
}
device_dict.save()
device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname)
device_dict.parameters = {
'interfaces': ['eth0', 'eth1'],
'sysfs': {
'eth0':
"/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1':
"/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"
},
'mac_addr': {
'eth0': "f0:de:f1:46:8c:21",
'eth1': "00:24:d7:9b:c0:8c"
},
'tags': {
'eth0': ['1G', '10G'],
'eth1': ['1G']
},
'map': {
'eth0': {
'192.168.0.2': 4
},
'eth1': {
'192.168.0.2': 6
}
}
}
device_dict.save()
target_group = "unit-test-only"
job_dict = split_multinode_yaml(self.factory.make_vland_job(),
target_group)
client_job = job_dict['client'][0]
client_handle, client_file_name = tempfile.mkstemp()
yaml.dump(client_job, open(client_file_name, 'w'))
# YAML device file, as required by lava-dispatch --target
device_yaml_file = os.path.realpath(
os.path.join(os.path.dirname(__file__), 'bbb-01.yaml'))
self.assertTrue(os.path.exists(device_yaml_file))
parser = JobParser()
bbb_device = NewDevice(device_yaml_file)
with open(client_file_name) as sample_job_data:
bbb_job = parser.parse(sample_job_data,
bbb_device,
4212,
None,
None,
None,
output_dir='/tmp/')
os.close(client_handle)
os.unlink(client_file_name)
self.assertIn('protocols', bbb_job.parameters)
self.assertIn(VlandProtocol.name, bbb_job.parameters['protocols'])
self.assertIn(MultinodeProtocol.name, bbb_job.parameters['protocols'])
def test_make_ssh_guest_yaml(self):
hostname = 'fakeqemu3'
device_dict = DeviceDictionary(hostname=hostname)
device_dict.parameters = self.conf
device_dict.save()
device = self.factory.make_device(self.device_type, hostname)
try:
jobs = TestJob.from_yaml_and_user(
self.factory.make_job_json(),
self.factory.make_user())
except DevicesUnavailableException as exc:
self.fail(exc)
sub_id = []
group_size = 0
path = os.path.dirname(os.path.join(__file__))
host_role = []
for job in jobs:
data = yaml.load(job.definition)
params = data['protocols']['lava-multinode']
params['target_group'] = 'replaced'
if not group_size:
group_size = params['group_size']
if job.sub_id.endswith('.0'):
self.assertFalse(job.dynamic_connection)
self.assertEqual(job.requested_device_type.name, device.device_type.name)
self.assertEqual(params['sub_id'], 0)
sub_id.append(params['sub_id'])
self.assertEqual(
data,
yaml.load(open(os.path.join(path, 'qemu-ssh-parent.yaml'), 'r').read())
)
self.assertEqual(job.device_role, 'host')
host_role.append(job.device_role)
else:
self.assertTrue(job.dynamic_connection)
self.assertNotIn(sub_id, params['sub_id'])
sub_id.append(params['sub_id'])
self.assertIsNone(job.requested_device_type)
deploy = [action for action in data['actions'] if 'deploy' in action][0]
self.assertEqual(deploy['deploy']['connection'], 'ssh')
# validate each job
if params['sub_id'] == 1:
self.assertEqual(
data,
yaml.load(open(os.path.join(path, 'qemu-ssh-guest-1.yaml'), 'r').read())
)
elif params['sub_id'] == 2:
self.assertEqual(
data,
yaml.load(open(os.path.join(path, 'qemu-ssh-guest-2.yaml'), 'r').read())
)
else:
self.fail("Unexpected sub_id parameter")
self.assertIsNone(job.requested_device_type)
self.assertIsNone(job.actual_device)
self.assertIsNone(job.requested_device)
host_role.append(data['host_role'])
self.assertFalse(any(role for role in host_role if role != 'host'))
self.assertEqual(len(sub_id), group_size)
self.assertEqual(sub_id, range(0, group_size))
def test_differing_vlan_tags(self):
"""
More devices of the requested type than needed by the test
with some devices having unsuitable vland interface tags.
"""
x86 = self.factory.make_device_type('x86')
x86_1 = self.factory.make_device(device_type=x86, hostname='x86-1')
device_dict = DeviceDictionary(hostname=x86_1.hostname)
device_dict.parameters = { # client, RJ45 10M 100M with separate 10G
'extends': 'x86.jinja2',
'interfaces': ['eth0', 'eth1', 'eth2'],
'sysfs': {
'eth0':
"/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1':
"/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1",
'eth2':
"/sys/devices/pci0000:00/0000:00:1c.2/0000:04:00.0/net/eth2"
},
'mac_addr': {
'eth0': "f0:de:f1:46:8c:22",
'eth1': "00:24:d7:9b:c0:8b",
'eth2': "00:24:d7:9b:c0:8c"
},
'tags': {
'eth0': [],
'eth1': ['RJ45', '10M', '100M', '1G'],
'eth2': ['SFP+', '10G']
},
'map': {
'eth0': {
'192.168.0.2': 5
},
'eth1': {
'192.168.0.2': 7
},
'eth2': {
'192.168.0.2': 12
}
}
}
device_dict.save()
x86_2 = self.factory.make_device(device_type=x86, hostname='x86-2')
device_dict = DeviceDictionary(hostname=x86_2.hostname)
device_dict.parameters = { # client, RJ45 10M 100M with separate 10G
'extends': 'x86.jinja2',
'interfaces': ['eth0', 'eth1', 'eth2'],
'sysfs': {
'eth0':
"/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1':
"/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1",
'eth2':
"/sys/devices/pci0000:00/0000:00:1c.2/0000:04:00.0/net/eth2"
},
'mac_addr': {
'eth0': "f0:de:f1:46:8c:22",
'eth1': "00:24:d7:9b:d0:8b",
'eth2': "00:24:d7:9b:d0:8c"
},
'tags': {
'eth0': [],
'eth1': ['RJ45', '10M', '100M', '1G'],
'eth2': ['SFP+', '10G']
},
'map': {
'eth0': {
'192.168.0.2': 14
},
'eth1': {
'192.168.0.2': 17
},
'eth2': {
'192.168.0.2': 22
}
}
}
device_dict.save()
x86_3 = self.factory.make_device(device_type=x86, hostname='x86-3')
device_dict = DeviceDictionary(hostname=x86_3.hostname)
device_dict.parameters = { # client, 40G, not 10G
'extends': 'x86.jinja2',
'interfaces': ['eth0', 'eth1', 'eth2'],
'sysfs': {
'eth0':
"/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1':
"/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1",
'eth2':
"/sys/devices/pci0000:00/0000:00:1c.2/0000:04:00.0/net/eth2"
},
'mac_addr': {
'eth0': "f0:de:f1:46:8c:22",
'eth1': "00:24:d7:9b:d0:8b",
'eth2': "00:24:d7:9b:d0:8c"
},
'tags': {
'eth0': [],
'eth1': ['RJ45', '10M', '100M', '1G'],
'eth2': ['QSFP+', '40G']
},
'map': {
'eth0': {
'192.168.0.2': 15
},
'eth1': {
'192.168.0.2': 16
},
'eth2': {
'192.168.0.2': 20
}
}
}
device_dict.save()
x86_4 = self.factory.make_device(device_type=x86, hostname='x86-4')
device_dict = DeviceDictionary(hostname=x86_4.hostname)
device_dict.parameters = { # client, RJ45 10M 100M with separate 10G
'extends': 'x86.jinja2',
'interfaces': ['eth0', 'eth1', 'eth2'],
'sysfs': {
'eth0':
"/sys/devices/pci0000:00/0000:00:19.0/net/eth0",
'eth1':
"/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1",
'eth2':
"/sys/devices/pci0000:00/0000:00:1c.2/0000:04:00.0/net/eth2"
},
'mac_addr': {
'eth0': "f0:de:f1:46:8c:22",
'eth1': "00:24:d7:9b:d0:8b",
'eth2': "00:24:d7:9b:d0:8c"
},
'tags': {
'eth0': [],
'eth1': ['RJ45', '10M', '100M', '1G'],
'eth2': ['SFP+', '10G']
},
'map': {
'eth0': {
'192.168.0.2': 14
},
'eth1': {
'192.168.0.2': 17
},
'eth2': {
'192.168.0.2': 22
}
}
}
device_dict.save()
devices = list(
Device.objects.filter(status=Device.IDLE).order_by('is_public'))
user = self.factory.make_user()
sample_job_file = os.path.join(os.path.dirname(__file__),
'sample_jobs', 'x86-vlan.yaml')
with open(sample_job_file, 'r') as test_support:
data = yaml.load(test_support)
vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user)
assignments = {}
for job in vlan_job:
device = find_device_for_job(job, devices)
self.assertIsNotNone(device)
self.assertEqual(device.device_type, job.requested_device_type)
# map has been defined
self.assertTrue(
match_vlan_interface(device, yaml.load(job.definition)))
assignments[job.device_role] = device
if device in devices:
devices.remove(device)
assign_jobs()
# reset state, pretend the assigned jobs have completed.
for job in TestJob.objects.all():
job.status = TestJob.COMPLETE
job.actual_device.status = Device.IDLE
job.actual_device.current_job = None
job.actual_device.save(update_fields=['status', 'current_job'])
job.save(update_fields=['status'])
# take x86_1 offline, forcing the idle list to include x86_3 for evaluation
x86_1.status = Device.OFFLINE
x86_1.save(update_fields=['status'])
x86_1.refresh_from_db()
devices = list(
Device.objects.filter(status=Device.IDLE).order_by('is_public'))
self.assertNotIn(x86_1, devices)
self.assertIn(x86_2, devices)
self.assertIn(x86_3, devices)
self.assertIn(x86_4, devices)
user = self.factory.make_user()
sample_job_file = os.path.join(os.path.dirname(__file__),
'sample_jobs', 'x86-vlan.yaml')
with open(sample_job_file, 'r') as test_support:
data = yaml.load(test_support)
vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user)
assignments = {}
for job in vlan_job:
device = find_device_for_job(job, devices)
self.assertIsNotNone(device)
self.assertEqual(device.device_type, job.requested_device_type)
# map has been defined
self.assertTrue(
match_vlan_interface(device, yaml.load(job.definition)))
assignments[job.device_role] = device
if device in devices:
devices.remove(device)
assign_jobs()
x86_1.refresh_from_db()
x86_2.refresh_from_db()
x86_3.refresh_from_db()
x86_4.refresh_from_db()
self.assertEqual(Device.STATUS_CHOICES[Device.OFFLINE],
Device.STATUS_CHOICES[x86_1.status])
self.assertEqual(Device.STATUS_CHOICES[Device.RESERVED],
Device.STATUS_CHOICES[x86_2.status])
self.assertEqual(Device.STATUS_CHOICES[Device.IDLE],
Device.STATUS_CHOICES[x86_3.status])
self.assertEqual(Device.STATUS_CHOICES[Device.RESERVED],
Device.STATUS_CHOICES[x86_4.status])