def test_design_inheritance(self, input_files, setup):
input_file = input_files.join("fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
ingester = Ingester()
ingester.enable_plugin(
'drydock_provisioner.ingester.plugins.yaml.YamlIngester')
orchestrator = Orchestrator(
state_manager=design_state, ingester=ingester)
design_status, design_data = orchestrator.get_effective_site(
design_ref)
assert len(design_data.baremetal_nodes) == 2
node = design_data.get_baremetal_node("controller01")
assert node.hardware_profile == 'HPGen9v3'
iface = node.get_applied_interface('bond0')
assert len(iface.get_hw_slaves()) == 2
iface = node.get_applied_interface('pxe')
assert len(iface.get_hw_slaves()) == 1
def test_bootaction_tarbuilder(self, input_files, deckhand_ingester,
setup):
objects.register_all()
input_file = input_files.join("deckhand_fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
design_status, design_data = deckhand_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
target_host = 'compute01'
ba = design_data.get_bootaction('helloworld')
action_id = ulid2.generate_binary_ulid()
assets = ba.render_assets(target_host, design_data, action_id)
assert len(assets) > 0
tarbytes = BootactionUtils.tarbuilder(assets)
assert tarbytes is not None
fileobj = io.BytesIO(tarbytes)
tarball = tarfile.open(mode='r:gz', fileobj=fileobj)
tarasset = tarball.getmember('/var/tmp/hello.sh')
assert tarasset.mode == 0o555
def test_node_filter_by_nodelabel(self, input_files, setup,
deckhand_orchestrator,
deckhand_ingester):
input_file = input_files.join("deckhand_fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
design_status, design_data = deckhand_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
nfs = {
'filter_set_type':
'intersection',
'filter_set': [
{
'filter_type': 'intersection',
'node_labels': {
'foo': 'baz'
},
},
],
}
node_list = deckhand_orchestrator.process_node_filter(nfs, design_data)
assert len(node_list) == 1
def test_node_filter_dict(self, input_files, setup, test_orchestrator):
input_file = input_files.join("fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
ingester = Ingester()
ingester.enable_plugin(
'drydock_provisioner.ingester.plugins.yaml.YamlIngester')
design_status, design_data = ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
nfs = {
'filter_set_type':
'intersection',
'filter_set': [
{
'filter_type': 'intersection',
'node_names': 'compute01',
},
],
}
node_list = test_orchestrator.process_node_filter(nfs, design_data)
assert len(node_list) == 1
def test_valid_param_reference(self, deckhand_ingester, input_files,
setup):
input_file = input_files.join("deckhand_fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
orchestrator = Orchestrator(state_manager=design_state,
ingester=deckhand_ingester)
design_status, design_data = orchestrator.get_effective_site(
design_ref)
assert len(design_data.baremetal_nodes) == 3
node = design_data.get_baremetal_node("compute01")
assert node.hardware_profile == 'HPGen9v3'
isolcpu = node.kernel_params.get('isolcpus', None)
# '2,4' is defined in the HardwareProfile as cpu_sets.sriov
assert isolcpu == '2,4'
hugepagesz = node.kernel_params.get('hugepagesz', None)
assert hugepagesz == '1G'
def parse_design(self, filename, input_files, deckhand_ingester):
input_file = input_files.join(filename)
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
return deckhand_ingester.ingest_data(design_state, design_ref)
def test_routedomain_omit_source_subnet(self, input_files, setup):
input_file = input_files.join("deckhand_routedomain.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
ingester = Ingester()
ingester.enable_plugin(
'drydock_provisioner.ingester.plugins.deckhand.DeckhandIngester')
orchestrator = Orchestrator(state_manager=design_state,
ingester=ingester)
design_status, design_data = orchestrator.get_effective_site(
design_ref)
assert design_status.status == hd_fields.ValidationResult.Success
net_rack3 = design_data.get_network('storage_rack3')
route_cidrs = list()
for r in net_rack3.routes:
if 'subnet' in r and r.get('subnet') is not None:
route_cidrs.append(r.get('subnet'))
assert '172.16.3.0/24' not in route_cidrs
def test_ingest_yaml(self, input_files, setup, yaml_ingester):
input_file = input_files.join("fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
design_status, design_data = yaml_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
assert design_status.status == objects.fields.ValidationResult.Success
assert len(design_data.host_profiles) == 2
assert len(design_data.baremetal_nodes) == 2
def test_rack_not_found(self, deckhand_ingester, input_files, setup):
objects.register_all()
input_file = input_files.join("deckhand_fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
design_status, design_data = deckhand_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
with pytest.raises(errors.DesignError):
design_data.get_rack('foo')
def test_no_baremetal_nodes(self, input_files, setup,
deckhand_orchestrator, deckhand_ingester):
input_file = input_files.join("deckhand_fullsite_no_nodes.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
design_status, design_data = deckhand_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
node_list = deckhand_orchestrator.process_node_filter(
None, design_data)
assert node_list == []
def test_ingest_deckhand_docref_exists(self, input_files, setup,
deckhand_ingester):
"""Test that each processed document has a doc_ref."""
input_file = input_files.join('deckhand_fullsite.yaml')
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
design_status, design_data = deckhand_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
assert design_status.status == objects.fields.ValidationResult.Success
for p in design_data.host_profiles:
assert p.doc_ref is not None
assert p.doc_ref.doc_schema == 'drydock/HostProfile/v1'
assert p.doc_ref.doc_name is not None
def test_ingest_deckhand(self, input_files, setup, deckhand_ingester):
objects.register_all()
input_file = input_files.join("deckhand_fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
design_status, design_data = deckhand_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
print("%s" % str(design_status.to_dict()))
assert design_status.status == objects.fields.ActionResult.Success
assert len(design_data.host_profiles) == 2
assert len(design_data.baremetal_nodes) == 2
def test_bootaction_parse(self, input_files, deckhand_ingester, setup):
objects.register_all()
input_file = input_files.join("invalid_bootaction.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
design_status, design_data = deckhand_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
assert design_status.status == objects.fields.ActionResult.Failure
print(str(design_status.to_dict()))
error_msgs = [m for m in design_status.message_list if m.error]
assert len(error_msgs) == 2
def test_rack_parse(self, deckhand_ingester, input_files, setup):
objects.register_all()
input_file = input_files.join("deckhand_fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
design_status, design_data = deckhand_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
print("%s" % str(design_status.to_dict()))
assert design_status.status == objects.fields.ActionResult.Success
rack = design_data.get_rack('rack1')
assert rack.location.get('grid') == 'EG12'
def test_rack_not_found(self, input_files, setup):
objects.register_all()
input_file = input_files.join("fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
ingester = Ingester()
ingester.enable_plugin(
'drydock_provisioner.ingester.plugins.yaml.YamlIngester')
design_status, design_data = ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
with pytest.raises(errors.DesignError):
design_data.get_rack('foo')
def test_ingest_full_site(self, input_files, setup):
objects.register_all()
input_file = input_files.join("fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
ingester = Ingester()
ingester.enable_plugin(
'drydock_provisioner.ingester.plugins.yaml.YamlIngester')
design_status, design_data = ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
assert len(design_data.host_profiles) == 2
assert len(design_data.baremetal_nodes) == 2
def test_bootaction_render(self, input_files, deckhand_ingester, setup):
objects.register_all()
input_file = input_files.join("deckhand_fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
design_status, design_data = deckhand_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
ba = design_data.get_bootaction('helloworld')
action_id = ulid2.generate_binary_ulid()
assets = ba.render_assets('compute01', design_data, action_id)
assert 'compute01' in assets[0].rendered_bytes.decode('utf-8')
def test_bootaction_parse(self, input_files, setup):
objects.register_all()
input_file = input_files.join("bootaction.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
ingester = Ingester()
ingester.enable_plugin(
'drydock_provisioner.ingester.plugins.yaml.YamlIngester')
design_status, design_data = ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
ba = design_data.get_bootaction('helloworld')
assert len(ba.asset_list) == 2
def test_bootaction_render_nodename(self, input_files, deckhand_ingester,
setup):
"""Test the bootaction render routine provides expected output."""
input_file = input_files.join("deckhand_fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
design_status, design_data = deckhand_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
ba = design_data.get_bootaction('helloworld')
action_id = ulid2.generate_binary_ulid()
assets = ba.render_assets('compute01', design_data, action_id,
design_ref)
assert 'compute01' in assets[0].rendered_bytes.decode('utf-8')
def test_rack_parse(self, input_files, setup):
objects.register_all()
input_file = input_files.join("fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
ingester = Ingester()
ingester.enable_plugin(
'drydock_provisioner.ingester.plugins.yaml.YamlIngester')
design_status, design_data = ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
rack = design_data.get_rack('rack1')
assert rack.location.get('grid') == 'EG12'
def test_node_filter_obj(self, input_files, setup, deckhand_orchestrator,
deckhand_ingester):
input_file = input_files.join("deckhand_fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
design_status, design_data = deckhand_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
nf = objects.NodeFilter()
nf.filter_type = 'intersection'
nf.node_names = ['compute01']
nfs = objects.NodeFilterSet(
filter_set_type='intersection', filter_set=[nf])
node_list = deckhand_orchestrator.process_node_filter(nfs, design_data)
assert len(node_list) == 1
def test_ingest_deckhand_repos(self, input_files, setup,
deckhand_ingester):
"""Test that the ingester properly parses repo definitions."""
input_file = input_files.join("deckhand_fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
design_status, design_data = deckhand_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
assert design_status.status == objects.fields.ValidationResult.Success
region_def = design_data.get_site()
assert len(region_def.repositories) == 1
assert region_def.repositories.remove_unlisted
for r in region_def.repositories:
assert 'docker' in r.url
def test_bootaction_render_key(self, input_files, deckhand_ingester,
setup):
"""Test that a bootaction can render the correct action_id and
action_key needed by the signalling API."""
input_file = input_files.join("deckhand_fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
design_status, design_data = deckhand_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
ba = design_data.get_bootaction('helloworld')
action_id = ulid2.generate_binary_ulid()
action_key = os.urandom(32)
assets = ba.render_assets('compute01', design_data, action_id,
action_key, design_ref)
assert action_key.hex() in assets[2].rendered_bytes.decode('utf-8')
assert ulid2.ulid_to_base32(
action_id) in assets[2].rendered_bytes.decode('utf-8')
def test_node_filter_obj(self, input_files, setup, test_orchestrator):
input_file = input_files.join("fullsite.yaml")
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
ingester = Ingester()
ingester.enable_plugin(
'drydock_provisioner.ingester.plugins.yaml.YamlIngester')
design_status, design_data = ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
nf = objects.NodeFilter()
nf.filter_type = 'intersection'
nf.node_names = ['compute01']
nfs = objects.NodeFilterSet(filter_set_type='intersection',
filter_set=[nf])
node_list = test_orchestrator.process_node_filter(nfs, design_data)
assert len(node_list) == 1
def drydock_state(setup):
state_mgr = DrydockState()
state_mgr.connect_db()
return state_mgr
def start_drydock(enable_keystone=True):
objects.register_all()
# Setup configuration parsing
cli_options = [
cfg.BoolOpt('debug',
short='d',
default=False,
help='Enable debug logging'),
]
config.config_mgr.conf.register_cli_opts(cli_options)
config.config_mgr.register_options(enable_keystone=enable_keystone)
config.config_mgr.conf(sys.argv[1:])
if config.config_mgr.conf.debug:
config.config_mgr.conf.set_override(name='log_level',
override='DEBUG',
group='logging')
# Setup root logger
logger = logging.getLogger(
config.config_mgr.conf.logging.global_logger_name)
logger.setLevel(config.config_mgr.conf.logging.log_level)
ch = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(levelname)s - %(filename)s:%(funcName)s - %(message)s'
)
ch.setFormatter(formatter)
logger.addHandler(ch)
# Specalized format for API logging
logger = logging.getLogger(
config.config_mgr.conf.logging.control_logger_name)
logger.propagate = False
formatter = logging.Formatter(
'%(asctime)s - %(levelname)s - %(user)s - %(req_id)s - %(external_ctx)s - %(message)s'
)
ch = logging.StreamHandler()
ch.setFormatter(formatter)
logger.addHandler(ch)
state = DrydockState()
state.connect_db()
input_ingester = Ingester()
input_ingester.enable_plugin(config.config_mgr.conf.plugins.ingester)
orchestrator = Orchestrator(enabled_drivers=config.config_mgr.conf.plugins,
state_manager=state,
ingester=input_ingester)
orch_thread = threading.Thread(target=orchestrator.watch_for_tasks)
orch_thread.start()
# Check if we have an API key in the environment
# Hack around until we move MaaS configs to the YAML schema
if 'MAAS_API_KEY' in os.environ:
config.config_mgr.conf.set_override(
name='maas_api_key',
override=os.environ['MAAS_API_KEY'],
group='maasdriver')
# Setup the RBAC policy enforcer
policy.policy_engine = policy.DrydockPolicy()
policy.policy_engine.register_policy()
# Ensure that the policy_engine is initialized before starting the API
wsgi_callable = api.start_api(state_manager=state,
ingester=input_ingester,
orchestrator=orchestrator)
# Now that loggers are configured, log the effective config
config.config_mgr.conf.log_opt_values(
logging.getLogger(config.config_mgr.conf.logging.global_logger_name),
logging.DEBUG)
return wsgi_callable