def step(context, name):
import os
import json
from chroma_core.models import Nid
from tests.unit.chroma_core.helpers import MockAgentRpc
# Skip setup if it was already done in a previous scenario.
if len(MockAgentRpc.mock_servers) > 0:
return
path = os.path.join(os.path.dirname(__file__),
"../../../../sample_data/%s.json" % name)
with open(path) as fh:
data = json.load(fh)
for host in data["hosts"]:
host["nids"] = [Nid.split_nid_string(n) for n in host["nids"]]
MockAgentRpc.mock_servers[host["address"]] = host
def load_filesystem_from_json(data):
# Since this is only ever used for the behave tests, and the behave tests
# are slated to be decommissioned at some point, we're just going to
# abandon all pretense that we might be loading a non-synthetic cluster.
from chroma_core.services.job_scheduler.job_scheduler_client import JobSchedulerClient
from tests.unit.chroma_core.helpers import synthetic_volume
from chroma_core.models import ManagedMgs, VolumeNode
from chroma_core.lib.cache import ObjectCache
from chroma_core.models import ManagedHost, ManagedTarget, ManagedTargetMount
from chroma_core.models import Nid
from tests.unit.chroma_core.helpers import synthetic_host
lookup = defaultdict(dict)
for host_info in data["hosts"]:
from tests.unit.chroma_core.helpers import MockAgentRpc
mock_host_info = MockAgentRpc.mock_servers[host_info["address"]]
# host, command = JobSchedulerClient.create_host(mock_host_info['fqdn'], mock_host_info['nodename'], ['manage_targets'], address = host_info['address'])
nids = [Nid.split_nid_string(n) for n in mock_host_info["nids"]]
# update mock_servers with list of Nid objects
MockAgentRpc.mock_servers[host_info["address"]]["nids"] = nids
host = synthetic_host(mock_host_info["address"],
nids=nids,
fqdn=mock_host_info["fqdn"],
nodename=mock_host_info["nodename"])
ObjectCache.add(ManagedHost, host)
host.state = "managed"
host.save()
lookup["hosts"][host_info["address"]] = host
def _create_volume_for_mounts(mounts):
# The test data doesn't give us a serial, so mung one out of the device paths
# on the basis that they're iSCSI-style
serial = mounts[0]["device_node"].split("/")[-1]
volume = synthetic_volume(serial=serial)
for mount in mounts:
VolumeNode.objects.create(host=lookup["hosts"][mount["host"]],
path=mount["device_node"],
primary=mount["primary"],
volume=volume)
return volume
for mgs_info in data["mgss"]:
volume = _create_volume_for_mounts(mgs_info["mounts"])
target, target_mounts = ManagedMgs.create_for_volume(volume.id)
target.uuid = uuid.uuid4().__str__()
target.ha_label = "%s_%s" % (target.name, uuid.uuid4().__str__()[0:6])
ObjectCache.add(ManagedTarget, target.managedtarget_ptr)
for tm in target_mounts:
ObjectCache.add(ManagedTargetMount, tm)
lookup["mgt"][mgs_info["mounts"][0]["host"]] = target
for fs_info in data["filesystems"]:
fs_bundle = {
"name": fs_info["name"],
"mgt": {
"id": lookup["mgt"][fs_info["mgs"]].id
},
"mdts": [],
"osts": [],
"conf_params": {},
}
for mdt_info in fs_info["mdts"]:
volume = _create_volume_for_mounts(mdt_info["mounts"])
# Although we create multiple volumes for the mdt, initially only add a single volume, once delete of MDT
# is possible we can add them all and then delete before adding. This code is structured for that fixed case.
if not fs_bundle["mdts"]:
fs_bundle["mdts"].append({
"volume_id": volume.id,
"conf_params": {}
})
for ost_info in fs_info["osts"]:
volume = _create_volume_for_mounts(ost_info["mounts"])
fs_bundle["osts"].append({
"volume_id": volume.id,
"conf_params": {}
})
fs, command = JobSchedulerClient.create_filesystem(fs_bundle)
def test_combined_mgs_mdt(self, invoke):
invoke.return_value = '{"Ok": []}'
def fixture_glob(g):
return glob.glob(
os.path.join(os.path.dirname(__file__),
"fixtures/test_combined_mgs_mdt", g))
for path in fixture_glob("*_nid.txt"):
address = os.path.basename(path).split("_")[0]
nids = [
Nid.split_nid_string(l.strip())
for l in open(path).readlines()
]
synthetic_host(address, nids)
host_data = {}
for path in fixture_glob("*detect_scan_output.txt"):
address = os.path.basename(path).split("_")[0]
data = json.load(open(path))["result"]
host_data[ManagedHost.objects.get(address=address)] = data
# Simplified volume construction:
# * Assume all device paths referenced in detection exist
# * Assume all devices visible on all hosts
# * Assume device node paths are identical on all hosts
devpaths = set()
for host, data in host_data.items():
for lt in data["local_targets"]:
for d in lt["device_paths"]:
if not d in devpaths:
devpaths.add(d)
volume = Volume.objects.create()
for host in host_data.keys():
VolumeNode.objects.create(volume=volume,
path=d,
host=host)
def _detect_scan_device_plugin(host, command, args=None):
self.assertIn(command, ["detect_scan", "device_plugin"])
if command == "detect_scan":
return host_data[host]
raise AgentException(
host, command, args,
"No device plugin data available in unit tests")
job = DetectTargetsJob.objects.create()
with mock.patch("chroma_core.lib.job.Step.invoke_agent",
new=mock.Mock(side_effect=_detect_scan_device_plugin)):
with mock.patch("chroma_core.models.Volume.storage_resource"):
synchronous_run_job(job)
self.assertEqual(ManagedFilesystem.objects.count(), 1)
self.assertEqual(ManagedFilesystem.objects.get().name, "test18fs")
self.assertEqual(ManagedOst.objects.count(), 8)
for t in ManagedTarget.objects.all():
self.assertEqual(t.immutable_state, True)
def assertMount(target_name, primary_host, failover_hosts=list()):
target = ManagedTarget.objects.get(name=target_name)
self.assertEquals(
ManagedTargetMount.objects.filter(target=target).count(),
1 + len(failover_hosts))
self.assertEquals(
ManagedTargetMount.objects.filter(
target=target,
primary=True,
host=ManagedHost.objects.get(
address=primary_host)).count(),
1,
)
for h in failover_hosts:
self.assertEquals(
ManagedTargetMount.objects.filter(
target=target,
primary=False,
host=ManagedHost.objects.get(address=h)).count(),
1,
)
assertMount("MGS", "kp-lustre-1-8-mgs-1")
assertMount("test18fs-MDT0000", "kp-lustre-1-8-mgs-1")
assertMount("test18fs-OST0000", "kp-lustre-1-8-oss-1")
assertMount("test18fs-OST0001", "kp-lustre-1-8-oss-1")
assertMount("test18fs-OST0002", "kp-lustre-1-8-oss-2")
assertMount("test18fs-OST0003", "kp-lustre-1-8-oss-2")
assertMount("test18fs-OST0004", "kp-lustre-1-8-oss-3")
assertMount("test18fs-OST0005", "kp-lustre-1-8-oss-3")
assertMount("test18fs-OST0006", "kp-lustre-1-8-oss-4")
assertMount("test18fs-OST0007", "kp-lustre-1-8-oss-4")
self.assertEqual(ManagedFilesystem.objects.get().state, "available")