def _dataset_info(self, child, attr_name):
dataset_name_attr = attr_name + "DatasetName"
dataset_type_attr = attr_name + "DatasetType"
if dataset_name_attr in child and dataset_type_attr in child:
dataset_name = child[dataset_name_attr].value
if dataset_name == "":
return
assert "." not in dataset_name, \
"Dataset name should not contain '.'"
dataset_type = child[dataset_type_attr].value
uppercase_attr = re.sub("([A-Z])", r"_\1", attr_name).upper()
return NDAttributeDatasetInfo(name=dataset_name,
type=dataset_type,
rank=2,
attr="%s.%s" %
(self.name, uppercase_attr))
def test_configure(self):
energy = LineGenerator("energy", "kEv", 13.0, 15.2, 2)
spiral = SpiralGenerator(["x", "y"], ["mm", "mm"], [0., 0.],
5.,
scale=2.0)
generator = CompoundGenerator([energy, spiral], [], [], 0.1)
generator.prepare()
fileDir = "/tmp"
formatName = "xspress3"
fileTemplate = "thing-%s.h5"
completed_steps = 0
steps_to_do = 38
part_info = {
"DET": [NDArrayDatasetInfo(2)],
"PANDA": [
NDAttributeDatasetInfo("I0", AttributeDatasetType.DETECTOR,
"COUNTER1.COUNTER", 2),
NDAttributeDatasetInfo("It", AttributeDatasetType.MONITOR,
"COUNTER2.COUNTER", 2),
NDAttributeDatasetInfo("t1x", AttributeDatasetType.POSITION,
"INENC1.VAL", 2)
],
"STAT": [CalculatedNDAttributeDatasetInfo("sum", "StatsTotal")],
}
infos = self.o.configure(self.context, completed_steps, steps_to_do,
part_info, generator, fileDir, formatName,
fileTemplate)
assert len(infos) == 8
assert infos[0].name == "xspress3.data"
assert infos[0].filename == "thing-xspress3.h5"
assert infos[0].type == DatasetType.PRIMARY
assert infos[0].rank == 4
assert infos[0].path == "/entry/detector/detector"
assert infos[0].uniqueid == "/entry/NDAttributes/NDArrayUniqueId"
assert infos[1].name == "xspress3.sum"
assert infos[1].filename == "thing-xspress3.h5"
assert infos[1].type == DatasetType.SECONDARY
assert infos[1].rank == 4
assert infos[1].path == "/entry/sum/sum"
assert infos[1].uniqueid == "/entry/NDAttributes/NDArrayUniqueId"
assert infos[2].name == "I0.data"
assert infos[2].filename == "thing-xspress3.h5"
assert infos[2].type == DatasetType.PRIMARY
assert infos[2].rank == 4
assert infos[2].path == "/entry/I0/I0"
assert infos[2].uniqueid == "/entry/NDAttributes/NDArrayUniqueId"
assert infos[3].name == "It.data"
assert infos[3].filename == "thing-xspress3.h5"
assert infos[3].type == DatasetType.MONITOR
assert infos[3].rank == 4
assert infos[3].path == "/entry/It/It"
assert infos[3].uniqueid == "/entry/NDAttributes/NDArrayUniqueId"
assert infos[4].name == "t1x.value"
assert infos[4].filename == "thing-xspress3.h5"
assert infos[4].type == DatasetType.POSITION_VALUE
assert infos[4].rank == 4
assert infos[4].path == "/entry/t1x/t1x"
assert infos[4].uniqueid == "/entry/NDAttributes/NDArrayUniqueId"
assert infos[5].name == "energy.value_set"
assert infos[5].filename == "thing-xspress3.h5"
assert infos[5].type == DatasetType.POSITION_SET
assert infos[5].rank == 1
assert infos[5].path == "/entry/detector/energy_set"
assert infos[5].uniqueid == ""
assert infos[6].name == "x.value_set"
assert infos[6].filename == "thing-xspress3.h5"
assert infos[6].type == DatasetType.POSITION_SET
assert infos[6].rank == 1
assert infos[6].path == "/entry/detector/x_set"
assert infos[6].uniqueid == ""
assert infos[7].name == "y.value_set"
assert infos[7].filename == "thing-xspress3.h5"
assert infos[7].type == DatasetType.POSITION_SET
assert infos[7].rank == 1
assert infos[7].path == "/entry/detector/y_set"
assert infos[7].uniqueid == ""
expected_xml_filename = "/tmp/BLOCK-HDF5-layout.xml"
# Wait for the start_future so the post gets through to our child
# even on non-cothread systems
self.o.start_future.result(timeout=1)
assert self.child.handled_requests.mock_calls == [
call.put('positionMode', True),
call.put('arrayCounter', 0),
call.put('dimAttDatasets', True),
call.put('enableCallbacks', True),
call.put('fileName', 'xspress3'),
call.put('filePath', '/tmp/'),
call.put('fileTemplate', '%sthing-%s.h5'),
call.put('fileWriteMode', 'Stream'),
call.put('lazyOpen', True),
call.put('positionMode', True),
call.put('swmrMode', True),
call.put('extraDimSize3', 1),
call.put('extraDimSize4', 1),
call.put('extraDimSize5', 1),
call.put('extraDimSize6', 1),
call.put('extraDimSize7', 1),
call.put('extraDimSize8', 1),
call.put('extraDimSize9', 1),
call.put('extraDimSizeN', 20),
call.put('extraDimSizeX', 2),
call.put('extraDimSizeY', 1),
call.put('numExtraDims', 1),
call.put('posNameDim3', ''),
call.put('posNameDim4', ''),
call.put('posNameDim5', ''),
call.put('posNameDim6', ''),
call.put('posNameDim7', ''),
call.put('posNameDim8', ''),
call.put('posNameDim9', ''),
call.put('posNameDimN', 'd1'),
call.put('posNameDimX', 'd0'),
call.put('posNameDimY', ''),
call.put('flushAttrPerNFrames', 10.0),
call.put('flushDataPerNFrames', 10.0),
call.put('xml', expected_xml_filename),
call.put('numCapture', 0),
call.post('start')
]
expected_xml = """<?xml version="1.0" ?>
<hdf5_layout>
<group name="entry">
<attribute name="NX_class" source="constant" type="string" value="NXentry" />
<group name="detector">
<attribute name="signal" source="constant" type="string" value="detector" />
<attribute name="axes" source="constant" type="string" value="energy_set,.,.,." />
<attribute name="NX_class" source="constant" type="string" value="NXdata" />
<attribute name="energy_set_indices" source="constant" type="string" value="0" />
<dataset name="energy_set" source="constant" type="float" value="13,15.2">
<attribute name="units" source="constant" type="string" value="kEv" />
</dataset>
<attribute name="x_set_indices" source="constant" type="string" value="1" />
<dataset name="x_set" source="constant" type="float" value="0.473264298891,-1.28806365331,-1.11933765723,0.721339144968,2.26130106714,2.3717213098,1.08574712174,-0.863941392256,-2.59791589857,-3.46951769442,-3.22399679412,-1.98374931946,-0.132541097885,1.83482458567,3.45008680308,4.36998121172,4.42670524204,3.63379270355,2.15784413199,0.269311496406">
<attribute name="units" source="constant" type="string" value="mm" />
</dataset>
<attribute name="y_set_indices" source="constant" type="string" value="1" />
<dataset name="y_set" source="constant" type="float" value="-0.64237113553,-0.500750778455,1.38930992616,1.98393756064,0.784917470231,-1.17377831157,-2.66405897615,-2.9669684623,-2.01825893141,-0.24129368636,1.72477821509,3.27215424484,3.98722048131,3.71781556747,2.5610299588,0.799047653518,-1.18858453138,-3.01284626565,-4.34725663835,-4.9755042398">
<attribute name="units" source="constant" type="string" value="mm" />
</dataset>
<dataset det_default="true" name="detector" source="detector">
<attribute name="NX_class" source="constant" type="string" value="SDS" />
</dataset>
</group>
<group name="sum">
<attribute name="signal" source="constant" type="string" value="sum" />
<attribute name="axes" source="constant" type="string" value="energy_set,.,.,." />
<attribute name="NX_class" source="constant" type="string" value="NXdata" />
<attribute name="energy_set_indices" source="constant" type="string" value="0" />
<hardlink name="energy_set" target="/entry/detector/energy_set" />
<attribute name="x_set_indices" source="constant" type="string" value="1" />
<hardlink name="x_set" target="/entry/detector/x_set" />
<attribute name="y_set_indices" source="constant" type="string" value="1" />
<hardlink name="y_set" target="/entry/detector/y_set" />
<dataset name="sum" ndattribute="StatsTotal" source="ndattribute" />
</group>
<group name="I0">
<attribute name="signal" source="constant" type="string" value="I0" />
<attribute name="axes" source="constant" type="string" value="energy_set,.,.,." />
<attribute name="NX_class" source="constant" type="string" value="NXdata" />
<attribute name="energy_set_indices" source="constant" type="string" value="0" />
<hardlink name="energy_set" target="/entry/detector/energy_set" />
<attribute name="x_set_indices" source="constant" type="string" value="1" />
<hardlink name="x_set" target="/entry/detector/x_set" />
<attribute name="y_set_indices" source="constant" type="string" value="1" />
<hardlink name="y_set" target="/entry/detector/y_set" />
<dataset name="I0" ndattribute="COUNTER1.COUNTER" source="ndattribute" />
</group>
<group name="It">
<attribute name="signal" source="constant" type="string" value="It" />
<attribute name="axes" source="constant" type="string" value="energy_set,.,.,." />
<attribute name="NX_class" source="constant" type="string" value="NXdata" />
<attribute name="energy_set_indices" source="constant" type="string" value="0" />
<hardlink name="energy_set" target="/entry/detector/energy_set" />
<attribute name="x_set_indices" source="constant" type="string" value="1" />
<hardlink name="x_set" target="/entry/detector/x_set" />
<attribute name="y_set_indices" source="constant" type="string" value="1" />
<hardlink name="y_set" target="/entry/detector/y_set" />
<dataset name="It" ndattribute="COUNTER2.COUNTER" source="ndattribute" />
</group>
<group name="t1x">
<attribute name="signal" source="constant" type="string" value="t1x" />
<attribute name="axes" source="constant" type="string" value="energy_set,.,.,." />
<attribute name="NX_class" source="constant" type="string" value="NXdata" />
<attribute name="energy_set_indices" source="constant" type="string" value="0" />
<hardlink name="energy_set" target="/entry/detector/energy_set" />
<attribute name="x_set_indices" source="constant" type="string" value="1" />
<hardlink name="x_set" target="/entry/detector/x_set" />
<attribute name="y_set_indices" source="constant" type="string" value="1" />
<hardlink name="y_set" target="/entry/detector/y_set" />
<dataset name="t1x" ndattribute="INENC1.VAL" source="ndattribute" />
</group>
<group name="NDAttributes" ndattr_default="true">
<attribute name="NX_class" source="constant" type="string" value="NXcollection" />
</group>
</group>
</hdf5_layout>"""
with open(expected_xml_filename) as f:
actual_xml = f.read().replace(">", ">\n")
assert actual_xml.splitlines() == expected_xml.splitlines()
def _create_dataset_infos(self, name, part_info, generator, filename):
# Update the dataset table
uniqueid = "/entry/NDAttributes/NDArrayUniqueId"
generator_rank = len(generator.dimensions)
# Get the detector name from the primary source
ndarray_infos = NDArrayDatasetInfo.filter_values(part_info)
assert len(ndarray_infos) in (0, 1), \
"More than one NDArrayDatasetInfo defined %s" % ndarray_infos
# Add the primary datasource
if ndarray_infos:
ndarray_info = ndarray_infos[0]
yield DatasetProducedInfo(name="%s.data" % name,
filename=filename,
type="primary",
rank=ndarray_info.rank + generator_rank,
path="/entry/detector/detector",
uniqueid=uniqueid)
# Add any secondary datasources
for calculated_info in \
CalculatedNDAttributeDatasetInfo.filter_values(part_info):
yield DatasetProducedInfo(
name="%s.%s" % (name, calculated_info.name),
filename=filename,
type="secondary",
rank=ndarray_info.rank + generator_rank,
path="/entry/%s/%s" %
(calculated_info.name, calculated_info.name),
uniqueid=uniqueid)
# Add all the other datasources
for dataset_info in NDAttributeDatasetInfo.filter_values(part_info):
if dataset_info.type == "detector":
# Something like I0
name = "%s.data" % dataset_info.name
type = "primary"
elif dataset_info.type == "monitor":
# Something like Iref
name = "%s.data" % dataset_info.name
type = "monitor"
elif dataset_info.type == "position":
# Something like x
name = "%s.value" % dataset_info.name
type = "position_value"
else:
raise AttributeError(
"Bad dataset type %r, should be in %s" %
(dataset_info.type, attribute_dataset_types))
yield DatasetProducedInfo(name=name,
filename=filename,
type=type,
rank=dataset_info.rank + generator_rank,
path="/entry/%s/%s" %
(dataset_info.name, dataset_info.name),
uniqueid=uniqueid)
# Add any setpoint dimensions
for dim in generator.axes:
yield DatasetProducedInfo(name="%s.value_set" % dim,
filename=filename,
type="position_set",
rank=1,
path="/entry/detector/%s_set" % dim,
uniqueid="")
def _make_layout_xml(self, generator, part_info):
# Make a root element with an NXEntry
root_el = ET.Element("hdf5_layout")
entry_el = ET.SubElement(root_el, "group", name="entry")
ET.SubElement(entry_el,
"attribute",
name="NX_class",
source="constant",
value="NXentry",
type="string")
# Check that there is only one primary source of detector data
ndarray_infos = NDArrayDatasetInfo.filter_values(part_info)
if not ndarray_infos:
# Still need to put the data in the file, so manufacture something
primary_rank = 1
else:
primary_rank = ndarray_infos[0].rank
# Make an NXData element with the detector data in it in
# /entry/detector/detector
data_el = self._make_nxdata("detector", primary_rank, entry_el,
generator)
det_el = ET.SubElement(data_el,
"dataset",
name="detector",
source="detector",
det_default="true")
ET.SubElement(det_el,
"attribute",
name="NX_class",
source="constant",
value="SDS",
type="string")
# Now add any calculated sources of data
for dataset_info in \
CalculatedNDAttributeDatasetInfo.filter_values(part_info):
# if we are a secondary source, use the same rank as the det
attr_el = self._make_nxdata(dataset_info.name,
primary_rank,
entry_el,
generator,
link=True)
ET.SubElement(attr_el,
"dataset",
name=dataset_info.name,
source="ndattribute",
ndattribute=dataset_info.attr)
# And then any other attribute sources of data
for dataset_info in NDAttributeDatasetInfo.filter_values(part_info):
# if we are a secondary source, use the same rank as the det
attr_el = self._make_nxdata(dataset_info.name,
dataset_info.rank,
entry_el,
generator,
link=True)
ET.SubElement(attr_el,
"dataset",
name=dataset_info.name,
source="ndattribute",
ndattribute=dataset_info.attr)
# Add a group for attributes
NDAttributes_el = ET.SubElement(entry_el,
"group",
name="NDAttributes",
ndattr_default="true")
ET.SubElement(NDAttributes_el,
"attribute",
name="NX_class",
source="constant",
value="NXcollection",
type="string")
xml = et_to_string(root_el)
return xml
def configure_and_check_output(self, on_windows=False):
energy = LineGenerator("energy", "kEv", 13.0, 15.2, 2)
spiral = SpiralGenerator(["x", "y"], ["mm", "mm"], [0.0, 0.0],
5.0,
scale=2.0)
generator = CompoundGenerator([energy, spiral], [], [], 0.1)
generator.prepare()
fileDir = "/tmp"
formatName = "xspress3"
fileTemplate = "thing-%s.h5"
completed_steps = 0
steps_to_do = 38
part_info = {
"DET": [NDArrayDatasetInfo(2)],
"PANDA": [
NDAttributeDatasetInfo.from_attribute_type(
"I0", AttributeDatasetType.DETECTOR, "COUNTER1.COUNTER"),
NDAttributeDatasetInfo.from_attribute_type(
"It", AttributeDatasetType.MONITOR, "COUNTER2.COUNTER"),
NDAttributeDatasetInfo.from_attribute_type(
"t1x", AttributeDatasetType.POSITION, "INENC1.VAL"),
],
"STAT": [CalculatedNDAttributeDatasetInfo("sum", "StatsTotal")],
}
if on_windows:
part_info["WINPATH"] = [FilePathTranslatorInfo("Y", "/tmp", "")]
infos = self.o.on_configure(
self.context,
completed_steps,
steps_to_do,
part_info,
generator,
fileDir,
formatName,
fileTemplate,
)
assert len(infos) == 8
assert infos[0].name == "xspress3.data"
assert infos[0].filename == "thing-xspress3.h5"
assert infos[0].type == DatasetType.PRIMARY
assert infos[0].rank == 4
assert infos[0].path == "/entry/detector/detector"
assert infos[0].uniqueid == "/entry/NDAttributes/NDArrayUniqueId"
assert infos[1].name == "xspress3.sum"
assert infos[1].filename == "thing-xspress3.h5"
assert infos[1].type == DatasetType.SECONDARY
assert infos[1].rank == 2
assert infos[1].path == "/entry/sum/sum"
assert infos[1].uniqueid == "/entry/NDAttributes/NDArrayUniqueId"
assert infos[2].name == "I0.data"
assert infos[2].filename == "thing-xspress3.h5"
assert infos[2].type == DatasetType.PRIMARY
assert infos[2].rank == 2
assert infos[2].path == "/entry/I0.data/I0.data"
assert infos[2].uniqueid == "/entry/NDAttributes/NDArrayUniqueId"
assert infos[3].name == "It.data"
assert infos[3].filename == "thing-xspress3.h5"
assert infos[3].type == DatasetType.MONITOR
assert infos[3].rank == 2
assert infos[3].path == "/entry/It.data/It.data"
assert infos[3].uniqueid == "/entry/NDAttributes/NDArrayUniqueId"
assert infos[4].name == "t1x.value"
assert infos[4].filename == "thing-xspress3.h5"
assert infos[4].type == DatasetType.POSITION_VALUE
assert infos[4].rank == 2
assert infos[4].path == "/entry/t1x.value/t1x.value"
assert infos[4].uniqueid == "/entry/NDAttributes/NDArrayUniqueId"
assert infos[5].name == "energy.value_set"
assert infos[5].filename == "thing-xspress3.h5"
assert infos[5].type == DatasetType.POSITION_SET
assert infos[5].rank == 1
assert infos[5].path == "/entry/detector/energy_set"
assert infos[5].uniqueid == ""
assert infos[6].name == "x.value_set"
assert infos[6].filename == "thing-xspress3.h5"
assert infos[6].type == DatasetType.POSITION_SET
assert infos[6].rank == 1
assert infos[6].path == "/entry/detector/x_set"
assert infos[6].uniqueid == ""
assert infos[7].name == "y.value_set"
assert infos[7].filename == "thing-xspress3.h5"
assert infos[7].type == DatasetType.POSITION_SET
assert infos[7].rank == 1
assert infos[7].path == "/entry/detector/y_set"
assert infos[7].uniqueid == ""
expected_xml_filename_local = "/tmp/BLOCK_HDF5-layout.xml"
if on_windows:
expected_xml_filename_remote = "Y:\\BLOCK_HDF5-layout.xml"
expected_filepath = "Y:" + os.sep
else:
expected_xml_filename_remote = expected_xml_filename_local
expected_filepath = "/tmp" + os.sep
# Wait for the start_future so the post gets through to our child
# even on non-cothread systems
self.o.start_future.result(timeout=1)
assert self.child.handled_requests.mock_calls == [
call.put("positionMode", True),
call.put("arrayCounter", 0),
call.put("dimAttDatasets", True),
call.put("enableCallbacks", True),
call.put("fileName", "xspress3"),
call.put("filePath", expected_filepath),
call.put("fileTemplate", "%sthing-%s.h5"),
call.put("fileWriteMode", "Stream"),
call.put("lazyOpen", True),
call.put("storeAttr", True),
call.put("swmrMode", True),
call.put("extraDimSize3", 1),
call.put("extraDimSize4", 1),
call.put("extraDimSize5", 1),
call.put("extraDimSize6", 1),
call.put("extraDimSize7", 1),
call.put("extraDimSize8", 1),
call.put("extraDimSize9", 1),
call.put("extraDimSizeN", 20),
call.put("extraDimSizeX", 2),
call.put("extraDimSizeY", 1),
call.put("numExtraDims", 1),
call.put("posNameDim3", ""),
call.put("posNameDim4", ""),
call.put("posNameDim5", ""),
call.put("posNameDim6", ""),
call.put("posNameDim7", ""),
call.put("posNameDim8", ""),
call.put("posNameDim9", ""),
call.put("posNameDimN", "d1"),
call.put("posNameDimX", "d0"),
call.put("posNameDimY", ""),
call.put("flushAttrPerNFrames", 0),
call.put("flushDataPerNFrames", 38),
call.put("xmlLayout", expected_xml_filename_remote),
call.put("numCapture", 0),
call.post("start"),
call.when_value_matches("arrayCounterReadback", greater_than_zero,
None),
]
with open(expected_xml_filename_local) as f:
actual_xml = f.read().replace(">", ">\n")
# Check the layout filename Malcolm uses for file creation
assert self.o.layout_filename == expected_xml_filename_local
return actual_xml