def main(file_name, jobs):
in1out2 = NMFunction(n=1, m=2)
in1out2.__metatype__ = "ADAPTER"
in1out2.set_parameter("function", datasets)
abs_reader = HigherOrderComponent(multiple_outputs_reader(m=3))
df_reader = HigherOrderComponent(multiple_outputs_reader(m=2))
feature_segmentation = ThicknessFeatureSegmentation()
feature_segmentation_out = NMFunction(m=3)
log_ratio = NMFunction(n=2)
log_ratio.set_parameter(
"function",
log_function)
average_abs = average()
average_df = average()
average_r = average()
network = {
in1out2: {
abs_reader: ("out", "in"),
df_reader: ("out1", "in"),
},
abs_reader: {
average_abs: ("out", "in"),
feature_segmentation: ("out1", "in"),
log_ratio: ("out2", "in1")
},
df_reader: {
log_ratio: ("out", "in"),
average_df: ("out1", "in")
},
feature_segmentation: {
feature_segmentation_out: ("out", "in")
},
log_ratio: {
average_r: ("out", "in")
},
feature_segmentation_out: {
average_abs: ("out", "in1"),
average_df: ("out1", "in1"),
average_r: ("out2", "in1")
},
}
pipeline = pypes.pipeline.Dataflow(network, n=jobs)
packet = pypes.packet.Packet()
packet.set("file_name", file_name)
pipeline.send(packet)
pipeline.close()
def run(self):
# Define our components entry point
while True:
# for each file name string waiting on our input port
for packet in self.receive_all("in"):
if packet is not None:
input_file = False
try:
log.debug("%s received %s",
self.__class__.__name__,
packet)
log.debug("with file name %s and data %s",
packet.get("file_name"),
packet.get("data"))
file_name = packet.get("file_name")
object_name = packet.get("data")
input_file = h5py.File(file_name)
input_object = [
dataset[...]
for dataset in input_file[object_name].values()
if isinstance(dataset, h5py.Dataset)]
attrs = [
dataset.attrs
for dataset in input_file[object_name].values()
if isinstance(dataset, h5py.Dataset)][0]
#add info from first dataset
for key, value in attrs.items():
packet.set(key, value)
log.debug("%s found %d datasets",
self.__class__.__name__,
len(input_object))
packet.set("data", input_object)
except:
log.error('%s failed while reading %s',
self.__class__.__name__,
file_name, exc_info=True)
finally:
if input_file:
input_file.close()
# send the packet to the next component
self.send('out', packet)
# yield the CPU, allowing another component to run
self.yield_ctrl()
def run(self):
# Define our components entry point
while True:
# for each file name string waiting on our input port
for packet in self.receive_all("in"):
log.debug("%s received %s %s",
self.__class__.__name__,
packet.get("file_name"),
packet.get("data"))
try:
file_name = packet.get("file_name")
object_name = packet.get("data")
input_file = h5py.File(file_name)
input_object = input_file[object_name]
self.files.append(input_file)
packet.set("data", input_object)
except:
log.error('%s failed while reading %s',
self.__class__.__name__,
file_name, exc_info=True)
# send the packet to the next component
self.send('out', packet)
# yield the CPU, allowing another component to run
self.yield_ctrl()