def process(input_data, output, config, monitor_name, observer):
"""
Integrate a set of data.
:param List[str] input_data: List of input filenames
:param str output: Filename of directory output
:param dict config: Dictionary to configure `pyFAI.worker.Worker`
:param IntegrationObserver observer: Observer of the processing
:param:
"""
worker = pyFAI.worker.Worker()
worker_config = config.copy()
json_monitor_name = worker_config.pop("monitor_name", None)
if monitor_name is None:
monitor_name = json_monitor_name
elif json_monitor_name is not None:
logger.warning("Monitor name from command line argument override the one from the configuration file.")
worker.set_config(worker_config, consume_keys=True)
worker.output = "raw"
# Check unused keys
for key in worker_config.keys():
# FIXME this should be read also
if key in ["application", "version"]:
continue
logger.warning("Configuration key '%s' from json is unused", key)
worker.safe = False # all processing are expected to be the same.
start_time = time.time()
if observer is not None:
observer.worker_initialized(worker)
# Skip invalide data
valid_data = []
for item in input_data:
if isinstance(item, six.string_types):
if os.path.isfile(item):
valid_data.append(item)
else:
if "::" in item:
try:
fabio.open(item)
valid_data.append(item)
except Exception:
logger.warning("File %s do not exists. File ignored.", item)
else:
logger.warning("File %s do not exists. File ignored.", item)
elif isinstance(item, fabio.fabioimage.FabioImage):
valid_data.append(item)
elif isinstance(item, numpy.ndarray):
valid_data.append(item)
else:
logger.warning("Type %s unsopported. Data ignored.", item)
if observer is not None:
observer.processing_started(len(valid_data))
# Integrate files one by one
for iitem, item in enumerate(valid_data):
logger.debug("Processing %s", item)
# TODO rework it as source
if isinstance(item, six.string_types):
kind = "filename"
fabio_image = fabio.open(item)
filename = fabio_image.filename
multiframe = fabio_image.nframes > 1
elif isinstance(item, fabio.fabioimage.FabioImage):
kind = "fabio-image"
fabio_image = item
multiframe = fabio_image.nframes > 1
filename = fabio_image.filename
elif isinstance(item, numpy.ndarray):
kind = "numpy-array"
filename = None
fabio_image = None
multiframe = False
if observer is not None:
observer.processing_data(iitem + 1, filename=filename)
if filename:
output_name = os.path.splitext(filename)[0]
else:
output_name = "array_%d" % iitem
if multiframe:
extension = "_pyFAI.h5"
else:
if worker.do_2D():
extension = ".azim"
else:
extension = ".dat"
output_name = "%s%s" % (output_name, extension)
if output:
if os.path.isdir(output):
basename = os.path.basename(output_name)
outpath = os.path.join(output, basename)
else:
outpath = os.path.abspath(output)
else:
outpath = output_name
if fabio_image is None:
if item.ndim == 3:
writer = HDF5Writer(outpath)
writer.init(fai_cfg=config)
for iframe, data in enumerate(item):
result = worker.process(data=data,
writer=writer)
if observer is not None:
if observer.is_interruption_requested():
break
observer.data_result(iitem, result)
else:
data = item
writer = DefaultAiWriter(outpath, worker.ai)
result = worker.process(data=data,
writer=writer)
if observer is not None:
observer.data_result(iitem, result)
else:
if multiframe:
writer = HDF5Writer(outpath, append_frames=True)
writer.init(fai_cfg=config)
for iframe in range(fabio_image.nframes):
fimg = fabio_image.getframe(iframe)
normalization_factor = get_monitor_value(fimg, monitor_name)
data = fimg.data
result = worker.process(data=data,
metadata=fimg.header,
normalization_factor=normalization_factor,
writer=writer)
if observer is not None:
if observer.is_interruption_requested():
break
observer.data_result(iitem, result)
writer.close()
else:
writer = DefaultAiWriter(outpath, worker.ai)
normalization_factor = get_monitor_value(fabio_image, monitor_name)
data = fabio_image.data
result = worker.process(data,
normalization_factor=normalization_factor,
writer=writer)
if observer is not None:
observer.data_result(iitem, result)
writer.close()
if observer is not None:
if observer.is_interruption_requested():
break
if observer is not None:
if observer.is_interruption_requested():
logger.info("Processing was aborted")
observer.processing_interrupted()
else:
observer.processing_succeeded()
observer.processing_finished()
logger.info("Processing done in %.3fs !", (time.time() - start_time))
return 0
def process(input_data,
output,
config,
monitor_name,
observer,
write_mode=HDF5Writer.MODE_ERROR):
"""
Integrate a set of data.
:param List[str] input_data: List of input filenames
:param str output: Filename of directory output
:param dict config: Dictionary to configure `pyFAI.worker.Worker`
:param IntegrationObserver observer: Observer of the processing
:param str write_mode: Specify options to deal with IO errors
"""
statistics = Statistics()
statistics.execution_started()
if observer is None:
# Create a null observer to avoid to deal with None
observer = IntegrationObserver()
worker = pyFAI.worker.Worker()
worker_config = config.copy()
json_monitor_name = worker_config.pop("monitor_name", None)
if monitor_name is None:
monitor_name = json_monitor_name
elif json_monitor_name is not None:
logger.warning(
"Monitor name from command line argument override the one from the configuration file."
)
worker.set_config(worker_config, consume_keys=True)
worker.output = "raw"
# Check unused keys
for key in worker_config.keys():
# FIXME this should be read also
if key in ["application", "version"]:
continue
logger.warning("Configuration key '%s' from json is unused", key)
worker.safe = False # all processing are expected to be the same.
observer.worker_initialized(worker)
# Skip invalide data
source = DataSource(statistics=statistics)
for item in input_data:
if isinstance(item, (str, )):
if os.path.isfile(item):
source.append(item)
else:
if "::" in item:
try:
# Only check that we can open the file
# It's low cost with HDF5
with fabio.open(item):
pass
source.append(item)
except Exception:
logger.warning("File %s do not exists. File ignored.",
item)
else:
logger.warning("File %s do not exists. File ignored.",
item)
elif isinstance(item, fabio.fabioimage.FabioImage):
source.append(item)
elif isinstance(item, numpy.ndarray):
source.append(item)
else:
logger.warning("Type %s unsopported. Data ignored.", item)
observer.processing_started(source.approximate_count())
writer = None
if output:
if "::" in output:
output, entry_path = output.split("::", 1)
else:
entry_path = None
if os.path.isdir(output):
writer = MultiFileWriter(output, mode=write_mode)
elif output.endswith(".h5") or output.endswith(".hdf5"):
writer = HDF5Writer(output,
hpath=entry_path,
append_frames=True,
mode=write_mode)
else:
output_path = os.path.abspath(output)
writer = MultiFileWriter(output_path, mode=write_mode)
else:
if source.is_single_multiframe():
basename = os.path.splitext(source.basename())[0]
output_filename = "%s_pyFAI.h5" % basename
writer = HDF5Writer(output_filename,
append_frames=True,
mode=write_mode)
else:
output_path = os.path.abspath(".")
writer = MultiFileWriter(None, mode=write_mode)
try:
writer.init(fai_cfg=config)
except IOError as e:
logger.error("Error while creating the writer: " + str(e.args[0]))
logger.error("Processing cancelled")
logger.info(
"To write HDF5, convenient options can be provided to decide what to do."
)
logger.info(
"Options: --delete (always delete the file) --append (create a new entry) --overwrite (overwrite this entry)"
)
writer.close()
return 1
# Integrate all the provided frames one by one
for data_info in source.frames():
logger.debug("Processing %s", item)
observer.processing_data(data_info,
approximate_count=source.approximate_count())
if data_info.fabio_image is not None:
normalization_factor = get_monitor_value(data_info.fabio_image,
monitor_name)
else:
normalization_factor = 1.0
if hasattr(writer, "prepare_write"):
writer.prepare_write(data_info, engine=worker.ai)
with statistics.time_processing():
result = worker.process(data=data_info.data,
normalization_factor=normalization_factor,
writer=writer)
# Store reference to input data if possible
if isinstance(writer, HDF5Writer) and (data_info.fabio_image
is not None):
fimg = data_info.fabio_image
if "dataset" in dir(fimg):
if isinstance(fimg.dataset, list):
for ds in fimg.dataset:
writer.set_hdf5_input_dataset(ds)
else:
writer.set_hdf5_input_dataset(fimg.dataset)
if observer.is_interruption_requested():
break
observer.data_result(data_info, result)
if observer.is_interruption_requested():
break
writer.close()
if observer.is_interruption_requested():
logger.error("Processing was aborted")
observer.processing_interrupted()
result = 2
else:
observer.processing_succeeded()
result = 0
observer.processing_finished()
statistics.execution_finished()
logger.info("[First frame] Preprocessing time: %.0fms",
statistics.preprocessing() * 1000)
logger.info("[Per frames] Reading time: %.0fms; Processing time: %.0fms",
statistics.reading_per_frame() * 1000,
statistics.processing_per_frame() * 1000)
logger.info("[Total] Reading time: %.3fs; Processing time: %.3fs",
statistics.total_reading(), statistics.total_processing())
logger.info("Execution done in %.3fs !", statistics.total_execution())
return result
def process(input_data, output, config, monitor_name, observer, write_mode=HDF5Writer.MODE_ERROR):
"""
Integrate a set of data.
:param List[str] input_data: List of input filenames
:param str output: Filename of directory output
:param dict config: Dictionary to configure `pyFAI.worker.Worker`
:param IntegrationObserver observer: Observer of the processing
:param str write_mode: Specify options to deal with IO errors
"""
statistics = Statistics()
statistics.execution_started()
if observer is None:
# Create a null observer to avoid to deal with None
observer = IntegrationObserver()
worker = pyFAI.worker.Worker()
worker_config = config.copy()
json_monitor_name = worker_config.pop("monitor_name", None)
if monitor_name is None:
monitor_name = json_monitor_name
elif json_monitor_name is not None:
logger.warning("Monitor name from command line argument override the one from the configuration file.")
worker.set_config(worker_config, consume_keys=True)
worker.output = "raw"
# Check unused keys
for key in worker_config.keys():
# FIXME this should be read also
if key in ["application", "version"]:
continue
logger.warning("Configuration key '%s' from json is unused", key)
worker.safe = False # all processing are expected to be the same.
observer.worker_initialized(worker)
# Skip invalide data
source = DataSource(statistics=statistics)
for item in input_data:
if isinstance(item, six.string_types):
if os.path.isfile(item):
source.append(item)
else:
if "::" in item:
try:
# Only check that we can open the file
# It's low cost with HDF5
with fabio.open(item):
pass
source.append(item)
except Exception:
logger.warning("File %s do not exists. File ignored.", item)
else:
logger.warning("File %s do not exists. File ignored.", item)
elif isinstance(item, fabio.fabioimage.FabioImage):
source.append(item)
elif isinstance(item, numpy.ndarray):
source.append(item)
else:
logger.warning("Type %s unsopported. Data ignored.", item)
observer.processing_started(source.approximate_count())
writer = None
if output:
if "::" in output:
output, entry_path = output.split("::", 1)
else:
entry_path = None
if os.path.isdir(output):
writer = MultiFileWriter(output, mode=write_mode)
elif output.endswith(".h5") or output.endswith(".hdf5"):
writer = HDF5Writer(output, hpath=entry_path, append_frames=True, mode=write_mode)
else:
output_path = os.path.abspath(output)
writer = MultiFileWriter(output_path, mode=write_mode)
else:
if source.is_single_multiframe():
basename = os.path.splitext(source.basename())[0]
output_filename = "%s_pyFAI.h5" % basename
writer = HDF5Writer(output_filename, append_frames=True, mode=write_mode)
else:
output_path = os.path.abspath(".")
writer = MultiFileWriter(None, mode=write_mode)
try:
writer.init(fai_cfg=config)
except IOError as e:
logger.error("Error while creating the writer: " + str(e.args[0]))
logger.error("Processing cancelled")
logger.info("To write HDF5, convenient options can be provided to decide what to do.")
logger.info("Options: --delete (always delete the file) --append (create a new entry) --overwrite (overwrite this entry)")
writer.close()
return 1
# Integrate all the provided frames one by one
for data_info in source.frames():
logger.debug("Processing %s", item)
observer.processing_data(data_info,
approximate_count=source.approximate_count())
if data_info.fabio_image is not None:
normalization_factor = get_monitor_value(data_info.fabio_image, monitor_name)
else:
normalization_factor = 1.0
if hasattr(writer, "prepare_write"):
writer.prepare_write(data_info, engine=worker.ai)
with statistics.time_processing():
result = worker.process(data=data_info.data,
normalization_factor=normalization_factor,
writer=writer)
if observer.is_interruption_requested():
break
observer.data_result(data_info, result)
if observer.is_interruption_requested():
break
writer.close()
if observer.is_interruption_requested():
logger.error("Processing was aborted")
observer.processing_interrupted()
result = 2
else:
observer.processing_succeeded()
result = 0
observer.processing_finished()
statistics.execution_finished()
logger.info("[First frame] Preprocessing time: %.0fms", statistics.preprocessing() * 1000)
logger.info("[Per frames] Reading time: %.0fms; Processing time: %.0fms", statistics.reading_per_frame() * 1000, statistics.processing_per_frame() * 1000)
logger.info("[Total] Reading time: %.3fs; Processing time: %.3fs", statistics.total_reading(), statistics.total_processing())
logger.info("Execution done in %.3fs !", statistics.total_execution())
return result