def setUp(self):
data = ReflectDataset()
x1 = np.linspace(0, 10, 5)
y1 = 2 * x1
e1 = np.ones_like(x1)
dx1 = np.ones_like(x1)
data.add_data((x1, y1, e1, dx1))
self.data = data
def setUp(self):
data = ReflectDataset()
x1 = np.linspace(0, 10, 5)
y1 = 2 * x1
e1 = np.ones_like(x1)
dx1 = np.ones_like(x1)
data.add_data((x1, y1, e1, dx1))
self.data = data
self.cwd = os.getcwd()
self.tmpdir = TemporaryDirectory()
os.chdir(self.tmpdir.name)
def setup_method(self, tmpdir):
self.pth = os.path.dirname(os.path.abspath(__file__))
data = ReflectDataset()
x1 = np.linspace(0, 10, 115)
y1 = 2 * x1
e1 = np.ones_like(x1)
dx1 = np.ones_like(x1)
data.add_data((x1, y1, e1, dx1))
self.data = data
self.cwd = os.getcwd()
self.tmpdir = tmpdir.strpath
os.chdir(self.tmpdir)
def reducer(self, callback=None):
"""
Reduce all the entries in reduction_entries
Parameters
----------
callback : callable
Function, `f(percent_finished)` that is called with the current
percentage progress of the reduction
"""
# refnx.reduce.reduce needs you to be in the directory where you're
# going to write files to
if self.output_directory:
os.chdir(self.output_directory)
# if no data directory was specified then assume it's the cwd
data_directory = self.data_directory
if not data_directory:
data_directory = "./"
def full_path(fname):
f = os.path.join(data_directory, fname)
return f
# if the streamed directory isn't mentioned then assume it's the same
# as the data directory
streamed_directory = self.streamed_directory
if not os.path.isdir(streamed_directory):
self.streamed_directory = data_directory
logging.info("-------------------------------------------------------"
"\nStarting reduction run")
logging.info(
"data_folder={data_directory}, trim_trailing=True, "
"lo_wavelength={low_wavelength}, "
"hi_wavelength={high_wavelength}, "
"rebin_percent={rebin_percent}, "
"normalise={monitor_normalisation}, "
"background={background_subtraction} "
"eventmode={streamed_reduction} "
"event_folder={streamed_directory}".format(**self.__dict__))
# sets up time slices for event reduction
if self.streamed_reduction:
eventmode = np.arange(self.stream_start, self.stream_end,
self.stream_duration)
eventmode = np.r_[eventmode, self.stream_end]
else:
eventmode = None
# are you manual beamfinding?
peak_pos = None
if self.manual_beam_find and self.manual_beam_finder is not None:
peak_pos = -1
idx = 0
cached_direct_beams = {}
for row, val in self.reduction_entries.items():
if not val["use"]:
continue
flood = None
if val["flood"]:
flood = full_path(val["flood"])
combined_dataset = None
# process entries one by one
for ref, db in zip(
["reflect-1", "reflect-2", "reflect-3"],
["direct-1", "direct-2", "direct-3"],
):
reflect = val[ref]
direct = val[db]
# if the file doesn't exist there's no point continuing
if (not os.path.isfile(full_path(reflect))) or (
not os.path.isfile(full_path(direct))):
continue
# which of the nspectra to reduce (or all)
ref_pn = PlatypusNexus(full_path(reflect))
if direct not in cached_direct_beams:
cached_direct_beams[direct] = PlatypusReduce(
direct, data_folder=data_directory)
reducer = cached_direct_beams[direct]
try:
reduced = reducer(
ref_pn,
scale=val["scale"],
h5norm=flood,
lo_wavelength=self.low_wavelength,
hi_wavelength=self.high_wavelength,
rebin_percent=self.rebin_percent,
normalise=self.monitor_normalisation,
background=self.background_subtraction,
manual_beam_find=self.manual_beam_finder,
peak_pos=peak_pos,
eventmode=eventmode,
event_folder=streamed_directory,
)
except Exception as e:
# typical Exception would be ValueError for non overlapping
# angles
logging.info(e)
continue
logging.info("Reduced {} vs {}, scale={}, angle={}".format(
reflect,
direct,
val["scale"],
reduced[1]["omega"][0, 0],
))
if combined_dataset is None:
combined_dataset = ReflectDataset()
fname = basename_datafile(reflect)
fname_dat = os.path.join(self.output_directory,
"c_{0}.dat".format(fname))
fname_xml = os.path.join(self.output_directory,
"c_{0}.xml".format(fname))
try:
combined_dataset.add_data(
reducer.data(),
requires_splice=True,
trim_trailing=True,
)
except ValueError as e:
# datasets don't overlap
logging.info(e)
continue
if combined_dataset is not None:
# after you've finished reducing write a combined file.
with open(fname_dat, "wb") as f:
combined_dataset.save(f)
with open(fname_xml, "wb") as f:
combined_dataset.save_xml(f)
logging.info("Written combined files: {} and {}".format(
fname_dat, fname_xml))
# can be used to create a progress bar
idx += 1
if callback is not None:
ok = callback(100 * idx / len(self.reduction_entries))
if not ok:
break
logging.info("\nFinished reduction run"
"-------------------------------------------------------")
def reduce_stitch(reflect_list,
direct_list,
background_list=None,
norm_file_num=None,
data_folder=None,
prefix='PLP',
trim_trailing=True,
save=True,
**kwds):
"""
Reduces a list of reflected beam run numbers and a list of corresponding
direct beam run numbers from the Platypus reflectometer. If there are
multiple reflectivity files they are spliced together.
Parameters
----------
reflect_list : list
Reflected beam run numbers, e.g. `[708, 709, 710]`
708 corresponds to the file PLP0000708.nx.hdf.
direct_list : list
Direct beam run numbers, e.g. `[711, 711, 711]`
background_list : list, optional
List of `bool` to control whether background subtraction is used
for each reduction, e.g. `[False, True, True]`. The default is to do
a background subtraction on all runs.
norm_file_num : int, optional
The run number for the water flood field correction.
data_folder : str, optional
Where is the raw data stored?
prefix : str, optional
The instrument filename prefix.
trim_trailing : bool, optional
When datasets are spliced together do you want to remove points in the
overlap region from the preceding dataset?
save : bool, optional
If `True` then the spliced file is written to a file (in the working
directory) with a name like: `c_PLP0000708.dat`.
kwds : dict, optional
Options passed directly to `refnx.reduce.platypusnexus.process`,
for processing of individual spectra. Look at that method docstring
for specification of options.
Returns
-------
combined_dataset, reduced_filename : refnx.dataset.ReflectDataset, str
The combined dataset and the file name of the reduced data, if it was
saved. If it wasn't saved `reduced_filename` is `None`.
Notes
-----
If `background` is in the supplied `kwds` it is ignored.
The `prefix` is used to specify the run numbers to a filename.
For example a run number of 10, and a prefix of `PLP` resolves to a
NeXus filename of 'PLP0000010.nx.hdf'.
Examples
--------
>>> from refnx.reduce import reduce_stitch
>>> dataset, fname = reduce_stitch([708, 709, 710],
... [711, 711, 711],
... rebin_percent=2)
"""
scale = kwds.get('scale', 1.)
kwds_copy = {}
kwds_copy.update(kwds)
kwds_copy.pop('background', None)
if not background_list:
background_list = [True] * len(reflect_list)
# now reduce all the files.
zipped = zip(reflect_list, direct_list, background_list)
combined_dataset = ReflectDataset()
if data_folder is None:
data_folder = os.getcwd()
if norm_file_num:
norm_datafile = number_datafile(norm_file_num, prefix=prefix)
kwds['h5norm'] = norm_datafile
if prefix == 'PLP':
reducer_klass = PlatypusReduce
else:
raise ValueError("Incorrect prefix specified")
for index, val in enumerate(zipped):
reflect_datafile = os.path.join(data_folder,
number_datafile(val[0], prefix=prefix))
direct_datafile = os.path.join(data_folder,
number_datafile(val[1], prefix=prefix))
reducer = reducer_klass(direct_datafile)
datasets, fnames = reducer.reduce(reflect_datafile,
save=save,
background=val[2],
**kwds_copy)
if not index:
datasets[0].scale(scale)
combined_dataset.add_data(datasets[0].data,
requires_splice=True,
trim_trailing=trim_trailing)
fname_dat = None
if save:
# this will give us <fname>.nx.hdf
# if reflect_list was an integer you'll get PLP0000708.nx.hdf
fname = number_datafile(reflect_list[0], prefix=prefix)
# now chop off .nx.hdf extension
fname = basename_datafile(fname)
fname_dat = 'c_{0}.dat'.format(fname)
with open(fname_dat, 'wb') as f:
combined_dataset.save(f)
fname_xml = 'c_{0}.xml'.format(fname)
with open(fname_xml, 'wb') as f:
combined_dataset.save_xml(f)
return combined_dataset, fname_dat
def reduce_stitch(reflect_list, direct_list, norm_file_num=None,
data_folder=None, trim_trailing=True, save=True, **kwds):
"""
Reduces a list of reflected beam run numbers and a list of corresponding
direct beam run numbers from the Platypus reflectometer. If there are
multiple reflectivity files they are spliced together.
Parameters
----------
reflect_list : list
Reflected beam run numbers, e.g. `[708, 709, 710]`
708 corresponds to the file PLP0000708.nx.hdf.
direct_list : list
Direct beam run numbers, e.g. `[711, 711, 711]`
norm_file_num : int, optional
The run number for the water flood field correction.
data_folder : str, optional
Where is the raw data stored?
trim_trailing : bool, optional
When datasets are spliced together do you want to remove points in the
overlap region from the preceding dataset?
save : bool, optional
If `True` then the spliced file is written to a file (in the working
directory) with a name like: `c_PLP0000708.dat`.
kwds : dict, optional
Options passed directly to `refnx.reduce.platypusnexus.process`,
for processing of individual spectra. Look at that method docstring
for specification of options.
Returns
-------
combined_dataset, reduced_filename : refnx.dataset.ReflectDataset, str
The combined dataset and the file name of the reduced data, if it was
saved. If it wasn't saved `reduced_filename` is `None`.
"""
scale = kwds.get('scale', 1.)
# now reduce all the files.
zipped = zip(reflect_list, direct_list)
combined_dataset = ReflectDataset()
if data_folder is None:
data_folder = os.getcwd()
if norm_file_num:
norm_datafile = number_datafile(norm_file_num)
kwds['h5norm'] = norm_datafile
for index, val in enumerate(zipped):
reflect_datafile = os.path.join(data_folder,
number_datafile(val[0]))
direct_datafile = os.path.join(data_folder,
number_datafile(val[1]))
reduced = ReducePlatypus(direct_datafile,
reflect=reflect_datafile,
save=save,
**kwds)
if not index:
reduced.scale(scale)
combined_dataset.add_data(reduced.data(),
requires_splice=True,
trim_trailing=trim_trailing)
fname = None
if save:
fname = 'c_PLP{0:07d}.dat'.format(reflect_list[0])
with open(fname, 'wb') as f:
combined_dataset.save(f)
fname = 'c_PLP{0:07d}.xml'.format(reflect_list[0])
with open(fname, 'wb') as f:
combined_dataset.save_xml(f)
return combined_dataset, fname
def reduce_stitch(
reflect_list,
direct_list,
data_folder=None,
prefix="PLP",
trim_trailing=True,
save=True,
scale=1.0,
reduction_options=None,
):
"""
Reduces a list of reflected beam run numbers and a list of corresponding
direct beam run numbers from the Platypus/Spatz reflectometers. If there
are multiple reflectivity files they are spliced together.
Parameters
----------
reflect_list : list
Reflected beam run numbers, e.g. `[708, 709, 710]`
708 corresponds to the file PLP0000708.nx.hdf.
direct_list : list
Direct beam run numbers, e.g. `[711, 711, 711]`
data_folder : str, optional
Where is the raw data stored?
prefix : str, optional
The instrument filename prefix.
trim_trailing : bool, optional
When datasets are spliced together do you want to remove points in the
overlap region from the preceding dataset?
save : bool, optional
If `True` then the spliced file is written to a file (in the working
directory) with a name like: `c_PLP0000708.dat`.
scale : float, optional
Scales the data by this value.
reduction_options : None, dict, or list of dict, optional
Options passed directly to `refnx.reduce.PlatypusNexus.process`,
for processing of individual spectra. Look at that method docstring
for specification of options. If an individual dict then the same
options are used to process all datasets. A list (or sequence) of
dict can be used to specify different options for each datasets. If
None, then a default set of reduction options will be used.
Returns
-------
combined_dataset, reduced_filename : refnx.dataset.ReflectDataset, str
The combined dataset and the file name of the reduced data, if it was
saved. If it wasn't saved `reduced_filename` is `None`.
Notes
-----
The `prefix` is used to specify the run numbers to a filename.
For example a run number of 10, and a prefix of `PLP` resolves to a
NeXus filename of 'PLP0000010.nx.hdf'.
Examples
--------
>>> from refnx.reduce import reduce_stitch
>>> dataset, fname = reduce_stitch([708, 709, 710],
... [711, 711, 711],
... reduction_options={"rebin_percent": 2})
"""
options = [ReductionOptions()] * len(reflect_list)
try:
if reduction_options is not None:
options = []
for i in range(len(reflect_list)):
if isinstance(reduction_options[i], dict):
options.append(reduction_options[i])
else:
options.append(ReductionOptions())
except KeyError:
# reduction_options may be an individual dict
if isinstance(reduction_options, dict):
options = [reduction_options] * len(reflect_list)
# now reduce all the files.
zipped = zip(reflect_list, direct_list, options)
combined_dataset = ReflectDataset()
if data_folder is None:
data_folder = os.getcwd()
if prefix == "PLP":
reducer_klass = PlatypusReduce
elif prefix == "SPZ":
reducer_klass = SpatzReduce
else:
raise ValueError("Incorrect prefix specified")
for index, val in enumerate(zipped):
reflect_datafile = os.path.join(data_folder,
number_datafile(val[0], prefix=prefix))
direct_datafile = os.path.join(data_folder,
number_datafile(val[1], prefix=prefix))
reducer = reducer_klass(direct_datafile)
datasets, fnames = reducer.reduce(reflect_datafile,
save=save,
**val[2])
if not index:
datasets[0].scale(scale)
combined_dataset.add_data(datasets[0].data,
requires_splice=True,
trim_trailing=trim_trailing)
fname_dat = None
if save:
# this will give us <fname>.nx.hdf
# if reflect_list was an integer you'll get PLP0000708.nx.hdf
fname = number_datafile(reflect_list[0], prefix=prefix)
# now chop off .nx.hdf extension
fname = basename_datafile(fname)
fname_dat = "c_{0}.dat".format(fname)
with open(fname_dat, "wb") as f:
combined_dataset.save(f)
fname_xml = "c_{0}.xml".format(fname)
with open(fname_xml, "wb") as f:
combined_dataset.save_xml(f)
return combined_dataset, fname_dat
def reducer(self, callback=None):
"""
Reduce all the entries in reduction_entries
Parameters
----------
callback : callable
Function, `f(percent_finished)` that is called with the current
percentage progress of the reduction
"""
# refnx.reduce.reduce needs you to be in the directory where you're
# going to write files to
if self.output_directory:
os.chdir(self.output_directory)
# if no data directory was specified then assume it's the cwd
data_directory = self.data_directory
if not data_directory:
data_directory = './'
def full_path(fname):
f = os.path.join(data_directory, fname)
return f
# if the streamed directory isn't mentioned then assume it's the same
# as the data directory
streamed_directory = self.streamed_directory
if not os.path.isdir(streamed_directory):
self.streamed_directory = data_directory
logging.info('-------------------------------------------------------'
'\nStarting reduction run')
logging.info(
'data_folder={data_directory}, trim_trailing=True, '
'lo_wavelength={low_wavelength}, '
'hi_wavelength={high_wavelength}, '
'rebin_percent={rebin_percent}, '
'normalise={monitor_normalisation}, '
'background={background_subtraction} '
'eventmode={streamed_reduction} '
'event_folder={streamed_directory}'.format(**self.__dict__))
# sets up time slices for event reduction
if self.streamed_reduction:
eventmode = np.arange(self.stream_start,
self.stream_end,
self.stream_duration)
eventmode = np.r_[eventmode, self.stream_end]
else:
eventmode = None
# are you manual beamfinding?
peak_pos = None
if (self.manual_beam_find and
self.manual_beam_finder is not None):
peak_pos = -1
idx = 0
cached_direct_beams = {}
for row, val in self.reduction_entries.items():
if not val['use']:
continue
flood = None
if val['flood']:
flood = val['flood']
combined_dataset = None
# process entries one by one
for ref, db in zip(['reflect-1', 'reflect-2', 'reflect-3'],
['direct-1', 'direct-2', 'direct-3']):
reflect = val[ref]
direct = val[db]
# if the file doesn't exist there's no point continuing
if ((not os.path.isfile(full_path(reflect))) or
(not os.path.isfile(full_path(direct)))):
continue
# which of the nspectra to reduce (or all)
ref_pn = PlatypusNexus(reflect)
if direct not in cached_direct_beams:
cached_direct_beams[direct] = PlatypusReduce(
direct,
data_folder=data_directory)
reducer = cached_direct_beams[direct]
reduced = reducer(
ref_pn, scale=val['scale'],
norm_file_num=flood,
lo_wavelength=self.low_wavelength,
hi_wavelength=self.high_wavelength,
rebin_percent=self.rebin_percent,
normalise=self.monitor_normalisation,
background=self.background_subtraction,
manual_beam_find=self.manual_beam_finder,
peak_pos=peak_pos,
eventmode=eventmode,
event_folder=streamed_directory)
logging.info(
'Reduced {} vs {}, scale={}, angle={}'.format(
reflect, direct, val['scale'],
reduced['omega'][0, 0]))
if combined_dataset is None:
combined_dataset = ReflectDataset()
fname = basename_datafile(reflect)
fname_dat = os.path.join(self.output_directory,
'c_{0}.dat'.format(fname))
fname_xml = os.path.join(self.output_directory,
'c_{0}.xml'.format(fname))
combined_dataset.add_data(reducer.data(),
requires_splice=True,
trim_trailing=True)
if combined_dataset is not None:
# after you've finished reducing write a combined file.
with open(fname_dat, 'wb') as f:
combined_dataset.save(f)
with open(fname_xml, 'wb') as f:
combined_dataset.save_xml(f)
logging.info(
'Written combined files: {} and {}'.format(
fname_dat, fname_xml))
# can be used to create a progress bar
idx += 1
if callback is not None:
ok = callback(100 * idx / len(self.reduction_entries))
if not ok:
break
logging.info('\nFinished reduction run'
'-------------------------------------------------------')
