def test_apply_vanadium(project_file, van_project_file, target_project_file):
"""Test applying vanadium to the raw data in project file
Parameters
----------
project_file : str
raw HiDRA project file to convert to 2theta pattern
van_project_file : str
raw HiDra vanadium file
target_project_file : str
target HiDRA
Returns
-------
"""
# Check files' existence
checkFileExists(project_file, feedback='assert')
checkFileExists(van_project_file, feedback='assert')
# Load data
# extract the powder patterns and add them to the project file
reducer = ReductionApp()
# instrument_file, calibration_file, mask, sub_runs
reducer.load_project_file(project_file)
reducer.reduce_data(sub_runs=None, instrument_file=None, calibration_file=None, mask=None,
van_file=van_project_file, num_bins=950)
reducer.save_diffraction_data(target_project_file)
def addPowderToProject(projectfile, calibration_file=None):
checkFileExists(projectfile, feedback='assert')
# extract the powder patterns and add them to the project file
reducer = ReductionApp()
# TODO should add versions for testing arguments: instrument_file, calibration_file, mask, sub_runs
reducer.load_project_file(projectfile)
reducer.reduce_data(sub_runs=None, instrument_file=None, calibration_file=calibration_file, mask=None)
reducer.save_diffraction_data(projectfile)
# tests for the created file
assert os.path.exists(projectfile)
def test_reduce_data(mask_file_name, filtered_counts, histogram_counts):
"""Verify NeXus converters including counts and sample log values"""
SUBRUNS = (1, 2, 3)
CENTERS = (69.99525, 80., 97.50225)
# reduce with PyRS/Python
hidra_ws = convertNeXusToProject('/HFIR/HB2B/IPTS-22731/nexus/HB2B_1017.ORIG.nxs.h5',
projectfile=None, skippable=True, mask_file_name=mask_file_name)
# verify that sample logs exist
sample_log_names = hidra_ws.get_sample_log_names()
# missing fields for HB2B_1017: SampleDescription, SampleId, SampleName, sub-run, Wavelength
# scan_index is not exposed through this method
# this list is imported from pyrs/core/nexus_conversion.py
EXPECTED_NAMES = DEFAULT_KEEP_LOGS.copy()
EXPECTED_NAMES.remove('SampleDescription')
EXPECTED_NAMES.remove('SampleId')
EXPECTED_NAMES.remove('SampleName')
EXPECTED_NAMES.remove('scan_index')
EXPECTED_NAMES.remove('sub-run')
EXPECTED_NAMES.remove('Wavelength')
assert len(sample_log_names) == len(EXPECTED_NAMES), 'Same number of log names'
for name in EXPECTED_NAMES: # check all expected names are found
assert name in sample_log_names
# verify subruns
np.testing.assert_equal(hidra_ws.get_sub_runs(), SUBRUNS)
for sub_run, total_counts in zip(hidra_ws.get_sub_runs(), filtered_counts):
counts_array = hidra_ws.get_detector_counts(sub_run)
np.testing.assert_equal(counts_array.shape, (1048576,))
assert np.sum(counts_array) == total_counts, 'mismatch in subrun={} for filtered data'.format(sub_run)
# Test reduction to diffraction pattern
reducer = ReductionApp()
reducer.load_hidra_workspace(hidra_ws)
reducer.reduce_data(sub_runs=None, instrument_file=None, calibration_file=None, mask=None)
# plot the patterns
if DIAGNOSTIC_PLOTS:
for sub_run, angle in zip(SUBRUNS, CENTERS):
x, y = reducer.get_diffraction_data(sub_run)
plt.plot(x, y, label='SUBRUN {} at {:.1f} deg'.format(sub_run, angle))
plt.legend()
plt.show()
# check ranges and total counts
for sub_run, angle, total_counts in zip(SUBRUNS, CENTERS, histogram_counts):
assert_label = 'mismatch in subrun={} for histogrammed data'.format(sub_run)
x, y = reducer.get_diffraction_data(sub_run)
assert x[0] < angle < x[-1], assert_label
# assert np.isnan(np.sum(y[1:])), assert_label
np.testing.assert_almost_equal(np.nansum(y), total_counts, decimal=1, err_msg=assert_label)
def test_texture_reduction(nexusfile, mask_file_name, gold_file):
"""Test the powder pattern calculator (service) with HB2B-specific reduction routine
Parameters
----------
project_file_name
mask_file_name
gold_file
"""
if not os.path.exists('/HFIR/HB2B/shared'):
pytest.skip('Unable to access HB2B archive')
CALIBRATION_FILE = "data/HB2B_calib_latest.json"
VANADIUM_FILE = "/HFIR/HB2B/IPTS-22731/nexus/HB2B_1115.nxs.h5"
# load gold file
gold_data_dict = parse_gold_file(gold_file)
# Parse input file
converter = NeXusConvertingApp(nexusfile, mask_file_name)
hidra_ws = converter.convert()
# Start reduction service
reducer = ReductionApp()
reducer.load_hidra_workspace(hidra_ws)
# Reduce raw counts
reducer.reduce_data(instrument_file=None,
calibration_file=CALIBRATION_FILE,
mask=None,
sub_runs=[],
van_file=VANADIUM_FILE,
eta_step=3.0)
for sub_run_i in list(gold_data_dict.keys()):
# Get gold data of pattern (i).
gold_data_i = gold_data_dict[sub_run_i]
# Get powder data of pattern (i).
pattern = reducer.get_diffraction_data(1, sub_run_i)
# validate correct two-theta reduction
np.testing.assert_allclose(pattern[0],
gold_data_dict[sub_run_i][0],
rtol=1E-8)
# remove NaN intensity arrays
pattern[1][np.where(np.isnan(pattern[1]))] = 0.
gold_data_i[1][np.where(np.isnan(gold_data_i[1]))] = 0.
# validate correct intesnity reduction
np.testing.assert_allclose(pattern[1],
gold_data_i[1],
rtol=1E-8,
equal_nan=True)
def _create_powder_patterns(hidra_workspace, instrument, calibration, mask,
subruns, project_file_name, append_mode):
logger.notice(
'Adding powder patterns to Hidra Workspace{}'.format(hidra_workspace))
reducer = ReductionApp(bool(options.engine == 'mantid'))
# reducer.load_project_file(projectfile)
# load HidraWorkspace
reducer.load_hidra_workspace(hidra_workspace)
reducer.reduce_data(instrument_file=instrument,
calibration_file=calibration,
mask=mask,
sub_runs=subruns,
van_file=None)
reducer.save_diffraction_data(project_file_name, append_mode)
def _view_raw(hidra_workspace, mask, subruns, engine):
reducer = ReductionApp(bool(engine == 'mantid'))
# reducer.load_project_file(projectfile)
reducer.load_hidra_workspace(hidra_workspace)
# interpret None to be first subrun
if not subruns:
subruns = [0]
# TODO pylint points out that this is a non-existant function
# plot raw detector counts without reduction but possibly with masking
reducer.plot_detector_counts(sub_run=subruns[0], mask=mask)
def reduce_hidra_workflow(nexus,
output_dir,
progressbar,
instrument=None,
calibration=None,
mask=None,
vanadium_file=None,
project_file_name=None):
"""Workflow of algorithms to reduce HB2B NeXus file to powder patterns
Parameters
----------
nexus
output_dir
progressbar
instrument
calibration : str
calibration file name
mask : str or None
Mask file (so far, only Mantid XML file)
vanadium_file : str or None
Vanadium file (reduced project file with vanadium counts at sub run 1)
project_file_name : str or None
if str, then the output file name won't use the default
Returns
-------
pyrs.core.workspaces.HidraWorkspace
HiDRA workspace
"""
# Init logger
logger = Logger('reduce_HB2B')
# Create project file (name) for default
if project_file_name is None:
project_file_name = os.path.basename(nexus).split('.')[0] + '.h5'
project_file_name = os.path.join(output_dir, project_file_name)
# Remove previous existing file
if os.path.exists(project_file_name):
# overwrite existing file
if os.access(project_file_name, os.W_OK):
# log information
logger.information('Will overwrite existing projectfile {}'.format(
project_file_name))
else:
# no permission
raise RuntimeError(
'User does not have permission to overwrite existing HiDRA project file {}'
''.format(project_file_name))
else:
# file does not exist so far
base_dir = os.path.dirname(project_file_name)
if not (os.path.exists(base_dir) and os.access(base_dir, os.W_OK)):
raise RuntimeError(
'User specified HiDRA project file path {} either does not exist or '
'user does not have write access.'.format(base_dir))
# END-IF-ELSE
# Set progress bar
if progressbar:
progressbar.setVisible(True)
progressbar.setValue(0.)
# process the data
converter = NeXusConvertingApp(nexus, mask)
hidra_ws = converter.convert(use_mantid=False)
# Update
if progressbar:
progressbar.setValue(50.)
# add powder patterns
# Calculate powder pattern
logger.notice(
'Adding powder patterns to Hidra Workspace {}'.format(hidra_ws))
# Initialize a reducer
reducer = ReductionApp()
# add workspace to reducer
reducer.load_hidra_workspace(hidra_ws)
# reduce
reducer.reduce_data(instrument_file=instrument,
calibration_file=calibration,
mask=None,
van_file=vanadium_file,
sub_runs=list(hidra_ws.get_sub_runs()))
if progressbar:
progressbar.setVisible(True)
progressbar.setValue(95.)
# Save
reducer.save_diffraction_data(project_file_name)
if progressbar:
progressbar.setVisible(True)
progressbar.setValue(100.)
return hidra_ws
def test_apply_mantid_mask():
"""Test auto reduction script with Mantid mask file applied
Returns
-------
"""
# Specify NeXus
nexus_file = 'data/HB2B_938.nxs.h5'
# Convert the NeXus to file to a project without mask and convert to 2theta diffraction pattern
no_mask_project_file = 'HB2B_938_no_mask.h5'
if os.path.exists(no_mask_project_file):
os.remove(no_mask_project_file)
# Convert to NeXust
no_mask_hidra_ws = convertNeXusToProject(nexus_file, no_mask_project_file, skippable=False,
mask_file_name=None)
mask_array = no_mask_hidra_ws.get_detector_mask(is_default=True)
assert mask_array is None, 'There shall not be any mask'
# Convert the nexus file to a project file and do the "simple" checks
no_mask_reducer = ReductionApp()
no_mask_reducer.load_project_file(no_mask_project_file)
no_mask_reducer.reduce_data(sub_runs=None, instrument_file=None, calibration_file=None, mask=None,
van_file=None, num_bins=950)
no_mask_reducer.save_diffraction_data(no_mask_project_file)
# Convert the NeXus to file to a project with mask and convert to 2theta diffraction pattern
project_file = 'HB2B_938_mask.h5'
if os.path.exists(project_file):
os.remove(project_file)
# Convert
masked_hidra_ws = convertNeXusToProject(nexus_file, project_file, skippable=False,
mask_file_name='data/HB2B_Mask_12-18-19.xml')
mask_array = masked_hidra_ws.get_detector_mask(True)
# check on Mask: num_masked_pixels = (135602,)
assert np.where(mask_array == 0)[0].shape[0] == 135602, 'Mask shall have 135602 pixels masked but not {}' \
''.format(np.where(mask_array == 0)[0].shape[0])
reducer = ReductionApp()
reducer.load_project_file(project_file)
# convert to diffraction pattern with mask
reducer.reduce_data(sub_runs=None, instrument_file=None, calibration_file=None, mask=mask_array,
van_file=None, num_bins=950)
reducer.save_diffraction_data(project_file)
# Compare range of 2theta
no_mask_data_set = no_mask_reducer.get_diffraction_data(sub_run=1)
masked_data_set = reducer.get_diffraction_data(sub_run=1)
print('[DEBUG...] No mask 2theta range: {}, {}'.format(no_mask_data_set[0].min(), no_mask_data_set[0].max()))
print('[DEBUG...] Masked 2theta range: {}, {}'.format(masked_data_set[0].min(), masked_data_set[0].max()))
# verify the masked reduced data shall have smaller or at least equal range of 2theta
assert no_mask_data_set[0].min() <= masked_data_set[0].min()
assert no_mask_data_set[0].max() >= masked_data_set[0].max()
def test_exclude_subruns(nexusfile, projectfile):
"""Test converting NeXus to project and convert to diffraction pattern
Note: project file cannot be the same as NeXus file as the output file will be
removed by pytest
Parameters
----------
nexusfile
projectfile
Returns
-------
"""
sub_runs = [2, 4, 5]
# convert the nexus file to a project file and do the "simple" checks
converter = NeXusConvertingApp(nexusfile, None)
hidra_ws = converter.convert()
reducer = ReductionApp()
reducer.load_hidra_workspace(hidra_ws)
reducer.reduce_data(instrument_file=None,
calibration_file=None,
mask=None,
sub_runs=sub_runs,
van_file=None)
reducer.save_diffraction_data(projectfile)
reduced_ws = HidraWorkspace('test_powder_pattern')
reduced_project = HidraProjectFile(projectfile)
reduced_ws.load_hidra_project(reduced_project, load_raw_counts=False, load_reduced_diffraction=True)
assert sub_runs == reduced_ws.get_sub_runs()
reducer.reduce_data(instrument_file=None,
calibration_file=None,
mask=None,
sub_runs=[],
van_file=None)
for sub_run in sub_runs:
np.testing.assert_allclose(reducer.get_diffraction_data(sub_run),
reduced_ws.get_reduced_diffraction_data(sub_run))
# cleanup
reduced_project.close()
os.remove(projectfile)