def subtract_single_frame(pathToFrames, pathToSubtracted, namePrefix,
singleFrame, maskFrame):
"""Subtracts a single frame from a dataset without any further alterations such as a flux correction.
pathToFrames ... string location of the folder which contains the frames
pathToSubtracted ... string location where the processed frames should be saved
namePrefix ... string short text that is added to each newly calculated frame
singleFrame ... fabio.frame this frame will be substracted from the dataset
maskFrame ... fabio.frame frame which contains all pixel that should be masked
"""
helping_tools.check_folder(pathToSubtracted)
singleData = singleFrame.data.copy()
print("Reading masks, please wait!")
maskUntrusted, maskDefective, maskHot = cbf_tools.generate_all_unwanted_pixel(
maskFrame, 1000000)
print("starting subtracting\n")
frameset = cbf_tools.Frameset(pathToFrames)
for fileName in frameset.generate_frame_names_from_template():
frame = fabio.open(fileName)
frame.data -= singleData.astype(
np.int32) # here is the actual frame subtraction
frame.data = frame.data.round().astype(
np.int32) # make resonable counts
frame.data = cbf_tools.restore_pixel_mask(frame, maskUntrusted,
maskDefective, maskHot)
fileName = os.path.basename(
fileName) # preparing writing to new location
frame.save(os.path.join(pathToSubtracted, namePrefix + fileName))
print("Frame subtracted from %s" % fileName, end='\r')
del frame # cleaning up memory
print("\nDone!")
def update_masked_intensity(sampleFrame,
pathToFrames,
frameNameTemplate,
pathToUpdated,
updatedPrefix,
updateIntensity=-1):
"""
Changes the already provided masked intisities from MASKED_BRAGG_INTENSITY with to a new intensity.
sampleFrame ... fabio.frame some frame from which the dimensions can be read.
pathToFrames ... string location of the folder which contains the frames.
frameNameTemplate ... string example of how the frames are named, allows for percent substitution.
pathToPunched ... string location where the processed frames should be saved.
punchedPrefix ... string prefix that should be put in front of the processed frames.
"""
frameShape = sampleFrame.data.shape
helping_tools.check_folder(pathToUpdated)
frameset = cbf_tools.Frameset(pathToFrames, frameNameTemplate)
# punching, the sorting function is used for a nice print out
for fileName in frameset.generate_frame_names_from_template():
print("Updating " + str(fileName), end="\r")
frame = fabio.open(fileName)
data = np.array(frame.data)
data[data <= MASKED_BRAGG_INTENSITY] = updateIntensity
frame.data = data
frame.write(
os.path.join(pathToUpdated,
updatedPrefix + os.path.basename(fileName)))
del frame # freeing memory
print("\nUpdating complete!")
def subtract_hybrid_background(pathToFrames, pathToSubtracted,
backgroundFrameNames, backgroundMixture, bgName,
maskFrame):
"""Generates a flux normalized background from multiple sources and subtracts it from the raw data.
pathToFrames ... string location of the folder which contains the frames
pathToSubtracted ... string location where the processed frames should be saved
backgroundFrameNames ... array[string] names of the background files which should be used
backgroundMixture ... array[float] contribution of each background frame for the final background image
bgName ... string prefix which should be added to the modified frames
maskFrame ... fabio.frame frame which contains all pixel that should be masked
"""
bgFluxes = []
bgData = []
bgCount = len(backgroundFrameNames)
for fileName in backgroundFrameNames:
bgFluxes.append(get_flux_from_file_name(fileName))
# grab the file extension, NOTE: extension includes the dot!!!!!!!!!!!
fileType = os.path.splitext(fileName)[1]
if fileType == '.h5':
bgData.append(cbf_tools.h5_to_numpy(fileName, ()))
elif fileType == '.cbf':
bgData.append(fabio.open(fileName).data)
helping_tools.check_folder(pathToSubtracted)
print("Reading masks, please wait!")
maskUntrusted, maskDefective, maskHot = cbf_tools.generate_all_unwanted_pixel(
maskFrame, 1000000)
print("starting subtracting\n")
frameset = cbf_tools.Frameset(pathToFrames)
for fileName in frameset.generate_frame_names_from_template():
frame = fabio.open(fileName)
frameFlux = cbf_tools.get_flux(frame)
# mix the background frame
bgAll = np.zeros(frame.data.shape)
for i in range(bgCount):
scale = frameFlux / bgFluxes[i]
bgAll += bgData[i] * scale * backgroundMixture[i]
frame.data -= bgAll # here is the actual backround subtraction
frame.data = frame.data.round().astype(
np.int32) # make resonable counts
frame.data = cbf_tools.restore_pixel_mask(frame, maskUntrusted,
maskDefective, maskHot)
fileName = os.path.basename(
fileName) # preparing writing to new location
frame.save(os.path.join(pathToSubtracted, bgName + fileName))
print("Background subtracted from %s" % fileName, end='\r')
del frame # cleaning up memory
print("\nDone!")
def punch_Bragg(XDSPunchFilePath,
sampleFrame,
pathToFrames,
frameNameTemplate,
pathToPunched,
punchedPrefix,
maskedIntensity=MASKED_BRAGG_INTENSITY):
"""
Masks all Bragg peaks using XDS BGK files. For an unkown reason, when XDS's BKGPIX.cbf is substracted from a raw frame, then all positive intensities represent Bragg peaks.
XDSPunchFilePath ... string location of the BKGPIX file create by XDS. NOTE: fabio can not read the that file by default, I used ALBULA to convert it to an h5 file!
sampleFrame ... fabio.frame some frame from which the dimensions can be read.
pathToFrames ... string location of the folder which contains the frames.
frameNameTemplate ... string example of how the frames are named, allows for percent substitution.
pathToPunched ... string location where the processed frames should be saved.
punchedPrefix ... string prefix that should be put in front of the processed frames.
maskedIntensity ... int placeholder intensity which identifies it as masked.
"""
frameShape = sampleFrame.data.shape
# generating the background data
bgType = os.path.splitext(XDSPunchFilePath)[1]
if bgType == ".h5":
bg = xds_tools.bgFromH5(XDSPunchFilePath, frameShape)
else:
raise TypeError("Background file not supported!")
helping_tools.check_folder(pathToPunched)
frameset = cbf_tools.Frameset(pathToFrames, frameNameTemplate)
# punching, the sorting function is used for a nice print out
for fileName in frameset.generate_frame_names_from_template():
print("Punching " + str(fileName), end="\r")
frame = fabio.open(fileName)
punched = frame.data - bg
for x in range(frameShape[0]):
for y in range(frameShape[1]):
if punched[x, y] >= 0:
frame.data[x, y] = MASKED_BRAGG_INTENSITY
frame.write(
os.path.join(pathToPunched,
punchedPrefix + os.path.basename(fileName)))
del frame # freeing memory
print("\nPunching complete!")
def SavGol_filter(pathToFrames, nameTemplate, frameRange, pathToFiltered,
namePrefix, maskFrame, subsize, windowLength, polyOrder):
"""Subtracts a flux normalized frame from a dataset.
pathToFrames ... string location of the folder which contains the frames
pathToSubtracted ... string location where the processed frames should be saved
namePrefix ... string short text that is added to each newly calculated frame
single ... fabio.frame this frame will be substracted from the dataset
maskFrame ... fabio.frame frame which contains all pixel that should be masked
"""
helping_tools.check_folder(pathToFiltered)
print("Reading masks, please wait!")
# maskUntrusted, maskDefective, maskHot = cbf_tools.generate_all_unwanted_pixel(maskFrame, 1000000)
print("starting filtering\n")
# generating frame paths and names for reading
frameset = cbf_tools.Frameset(pathToFrames, nameTemplate)
frameset.setSize = frameRange
fileNames = frameset.generate_frame_names_from_template()
# generating frame paths and names for writing
frameset = cbf_tools.Frameset(pathToFiltered, namePrefix + nameTemplate)
frameset.setSize = frameRange
newFiles = frameset.generate_frame_names_from_template()
templateFrame = fabio.open(fileNames[0])
# determination of how many tiles are necessary for the subdivition of the frames
tilesx = int(templateFrame.data.shape[0] / subsize) + 1
tilesy = int(templateFrame.data.shape[1] / subsize) + 1
for subx in range(tilesx):
for suby in range(tilesy):
print("\nWorking on sub %i of %i" %
((suby * (subx + 1) + subx), tilesx * tilesy))
# generation of the subframe size taking the border regions into account
if (subx + 2) > tilesx:
width = templateFrame.data.shape[0] - subx * subsize
else:
width = subsize
if (suby + 2) > tilesy:
height = templateFrame.data.shape[1] - suby * subsize
else:
height = subsize
print("Width %i, height %i" % (width, height))
subFrame = np.zeros((width, height, frameRange))
for i in range(frameRange):
print("Reading frame " + fileNames[i]) #, end="\r")
frame = fabio.open(fileNames[i])
subFrame[:, :,
i] = frame.data[subx * subsize:subx * subsize + width,
suby * subsize:suby * subsize +
height].copy()
del frame # cleaning memory
print("\nApplying SavGol filter...")
for x in range(subFrame.shape[0]):
for y in range(subFrame.shape[1]):
print(x, y, end="\r")
filterLine = subFrame[x, y, :]
subFrame[x,
y, :] = scipy.signal.savgol_filter(
filterLine,
windowLength,
polyOrder,
mode='wrap').copy()
subframe = subFrame.astype(np.int32)
for i in range(frameRange):
print("Writing frame " + newFiles[i]) #, end="\r")
frame = fabio.open(newFiles[i])
frame.data[subx * subsize:subx * subsize + width, suby *
subsize:suby * subsize + height] = subFrame[:, :, i]
frame.save(
os.path.join(pathToFiltered, namePrefix +
os.path.basename(frame.filename)))
del frame # cleaning memory
print("\nDone!")
def generate_subframe_background_percentile(pathToFrames,
pathToBackground,
nameTemplate,
frameRange,
subsize,
percentile,
outputName,
outputModifiers=None):
"""Creates a background by only reading in parts of frames and puzzeling these parts together.
pathToFrames ... string location of the folder which contains the frames
pathToBackground ... string location where the background frame should be placed
nameTemplate ... string format of the frame names, allows percent substitution
frameRange ... int maximum number of frames over which to run the algorithm
subsize ... int number of pixels in x and y directions to determine the subframe size
this is used to save memory
percentile ... numeric the percentile of the frames which should be considered as background
outputName ... string name of the finished background frame, allows percent substituiton
outputModifiers ... string plus-sign seperated string list, these modfieres are used to susbtitute outputName
"""
# parse the modifiers
outputModifiers = helping_tools.parse_substition_modifiers(outputModifiers)
fileNames = []
frameset = cbf_tools.Frameset(pathToFrames, nameTemplate)
fileNames = frameset.generate_frame_names_from_template()
templateFrame = fabio.open(fileNames[0]) # just a prototype
bg = np.zeros((templateFrame.data.shape[0], templateFrame.data.shape[1]))
# determination of how many tiles are necessary for the subdivition of the frames
tilesx = int(templateFrame.data.shape[0] / subsize) + 1
tilesy = int(templateFrame.data.shape[1] / subsize) + 1
for subx in range(tilesx):
for suby in range(tilesy):
print("\nWorking on sub %i of %i" % ((subx + 1) *
(suby + 1), tilesx * tilesy))
# generation of the subframe size taking the border regions into account
if (subx + 2) > tilesx:
width = templateFrame.data.shape[0] - subx * subsize
else:
width = subsize
if (suby + 2) > tilesy:
height = templateFrame.data.shape[1] - suby * subsize
else:
height = subsize
print("Width %i, height %i" % (width, height))
subFrame = np.zeros((width, height, frameRange))
for i in range(frameRange):
print("Reading frame " + fileNames[i], end="\r")
frame = fabio.open(fileNames[i])
subFrame[:, :,
i] = frame.data[subx * subsize:subx * subsize + width,
suby * subsize:suby * subsize +
height].copy()
del frame # cleaning memory
print("\nCalculating percentile")
bg[subx * subsize:subx * subsize + width,
suby * subsize:suby * subsize + height] = get_percentile(
subFrame, subFrame.shape, percentile)
helping_tools.check_folder(pathToBackground)
# create and write the flux monitor
fluxFileName = "fluxmonitor_" + outputName + ".csv"
flux = cbf_tools.average_flux(pathToFrames, nameTemplate, pathToBackground,
fluxFileName % outputModifiers, frameRange)
# writing the background file
templateFrame.data = bg.astype(np.int32)
fileName, fileExtension = os.path.splitext(outputName)
# splicing the average flux into the file name and prepare the extension
outputName = fileName + "_flux_" + str(flux) + ".cbf"
# write the cbf file
templateFrame.write(
os.path.join(pathToBackground, outputName % outputModifiers))
# write the h5 file
cbf_tools.frame_to_h5(templateFrame,
os.path.join(pathToBackground, outputName + ".h5"),
outputModifiers)