def subtract_single_frame(pathToFrames, pathToSubtracted, namePrefix, singleFrame, maskFrame): """Subtracts a single frame from a dataset without any further alterations such as a flux correction. pathToFrames ... string location of the folder which contains the frames pathToSubtracted ... string location where the processed frames should be saved namePrefix ... string short text that is added to each newly calculated frame singleFrame ... fabio.frame this frame will be substracted from the dataset maskFrame ... fabio.frame frame which contains all pixel that should be masked """ helping_tools.check_folder(pathToSubtracted) singleData = singleFrame.data.copy() print("Reading masks, please wait!") maskUntrusted, maskDefective, maskHot = cbf_tools.generate_all_unwanted_pixel( maskFrame, 1000000) print("starting subtracting\n") frameset = cbf_tools.Frameset(pathToFrames) for fileName in frameset.generate_frame_names_from_template(): frame = fabio.open(fileName) frame.data -= singleData.astype( np.int32) # here is the actual frame subtraction frame.data = frame.data.round().astype( np.int32) # make resonable counts frame.data = cbf_tools.restore_pixel_mask(frame, maskUntrusted, maskDefective, maskHot) fileName = os.path.basename( fileName) # preparing writing to new location frame.save(os.path.join(pathToSubtracted, namePrefix + fileName)) print("Frame subtracted from %s" % fileName, end='\r') del frame # cleaning up memory print("\nDone!")
def update_masked_intensity(sampleFrame, pathToFrames, frameNameTemplate, pathToUpdated, updatedPrefix, updateIntensity=-1): """ Changes the already provided masked intisities from MASKED_BRAGG_INTENSITY with to a new intensity. sampleFrame ... fabio.frame some frame from which the dimensions can be read. pathToFrames ... string location of the folder which contains the frames. frameNameTemplate ... string example of how the frames are named, allows for percent substitution. pathToPunched ... string location where the processed frames should be saved. punchedPrefix ... string prefix that should be put in front of the processed frames. """ frameShape = sampleFrame.data.shape helping_tools.check_folder(pathToUpdated) frameset = cbf_tools.Frameset(pathToFrames, frameNameTemplate) # punching, the sorting function is used for a nice print out for fileName in frameset.generate_frame_names_from_template(): print("Updating " + str(fileName), end="\r") frame = fabio.open(fileName) data = np.array(frame.data) data[data <= MASKED_BRAGG_INTENSITY] = updateIntensity frame.data = data frame.write( os.path.join(pathToUpdated, updatedPrefix + os.path.basename(fileName))) del frame # freeing memory print("\nUpdating complete!")
def subtract_hybrid_background(pathToFrames, pathToSubtracted, backgroundFrameNames, backgroundMixture, bgName, maskFrame): """Generates a flux normalized background from multiple sources and subtracts it from the raw data. pathToFrames ... string location of the folder which contains the frames pathToSubtracted ... string location where the processed frames should be saved backgroundFrameNames ... array[string] names of the background files which should be used backgroundMixture ... array[float] contribution of each background frame for the final background image bgName ... string prefix which should be added to the modified frames maskFrame ... fabio.frame frame which contains all pixel that should be masked """ bgFluxes = [] bgData = [] bgCount = len(backgroundFrameNames) for fileName in backgroundFrameNames: bgFluxes.append(get_flux_from_file_name(fileName)) # grab the file extension, NOTE: extension includes the dot!!!!!!!!!!! fileType = os.path.splitext(fileName)[1] if fileType == '.h5': bgData.append(cbf_tools.h5_to_numpy(fileName, ())) elif fileType == '.cbf': bgData.append(fabio.open(fileName).data) helping_tools.check_folder(pathToSubtracted) print("Reading masks, please wait!") maskUntrusted, maskDefective, maskHot = cbf_tools.generate_all_unwanted_pixel( maskFrame, 1000000) print("starting subtracting\n") frameset = cbf_tools.Frameset(pathToFrames) for fileName in frameset.generate_frame_names_from_template(): frame = fabio.open(fileName) frameFlux = cbf_tools.get_flux(frame) # mix the background frame bgAll = np.zeros(frame.data.shape) for i in range(bgCount): scale = frameFlux / bgFluxes[i] bgAll += bgData[i] * scale * backgroundMixture[i] frame.data -= bgAll # here is the actual backround subtraction frame.data = frame.data.round().astype( np.int32) # make resonable counts frame.data = cbf_tools.restore_pixel_mask(frame, maskUntrusted, maskDefective, maskHot) fileName = os.path.basename( fileName) # preparing writing to new location frame.save(os.path.join(pathToSubtracted, bgName + fileName)) print("Background subtracted from %s" % fileName, end='\r') del frame # cleaning up memory print("\nDone!")
def punch_Bragg(XDSPunchFilePath, sampleFrame, pathToFrames, frameNameTemplate, pathToPunched, punchedPrefix, maskedIntensity=MASKED_BRAGG_INTENSITY): """ Masks all Bragg peaks using XDS BGK files. For an unkown reason, when XDS's BKGPIX.cbf is substracted from a raw frame, then all positive intensities represent Bragg peaks. XDSPunchFilePath ... string location of the BKGPIX file create by XDS. NOTE: fabio can not read the that file by default, I used ALBULA to convert it to an h5 file! sampleFrame ... fabio.frame some frame from which the dimensions can be read. pathToFrames ... string location of the folder which contains the frames. frameNameTemplate ... string example of how the frames are named, allows for percent substitution. pathToPunched ... string location where the processed frames should be saved. punchedPrefix ... string prefix that should be put in front of the processed frames. maskedIntensity ... int placeholder intensity which identifies it as masked. """ frameShape = sampleFrame.data.shape # generating the background data bgType = os.path.splitext(XDSPunchFilePath)[1] if bgType == ".h5": bg = xds_tools.bgFromH5(XDSPunchFilePath, frameShape) else: raise TypeError("Background file not supported!") helping_tools.check_folder(pathToPunched) frameset = cbf_tools.Frameset(pathToFrames, frameNameTemplate) # punching, the sorting function is used for a nice print out for fileName in frameset.generate_frame_names_from_template(): print("Punching " + str(fileName), end="\r") frame = fabio.open(fileName) punched = frame.data - bg for x in range(frameShape[0]): for y in range(frameShape[1]): if punched[x, y] >= 0: frame.data[x, y] = MASKED_BRAGG_INTENSITY frame.write( os.path.join(pathToPunched, punchedPrefix + os.path.basename(fileName))) del frame # freeing memory print("\nPunching complete!")
def SavGol_filter(pathToFrames, nameTemplate, frameRange, pathToFiltered, namePrefix, maskFrame, subsize, windowLength, polyOrder): """Subtracts a flux normalized frame from a dataset. pathToFrames ... string location of the folder which contains the frames pathToSubtracted ... string location where the processed frames should be saved namePrefix ... string short text that is added to each newly calculated frame single ... fabio.frame this frame will be substracted from the dataset maskFrame ... fabio.frame frame which contains all pixel that should be masked """ helping_tools.check_folder(pathToFiltered) print("Reading masks, please wait!") # maskUntrusted, maskDefective, maskHot = cbf_tools.generate_all_unwanted_pixel(maskFrame, 1000000) print("starting filtering\n") # generating frame paths and names for reading frameset = cbf_tools.Frameset(pathToFrames, nameTemplate) frameset.setSize = frameRange fileNames = frameset.generate_frame_names_from_template() # generating frame paths and names for writing frameset = cbf_tools.Frameset(pathToFiltered, namePrefix + nameTemplate) frameset.setSize = frameRange newFiles = frameset.generate_frame_names_from_template() templateFrame = fabio.open(fileNames[0]) # determination of how many tiles are necessary for the subdivition of the frames tilesx = int(templateFrame.data.shape[0] / subsize) + 1 tilesy = int(templateFrame.data.shape[1] / subsize) + 1 for subx in range(tilesx): for suby in range(tilesy): print("\nWorking on sub %i of %i" % ((suby * (subx + 1) + subx), tilesx * tilesy)) # generation of the subframe size taking the border regions into account if (subx + 2) > tilesx: width = templateFrame.data.shape[0] - subx * subsize else: width = subsize if (suby + 2) > tilesy: height = templateFrame.data.shape[1] - suby * subsize else: height = subsize print("Width %i, height %i" % (width, height)) subFrame = np.zeros((width, height, frameRange)) for i in range(frameRange): print("Reading frame " + fileNames[i]) #, end="\r") frame = fabio.open(fileNames[i]) subFrame[:, :, i] = frame.data[subx * subsize:subx * subsize + width, suby * subsize:suby * subsize + height].copy() del frame # cleaning memory print("\nApplying SavGol filter...") for x in range(subFrame.shape[0]): for y in range(subFrame.shape[1]): print(x, y, end="\r") filterLine = subFrame[x, y, :] subFrame[x, y, :] = scipy.signal.savgol_filter( filterLine, windowLength, polyOrder, mode='wrap').copy() subframe = subFrame.astype(np.int32) for i in range(frameRange): print("Writing frame " + newFiles[i]) #, end="\r") frame = fabio.open(newFiles[i]) frame.data[subx * subsize:subx * subsize + width, suby * subsize:suby * subsize + height] = subFrame[:, :, i] frame.save( os.path.join(pathToFiltered, namePrefix + os.path.basename(frame.filename))) del frame # cleaning memory print("\nDone!")
def generate_subframe_background_percentile(pathToFrames, pathToBackground, nameTemplate, frameRange, subsize, percentile, outputName, outputModifiers=None): """Creates a background by only reading in parts of frames and puzzeling these parts together. pathToFrames ... string location of the folder which contains the frames pathToBackground ... string location where the background frame should be placed nameTemplate ... string format of the frame names, allows percent substitution frameRange ... int maximum number of frames over which to run the algorithm subsize ... int number of pixels in x and y directions to determine the subframe size this is used to save memory percentile ... numeric the percentile of the frames which should be considered as background outputName ... string name of the finished background frame, allows percent substituiton outputModifiers ... string plus-sign seperated string list, these modfieres are used to susbtitute outputName """ # parse the modifiers outputModifiers = helping_tools.parse_substition_modifiers(outputModifiers) fileNames = [] frameset = cbf_tools.Frameset(pathToFrames, nameTemplate) fileNames = frameset.generate_frame_names_from_template() templateFrame = fabio.open(fileNames[0]) # just a prototype bg = np.zeros((templateFrame.data.shape[0], templateFrame.data.shape[1])) # determination of how many tiles are necessary for the subdivition of the frames tilesx = int(templateFrame.data.shape[0] / subsize) + 1 tilesy = int(templateFrame.data.shape[1] / subsize) + 1 for subx in range(tilesx): for suby in range(tilesy): print("\nWorking on sub %i of %i" % ((subx + 1) * (suby + 1), tilesx * tilesy)) # generation of the subframe size taking the border regions into account if (subx + 2) > tilesx: width = templateFrame.data.shape[0] - subx * subsize else: width = subsize if (suby + 2) > tilesy: height = templateFrame.data.shape[1] - suby * subsize else: height = subsize print("Width %i, height %i" % (width, height)) subFrame = np.zeros((width, height, frameRange)) for i in range(frameRange): print("Reading frame " + fileNames[i], end="\r") frame = fabio.open(fileNames[i]) subFrame[:, :, i] = frame.data[subx * subsize:subx * subsize + width, suby * subsize:suby * subsize + height].copy() del frame # cleaning memory print("\nCalculating percentile") bg[subx * subsize:subx * subsize + width, suby * subsize:suby * subsize + height] = get_percentile( subFrame, subFrame.shape, percentile) helping_tools.check_folder(pathToBackground) # create and write the flux monitor fluxFileName = "fluxmonitor_" + outputName + ".csv" flux = cbf_tools.average_flux(pathToFrames, nameTemplate, pathToBackground, fluxFileName % outputModifiers, frameRange) # writing the background file templateFrame.data = bg.astype(np.int32) fileName, fileExtension = os.path.splitext(outputName) # splicing the average flux into the file name and prepare the extension outputName = fileName + "_flux_" + str(flux) + ".cbf" # write the cbf file templateFrame.write( os.path.join(pathToBackground, outputName % outputModifiers)) # write the h5 file cbf_tools.frame_to_h5(templateFrame, os.path.join(pathToBackground, outputName + ".h5"), outputModifiers)