def test_getH5FilePath_ref(self): refH5Master1 = "ref-UPF2-UPF2__4_1_1_master.h5" refH5Data1 = "ref-UPF2-UPF2__4_1_1_data_000001.h5" file1 = "ref-UPF2-UPF2__4_1_0001.h5" h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath( file1, hasOverlap=True) self.assertEqual(refH5Master1, str(h5MasterFilePath)) self.assertEqual(refH5Data1, str(h5DataFilePath)) refH5Master2 = "ref-UPF2-UPF2__4_1_2_master.h5" refH5Data2 = "ref-UPF2-UPF2__4_1_2_data_000001.h5" file2 = "ref-UPF2-UPF2__4_1_0002.h5" h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath( file2, hasOverlap=True) self.assertEqual(refH5Master2, str(h5MasterFilePath)) self.assertEqual(refH5Data2, str(h5DataFilePath))
def test_readEiger4mHeader(self): referenceDataPath = self.dataPath / 'ReadImageHeader_Eiger4M.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath( inData['imagePath'][0]) dictHeader = ReadImageHeader.readHdf5Header(h5MasterFilePath) self.assertEqual(dictHeader['description'], 'Dectris Eiger 4M')
def test_readEiger16mHeader(self): referenceDataPath = self.dataPath / "ReadImageHeader_Eiger16M.json" inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath( inData["imagePath"][0] ) dictHeader = ReadImageHeader.readHdf5Header(h5MasterFilePath) self.assertEqual(dictHeader["description"], "Dectris EIGER2 CdTe 16M")
def loadTestImage(imageFileName): """ This method tries to download images from http://www.edna-site.org/data/tests/images """ imageDirPath = getTestImageDirPath() if not imageDirPath.exists(): imageDirPath.mkdir(mode=0o777, parents=True) # Check if h5 data if imageFileName.endswith(".h5"): imagePath = imageDirPath / imageFileName hasOverlap = imagePath.name.startswith("ref-") h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath(imagePath, hasOverlap=hasOverlap) listImagePath = [h5MasterFilePath, h5DataFilePath] else: listImagePath = [imageDirPath / imageFileName] for imagePath in listImagePath: if not imagePath.exists(): logger.info( "Trying to download image %s" % str(imagePath) + ", timeout set to %d s" % MAX_DOWNLOAD_TIME ) if "http_proxy" in os.environ: dictProxies = {'http': os.environ["http_proxy"]} proxy_handler = ProxyHandler(dictProxies) opener = build_opener(proxy_handler).open else: opener = urlopen timer = threading.Timer(MAX_DOWNLOAD_TIME + 1, __timeoutDuringDownload) timer.start() data = opener("%s/%s" % (URL_EDNA_SITE, imagePath.name), data=None, timeout=MAX_DOWNLOAD_TIME ).read() timer.cancel() try: open(str(imagePath), "wb").write(data) except IOError: raise IOError( "Unable to write downloaded data to disk at %s" % imagePath ) if os.path.exists(str(imagePath)): logger.info("Image %s successfully downloaded." % imagePath) else: raise RuntimeError( "Could not automatically download test image %r!\n" + "If you are behind a firewall, " + "please set the environment variable http_proxy.\n" + "Otherwise please try to download the images manually from\n" + "http://www.edna-site.org/data/tests/images" % imageFileName)
def createHdf5HeaderData(cls, imagePath, skipNumberOfImages=False, hasOverlap=False, isFastMesh=False): h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath( pathlib.Path(imagePath), isFastMesh=isFastMesh, hasOverlap=hasOverlap) # Waiting for file timedOut, finalSize = UtilsPath.waitForFile(h5MasterFilePath, expectedSize=100000, timeOut=DEFAULT_TIME_OUT) if timedOut: errorMessage = "Timeout when waiting for image %s" % imagePath logger.error(errorMessage) raise BaseException(errorMessage) dictHeader = cls.readHdf5Header(h5MasterFilePath) description = dictHeader['description'] if 'Eiger 4M' in description: detectorName = 'EIGER 4M' detectorType = 'eiger4m' numberPixelX = 2070 numberPixelY = 2167 else: raise RuntimeError( '{0} cannot read image header from images with detector type {1}' .format(cls.__class__.__name__, description)) # Find out size of data set prefix = str(h5MasterFilePath).split('master')[0] listDataImage = [] noImages = 0 if not skipNumberOfImages: for data in dictHeader['data']: dataFilePath = prefix + data + '.h5' timedOut, finalSize = UtilsPath.waitForFile( dataFilePath, expectedSize=1000000, timeOut=DEFAULT_TIME_OUT) if timedOut: raise RuntimeError( 'Timeout waiting for file {0}'.format(dataFilePath)) # listDataImage.append({ # 'path': dataFilePath # }) f = h5py.File(dataFilePath, 'r') dataShape = f['entry']['data']['data'].shape noImages += dataShape[0] f.close() experimentalCondition = {} # Pixel size and beam position detector = { 'numberPixelX': int(numberPixelX), 'numberPixelY': int(numberPixelY), 'pixelSizeX': round(dictHeader['x_pixel_size'] * 1000, 3), 'pixelSizeY': round(dictHeader['y_pixel_size'] * 1000, 3), 'beamPositionX': round( float(dictHeader['beam_center_x'] * dictHeader['x_pixel_size'] * 1000), 3), 'beamPositionY': round( float(dictHeader['beam_center_y'] * dictHeader['y_pixel_size'] * 1000), 3), 'distance': round(float(dictHeader['detector_distance']) * 1000, 3), 'serialNumber': dictHeader['detector_number'], 'name': detectorName, 'type': detectorType } experimentalCondition['detector'] = detector # Beam object beam = { 'wavelength': round(float(dictHeader['wavelength']), 6), 'exposureTime': round(float(dictHeader['count_time']), 6) } experimentalCondition['beam'] = beam # Goniostat object goniostat = {} rotationAxisStart = round(float(dictHeader['omega_start']), 4) oscillationWidth = round(float(dictHeader['omega_increment']), 4) goniostat['rotationAxisStart'] = rotationAxisStart goniostat[ 'rotationAxisEnd'] = rotationAxisStart + oscillationWidth * noImages goniostat['oscillationWidth'] = oscillationWidth experimentalCondition['goniostat'] = goniostat # Create the image object masterImage = { 'path': imagePath, 'date': dictHeader['data_collection_date'], 'number': 1 } # imageNumber = UtilsImage.getImageNumber(imagePath) # image['number'] = imageNumber subWedge = { 'experimentalCondition': experimentalCondition, 'image': [masterImage] + listDataImage } return subWedge
def run(self, inData): # Format suffix format = inData.get("format", "jpeg") if format == "jpeg": thumbSuffix = ".jpeg" elif format == "jpg": thumbSuffix = ".jpg" else: raise RuntimeError("Unsupported format: {0}".format(format)) # Loop through all images listJPEGTask = [] listThumbTask = [] forcedOutputDirectory = inData.get("forcedOutputDirectory", None) for imagePath in inData["image"]: # Check image file extension imageFileName, suffix = os.path.splitext( os.path.basename(imagePath)) if not suffix in [".img", ".marccd", ".mccd", ".cbf", ".h5"]: raise RuntimeError( "Unknown image file name extension for pyarch thumbnail generator: %s" % imagePath) # Wait for image file if suffix == ".h5": h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath( imagePath, isFastMesh=True) waitFilePath = h5DataFilePath else: waitFilePath = imagePath expectedSize = self.getExpectedSize(imagePath) hasTimedOut, finalSize = UtilsPath.waitForFile( waitFilePath, expectedSize=expectedSize, timeOut=600) if hasTimedOut: raise RuntimeError( "Waiting for file {0} timed out!".format(imagePath)) outputFileName = imageFileName + thumbSuffix if forcedOutputDirectory is not None: outputPath = os.path.join(forcedOutputDirectory, outputFileName) else: outputPath = None # Create JPEG with resolution rings inDataReadHeader = { "imagePath": [imagePath], "skipNumberOfImages": True, "isFastMesh": True } readHeader = ReadImageHeader( inData=inDataReadHeader, workingDirectorySuffix=imageFileName, ) readHeader.execute() experimentalCondition = readHeader.outData["subWedge"][0][ "experimentalCondition"] detector = experimentalCondition["detector"] beam = experimentalCondition["beam"] inDataCreateJPEG = { "image": imagePath, "height": 1024, "width": 1024, "outputFileName": outputFileName, "outputPath": outputPath, "doResolutionRings": True, "pixelSizeX": detector["pixelSizeX"], "pixelSizeY": detector["pixelSizeY"], "beamPositionX": detector["beamPositionX"], "beamPositionY": detector["beamPositionY"], "distance": detector["distance"], "wavelength": beam["wavelength"], } createJPEG = CreateThumbnail(inData=inDataCreateJPEG, workingDirectorySuffix=imageFileName + "_JPEG") createJPEG.start() listJPEGTask.append(createJPEG) # Create thumbnail outputFileName = imageFileName + ".thumb" + thumbSuffix if forcedOutputDirectory is not None: outputPath = os.path.join(forcedOutputDirectory, outputFileName) else: outputPath = None inDataCreateThumb = { "image": imagePath, "height": 256, "width": 256, "outputFileName": outputFileName, "outputPath": outputPath, "doResolutionRings": True, "pixelSizeX": detector["pixelSizeX"], "pixelSizeY": detector["pixelSizeY"], "beamPositionX": detector["beamPositionX"], "beamPositionY": detector["beamPositionY"], "distance": detector["distance"], "wavelength": beam["wavelength"], } createThumb = CreateThumbnail( inData=inDataCreateThumb, workingDirectorySuffix=imageFileName + "_thumbnail") createThumb.start() listThumbTask.append(createThumb) outData = {"pathToJPEGImage": [], "pathToThumbImage": []} for task in listJPEGTask: task.join() if forcedOutputDirectory: outData["pathToJPEGImage"].append(task.outData["thumbNail"]) else: pyarchPath = self.copyThumbnailToPyarch(task) outData["pathToJPEGImage"].append(pyarchPath) for task in listThumbTask: task.join() if forcedOutputDirectory: outData["pathToThumbImage"].append(task.outData["thumbNail"]) else: pyarchPath = self.copyThumbnailToPyarch(task) outData["pathToThumbImage"].append(pyarchPath) return outData
def createThumbnail( image, format="jpg", height=512, width=512, outputPath=None, minLevel=0, maxLevel=99.95, dilatation=4, workingDirectory=None, outputFileName=None, doResolutionRings=False, pixelSizeX=None, pixelSizeY=None, beamPositionX=None, beamPositionY=None, distance=None, wavelength=None, ): imageFileName = os.path.basename(image) imagePath = image imageNameWithoutSuffix, imageSuffix = os.path.splitext(imageFileName) if imageSuffix == ".h5": imageNumber = UtilsImage.getImageNumber(image) h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath( image, isFastMesh=True) noTrials = 5 fabioImage = None while noTrials > 0: try: fabioImage = fabio.openimage.openimage(h5MasterFilePath) noTrials = 0 except Exception as e: logger.debug("Error when trying to open {0}: {1}".format( h5MasterFilePath, e)) logger.debug( "Sleeping 5s and trying again, {0} trials left".format( noTrials)) noTrials -= 1 time.sleep(5) if fabioImage is None: raise RuntimeError( "Cannot open file {0} with fabio".format(h5MasterFilePath)) logger.debug("No frames: {0}".format(fabioImage.nframes)) if imageNumber < fabioImage.nframes: numpyImage = fabioImage.getframe(imageNumber).data else: numpyImage = fabioImage.data if numpyImage.dtype == numpy.dtype("uint32"): numpyImage = numpy.where(numpyImage > 65536 * 65536 - 2, 0, numpyImage) else: numpyImage = numpy.where(numpyImage > 256 * 256 - 2, 0, numpyImage) else: fabioImage = fabio.openimage.openimage(image) numpyImage = fabioImage.data # Default format suffix = "jpg" pilFormat = "JPEG" if format is not None: if format.lower() == "png": suffix = "png" pilFormat = "PNG" # The following code has been adapted from EDPluginExecThumbnail written by J.Kieffer dtype = numpyImage.dtype sortedArray = numpyImage.flatten() sortedArray.sort() numpyImage = numpy.maximum(numpyImage, int(minLevel) * numpy.ones_like(numpyImage)) maxLevel = sortedArray[int( round(float(maxLevel) * sortedArray.size / 100.0))] if maxLevel < 25: maxLevel = 25 numpyImage = numpy.minimum(numpyImage, maxLevel * numpy.ones_like(numpyImage)) numpyImage = scipy.ndimage.morphology.grey_dilation( numpyImage, (dilatation, dilatation)) mumpyImageFloat = (numpyImage.astype(numpy.float32)) / float(maxLevel) numpyImageInt = (mumpyImageFloat * 255.0).astype(numpy.uint8) # Check if we should do resolution rings listResolution = [] if doResolutionRings: delta = (height + width) / 2000 if delta < 1.0: delta = 1.0 centreX = beamPositionX / pixelSizeX centreY = beamPositionY / pixelSizeY sizeY, sizeX = numpyImageInt.shape averageSize = (sizeX + sizeY) / 2.0 yy, xx = numpy.mgrid[:sizeY, :sizeX] circle = (xx - centreX)**2 + (yy - centreY)**2 for resolution in [1.0, 1.1, 1.2, 1.5, 2.0, 3.0, 4.0]: import math theta = math.asin(wavelength / (2 * resolution)) radius = math.tan(2 * theta) * distance / pixelSizeX listResolution.append((resolution, radius / averageSize)) numpyImageInt = numpy.where( numpy.logical_and(circle < (radius + delta)**2, circle > (radius - delta)**2), 254, numpyImageInt) pilOutputImage = ImageOps.invert(Image.fromarray(numpyImageInt, 'L')) if height is not None and width is not None: pilOutputImage = pilOutputImage.resize((width, height), Image.ANTIALIAS) width, height = pilOutputImage.size for resolution, distance in listResolution: centreX = width / 2 centreY = height / 2 textfont = ImageFont.truetype( "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", int(height / 30), encoding="unic") resolutionText = "{0} Å".format(resolution) imageEditable = ImageDraw.Draw(pilOutputImage) newDistance = distance * (height + width) / 2.0 / math.sqrt(2) imageEditable.text((centreX + newDistance - width / 20, centreY + newDistance - height / 20), resolutionText, 0, font=textfont) if width * height > ImageFile.MAXBLOCK: ImageFile.MAXBLOCK = width * height if outputPath is None: if outputFileName is None: outputPath = os.path.join( workingDirectory, os.path.splitext(imageFileName)[0] + "." + suffix) else: outputPath = os.path.join(workingDirectory, outputFileName) pilOutputImage.save(outputPath, pilFormat, quality=85, optimize=True) logger.info("Output thumbnail path: %s" % outputPath) return outputPath
def generateImageLinks(in_data, working_directory=None): first_sub_wedge = in_data["subWedge"][0] first_image_path = first_sub_wedge["image"][0]["path"] prefix = UtilsImage.getPrefix(first_image_path) suffix = UtilsImage.getSuffix(first_image_path) if suffix == "h5": lowest_xds_image_number = 1 highest_xds_image_number = 1 h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath( first_image_path, hasOverlap=True, isFastMesh=False) h5MasterFile = os.path.basename((str(h5MasterFilePath))) h5DataFile = os.path.basename((str(h5DataFilePath))) list_image_link = [ [str(h5MasterFilePath), h5MasterFile], [str(h5DataFilePath), h5DataFile], ] if working_directory is not None: os.symlink(str(h5MasterFilePath), h5MasterFile) os.symlink(str(h5DataFilePath), h5DataFile) template = h5MasterFile.replace("master", "??????") lowest_xds_image_number = None highest_xds_image_number = None for subwedge in in_data["subWedge"]: image_list = subwedge["image"] for image_dict in image_list: image_path = image_dict["path"] image_number = UtilsImage.getImageNumber(image_path) if lowest_xds_image_number is None or lowest_xds_image_number > image_number: lowest_xds_image_number = image_number if highest_xds_image_number is None or highest_xds_image_number < image_number: highest_xds_image_number = image_number else: template = "%s_xdslink_?????.%s" % (prefix, suffix) xds_lowest_image_number_global = 1 # First we have to find the smallest goniostat rotation axis start: oscillation_start_min = 0 # Loop through the list of sub wedges list_of_list = [] lowest_xds_image_number = None highest_xds_image_number = None list_spot_range = [] for sub_wedge in in_data["subWedge"]: list_image_link = [] image_list = sub_wedge["image"] goniostat = sub_wedge["experimentalCondition"]["goniostat"] oscillation_start = goniostat["rotationAxisStart"] oscillation_range = goniostat["oscillationWidth"] # First find the lowest and highest image numbers lowest_image_number = None for image in image_list: image_number = image["number"] if lowest_image_number is None or image_number < lowest_image_number: lowest_image_number = image_number # Loop through the list of images spot_range_min = None spot_range_max = None for image in image_list: image_number = image["number"] image_oscillation_start = ( oscillation_start + (image_number - lowest_image_number) * oscillation_range) # if xdsLowestImageNumberGlobal is None: # xdsLowestImageNumberGlobal = 1 + int((imageOscillationStart - oscillationStartMin) / oscillationRange) xds_image_number = xds_lowest_image_number_global + int( (image_oscillation_start - oscillation_start_min) / oscillation_range + 0.5) print( xds_image_number, image_oscillation_start, oscillation_start_min, oscillation_range, ) source_path = image["path"] target = "%s_xdslink_%05d.%s" % (prefix, xds_image_number, suffix) print([source_path, target]) list_image_link.append([source_path, target]) if working_directory is not None and not os.path.exists( target): os.symlink(source_path, target) if (lowest_xds_image_number is None or lowest_xds_image_number > xds_image_number): lowest_xds_image_number = xds_image_number if (highest_xds_image_number is None or highest_xds_image_number < xds_image_number): highest_xds_image_number = xds_image_number if spot_range_min is None or spot_range_min > xds_image_number: spot_range_min = xds_image_number if spot_range_max is None or spot_range_max < xds_image_number: spot_range_max = xds_image_number list_spot_range.append([spot_range_min, spot_range_max]) list_of_list.append(list_image_link) previous_exclude_data_range_max = 1 list_exclude_data_range = [] for spot_range_min, spot_range_max in list_spot_range: if spot_range_min > previous_exclude_data_range_max + 1: list_exclude_data_range.append( [previous_exclude_data_range_max, spot_range_min - 1]) previous_exclude_data_range_max = spot_range_max + 1 dictImageLinks = { "imageLink": list_of_list, "spotRange": list_spot_range, "dataRange": [lowest_xds_image_number, highest_xds_image_number], "excludeDataRange": list_exclude_data_range, "template": template, } return dictImageLinks
def createHdf5HeaderData(cls, imagePath, skipNumberOfImages=False, hasOverlap=False, isFastMesh=True): h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath( pathlib.Path(imagePath), isFastMesh=isFastMesh, hasOverlap=hasOverlap) # Waiting for file timedOut, finalSize = UtilsPath.waitForFile(h5MasterFilePath, expectedSize=2000000, timeOut=DEFAULT_TIME_OUT) if timedOut: errorMessage = "Timeout when waiting for image %s" % imagePath logger.error(errorMessage) raise BaseException(errorMessage) logger.info("Final size for {0}: {1}".format(h5MasterFilePath, finalSize)) noTrialsLeft = 5 dictHeader = None while noTrialsLeft > 0: try: dictHeader = cls.readHdf5Header(h5MasterFilePath) noTrialsLeft = 0 except Exception as e: logger.warning( "Cannot read header from {0}, no trials left: {1}".format( h5MasterFilePath, noTrialsLeft)) time.sleep(5) noTrialsLeft -= 1 if dictHeader is None: raise RuntimeError( "Cannot read header from {0}!".format(h5MasterFilePath)) description = dictHeader["description"] if "Eiger 4M" in description: detectorName = "EIGER 4M" detectorType = "eiger4m" numberPixelX = 2070 numberPixelY = 2167 elif "eiger" in description.lower() and "16M" in description: detectorName = "EIGER 16M" detectorType = "eiger16m" numberPixelX = 4148 numberPixelY = 4362 else: raise RuntimeError( "{0} cannot read image header from images with detector type {1}" .format(cls.__class__.__name__, description)) # Image number image_number = UtilsImage.getImageNumber(imagePath) # Find out size of data set prefix = str(h5MasterFilePath).split("master")[0] listDataImage = [] noImages = 0 if not skipNumberOfImages: for data in dictHeader["data"]: dataFilePath = prefix + data + ".h5" timedOut, finalSize = UtilsPath.waitForFile( dataFilePath, expectedSize=100000, timeOut=DEFAULT_TIME_OUT) if timedOut: raise RuntimeError( "Timeout waiting for file {0}".format(dataFilePath)) # listDataImage.append({ # 'path': dataFilePath # }) f = h5py.File(dataFilePath, "r") dataShape = f["entry"]["data"]["data"].shape noImages += dataShape[0] f.close() experimentalCondition = {} # Pixel size and beam position detector = { "numberPixelX": int(numberPixelX), "numberPixelY": int(numberPixelY), "pixelSizeX": round(dictHeader["x_pixel_size"] * 1000, 3), "pixelSizeY": round(dictHeader["y_pixel_size"] * 1000, 3), "beamPositionX": round( float(dictHeader["beam_center_x"] * dictHeader["x_pixel_size"] * 1000), 3, ), "beamPositionY": round( float(dictHeader["beam_center_y"] * dictHeader["y_pixel_size"] * 1000), 3, ), "distance": round(float(dictHeader["detector_distance"]) * 1000, 3), "serialNumber": dictHeader["detector_number"], "name": detectorName, "type": detectorType, } experimentalCondition["detector"] = detector # Beam object beam = { "wavelength": round(float(dictHeader["wavelength"]), 6), "exposureTime": round(float(dictHeader["count_time"]), 6), } experimentalCondition["beam"] = beam # Goniostat object goniostat = {} rotationAxisStart = round(float(dictHeader["omega_start"]), 4) oscillationWidth = round(float(dictHeader["omega_range_average"]), 4) # Offset for the image number rotationAxisStart += (image_number - 1) * oscillationWidth goniostat["rotationAxisStart"] = rotationAxisStart goniostat["rotationAxisEnd"] = rotationAxisStart + oscillationWidth goniostat["oscillationWidth"] = oscillationWidth experimentalCondition["goniostat"] = goniostat # Create the image object image_dict = { "path": imagePath, "date": dictHeader["data_collection_date"], "number": 1, } # imageNumber = UtilsImage.getImageNumber(imagePath) # image['number'] = imageNumber subWedge = { "experimentalCondition": experimentalCondition, "image": [image_dict], } return subWedge