def test_pipeline(self): # Make sure the test file has been unzipped. if not os.path.isfile(_TEST_INPUT_FILE2): if VERBOSE: print("Unzipping", _TEST_INPUT_FILE2_ZIP) zf = ZipFile(_TEST_INPUT_FILE2_ZIP) zf.extractall(_datapath) # Test the alternative pipeline API for the simulator. test_map = MiriIlluminationModel(_TEST_INPUT_FILE2) test_map.set_instrument_metadata(_DEFAULT_SCA) sca = SensorChipAssembly3(logger=LOGGER) with warnings.catch_warnings(): # Suppress FITS header warnings. warnings.simplefilter("ignore") exposure_data = sca.simulate_pipe( test_map, scale=1.0, fringemap=None, readout_mode=_DEFAULT_READOUT, subarray=None, nints=1, ngroups=10, temperature=6.5, cosmic_ray_mode='NONE', simulate_dark_current=SIMULATE_DARK, verbose=0) # The simulated data must have a valid shape and contain a # range of values. self.assertTrue(exposure_data.data.shape[0] > 0) self.assertTrue(exposure_data.data.shape[1] > 0) self.assertNotAlmostEqual(exposure_data.data.min(), exposure_data.data.max()) del test_map, exposure_data, sca
def test_subarray_modes(self): # Test the extraction of subarrays from full frame data. test_map = MiriIlluminationModel(_TEST_INPUT_FILE2) map_data = test_map.get_illumination() # The illumination data is assumed to have a valid shape and be # filled with a range of values (the data should have a slope). self.assertTrue(map_data.shape[0] > 0) self.assertTrue(map_data.shape[1] > 0) self.assertNotAlmostEqual(map_data.min(), map_data.max()) if VERBOSE: print("\nTesting FULL to subarray modes: ", end='') if TEST_MODE_COVERAGE > 0.99: modes_to_test = list(detector_properties['SUBARRAY'].keys()) random.shuffle(modes_to_test) else: nsamples = int(0.5 + len(list(detector_properties['SUBARRAY'].keys())) * \ TEST_MODE_COVERAGE) nsamples = max(nsamples, MINSAMPLES) modes_to_test = random.sample( \ list(detector_properties['SUBARRAY'].keys()), nsamples) sca = SensorChipAssembly3(logger=LOGGER) with warnings.catch_warnings(): # Suppress FITS header warnings. warnings.simplefilter("ignore") for submode in modes_to_test: if VERBOSE: print(submode + ' ', end='') sca.setup(_DEFAULT_SCA, readout_mode=_DEFAULT_READOUT, subarray=submode, inttime=None, ngroups=2, nints=1, temperature=6.5, cosmic_ray_mode='NONE', simulate_dark_current=SIMULATE_DARK, verbose=0) sca.set_illumination(test_map) simulated_data = sca.exposure() # The simulated data must have a valid shape and contain a # range of values. (A more specific test is difficult because # read noise is included. The accuracy of subarray extraction # should already have been tested in the test_detector unit # tests.) self.assertTrue(simulated_data.shape[0] > 0) self.assertTrue(simulated_data.shape[1] > 0) self.assertNotAlmostEqual(simulated_data.min(), simulated_data.max()) if VERBOSE: print("", end='') del test_map, sca
columns = subarray[3] if verbose > 0: print("Defining test data for subarray %s." % subarray_str) print("Setting columns=%d, rows=%d." % (rows, columns)) except (KeyError, IndexError): strg = "Unrecognised or badly defined subarray mode: %s" % \ subarray_str raise AttributeError(strg) # Create a new illumination data datavalues = create_cross_data(rows, columns, minvalue=minvalue, maxvalue=maxvalue, addfifth=addfifth) wdata = 5.0 * np.ones_like(datavalues) map_object = MiriIlluminationModel(intensity=datavalues, wavelength=wdata) map_object.set_instrument_metadata("MIRIMAGE") if subarray_str: map_object.set_subarray_metadata(subarray_str) if verbose > 1: print(map_object) map_object.plot() # Save the illumination map to a FITS file. map_object.save(outputfile, overwrite=overwrite) if verbose > 0: print("New illumination map saved to %s." % outputfile)
def load_illumination_map(filename, ftype='FITS', metadata=None, scale=1.0, add_wavelength=True, wavmin=1.0, wavmax=30.0): """ Create an IlluminationMap object from a file of detector illumination data. :Parameters: filename: string or tuple of strings The name(s) of the file(s) to be opened. ftype: string, optional, default='FITS' The type of file from which to read the data. * If 'STSCI', the illumination map is read from a standard MIRI illumination data file, based on the STScI data model. * If 'FITS', intensity and wavelength data are obtained from a single FITS file containing INTENSITY, WAVELENGTH and DIRECTION extensions (with the latter two extensions being optional). * If 'ASCII', intensity and wavelength data are read from a list of 1, 2 or 3 ASCII files (with the latter two files being optional). Intensity values are multiplied by the given scale factor. ASCII files describing an illumination map define a value for every pixel. They should contain nrows lines, each of which contains ncolumns of blank-separated numbers; i.e. the default format expected by the numpy.loadtxt() function. * If 'IMAGE' (or 'JPEG'), intensity data are read from a standard image format file (such as JPEG, GIF or PPM) whose contents are converted into a composite image. A simple wavelength image may be added varying between a specified minimum and maximum wavelength, or wavelength data may be ignored. Intensity values are multiplied by the given scale factor. metadata: dictionary-like object, optional An object containing metadata to be associated with the data. It could be a plain Python dictionary, a Metadata object or a pyFits Header object, as long as it supports keyword operations. (Only used if the input file is ASCII or IMAGE.) scale: float, optional, default=1.0 An optional scale factor to apply to the intensity data. This is more useful for ftype='IMAGE' data in cases where the data do not contain a photon flux. add_wavelength: boolean, optional, default=True Add a test wavelength map to the data. This parameter is valid only for ftype='IMAGE' data. wavmin: float, optional, default=1.0 Minimum wavelength associated with data in microns. This parameter is valid only for ftype='IMAGE' data. wavmax: float, optional, default=30.0 Maximum wavelength associated with data in microns. This parameter is valid only for ftype='IMAGE' data. :Raises: ValueError Raised if any of the parameters are out of range. IOError Raised if there is an error opening, reading or interpreting the data from the input file. ImportError Raised if the Python Imaging Library (PIL) is not available. :Returns: A new MiriIlluminationModel object. """ # Attempt to open and then read the given file if ftype == 'STSCI' or ftype == 'STScI': # Read a MIRI data model in standard STScI format. illumination_map = MiriIlluminationModel(filename) illumination_map.apply_scale(scale) return illumination_map elif ftype == 'FITS': # Read a FITS file in the agreed detector illumination format. # The file must contain a primary FITS header and an INTENSITY # extension. It may also contain WAVELENGTH and DIRECTION # extensions. try: hdulist = pyfits.open(filename) # Read the contents of the file header = hdulist[0].header description = 'Metadata extracted from %s' % filename metadata = Metadata(description) metadata.from_fits_header(header) intensity = scale * hdulist['INTENSITY'].data intensity_header = hdulist['INTENSITY'].header intensity_metadata = Metadata('Intensity metadata') intensity_metadata.from_fits_header(intensity_header) except Exception as e: # If the file could not be read re-raise the exception # with a more meaningful error message. strg = "%s: Could not read illumination data file %s.\n %s" % \ (e.__class__.__name__, filename, e) raise IOError(strg) # The wavelength data is optional try: wavelength = hdulist['WAVELENGTH'].data wavelength_header = hdulist['WAVELENGTH'].header wavelength_metadata = Metadata('Wavelength metadata') wavelength_metadata.from_fits_header(wavelength_header) except (KeyError, AttributeError): wavelength = None wavelength_metadata = None hdulist.close() elif ftype == 'ASCII': if isinstance(filename, str): # A single string is provided - there is just an intensity file intensity = scale * np.loadtxt(filename) intensity_metadata = None wavelength = None wavelength_metadata = None else: # A list of strings has been provided. Attempt to open each # file, ignoring the wavelength file if not specified or it # doesn't exist. intensity = scale * np.loadtxt(filename[0]) intensity_metadata = None try: wavelength = np.loadtxt(filename[1]) except Exception: wavelength = None wavelength_metadata = None elif ftype == 'IMAGE' or ftype == 'JPEG': if _PIL_AVAILABLE: # Convert to a numpy array of the correct shape, reducing the # amplitude by the given scale factor. data = _np_from_jpeg(filename) intensity = scale * data.astype(np.float32) intensity_metadata = None # Add some test wavelength data if required. The wavelength will # increase linearly from bottom to top over the range specified. if add_wavelength: wavelength = np.empty_like(intensity) wav = np.linspace(wavmin, wavmax, wavelength.shape[1]) wavelength[:, :] = wav wavelength = np.transpose(wavelength) else: wavelength = None wavelength_metadata = None else: strg = "Sorry, file format %s can't be processed " \ "because the Python Imaging Library is not available." % ftype raise ImportError(strg) else: # Other file formats are not yet supported strg = "Sorry, file format %s is not supported." % ftype raise ValueError(strg) # Bail out if the intensity data has not been read successfully. if intensity is None: strg = "No intensity data could be found within file.\n %s" % \ filename raise ValueError(strg) # Ensure that the intensity and wavelength arrays are # compatible. illumination_shape = intensity.shape[-2:] if len(intensity.shape) > 2: slices = intensity.shape[0] else: slices = 0 # If the wavelength array is 1-D and the intensity array is 3-D # the wavelength array needs to be reshaped into a 3-D array. if wavelength is not None: if len(wavelength.shape) == 1 and len(intensity.shape) > 2: sz1 = wavelength.shape[0] wavelength.shape = [sz1, 1, 1] if check_wavelength(wavelength, illumination_shape, slices) == False: print("WARNING: Wavelength array has unexpected size - ignoring it.") wavelength = None # Finally, create and return an IlluminationMap object illum = MiriIlluminationModel(intensity=intensity, wavelength=wavelength) return illum
def create_test_data(rows, columns, pattern, values, metadata=None, cloneimage=None, add_wavelength=True, wavmin=1.0, wavmax=30.0, seedvalue=None): """ Create an data object object containing a test pattern, which can be useful for generating artificial dark and flat-field data, for example. :Parameters: rows: int Number of rows columns: int Number of columns pattern: string The type of test pattern. CONSTANT - A constant value. useful for flat-field tests. SLOPE - A flat surface of constant slope. Useful for test data. BOX - An image with a central bright box. TESTIMAGE - A regular grid of bright images on a constant background. DARKMAP - A dark current multiplier map containing random hot pixels. BADPIXEL - A bad pixel map containing random dead pixels. Other types TBD. values: float or tuple of floats Values to be used to create the requested type of data. For CONSTANT: * values contains the constant value For SLOPE * values[0] contains the constant * values[1] contains the row slope * values[2] contains the column slope For BOX * values[0] contains the constant background * values[1] contains the peak brightness For TESTIMAGE * values[0] contains the constant background * values[1] contains the peak brightness * values[2] contains the row spacing in pixels * values[3] contains the column spacing in pixels For DARKMAP * values[0] contains the minimum dark multipler (normally <=1.0) * values[1] contains the maximum dark multipler (normally >=1.0) * values[2] contains the number of random hot pixels * values[3] contains the hot pixel multipler (normally >1000) For BADPIXEL * values contains the number of random dead pixels metadata: Metadata object, optional The primary metadata describing the data. If None, the metadata is ignored. cloneimage: array_like, optional A image to be cloned within the TESTIMAGE, DARKMAP or BADPIXEL patterns. If specified, some of the abnormal pixels will be stamped with this image. If not specified only single pixels are modified. add_wavelength: boolean, optional, default=True Add a test wavelength map to the data. Not valid for DARKMAP or BADPIXEL. wavmin: float, optional, default=1.0 Minimum wavelength associated with data in microns. wavmax: float, optional, default=30.0 Maximum wavelength associated with data in microns. seedvalue: int, optional, default=None The seed to be sent to the np.random number generator before generating the test data. If not specified, a value of None will be sent, which randomises the seed. :Raises: TypeError Raised if any of the parameters are of the wrong type, size or shape. :Returns: For the DARKMAP pattern, a new MeasuredModel object. For the BADPIXEL pattern, a new MiriBadPixelModel object. For other patterns, a new MiriIlluminationModel object. """ # Set the seed for the np.random function. np.random.seed(seedvalue) # Test the requested data size try: rows = int(rows) columns = int(columns) except (TypeError, ValueError): strg = "Row and column values must be integers." raise TypeError(strg) # Define an intensity array containing the defined pattern. if pattern == 'CONSTANT': # A constant level. try: datavalues = float(values) * np.ones([rows, columns], dtype=np.float32) except (TypeError, ValueError) as e: strg = "CONSTANT pattern needs a single floating point value" strg += "\n %s" % e raise TypeError(strg) elif pattern == 'SLOPE': # A flat, sloping surface. try: datavalues = \ np.fromfunction( lambda i,j: values[0] + i*values[1] + j*values[2], [rows,columns]) except (TypeError, ValueError, IndexError) as e: strg = "SLOPE pattern needs a tuple of 3 floats" strg += "\n %s" % e raise TypeError(strg) elif pattern == "BOX": # A box image. try: dmin = float(values[0]) dmax = float(values[1]) djump = dmax - dmin except (TypeError, ValueError, IndexError) as e: strg = "BOX pattern needs a tuple of (float, float)" strg += "\n %s" % e raise TypeError(strg) # Initialise the array full of background values and set a # central box to the maximum value. datavalues = dmin * np.ones([rows, columns], dtype=np.float32) rmin = rows / 3 rmax = rmin * 2 cmin = columns / 3 cmax = cmin * 2 datavalues[rmin:rmax, cmin:cmax] += djump elif pattern == "TESTIMAGE": # A grid of test images. try: dmin = float(values[0]) dmax = float(values[1]) djump = dmax - dmin rowstep = int(values[2]) colstep = int(values[3]) except (TypeError, ValueError, IndexError) as e: strg = "TESTIMAGE pattern needs a tuple of (float, float, int, int)" strg += "\n %s" % e raise TypeError(strg) # Initialise the array full of background values. datavalues = dmin * np.ones([rows, columns], dtype=np.float32) if cloneimage is not None: cloneimage = np.asarray(cloneimage) cloneimage *= djump lastrow = rows - cloneimage.shape[0] + 1 lastcol = columns - cloneimage.shape[1] + 1 else: lastrow = rows lastcol = columns for row in range(0, lastrow, rowstep): for col in range(0, lastcol, colstep): # Stamp either the clone image or a single pixel. if (cloneimage is not None): for xx in range(0, cloneimage.shape[0]): rr = row + xx for yy in range(0, cloneimage.shape[1]): cc = col + yy datavalues[rr, cc] += cloneimage[xx, yy] else: datavalues[row, col] = djump elif pattern == 'DARKMAP': # A dark current multipler map. There is never a wavelength array add_wavelength = False # The data contains 1.0 unless otherwise specified. datavalues = np.ones([rows, columns], dtype=np.float32) try: dmin = float(values[0]) dmax = float(values[1]) nhot = int(values[2]) dhot = float(values[3]) except (TypeError, ValueError, IndexError) as e: strg = "DARKMAP pattern needs a tuple of (float, float, int, float)" strg += "\n %s" % e raise TypeError(strg) # Rather than set every individual pixel (which would be inefficient) # create a map containing zones of different dark current multiplers. DARK_ZONES = 4 rstep = datavalues.shape[0] // DARK_ZONES cstep = datavalues.shape[1] // DARK_ZONES for row in range(0, rows - rstep, rstep): for column in range(0, columns - rstep, cstep): if dmin < dmax: rvalue = float(np.random.uniform(dmin, dmax)) else: # No variation rvalue = dmin datavalues[row:row + rstep, column:column + rstep] = rvalue if cloneimage is not None: cloneimage = np.asarray(cloneimage) cloneimage *= dhot lastrow = rows - cloneimage.shape[0] + 1 lastcol = columns - cloneimage.shape[1] + 1 else: lastrow = rows lastcol = columns for zap in range(0, nhot): hit_row = int(np.random.uniform(0, lastrow)) hit_column = int(np.random.uniform(0, lastcol)) # Stamp only a small percent of the hot pixels with the clone image. MAX_CLONED_IMAGE_PERCENT = 5.0 if (cloneimage is not None) and \ (np.random.uniform(0, 100) < MAX_CLONED_IMAGE_PERCENT): for xx in range(0, cloneimage.shape[0]): rr = hit_row + xx for yy in range(0, cloneimage.shape[1]): cc = hit_column + yy datavalues[rr, cc] += cloneimage[xx, yy] else: datavalues[hit_row, hit_column] = dhot elif pattern == 'BADPIXEL': # A bad pixel map. There is never a wavelength array. add_wavelength = False datavalues = np.zeros([rows, columns], dtype=np.uint16) try: ndead = int(values) except (TypeError, ValueError) as e: strg = "BADPIXEL pattern needs an integer bad pixel count" strg += "\n %s" % e raise TypeError(strg) if cloneimage is not None: cloneimage = np.asarray(cloneimage) cloneimage *= _BAD_PIXEL lastrow = rows - cloneimage.shape[0] + 1 lastcol = columns - cloneimage.shape[1] + 1 else: lastrow = rows lastcol = columns for zap in range(0, ndead): hit_row = int(np.random.uniform(0, lastrow)) hit_column = int(np.random.uniform(0, lastcol)) # Stamp only a small percent of the bad pixels with the clone image. if (cloneimage is not None) and \ (np.random.uniform(0, 100) < _MAX_CLONED_IMAGE_PERCENT): # TODO: I can't get this slicing to work. # uptorow = hit_row + cloneimage.shape[0] - 1 # uptocol = hit_column + cloneimage.shape[1] - 1 # datavalues[hit_row:uptorow,hit_column:uptocol] = cloneimage for xx in range(0, cloneimage.shape[0]): rr = hit_row + xx for yy in range(0, cloneimage.shape[1]): cc = hit_column + yy datavalues[rr, cc] = cloneimage[xx, yy] else: datavalues[hit_row, hit_column] = _BAD_PIXEL # flagvalues=(_GOOD_PIXEL,_BAD_PIXEL) # flagnames=('GOOD','BAD') # TODO: Add FRINGE pattern? else: # Other data types are not supported strg = "Sorry, test data pattern %s is not supported." % pattern raise ValueError(strg) # Add some test wavelength data if required. The wavelength will # increase linearly from bottom to top over the range specified. if add_wavelength: wavelength = np.empty_like(datavalues) wav = np.linspace(wavmin, wavmax, wavelength.shape[1]) wavelength[:, :] = wav # wavelength = np.transpose(wavelength) else: wavelength = None data_metadata = None wavelength_metadata = None # Finally, create and return an appropriate object if pattern == 'BADPIXEL': data_object = MiriBadPixelMaskModel(dq=datavalues) # data_object = BadPixelMap(datavalues, flagvalues=flagvalues, # flagnames=flagnames, metadata=metadata, # dq_metadata=data_metadata) elif pattern == 'DARKMAP': data_object = MiriMeasuredModel(data=datavalues) # elif pattern == 'FRINGEMAP': # data_object = MiriMeasuredModel(data=datavalues) else: data_object = MiriIlluminationModel(intensity=datavalues, wavelength=wavelength) return data_object
def test_subarray_to_subarray(self): # Test the extraction of a subarray from input data containing # the same subarray. if VERBOSE: print("\nTesting subarray to subarray modes: ", end='') if TEST_MODE_COVERAGE > 0.99: modes_to_test = list(detector_properties['SUBARRAY'].keys()) random.shuffle(modes_to_test) else: nsamples = int(0.5 + len(list(detector_properties['SUBARRAY'].keys())) * \ TEST_MODE_COVERAGE) nsamples = max(nsamples, MINSAMPLES) modes_to_test = random.sample( \ list(detector_properties['SUBARRAY'].keys()), nsamples) with warnings.catch_warnings(): # Suppress FITS header warnings. warnings.simplefilter("ignore") for submode in modes_to_test: print(submode + ' ', end='') subarray = detector_properties.get('SUBARRAY', submode) if subarray is not None: testvalues = np.fromfunction( lambda i, j: 2.0 + i * 1.0 + j * 1.0, [subarray[2], subarray[3]]) wavelength = np.empty_like(testvalues) wav = np.linspace(5.0, 25.0, wavelength.shape[1]) wavelength[:, :] = wav test_map = MiriIlluminationModel(intensity=testvalues, wavelength=wavelength) map_data = test_map.get_illumination() # The illumination data is assumed to have a valid shape and be # filled with a range of values (the data should have a slope). self.assertTrue(map_data.shape[0] > 0) self.assertTrue(map_data.shape[1] > 0) self.assertNotAlmostEqual(map_data.min(), map_data.max()) sca = SensorChipAssembly1(logger=LOGGER) sca.setup(_DEFAULT_SCA, readout_mode=_DEFAULT_READOUT, subarray=submode, inttime=None, ngroups=2, nints=1, temperature=6.5, cosmic_ray_mode='NONE', simulate_dark_current=SIMULATE_DARK, verbose=0) sca.set_illumination(test_map, subarray_input=submode) simulated_data = sca.exposure() # The simulated data must have a valid shape and contain a # range of values. (A more specific test is difficult because # read noise is included. The accuracy of subarray extraction # should already have been tested in the test_detector unit # tests.) self.assertTrue(simulated_data.shape[0] > 0) self.assertTrue(simulated_data.shape[1] > 0) self.assertNotAlmostEqual(simulated_data.min(), simulated_data.max()) del sca del test_map if VERBOSE: print("", end='')