def get_dark_frame(self):
'''
takes however many dark files that are specified in the pipe.yml and computes the counts/pixel/sec for the sum
of all the dark obs. This creates a stitched together long dark obs from all of the smaller obs given. This
is useful for legacy data where there may not be a specified dark observation but parts of observations where
the filter wheel was closed.
If self.use_wavecal is True then a dark is not subtracted off since this just takes into account total counts
and not energy information
:return: expected dark counts for each pixel over a flat observation
'''
if not self.dark_h5_file_names:
dark_frame = np.zeros_like(self.spectralCubes[0][:, :, 0])
else:
self.dark_start = [self.cfg.flatcal.dark_data['start_times']]
self.dark_int = [self.cfg.flatcal.dark_data['int_times']]
self.dark_h5_file_names = [
os.path.join(self.h5_directory,
str(t) + '.h5') for t in self.dark_start
]
frames = np.zeros((140, 146, len(self.dark_start)))
getLogger(__name__).info('Loading dark frames for Laser flat')
for i, file in enumerate(self.dark_h5_file_names):
obs = Photontable(file)
frame = obs.getPixelCountImage(
integrationTime=self.dark_int[i])['image']
frames[:, :, i] = frame
total_counts = np.sum(frames, axis=2)
total_int_time = float(np.sum(self.dark_int))
counts_per_sec = total_counts / total_int_time
dark_frame = counts_per_sec
return dark_frame
def __init__(self):
self.endtime = -1
self.begintime = 0
self.tick_duration = 1e-6
self.ticks_per_sec = int(1.0 / self.tick_duration)
self.nRow = 140
self.nCol = 146
self.h5file = 'test'
self.obs = Photontable(self.h5file)
def __init__(self, configuration=None, h5_file_names=None, solution_name='solution.npz', interpolation=None,
use_satellite_spots=True, obj_pos=None):
self.interpolation = interpolation
self.use_satellite_spots = use_satellite_spots
self.obj_pos = obj_pos
self.flux_spectra = None
self.flux_effTime = None
self.wvl_bin_edges = None
self.std_spectrum = None
self.std_wvls = None
self.std_flux = None
self.rebin_std_wvls = None
self.rebin_std_flux = None
self.obs = None
self.wvl_bin_widths = None
self.curve = None
self.image = None
self.bb_flux = None
self.bb_wvls = None
self.conv_wvls = None
self.conv_flux = None
self.data = None
self.rebin_plot_data = None
self.wvl_bin_centers = None
self.flux_spectrum = None
self.cube = None
self.aperture_centers = None
self.aperture_radii = None
self.errors=None
self.contrast = None
if h5_file_names:
self.obs = [Photontable(f) for f in h5_file_names]
if configuration:
self.cfg = Configuration(configuration) if not isinstance(configuration, Configuration) else configuration
self.obj_pos = self.cfg.obj_pos
self.solution_name = solution_name
self.use_satellite_spots = self.cfg.use_satellite_spots
self.interpolation = self.cfg.interpolation
self.obs = [Photontable(f) for f in self.cfg.h5_file_names]
self.wvl_bin_widths = self.cfg.wvl_bin_widths
self.wvl_bin_edges = self.cfg.wvl_bin_edges
self.data = self.cfg.data
self.wvl_bin_centers = self.cfg.wvl_bin_centers
self.contrast = np.zeros_like(self.wvl_bin_centers)
self.aperture_centers = np.zeros((len(self.wvl_bin_centers), 2))
self.aperture_radii = np.zeros_like(self.wvl_bin_centers)
self.platescale = self.data.wcscal.platescale
self.solution = ResponseCurve(configuration=self.cfg, curve=self.curve, wvl_bin_widths=self.wvl_bin_widths,
wvl_bin_centers= self.wvl_bin_centers, cube=self.cube,
solution_name=self.solution_name, errors=self.errors)
def fetchimg(ob, kwargs):
of = Photontable(ob.h5)
if kwargs['nwvl'] > 1:
im = of.getSpectralCube(hdu=True, **kwargs)
else:
kwargs.pop('wvlN', None)
im = of.getPixelCountImage(hdu=True, **kwargs)
del of
#TODO move to photontable and fetch it from an import
im.headerh['PIXELA'] = 10.0**2
return im
def make_spectralcube(self):
'''
called from loadFlatSpectra
:return:
list of counts per second cubes, list of all of the integration times used to make the count per second cubes,
bag pixel mask
'''
wavelengths = self.wavelengths
nWavs = len(self.wavelengths)
ntimes = self.cfg.flatcal.nchunks
if np.any(
self.intTime * self.cfg.flatcal.nchunks > self.exposure_times):
ntimes = int((self.exposure_times / self.intTime).max())
getLogger(__name__).info(
'Number of chunks * chunk time is longer than the laser exposure. Using full'
'length of exposure ({} chunks)'.format(ntimes))
cps_cube_list = np.zeros([ntimes, self.xpix, self.ypix, nWavs])
mask = np.zeros([self.xpix, self.ypix, nWavs])
int_times = np.zeros([self.xpix, self.ypix, nWavs])
if self.use_wavecal:
delta_list = wavelengths / self.r_list
for iwvl, wvl in enumerate(wavelengths):
obs = Photontable(self.h5_file_names[iwvl])
# mask out hot pixels before finding the mean
if not obs.info['isBadPixMasked'] and not self.use_wavecal:
getLogger(__name__).info(
'Hot pixel masking laser h5 file for flatcal')
badpix.mask_hot_pixels(self.h5_file_names[iwvl])
bad_mask = obs.flagMask(pixelflags.PROBLEM_FLAGS)
mask[:, :, iwvl] = bad_mask
if self.use_wavecal:
counts = obs.getTemporalCube(
integrationTime=self.intTime * self.cfg.flatcal.nchunks,
timeslice=self.intTime,
startw=wvl - (delta_list[iwvl] / 2.),
stopw=wvl + (delta_list[iwvl] / 2.))
else:
counts = obs.getTemporalCube(
integrationTime=self.cfg.flatcal.chunk_time *
self.cfg.flatcal.nchunks,
timeslice=self.intTime)
getLogger(__name__).info('Loaded {}nm spectral cube'.format(wvl))
cps_cube = counts[
'cube'] / self.intTime # TODO move this division outside of the loop
cps_cube = np.moveaxis(cps_cube, 2, 0)
int_times[:, :, iwvl] = self.intTime
cps_cube_list[:, :, :, iwvl] = cps_cube
return cps_cube_list, int_times, mask
def handle_existing(self):
""" Handles existing h5 files, deleting them if appropriate"""
if os.path.exists(self.cfg.h5file):
if self.force:
getLogger(__name__).info('Remaking {} forced'.format(
self.cfg.h5file))
done = False
else:
try:
done = Photontable(
self.cfg.h5file).duration >= self.cfg.inttime
if not done:
getLogger(__name__).info(
('{} does not contain full duration, '
'will remove and rebuild').format(
self.cfg.h5file))
except:
done = False
getLogger(__name__).info(
('{} presumed corrupt,'
' will remove and rebuild').format(self.cfg.h5file),
exc_info=True)
if not done:
try:
os.remove(self.cfg.h5file)
getLogger(__name__).info('Deleted {}'.format(
self.cfg.h5file))
except FileNotFoundError:
pass
else:
getLogger(__name__).info(
'H5 {} already built. Remake not requested. Done.'.format(
self.cfg.h5file))
self.done = True
def getPixelPhotonList(filename, xCoord, yCoord, **kwargs):
"""
Gets the list of photon arrival times from a H5 file
INPUTS:
filename - Name of H5 data file
xCoord -
yCoord -
**kwargs - keywords for Obsfile getpixelphotonlist()
OUTPUTS:
ts - timestamps in us of photon arrival times
"""
obs = Photontable(filename)
photonList = obs.getPixelPhotonList(xCoord, yCoord, **kwargs)
times = photonList['Time']
print("#photons: " + str(len(times)))
del obs #make sure to close files nicely
return times
def plot_counts_on_array(self, obsFile, timeStep=10):
obs = Photontable(obsFile)
photons = obs.photonTable.read()
hist = np.bincount((photons['Time'] / timeStep).astype(int))
timestamps = np.linspace(0, photons['Time'].max(), len(hist))
plt.plot(timestamps, hist, linewidth=0.3)
plt.xlabel('Time (microseconds)')
plt.ylabel('Counts on Array')
plt.title(f"Array Count Time Stream ({obsFile})")
def determine_mean_photperRID(self):
ret = {}
# return {d:1 for d in self.datasets}
for d in self.datasets:
if d in self.count_images:
cnt_img = self.count_images[d]
else:
res = list(self.query_iter((SHORTT_QUERY), dataset=d))
i = np.array([r.queryt for r in res]).argmin()
fast_file = res[i].file
print('This might take a while: {:.0f} s'.format(
res[i].queryt))
of = Photontable(fast_file)
cnt_img = of.getPixelCountImage(firstSec=30,
integrationTime=5)['image']
cnt_img *= of.info['expTime'] / 5
del of
self.count_images[d] = cnt_img
ret[d] = cnt_img.sum() / (cnt_img > 0).sum()
return ret
def loadData(self):
getLogger(__name__).info('Loading calibration data from {}'.format(
self.h5file))
self.obs = Photontable(self.h5file)
if not self.obs.wavelength_calibrated:
raise RuntimeError('Photon data is not wavelength calibrated.')
self.beamImage = self.obs.beamImage
self.wvlFlags = pixelflags.beammap_flagmap_to_h5_flagmap(
self.obs.beamFlagImage)
self.xpix = self.obs.nXPix
self.ypix = self.obs.nYPix
self.energies = [(c.h * c.c) / (i * 10**(-9) * c.e)
for i in self.wavelengths]
middle = int(len(self.wavelengths) / 2.0)
self.energyBinWidth = self.energies[middle] / (5.0 *
self.r_list[middle])
self.wvlBinEdges = Photontable.makeWvlBins(self.energyBinWidth,
self.wvlStart, self.wvlStop)
self.wavelengths = self.wvlBinEdges[:-1] + np.diff(self.wvlBinEdges)
self.wavelengths = self.wavelengths.flatten()
self.sol = False
time.time())
args = parser.parse_args()
flattner = WhiteCalibrator(args.cfgfile,
cal_file_name='calsol_{}.h5'.format(timestamp))
if not os.path.isfile(flattner.cfg.h5file):
raise RuntimeError('Not up to date')
b2h_config = bin2hdf.Bin2HdfConfig(datadir=flattner.cfg.paths.data,
beamfile=flattner.cfg.beammap.file,
outdir=flattner.paths.out,
starttime=flattner.cfg.start_time,
inttime=flattner.cfg.expTime,
x=flattner.cfg.beammap.ncols,
y=flattner.cfg.beammap.ncols)
bin2hdf.makehdf(b2h_config, maxprocs=1)
getLogger(__name__).info('Made h5 file at {}.h5'.format(
flattner.cfg.start_time))
obsfile = Photontable(flattner.h5file, mode='write')
if not obsfile.wavelength_calibrated:
obsfile.applyWaveCal(wavecal.load_solution(flattner.cfg.wavesol))
getLogger(__name__).info('Applied Wavecal {} to {}.h5'.format(
flattner.cfg.wavesol, flattner.h5file))
if args.h5only:
exit()
flattner.makeCalibration()
def mask_hot_pixels(file,
method='hpm_flux_threshold',
step=30,
startt=0,
stopt=None,
ncpu=1,
**methodkw):
"""
This routine is the main code entry point of the bad pixel masking code.
Takes an obs. file as input and writes a 'bad pixel table' to that h5 file where each entry is an indicator of
whether the pixel was good, dead, hot, or cold. Defaults should be somewhat reasonable for a typical on-sky image.
HPCut method is interchangeable with any of the methods listed here.
The HOT and DEAD masks are combined into a single BAD mask at the end
Required Input:
:param obsfile: user passes an obsfile instance here
:param startt Scalar Integer. Timestamp at which to begin bad pixel masking, default = 0
:param stopt Scalar Integer. Timestamp at which to finish bad pixel masking, default = -1
(run to the end of the file)
:param step Scalar Integer. Number of seconds to do the bad pixel masking over (should be an integer
number of steps through the obsfile), default = 30
:param hpcutmethod String. Method to use to detect hot pixels. Options are:
hpm_median_movingbox
hpm_flux_threshold
hpm_laplacian
hpm_cps_cut
Other Input:
Appropriate args and kwargs that go into the chosen hpcut function
:return:
Applies relevant pixelflags - see pixelflags.py
"""
obs = Photontable(file)
if obs.info['isBadPixMasked']:
getLogger(__name__).info('{} is already bad pixel calibrated'.format(
obs.fileName))
return
if stopt is None:
stopt = obs.getFromHeader('expTime')
assert startt < stopt
if step > stopt - startt:
getLogger(__name__).warning((
'Hot pixel step time longer than exposure time by {:.0f} s, using full '
'exposure').format(abs(stopt - startt - step)))
step = stopt - startt
step_starts = np.arange(
startt, stopt, step,
dtype=int) # Start time for each step (in seconds).
step_ends = step_starts + int(step) # End time for each step
step_ends[step_ends > stopt] = int(
stopt
) # Clip any time steps that run over the end of the requested time range.
# Initialise stack of masks, one for each time step
hot_masks = np.zeros([obs.nXPix, obs.nYPix, step_starts.size], dtype=bool)
cold_masks = np.zeros([obs.nXPix, obs.nYPix, step_starts.size], dtype=bool)
func = globals()[method]
# Generate a stack of bad pixel mask, one for each time step
for i, each_time in enumerate(step_starts):
getLogger(__name__).info('Processing time slice: {} - {} s'.format(
each_time, each_time + step))
raw_image_dict = obs.getPixelCountImage(
firstSec=each_time,
integrationTime=step,
applyWeight=True,
applyTPFWeight=True,
scaleByEffInt=method == 'hpm_cps_cut')
bad_pixel_solution = func(raw_image_dict['image'], **methodkw)
hot_masks[:, :, i] = bad_pixel_solution['hot_mask']
cold_masks[:, :, i] = bad_pixel_solution['cold_mask']
unstable_mask = np.zeros((obs.nXPix, obs.nYPix), dtype=bool)
for x in range(obs.nXPix):
for y in range(obs.nYPix):
vals = np.zeros(len(hot_masks[x, y, :]), dtype=bool)
for i, mask in enumerate(hot_masks[x, y, :]):
vals[i] = mask
if not all(vals) or all(vals):
unstable_mask[x, y] = False
else:
unstable_mask[x, y] = True
# Combine the bad pixel masks into a master mask
obs.enablewrite()
obs.applyBadPixelMask(np.all(hot_masks, axis=-1),
np.all(cold_masks, axis=-1), unstable_mask)
obs.disablewrite()
9459, 9460, 9461, 9462, 9463, 9464, 9465, 9466, 9467, 9468, 9469, 9470,
9471, 9472, 9473, 9474, 9475, 9476, 9477, 9478, 9479, 9480, 9481, 9482,
9483, 9484, 9485, 9486, 9487, 9488, 9489, 9490, 9491, 9492, 9493, 9494,
9495, 9496, 9497, 9498, 9499, 9500, 9501, 9502, 9503, 9504, 9505
])
print('Warning: Only run this script once per file')
print(fns)
tmp = ''
while tmp not in ['Y', 'N']:
tmp = input('Are you sure you want to run this script? (Y/N)')
tmp = tmp.upper()
if tmp == 'Y':
for fn in fns:
print('Adjusting flags in: ' + fn)
obs = Photontable(fn, mode='w')
resIDs = obs.beamImage[:, :]
flags = obs.beamFlagImage[:, :]
resID_noBeammap = resIDs[np.where(np.bitwise_and(flags, 1) > 0)]
resID_beammap = resIDs[np.where(np.bitwise_and(flags, 1) == 0)]
print('ResIDs with a DAC Tone that failed the beammap:')
print(resID_withTone[np.where(np.isin(resID_withTone,
resID_noBeammap))])
print(
'ResIDs without a DAC Tone that passed the beammaped (should be none):'
)
print(resID_beammap[np.where(
np.isin(resID_beammap, resID_withTone, invert=True))])
resID_wvlCal = resIDs[np.where(np.bitwise_and(flags, 2) == 0)]
class WhiteCalibrator(FlatCalibrator):
"""
Opens flat file using parameters from the param file, sets wavelength binning parameters, and calculates flat
weights for flat file. Writes these weights to a h5 file and plots weights both by pixel
and in wavelength-sliced images.
"""
def __init__(self, config=None, cal_file_name='flatsol_{start}.h5'):
"""
Reads in the param file and opens appropriate flat file. Sets wavelength binning parameters.
"""
super().__init__(config)
self.startTime = self.cfg.start_time
self.expTime = self.cfg.exposure_time
self.h5file = self.cfg.get(
'h5file',
os.path.join(self.cfg.paths.out,
str(self.startTime) + '.h5'))
self.flatCalFileName = self.cfg.get(
'flatname',
os.path.join(self.cfg.paths.database,
cal_file_name.format(start=self.startTime)))
def loadData(self):
getLogger(__name__).info('Loading calibration data from {}'.format(
self.h5file))
self.obs = Photontable(self.h5file)
if not self.obs.wavelength_calibrated:
raise RuntimeError('Photon data is not wavelength calibrated.')
self.beamImage = self.obs.beamImage
self.wvlFlags = pixelflags.beammap_flagmap_to_h5_flagmap(
self.obs.beamFlagImage)
self.xpix = self.obs.nXPix
self.ypix = self.obs.nYPix
self.energies = [(c.h * c.c) / (i * 10**(-9) * c.e)
for i in self.wavelengths]
middle = int(len(self.wavelengths) / 2.0)
self.energyBinWidth = self.energies[middle] / (5.0 *
self.r_list[middle])
self.wvlBinEdges = Photontable.makeWvlBins(self.energyBinWidth,
self.wvlStart, self.wvlStop)
self.wavelengths = self.wvlBinEdges[:-1] + np.diff(self.wvlBinEdges)
self.wavelengths = self.wavelengths.flatten()
self.sol = False
def loadFlatSpectra(self):
"""
Reads the flat data into a spectral cube whose dimensions are determined
by the number of x and y pixels and the number of wavelength bins.
Each element will be the spectral cube for a time chunk
Find factors to correct nonlinearity due to deadtime in firmware
To be used for whitelight flat data
"""
self.spectralCubes = []
self.cubeEffIntTimes = []
for firstSec in range(0, self.expTime,
self.intTime): # for each time chunk
cubeDict = self.obs.getSpectralCube(firstSec=firstSec,
integrationTime=self.intTime,
applyWeight=False,
applyTPFWeight=False,
wvlBinEdges=self.wvlBinEdges)
cube = cubeDict['cube'] / cubeDict['effIntTime'][:, :, None]
# TODO: remove when we have deadtime removal code in pipeline
cube /= (1 - cube.sum(axis=2) * self.deadtime)[:, :, None]
bad = np.isnan(
cube
) # TODO need to update masks to note why these 0s appeared
cube[bad] = 0
self.spectralCubes.append(cube)
self.cubeEffIntTimes.append(cubeDict['effIntTime'])
msg = 'Loaded Flat Spectra for seconds {} to {}'.format(
int(firstSec),
int(firstSec) + int(self.intTime))
getLogger(__name__).info(msg)
self.spectralCubes = np.array(self.spectralCubes[0])
# Haven't had a chance to check if the following lines break - find me if you come across this before me - Sarah
self.cubeEffIntTimes = np.array(self.cubeEffIntTimes)
self.countCubes = self.cubeEffIntTimes * self.spectralCubes
class Cosmic:
def __init__(self):
self.endtime = -1
self.begintime = 0
self.tick_duration = 1e-6
self.ticks_per_sec = int(1.0 / self.tick_duration)
self.nRow = 140
self.nCol = 146
self.h5file = 'test'
self.obs = Photontable(self.h5file)
def findCosmics(self,
stride=10,
threshold=100,
population_max=2000,
nsigma=5,
pps_stride=10000):
"""
Find cosmics ray suspects. Histogram the number of photons
recorded at each timeStamp. When the number of photons in a
group of stride timeStamps is greater than threshold in second
iSec, add (iSec,timeStamp) to cosmicTimeLists. Also keep
track of the histogram of the number of photons per stride
timeStamps.
return a dictionary of 'populationHg', 'cosmicTimeLists',
'binContents', 'timeHgValues', 'interval', 'frameSum', and 'pps'
populationHg is a histogram of the number of photons in each time bin.
This is a poisson distribution with a long tail due to cosmic events
cosmicTimeLists is a numpy array of all the sequences that are
suspects for cosmic rays
binContents corresponds to cosmicTimeLists. For each time in
cosmicTimeLists, binContents is the number of photons detected
at that time.
timeHgValues is a histogram of the number of photons in each time
interval
frameSum is a two dimensional numpy array of the number of photons
detected by each pixel
interval is the interval of data to be masked out
pps is photons per second, calculated every ppsStride bins.
"""
###self.logger.info("findCosmics: begin stride=%d threshold=%d populationMax=%d nSigma=%d writeCosmicMask=%s")
exptime = self.endtime - self.begintime
nbins = int(np.round(self.ticks_per_sec * exptime + 1))
bins = np.arange(0, nbins, 1)
timehist_values, framesum = self.get_timehist_and_framesum(
self.begintime, self.endtime)
remainder = len(timehist_values) % pps_stride
if remainder > 0:
temp = timehist_values[:-remainder]
else:
temp = timehist_values
pps_time = (pps_stride * self.tick_duration)
pps = np.sum(temp.reshape(-1, pps_stride), axis=1) / pps_time
cr_pop = Cosmic.population_from_timehist_values(
timehist_values, population_max, stride, threshold)
# now build up all of the intervals in seconds
i = interval()
icount = 0
seconds_per_tick = self.tick_duration
for cosmic_time in cr_pop['cosmic_time_list']:
t0 = self.begintime + cosmic_time * seconds_per_tick
dt = stride * seconds_per_tick
t1 = t0 + dt
left = max(self.begintime, t0 - nsigma * dt)
right = min(self.endtime, t1 + 2 * nsigma * dt)
i = i | interval[left, right]
icount += 1
times_masked = Cosmic.count_masked_bins(i)
ppm_masked = 1000000 * times_masked / (self.endtime - self.begintime)
return_val = {}
return_val['timehist_values'] = timehist_values
return_val['population_hist'] = cr_pop['population_hist']
return_val['cosmic_timelist'] = cr_pop['cosmic_timelist']
return_val['bin_contents'] = cr_pop['bin_contents']
return_val['framesum'] = framesum
return_val['interval'] = i
return_val['ppm_masked'] = ppm_masked
return_val['pps'] = pps
return_val['pps_time'] = pps_time
return return_val
def get_timehist_and_framesum(self, begintime, endtime):
integrationtime = endtime - begintime
nbins = int(np.round(self.ticks_per_sec * integrationtime + 1))
timehist_values = np.zeros(nbins, dtype=np.int64)
framesum = np.zeros((self.nRow, self.nCol))
for iRow in range(self.nRow):
for iCol in range(self.nCol):
gtpl = self.obs.getPixelPhotonList(
iRow,
iCol,
firstSec=begintime,
integrationtime=integrationtime)
timestamps = gtpl['timestamps']
if timestamps.size > 0:
timestamps = (timestamps - begintime) * self.ticks_per_sec
ts32 = np.round(timestamps).astype(np.uint32)
ts_binner.ts_binner32(ts32, timehist_values)
framesum[iRow, iCol] += ts32.size
return timehist_values, framesum
@staticmethod
def count_masked_bins(mask_interval):
return_val = 0
for x in mask_interval:
return_val += x[1] - x[0]
return return_val
@staticmethod
def population_from_timehist_values(timehist_values, population_max,
stride, threshold):
"""
Rebin the timehist_values histogram by combining stride bins. If
stride > 1, then bin a second time after shifting by stride/2
Create population_hist, a histogram of the number of photons in
the large bins. Also, create (and then sort) a list
cosmic_timelist of the start of bins (in original time units)
of overpopulated bins that have more than threshold number of
photons.
return a dictionary containing population_hist and cosmic_timelist
"""
pop_range = (-0.5, population_max - 0.5)
if stride == 1:
population_hist = np.histogram(timehist_values,
population_max,
range=pop_range)
cosmic_timelist = np.where(timehist_values > threshold)[0]
bin_contents = np.extract(timehist_values > threshold,
timehist_values)
else:
# rebin the timehist_values before counting the populations
length = timehist_values.size
remainder = length % stride
if remainder == 0:
end = length
else:
end = -remainder
timehist_values_trimmed = timehist_values[0:end]
timehist_values_rebinned_0 = np.reshape(
timehist_values_trimmed, [length / stride, stride]).sum(axis=1)
population_hist_0 = np.histogram(timehist_values_rebinned_0,
population_max,
range=pop_range)
cosmic_timelist_0 = stride * np.where(
timehist_values_rebinned_0 > threshold)[0]
bin_contents_0 = np.extract(timehist_values_rebinned_0 > threshold,
timehist_values_rebinned_0)
timehist_values_rebinned_1 = np.reshape(
timehist_values_trimmed[stride / 2:-stride / 2],
[(length - stride) / stride, stride]).sum(axis=1)
population_hist_1 = np.histogram(timehist_values_rebinned_1,
population_max,
range=pop_range)
cosmic_timelist_1 = (stride / 2) + stride * np.where(
timehist_values_rebinned_1 > threshold)[0]
bin_contents_1 = np.extract(timehist_values_rebinned_1 > threshold,
timehist_values_rebinned_1)
population_hist = (population_hist_0[0] + population_hist_1[0],
population_hist_0[1])
cosmic_timelist = np.concatenate(
(cosmic_timelist_0, cosmic_timelist_1))
bin_contents = np.concatenate((bin_contents_0, bin_contents_1))
args = np.argsort(cosmic_timelist)
cosmic_timelist = cosmic_timelist[args]
bin_contents = bin_contents[args]
cosmic_timelist.sort()
return_val = {}
return_val['population_hist'] = population_hist
return_val['cosmic_timelist'] = cosmic_timelist
return_val['bin_contents'] = bin_contents
return return_val
planet_photons = np.array(planet_photons).T.astype(np.float32)
planet_photons = np.insert(planet_photons,
obj=1,
values=cam.phase_cal(ap.wvl_range[0]),
axis=0)
planet_photons[0] = (planet_photons[0] + abs_step) * sp.sample_time
photons = np.concatenate((star_photons, planet_photons), axis=1)
# cam.save_photontable(photonlist=photons, index=None, populate_subsidiaries=False)
stem = cam.arange_into_stem(photons.T,
(cam.array_size[1], cam.array_size[0]))
stem = list(map(list, zip(*stem)))
stem = cam.remove_close(stem)
photons = cam.ungroup(stem)
photons = photons[[0, 1, 3, 2]]
# grid(cam.rebin_list(photons), show=False)
cam.populate_photontable(photons=photons, finalise=False)
cam.populate_photontable(photons=[], finalise=True)
grid(cam.rebin_list(photons), show=False)
# to look at the photontable
obs = Photontable(iop.photonlist)
print(obs.photonTable)
# to plot an image of all the photons on the array
image = obs.getPixelCountImage(integrationTime=None)['image']
quick2D(image, show=False, title='Const planet photons')
plt.show(block=True)
def getTotalCountsAndDuration(self, obsFilePath):
counts = np.zeros((len(self.resIDs), len(BF_LASER_WAVELENGTHS)))
cosmicCounts = np.zeros((len(self.resIDs), len(BF_LASER_WAVELENGTHS)))
durations = np.zeros((len(self.resIDs), len(BF_LASER_WAVELENGTHS)))
thresh = 4
if thresh is not None:
print(
f"Using a threshold of {thresh} counts per bin to identify cosmic ray events in {obsFilePath}"
)
obs = Photontable(obsFilePath)
cosmicPeaks, cosmicCutout = self.findCosmicRayTimeStamps(
obs.photonTable.read(), threshold=thresh)
s1 = time.time()
for count1, j in enumerate(self.resIDs):
if (count1 % 40) == 0:
print(
f"{count1 / len(self.resIDs) * 100:.2f}% done with obsFile {obsFilePath}"
)
if obs.duration >= 60:
duration = 60
else:
duration = obs.duration
timeRemoved = len(cosmicCutout) / 1e6
for count2, _ in enumerate(BF_LASER_WAVELENGTHS):
photonList = obs.getPixelPhotonList(
resid=j,
wvlStart=self.wlLimits[0][count2],
wvlStop=self.wlLimits[1][count2],
integrationTime=60)
cosmicCount = [
np.any(np.in1d(cosmicCutout, k))
for k in photonList['Time']
]
photonsToRemove = np.sum(cosmicCount)
counts[count1][count2] = len(photonList)
durations[count1][count2] = (duration - timeRemoved)
cosmicCounts[count1][count2] = photonsToRemove
temp = np.concatenate((self.resIDs[:, None], counts, cosmicCounts,
durations[:, 0][:, None]),
axis=1)
data = np.core.records.fromarrays(
temp.transpose(),
names=
'ResID, bin1, bin2, bin3, bin4, bin5, bin1cosmics, bin2cosmics, bin3cosmics, bin4cosmics, bin5cosmics, duration',
formats='i4, f8, f8, f8, f8, f8, f8, f8, f8, f8, f8, f8')
e1 = time.time()
print(
f"--- It took {int(e1 - s1)} seconds to generate counts from Photontable {obsFilePath} ---"
)
obs.file.close()
return data
def test_settings(cfg,
settings,
queries,
do_build=True,
do_query=True,
cleanup=True,
recovery=''):
if recovery:
try:
with open(recovery, 'rb') as f:
results = pickle.load(f)
except IOError:
results = {}
builders = []
h5s = ['{}_{}.h5'.format(cfg.h5file[:-3], setting_id(s)) for s in settings]
for s, h5 in zip(settings, h5s):
builder = bin2hdf.HDFBuilder(copy.copy(cfg))
builder.cfg.user_h5file = h5
builder.kwargs = s
builders.append(builder)
needed = {b.cfg.h5file: False for b in builders}
for b in builders:
if b.cfg.h5file not in results:
needed[b.cfg.h5file] = do_build
elif do_query:
for q in queries:
if str(q) not in results[b.cfg.h5file]:
needed[b.cfg.h5file] = do_build
break
for b in builders:
if not (needed[b.cfg.h5file] and not os.path.exists(b.cfg.h5file)):
continue
tic = time.time()
b.run()
toc = time.time()
results[b.cfg.h5file] = dict(creation=toc - tic)
of = Photontable(b.cfg.h5file)
results[b.cfg.h5file].update(
dict(size=os.stat(b.cfg.h5file).st_size / 1024**2,
nphotons=len(of.photonTable),
file_nfo=str(of),
table_nfo=repr(of.photonTable)))
del of
if recovery:
with open(recovery, 'wb') as f:
pickle.dump(results, f)
if not do_query:
queries = []
for q in queries:
for b in builders:
key = b.cfg.h5file
if key not in results or not os.path.exists(b.cfg.h5file):
getLogger(__name__).info(
f'No result record/file for {b.cfg.h5file}')
continue
if str(q) in results[key]:
continue
of = Photontable(b.cfg.h5file)
tic = time.time()
result = of.query(**q)
toc = time.time()
del of
results[key][str(q)] = {'time': toc - tic, 'nphotons': len(result)}
if recovery:
with open(recovery, 'wb') as f:
pickle.dump(results, f)
if cleanup:
for f in h5s:
os.remove(f)
return results
def tsectest(f):
of = Photontable(f)
# p=LineProfiler(of.photonTable._where)
# p.enable()
of.query(startt=0, intt=30)
def get_transforms(ditherfile,
datadir,
wvl_start=None,
wvl_stop=None,
fwhm_guess=3.0,
fit_power=1,
CONEX_ERROR=0.0001,
plot=False):
dither = MKIDDitheredObservation(os.path.basename(ditherfile), ditherfile,
None, None)
obs_files = [
os.path.join(datadir, '{}.h5'.format(o.start)) for o in dither.obs
]
box_size = fwhm_guess * 10
debug_images = []
pixel_positions = []
source_est = []
for file in obs_files:
obs = Photontable(file)
data = obs.getPixelCountImage(applyWeight=False,
exclude_flags=pixelflags.PROBLEM_FLAGS,
wvlStart=wvl_start,
wvlStop=wvl_stop)['image']
data = np.transpose(data)
debug_images.append(data)
mean, median, std = stats.sigma_clipped_stats(data,
sigma=3.0,
mask_value=0)
mask = np.zeros_like(data, dtype=bool)
mask[data == 0] = True
sources = DAOStarFinder(fwhm=fwhm_guess,
threshold=5. * std)(data - median, mask=mask)
source = sources[sources['flux'].argmax()]
source_est.append((source['xcentroid'], source['ycentroid']))
position = centroids.centroid_sources(data,
source['xcentroid'],
source['ycentroid'],
box_size=int(box_size),
centroid_func=centroid_2dg,
mask=mask)
pixel_positions.append((position[0][0], position[1][0]))
pixel_positions = np.array(pixel_positions)
conex_positions = np.array(dither.pos)
xform_con2pix = tf.estimate_transform('polynomial',
conex_positions,
pixel_positions,
order=fit_power)
xform_pix2con = tf.estimate_transform('polynomial',
pixel_positions,
conex_positions,
order=fit_power)
if plot:
axis = int(round(np.sqrt(len(obs_files)), 0))
fig, axs = plt.subplots(axis, axis, figsize=(20, 15))
i = 0
j = 0
for index, image in enumerate(debug_images[:-1]):
axs[i, j].imshow(image,
origin='lower',
interpolation='nearest',
cmap='viridis')
axs[i, j].add_patch(
Rectangle(
(source_est[index][0] -
(box_size / 2), source_est[index][1] - (box_size / 2)),
box_size,
box_size,
linewidth=1,
edgecolor='r',
fill=None))
marker = '+'
ms, mew = 30, 2.
axs[i, j].plot(source_est[index][0],
source_est[index][1],
color='red',
marker=marker,
ms=ms,
mew=mew)
axs[i, j].plot(pixel_positions[index][0],
pixel_positions[index][1],
color='blue',
marker=marker,
ms=ms,
mew=mew)
axs[i, j].set_title(
'Red + = Estimate, Blue + = Centroid for Dither Pos %i' %
index)
if (index + 1) % axis == 0 and (index + 1) != len(obs_files):
i += 1
j = 0
elif index != len(obs_files) - 1:
j += 1
plt.show()
_plotresiduals(xform_con2pix, conex_positions, pixel_positions)
return xform_con2pix, xform_pix2con