def run(self,
sensor_id1,
infiles1,
gains1,
bias_frame1=None,
dark_frame1=None,
crosstalk_matrix_file=None,
**kwargs):
## Parse kwargs for separate victim CCD inputs
try:
sensor_id2 = kwargs['sensor_id2']
except KeyError:
sensor_id2 = sensor_id1
infiles2 = infiles1
gains2 = gains1
bias_frame2 = bias_frame1
dark_frame2 = dark_frame1
else:
infiles2 = kwargs['infiles2']
gains2 = kwargs['gains2']
bias_frame2 = kwargs['bias_frame2']
dark_frame2 = kwargs['dark_frame2']
infiles_list = [(infiles1[i], infiles2[i])
for i in range(len(infiles1))]
all_amps = imutils.allAmps(infiles1[0])
## Create new matrix or modify existing
if crosstalk_matrix_file is not None:
crosstalk_matrix = CrosstalkMatrix.from_fits(crosstalk_matrix_file)
outfile = crosstalk_matrix_file
else:
crosstalk_matrix = CrosstalkMatrix(sensor_id1,
victim_id=sensor_id2,
namps=len(all_amps))
outfile = self.config.outfile
for infile1, infile2 in infiles_list:
ccd1 = MaskedCCD(infile1,
bias_frame=bias_frame1,
dark_frame=dark_frame1)
ccd2 = MaskedCCD(infile2,
bias_frame=bias_frame2,
dark_frame=dark_frame2)
num_aggressors = 0
signals = []
## Search each amp for aggressor
for i in all_amps:
imarr1 = ccd1.unbiased_and_trimmed_image(
i).getImage().getArray() * gains1[i]
smoothed = gaussian_filter(imarr1, 20)
y, x = np.unravel_index(smoothed.argmax(), smoothed.shape)
stamp1 = make_stamp(imarr1, y, x)
ly, lx = stamp1.shape
Y, X = np.ogrid[-ly / 2:ly / 2, -lx / 2:lx / 2]
mask = X * X + Y * Y <= 20 * 20
signal = np.mean(stamp1[mask])
if signal > self.config.threshold:
signals.append(signal)
row = {}
## Calculate crosstalk for each victim amp
for j in all_amps:
imarr2 = ccd2.unbiased_and_trimmed_image(
j).getImage().getArray() * gains2[j]
stamp2 = make_stamp(imarr2, y, x)
row[j] = crosstalk_fit(stamp1,
stamp2,
noise=7.0,
num_iter=self.config.num_iter,
nsig=self.config.nsig)
crosstalk_matrix.set_row(i, row)
if num_aggressors == self.config.aggressors_per_image:
break
crosstalk_matrix.signal = np.median(np.asarray(signals))
if sensor_id1 == sensor_id2:
crosstalk_matrix.set_diagonal(0.)
crosstalk_matrix.write_fits(outfile, overwrite=True)
def run(self,
sensor_name,
infiles,
bias_frame=None,
dark_frame=None,
linearity_correction=None):
if not isinstance(infiles, list):
infiles = [infiles]
## Get sensor information from header
all_amps = imutils.allAmps(infiles[0])
with fits.open(infiles[0]) as hdulist:
lsst_num = hdulist[0].header['LSST_NUM']
teststand = hdulist[0].header['TSTAND']
manufacturer = lsst_num[:3]
## Interface with SQL database
database = self.config.database
logging.info(
"{0} Running CrosstalkSatelliteTask using database {1}".format(
datetime.now(), database))
with db_session(database) as session:
## Get sensor from database
try:
sensor = Sensor.from_db(session, sensor_name=sensor_name)
except NoResultFound:
sensor = Sensor(sensor_name=sensor_name,
lsst_num=lsst_num,
manufacturer=manufacturer,
namps=len(all_amps))
sensor.segments = {
i: Segment(segment_name=AMP2SEG[i], amplifier_number=i)
for i in all_amps
}
sensor.add_to_db(session)
session.commit()
logging.info("{0} New sensor {1} added to database".format(
datetime.now(), sensor_name))
## Set configuration and analysis settings
width = self.config.width
restrict_to_side = self.config.restrict_to_side
canny_sigma = self.config.canny_sigma
low_threshold = self.config.low_threshold
high_threshold = self.config.high_threshold
if len(infiles) > 1:
is_coadd = True
else:
is_coadd = False
ccds = [
MaskedCCD(infile,
bias_frame=bias_frame,
dark_frame=dark_frame,
linearity_correction=linearity_correction)
for infile in infiles
]
logging.info("{0} ".format(datetime.now()) + \
"Processing files: {}".format(' '.join(map(str, infiles))))
## Aggressor amplifiers
for i in all_amps:
aggressor_images = [
ccd.unbiased_and_trimmed_image(i).getImage()
for ccd in ccds
]
aggressor_imarr = imutils.stack(aggressor_images).getArray()
## Find aggressor regions
tested_angles = np.linspace(-np.pi / 2, np.pi / 2, 1000)
edges = feature.canny(aggressor_imarr,
sigma=canny_sigma,
low_threshold=low_threshold,
high_threshold=high_threshold)
h, theta, d = hough_line(edges, theta=tested_angles)
_, angle, dist = hough_line_peaks(h, theta, d)
if len(angle) != 2:
continue
read_noise = calculate_read_noise(ccds[0], i) * np.sqrt(
2. / len(ccds))
mean_angle = np.mean(angle)
mean_dist = np.mean(dist)
mask = satellite_mask(aggressor_imarr,
mean_angle,
mean_dist,
width=width)
signal = np.max(aggressor_imarr[~mask])
## Victim amplifiers
if restrict_to_side:
if i < 9:
vic_amps = range(1, 9)
else:
vic_amps = range(9, 17)
else:
vic_amps = all_amps
for j in vic_amps:
victim_images = [
ccd.unbiased_and_trimmed_image(j).getImage()
for ccd in ccds
]
victim_imarr = imutils.stack(victim_images).getArray()
res = crosstalk_fit(aggressor_imarr,
victim_imarr,
mask,
noise=read_noise)
## Add result to database
result = Result(aggressor_id=sensor.segments[i].id,
victim_id=sensor.segments[j].id,
aggressor_signal=signal,
coefficient=res[0],
error=res[4],
methodology='MODEL_LSQ',
teststand=teststand,
image_type='satellite',
analysis='CrosstalkSatelliteTask',
is_coadd=is_coadd,
z_offset=res[1],
y_tilt=res[2],
x_tilt=res[3])
result.add_to_db(session)
logging.info(
"{0} Injested C({1},{2}) for signal {3:.1f}".format(
datetime.now(), i, j, signal))
logging.info("{0} Task completed successfully.".format(
datetime.now()))
def run(self, inputExp, rawExp):
## run() method
outputRatios = defaultdict(lambda: defaultdict(dict))
outputFluxes = defaultdict(lambda: defaultdict(dict))
outputZOffsets = defaultdict(lambda: defaultdict(dict))
outputYTilts = defaultdict(lambda: defaultdict(dict))
outputXTilts = defaultdict(lambda: defaultdict(dict))
badPixels = list(self.config.badMask)
targetDetector = inputExp.getDetector()
targetChip = targetDetector.getName()
targetIm = inputExp.getMaskedImage()
## loop on sourceExp would go here
sourceExp = inputExp ## Ignore other exposures for now
sourceDetector = sourceExp.getDetector()
sourceChip = sourceDetector.getName()
sourceIm = sourceExp.getMaskedImage()
bad = sourceIm.getMask().getPlaneBitMask(badPixels)
self.log.info("Measuring crosstalk from source: %s", sourceChip)
ratioDict = defaultdict(lambda: defaultdict(list))
zoffsetDict = defaultdict(lambda: defaultdict(list))
ytiltDict = defaultdict(lambda: defaultdict(list))
xtiltDict = defaultdict(lambda: defaultdict(list))
extractedCount = 0
for sourceAmp in sourceDetector:
sourceAmpName = sourceAmp.getName()
sourceAmpImage = sourceIm[sourceAmp.getBBox()]
sourceMask = sourceAmpImage.mask.array
sourceAmpArray = sourceAmpImage.image.array
tested_angles = np.linspace(-np.pi / 2, np.pi / 2, 1000)
edges = feature.canny(sourceAmpArray,
sigma=self.config.cannySigma,
low_threshold=self.config.thresholdLow,
high_threshold=self.config.thresholdHigh)
h, theta, d = hough_line(edges, theta=tested_angles)
_, angle, dist = hough_line_peaks(h, theta, d)
if len(angle) != 2:
continue
mean_angle = np.mean(angle)
mean_dist = np.mean(dist)
select = mixCrosstalk.satellite_mask(sourceAmpArray,
mean_angle,
mean_dist,
width=self.config.maskWidth)
signal = np.max(sourceAmpArray[select])
self.log.debug(" Source amplifier: %s", sourceAmpName)
outputFluxes[sourceChip][sourceAmpName] = [float(signal)]
for targetAmp in targetDetector:
# iterate over targetExposure
targetAmpName = targetAmp.getName()
if sourceAmpName == targetAmpName and sourceChip == targetChip:
ratioDict[sourceAmpName][targetAmpName] = []
continue
self.log.debug(" Target amplifier: %s", targetAmpName)
if self.config.correctNoiseCovariance:
covariance = mixCrosstalk.calculate_covariance(
rawExp, sourceAmp, targetAmp)
else:
noise = np.asarray(
[[sourceAmp.getReadNoise() / sourceAmp.getGain(), 0.],
[0.,
targetAmp.getReadNoise() / targetAmp.getGain()]])
covariance = np.square(noise)
targetAmpImage = CrosstalkCalib.extractAmp(
targetIm.image,
targetAmp,
sourceAmp,
isTrimmed=self.config.isTrimmed)
targetAmpArray = targetAmpImage.array
results = mixCrosstalk.crosstalk_fit(
sourceAmpArray,
targetAmpArray,
select,
covariance=covariance,
correct_covariance=self.config.correctNoiseCovariance,
seed=189)
ratioDict[targetAmpName][sourceAmpName] = [float(results[0])]
zoffsetDict[targetAmpName][sourceAmpName] = [float(results[1])]
ytiltDict[targetAmpName][sourceAmpName] = [float(results[2])]
xtiltDict[targetAmpName][sourceAmpName] = [float(results[3])]
extractedCount += 1
self.log.info("Extracted %d pixels from %s -> %s", extractedCount,
sourceChip, targetChip)
outputRatios[targetChip][sourceChip] = ratioDict
outputZOffsets[targetChip][sourceChip] = zoffsetDict
outputYTilts[targetChip][sourceChip] = ytiltDict
outputXTilts[targetChip][sourceChip] = xtiltDict
return pipeBase.Struct(outputRatios=ddict2dict(outputRatios),
outputFluxes=ddict2dict(outputFluxes),
outputZOffsets=ddict2dict(outputZOffsets),
outputYTilts=ddict2dict(outputYTilts),
outputXTilts=ddict2dict(outputXTilts))
def run(self,
sensor_name,
infiles,
bias_frame=None,
dark_frame=None,
linearity_correction=None):
if not isinstance(infiles, list):
infiles = [infiles]
## Get sensor information from header
all_amps = imutils.allAmps(infiles[0])
with fits.open(infiles[0]) as hdulist:
lsst_num = hdulist[0].header['LSST_NUM']
teststand = hdulist[0].header['TSTAND']
manufacturer = lsst_num[:3]
## Interface with SQL database
database = self.config.database
logging.info(
"{0} Running CrosstalkColumnTask using database {1}".format(
datetime.now(), database))
with db_session(database) as session:
## Get sensor from database
try:
sensor = Sensor.from_db(session, sensor_name=sensor_name)
except NoResultFound:
sensor = Sensor(sensor_name=sensor_name,
lsst_num=lsst_num,
manufacturer=manufacturer,
namps=len(all_amps))
sensor.segments = {
i: Segment(segment_name=AMP2SEG[i], amplifier_number=i)
for i in all_amps
}
sensor.add_to_db(session)
session.commit()
logging.info("{0} New sensor {1} added to database".format(
datetime.now(), sensor_name))
## Set configuration and analysis settings
ly = self.config.length_y
lx = self.config.length_x
threshold = self.config.threshold
if len(infiles) > 1:
is_coadd = True
else:
is_coadd = False
ccds = [
MaskedCCD(infile,
bias_frame=bias_frame,
dark_frame=dark_frame,
linearity_correction=linearity_correction)
for infile in infiles
]
logging.info("{0} ".format(datetime.now()) + \
"Processing files: {}".format(' '.join(map(str, infiles))))
## Aggressor amplifiers
for i in all_amps:
## Find aggressor regions
exptime = 1
gain = 1
bp = BrightPixels(ccds[0], i, exptime, gain, ethresh=threshold)
pixels, columns = bp.find()
if len(columns) == 0:
continue
col = columns[0]
read_noise = calculate_read_noise(ccds[0], i) * np.sqrt(
2. / len(ccds))
aggressor_images = [
ccd.unbiased_and_trimmed_image(i).getImage()
for ccd in ccds
]
aggressor_imarr = imutils.stack(aggressor_images).getArray()
signal = np.mean(aggressor_imarr[:, col])
mask = rectangular_mask(aggressor_imarr,
1000,
col,
ly=ly,
lx=lx)
## Victim amplifiers
for j in all_amps:
victim_images = [
ccd.unbiased_and_trimmed_image(j).getImage()
for ccd in ccds
]
victim_imarr = imutils.stack(victim_images).getArray()
## Add crosstalk result to database
res = crosstalk_fit(aggressor_imarr,
victim_imarr,
mask,
noise=read_noise)
result = Result(aggressor_id=sensor.segments[i].id,
victim_id=sensor.segments[j].id,
aggressor_signal=signal,
coefficient=res[0],
error=res[4],
methodology='MODEL_LSQ',
image_type='brightcolumn',
teststand=teststand,
analysis='CrosstalkColumnTask',
is_coadd=is_coadd,
z_offset=res[1],
y_tilt=res[2],
x_tilt=res[3])
result.add_to_db(session)
logging.info(
"{0} Injested C({1},{2}) for signal {3:.1f}".format(
datetime.now(), i, j, signal))
logging.info("{0} Task completed successfully.".format(
datetime.now()))
def run(self, inputExp, rawExp):
## run() method
outputRatios = defaultdict(lambda: defaultdict(dict))
outputFluxes = defaultdict(lambda: defaultdict(dict))
outputZOffsets = defaultdict(lambda: defaultdict(dict))
outputYTilts = defaultdict(lambda: defaultdict(dict))
outputXTilts = defaultdict(lambda: defaultdict(dict))
badPixels = list(self.config.badMask)
targetDetector = inputExp.getDetector()
targetChip = targetDetector.getName()
targetIm = inputExp.getMaskedImage()
## loop on sourceExp would go here
sourceExp = inputExp ## Ignore other exposures for now
sourceDetector = sourceExp.getDetector()
sourceChip = sourceDetector.getName()
sourceIm = sourceExp.getMaskedImage()
bad = sourceIm.getMask().getPlaneBitMask(badPixels)
self.log.info("Measuring crosstalk from source: %s", sourceChip)
ratioDict = defaultdict(lambda: defaultdict(list))
zoffsetDict = defaultdict(lambda: defaultdict(list))
ytiltDict = defaultdict(lambda: defaultdict(list))
xtiltDict = defaultdict(lambda: defaultdict(list))
extractedCount = 0
for sourceAmp in sourceDetector:
sourceAmpName = sourceAmp.getName()
sourceAmpImage = sourceIm[sourceAmp.getBBox()]
sourceMask = sourceAmpImage.mask.array
sourceAmpArray = sourceAmpImage.image.array
columns = mixCrosstalk.find_bright_columns(sourceAmpArray,
self.config.threshold)
if len(columns) == 0: continue
select = mixCrosstalk.rectangular_mask(sourceAmpArray,
1000,
columns[0],
ly=self.config.maskLengthY,
lx=self.config.maskLengthX)
signal = np.mean(sourceAmpArray[:, columns[0]])
self.log.debug(" Source amplifier: %s", sourceAmpName)
outputFluxes[sourceChip][sourceAmpName] = [float(signal)]
for targetAmp in targetDetector:
# iterate over targetExposure
targetAmpName = targetAmp.getName()
if sourceAmpName == targetAmpName and sourceChip == targetChip:
ratioDict[sourceAmpName][targetAmpName] = []
continue
self.log.debug(" Target amplifier: %s", targetAmpName)
if self.config.correctNoiseCovariance:
covariance = mixCrosstalk.calculate_covariance(
rawExp, sourceAmp, targetAmp)
else:
noise = np.asarray(
[[sourceAmp.getReadNoise() / sourceAmp.getGain(), 0.],
[0.,
targetAmp.getReadNoise() / targetAmp.getGain()]])
covariance = np.square(noise)
targetAmpImage = CrosstalkCalib.extractAmp(
targetIm.image,
targetAmp,
sourceAmp,
isTrimmed=self.config.isTrimmed)
targetAmpArray = targetAmpImage.array
results = mixCrosstalk.crosstalk_fit(
sourceAmpArray,
targetAmpArray,
select,
covariance=covariance,
correct_covariance=self.config.correctNoiseCovariance,
seed=189)
ratioDict[targetAmpName][sourceAmpName] = [float(results[0])]
zoffsetDict[targetAmpName][sourceAmpName] = [float(results[1])]
ytiltDict[targetAmpName][sourceAmpName] = [float(results[2])]
xtiltDict[targetAmpName][sourceAmpName] = [float(results[3])]
extractedCount += 1
self.log.info("Extracted %d pixels from %s -> %s", extractedCount,
sourceChip, targetChip)
outputRatios[targetChip][sourceChip] = ratioDict
outputZOffsets[targetChip][sourceChip] = zoffsetDict
outputYTilts[targetChip][sourceChip] = ytiltDict
outputXTilts[targetChip][sourceChip] = xtiltDict
return pipeBase.Struct(outputRatios=ddict2dict(outputRatios),
outputFluxes=ddict2dict(outputFluxes),
outputZOffsets=ddict2dict(outputZOffsets),
outputYTilts=ddict2dict(outputYTilts),
outputXTilts=ddict2dict(outputXTilts))