def jaccard_distance(u, v):
"""return jaccard distance"""
u = numpy.asarray(u)
v = numpy.asarray(v)
return (numpy.double(numpy.bitwise_and((u != v),
numpy.bitwise_or(u != 0, v != 0)).sum())
/ numpy.double(numpy.bitwise_or(u != 0, v != 0).sum()))
def simulatestep(ps, pm, pv, teilchenort, teilchenmobil, number, vel):
zzv = np.random.random(number)
zzv2 = zzv < ps
zzv3 = zzv < pm
#print "ss"
# print teilchen, number
#zzvVel = np.random.randint(-1,2, size=number)
# zzvVel2 = np.random.random(number)
#zzvVel3 = (zzvVel2 < pv) * zzvVel
#print zzvVel
#berechne neuen Zustand für die Teilchen# bin mobil, wenn
# vorher mobil und es bleibe (zzv3, pm)
# oder: war nicht mobil und bleibe nicht (invertiert zu oder)
mobilneu = np.bitwise_or(np.bitwise_and(teilchenmobil, zzv3),(np.invert(np.bitwise_or(teilchenmobil, zzv2))))
#vel = vel + zzvVel3 * mobilneu
#vel = vel * mobilneu
#print vel
#np.clip(vel, 0, 12, vel)
#print "vel, ss", vel
# wenn mobil, addiere 1 zum Ort
teilchenortneu = teilchenort + vel*mobilneu
return teilchenortneu, mobilneu, vel
def pattern_distance_jaccard(a, b):
"""
Computes a distance measure for two binary patterns based on their
Jaccard-Needham distance, defined as
.. math::
d_J(a,b) = 1 - J(a,b) = \\frac{|a \\cup b| - |a \\cap b|}{|a \\cup b|}.
The similarity measure takes values on the closed interval [0, 1],
where a value of 1 is attained for disjoint, i.e. maximally dissimilar
patterns a and b and a value of 0 for the case of :math:`a=b`.
Parameters
----------
a : list or array, int or bool
Input pattern
b : list or array, int or bool
Input pattern
Returns
-------
dist : double
Jaccard distance between `a` and `b`.
"""
# Note: code taken from scipy. Duplicated as only numpy references wanted for base functionality
a = np.atleast_1d(a).astype(bool)
b = np.atleast_1d(b).astype(bool)
dist = (np.double(np.bitwise_and((a != b), np.bitwise_or(a != 0, b != 0)).sum())
/ np.double(np.bitwise_or(a != 0, b != 0).sum()))
return dist
def __invertibleToRGB(self, rgbVarr, varrIdx, colorLutStruct):
'''
Decode an RGB image rendered in INVERTIBLE_LUT mode. The image encodes
float values as colors, so the RGB value is first decoded into its
represented float value and then the color table is applied.
'''
w0 = np.left_shift(
rgbVarr[varrIdx[0], varrIdx[1], 0].astype(np.uint32), 16)
w1 = np.left_shift(
rgbVarr[varrIdx[0], varrIdx[1], 1].astype(np.uint32), 8)
w2 = rgbVarr[varrIdx[0], varrIdx[1], 2]
value = np.bitwise_or(w0, w1)
value = np.bitwise_or(value, w2)
value = np.subtract(value.astype(np.int32), 1)
normalized_val = np.divide(value.astype(float), 0xFFFFFE)
# Map float value to color lut (use a histogram to support non-uniform
# colormaps (fetch bin indices))
colorLut = colorLutStruct.lut
bins = colorLutStruct.adjustedBins
idx = np.digitize(normalized_val, bins)
idx = np.subtract(idx, 1)
valueImage = np.zeros([rgbVarr.shape[0], rgbVarr.shape[1]],
dtype=np.uint32)
valueImage[varrIdx[0], varrIdx[1]] = idx
return colorLut[valueImage]
def valuewriter(self, imageSlice, fname, vrange):
""" Takes in either a (1C) float or a RGB (3C) buffer and writes it as
an image file."""
# Adjust the filename, replace png with .im
baseName, ext = os.path.splitext(fname)
adjustedName = baseName + self.floatExtension()
dimensions = imageSlice.shape
if len(dimensions) == 2 and imageSlice.dtype == numpy.float32:
# Given as single channel floating point buffer.
self.zwriter(imageSlice, adjustedName)
elif (len(dimensions) > 2) and (dimensions[2] == 3):
#if self.dontConvertValsToFloat
# self.rgbwriter(imageSlice, fname)
# return
# Given an RGB buffer
# Convert it to a floating point buffer for consistency
# TODO: just one copy of this code in cinema
w0 = numpy.left_shift(imageSlice[:,:,0].astype(numpy.uint32), 16)
w1 = numpy.left_shift(imageSlice[:,:,1].astype(numpy.uint32), 8)
w2 = imageSlice[:,:,2].astype(numpy.uint32)
value = numpy.bitwise_or(w0,w1)
value = numpy.bitwise_or(value,w2)
value = numpy.subtract(value, 1)
value = value.astype(numpy.float32)
adjusted_val = numpy.divide(value, float(0xFFFFFE))
self.zwriter(adjusted_val, adjustedName)
else:
raise ValueError("Invalid dimensions for a value raster.")
def replace_vals(filename, replace, delim=','):
'''
Replace the values in filename with specified values in replace_values
Parameters
----------
filename : string
Will be read into a rec array
replace_values : tuple
First object is value to replace and second object is what to replace
it with
'''
data = csv2rec(filename, delimiter=delim, missing=replace[0])
for nm in data.dtype.names:
try:
# Missing float
isNaN = (np.isnan(data[nm]))
except:
isNaN = np.zeros(len(data[nm]), dtype=bool)
isBlank = np.array([it == '' for it in data[nm]])
isMinusOne = (data[nm] == -1)# Missing int
# Missing other
isNone = np.array([i == None for i in data[nm]])
ind = np.bitwise_or(isNaN, isBlank)
ind = np.bitwise_or(ind, isMinusOne)
ind = np.bitwise_or(ind, isNone)
data[nm][ind] = replace[1]
return data
def place_ship(self, position, length, orientation):
"""
Return None if ship cannot be placed
"""
ship = None
if orientation == 'H':
zeros = np.zeros(self.width * self.height, dtype='int8')
if (position[0] + length) > self.width:
return None
for i in range(length):
zeros[position[1] * self.width + position[0]+i] = 1
if np.all(np.bitwise_and(self._layout, zeros) == 0):
self._layout = np.bitwise_or(self._layout, zeros)
ship = Ship(position, length, orientation)
elif orientation == 'V':
zeros = np.zeros(self.width * self.height, dtype='int8')
if (position[1] + length) > self.height:
return None
for i in range(length):
zeros[(position[1] + i) * self.width + position[0]] = 1
if np.all(np.bitwise_and(self._layout, zeros) == 0):
self._layout = np.bitwise_or(self._layout, zeros)
ship = Ship(position, length, orientation)
if ship:
self._ships.append(ship)
return ship
def filter(mask, cube, header, clipMethod, threshold, rmsMode, fluxRange, verbose):
err.message("Running threshold finder.")
# Sanity checks of user input
err.ensure(
clipMethod in {"absolute", "relative"},
"Threshold finder failed. Illegal clip method: '" + str(clipMethod) + "'.")
err.ensure(
rmsMode in {"std", "mad", "gauss", "negative"},
"Threshold finder failed. Illegal RMS mode: '" + str(rmsMode) + "'.")
err.ensure(
fluxRange in {"positive", "negative", "all"},
"Threshold finder failed. Illegal flux range: '" + str(fluxRange) + "'.")
# Scale threshold by RMS if requested
if clipMethod == "relative":
threshold *= GetRMS(cube, rmsMode=rmsMode, fluxRange=fluxRange, zoomx=1, zoomy=1, zoomz=1, verbose=verbose)
# Print some information and check sign of threshold
err.message(" Using threshold of " + str(threshold) + ".")
err.ensure(threshold >= 0.0, "Threshold finder failed. Threshold value is negative.")
# Run the threshold finder, setting bit 1 of the mask for |cube| >= |threshold|:
np.bitwise_or(mask, np.greater_equal(np.absolute(cube), threshold), out=mask)
return
def _combine_masks(self):
"""Combine multiple mask planes.
Sets the detected mask bit if any image has a detection,
and sets other bits only if set in all images.
Returns
-------
np.ndarray
The combined mask plane.
"""
mask_arr = (exp.getMaskedImage().getMask() for exp in self.exposures)
detected_mask = None
mask_use = None
for mask in mask_arr:
mask_vals = mask.getArray()
if mask_use is None:
mask_use = mask_vals
else:
mask_use = np.bitwise_and(mask_use, mask_vals)
detected_bit = mask.getPlaneBitMask('DETECTED')
if detected_mask is None:
detected_mask = mask_vals & detected_bit
else:
detected_mask = np.bitwise_or(detected_mask, (mask_vals & detected_bit))
mask_vals_return = np.bitwise_or(mask_use, detected_mask)
return mask_vals_return
def read_ports(self):
key = psychopy.event.getKeys(keyList=['1','2','3','k'])
ports = np.asarray([False,False,False])
if key:
if not key[0] in self._key_pressed: self._key_pressed.append(key[0])
if 'k' in self._key_pressed and '1' in self._key_pressed:
ports = np.bitwise_or(ports,[True,False,False])
psychopy.event.clearEvents()
print(self._key_pressed)
self._key_pressed.remove('k')
self._key_pressed.remove('1')
if 'k' in self._key_pressed and '2' in self._key_pressed:
ports = np.bitwise_or(ports,[False,True,False])
psychopy.event.clearEvents()
print(self._key_pressed)
self._key_pressed.remove('k')
self._key_pressed.remove('2')
if 'k' in self._key_pressed and '3' in self._key_pressed:
ports = np.bitwise_or(ports,[False,False,True])
psychopy.event.clearEvents()
print(self._key_pressed)
self._key_pressed.remove('k')
self._key_pressed.remove('3')
return self.get_ports()[ports]
def reset_bad_gain(pdq, gain):
"""
For pixels in the gain array that are either non-positive or NaN, reset the
the corresponding pixels in the pixel DQ array to NO_GAIN_VALUE and
DO_NOT_USE so that they will be ignored.
Parameters
----------
pdq : int, 2D array
pixel dq array of input model
gain : float32, 2D array
gain array from reference file
Returns
-------
pdq : int, 2D array
pixleldq array of input model, reset to NO_GAIN_VALUE and DO_NOT_USE
for pixels in the gain array that are either non-positive or NaN.
"""
wh_g = np.where( gain <= 0.)
if len(wh_g[0]) > 0:
pdq[wh_g] = np.bitwise_or( pdq[wh_g], dqflags.pixel['NO_GAIN_VALUE'] )
pdq[wh_g] = np.bitwise_or( pdq[wh_g], dqflags.pixel['DO_NOT_USE'] )
wh_g = np.where( np.isnan( gain ))
if len(wh_g[0]) > 0:
pdq[wh_g] = np.bitwise_or( pdq[wh_g], dqflags.pixel['NO_GAIN_VALUE'] )
pdq[wh_g] = np.bitwise_or( pdq[wh_g], dqflags.pixel['DO_NOT_USE'] )
return pdq
def get_keyv(iarr, level):
i1, i2, i3 = (v.astype("int64") for v in iarr)
i1 = spread_bitsv(i1, level)
i2 = spread_bitsv(i2, level) << 1
i3 = spread_bitsv(i3, level) << 2
np.bitwise_or(i1, i2, i1)
np.bitwise_or(i1, i3, i1)
return i1
def mas_combining4(row):
rows = ['DER_mass_MMC','DER_mass_vis','DER_mass_transverse_met_lep']
frame = row[rows[0]] + row[rows[1]] + row[rows[2]]
mask = np.bitwise_or(row[rows[0]]==-999, row[rows[1]]==-999.)
mask = np.bitwise_or(mask, row[rows[1]]==-999.)
frame[mask] = -999.
frame.name = 'MNew_mc4'
return frame
def partBy2(x):
x = np.bitwise_and(x, 0x1f00000000ffff)
x = np.bitwise_and(np.bitwise_or(x, np.left_shift(x, 32)), 0x1f00000000ffff)
x = np.bitwise_and(np.bitwise_or(x, np.left_shift(x, 16)), 0x1f0000ff0000ff)
x = np.bitwise_and(np.bitwise_or(x, np.left_shift(x, 8)), 0x100f00f00f00f00f)
x = np.bitwise_and(np.bitwise_or(x, np.left_shift(x, 4)), 0x10c30c30c30c30c3)
x = np.bitwise_and(np.bitwise_or(x, np.left_shift(x, 2)), 0x1249249249249249)
return x
def __call__(self):
if hasattr(self.ctx.params, 'cleaner') and self.ctx.params.cleaner != "None":
#load module
mod = importlib.import_module(self.ctx.params.cleaner)
rfi_mask_vx, rfi_mask_vy = mod.rm_rfi(self.ctx)
self.ctx.tod_vx.mask = np.bitwise_or(self.ctx.tod_vx.mask, rfi_mask_vx)
self.ctx.tod_vy.mask = np.bitwise_or(self.ctx.tod_vy.mask, rfi_mask_vy)
def colRowIsOnSciencePixelList(self, col, row, padding=DEFAULT_PADDING):
"""similar to colRowIsOnSciencePixelList() but takes lists as input"""
out = np.ones(len(col), dtype=bool)
col_arr = np.array(col)
row_arr = np.array(row)
mask = np.bitwise_or(col_arr < 12. - padding, col_arr > 1111 + padding)
out[mask] = False
mask = np.bitwise_or(row_arr < 20. - padding, row_arr > 1043 + padding)
out[mask] = False
return out
def _numpy(self, data, weights, shape):
q = self.quantity(data)
self._checkNPQuantity(q, shape)
self._checkNPWeights(weights, shape)
weights = self._makeNPWeights(weights, shape)
newentries = weights.sum()
import numpy
selection = numpy.isnan(q)
numpy.bitwise_not(selection, selection)
subweights = weights.copy()
subweights[selection] = 0.0
self.nanflow._numpy(data, subweights, shape)
# avoid nan warning in calculations by flinging the nans elsewhere
numpy.bitwise_not(selection, selection)
q = numpy.array(q, dtype=numpy.float64)
q[selection] = 0.0
weights = weights.copy()
weights[selection] = 0.0
if all(isinstance(v, Count) and v.transform is identity for c, v in self.bins) and numpy.all(numpy.isfinite(q)) and numpy.all(numpy.isfinite(weights)):
h, _ = numpy.histogram(q, [float("-inf")] + [(c1 + c2)/2.0 for (c1, v1), (c2, v2) in zip(self.bins[:-1], self.bins[1:])] + [float("inf")], weights=weights)
for hi, (c, v) in zip(h, self.bins):
v.fill(None, float(hi))
else:
selection = numpy.empty(q.shape, dtype=numpy.bool)
selection2 = numpy.empty(q.shape, dtype=numpy.bool)
for index in xrange(len(self.bins)):
if index == 0:
high = (self.bins[index][0] + self.bins[index + 1][0])/2.0
numpy.greater_equal(q, high, selection)
elif index == len(self.bins) - 1:
low = (self.bins[index - 1][0] + self.bins[index][0])/2.0
numpy.less(q, low, selection)
else:
low = (self.bins[index - 1][0] + self.bins[index][0])/2.0
high = (self.bins[index][0] + self.bins[index + 1][0])/2.0
numpy.less(q, low, selection)
numpy.greater_equal(q, high, selection2)
numpy.bitwise_or(selection, selection2, selection)
subweights[:] = weights
subweights[selection] = 0.0
self.bins[index][1]._numpy(data, subweights, shape)
# no possibility of exception from here on out (for rollback)
self.entries += float(newentries)
def color(img_on, img_off):
img_off = cv2.imread(img_off)
img_on = cv2.imread(img_on)
img = np.array((np.array(img_on, dtype=np.int16)-np.array(img_off, dtype=np.int16)).clip(0,255), dtype=np.uint8)
img = cv2.medianBlur(img,3)
cv2.imshow("soustracted",cv2.resize(img, (img.shape[1]/2,img.shape[0]/2)))
cv2.waitKey(0)
lower = np.array([0, 0, 20], dtype=np.uint8)
upper = np.array([5, 5, 255], dtype=np.uint8)
mask0 = cv2.inRange(img, lower, upper)
cv2.imshow("color_threshold_red",cv2.resize(mask0, (img.shape[1]/2,img.shape[0]/2)))
cv2.waitKey(0)
img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
lower = np.array([0, 170, 20], dtype=np.uint8)
upper = np.array([10, 255, 255], dtype=np.uint8)
mask1 = cv2.inRange(img_hsv, lower, upper)
lower = np.array([170, 170, 20], dtype=np.uint8)
upper = np.array([180, 255, 255], dtype=np.uint8)
mask2 = cv2.inRange(img_hsv, lower, upper)
mask = np.bitwise_or(mask1, mask2)
cv2.imshow("hsv_threshold_low_brightness",cv2.resize(mask, (mask.shape[1]/2,mask.shape[0]/2)))
cv2.waitKey(0)
lower = np.array([80, 0, 200], dtype=np.uint8)
upper = np.array([100, 255, 255], dtype=np.uint8)
mask3 = cv2.inRange(img_hsv, lower, upper)
cv2.imshow("hsv_threshold_high_brightness",cv2.resize(mask3, (mask3.shape[1]/2,mask3.shape[0]/2)))
cv2.waitKey(0)
mask = np.bitwise_or(mask, mask3)
mask = np.bitwise_or(mask0, mask)
mask = cv2.GaussianBlur(mask,(3,3),0)
mask = cv2.inRange(mask, np.array([250]), np.array([255]))
res = cv2.bitwise_and(img_on, img_on, mask=mask)
tmp = np.zeros(res.shape)
for line in range(res.shape[0]):
moments = cv2.moments(res[line,:,2])
if(moments['m00'] != 0):
tmp[line][round(moments['m01']/moments['m00'])] = [0,255,0]
cv2.imshow("hsv_or_color",cv2.resize(mask, (mask.shape[1]/2,mask.shape[0]/2)))
cv2.waitKey(0)
cv2.imshow("result",cv2.resize(tmp, (res.shape[1]/2,res.shape[0]/2)))
cv2.waitKey(0)
def gameoflife(W,H,ITER,DIST,random_state):
random.setstate(random_state)
LIVING_LOW= 2
LIVING_HIGH = 3
ALIVE = 3
full = np.zeros((W+2,H+2), dtype=np.long, dist=DIST)
dead = np.zeros((W,H), dtype=np.long, dist=DIST)
live = np.zeros((W,H), dtype=np.long, dist=DIST)
live2 = np.zeros((W,H), dtype=np.long, dist=DIST)
neighbors = np.zeros((W,H), dtype=np.long, dist=DIST)
cells = full[1:W+1,1:H+1]
ul = full[0:W, 0:H]
um = full[0:W, 1:H+1]
ur = full[0:W, 2:H+2]
ml = full[1:W+1, 0:H]
mr = full[1:W+1, 2:H+2]
ll = full[2:W+2, 0:H]
lm = full[2:W+2, 1:H+1]
lr = full[2:W+2, 2:H+2]
for i in xrange(W):
for j in range(H):
if random.random() > .8:
cells[i][j] = 1
for i in xrange(ITER):
# zero neighbors
np.bitwise_and(neighbors,0,neighbors)
# count neighbors
neighbors += ul
neighbors += um
neighbors += ur
neighbors += ml
neighbors += mr
neighbors += ll
neighbors += lm
neighbors += lr
# extract live cells neighbors
np.multiply(neighbors, cells, live)
# find all living cells among the already living
np.equal(live, LIVING_LOW, live2)
np.equal(live, LIVING_HIGH, live)
# merge living cells into 'live'
np.bitwise_or(live, live2, live)
# extract dead cell neighbors
np.equal(cells, 0, dead)
dead *= neighbors
np.equal(dead,ALIVE,dead)
# make sure all threads have read their values
np.bitwise_or(live, dead, cells)
return full
def simulatestep(ps, pm, teilchenort, teilchenmobil, number):
zzv = np.random.random(number)
zzv2 = zzv < ps
zzv3 = zzv < pm
#berechne neuen Zustand für die Teilchen
# vorher mobil und es bleibe (zzv3, pm)
# oder: war nicht mobil und bleibe nicht (invertiert zu oder)
mobilneu = np.bitwise_or(np.bitwise_and(teilchenmobil, zzv3),(np.invert(np.bitwise_or(teilchenmobil, zzv2))))
# wenn mobil, addiere 1 zum Ort
teilchenortneu = teilchenort + mobilneu
return teilchenortneu, mobilneu
def simulatestep(p_in, teilchenort, teilchenmobil, number, vel):
psvektor = np.array([ps] * number)
zzv = np.random.random(number)
#bleibe stationär
zzv2 = zzv < psvektor
#bleibe mobil
zzv3 = zzv < p_in
#print "pin", p_in
'''print "vektoren"
print p_in
print zzv
print zzv2
print zzv3'''
#berechne neuen Zustand für die Teilchen
# bin mobil, wenn
# vorher mobil und es bleibe (zzv3, bestimmt durch p_in, variable WKeiten)
# oder: nicht mobil und nicht bleibe (zzv2, bestimmt durch gammavektor, fest TODO: geht das geschickter?)
mobilneu = np.bitwise_or(np.bitwise_and(teilchenmobil, zzv3),(np.invert(np.bitwise_or(teilchenmobil, zzv2))))
# wenn mobil, addiere vel zum Ort
teilchenortneu = teilchenort + vel*mobilneu
#print "mobneu", mobilneu
vel = np.clip(vel, velmin, velmax)
# berechne neue Geschwindigkeiten; falls mobil addiere was dazu, clip davor und danach zum Schutz
velneu = (vel + ((velmax-vel)/veldivisor)) * mobilneu # wenn nicht mobil, schneller, wenn mobil
#print "velneu", velneu
#vel ist mindestens 1, für den neuen Ort wird mobil ja eh noch mal befragt
velneu = np.clip(velneu, 0, velmax)
#print max(velneu)
#print "p_in", p_in
#time.sleep(1)
# wo ich nicht mobil bin: gamma0 -> p1: [0, g0, 0,0,0, g0, g0, 0...]
p1 = np.invert(mobilneu) * gamma0
#wo ich mobil bin, erhöht sich p TODO p2: [p, 0, p,p,p,0,0,p,...]
p2 = mobilneu * (gamma0 + vel/gammadivisor)
#print "veldings", 1/vel
#print gamma0/gammadivisor
#print "p2", p2
p_out = p1 + p2
#print "pout", p_out
p_out = np.clip(p_out, gamma0, 0.9999)
#print velneu,'\n', p_out
return teilchenortneu, mobilneu, velneu, p_out
def update_flt(raw):
# Copy Pipeline flt to ir_aux_files directory
# Add Gabe's extra bad pixels to FLT DQ
# Flag persistence pixels (>0.6*ERR) in FLT DQ if persist.fits exists
# Replace FLT data with flattened ramp from make_IMA_FLT
# Add header keywords indicating when this script was run
# Write fixes into original FLT to preserve WCS transformations
flt = raw.replace('raw.fits', 'flt.fits')
if not os.path.exists(flt):
orig_flt = glob.glob('../targets/*/{}'.format(flt))[0]
# print 'BEGINNING FLT UPDATE'
# print '________________________________________________________'
shutil.copy(orig_flt, '.')
else:
print 'FLT {} already in directory, skipping'.format(flt)
return
hdu = fits.open(flt, mode='update')
orig_dq = hdu['DQ'].data
new_flags = fits.getdata('/astro/pabeta/wfc3/data/badpix_spars200_Nov9.fits')
if orig_dq.shape == (1014,1014):
new_dq = np.bitwise_or(orig_dq,new_flags)
else:
hdr1 = hdu[1].header
ltv1, ltv2 = abs(hdr1['LTV1']), abs(hdr1['LTV2'])
naxis1, naxis2 = hdr1['NAXIS1'], hdr1['NAXIS2']
flags_sub = new_flags[ltv2:ltv2+naxis2,ltv1:ltv1+naxis1]
new_dq = np.bitwise_or(orig_dq,flags_sub)
today = datetime.today()
date = '{}-{}-{}'.format(today.year,today.month,today.day)
hdu[0].header['BPIXFLAG'] = date
proposid = hdu[0].header['PROPOSID']
pers_path = '/grp/hst/wfc3a/GO_Links/{}/Visit*/Persist/{}'.format(proposid,hdu[0].header['ROOTNAME'].lower()+'_persist.fits')
if os.path.exists(pers_path):
err = hdu['ERR'].data
pers_flags = np.zeros(err.shape, dtype=np.uint16)
pers_data = fits.getdata(pers_path)
pers_flags[pers_data>0.6*err] = 1024
new_dq = np.bitwise_or(new_dq,pers_flags)
hdu[0].header['PERSFLAG'] = date
print 'ADDING BAD PIXELS TO DQ ARRAY FOR {}'.format(flt)
hdu['DQ'].data = new_dq
flb = flt.replace('flt.fits','flb.fits')
if os.path.exists(flb):
flb_data = fits.getdata(flb,1)
hdu['SCI'].data = flb_data
print 'REPLACING PIPELINE FLT DATA WITH FLATTENED RAMP FOR {}'.format(flt)
hdu[0].header['FLATRAMP'] = date
hdu.close()
return flt
def simulatestep(ps, pm, teilchenort, teilchenmobil):
number = len(teilchenort)
# print teilchen, number
zzv = np.random.random(number)
zzv2 = zzv < ps
zzv3 = zzv < pm
#berechne neuen Zustand für die Teilchen
mobilneu = np.bitwise_or(np.bitwise_and(teilchenmobil, zzv3),(np.invert(np.bitwise_or(teilchenmobil, zzv2))))
# wenn mobil, addiere 1 zum Ort
teilchenortneu = teilchenort + mobilneu
return teilchenortneu, mobilneu
def apply_flat_field (science, flat ):
"""
Short Summary
-------------
Flat fields the data and error arrays, and updates data quality array
based on bad pixels in flat field arrays. Applies portion of flat field
corresponding to science image subarray.
Parameters
----------
science: JWST data model
input science data model
flat: JWST data model
flat field data model
Returns
-------
None
"""
# If the input science data model is a subarray, extract the same
# subarray from the flatfield model
if ref_matches_sci (flat, science):
flat_data = flat.data
flat_dq = flat.dq
else:
log.info("Extracting matching subarray from flat")
flat_data = get_subarray(flat.data, science)
flat_dq = get_subarray(flat.dq, science)
# For pixels whose flat is either NaN or NO_FLAT_FIELD, update their DQ to
# indicate that no flat is applied to those pixels
flat_dq[np.isnan(flat_data)] = np.bitwise_or(flat_dq[np.isnan(flat_data)],
dqflags.pixel['NO_FLAT_FIELD'])
# Replace NaN's in flat with 1's
flat_data[np.isnan(flat_data)] = 1.0
# Reset flat values of pixels having DQ values containing NO_FLAT_FIELD
# to 1.0, so that no flat fielding correction is made
wh_dq = np.bitwise_and( flat_dq, dqflags.pixel['NO_FLAT_FIELD'])
flat_data[ wh_dq == dqflags.pixel['NO_FLAT_FIELD'] ] = 1.0
# Flatten data and error arrays
science.data /= flat_data
science.err /= flat_data
# Combine the science and flat DQ arrays
science.dq = np.bitwise_or(science.dq, flat_dq)
def CreateTimeMatrix(DataFile):
"""
"""
phofile = PhoAlign + os.path.sep + DataFile + '.lab'
sylfile = SylAlign + os.path.sep + DataFile + '.lab'
worfile = WorAlign + os.path.sep + DataFile + '.lab'
phodata = htslab.read_full_lab(phofile)
syldata = htslab.read_full_lab(sylfile)
wordata = htslab.read_full_lab(worfile)
if len(wordata[1]) != len(syldata[1]) or wordata[1][-1] != syldata[1][-1]:
print "\t Unequal Length %s" % (DataFile)
return 0
DataTime = np.int(syldata[1][-1])
# default, update at every frame level
dataMat = np.bitwise_or(np.zeros([DataTime], dtype=np.int8), bitInfo['fra'])
preSyl = ''
preWor = ''
for idx1 in xrange(len(syldata[0])):
frameStart = syldata[0][idx1]
frameEnd = syldata[1][idx1]
# update the phoneme state
dataMat[frameStart] = np.bitwise_or(dataMat[frameStart], bitInfo['pho'])
syllabel = syldata[2][idx1]
worlabel = wordata[2][idx1]
if syllabel != preSyl:
dataMat[frameStart] = np.bitwise_or(dataMat[frameStart], bitInfo['syl'])
if worlabel != preWor:
dataMat[frameStart] = np.bitwise_or(dataMat[frameStart], bitInfo['wor'])
if len(preWor)==0 or preWor == phraseSym or worlabel == phraseSym:
dataMat[frameStart] = np.bitwise_or(dataMat[frameStart], bitInfo['phr'])
preSyl = syllabel
preWor = worlabel
if CheckBinary:
pholabel = phodata[2][idx1]
for t in range(np.int(frameStart), np.int(frameEnd)):
print "%d, %s [%s %s %s]" % (t,np.binary_repr(dataMat[t], len(bitInfo)),
pholabel[0:6], syllabel[0:6], worlabel[0:6])
py_rw.write_raw_mat(dataMat, DataDir+os.path.sep+DataFile+'.bin', 'u1')
return DataTime
def embedPayload(self,payload,override=False):
if type(payload) is not Payload:
raise TypeError("Incorrect input type")
if len(payload.img.shape)==3:
color=3
else:
color=1
payload_size=8*payload.img.shape[0]*payload.img.shape[1]*color
if len(self.img.shape)==3:
color1=3
else:
color1=1
carrier_size=self.img.shape[0]*self.img.shape[1]
if payload_size>carrier_size*color1:
raise ValueError("Carrier can not hold payload")
if self.payloadExists()==True and override ==False:
raise Exception("Payload exist and can not override")
xml_list=numpy.ndarray(shape=(len(payload.xml),1),dtype=numpy.uint8)
cnt=0
for item in payload.xml:
xml_list[cnt][0]=ord(item)
cnt+=1
xml_binary=numpy.unpackbits(xml_list)
image=copy.deepcopy(self.img)
if color1==3:
reshaped_image=image.flatten()
ordered_image=numpy.concatenate((reshaped_image[0::3],reshaped_image[1::3],reshaped_image[2::3]))
embed_image=ordered_image[:len(xml_binary)]
else:
ordered_image=image.flatten()
embed_image=ordered_image[:len(xml_binary)]
clean_matrix=numpy.ndarray(shape=embed_image.shape,dtype=numpy.uint8)
clean_matrix.fill(254)
numpy.bitwise_and(clean_matrix,embed_image,embed_image)
numpy.bitwise_or(embed_image,xml_binary,embed_image)
reshaped_image=numpy.concatenate((embed_image,ordered_image[len(xml_binary):]))
if color1==3:
reshaped_image=reshaped_image.reshape(3,-1)
reshaped_image=numpy.dstack(tuple(reshaped_image))
image=reshaped_image.reshape(image.shape)
else:
image=reshaped_image.reshape(image.shape)
return image
def _get_voc_color_map(n=256):
color_map = np.zeros((n, 3))
for i in xrange(n):
r = b = g = 0
cid = i
for j in xrange(0, 8):
r = np.bitwise_or(r, np.left_shift(np.unpackbits(np.array([cid], dtype=np.uint8))[-1], 7-j))
g = np.bitwise_or(g, np.left_shift(np.unpackbits(np.array([cid], dtype=np.uint8))[-2], 7-j))
b = np.bitwise_or(b, np.left_shift(np.unpackbits(np.array([cid], dtype=np.uint8))[-3], 7-j))
cid = np.right_shift(cid, 3)
color_map[i][0] = r
color_map[i][1] = g
color_map[i][2] = b
return color_map
def labelcolormap(N=256):
cmap = np.zeros((N, 3))
for i in xrange(0, N):
id = i
r, g, b = 0, 0, 0
for j in xrange(0, 8):
r = np.bitwise_or(r, (bitget(id, 0) << 7-j))
g = np.bitwise_or(g, (bitget(id, 1) << 7-j))
b = np.bitwise_or(b, (bitget(id, 2) << 7-j))
id = (id >> 3)
cmap[i, 0] = r
cmap[i, 1] = g
cmap[i, 2] = b
cmap = cmap.astype(np.float32) / 255
return cmap
def get_training_image_segmentations(self, image_indices, tooth_indices):
segmentations = []
mirrored_segmentations = []
for image_index in image_indices:
combined_segmentation = np.uint8(np.zeros(self._training_segmentations[image_index][0].shape))
combined_segmentation_mirrored = np.uint8(np.zeros(self._training_segmentations[image_index][0].shape))
for tooth_index in tooth_indices:
combined_segmentation = np.bitwise_or(combined_segmentation,
self._training_segmentations[image_index][tooth_index])
combined_segmentation_mirrored = np.bitwise_or(combined_segmentation_mirrored,
self._training_segmentations_mirrored[image_index][
tooth_index])
segmentations.append(combined_segmentation)
mirrored_segmentations.append(combined_segmentation_mirrored)
return segmentations, mirrored_segmentations
def get_detect(img,body_position):
region_upper=int(img.shape[0]*0.3)
region_lower=int(img.shape[0]*0.6)
if body_position[0]<(img.shape[1]/2.0):
region_left=body_position[0]+30
region_right=img.shape[1]-30
else:
region_left=30
region_right=body_position[0]-30
region = img[region_upper:region_lower, region_left:region_right]
edge_list=[0,0,0,0]
for i in range(3):
region_gray=cv2.cvtColor(region,cv2.COLOR_BGR2HSV)[:,:,i]
edge_list[i]=cv2.Canny(region_gray,150,200)
region_gray=cv2.cvtColor(region,cv2.COLOR_BGR2GRAY)
region_gray = cv2.GaussianBlur(region_gray, (5, 5), 0)
edge_list[3] = cv2.Canny(region_gray, 110, 200,apertureSize=5)
edge_list[1]=np.bitwise_or(edge_list[0],edge_list[1])
edge_list[2]=np.bitwise_or(edge_list[2],edge_list[1])
edge_final=np.bitwise_or(edge_list[3],edge_list[2])
contours= cv2.findContours(edge_final, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[1]
y=99999
for contour in contours:
sorted_top=sorted(contour,key=lambda contour:contour[0][1])
if sorted_top[0][0][1]<y:
raw_x = x = sorted_top[0][0][0]
raw_y = y = sorted_top[0][0][1]
print((int(x+ region_left), int(y+ region_upper)))
mask = np.zeros((region_lower-region_upper+ 2, region_right-region_left + 2), np.uint8)
mask=cv2.floodFill(region, mask, (raw_x, raw_y+16), [255,25,255])[2]
cv2.circle(img, (int(x + region_left), int(y + region_upper)), 5, (255, 0, 5), -1)
M = cv2.moments(mask)
x = int(M["m10"] / M["m00"])
y = int(M["m01"] / M["m00"])
if y<raw_y or abs(x-raw_x)>40:
x=raw_x;y=raw_y
y += region_upper
x += region_left
y = (-abs(x-body_position[0])/pow(3,0.5)+body_position[1])
else:
y += region_upper
x += region_left
cv2.circle(img, (int(x),int(y)), 5, (0,0,255), -1)
cv2.imwrite("/home/dvaliu/temp/test_center.png",img)
return [x, y]
'''
x = np.linspace(10, 20, 5)
print(x) # [10, 12.5, 15, 17.5, 20]
y = np.linspace(10, 20, 11) # [10,11,12,13,14,15,16,17,18,19,20]
'''
# 位操作
bitwise_and 对数组元素执行位 与 操作 1&1 -> 1 1&0 -> 0 0&0 -> 0 两个都是1才是1
bitwise_or 对数组元素执行位 或 操作 1|1 -> 1 1|0 -> 1 0|0 -> 0 有一个是1就是1
left_shift 向左移动二进制表示的位 0011 向左移1位，用0来补位 1100 1100000 左移1位 1000000 顶部1位丢弃掉
right_shift 向右移动二进制表示的位 1100 向右移1位，用0来补位 0110 0000011 右移1位 0000001 尾部1位丢弃掉了
'''
a = 2 # 10
b = 3 # 11
print(np.bitwise_and(a, b)) # 10 -> 1*2的1次方 + 0*2的0次方 -> 2
print(np.bitwise_or(a, b)) # 11 -> 1*2的1次方 + 1*2的0次方 -> 3
print(np.left_shift(a, 1)) # 10 左移动1位 0100 -> 4
print(np.left_shift(a, 2)) # 10 左移动2位 1000 -> 8
print(np.right_shift(a, 1)) # 10 右移动1位 01 -> 1
'''
# 字符串函数
add() 两个str或Unicode数组的逐个字符串连接
replace() 替换
split() 分割
title() 返回输入字符串的按元素标题替换版本，其中每个单词的首字母都大些
multiply() 返回按元素多重连接后的字符串
'''
print(np.char.multiply('hello ', 3)) # hello hello hello
print(np.char.title('hello xfz')) # Hello Xfz
'''
# 算数函数
def compute_filter_responses(self):
"""Compute the motion-energy filters' response to the stimuli.
Parameters
----------
stimulus : 3D np.array (n, vdim, hdim)
The movie frames.
stimulus_fps : scalar
The temporal frequency of the stimulus
gabor_temporal_window : scalar, None
The number of frames in one filter.
If None, it defaults to floor(2/3) of `stimulus_fps`
Similar to Nishimoto, 2011.
quadrature_combination : function, optional
Specifies how to combine the channel reponses quadratures.
The function must take the sin and cos as arguments in order.
Defaults to: (sin^2 + cos^2)^1/2
output_nonlinearity : function, optional
Passes the channels (after `quadrature_combination`) through a
non-linearity. The function input is the (`n`,`nfilters`) array.
Defaults to: ln(x + 1)
dozscore : bool, optional
Whether to z-score the channel responses in time
moten_pyramid_parameters: dict
See :func:`mk_moten_pyramid_params` for details on parameters
specifiying a motion-energy pyramid.
Returns
-------
filter_responses : np.array, (n, nfilters)
"""
# stimulus = stimulus.reshape(stimulus.shape[0], -1)
# if gabor_temporal_window is None:
# gabor_temporal_window = int(stimulus_fps*(2./3.))
# gabor_parameters = mk_moten_pyramid_params(stimulus_fps,
# gabor_temporal_window,
# aspect_ratio=aspect_ratio,
# **moten_pyramid_parameters)
channels = []
for idx, gabor_param in enumerate(self.param_list):
gabor = self.make_3d_gabor((self.horizontal_dim, self.vertical_dim, self.temporal_window),
*gabor_param[:-3],
aspect_ratio=self.aspect_ratio)
gabor0, gabor90, tgabor0, tgabor90 = gabor
phase = gabor_param[7]
channel_sin, channel_cos = self.dotdelay_frames(gabor0, gabor90,
tgabor0, tgabor90)
channel_sin, channel_cos = np.float32(channel_sin), np.float32(channel_cos)
if phase == 0:
channels.append(channel_sin)
elif phase == 1:
channels.append(channel_cos)
elif phase == 2:
channels.append(channel_sin**2)
elif phase == 3:
channels.append(channel_cos**2)
channels = np.asarray(channels).T
if self.zscore_each:
# from scipy.stats import zscore
if self._means is None:
self._means = np.mean(channels, axis=0)
if self._stds is None:
self._stds = np.std(channels, axis=0)
channels -= self._means
channels /= self._stds
elif self.zscore_by_type:
if self._means is None or self._stds is None:
usf = np.unique(self.param_list[:,9])
utf = np.unique(self.param_list[:,5])
ugg = np.unique(self.param_list[:,8])
self._means = np.zeros(len(self.param_list))
self._stds = np.zeros(len(self.param_list))
for sf in usf:
for tf in utf:
for gg in ugg:
idx_l = np.bitwise_and(np.bitwise_and(np.bitwise_and(self.param_list[:,9] == sf, self.param_list[:,5]==tf), np.bitwise_or(self.param_list[:,7]==0, self.param_list[:,7]==1)), self.param_list[:,8]==gg)
idx_q = np.bitwise_and(np.bitwise_and(np.bitwise_and(self.param_list[:,9] == sf, self.param_list[:,5]==tf), np.bitwise_or(self.param_list[:,7]==2, self.param_list[:,7]==3)), self.param_list[:,8]==gg)
self._means[idx_l] = np.mean(channels[:,idx_l])
self._means[idx_q] = np.mean(channels[:,idx_q])
self._stds[idx_l] = np.sqrt(np.mean((channels[:,idx_l] - self._means[idx_l])**2))
self._stds[idx_q] = np.sqrt(np.mean((channels[:,idx_q] - self._means[idx_q])**2))
channels -= self._means
channels /= self._stds
return channels
def DelayFunc(self):
# data analysed the old way to get delay range plot
lastTag = np.max(self.ttagBuf.rawtags)
firstTag = (lastTag - np.int(
np.ceil(self.exptime / self.ttagBuf.resolution))).astype(np.uint64)
rawPos = np.nonzero(
np.bitwise_and(self.ttagBuf.rawtags > firstTag,
self.ttagBuf.rawtags < lastTag))[0]
rawTags = self.ttagBuf.rawtags[rawPos]
rawChan = self.ttagBuf.rawchannels[rawPos]
rawAll = np.vstack((rawTags, rawChan))
newAll = np.sort(rawAll, axis=0)
newTags = rawAll[0]
newChan = rawAll[1]
# filter data to plot
selDelay = self.cmbChannels.currentIndex()
if selDelay > 8:
selTags = newTags
selChan = newChan
else:
if self.curTab == 0:
ch1 = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])
ch2 = np.array([3, 4, 5, 3, 4, 5, 3, 4, 5])
elif self.curTab == 1:
ch1 = np.array([0, 0, 0, 1, 1, 1, -1, -1, -1])
ch2 = np.array([2, 3, -1, 2, 3, -1, 2, 3, -1])
selPos = np.nonzero(
np.bitwise_or(newChan == ch1[selDelay],
newChan == ch2[selDelay]))[0]
selTags = newTags[selPos]
selChan = newChan[selPos]
# add delays to tags
selTags = selTags + np.around(
self.delay[selChan] / self.ttagBuf.resolution).astype(np.int64)
# compute delay histogram
delayEdges = np.arange(
-np.around(self.delayRange / self.ttagBuf.resolution).astype(
np.int32),
np.around(self.delayRange / self.ttagBuf.resolution).astype(
np.int32), 2)
selTags = selTags.astype(np.int64)
selChan = selChan.astype(np.int8)
if selDelay > 8:
if self.curTab == 0:
chMap = np.array([0, 0, 0, 1, 1, 1])
elif self.curTab == 1:
chMap = np.array([0, 0, 1, 1])
selChan = chMap[selChan]
t12diff = np.diff(selTags)
chdiff = np.diff(selChan)
t12diff = t12diff[chdiff != 0] * np.sign(chdiff[chdiff != 0])
count, bins = np.histogram(t12diff,
bins=delayEdges,
range=(np.amin(delayEdges),
np.amax(delayEdges)))
binCenter = (bins[1:] + bins[:-1]) / 2
delays = binCenter * self.ttagBuf.resolution * 1e9
bg = pg.BarGraphItem(x=delays, height=count, width=0.1, brush='b')
self.pltDelay.clear()
# fit results if the fit box is checked
if self.chkFit.isChecked():
gaussian = lambda x, A, x0, s: A * np.exp(-(x - x0)**2 /
(2 * s**2))
p0 = [np.max(count), delays[np.argmax(count)], 0.3]
popt, perr = curve_fit(gaussian, delays, count, p0=p0)
x = np.linspace(np.amin(delays), np.amax(delays), 1000)
yfit = gaussian(x, *popt)
self.pltDelay.plot(x, yfit, pen='r')
self.lblMean.setText("{:.4f}".format(popt[1]))
self.lblStd.setText("{:.4f}".format(popt[2]))
self.pltDelay.addItem(bg)
import numpy as np
print 'Binary equivalents of 13 and 17:'
a,b = 13,17
print bin(a), bin(b)
print np.bitwise_and(13, 17)
print np.bitwise_or(13, 17)
print np.left_shift(10,2)
#String functions
print np.char.add(['hello'],[' xyz'])
print np.char.add(['hello', 'hi'],[' abc', ' xyz'])
print np.char.multiply('Hello ',3)
print np.char.center('hello', 20,fillchar = '*')
print np.char.capitalize('hello world')
print np.char.title('hello how are you?')
print np.char.lower('HELLO')
print np.char.replace ('He is a good boy', 'is', 'was')
a = np.char.encode('hello', 'cp500')
#trignometroc functions
a = np.array([0,30,45,60,90])
c=np.sin(a*np.pi/180)
print np.sin(a*np.pi/180)
inv = np.arcsin(a)
print inv
print np.degrees(inv)
a = np.array([-1.7, 1.5, -0.2, 0.6, 10])
print np.floor(a)
def threshold_grad(grad, activeset, threshold=.5):
absgrad = np.max(np.abs(grad), axis=0)
activeset = np.bitwise_or(activeset, absgrad >= threshold*np.max(absgrad, axis=0))
grad[:, ~activeset] = 0
return (grad, activeset)
def create_combined(self, im_1_a, im_2_a):
"""
Short Summary
-------------
Create combined image from aligned input images. In the combined image:
The SCI pixel values are set by:
1. use pixels that are good (based on DQ) in image #1, else
2. for pixels that are bad in image #1, use good image #2 pixels, else
3. for pixels that are bad in both images, leave as default (0)
The DQ pixel values are similarly set, by:
1. use pixels that are good in image #1, else
2. for pixels that are bad in image #1, use good image #2 pixels, else
3. for pixels that are bad in both images, add a 'DO_NOT_USE' value to
the corresponding DQ value in image #1.
The ERR pixel values are set by:
1. use pixels that are good in image #1, else
2. for pixels that are bad in image #1, use good image #2 pixels, else
3. for pixels that are bad in both images, leave as default (0)
The WCS of the output model is set to the WCS of the 1st input
Parameters
----------
im_1_a: 2D Data Model
aligned image from input #1
im_2_a: 2D Data Model
aligned image from input #2
Returns
-------
data_comb: 2d float array
combined SCI array
dq_comb: 2d integer array
combined DQ array
err_comb: 2d float array
combined ERR array
wcs_1: pywcs.WCS object
wcs information for the 1st input
"""
sci_data_1 = im_1_a.data.copy()
sci_data_2 = im_2_a.data.copy()
dq_data_1 = im_1_a.dq.copy()
dq_data_2 = im_2_a.dq.copy()
err_data_1 = im_1_a.err.copy()
err_data_2 = im_2_a.err.copy()
wh_1_good = (dq_data_1 == 0) # Where ALIGNED #1 pixels are good
wh_1_bad_2_good = (dq_data_1 != 0) & (dq_data_2 == 0)
wh_1_bad_2_bad = (dq_data_1 != 0) & (dq_data_2 != 0)
# Populate combined SCI, DQ and ERR arrays
data_comb = sci_data_1 * 0 # Pixels that are bad in both will stay 0
data_comb[wh_1_good] = sci_data_1[wh_1_good]
data_comb[wh_1_bad_2_good] = sci_data_2[wh_1_bad_2_good]
dq_comb = dq_data_1.copy()
dq_comb[wh_1_bad_2_good] = dq_data_2[wh_1_bad_2_good]
dq_comb[wh_1_bad_2_bad] = np.bitwise_or(dqflags.group['DO_NOT_USE'],
dq_comb[wh_1_bad_2_bad])
err_comb = err_data_1.copy()
err_comb[wh_1_bad_2_good] = err_data_2[wh_1_bad_2_good]
err_comb[wh_1_bad_2_bad] = 0.
# Use wcs of input 1 for combination
wcs_1 = self.input_1.get_fits_wcs('SCI')
return data_comb, dq_comb, err_comb, wcs_1
def find_bad_pix(
filenames,
uncal_filenames=None,
jump_filenames=None,
fitopt_filenames=None,
clipping_sigma=5.,
max_clipping_iters=5,
noisy_threshold=5,
max_saturated_fraction=0.5,
max_jump_limit=10,
jump_ratio_threshold=5,
early_cutoff_fraction=0.25,
pedestal_sigma_threshold=5,
rc_fraction_threshold=0.8,
low_pedestal_fraction=0.8,
high_cr_fraction=0.8,
flag_values={
'hot': ['HOT'],
'rc': ['RC'],
'low_pedestal': ['OTHER_BAD_PIXEL'],
'high_cr': ["TELEGRAPH"]
},
do_not_use=['hot', 'rc', 'low_pedestal', 'high_cr'],
outfile=None,
plot=False):
"""MAIN FUNCTION
Parameters
----------
filenames : list
List of dark current slope files. These should be slope images.
uncal_filenames : list
List of uncal files. Should have a 1-to-1 correspondence to the
files in ``filenames``. If None, the scipt will look in the same
directory containing ``filenames``, and assume that the only
difference in filename is that rate.fits is replaced with
uncal.fits. Uncal files are only used when working with MIRI
data.
jump_filenames : list
List of exposures output from the jump step of the pipeline.
Should have a 1-to-1 correspondence to the
files in ``filenames``. If None, the scipt will look in the same
directory containing ``filenames``, and assume that the only
difference in filename is that rate.fits is replaced with
jump.fits
fitopt_filenames : list
List of exposures from the optional output from the ramp_fitting
step of the pipeline. Should have a 1-to-1 correspondence to the
files in ``filenames``. If None, the scipt will look in the same
directory containing ``filenames``, and assume that the only
difference in filename is that rate.fits is replaced with
fitopt.fits
clipping_sigma : int
Number of sigma to use when sigma-clipping the 2D array of
standard deviation values from the dark current slope files.
The sigma-clipped mean and standard deviation are used to locate
noisy pixels.
max_clipping_iters : int
Maximum number of iterations to use when sigma clipping to find
the mean and standard deviation values that are used when
locating noisy pixels.
noisy_threshold : int
Number of sigma above the mean noise (associated with the slope)
to use as a threshold for identifying noisy pixels.
max_saturated_fraction : float
When identifying pixels that are fully saturated (in all groups
of an integration), this is the fraction of integrations within
which a pixel must be fully saturated before flagging it as HOT
max_jump_limit : int
The maximum number of jumps a pixel can have in an integration
before it is flagged as a ``high jump`` pixel (which may be
flagged as noisy later)
jump_ratio_threshold : int
Cutoff for the ratio of jumps early in the ramp to jumps later in
the ramp. Pixels with a ratio greater than this value (and which
also have a high total number of jumps) will be flagged as
potential (I)RC pixels.
early_cutoff_fraction : float
Fraction of the integration to use when comparing the jump rate
early in the integration to that across the entire integration.
Must be <= 0.5
pedestal_sigma_threshold : int
Used when searching for RC pixels via the pedestal image. Pixels
with pedestal values more than ``pedestal_sigma_threshold`` above
the mean are flagged as potential RC pixels
rc_fraction_threshold : float
Used when searching for RC pixels. This is the fraction of input
files within which the pixel must be identified as an RC pixel
before it will be flagged as a permanent RC pixel
low_pedestal_fraction : float
This is the fraction of input files within which a pixel must be
identified as a low pedestal pixel before it will be flagged as
a permanent low pedestal pixel
high_cr_fraction : float
This is the fraction of input files within which a pixel must be
flagged as having a high number of jumps before it will be flagged
as permanently noisy
flag_values : dict
This dictionary maps the types of bad pixels searched for to the
flag mnemonics to use when creating the bad pixel file. Keys are
the types of bad pixels searched for, and values are lists that
include mnemonics recognized by the jwst calibration pipeline
e.g. {'hot': ['HOT'], 'rc': ['RC'], 'low_pedestal': ['OTHER_BAD_PIXEL'], 'high_cr': ["TELEGRAPH"]}
do_not_use : list
List of bad pixel types to be flagged as DO_NOT_USE
e.g. ['hot', 'rc', 'low_pedestal', 'high_cr']
plot : bool
If True, produce plots of intermediate results.
outfile : str
Name of fits file to save the resulting bad pixel mask to
"""
# Currently the code stipulates that 5 good values of the slope are
# needed in each pixel in order to determine a good stdev value. So
# let's check the number of input files here and quit if there are
# fewer than 5.
if len(filenames) < 5:
print(filenames)
raise ValueError(
"ERROR: >5 input files are required to find bad pixels from darks."
)
# Add DO_NOT_USE to all requested types of bad pixels
do_not_use = [element.lower() for element in do_not_use]
for key in flag_values:
if key.lower() in do_not_use:
flag_values[key].append('DO_NOT_USE')
# Form the outfile and outdir
if outfile is None:
outfile = 'badpixels_from_darks.fits'
outdir = os.path.dirname(outfile)
if not outdir:
outdir = '.'
# Read in the slope data. Strip off reference pixels.
# Return a 3D array of slopes and a 3D array mapping where the
# science pixels are.
print('Reading slope files...')
# instrument,slopes, refpix_additions = read_slope_files(filenames)
instrument, slopes, indexes, refpix_additions = read_slope_integrations(
filenames)
shape_slope = slopes.shape
# Calculate the mean and standard deviation through the stack for
# each pixel. Assuming that we are looking for noisy pixels, we don't
# want to do any sigma clipping on the inputs here, right?
mean_slope = np.mean(slopes, axis=0)
std_slope = np.std(slopes, axis=0)
hdout = fits.PrimaryHDU(mean_slope)
hdout.writeto('average_of_slopes.fits', overwrite=True)
hdout = fits.PrimaryHDU(std_slope)
hdout.writeto('sigma_of_slopes.fits', overwrite=True)
# Use sigma-cliping when calculating the mean and standard deviation
# of the standard deviations
clipped_stdevs, cliplow, cliphigh = sigma_clip(std_slope,
sigma=clipping_sigma,
maxiters=max_clipping_iters,
masked=False,
return_bounds=True)
avg_of_std = np.mean(clipped_stdevs)
std_of_std = np.std(clipped_stdevs)
cut_limit = avg_of_std + std_of_std * noisy_threshold
# Identify noisy pixels as those with noise values more than
# noisy_threshold*sigma above the average noise level
# noisy = std_slope > cut_limit # not a good stat we need to remove slopes with cr hits
# Plot histogram to later compare with better std_slope only containing
# slopes with no jumps detected.
if plot:
xhigh = avg_of_std + std_of_std * noisy_threshold
plot_image(std_slope, xhigh, outdir, "Pixel Standard devations",
"pixel_std_withjumps.png")
nbins = 5000
titleplot = 'Histogram of Pixel Slope STD with cosmic ray jumps: Clipped Ave ' + \
'{:6.4f}'.format(avg_of_std) + ' Std ' + '{:6.4f}'.format(std_of_std)
plot_histogram_stats(std_slope,
cut_limit,
nbins,
outdir,
titleplot,
"histo_std_withjumps.png",
xaxis_log=True)
# Read in the optional outputs from the ramp-fitting step, so that
# we can look at the y-intercepts and the jump flags
saturated = np.zeros(slopes.shape)
rc_from_pedestal = np.zeros(slopes.shape)
low_pedestal = np.zeros(slopes.shape)
high_cr_rate = np.zeros(slopes.shape)
rc_from_flags = np.zeros(slopes.shape)
slope_stack = []
islope_stack = []
total_ints = 0
counter = 0
for i, filename in enumerate(filenames):
# Read in the ramp and get the data and dq arrays
jump_file = None
if jump_filenames is not None:
jump_file = jump_filenames[i]
else:
for suffix in RATE_FILE_SUFFIXES:
if suffix in filename:
slope_suffix = '{}.fits'.format(suffix)
jump_file = filename.replace(slope_suffix, '_jump.fits')
break
if jump_file is None:
raise ValueError("ERROR: Unrecognized slope filename suffix.")
if not os.path.isfile(jump_file):
raise FileNotFoundError(
"ERROR: Jump file {} not found.".format(jump_file))
print('Opening Jump File {}'.format(jump_file))
groupdq = dq_flags.get_groupdq(jump_file, refpix_additions)
cr_map = dq_flags.flag_map(groupdq, 'JUMP_DET')
# Get slope data corresponding to this file by extracting the
# appropriate frames from the ``slopes`` stack
slope = slopes[indexes[i]:indexes[i + 1], :, :]
# Read in the fitops file associated with the exposure and get
# the pedestal array (y-intercept)
if fitopt_filenames is not None:
pedestal_file = fitopt_filenames[i]
else:
pedestal_file = filename.replace(slope_suffix, '_fitopt.fits')
if not os.path.isfile(pedestal_file):
raise FileNotFoundError(
"ERROR: Pedestal file {} not found.".format(pedestal_file))
print('Opening Pedestal File {}'.format(pedestal_file))
pedestal = read_pedestal_data(pedestal_file, refpix_additions)
# for MIRI the zero point of the ramp drifts with time. Adjust the
# pedestal to be a relative pedestal wrt to group 2
if instrument == 'MIRI':
if uncal_filenames is not None:
uncal_file = uncal_filenames[i]
else:
uncal_file = filename.replace(slope_suffix, '_uncal.fits')
if not os.path.isfile(uncal_file):
raise FileNotFoundError(
"ERROR: Uncal file {} not found.".format(uncal_file))
group2 = extract_group2(uncal_file, refpix_additions)
pedestal_org = copy.deepcopy(pedestal)
pedestal = np.fabs(group2 - pedestal)
# Work one integration at a time
for int_num in range(pedestal.shape[0]):
# pull out the DQ of the first group. This will be use to remove
# Low pedestal values that have a pedestal of 0 because they are
# saturated on group 1.
first_group = groupdq[int_num, 0, :, :]
pedestal_int = pedestal[int_num, :, :]
slope_int = slope[int_num, :, :]
clipped_pedestal, cliplow, cliphigh = sigmaclip(pedestal_int,
low=3.,
high=3.)
mean_pedestal = np.mean(clipped_pedestal)
std_pedestal = np.std(clipped_pedestal)
rc_from_pedestal[counter, :, :] += pedestal_int > (
mean_pedestal + std_pedestal * pedestal_sigma_threshold)
# Pixels with abnormally low pedestal values
pedestal_low = pedestal_int < (
mean_pedestal - std_pedestal * pedestal_sigma_threshold)
first_group_sat = np.bitwise_and(first_group,
dqflags.pixel['SATURATED'])
# do not allow pixels saturated on group 1 to be marked as low pedestal
pedestal_results = np.logical_and(pedestal_low,
(first_group_sat == 0))
low_pedestal[counter, :, :] += pedestal_results
# Find pixels that are saturated in all groups. These will have
# a pedestal value of 0 (according to the pipeline documentation).
# These should end up flagged as HOT and DO_NOT_USE
# Remove all the cases where ped = 0, but group 1 is not saturated
# This can be dead pixels
if instrument == 'MIRI':
pedestal_int = pedestal_org[int_num, :, :]
saturated[counter, :, :] += saturated_in_all_groups(
pedestal_int, first_group_sat)
# Find pixels that have an abnormally high number of jumps, as
# well as those that have most of their jumps concentrated in the
# early part of the integration. The latter are possibly RC or IRC
# pixels
many_jumps, rc_candidates, number_of_jumps =\
find_pix_with_many_jumps(cr_map[int_num, :, :, :], max_jump_limit=10,
jump_ratio_threshold=5,
early_cutoff_fraction=0.25)
high_cr_rate[counter, :, :] += many_jumps
rc_from_flags[counter, :, :] += rc_candidates
# using the number_of_jumps (a per integration value) create a clean set of
# pixel slopes with no cosmic rays
clean_slopes, iclean_slopes = slopes_not_cr(
slope_int, number_of_jumps)
slope_stack.append(clean_slopes)
islope_stack.append(iclean_slopes)
total_ints += 1
counter += 1
# now find the mean and standard deviation of the "clean" pixel slopes
clean_mean_slope, clean_std_slope, num_good = combine_clean_slopes(
slope_stack, islope_stack)
hdout = fits.PrimaryHDU(clean_mean_slope)
hdout.writeto('average_of_slopes_nojumps.fits', overwrite=True)
hdout = fits.PrimaryHDU(clean_std_slope)
hdout.writeto('sigma_of_slopes_nojumps.fits', overwrite=True)
num_good_slopes = num_good.astype(np.int16)
hdout = fits.PrimaryHDU(num_good_slopes)
hdout.writeto('number_of_slopes_nojumps.fits', overwrite=True)
# Use sigma-cliping to remove large outliers to have clean stats to flag
# noisy pixels.
# removing nans from clean_std_slope because it causes warning messages to be print
clean_std_slope_nonan = clean_std_slope[np.isfinite(clean_std_slope)]
clipped_stdevs, cliplow, cliphigh = sigma_clip(clean_std_slope_nonan,
sigma=clipping_sigma,
maxiters=max_clipping_iters,
masked=False,
return_bounds=True)
avg_of_std = np.mean(clipped_stdevs)
std_of_std = np.std(clipped_stdevs)
cut_limit = avg_of_std + std_of_std * noisy_threshold
# assigning nans from clean_std_slope to very large values that will be cut
# because it causes warning messages to be print
values_nan = np.isnan(clean_std_slope)
clean_std_slope[values_nan] = avg_of_std + std_of_std * 50
noisy = clean_std_slope > cut_limit
num_noisy = len(np.where(noisy)[0])
if plot:
# plot the number of good slopes per pixel
max_values = np.amax(num_good)
plot_image(num_good, max_values, outdir,
"Number of Good slopes/pixel ", "clean_pixel_number.png")
# plot the standard deviation of pixels slope after eliminating
# values having jumps detectect in ramp
xhigh = avg_of_std + std_of_std
plot_image(clean_std_slope, xhigh, outdir,
"Clean Pixel Standard devations", "clean_pixel_std.png")
# plot the histogram before the clipping
nbins = 5000
titleplot = 'Histogram of Clean Pixel Slope STD Average ' + \
'{:6.4f}'.format(avg_of_std) + ' Std ' + '{:6.4f}'.format(std_of_std)
plot_histogram_stats(clean_std_slope,
cut_limit,
nbins,
outdir,
titleplot,
"histo_clean_std.png",
xaxis_log=True)
# Look through the stack of saturated pixels and keep those saturated
# more than N% of the time
fully_saturated = np.sum(saturated, axis=0) / total_ints
fully_saturated[fully_saturated < max_saturated_fraction] = 0
fully_saturated = np.ceil(fully_saturated).astype(int)
fully_saturated = apply_flags(fully_saturated, flag_values['hot'])
num_saturated = len(np.where(fully_saturated != 0)[0])
print('\n\nFound {} fully saturated pixels.'.format(num_saturated))
# How do we want to combine these to identify RC pixels?
rc_pedestal = np.sum(rc_from_pedestal, axis=0) / total_ints
rc_flags = np.sum(rc_from_flags, axis=0) / total_ints
rc_from_pedestal_only = (rc_pedestal > rc_fraction_threshold).astype(int)
rc_from_jumps_only = (rc_flags > rc_fraction_threshold).astype(int)
num_rc_ped = len(np.where(rc_from_pedestal_only != 0)[0])
num_rc_jump = len(np.where(rc_from_jumps_only != 0)[0])
print("Found {} RC pixels from pedestal search".format(num_rc_ped))
print("Found {} RC pixels from Jump search".format(num_rc_jump))
rc = ((rc_pedestal > rc_fraction_threshold) |
(rc_flags > rc_fraction_threshold))
rc = apply_flags(rc.astype(int), flag_values['rc'])
num_rc = len(np.where(rc != 0)[0])
print('Found {} RC pixels.'.format(num_rc))
# Low pedestal pixels
low_pedestal_vals = np.sum(low_pedestal, axis=0) / total_ints
low_ped = low_pedestal_vals > low_pedestal_fraction
# Pixels that are saturated on the first group will have a PEDESTAL value
# of 0. Pull these out of this set (these are hot pixels)
low_ped = apply_flags(low_ped.astype(int), flag_values['low_pedestal'])
num_low_ped = len(np.where(low_ped != 0)[0])
print('Found {} low pedestal pixels.'.format(num_low_ped))
# Pixels with lots of CR flags should be added to the list of noisy pixels?
high_cr = np.sum(high_cr_rate, axis=0) / total_ints
noisy_second_pass = high_cr > high_cr_fraction
combined_noisy = np.bitwise_or(noisy, noisy_second_pass)
combined_noisy = apply_flags(combined_noisy.astype(int),
flag_values['high_cr'])
num_high_cr = len(np.where(noisy_second_pass != 0)[0])
print('Found {} pixels with a high number of jumps.'.format(num_high_cr))
print('Found {} pixels with noise above the threshold.'.format(num_noisy))
num_combined_noisy = len(np.where(combined_noisy != 0)[0])
print(
'Combining noisy and high jump pixels, found {} noisy pixels.'.format(
num_combined_noisy))
# Combine the various flavors of bad pixels into a final DQ map
bad_pixels = combine_bad_pixel_types(fully_saturated, rc, low_ped,
combined_noisy)
# Add the reference pixels back into the bad pixel map
bad_pixels = add_refpix(bad_pixels, refpix_additions)
# Create DQ definitions to be saved with the output file
dq_def = create_dqdef()
# Save the bad pixel mask to a fits file
# Eventually this routine will be called as part of the dark current reference file
# generator, and the bad pixel mask will be saved in the DQ extension of the
# reference file
h0 = fits.PrimaryHDU(fully_saturated)
h0.header['EXTNAME'] = 'SATURATED'
h1a = fits.ImageHDU(rc_from_pedestal_only)
h1a.header['EXTNAME'] = 'RC_FROM_PED'
h1b = fits.ImageHDU(rc_from_jumps_only)
h1b.header['EXTNAME'] = 'RC_FROM_JUMPS'
h1 = fits.ImageHDU(rc)
h1.header['EXTNAME'] = 'RC'
h2 = fits.ImageHDU(low_ped)
h2.header['EXTNAME'] = 'LOW_PEDESTAL'
h3 = fits.ImageHDU(noisy.astype(int))
h3.header['EXTNAME'] = 'NOISY'
h4 = fits.ImageHDU(noisy_second_pass.astype(int))
h4.header['EXTNAME'] = 'MANY_CRS'
h5 = fits.ImageHDU(combined_noisy)
h5.header['EXTNAME'] = 'NOISY_AND_CRS'
hlist = fits.HDUList([h0, h1a, h1b, h1, h2, h3, h4, h5])
hlist.writeto(outfile, overwrite=True)
print('Multi-extension file with individual types of bad pixels saved to:')
print(outfile)
return bad_pixels
continue
img[y][x] = 255
q.put((x + 1, y))
q.put((x - 1, y))
q.put((x, y + 1))
q.put((x, y - 1))
# check if circle was complete
if not IsBounded:
img[:, :] = 0
return
cv2.namedWindow('Window')
cv2.setMouseCallback('Window', interactive_drawing)
while True:
cv2.imshow('Window', np.bitwise_or(img, mask_img))
k = cv2.waitKey(1) & 0xFF
if k == ord('f'):
fill = True
elif k == ord('d'):
fill = False
elif k == ord('q'):
break
elif k == ord('r'):
mask_img[:, :] = 0
elif k == ord('s'):
cv2.imshow('Mask', mask_img)
cv2.imwrite("/Users/btse/Desktop/Result.png", mask_img)
cv2.destroyAllWindows()
def flag_cr(sci_image, blot_image, **pars):
"""Masks outliers in science image.
Mask blemishes in dithered data by comparing a science image
with a model image and the derivative of the model image.
Parameters
----------
sci_image : ImageModel
the science data
blot_image : ImageModel
the blotted median image of the dithered science frames
pars : dict
the user parameters for Outlier Detection
Default parameters:
grow = 1 # Radius to mask [default=1 for 3x3]
ctegrow = 0 # Length of CTE correction to be applied
snr = "5.0 4.0" # Signal-to-noise ratio
scale = "1.2 0.7" # scaling factor applied to the derivative
backg = 0 # Background value
"""
grow = pars.get('grow', 1)
ctegrow = pars.get('ctegrow', 0) # not provided by outlierpars
backg = pars.get('backg', 0)
snr1, snr2 = [float(val) for val in pars.get('snr', '5.0 4.0').split()]
scl1, scl2 = [float(val) for val in pars.get('scale', '1.2 0.7').split()]
if not sci_image.meta.background.subtracted:
# Include background back into blotted image for comparison
subtracted_background = sci_image.meta.background.level
log.debug("Subtracted background: {}".format(subtracted_background))
if subtracted_background is None:
subtracted_background = backg
exptime = sci_image.meta.exposure.exposure_time
sci_data = sci_image.data * exptime
blot_data = blot_image.data * exptime
blot_deriv = abs_deriv(blot_data)
err_data = np.nan_to_num(sci_image.err)
# Define output cosmic ray mask to populate
cr_mask = np.zeros(sci_image.shape, dtype=np.uint8)
#
#
# COMPUTATION PART I
#
#
# Model the noise and create a CR mask
diff_noise = np.abs(sci_data - blot_data)
# ta = np.sqrt(np.abs(blot_data + subtracted_background) + rn ** 2)
ta = np.sqrt(np.abs(blot_data + subtracted_background) + err_data**2)
t2 = scl1 * blot_deriv + snr1 * ta
tmp1 = np.logical_not(np.greater(diff_noise, t2))
# Convolve mask with 3x3 kernel
kernel = np.ones((3, 3), dtype=np.uint8)
tmp2 = np.zeros(tmp1.shape, dtype=np.int32)
ndimage.convolve(tmp1, kernel, output=tmp2, mode='nearest', cval=0)
#
#
# COMPUTATION PART II
#
#
# Create a second CR Mask
xt2 = scl2 * blot_deriv + snr2 * ta
np.logical_not(np.greater(diff_noise, xt2) & np.less(tmp2, 9), cr_mask)
#
#
# COMPUTATION PART III
#
#
# Flag additional cte 'radial' and 'tail' pixels surrounding CR
# pixels as CRs
# In both the 'radial' and 'length' kernels below, 0=good and
# 1=bad, so that upon convolving the kernels with cr_mask, the
# convolution output will have low->bad and high->good from which
# 2 new arrays are created having 0->bad and 1->good. These 2 new
# arrays are then AND'ed to create a new cr_mask.
# recast cr_mask to int for manipulations below; will recast to
# Bool at end
cr_mask_orig_bool = cr_mask.copy()
cr_mask = cr_mask_orig_bool.astype(np.int8)
# make radial convolution kernel and convolve it with original cr_mask
cr_grow_kernel = np.ones((grow, grow))
cr_grow_kernel_conv = cr_mask.copy()
ndimage.convolve(cr_mask, cr_grow_kernel, output=cr_grow_kernel_conv)
# make tail convolution kernel and (shortly) convolve it with
# original cr_mask
cr_ctegrow_kernel = np.zeros((2 * ctegrow + 1, 2 * ctegrow + 1))
cr_ctegrow_kernel_conv = cr_mask.copy()
# which pixels are masked by tail kernel depends on readout direction
# We could put useful info in here for CTE masking if needed. Code
# remains below. For now, we set to zero, which turns off CTE masking.
ctedir = 0
if (ctedir == 1):
cr_ctegrow_kernel[0:ctegrow, ctegrow] = 1
if (ctedir == -1):
cr_ctegrow_kernel[ctegrow + 1:2 * ctegrow + 1, ctegrow] = 1
if (ctedir == 0):
pass
# finally do the tail convolution
ndimage.convolve(cr_mask, cr_ctegrow_kernel, output=cr_ctegrow_kernel_conv)
# select high pixels from both convolution outputs; then 'and' them to
# create new cr_mask
where_cr_grow_kernel_conv = np.where(cr_grow_kernel_conv < grow * grow, 0,
1)
where_cr_ctegrow_kernel_conv = np.where(cr_ctegrow_kernel_conv < ctegrow,
0, 1)
# combine masks and cast back to Bool
np.logical_and(where_cr_ctegrow_kernel_conv, where_cr_grow_kernel_conv,
cr_mask)
cr_mask = cr_mask.astype(bool)
count_sci = np.count_nonzero(sci_image.dq)
count_cr = np.count_nonzero(cr_mask)
log.debug("Pixels in input DQ: {}".format(count_sci))
log.debug("Pixels in cr_mask: {}".format(count_cr))
# Update the DQ array in the input image in place
np.bitwise_or(sci_image.dq, np.invert(cr_mask) * CRBIT, sci_image.dq)
def find_table(img, o_img_height, o_img_width):
## find white border
DoB = zc.get_DoB(img, 1, 31, method = 'Average')
mask_white = zc.color_inrange(DoB, 'HSV', V_L = 10)
## find purple table (roughly)
mask_table = zc.color_inrange(img, 'HSV', H_L = 130, H_U = 160, S_L = 50, V_L = 50, V_U = 220)
mask_table, _ = zc.get_big_blobs(mask_table, min_area = 50)
mask_table = cv2.morphologyEx(mask_table, cv2.MORPH_CLOSE, zc.generate_kernel(7, 'circular'), iterations = 1)
mask_table, _ = zc.find_largest_CC(mask_table)
if mask_table is None:
rtn_msg = {'status': 'fail', 'message' : 'Cannot find table'}
return (rtn_msg, None)
mask_table_convex, _ = zc.make_convex(mask_table.copy(), app_ratio = 0.005)
mask_table = np.bitwise_or(mask_table, mask_table_convex)
mask_table_raw = mask_table.copy()
## fine tune the purple table based on white border
mask_white = np.bitwise_and(np.bitwise_not(mask_table), mask_white)
for i in range(15):
mask_table = zc.expand(mask_table, 3)
mask_table = np.bitwise_and(np.bitwise_not(mask_white), mask_table)
if i % 4 == 3:
mask_table, _ = zc.make_convex(mask_table, app_ratio = 0.01)
mask_table = np.bitwise_and(np.bitwise_not(mask_white), mask_table)
mask_table, _ = zc.find_largest_CC(mask_table)
mask_table, hull_table = zc.make_convex(mask_table, app_ratio = 0.01)
## check if table is big enough
table_area = cv2.contourArea(hull_table)
table_area_percentage = float(table_area) / img.shape[0] / img.shape[1]
if table_area_percentage < 0.06:
rtn_msg = {'status' : 'fail', 'message' : "Detected table too small: %f" % table_area_percentage}
return (rtn_msg, None)
## find top line of table
hull_table = np.array(zc.sort_pts(hull_table[:,0,:], order_first = 'y'))
ul = hull_table[0]
ur = hull_table[1]
if ul[0] > ur[0]:
t = ul; ul = ur; ur = t
i = 2
# the top two points in the hull are probably on the top line, but may not be the corners
while i < hull_table.shape[0] and hull_table[i, 1] - hull_table[0, 1] < 80:
pt_tmp = hull_table[i]
if pt_tmp[0] < ul[0] or pt_tmp[0] > ur[0]:
# computing the area of the part of triangle that lies inside the table
triangle = np.vstack([pt_tmp, ul, ur]).astype(np.int32)
mask_triangle = np.zeros_like(mask_table)
cv2.drawContours(mask_triangle, [triangle], 0, 255, -1)
pts = mask_table_raw[mask_triangle.astype(bool)]
if np.sum(pts == 255) > 10:
break
if pt_tmp[0] < ul[0]:
ul = pt_tmp
else:
ur = pt_tmp
i += 1
else:
break
ul = [int(x) for x in ul]
ur = [int(x) for x in ur]
## sanity checks about table top line detection
if zc.euc_dist(ul, ur) ** 2 * 2.5 < table_area:
rtn_msg = {'status' : 'fail', 'message' : "Table top line too short"}
return (rtn_msg, None)
if abs(zc.line_angle(ul, ur)) > 0.4:
rtn_msg = {'status' : 'fail', 'message' : "Table top line tilted too much"}
return (rtn_msg, None)
# check if two table sides form a reasonable angle
mask_table_bottom = mask_table.copy()
mask_table_bottom[:-30] = 0
p_left_most = zc.get_edge_point(mask_table_bottom, (-1, 0))
p_right_most = zc.get_edge_point(mask_table_bottom, (1, 0))
if p_left_most is None or p_right_most is None:
rtn_msg = {'status' : 'fail', 'message' : "Table doesn't occupy bottom part of image"}
return (rtn_msg, None)
left_side_angle = zc.line_angle(ul, p_left_most)
right_side_angle = zc.line_angle(ur, p_right_most)
angle_diff = zc.angle_dist(left_side_angle, right_side_angle, angle_range = math.pi * 2)
if abs(angle_diff) > 1.8:
rtn_msg = {'status' : 'fail', 'message' : "Angle between two side edge not right"}
return (rtn_msg, None)
## rotate to make opponent upright, use table edge as reference
pts1 = np.float32([ul,ur,[ul[0] + (ur[1] - ul[1]), ul[1] - (ur[0] - ul[0])]])
pts2 = np.float32([[0, o_img_height], [o_img_width, o_img_height], [0, 0]])
M = cv2.getAffineTransform(pts1, pts2)
img[np.bitwise_not(zc.get_mask(img, rtn_type = "bool", th = 3)), :] = [3,3,3]
img_rotated = cv2.warpAffine(img, M, (o_img_width, o_img_height))
## sanity checks about rotated opponent image
bool_img_rotated_valid = zc.get_mask(img_rotated, rtn_type = "bool")
if float(bool_img_rotated_valid.sum()) / o_img_width / o_img_height < 0.7:
rtn_msg = {'status' : 'fail', 'message' : "Valid area too small after rotation"}
return (rtn_msg, None)
rtn_msg = {'status' : 'success'}
return (rtn_msg, (img_rotated, mask_table, M))
def getJaccardError(img1, img2):
img_intersection = np.sum(np.bitwise_and(img1, img2))
img_union = np.sum(np.bitwise_or(img1, img2))
jacc_error = float(img_intersection) / float(img_union)
return jacc_error
def bitwiseOrData(data1, data2):
return numpy.bitwise_or(data1, data2)
def time_char_or(self):
np.bitwise_or(self.c, 0, out=self.c)
np.bitwise_or(0, self.c, out=self.c)
def _activity(run,
baseline_activity=0.,
baseline_sigma=3.0,
trace_type='deconvolved'):
"""
Get the activity levels for the temporal classifier.
Parameters
----------
run : Run
baseline_activity : float
Scale factor of baseline activity.
'temporal-prior-baseline-activity' in classifier parameters.
baseline_sigma : float
Scale factor of baseline variance.
'temporal-prior-baseline-sigma' in classifier parameters.
trace_type : {'deconvolved'}
This only works on deconvolved data for now.
Returns
-------
baseline activity, variance of activity, outliers
"""
if trace_type != 'deconvolved':
raise ValueError(
'Temporal classifier only implemented for deconvolved data.')
if run.run_type == 'spontaneous' and 'sated' in run.tags:
runs = run.parent.runs(run_types=['spontaneous'], tags=['sated'])
spontaneous = True
elif run.run_type == 'spontaneous' and 'hungry' in run.tags:
runs = run.parent.runs(run_types=['spontaneous'], tags=['hungry'])
spontaneous = True
elif run.run_type == 'training':
runs = run.parent.runs(run_types=['training'])
spontaneous = False
else:
raise ValueError(
'Unknown run_type and tags, not sure how to calculate activity.')
baseline, variance, outliers = None, None, None
if spontaneous:
popact, outliers = [], []
for r in runs:
t2p = r.trace2p()
pact = t2p.trace('deconvolved')
fmin = t2p.lastonset()
mask = t2p.inactivity()
mask[:fmin] = False
if len(popact):
popact = np.concatenate([popact, pact[:, mask]], axis=1)
else:
popact = pact[:, mask]
trs = t2p.trace('deconvolved')[:, fmin:]
cellact = np.nanmean(trs, axis=1)
outs = cellact > np.nanmedian(cellact) + 2 * np.std(cellact)
if len(outliers) == 0:
outliers = outs
else:
outliers = np.bitwise_or(outliers, outs)
if len(popact):
popact = np.nanmean(popact[np.invert(outliers), :], axis=0)
baseline = np.median(popact)
variance = np.std(popact)
outliers = outliers
else:
popact = []
for r in runs:
t2p = r.trace2p()
ncells = t2p.ncells
pact = np.nanmean(t2p.trace('deconvolved'), axis=0)
skipframes = int(t2p.framerate * 4)
for cs in ['plus*', 'neutral*', 'minus*', 'pavlovian*']:
onsets = t2p.csonsets(cs)
for ons in onsets:
pact[ons:ons + skipframes] = np.nan
popact = np.concatenate([popact, pact[np.isfinite(pact)]])
if len(popact):
# baseline = np.median(popact)
# Exclude extremes
percent = 2.0
popact = np.sort(popact)
trim = int(percent * popact.size / 100.)
popact = popact[trim:-trim]
baseline = np.median(
popact) # Moved to after extreme exclusion on 190326
variance = np.std(popact)
outliers = np.zeros(ncells, dtype=bool)
if baseline is None:
baseline, variance = 0.01, 0.08 * baseline_sigma
else:
baseline *= baseline_activity
variance *= baseline_sigma
return baseline, variance, outliers
def do_correction(input_model, ref_model):
"""
Short Summary
-------------
Execute all tasks for saturation, including using a saturation reference
file.
Parameters
----------
input_model: data model object
The input science data to be corrected
ref_model: data model object
Saturation reference file mode object
Returns
-------
output_model: data model object
object having GROUPDQ array saturation flags set
"""
ramparr = input_model.data
# Was IRS2 readout used?
is_irs2_format = x_irs2.is_irs2(input_model)
if is_irs2_format:
irs2_mask = x_irs2.make_mask(input_model)
# Create the output model as a copy of the input
output_model = input_model.copy()
groupdq = output_model.groupdq
# Extract subarray from reference file, if necessary
if reffile_utils.ref_matches_sci(input_model, ref_model):
satmask = ref_model.data
dqmask = ref_model.dq
else:
log.info('Extracting reference file subarray to match science data')
ref_sub_model = reffile_utils.get_subarray_model(
input_model, ref_model)
satmask = ref_sub_model.data.copy()
dqmask = ref_sub_model.dq.copy()
ref_sub_model.close()
# For pixels flagged in reference file as NO_SAT_CHECK, set the dq mask
# and saturation mask
wh_sat = np.bitwise_and(dqmask, dqflags.pixel['NO_SAT_CHECK'])
dqmask[wh_sat ==
dqflags.pixel['NO_SAT_CHECK']] = dqflags.pixel['NO_SAT_CHECK']
satmask[wh_sat == dqflags.pixel['NO_SAT_CHECK']] = HUGE_NUM
# Correct saturation values for NaNs in the ref file
correct_for_NaN(satmask, dqmask)
dq_flag = dqflags.group['SATURATED']
nints = ramparr.shape[0]
ngroups = ramparr.shape[1]
detector = input_model.meta.instrument.detector
flagarray = np.zeros(ramparr.shape[-2:], dtype=groupdq.dtype)
for ints in range(nints):
for plane in range(ngroups):
# Update the 4D groupdq array with the saturation flag. The
# flag is set in the current plane and all following planes.
if is_irs2_format:
sci_temp = x_irs2.from_irs2(ramparr[ints, plane, :, :],
irs2_mask, detector)
flag_temp = np.where(sci_temp >= satmask, dq_flag, 0)
# Copy flag_temp into flagarray.
x_irs2.to_irs2(flagarray, flag_temp, irs2_mask, detector)
else:
flagarray[:, :] = np.where(
ramparr[ints, plane, :, :] >= satmask, dq_flag, 0)
np.bitwise_or(groupdq[ints, plane:, :, :], flagarray,
groupdq[ints, plane:, :, :])
output_model.groupdq = groupdq
if is_irs2_format:
pixeldq_temp = x_irs2.from_irs2(output_model.pixeldq, irs2_mask,
detector)
pixeldq_temp = np.bitwise_or(pixeldq_temp, dqmask)
x_irs2.to_irs2(output_model.pixeldq, pixeldq_temp, irs2_mask, detector)
else:
output_model.pixeldq = np.bitwise_or(output_model.pixeldq, dqmask)
return output_model
def temporal_prior(run,
pars,
nan_cells=None,
replace_data=None,
replace_temporal_prior=None):
"""Given a run and trained model, classify reactivations.
Parameters
----------
run : Run
Run that the classifier will be applied to.
model : aode.AODE
A fully-trained model.
pars : dict
Parameters for classifier.
nan_cells : np.ndarray of bool
Cell mask of cells to exclude from temporal prior. Activity outliers
are also automatically removed.
merge_cses : optional, list
List of class names. Merges the results of later values into the first
one.
replace_data : matrix (ncells, ntimes)
Replacement data if not None
replace_priors : dict of floats
The prior probabilities for each category to replace that in parameters
replace_temporal_prior : vector
A replacement temporal prior to generate the frame priors
replace_integrate_frames : int
Can change the frame integration number without changing the parameters.
This is useful if the data are already passed through a rolling max filter.
Returns
-------
dict
Results dict.
"""
if replace_temporal_prior is not None:
return replace_temporal_prior
if replace_data is not None:
full_traces = replace_data
else:
full_traces = run.trace2p().trace(pars['trace-type'])
if nan_cells is None:
nan_cells = np.any(np.invert(np.isfinite(full_traces)), axis=1)
if pars['remove-stim'] and replace_data is None:
t2p = run.trace2p()
pre_pad_s = pars['classification-ms'] / 1000. / 2.
post_pad_s = 0.0 + pre_pad_s
pav_post_pad_s = 0.5 + pre_pad_s
stim_mask = t2p.stim_mask(pre_pad_s=pre_pad_s,
post_pad_s=post_pad_s,
pav_post_pad_s=pav_post_pad_s)
else:
stim_mask = None
if pars['temporal-dependent-priors']:
if 'temporal-prior-fwhm-frames' not in pars:
t2p = run.trace2p()
pars = _convert_time_to_frames(deepcopy(pars), t2p.framerate)
baseline, variance, outliers = _activity(
run,
baseline_activity=pars['temporal-prior-baseline-activity'],
baseline_sigma=pars['temporal-prior-baseline-sigma'],
trace_type=pars['trace-type'])
# Make sure cells with any NaN's and outliers are removed from the
# temporal prior
outliers = np.bitwise_or(outliers, nan_cells)
tpriorvec = aode.temporal_prior(
full_traces[np.invert(outliers), :],
actmn=baseline,
actvar=variance,
fwhm=pars['temporal-prior-fwhm-frames'],
stim_mask=stim_mask)
else:
# NOTE: this hasn't really been tested
# Default to equal probability
tpriorvec = np.ones(full_traces.shape[1])
return tpriorvec
def find_overlapping_regions(true_regions, test_regions):
true_boxes = []
true_classes = []
test_boxes = []
test_classes = []
test_scores = []
img_dims = true_regions['hsv_img'].shape[:2]
for r in true_regions['regions']:
true_boxes.append(compute_bbox(r['points']))
true_classes.append(r['label'])
for r in test_regions:
test_boxes.append(compute_bbox(r['contour']))
max_prob = max(r['prob'].items(), key=itemgetter(1))
test_classes.append(max_prob[0])
test_scores.append(max_prob[1])
# now we are ready to find the overlaps, we'll keep track of them with a dictionary
# where the keys are the true region's index. The values will be a dictionary of
# overlapping test regions, organized into 2 groups:
# - matching overlaps: overlaps where the test & true region labels agree
# - non-matching overlaps: overlaps where the test & true region labels differ
#
# Each one of those groups will be keys with a value of another list of dictionaries,
# with the overlapping test region index along with the IoU value.
# There are 2 other cases to cover:
# - true regions with no matching overlaps (i.e. missed regions)
# - test regions with no matching overlaps (i.e. false positives)
overlaps = {}
true_match_set = set()
test_match_set = set()
for i, r1 in enumerate(true_boxes):
true_mask = None # reset to None, will compute as needed
for j, r2 in enumerate(test_boxes):
if not do_boxes_overlap(r1, r2):
continue
# So you're saying there's a chance?
# If we get here, there is a chance for an overlap but it is not guaranteed,
# we'll need to check the contours' pixels
if true_mask is None:
# we've never tested against this contour yet, so render it
true_mask = make_boolean_mask(
true_regions['regions'][i]['points'], img_dims)
# and render the test contour
test_mask = make_boolean_mask(test_regions[j]['contour'], img_dims)
intersect_mask = np.bitwise_and(true_mask, test_mask)
intersect_area = intersect_mask.sum()
if not intersect_area > 0:
# the bounding boxes overlapped, but the contours didn't, skip it
continue
union_mask = np.bitwise_or(true_mask, test_mask)
true_match_set.add(i)
test_match_set.add(j)
if i not in overlaps:
overlaps[i] = {
'true_label': true_classes[i],
'true': [],
'false': []
}
test_result = {
'test_index': j,
'iou': intersect_area / union_mask.sum()
}
if true_classes[i] == test_classes[j]:
overlaps[i]['true'].append(test_result)
else:
overlaps[i]['false'].append(test_result)
missed_regions = true_match_set.symmetric_difference(
(range(0, len(true_boxes))))
false_positives = test_match_set.symmetric_difference(
(range(0, len(test_boxes))))
return {
'overlaps': overlaps,
'missed_regions': missed_regions,
'false_positives': false_positives
}
def apply_filter(self, entry: DetectionAnnotation, is_empty):
return np.where(np.bitwise_or(entry.x_maxs - entry.x_mins <= 0, entry.y_maxs - entry.y_mins <= 0))[0]
def tanimoto(x, y):
x = np.asarray(x, np.bool)
y = np.asarray(y, np.bool)
return -np.log2(
np.double(np.bitwise_and(x, y).sum()) /
np.double(np.bitwise_or(x, y).sum()))
def jaccard(x, y):
x = np.asarray(x, np.bool)
y = np.asarray(y, np.bool)
return 1 - np.double(np.bitwise_and(x, y).sum()) / np.double(
np.bitwise_or(x, y).sum())
def clock_tec_solveMH(obs_phase,
freqs,
times,
m0,
cov,
Cd_error,
Ct_ratio,
plot=False):
'''Solves for the terms phase(CS,TEC,delay) = CS + e^2/(4pi ep0 me c) * TEC/nu + 2pi*nu*delay
Assumes phase is in units of radians, freqs in is units of Hz,
and times is in units of seconds with arbitrary offset
obs_phase is shape (num_freqs, num_times)'''
binning = 50
convergence = binning**2 * 3
def calc_phase(m, freqs):
phase = np.multiply.outer(np.ones(
len(freqs)), m[:, 2]) + 8.44797256e-7 * TECU * np.multiply.outer(
1. / freqs, m[:, 1]) + 2. * np.pi * np.multiply.outer(
freqs, m[:, 0])
return phase
def neglogL(obs_phase, phase, CdCt):
L2 = obs_phase - phase
L2 *= L2
L2 /= (CdCt + 1e-15)
return np.sum(L2, axis=0) / 2.
def sample_prior(last, cov):
"""Last is tau,tec,cs in matrix of size [len(times),3], return similar shaped next point"""
return last + np.random.multivariate_normal(
mean=[0, 0, 0], cov=cov, size=last.shape[0])
cs = m0[:, 2]
tec = m0[:, 1]
tau = m0[:, 0]
print("Initial CS: {}".format(cs))
print("Initial TEC: {}".format(tec))
print("Initial delay: {}".format(tau))
m = m0.copy()
# if plot:
# plt.plot(times,cs0,label="CS0")
# plt.plot(times,tec0,label="TEC0")
# plt.plot(times,delay0,label="delay0")
# plt.legend(frameon=False)
# plt.show()
Ct = (Ct_ratio * np.abs(obs_phase))**2
Cd = (Cd_error * np.pi / 180.)**2
CdCt = Cd + Ct
Si = neglogL(obs_phase, calc_phase(m, freqs), CdCt)
print("Initial Si: {}".format(Si))
max_iter = 100 * convergence
posterior = np.zeros([convergence, len(times), 3], dtype=np.double)
multiplicity = np.zeros([convergence, len(times)], dtype=np.double)
posterior[0, :, :] = m
minS = Si
minSol = m.copy()
accepted = np.ones(len(times), dtype=np.int)
cov_prior = np.diag([1e-10, 1e-6, 0.5])**2 + cov
iter = 1
while np.max(accepted) < convergence and iter < max_iter:
#sample
last = np.array(
[posterior[accepted[i] - 1, i, :] for i in range(len(times))])
m_j = sample_prior(last, cov_prior)
Sj = neglogL(obs_phase, calc_phase(m_j, freqs), CdCt)
Lj = np.exp(-Sj)
accept_mask = np.bitwise_or(
Sj < Si,
np.log(np.random.uniform(size=len(Sj))) < Si - Sj)
#print(accept_mask)
Si[accept_mask] = Sj[accept_mask]
for i in range(len(times)):
if accept_mask[i]:
posterior[accepted[i], i, :] = m_j[i, :]
multiplicity[accepted[i], i] += 1
accepted[i] += 1
else:
multiplicity[accepted[i] - 1, i] += 1
if np.any(accept_mask):
#print(m_j)
#print("{} accepted".format(np.sum(accept_mask)))
pass
maxL_mask = Sj < minS
minSol[maxL_mask, :] = m_j[maxL_mask]
minS[maxL_mask] = Sj[maxL_mask]
iter += 1
if iter != max_iter:
print("Converged in {} steps with mean acceptance rate of {}".format(
iter,
np.mean(accepted) / iter))
posteriors = []
multiplicities = []
means = []
stds = []
maxLs = []
for i in range(len(times)):
posteriors.append(posterior[:accepted[i], i, :])
multiplicities.append(multiplicity[:accepted[i], i])
means.append(
np.sum(posteriors[i].T * multiplicities[i], axis=1) /
np.sum(multiplicities[i]))
stds.append(
np.sqrt(
np.sum(posteriors[i].T**2 * multiplicities[i], axis=1) /
np.sum(multiplicities[i]) - means[i]**2))
maxLs.append(minSol[i, :])
print("Sol {}, (Gaussian) sol is {} +- {}".format(
i, means[i], stds[i]))
print(" maxL sol is {}".format(maxLs[i]))
if plot:
plt.hist(posteriors[0][:, 0], weights=multiplicities[0], label='tau')
plt.legend(frameon=False)
plt.show()
plt.hist(posteriors[0][:, 1], weights=multiplicities[0], label='tec')
plt.legend(frameon=False)
plt.show()
plt.hist(posteriors[0][:, 2], weights=multiplicities[0], label='cs')
plt.legend(frameon=False)
plt.show()
return maxLs
import cv2
import numpy as np
img1 = cv2.imread('log_3.png', 0)
img2 = cv2.imread('log_4.png', 0)
img3 = cv2.imread('blankl.png', 0)
f, c = img1.shape
for i in range(f):
for j in range(c):
img3[i][j] = np.bitwise_or(img1[i][j], img2[i][j])
cv2.imshow('res', img3)
del PParr, PVarr, PMarr
# Make a mapping of where the particles should go.
dest = na.zeros(size, dtype='int8')
done = na.zeros(size, dtype='bool')
for i, b in enumerate(boundaries):
# We go backwards, smallest to largest grid
j = len(boundaries) - i - 1
LE, RE = boundaries[j]
is_inside = ( (PPall.T >= LE).all(axis=1) * \
(PPall.T < RE).all(axis=1) )
# We only change ones that haven't been assigned before.
is_inside = is_inside * na.invert(done)
dest += j * is_inside
# Keep track of which particles have already been assigned.
done = na.bitwise_or(is_inside, done)
# Clean up.
del is_inside, done
# Find the sizes of the final files using histogram.
bins = na.arange(len(boundaries) + 1)
sizes, bins = na.histogram(dest, bins)
# For the mass field, we need to fix the masses for particles that change
# assignments. A negative value of diff means that the particle went to a
# less refined grid, so it should drop in mass. And visa versa for positive
# diffs.
refinement = float(refinement)**3
diff = dest - source
multi = na.power(refinement, diff)
def time_int_or(self):
np.bitwise_or(self.i, 0, out=self.i)
np.bitwise_or(0, self.i, out=self.i)
def test_ascii():
nobj = 5000
numpy.random.seed(8675309)
x = numpy.random.random_sample(nobj)
y = numpy.random.random_sample(nobj)
z = numpy.random.random_sample(nobj)
ra = numpy.random.random_sample(nobj)
dec = numpy.random.random_sample(nobj)
r = numpy.random.random_sample(nobj)
w = numpy.random.random_sample(nobj)
g1 = numpy.random.random_sample(nobj)
g2 = numpy.random.random_sample(nobj)
k = numpy.random.random_sample(nobj)
flags = numpy.zeros(nobj).astype(int)
for flag in [1, 2, 4, 8, 16]:
sub = numpy.random.random_sample(nobj) < 0.1
flags[sub] = numpy.bitwise_or(flags[sub], flag)
file_name = os.path.join('data', 'test.dat')
with open(file_name, 'w') as fid:
# These are intentionally in a different order from the order we parse them.
fid.write('# ra,dec,x,y,k,g1,g2,w,flag,z,r\n')
for i in range(nobj):
fid.write((('%.8f ' * 10) + '%d\n') %
(ra[i], dec[i], x[i], y[i], k[i], g1[i], g2[i], w[i],
z[i], r[i], flags[i]))
# Check basic input
config = {
'x_col': 3,
'y_col': 4,
'z_col': 9,
'x_units': 'rad',
'y_units': 'rad',
'w_col': 8,
'g1_col': 6,
'g2_col': 7,
'k_col': 5,
}
cat1 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(cat1.x, x)
numpy.testing.assert_almost_equal(cat1.y, y)
numpy.testing.assert_almost_equal(cat1.z, z)
numpy.testing.assert_almost_equal(cat1.w, w)
numpy.testing.assert_almost_equal(cat1.g1, g1)
numpy.testing.assert_almost_equal(cat1.g2, g2)
numpy.testing.assert_almost_equal(cat1.k, k)
# Check flags
config['flag_col'] = 11
cat2 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(cat2.w[flags == 0], w[flags == 0])
numpy.testing.assert_almost_equal(cat2.w[flags != 0], 0.)
# Check ok_flag
config['ok_flag'] = 4
cat3 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(
cat3.w[numpy.logical_or(flags == 0, flags == 4)],
w[numpy.logical_or(flags == 0, flags == 4)])
numpy.testing.assert_almost_equal(
cat3.w[numpy.logical_and(flags != 0, flags != 4)], 0.)
# Check ignore_flag
del config['ok_flag']
config['ignore_flag'] = 16
cat4 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(cat4.w[flags < 16], w[flags < 16])
numpy.testing.assert_almost_equal(cat4.w[flags >= 16], 0.)
# Check different units for x,y
config['x_units'] = 'arcsec'
config['y_units'] = 'arcsec'
del config['z_col']
cat5 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(cat5.x, x * (pi / 180. / 3600.))
numpy.testing.assert_almost_equal(cat5.y, y * (pi / 180. / 3600.))
config['x_units'] = 'arcmin'
config['y_units'] = 'arcmin'
cat5 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(cat5.x, x * (pi / 180. / 60.))
numpy.testing.assert_almost_equal(cat5.y, y * (pi / 180. / 60.))
config['x_units'] = 'deg'
config['y_units'] = 'deg'
cat5 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(cat5.x, x * (pi / 180.))
numpy.testing.assert_almost_equal(cat5.y, y * (pi / 180.))
del config['x_units'] # Default is radians
del config['y_units']
cat5 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(cat5.x, x)
numpy.testing.assert_almost_equal(cat5.y, y)
# Check ra,dec
del config['x_col']
del config['y_col']
config['ra_col'] = 1
config['dec_col'] = 2
config['r_col'] = 10
config['ra_units'] = 'rad'
config['dec_units'] = 'rad'
cat6 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(cat6.ra, ra)
numpy.testing.assert_almost_equal(cat6.dec, dec)
config['ra_units'] = 'deg'
config['dec_units'] = 'deg'
cat6 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(cat6.ra, ra * (pi / 180.))
numpy.testing.assert_almost_equal(cat6.dec, dec * (pi / 180.))
config['ra_units'] = 'hour'
config['dec_units'] = 'deg'
cat6 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(cat6.ra, ra * (pi / 12.))
numpy.testing.assert_almost_equal(cat6.dec, dec * (pi / 180.))
# Check using a different delimiter, comment marker
csv_file_name = os.path.join('data', 'test.csv')
with open(csv_file_name, 'w') as fid:
# These are intentionally in a different order from the order we parse them.
fid.write('% This file uses commas for its delimiter')
fid.write('% And more than one header line.')
fid.write('% Plus some extra comment lines every so often.')
fid.write('% And we use a weird comment marker to boot.')
fid.write('% ra,dec,x,y,k,g1,g2,w,flag\n')
for i in range(nobj):
fid.write((('%.8f,' * 10) + '%d\n') %
(ra[i], dec[i], x[i], y[i], k[i], g1[i], g2[i], w[i],
z[i], r[i], flags[i]))
if i % 100 == 0:
fid.write('%%%% Line %d\n' % i)
config['delimiter'] = ','
config['comment_marker'] = '%'
cat7 = treecorr.Catalog(csv_file_name, config)
numpy.testing.assert_almost_equal(cat7.ra, ra * (pi / 12.))
numpy.testing.assert_almost_equal(cat7.dec, dec * (pi / 180.))
numpy.testing.assert_almost_equal(cat7.r, r)
numpy.testing.assert_almost_equal(cat7.g1, g1)
numpy.testing.assert_almost_equal(cat7.g2, g2)
numpy.testing.assert_almost_equal(cat7.w[flags < 16], w[flags < 16])
numpy.testing.assert_almost_equal(cat7.w[flags >= 16], 0.)
# Check flip_g1, flip_g2
del config['delimiter']
del config['comment_marker']
config['flip_g1'] = True
cat8 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(cat8.g1, -g1)
numpy.testing.assert_almost_equal(cat8.g2, g2)
config['flip_g2'] = 'true'
cat8 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(cat8.g1, -g1)
numpy.testing.assert_almost_equal(cat8.g2, -g2)
config['flip_g1'] = 'n'
config['flip_g2'] = 'yes'
cat8 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(cat8.g1, g1)
numpy.testing.assert_almost_equal(cat8.g2, -g2)
# Check overriding values with kwargs
cat8 = treecorr.Catalog(file_name, config, flip_g1=True, flip_g2=False)
numpy.testing.assert_almost_equal(cat8.g1, -g1)
numpy.testing.assert_almost_equal(cat8.g2, g2)
def increse(d, k):
d[k] -= 1
return d[k]
# print("me.py file_path. eg: me.py D:/program/x86/KingRoot/adb.exe")
fpath = sys.argv[1]
# fpath = "D:/program/x86/KingRoot/adb.exe"
print("process:%s" % fpath)
fbin = np.fromfile(fpath, dtype=np.uint8)
bin5 = np.resize(fbin, int(len(fbin) / 5) * 5)
bin = np.reshape(bin5, (-1, 5))
xy, color_byte = bin[:, 0:2], bin[:, 2:5]
xy_ = [np.uint16(np.bitwise_or(np.left_shift(i[0], 8), i[1])) for i in xy]
xy_unique, xy_index, xy_counts = np.unique(xy_,
return_index=True,
return_counts=True)
mod = np.average(xy_counts) * np.std(xy_counts) * 2
xy_unique_count = dict(zip(xy_unique, xy_counts))
xy_z = [np.array([i, increse(xy_unique_count, i)]) for i in xy_]
xyz = np.array([
np.array([
np.uint8(np.right_shift(np.bitwise_and(i[0], 0xff00), 8)),
np.uint8(np.bitwise_and(i[0], 0x00ff)),
np.uint32(i[1] % mod)
]) for i in xy_z
])
x, y, z = xyz[:, 0:1], xyz[:, 1], xyz[:, 2]
color0To1 = np.true_divide(color_byte, 256)
def to_framebuffer_or(self):
self._fb1 = _np.bitwise_or(self._fb1, self._matrix) # pylint: disable=no-member
def mark_transit_cadences(time,
period_days,
epoch_bkjd,
duration_days,
num_durations=1,
flags=None):
"""Create a logical array indicating which cadences are
affected by a transit.
Parameters
----------
time : array_like
Numpy 1D array of cadence times.
period_days : float
Transit period in days.
epoch_bkjd : float
Transit epoch.
duration_days : float
Duration of transit (start to finish). If you select a duration from
first to last contact, all cadences affected by transit are selected.
If you only select 2 to 3 contacts, only the interior region of the
transit is selected.
num_durations : int
How much of the lightcurve on either side of the transit to mark.
Default means to mark 1/2 the transit duration on either side of the
transit center.
flags : array_like or `None`
If set, must be an array of booleans of length ``time``.
Cadences where this array is `True` are ignored in the calculation.
This is useful if some of the entries of time are NaN.
Returns
-------
idx : array_like
Array of booleans with the same length as ``time``.
Element set to `True` are affected by transits.
Raises
------
ValueError
Invalid input.
"""
if flags is None:
flags = np.zeros_like(time, dtype=np.bool_)
good_time = time[~flags]
i0 = np.floor((np.min(good_time) - epoch_bkjd) / period_days)
i1 = np.ceil((np.max(good_time) - epoch_bkjd) / period_days)
if not np.isfinite(i0) or not np.isfinite(i1):
raise ValueError('Error bounding transit time')
transit_times = epoch_bkjd + period_days * np.arange(i0, i1 + 1)
big_num = sys.float_info.max # A large value that isn't NaN
max_diff = 0.5 * duration_days * num_durations
idx = np.zeros_like(time, dtype=np.bool8)
for tt in transit_times:
diff = time - tt
diff[flags] = big_num
if not np.all(np.isfinite(diff)):
raise ValueError('NaN found in diff of transit time')
idx = np.bitwise_or(idx, np.fabs(diff) < max_diff)
if not np.any(idx):
warnings.warn('No cadences found matching transit locations')
return idx
def draw_with_fb_or(self):
self._matrix = _np.bitwise_or(self._fb1, self._matrix) # pylint: disable=no-member
return bytes(self)
def find_CRs(data, gdq, read_noise, rej_threshold, nframes):
"""
Find CRs/Jumps in each integration within the input data array.
The input data array is assumed to be in units of electrons, i.e. already
multiplied by the gain. We also assume that the read noise is in units of
electrons.
"""
# Get data characteristics
(nints, ngroups, nrows, ncols) = data.shape
# Create array for output median slope images
median_slopes = np.zeros((nints, nrows, ncols), dtype=np.float32)
# Square the read noise values, for use later
read_noise_2 = read_noise * read_noise
# Reset saturated values in input data array to NaN, so they don't get
# used in any of the subsequent calculations
data[gdq == dqflags.group['SATURATED']] = np.NaN
# Loop over multiple integrations
for integration in range(nints):
# Roll the ngroups axis of data arrays to the end, to make
# memory access to the values for a given pixel faster
rdata = np.rollaxis(data[integration], 0, 3)
# Compute first differences of adjacent groups up the ramp
first_diffs = np.diff(rdata, axis=2)
nans = np.where(np.isnan(first_diffs))
first_diffs[nans] = 100000.
positive_first_diffs = np.abs(first_diffs)
diffsum = positive_first_diffs.sum
# Make all the first diffs for saturated groups be equal to
# 100,000 to put them above the good values in the sorted index
matching_array = np.ones(shape=(
positive_first_diffs.shape[0], positive_first_diffs.shape[1],
positive_first_diffs.shape[2])) * 100000
sat_groups = (positive_first_diffs == matching_array)
number_sat_groups = (sat_groups * 1).sum(axis=2)
ndiffs = ngroups - 1
sort_index = np.argsort(positive_first_diffs)
med_diffs = return_clipped_median(ndiffs, number_sat_groups,
positive_first_diffs, sort_index)
# Save initial estimate of the median slope for all pixels
median_slopes[integration] = med_diffs
# Compute uncertainties as the quadrature sum of the poisson noise
# in the first difference signal and read noise. Because the first
# differences can be biased by CRs/jumps, we use the median signal
# for computing the poisson noise. Here sigma correctly has the
# read noise taking into account the fact that multiple frames were
# averaged into each group.
poisson_noise = np.sqrt(np.abs(med_diffs))
sigma = np.sqrt(poisson_noise * poisson_noise + read_noise_2 / nframes)
# Reset sigma to exclude pixels with both readnoise and signal=0
wh_sig = np.where(sigma == 0.)
if len(wh_sig[0] > 0):
log.debug('Twopt found %d pixels with sigma=0' % (len(wh_sig[0])))
log.debug('which will be reset so that no jump will be detected')
sigma[wh_sig] = HUGE_NUM
# Compute distance of each sample from the median in units of sigma;
# note that the use of "abs" means we'll detect both positive and
# negative outliers
ratio = np.abs(first_diffs -
med_diffs[:, :, np.newaxis]) / sigma[:, :, np.newaxis]
# Find the group index of the max outlier in each pixel
# (the RHS is identical to the previous versions:
# max_index = np.nanargmax (ratio, axis=2)
max_index1 = sort_index[:, :, ngroups - 2]
# Get indices of highest values (may be outliers) that are above the
# rejection threshold
r, c = np.indices(max_index1.shape)
row1, col1 = np.where(
ratio[r, c, max_index1 - number_sat_groups] > rej_threshold)
log.debug(
'From highest outlier Twopt found %d pixels with at least one CR' %
(len(row1)))
number_pixels_with_cr = len(row1)
for j in range(number_pixels_with_cr):
pixel_masked_diffs = first_diffs[row1[j], col1[j]]
pixel_rn2 = read_noise_2[row1[j], col1[j]]
pixel_sat_groups = number_sat_groups[row1[j], col1[j]]
sorted_index_of_cr = max_index1[row1[j],
col1[j]] - pixel_sat_groups
# Create a CR mask and set 1st CR to be found
# cr_mask=0 designates a CR
pixel_cr_mask = np.ones(pixel_masked_diffs.shape, dtype=bool)
number_CRs_found = 1
pixel_sorted_index = sort_index[row1[j], col1[j], :]
pixel_cr_mask[pixel_sorted_index[ndiffs - pixel_sat_groups -
1]] = 0
new_CR_found = True
# Loop over all the found CRs and see if there is more than one CR, setting the mask as you go
while new_CR_found and (
(ndiffs - number_CRs_found - pixel_sat_groups) > 1):
new_CR_found = False
pixel_med_diff = return_clipped_median(
ndiffs, number_CRs_found + pixel_sat_groups,
pixel_masked_diffs, pixel_sorted_index)
poisson_noise = np.sqrt(np.abs(pixel_med_diff))
sigma = np.sqrt(poisson_noise * poisson_noise +
pixel_rn2 / nframes)
ratio = np.abs(pixel_masked_diffs - pixel_med_diff) / sigma
pixel_sorted_ratio = ratio[pixel_sorted_index[:]]
# Check if largest remaining difference is above threshold
if ratio[pixel_sorted_index[ndiffs - number_CRs_found -
pixel_sat_groups -
1]] > rej_threshold:
new_CR_found = True
pixel_cr_mask[pixel_sorted_index[ndiffs -
number_CRs_found -
pixel_sat_groups - 1]] = 0
number_CRs_found += 1
# Found all CRs. Set CR flags in input DQ array for this pixel
gdq[integration, 1:, row1[j], col1[j]] = \
np.bitwise_or(gdq[integration, 1:, row1[j], col1[j]],
dqflags.group['JUMP_DET'] * np.invert(pixel_cr_mask))
# Save the CR-cleaned median slope for this pixel
if not new_CR_found: # the loop ran at least one time
median_slopes[integration, row1[j], col1[j]] = pixel_med_diff
# Next pixel with an outlier (j loop)
# Next integration (integration loop)
return median_slopes
