def realDataPng(realOtPath, workPath, tSampleNamePart, tModelNamePart1,
tModelNamePart2):
X, Y, s2n = getRealData(realOtPath, workPath, tSampleNamePart)
print(X.shape)
print(Y.shape)
print(s2n.shape)
tIdx = [
370, 389, 419, 451, 459, 557, 559, 579, 3463, 5238, 7010, 7131, 7460,
7748, 9139, 9698, 12605, 13183, 20547, 20554, 20558, 20561, 20566,
20568, 20575, 20586, 20589, 20642, 20644, 20646, 20648, 20717, 20726,
20730, 20734, 20736, 20739, 20746, 20768, 20771, 20868, 20871, 20884,
20885
]
tIdx = np.array(tIdx)
X = X[tIdx]
Y = Y[tIdx]
s2n = s2n[tIdx]
print(X.shape)
print(Y.shape)
print(s2n.shape)
falseImg = X
falseS2n = s2n
falsePred = Y
print("\n\n***********************")
print("model1 True, model2 False: %d" % (falseImg.shape[0]))
for i in range(falseImg.shape[0]):
objWidz = zscale_image(falseImg[i][0])
tmpWidz = zscale_image(falseImg[i][1])
resiWidz = zscale_image(falseImg[i][2])
if objWidz.shape[0] == 0:
objWidz = falseImg[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = falseImg[i][1]
if resiWidz.shape[0] == 0:
resiWidz = falseImg[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(6, 2))
axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray')
axes.flat[1].set_title("pbb=%.2f,s2n=%.2f" %
(falsePred[i][1], falseS2n[i]))
plt.show()
def getSubImages(storePath, skyListFile, otName):
workPath = 'twork/%s' % (otName)
os.system("rm -rf %s" % (workPath))
if not os.path.exists(workPath):
os.system("mkdir -p %s" % (workPath))
size = 100
skyList = np.loadtxt(skyListFile, dtype='str', delimiter=',')
for tsky in skyList:
#190208,2568,19,3365.45,3507.93,G044_mon_objt_190208T12320486.fit
#print(tsky)
dateStr = tsky[0]
cam_id = int(tsky[2])
mountNum = int(cam_id / 5) + 1
ccdNum = cam_id % 5
camName = '0%d%d' % (mountNum, ccdNum)
tpath1 = '%s/%s/G00%d_%s' % (storePath, dateStr, mountNum, camName)
fname = tsky[5]
x = float(tsky[3]) + 20
y = float(tsky[4])
print("%s/%s" % (tpath1, fname))
#subImg = getWindowImgs(tpath1, fname, x, y, size)
subImg = getWindowImgs(storePath, fname, x, y, size)
subImgz = zscale_image(subImg)
subImgz = scipy.ndimage.zoom(subImgz, 4, order=0)
imgpre = fname.split(".")[0]
tImgName = "%s.jpg" % (imgpre)
savePath = "%s/%s" % (workPath, tImgName)
Image.fromarray(subImgz).save(savePath)
def getWindowImgs2(self, objImg, tmpImg, resiImg, poslist, size):
objPath = "%s/%s" % (self.tmpDir, objImg)
tmpPath = "%s/%s" % (self.tmpDir, tmpImg)
resiPath = "%s/%s" % (self.tmpDir, resiImg)
objData = fits.getdata(objPath)
tmpData = fits.getdata(tmpPath)
resiData = fits.getdata(resiPath)
i = 0
subImgs = []
for tpos in poslist:
objWid = self.getWindowImg(objData, (tpos[0], tpos[1]), size)
tmpWid = self.getWindowImg(tmpData, (tpos[2], tpos[3]), size)
resiWid = self.getWindowImg(resiData, (tpos[4], tpos[5]), size)
if len(resiWid) > 0 and len(objWid) > 0 and len(tmpWid) > 0:
#self.log.debug(tpos)
#cv2.circle(img, center, radius, color, thickness=1, lineType=8, shift=0)
#cv2.circle(objWid,(int(size/2),int(size/2)),2,(255,0,0),1)
#cv2.circle(tmpWid,(int(size/2),int(size/2)),2,(255,0,0),1)
#cv2.circle(resiWid,(int(size/2),int(size/2)),2,(255,0,0),1)
objWidz = zscale_image(objWid)
tmpWidz = zscale_image(tmpWid)
resiWidz = zscale_image(resiWid)
#异常残差图像处理，如果scale失败：1）等于原diffImg；2）直接量化到255
if resiWidz.shape[0] == 21 and resiWidz.shape[1] == 21:
subImgs.append([objWidz, tmpWidz, resiWidz])
#conWid = np.concatenate((objWidz, tmpWidz, resiWidz), axis=1)
#plt.imshow(conWid, cmap='gray')
#plt.show()
i = i + 1
#if i>20:
# break
return subImgs
def saveImgs(imgs, tpath, zoomScale=4):
if not os.path.exists(tpath):
os.system("mkdir -p %s" % (tpath))
for i in range(imgs.shape[0]):
X = imgs[i]
tobjImg = zscale_image(X[0])
tTempImg = zscale_image(X[1])
tResiImg = zscale_image(X[2])
if zoomScale != 1:
tobjImg = scipy.ndimage.zoom(tobjImg, zoomScale, order=0)
tTempImg = scipy.ndimage.zoom(tTempImg, zoomScale, order=0)
tResiImg = scipy.ndimage.zoom(tResiImg, zoomScale, order=0)
xspace = np.ones((tobjImg.shape[0], 10), np.uint8) * 255
timg = np.concatenate((tobjImg, xspace, tTempImg, xspace, tResiImg),
axis=1)
savePath = "%s/%05d.jpg" % (tpath, i)
Image.fromarray(timg).save(savePath)
def showImage(imgs, props, predY, label, showNum=10):
maxEllip = 0.5
for i in range(imgs.shape[0]):
if i>=showNum:
break
s2n = 1.087/props[i,12].astype(np.float)
#print(props[i])
X = props[i][0]
Y = props[i][1]
#if math.fabs(Y-2616.9944)>5:
# continue
ELONGATION = props[i][5]
ELLIPTICITY = props[i][6]
if ELLIPTICITY>maxEllip:
continue
print("ELONGATION=%.2f,ELLIPTICITY=%.2f,X=%.2f,Y=%.2f"%(ELONGATION,ELLIPTICITY,X, Y))
objWidz = zscale_image(imgs[i][0])
tmpWidz = zscale_image(imgs[i][1])
resiWidz = zscale_image(imgs[i][2])
if objWidz.shape[0] == 0:
objWidz = imgs[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = imgs[i][1]
if resiWidz.shape[0] == 0:
resiWidz = imgs[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(3, 3))
axes.flat[0].imshow(objWidz, interpolation = "nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation = "nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation = "nearest", cmap='gray')
axes.flat[1].set_title("predicted pbb=%.2f, label=%s, s2n=%.2f,ellip=%.2f\n"%(predY[i][1],label,s2n,ELLIPTICITY))
plt.show()
def test(workPath, tModelNamePart):
dataPath = "%s/data" % (workPath)
tpath = "%s/test.npz" % (dataPath)
print(tpath)
tdata = np.load(tpath)
X_test = tdata['X']
Y_test = tdata['Y']
s2n_test = tdata['s2n']
print(X_test.shape)
print(Y_test.shape)
print(s2n_test.shape)
K.set_image_data_format('channels_first')
modelName = "model_80w_20190403_dropout_train8_09.h5"
#model = load_model("%s/%s"%(workPath, modelName))
model = load_model(
"%s/%s" % (workPath, modelName),
custom_objects={'concatenate': keras.layers.concatenate})
Y_pred = model.predict(X_test)
pbb_threshold = 0.5
pred_labels = np.array((Y_pred[:, 1] > pbb_threshold), dtype="int")
print(modelName)
print("Correctly classified %d out of %d" %
((pred_labels == Y_test[:, 1].astype(int)).sum(), Y_test.shape[0]))
print("accuracy = %f" %
(1. *
(pred_labels == Y_test[:, 1].astype(int)).sum() / Y_test.shape[0]))
TIdx = Y_test[:, 1] == 1
FIdx = Y_test[:, 1] == 0
T_pred_rst = pred_labels[TIdx]
F_pred_rst = pred_labels[FIdx]
print(T_pred_rst.shape)
print(F_pred_rst.shape)
TP = ((T_pred_rst == 1).astype(int)).sum()
TN = ((F_pred_rst == 0).astype(int)).sum()
FP = ((F_pred_rst == 1).astype(int)).sum()
FN = ((T_pred_rst == 0).astype(int)).sum()
print("total=%d,TP=%d,TN=%d,FP=%d,FN=%d" %
(Y_test.shape[0], TP, TN, FP, FN))
accuracy = (TP + TN) * 1.0 / (TP + FN + TN + FP)
precision = (TP) * 1.0 / (TP + FP)
recall = (TP) * 1.0 / (TP + FN)
f1_score = 2.0 * (precision * recall) / (precision + recall)
print("accuracy=%f%%" % (accuracy * 100))
print("precision=%f%%" % (precision * 100))
print("recall=%f%%" % (recall * 100))
print("f1_score=%f%%" % (f1_score * 100))
falseImg = X_test[pred_labels != Y_test[:, 1]]
falseLabel = Y_test[pred_labels != Y_test[:, 1]]
falsePred = Y_pred[pred_labels != Y_test[:, 1]]
falseS2n = s2n_test[pred_labels != Y_test[:, 1]]
tnum1 = 0
tnum2 = 0
for i in range(falseImg.shape[0]):
if falseLabel[i, 1] == 1:
tnum1 = tnum1 + 1
else:
tnum2 = tnum2 + 1
print("total %d, miss classified %d" %
(Y_test.shape[0], falseImg.shape[0]))
print("True classified as False %d" % (tnum1))
print("False classified as True %d" % (tnum2))
print("\n\n***********************")
print("image of True classified as False")
for i in range(falseImg.shape[0]):
if falseLabel[i, 1] == 1:
objWidz = zscale_image(falseImg[i][0])
tmpWidz = zscale_image(falseImg[i][1])
resiWidz = zscale_image(falseImg[i][2])
if objWidz.shape[0] == 0:
objWidz = falseImg[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = falseImg[i][1]
if resiWidz.shape[0] == 0:
resiWidz = falseImg[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(6, 2))
axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray')
axes.flat[2].set_title("predicted pbb = " +
str(np.round(falsePred[i][1], 2)) +
", label = " + str(falseLabel[i, 1]) +
", s2n = " + str(falseS2n[i]))
plt.show()
print("\n\n***********************")
print("image of False classified as True")
for i in range(falseImg.shape[0]):
if falseLabel[i, 1] == 0:
objWidz = zscale_image(falseImg[i][0])
tmpWidz = zscale_image(falseImg[i][1])
resiWidz = zscale_image(falseImg[i][2])
if objWidz.shape[0] == 0:
objWidz = falseImg[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = falseImg[i][1]
if resiWidz.shape[0] == 0:
resiWidz = falseImg[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(6, 2))
axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray')
axes.flat[2].set_title("predicted pbb = " +
str(np.round(falsePred[i][1], 2)) +
", label = " + str(falseLabel[i, 1]) +
", s2n = " + str(falseS2n[i]))
plt.show()
trueImg = X_test[pred_labels == Y_test[:, 1]]
trueLabel = Y_test[pred_labels == Y_test[:, 1]]
truePred = Y_pred[pred_labels == Y_test[:, 1]]
trueS2n = s2n_test[pred_labels == Y_test[:, 1]]
showNum = 50
tnum = 0
print("\n\n***********************")
print("image of True classified as True")
for i in range(trueImg.shape[0]):
if trueLabel[i, 1] == 1:
objWidz = zscale_image(trueImg[i][0])
tmpWidz = zscale_image(trueImg[i][1])
resiWidz = zscale_image(trueImg[i][2])
if objWidz.shape[0] == 0:
objWidz = falseImg[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = falseImg[i][1]
if resiWidz.shape[0] == 0:
resiWidz = falseImg[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(6, 2))
axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray')
axes.flat[2].set_title("predicted pbb = " +
str(np.round(truePred[i][1], 2)) +
", label = " + str(trueLabel[i, 1]) +
", s2n = " + str(trueS2n[i]))
plt.show()
tnum = tnum + 1
if tnum > showNum:
break
tnum = 0
print("\n\n***********************")
print("image of False classified as False")
for i in range(trueImg.shape[0]):
if trueLabel[i, 1] == 0:
objWidz = zscale_image(trueImg[i][0])
tmpWidz = zscale_image(trueImg[i][1])
resiWidz = zscale_image(trueImg[i][2])
if objWidz.shape[0] == 0:
objWidz = falseImg[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = falseImg[i][1]
if resiWidz.shape[0] == 0:
resiWidz = falseImg[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(6, 2))
axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray')
axes.flat[2].set_title("predicted pbb = " +
str(np.round(truePred[i][1], 2)) +
", label = " + str(trueLabel[i, 1]) +
", s2n = " + str(trueS2n[i]))
plt.show()
tnum = tnum + 1
if tnum > showNum:
break
def imgTransform(data1, data2, data3, transMethod='none'):
data1[data1 < 0] = 0
data2[data2 < 0] = 0
data3[data3 < 0] = 0
tmax1 = np.max(data1)
tmax2 = np.max(data2)
tmax3 = np.max(data3)
tmax = np.max([tmax1, tmax2, tmax3])
tminMaxValue = 50
if transMethod == 'eachMax':
if tmax1 > tminMaxValue and tmax2 > tminMaxValue and tmax3 > tminMaxValue:
data1 = data1 * 255.0 / tmax1
data2 = data2 * 255.0 / tmax2
data3 = data3 * 255.0 / tmax3
elif tmax1 > tminMaxValue and tmax2 <= tminMaxValue and tmax3 > tminMaxValue:
data1 = data1 * 255.0 / tmax1
data2 = data2
data3 = data3 * 255.0 / tmax3
else:
data1, data2, data3 = np.array([]), np.array([]), np.array([])
elif transMethod == 'unionMax':
if tmax < tminMaxValue:
data1, data2, data3 = np.array([]), np.array([]), np.array([])
else:
data1 = data1 * 255.0 / tmax
data2 = data2 * 255.0 / tmax
data3 = data3 * 255.0 / tmax
elif transMethod == 'zscale':
tdata1 = zscale_image(data1)
tdata2 = zscale_image(data2)
tdata3 = zscale_image(data3)
if tdata1.shape[0] == 0:
tmin = np.min(data1)
tmax = np.max(data1)
tdata1 = (((data1 - tmin) / (tmax - tmin)) * 255).astype(np.uint8)
if tdata2.shape[0] == 0:
tmin = np.min(data2)
tmax = np.max(data2)
tdata2 = (((data2 - tmin) / (tmax - tmin)) * 255).astype(np.uint8)
if tdata3.shape[0] == 0:
tmin = np.min(data3)
tmax = np.max(data3)
tdata3 = (((data3 - tmin) / (tmax - tmin)) * 255).astype(np.uint8)
data1 = tdata1
data2 = tdata2
data3 = tdata3
if transMethod == 'eachMax' or transMethod == 'unionMax' or transMethod == 'zscale':
data1[data1 > 255] = 255
data2[data2 > 255] = 255
data3[data3 > 255] = 255
data1 = data1.astype(np.uint8)
data2 = data2.astype(np.uint8)
data3 = data3.astype(np.uint8)
elif transMethod == 'none': #preMethod4
data1[data1 > 65535] = 65535
data2[data2 > 65535] = 65535
data3[data3 > 65535] = 65535
data1 = data1.astype(np.uint16)
data2 = data2.astype(np.uint16)
data3 = data3.astype(np.uint16)
return data1, data2, data3
def doClassifyAndUpload(self,
subImgPath,
totFile,
fotFile,
fullImgPath,
newImg,
tmpImg,
resImg,
origName,
serverIP,
prob=0.01,
maxNEllip=0.6,
maxMEllip=0.5,
reverse=False):
self.log.debug("start new thread classifyAndUpload %s" % (origName))
starttime = datetime.now()
self.pbb_threshold = prob
try:
nameBase = origName[:origName.index(".")]
tParms1 = self.doClassifyFile(subImgPath, totFile)
if tParms1.shape[0] > 0:
tParms1 = tParms1[tParms1[:, 6] < maxNEllip]
#tParms1 = tParms1[(tParms1[:,6]<maxMEllip) & (tParms1[:,17]>=prob)]
if tParms1.shape[0] > 0:
tflags1 = np.ones((tParms1.shape[0], 1)) #OT FLAG
tParms1 = np.concatenate((tParms1, tflags1), axis=1)
tParms2 = self.doClassifyFile(subImgPath, fotFile)
if tParms2.shape[0] > 0:
tParms2 = tParms2[(tParms2[:, 6] < maxMEllip)
& (tParms2[:, 17] >= prob)]
if tParms2.shape[0] > 0:
tflags2 = np.zeros((tParms2.shape[0], 1)) #OT FLAG
tParms2 = np.concatenate((tParms2, tflags2), axis=1)
if tParms1.shape[0] > 0 and tParms2.shape[0] > 0 and tParms2.shape[
0] < 50:
tParms = np.concatenate((tParms1, tParms2), axis=0)
elif tParms1.shape[0] > 0:
tParms = tParms1
elif tParms2.shape[0] > 0 and tParms2.shape[0] < 25:
tParms = tParms2
else:
tParms = np.array([])
if tParms.shape[0] > 0:
tSubImgs, tParms = getWindowImgs(fullImgPath, newImg, tmpImg,
resImg, tParms, 100)
if tParms.shape[0] > 0:
self.log.info(
"after classified, %s total get %d sub images" %
(origName, tSubImgs.shape[0]))
i = 1
timgNames = []
for timg in tSubImgs:
objWid, tmpWid, resiWid = timg[0], timg[1], timg[2]
objWidz = zscale_image(objWid)
tmpWidz = zscale_image(tmpWid)
resiWidz = zscale_image(resiWid)
objWidz = scipy.ndimage.zoom(objWidz, 2, order=0)
tmpWidz = scipy.ndimage.zoom(tmpWidz, 2, order=0)
resiWidz = scipy.ndimage.zoom(resiWidz, 2, order=0)
objWidz = objWidz.reshape(objWidz.shape[0],
objWidz.shape[1],
1).repeat(3, 2)
tmpWidz = tmpWidz.reshape(tmpWidz.shape[0],
tmpWidz.shape[1],
1).repeat(3, 2)
resiWidz = resiWidz.reshape(resiWidz.shape[0],
resiWidz.shape[1],
1).repeat(3, 2)
shift00 = 3
cv2.circle(objWidz,
(int(objWidz.shape[0] / 2) - shift00,
int(objWidz.shape[1] / 2) - shift00),
10, (0, 255, 0),
thickness=2)
cv2.circle(tmpWidz,
(int(tmpWidz.shape[0] / 2) - shift00,
int(tmpWidz.shape[1] / 2) - shift00),
10, (0, 255, 0),
thickness=2)
cv2.circle(resiWidz,
(int(resiWidz.shape[0] / 2) - shift00,
int(resiWidz.shape[1] / 2) - shift00),
10, (0, 255, 0),
thickness=2)
spaceLine = np.ones(
(objWidz.shape[0], 5), dtype=np.uint8) * 255
spaceLine = spaceLine.reshape(spaceLine.shape[0],
spaceLine.shape[1],
1).repeat(3, 2)
sub2Con = np.concatenate(
(objWidz, spaceLine, tmpWidz, spaceLine, resiWidz),
axis=1)
tImgName = "%s_%05d.jpg" % (nameBase, i)
timgNames.append(tImgName)
savePath = "%s/%s" % (fullImgPath, tImgName)
Image.fromarray(sub2Con).save(savePath)
i = i + 1
catName = tImgName = "%s.cat" % (nameBase)
catPath = "%s/%s" % (fullImgPath, catName)
fp0 = open(catPath, 'w')
#fp0.write(self.theader)
i = 0
for td in tParms:
tstr = "%.4f,%.4f,%.2f,%.2f,%.2f,%.3f,%.3f,%.3f,%.2f,%.2f,%d,%.4f,%.4f,%.4f,%.4f,%f,%f,%.3f,%d,%s\n"%\
(td[0],td[1],td[2],td[3],td[4],td[5],td[6],td[7],td[8],td[9],td[10],td[11],td[12],td[13],
td[14],td[15],td[16],td[17],td[18], timgNames[i])
fp0.write(tstr)
i = i + 1
fp0.close()
self.doUpload(fullImgPath, [catName], 'diffot1', serverIP)
self.doUpload(fullImgPath, timgNames, 'diffot1img',
serverIP)
if tParms.shape[0] == 0:
self.log.info("after classified, no OT candidate left")
if tParms2.shape[0] >= 50:
self.log.error(
"too more matched OT candidate, skip upload matched to db: after classified, %s total get %d matchend sub images"
% (origName, tParms2.shape[0]))
os.system("rm -rf %s" % (fullImgPath))
except Exception as e:
self.log.error('classifyAndUpload error')
self.log.error(str(e))
tstr = traceback.format_exc()
self.log.error(tstr)
endtime = datetime.now()
runTime = (endtime - starttime).seconds
self.log.info("********** classifyAndUpload %s use %d seconds" %
(origName, runTime))
def simulateImage1(self, objCat, objImg, tmpCat, tmpImg, tmpOtNum=100):
#对同一幅图多次仿真，只用提取一次仿真psf模板
if self.otImgNum < tmpOtNum:
self.getTmpOtImgs(tmpCat, tmpImg, otNum=tmpOtNum)
otAs, deltaXY = self.randomOTAPos(objCat)
destImg = "%s/%s" % (self.tmpDir, objImg)
with fits.open(destImg) as hdul:
tflag = 1
data0 = hdul[0].data
for posamag in otAs:
posa = (posamag[0], posamag[1])
mag_add = posamag[2]
rimgIdx = randint(0, self.otImgNum - 1)
psft = self.otImgs[rimgIdx]
flux_ratio = 10**((10 - mag_add) / 2.5)
data0 = self.addStar(data0, psft, posa, flux_ratio=flux_ratio)
if tflag == 0:
ctrX = math.ceil(posa[0])
ctrY = math.ceil(posa[1])
minx = ctrX - 16
maxx = ctrX + 16
miny = ctrY - 16
maxy = ctrY + 16
plt.clf()
fig, axes = plt.subplots(1, 4, figsize=(6, 8))
axes.flat[0].imshow(psft,
interpolation="nearest",
cmap='gray')
psftz = zscale_image(psft)
axes.flat[1].imshow(psftz,
interpolation="nearest",
cmap='gray')
widImg = data0[miny:maxy, minx:maxx]
axes.flat[2].imshow(widImg,
interpolation="nearest",
cmap='gray')
widImgz = zscale_image(widImg)
axes.flat[3].imshow(widImgz,
interpolation="nearest",
cmap='gray')
plt.show()
tflag = 1
outpre = objImg.split(".")[0]
regfile = "%s_simaddstar1_ds9.reg" % (outpre)
posfile = "%s_simaddstar1_pos.cat" % (outpre)
outfile = "%s_simaddstar1.fit" % (outpre)
regPath = "%s/%s" % (self.tmpDir, regfile)
posPath = "%s/%s" % (self.tmpDir, posfile)
outPath = "%s/%s" % (self.tmpDir, outfile)
hdul_new = fits.HDUList(hdul)
hdul_new.writeto(outPath, overwrite=True)
hdul_new.close()
print("output simulated fits file to %s " % (outfile))
fp1 = open(regPath, 'w')
fp2 = open(posPath, 'w')
for posamag in otAs:
posa = (posamag[0], posamag[1])
mag_add = posamag[2]
fp1.write(
"image;circle(%.2f,%.2f,%.2f) # color=green width=1 text={%.2f} font=\"times 7\"\n"
% (posa[0], posa[1], 10.0, mag_add))
fp2.write("%6.2f %6.2f %2.1f\n" % (posa[0], posa[1], mag_add))
fp1.close()
fp2.close()
print("simulateImageByAddStar1 done.")
return outfile, posfile, deltaXY, self.otImgs
def doClassifyAndUpload(self,
subImgPath,
totFile,
fotFile,
fullImgPath,
newImg,
tmpImg,
resImg,
origName,
serverIP,
runName,
skyName,
tcatParm,
prob=0.0000001,
maxNEllip=0.6,
maxMEllip=0.5,
reverse=False):
self.log.info("start new thread classifyAndUpload %s" % (origName))
print("start new thread classifyAndUpload %s" % (origName))
minFwhm = 1.5
maxFwhm = 3.0
starttime = datetime.now()
self.pbb_threshold = prob
try:
nameBase = origName[:origName.index(".")]
dateStr = nameBase.split('_')[3][:6]
camName = nameBase[:4]
cmbNum = 5
#tidx = nameBase.index('_c')+2
#cmbNum = nameBase[tidx:tidx+3] #'G021_tom_objt_190109T13531492_c005.fit'
crossTaskName = "%s_%s_%s_C%03d" % (dateStr, camName, skyName,
cmbNum)
if runName != 'p1':
crossTaskName = "%s_%s" % (crossTaskName, runName)
#self.log.info("crossTaskName %s"%(crossTaskName))
#print("crossTaskName %s"%(crossTaskName))
tParms1t = np.array([])
tParms2t = np.array([])
tParms1, obsUtc1 = self.doClassifyFile(subImgPath, totFile)
#print("doClassifyAndUpload 001")
if tParms1.shape[0] > 0:
tParms1t = tParms1[(tParms1[:, 6] < maxNEllip)
& (tParms1[:, 9] < maxFwhm) &
(tParms1[:, 9] > minFwhm)]
#tParms1t = tParms1[(tParms1[:,6]<maxMEllip) & (tParms1[:,15]>=prob)]
if tParms1t.shape[0] > 0:
tflags1t = np.ones((tParms1t.shape[0], 1)) #OT FLAG
tParms1t = np.concatenate((tParms1t, tflags1t), axis=1)
#print("doClassifyAndUpload 002")
tParms2, obsUtc2 = self.doClassifyFile(subImgPath, fotFile)
if tParms2.shape[0] > 0:
#tParms2t = tParms2[(tParms2[:,6]<maxMEllip) & (tParms2[:,15]>=prob)&(tParms2[:,9]<maxFwhm)&(tParms2[:,9]>minFwhm)]
tParms2t = tParms2[(tParms2[:, 6] < maxMEllip)
& (tParms2[:, 9] < maxFwhm) &
(tParms2[:, 9] > minFwhm)]
if tParms2t.shape[0] > 0:
tflags2t = np.zeros((tParms2t.shape[0], 1)) #OT FLAG
tParms2t = np.concatenate((tParms2t, tflags2t), axis=1)
'''
tParms3t = np.array([])
badotFile = fotFile.replace('fotimg', 'badimg2')
tParms3, obsUtc3 = self.doClassifyFile(subImgPath, badotFile)
if tParms3.shape[0]>0:
tParms3t = tParms3[(tParms3[:,6]<maxMEllip) & (tParms3[:,15]>=prob)]
if tParms3t.shape[0]>0:
tflags3t = np.zeros((tParms3t.shape[0],1)) #OT FLAG
tParms3t = np.concatenate((tParms3t, tflags3t), axis=1)
'''
print("classify result: %d tot(%d), %d fot(%d)" %
(tParms1t.shape[0], tParms1.shape[0], tParms2t.shape[0],
tParms2.shape[0]))
#print("doClassifyAndUpload 003")
if tParms1t.shape[0] > 0 and tParms1t.shape[
0] < 100 and tParms2t.shape[0] > 0 and tParms2t.shape[
0] < 100:
tParms = np.concatenate((tParms1t, tParms2t), axis=0)
elif tParms1t.shape[0] > 0 and tParms1t.shape[0] < 100:
tParms = tParms1t
elif tParms2t.shape[0] > 0 and tParms2t.shape[0] < 100:
tParms = tParms2t
else:
tParms = np.array([])
#print("doClassifyAndUpload 004")
if tParms.shape[0] == 0:
#print("doClassifyAndUpload 005")
self.log.info("after classified, no OT candidate left")
elif tParms.shape[0] > 0:
#print("doClassifyAndUpload 006")
tSubImgs, tParms = getWindowImgs(fullImgPath, newImg, tmpImg,
resImg, tParms, 100)
if tParms.shape[0] > 0:
self.log.info(
"after classified, %s total get %d sub images" %
(origName, tSubImgs.shape[0]))
#print("after classified, %s total get %d sub images"%(origName, tSubImgs.shape[0]))
i = 1
timgNames = []
for timg in tSubImgs:
objWid, tmpWid, resiWid = timg[0], timg[1], timg[2]
objWidz = zscale_image(objWid)
tmpWidz = zscale_image(tmpWid)
resiWidz = zscale_image(resiWid)
objWidz = scipy.ndimage.zoom(objWidz, 2, order=0)
tmpWidz = scipy.ndimage.zoom(tmpWidz, 2, order=0)
resiWidz = scipy.ndimage.zoom(resiWidz, 2, order=0)
objWidz = objWidz.reshape(objWidz.shape[0],
objWidz.shape[1],
1).repeat(3, 2)
tmpWidz = tmpWidz.reshape(tmpWidz.shape[0],
tmpWidz.shape[1],
1).repeat(3, 2)
resiWidz = resiWidz.reshape(resiWidz.shape[0],
resiWidz.shape[1],
1).repeat(3, 2)
shift00 = 3
cv2.circle(objWidz,
(int(objWidz.shape[0] / 2) - shift00,
int(objWidz.shape[1] / 2) - shift00),
10, (0, 255, 0),
thickness=2)
cv2.circle(tmpWidz,
(int(tmpWidz.shape[0] / 2) - shift00,
int(tmpWidz.shape[1] / 2) - shift00),
10, (0, 255, 0),
thickness=2)
cv2.circle(resiWidz,
(int(resiWidz.shape[0] / 2) - shift00,
int(resiWidz.shape[1] / 2) - shift00),
10, (0, 255, 0),
thickness=2)
spaceLine = np.ones(
(objWidz.shape[0], 5), dtype=np.uint8) * 255
spaceLine = spaceLine.reshape(spaceLine.shape[0],
spaceLine.shape[1],
1).repeat(3, 2)
sub2Con = np.concatenate(
(objWidz, spaceLine, tmpWidz, spaceLine, resiWidz),
axis=1)
tstrs = tmpImg.split(
'_') #G044_mon_objt_190128T10264248.fit
if len(tstrs) == 4:
tmpDateTime = tstrs[3]
cv2.putText(
sub2Con, #numpy array on which text is written
tmpDateTime[:tmpDateTime.index('.')], #text
(objWidz.shape[1] + 25, objWidz.shape[0] -
10), #position at which writing has to start
cv2.FONT_HERSHEY_SIMPLEX, #font family
0.5, #font size
(0, 255, 0), #font color
2)
tImgName = "%s_%05d.jpg" % (nameBase, i)
timgNames.append(tImgName)
savePath = "%s/%s" % (fullImgPath, tImgName)
Image.fromarray(sub2Con).save(savePath)
i = i + 1
catName = tImgName = "%s.cat" % (nameBase)
catPath = "%s/%s" % (fullImgPath, catName)
fp0 = open(catPath, 'w')
fp0.write(self.theader2)
t2oX = tcatParm[-2]
t2oY = tcatParm[-1]
tx = tParms[:, 0]
ty = tParms[:, 1]
ox = t2oX(tx, ty)
oy = t2oY(tx, ty)
i = 0
for td in tParms:
#print(td)
#print(td[15])
tstr = self.catFormate%\
(ox[i],oy[i],td[2],td[3],td[4],td[5],td[6],td[7],td[8],td[9],td[10],td[11],td[12],td[13],
td[14],td[15],td[16], timgNames[i], obsUtc1, td[0],td[1])
fp0.write(tstr)
i = i + 1
fp0.close()
self.doUpload(fullImgPath, [catName], 'crossOTList',
serverIP, crossTaskName)
self.doUpload(fullImgPath, timgNames, 'crossOTStamp',
serverIP, crossTaskName)
if tParms1.shape[0] >= 100 or tParms2.shape[0] >= 100:
self.log.error(
"too more unmatched or matched OT candidate, skip upload matched to db: after classified, %s total get %d matchend sub images"
% (origName, tParms2.shape[0]))
os.system("rm -rf %s" % (fullImgPath))
except Exception as e:
self.log.error('classifyAndUpload error')
self.log.error(str(e))
tstr = traceback.format_exc()
self.log.error(tstr)
endtime = datetime.now()
runTime = (endtime - starttime).seconds
self.log.info("********** classifyAndUpload %s use %d seconds" %
(origName, runTime))
def test():
tpath2 = "/home/xy/Downloads/myresource/deep_data2/gwac_ot2_apart"
tpath3 = "/home/xy/Downloads/myresource/deep_data2/simot/rest_data_0929"
dateStr = datetime.strftime(datetime.now(), "%Y%m%d")
workPath = "/home/xy/Downloads/myresource/deep_data2/simot/train_%s" % (
dateStr)
if not os.path.exists(workPath):
os.system("mkdir %s" % (workPath))
print("work path is %s" % (workPath))
X, Y, s2n = getData3(tpath3, tpath2, workPath)
print(X.shape)
print(Y.shape)
print(s2n.shape)
N_data = X.shape[0]
N_train = int(N_data * 0.9)
print("train: %d, test: %d" % (N_train, N_data - N_train))
X_train, Y_train, s2n_train = X[:N_train], Y[:N_train], s2n[:N_train]
X_test, Y_test, s2n_test = X[N_train:], Y[N_train:], s2n[N_train:]
from keras.models import load_model
model = load_model("%s/model_128_5_RealFOT.h5" % (workPath))
Y_pred = model.predict(X_test)
pbb_threshold = 0.5
pred_labels = np.array((Y_pred[:, 1] > pbb_threshold), dtype="int")
print("Correctly classified %d out of %d" %
((pred_labels == Y_test[:, 1].astype(int)).sum(), Y_test.shape[0]))
print("accuracy = %f" %
(1. *
(pred_labels == Y_test[:, 1].astype(int)).sum() / Y_test.shape[0]))
TIdx = Y_test[:, 1] == 1
FIdx = Y_test[:, 1] == 0
T_pred_rst = pred_labels[TIdx]
F_pred_rst = pred_labels[FIdx]
print(T_pred_rst.shape)
print(F_pred_rst.shape)
TP = ((T_pred_rst == 1).astype(int)).sum()
TN = ((F_pred_rst == 0).astype(int)).sum()
FP = ((F_pred_rst == 1).astype(int)).sum()
FN = ((T_pred_rst == 0).astype(int)).sum()
print("total=%d,TP=%d,TN=%d,FP=%d,FN=%d" %
(Y_test.shape[0], TP, TN, FP, FN))
accuracy = (TP + TN) * 1.0 / (TP + FN + TN + FP)
precision = (TP) * 1.0 / (TP + FP)
recall = (TP) * 1.0 / (TP + FN)
f1_score = 2.0 * (precision * recall) / (precision + recall)
print("accuracy=%f%%" % (accuracy * 100))
print("precision=%f%%" % (precision * 100))
print("recall=%f%%" % (recall * 100))
print("f1_score=%f%%" % (f1_score * 100))
falseImg = X_test[pred_labels != Y_test[:, 1]]
falseLabel = Y_test[pred_labels != Y_test[:, 1]]
falsePred = Y_pred[pred_labels != Y_test[:, 1]]
falseS2n = s2n_test[pred_labels != Y_test[:, 1]]
tnum1 = 0
tnum2 = 0
for i in range(falseImg.shape[0]):
if falseLabel[i, 1] == 1:
tnum1 = tnum1 + 1
else:
tnum2 = tnum2 + 1
print("total %d, miss classified %d" %
(Y_test.shape[0], falseImg.shape[0]))
print("True classified as False %d" % (tnum1))
print("False classified as True %d" % (tnum2))
print("\n\n***********************")
print("image of True classified as False")
for i in range(falseImg.shape[0]):
if falseLabel[i, 1] == 1:
objWidz = zscale_image(falseImg[i][0])
tmpWidz = zscale_image(falseImg[i][1])
resiWidz = zscale_image(falseImg[i][2])
if objWidz.shape[0] == 0:
objWidz = falseImg[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = falseImg[i][1]
if resiWidz.shape[0] == 0:
resiWidz = falseImg[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(3, 3))
axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray')
axes.flat[2].set_title("predicted pbb = " +
str(np.round(falsePred[i][1], 2)) +
", label = " + str(falseLabel[i, 1]) +
", s2n = " + str(falseS2n[i]))
plt.show()
print("\n\n***********************")
print("image of False classified as True")
for i in range(falseImg.shape[0]):
if falseLabel[i, 1] == 0:
objWidz = zscale_image(falseImg[i][0])
tmpWidz = zscale_image(falseImg[i][1])
resiWidz = zscale_image(falseImg[i][2])
if objWidz.shape[0] == 0:
objWidz = falseImg[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = falseImg[i][1]
if resiWidz.shape[0] == 0:
resiWidz = falseImg[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(3, 3))
axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray')
axes.flat[2].set_title("predicted pbb = " +
str(np.round(falsePred[i][1], 2)) +
", label = " + str(falseLabel[i, 1]) +
", s2n = " + str(falseS2n[i]))
plt.show()
trueImg = X_test[pred_labels == Y_test[:, 1]]
trueLabel = Y_test[pred_labels == Y_test[:, 1]]
truePred = Y_pred[pred_labels == Y_test[:, 1]]
trueS2n = s2n_test[pred_labels == Y_test[:, 1]]
showNum = 50
tnum = 0
print("\n\n***********************")
print("image of True classified as True")
for i in range(trueImg.shape[0]):
if trueLabel[i, 1] == 1:
objWidz = zscale_image(trueImg[i][0])
tmpWidz = zscale_image(trueImg[i][1])
resiWidz = zscale_image(trueImg[i][2])
if objWidz.shape[0] == 0:
objWidz = falseImg[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = falseImg[i][1]
if resiWidz.shape[0] == 0:
resiWidz = falseImg[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(3, 3))
axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray')
axes.flat[2].set_title("predicted pbb = " +
str(np.round(truePred[i][1], 2)) +
", label = " + str(trueLabel[i, 1]) +
", s2n = " + str(trueS2n[i]))
plt.show()
tnum = tnum + 1
if tnum > showNum:
break
tnum = 0
print("\n\n***********************")
print("image of False classified as False")
for i in range(trueImg.shape[0]):
if trueLabel[i, 1] == 0:
objWidz = zscale_image(trueImg[i][0])
tmpWidz = zscale_image(trueImg[i][1])
resiWidz = zscale_image(trueImg[i][2])
if objWidz.shape[0] == 0:
objWidz = falseImg[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = falseImg[i][1]
if resiWidz.shape[0] == 0:
resiWidz = falseImg[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(3, 3))
axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray')
axes.flat[2].set_title("predicted pbb = " +
str(np.round(truePred[i][1], 2)) +
", label = " + str(trueLabel[i, 1]) +
", s2n = " + str(trueS2n[i]))
plt.show()
tnum = tnum + 1
if tnum > showNum:
break
def outMisMatch(workPath, imgsFile, destName, tModelNamePart):
dataPath = "%s/data" % (workPath)
tpath = "%s/%s.npz" % (dataPath, imgsFile)
print(tpath)
destPath = "%s/%s/%s" % (dataPath, destName, imgsFile)
if not os.path.exists(destPath):
os.system("mkdir -p %s" % (destPath))
destPath0 = "%s/0" % (destPath)
if not os.path.exists(destPath0):
os.system("mkdir %s" % (destPath0))
destPath1 = "%s/1" % (destPath)
if not os.path.exists(destPath1):
os.system("mkdir %s" % (destPath1))
tdata = np.load(tpath)
X_test = tdata['X']
Y_test = tdata['Y']
s2n_test = tdata['s2n']
print(X_test.shape)
print(Y_test.shape)
print(s2n_test.shape)
K.set_image_data_format('channels_first')
modelName = "model_80w_20190403_dropout_train11_09.h5"
model = load_model("%s/%s" % (workPath, modelName))
Y_pred = model.predict(X_test)
pbb_threshold = 0.5
pred_labels = np.array((Y_pred[:, 1] >= pbb_threshold), dtype="int")
print("Correctly classified %d out of %d" %
((pred_labels == Y_test[:, 1].astype(int)).sum(), Y_test.shape[0]))
print("accuracy = %f" %
(1. *
(pred_labels == Y_test[:, 1].astype(int)).sum() / Y_test.shape[0]))
TIdx = Y_test[:, 1] == 1
FIdx = Y_test[:, 1] == 0
T_pred_rst = pred_labels[TIdx]
F_pred_rst = pred_labels[FIdx]
print(T_pred_rst.shape)
print(F_pred_rst.shape)
TP = ((T_pred_rst == 1).astype(int)).sum()
TN = ((F_pred_rst == 0).astype(int)).sum()
FP = ((F_pred_rst == 1).astype(int)).sum()
FN = ((T_pred_rst == 0).astype(int)).sum()
print("total=%d,TP=%d,TN=%d,FP=%d,FN=%d" %
(Y_test.shape[0], TP, TN, FP, FN))
for i in range(X_test.shape[0]):
try:
ty = Y_test[i][1]
if ty != pred_labels[i]:
objWidz = zscale_image(X_test[i][0])
tmpWidz = zscale_image(X_test[i][1])
resiWidz = zscale_image(X_test[i][2])
if objWidz.shape[0] == 0:
objWidz = X_test[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = X_test[i][1]
if resiWidz.shape[0] == 0:
resiWidz = X_test[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(6, 2))
axes.flat[0].imshow(objWidz,
interpolation="nearest",
cmap='gray')
axes.flat[1].imshow(tmpWidz,
interpolation="nearest",
cmap='gray')
axes.flat[2].imshow(resiWidz,
interpolation="nearest",
cmap='gray')
axes.flat[1].set_title("pbb = " +
str(np.round(Y_pred[i][1], 2)) +
", label = " + str(Y_pred[i, 1]) +
", s2n = " + str(s2n_test[i]))
#plt.show()
savePath = "%s/%d/%06d.jpg" % (destPath, ty, i)
plt.savefig(savePath, bbox_inches='tight')
#break
except Exception as e:
tstr = traceback.format_exc()
print(tstr)
def test(totpath, fotpath, workPath, tSampleNamePart, tModelNamePart):
X, Y, s2n = getData2(totpath, fotpath, workPath, tSampleNamePart)
print(X.shape)
print(Y.shape)
print(s2n.shape)
N_data = X.shape[0]
N_train = int(N_data * 0.9)
print("train: %d, test: %d" % (N_train, N_data - N_train))
X_train, Y_train, s2n_train = X[:N_train], Y[:N_train], s2n[:N_train]
X_test, Y_test, s2n_test = X[N_train:], Y[N_train:], s2n[N_train:]
from keras.models import load_model
modelName = "model_RealFOT_%s.h5" % (tModelNamePart)
model = load_model("%s/%s" % (workPath, modelName))
Y_pred = model.predict(X_test)
pbb_threshold = 0.5
pred_labels = np.array((Y_pred[:, 1] > pbb_threshold), dtype="int")
print("Correctly classified %d out of %d" %
((pred_labels == Y_test[:, 1].astype(int)).sum(), Y_test.shape[0]))
print("accuracy = %f" %
(1. *
(pred_labels == Y_test[:, 1].astype(int)).sum() / Y_test.shape[0]))
TIdx = Y_test[:, 1] == 1
FIdx = Y_test[:, 1] == 0
T_pred_rst = pred_labels[TIdx]
F_pred_rst = pred_labels[FIdx]
print(T_pred_rst.shape)
print(F_pred_rst.shape)
TP = ((T_pred_rst == 1).astype(int)).sum()
TN = ((F_pred_rst == 0).astype(int)).sum()
FP = ((F_pred_rst == 1).astype(int)).sum()
FN = ((T_pred_rst == 0).astype(int)).sum()
print("total=%d,TP=%d,TN=%d,FP=%d,FN=%d" %
(Y_test.shape[0], TP, TN, FP, FN))
accuracy = (TP + TN) * 1.0 / (TP + FN + TN + FP)
precision = (TP) * 1.0 / (TP + FP)
recall = (TP) * 1.0 / (TP + FN)
f1_score = 2.0 * (precision * recall) / (precision + recall)
print("accuracy=%f%%" % (accuracy * 100))
print("precision=%f%%" % (precision * 100))
print("recall=%f%%" % (recall * 100))
print("f1_score=%f%%" % (f1_score * 100))
falseImg = X_test[pred_labels != Y_test[:, 1]]
falseLabel = Y_test[pred_labels != Y_test[:, 1]]
falsePred = Y_pred[pred_labels != Y_test[:, 1]]
falseS2n = s2n_test[pred_labels != Y_test[:, 1]]
tnum1 = 0
tnum2 = 0
for i in range(falseImg.shape[0]):
if falseLabel[i, 1] == 1:
tnum1 = tnum1 + 1
else:
tnum2 = tnum2 + 1
print("total %d, miss classified %d" %
(Y_test.shape[0], falseImg.shape[0]))
print("True classified as False %d" % (tnum1))
print("False classified as True %d" % (tnum2))
print("\n\n***********************")
print("image of True classified as False")
for i in range(falseImg.shape[0]):
if falseLabel[i, 1] == 1:
objWidz = zscale_image(falseImg[i][0])
tmpWidz = zscale_image(falseImg[i][1])
resiWidz = zscale_image(falseImg[i][2])
if objWidz.shape[0] == 0:
objWidz = falseImg[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = falseImg[i][1]
if resiWidz.shape[0] == 0:
resiWidz = falseImg[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(6, 2))
axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray')
axes.flat[2].set_title("predicted pbb = " +
str(np.round(falsePred[i][1], 2)) +
", label = " + str(falseLabel[i, 1]) +
", s2n = " + str(falseS2n[i]))
plt.show()
print("\n\n***********************")
print("image of False classified as True")
for i in range(falseImg.shape[0]):
if falseLabel[i, 1] == 0:
objWidz = zscale_image(falseImg[i][0])
tmpWidz = zscale_image(falseImg[i][1])
resiWidz = zscale_image(falseImg[i][2])
if objWidz.shape[0] == 0:
objWidz = falseImg[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = falseImg[i][1]
if resiWidz.shape[0] == 0:
resiWidz = falseImg[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(6, 2))
axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray')
axes.flat[2].set_title("predicted pbb = " +
str(np.round(falsePred[i][1], 2)) +
", label = " + str(falseLabel[i, 1]) +
", s2n = " + str(falseS2n[i]))
plt.show()
trueImg = X_test[pred_labels == Y_test[:, 1]]
trueLabel = Y_test[pred_labels == Y_test[:, 1]]
truePred = Y_pred[pred_labels == Y_test[:, 1]]
trueS2n = s2n_test[pred_labels == Y_test[:, 1]]
showNum = 50
tnum = 0
print("\n\n***********************")
print("image of True classified as True")
for i in range(trueImg.shape[0]):
if trueLabel[i, 1] == 1:
objWidz = zscale_image(trueImg[i][0])
tmpWidz = zscale_image(trueImg[i][1])
resiWidz = zscale_image(trueImg[i][2])
if objWidz.shape[0] == 0:
objWidz = falseImg[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = falseImg[i][1]
if resiWidz.shape[0] == 0:
resiWidz = falseImg[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(6, 2))
axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray')
axes.flat[2].set_title("predicted pbb = " +
str(np.round(truePred[i][1], 2)) +
", label = " + str(trueLabel[i, 1]) +
", s2n = " + str(trueS2n[i]))
plt.show()
tnum = tnum + 1
if tnum > showNum:
break
tnum = 0
print("\n\n***********************")
print("image of False classified as False")
for i in range(trueImg.shape[0]):
if trueLabel[i, 1] == 0:
objWidz = zscale_image(trueImg[i][0])
tmpWidz = zscale_image(trueImg[i][1])
resiWidz = zscale_image(trueImg[i][2])
if objWidz.shape[0] == 0:
objWidz = falseImg[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = falseImg[i][1]
if resiWidz.shape[0] == 0:
resiWidz = falseImg[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(6, 2))
axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray')
axes.flat[2].set_title("predicted pbb = " +
str(np.round(truePred[i][1], 2)) +
", label = " + str(trueLabel[i, 1]) +
", s2n = " + str(trueS2n[i]))
plt.show()
tnum = tnum + 1
if tnum > showNum:
break
def realDataTest(realOtPath, workPath, tSampleNamePart, tModelNamePart1,
tModelNamePart2):
X, Y, s2n = getRealData(realOtPath, workPath, tSampleNamePart)
subNum = 1000
X = X[:subNum]
Y = Y[:subNum]
s2n = s2n[:subNum]
print(X.shape)
print(Y.shape)
print(s2n.shape)
X_test, Y_test, s2n_test = X, Y, s2n
y = np.zeros(X_test.shape[0])
Y_test = np.array([np.logical_not(y), y]).transpose()
from keras.models import load_model
modelName1 = "model_RealFOT_%s.h5" % (tModelNamePart1)
model = load_model("%s/%s" % (workPath, modelName1))
Y_pred1 = model.predict(X_test)
modelName2 = "model_RealFOT_%s.h5" % (tModelNamePart2)
model = load_model("%s/%s" % (workPath, modelName2))
Y_pred2 = model.predict(X_test)
pbb_threshold = 0.5
pred_labels1 = np.array((Y_pred1[:, 1] > pbb_threshold), dtype="int")
pred_labels2 = np.array((Y_pred2[:, 1] > pbb_threshold), dtype="int")
model1True = pred_labels1[pred_labels1 == Y_test[:, 1]]
model2True = pred_labels2[pred_labels2 == Y_test[:, 1]]
print("model1 correct=%d" % (model1True.shape[0]))
print("model2 correct=%d" % (model2True.shape[0]))
model1True = pred_labels1[pred_labels1 == 1]
model2True = pred_labels2[pred_labels2 == 1]
print("model1 True=%d" % (model1True.shape[0]))
print("model2 True=%d" % (model2True.shape[0]))
diffIdx = pred_labels1 != pred_labels2
m1t = (pred_labels1 == 1) & diffIdx
m1f = (pred_labels1 == 0) & diffIdx
m2t = (pred_labels2 == 1) & diffIdx
m2f = (pred_labels2 == 0) & diffIdx
m1tImg = X_test[m1t]
m1fImg = X_test[m1f]
m2tImg = X_test[m2t]
m2fImg = X_test[m2f]
m1tS2n = s2n_test[m1t]
m1fS2n = s2n_test[m1f]
m2tS2n = s2n_test[m2t]
m2fS2n = s2n_test[m2f]
m1tPred = Y_pred1[m1t]
m1fPred = Y_pred1[m1f]
m2tPred = Y_pred2[m2t]
m2fPred = Y_pred2[m2f]
print("m1t=%d" % (m1tImg.shape[0]))
print("m1f=%d" % (m1fImg.shape[0]))
print("m2t=%d" % (m2tImg.shape[0]))
print("m2f=%d" % (m2fImg.shape[0]))
#totImgPath = 'totimg'
#for i, timg in enumerate(m1tImg):
# savePath = "%s/%05d.png"%(totImgPath, i)
showNum = 100
falseImg = m1tImg[:showNum]
falseS2n = m1tS2n[:showNum]
falsePred = m1tPred[:showNum]
print("\n\n***********************")
print("model1 True, model2 False: %d" % (falseImg.shape[0]))
for i in range(falseImg.shape[0]):
objWidz = zscale_image(falseImg[i][0])
tmpWidz = zscale_image(falseImg[i][1])
resiWidz = zscale_image(falseImg[i][2])
if objWidz.shape[0] == 0:
objWidz = falseImg[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = falseImg[i][1]
if resiWidz.shape[0] == 0:
resiWidz = falseImg[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(6, 2))
axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray')
axes.flat[1].set_title("pbb=%.2f,s2n=%.2f" %
(falsePred[i][1], falseS2n[i]))
plt.show()
falseImg = m1fImg[:showNum]
falseS2n = m1fS2n[:showNum]
falsePred = m1fPred[:showNum]
print("\n\n***********************")
print("model1 True, model2 False: %d" % (falseImg.shape[0]))
for i in range(falseImg.shape[0]):
objWidz = zscale_image(falseImg[i][0])
tmpWidz = zscale_image(falseImg[i][1])
resiWidz = zscale_image(falseImg[i][2])
if objWidz.shape[0] == 0:
objWidz = falseImg[i][0]
if tmpWidz.shape[0] == 0:
tmpWidz = falseImg[i][1]
if resiWidz.shape[0] == 0:
resiWidz = falseImg[i][2]
plt.clf()
fig, axes = plt.subplots(1, 3, figsize=(6, 2))
axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray')
axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray')
axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray')
axes.flat[1].set_title("pbb=%.2f,s2n=%.2f" %
(falsePred[i][1], falseS2n[i]))
plt.show()
def getRealData(destPath):
from gwac_util import zscale_image
ot2path = '/home/xy/Downloads/myresource/deep_data2/simot/ot2_img_collection_180906_20180907'
ot2listFile = '%s/list.txt' % (ot2path)
ot2list = np.genfromtxt(ot2listFile, dtype='str')
print(ot2list.shape)
print(ot2list[:3])
X = []
badImgNum = 0
badIdx = []
#timgbinPath = '%s/gwac_otimg_nozscale.npz'%(destPath)
timgbinPath = '%s/gwac_otimg.npz' % (destPath)
if os.path.exists(timgbinPath):
print("bin file exist, read from %s" % (timgbinPath))
timgbin = np.load(timgbinPath)
X = timgbin['ot']
ot2prop = timgbin['ot2prop']
else:
for i in range(ot2list.shape[0]):
tot2 = ot2list[i]
objPath = "%s/%s" % (ot2path, tot2[1])
refPath = "%s/%s" % (ot2path, tot2[2])
diffPath = "%s/%s" % (ot2path, tot2[3])
objImg = readStampFromFits(objPath)
refImg = readStampFromFits(refPath)
diffImg = readStampFromFits(diffPath)
if objImg.shape[0] == 0 or refImg.shape[0] == 0 or diffImg.shape[
0] == 0:
continue
objImgz = zscale_image(objImg)
refImgz = zscale_image(refImg)
diffImgz = zscale_image(diffImg)
'''
objImgz = objImg
refImgz = refImg
diffImgz = diffImg
'''
#异常残差图像处理，如果scale失败：1）等于原diffImg；2）直接量化到255
if diffImgz.shape[0] != 12 or diffImgz.shape[1] != 12:
print("%d, %s" % (i, tot2[1]))
badIdx.append(i)
tmin = np.min(diffImg)
tmax = np.max(diffImg)
diffImgz = (((diffImg - tmin) / (tmax - tmin)) * 255).astype(
np.uint8)
#diffImgz = diffImg
badImgNum = badImgNum + 1
X.append([objImgz, refImgz, diffImgz])
X = np.array(X)
'''
for i in range(X.shape[0]):
tot2 = ot2list[i]
tlabel = tot2[4]
ttype = tot2[5]
if ttype=='2':
fig, axes = plt.subplots(1, 3, figsize=(3, 1))
axes.flat[0].imshow(X[i][0], cmap='gray')
axes.flat[1].imshow(X[i][1], cmap='gray')
axes.flat[2].imshow(X[i][2], cmap='gray')
axes.flat[1].set_title("%d, %s, look=%s, type=%s"%(i, tot2[1][:14], tlabel, ttype))
plt.show()
'''
ot2prop = ot2list
print("save bin fiel to %s" % (timgbinPath))
np.savez_compressed(timgbinPath, ot=X, ot2prop=ot2prop)
print("bad image %d" % (badImgNum))
return X, ot2prop
