Esempio n. 1
0
    def diffImage(self, imgName, tmplParms):

        starttime = datetime.now()

        resultFlag = True
        oImgPre = imgName.split(".")[0]
        dateStr = imgName.split('_')[3][:6]

        os.system("rm -rf %s/*" % (self.diff))

        status = tmplParms[0]
        #tmplImgName = tmplParms[1][-1][0]
        tmplImgName = tmplParms[1][0][0]
        tmplImgPre = tmplImgName.split(".")[0]
        if status == '1':
            ttmplPath = self.tmplAlignDir
        else:
            ttmplPath = self.tmplDiffDir

        ofitPath = "%s/%s.fit" % (self.doDiffTmpl, tmplImgPre)
        if not os.path.exists(ofitPath):
            os.system("rm -rf %s/*" % (self.doDiffTmpl))
            os.system("cp %s/%s.fit %s/%s.fit" %
                      (ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre))
            os.system("cp %s/%s.cat %s/%s.cat" %
                      (ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre))
            os.system("cp %s/%s_badpix.cat %s/%s_badpix.cat" %
                      (ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre))

        os.system("cp %s/%s.fit %s/ti.fit" %
                  (self.doDiffTmpl, tmplImgPre, self.diff))
        os.system("cp %s/%s.cat %s/ti.cat" %
                  (self.doDiffTmpl, tmplImgPre, self.diff))
        os.system("cp %s/%s_badpix.cat %s/ti_badpix.cat" %
                  (self.doDiffTmpl, tmplImgPre, self.diff))

        os.system("cp %s/%s_align.fit %s/oi.fit" %
                  (self.alignDir, oImgPre, self.diff))
        os.system("cp %s/%s_align.cat %s/oi.cat" %
                  (self.alignDir, oImgPre, self.diff))

        theader = fits.getheader("%s/oi.fit" % (self.diff))
        dateObs = theader['DATE-OBS']
        timeObs = theader['TIME-OBS']
        dtStr = "%sT%s" % (dateObs, timeObs)

        objTmpResi, runSuccess = self.tools.runHotpants(
            'oi.fit', 'ti.fit', self.diff)
        if not runSuccess:
            self.log.error("diffImage failure: %s" % (imgName))
            return False

        fpar = 'sex_diff.par'
        #sexConf=['-DETECT_MINAREA','3','-DETECT_THRESH','2.5','-ANALYSIS_THRESH','2.5']
        sexConf = [
            '-DETECT_MINAREA', '5', '-DETECT_THRESH', '2.5',
            '-ANALYSIS_THRESH', '2.5'
        ]
        resiCat, isSuccess = self.tools.runSextractor(objTmpResi, self.diff,
                                                      self.diff, fpar, sexConf)
        if not isSuccess:
            self.log.error("diffImage runSextractor failure")
            return isSuccess

        mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
            self.diff, resiCat, 'oi.cat', 1)  #1 and 5

        badPix2Path = "%s/%s" % (self.diff, nmhFile)
        if os.path.exists(badPix2Path):
            badPixProps2 = np.loadtxt(badPix2Path)
        else:
            badPixProps2 = np.array([])

        mchRadius = 10  #15 10
        mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
            self.diff, mchFile, 'ti.cat', mchRadius)

        fotPath = "%s/%s" % (self.diff, mchFile)
        if os.path.exists(fotPath):
            fotProps = np.loadtxt(fotPath)
        else:
            fotProps = np.array([])

        mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
            self.diff, nmhFile, 'ti_badpix.cat', 1)  #1 and 5
        os.system("cp %s/%s %s/%s_tot.cat" %
                  (self.diff, nmhFile, self.diffCatDir, oImgPre))

        totPath = "%s/%s" % (self.diff, nmhFile)
        if os.path.exists(totPath):
            totProps = np.loadtxt(totPath)
        else:
            totProps = np.array([])

        badPixPath = "%s/ti_badpix.cat" % (self.diff)
        if os.path.exists(badPixPath):
            badPixProps = np.loadtxt(badPixPath)
        else:
            badPixProps = np.array([])

        #badPixProps = np.array([])
        tstr = "orgBadPix %d, nmBad %d, match %d, noMatch %d" % (
            badPixProps.shape[0], badPixProps2.shape[0], fotProps.shape[0],
            totProps.shape[0])
        self.log.info(tstr)
        print(tstr)

        #size = self.subImgSize
        size = 68
        if totProps.shape[0] < 500 and totProps.shape[0] > 0:

            wcsPath = "%s/%s.wcs" % (ttmplPath, tmplImgPre)
            wcs = WCS(wcsPath)

            searchRadius = 0.05
            mpQuery = QueryMinorPlanet()
            mpQuery.connDb()
            totSubImgs, totParms = getWindowImgs(self.diff, 'oi.fit', 'ti.fit',
                                                 objTmpResi, totProps, size)
            if totParms.shape[0] > 0:
                tXY = totParms[:, 0:2]
                #tdates = np.repeat(dtStr,tXY.shape[0]).reshape((tXY.shape[0],1))
                tRaDec = wcs.all_pix2world(tXY, 1)
                totParms = np.concatenate((totParms, tRaDec), axis=1)

                mchDis = []
                for tpos in tRaDec:
                    ra = tpos[0]
                    dec = tpos[1]
                    tdis, tmag = mpQuery.matchMP(ra, dec, dateStr, dtStr,
                                                 searchRadius + 0.001)
                    mchDis.append([tdis, tmag])

                mchDis = np.array(mchDis)
                totParms = np.concatenate((totParms, mchDis), axis=1)

                totSubImgs1 = totSubImgs[mchDis[:, 0] >= searchRadius]
                totParms1 = totParms[mchDis[:, 0] >= searchRadius]
                mpSubImgs1 = totSubImgs[mchDis[:, 0] < searchRadius]
                mpParms1 = totParms[mchDis[:, 0] < searchRadius]

                tstr0 = "%s has %d tot, contain %d minorplanet." % (
                    imgName, totParms.shape[0], mpParms1.shape[0])
                print(tstr0)
                self.log.info(tstr0)

                if totSubImgs1.shape[0] > 0:
                    totpath = '%s/%s_totimg.npz' % (self.destDir, oImgPre)
                    np.savez_compressed(totpath,
                                        imgs=totSubImgs1,
                                        parms=totParms1,
                                        obsUtc=dtStr)
                    resiImgs = []
                    for timg in totSubImgs1:
                        resiImgs.append(timg[2])

                    preViewPath = "%s/%s_tot.jpg" % (self.subImgViewDir,
                                                     oImgPre)
                    #if not os.path.exists(preViewPath):
                    psfView = genPSFView(resiImgs)
                    Image.fromarray(psfView).save(preViewPath)

                if mpSubImgs1.shape[0] > 0:
                    self.sendMsg(tstr0)
                    totpath = '%s/%s_mpimgt.npz' % (self.destDir, oImgPre)
                    np.savez_compressed(totpath,
                                        imgs=mpSubImgs1,
                                        parms=mpParms1,
                                        obsUtc=dtStr)
                    resiImgs = []
                    for timg in mpSubImgs1:
                        resiImgs.append(timg[2])
                    preViewPath = "%s/%s_mpt.jpg" % (self.subImgViewDir,
                                                     oImgPre)
                    #if not os.path.exists(preViewPath):
                    psfView = genPSFView(resiImgs)
                    Image.fromarray(psfView).save(preViewPath)

                    os.system(
                        "cp %s/%s %s/%s_resi.fit" %
                        (self.diff, objTmpResi, self.diffImgDir, oImgPre))
                    os.system("cp %s/%s %s/%s_resi.cat" %
                              (self.diff, resiCat, self.diffCatDir, oImgPre))

                else:

                    alignImgName = "%s/%s_align.fit" % (self.alignDir, oImgPre)
                    os.system("rm -rf %s" % (alignImgName))
                    alignImgName = "%s/%s_align.cat" % (self.alignDir, oImgPre)
                    os.system("rm -rf %s" % (alignImgName))
                    alignImgName = "%s/%s.cat" % (self.catDir, oImgPre)
                    os.system("rm -rf %s" % (alignImgName))

            if fotProps.shape[0] > 0 and fotProps.shape[0] < 500:
                fotSubImgs, fotParms = getWindowImgs(self.diff, 'oi.fit',
                                                     'ti.fit', objTmpResi,
                                                     fotProps, size)
                if fotParms.shape[0] > 0:
                    tXY = fotParms[:, 0:2]
                    #tdates = np.repeat(dtStr,tXY.shape[0]).reshape((tXY.shape[0],1))
                    tRaDec = wcs.all_pix2world(tXY, 1)
                    fotParms = np.concatenate((fotParms, tRaDec), axis=1)

                    if fotSubImgs.shape[0] > 0:
                        totpath = '%s/%s_fotimg.npz' % (self.destDir, oImgPre)
                        np.savez_compressed(totpath,
                                            imgs=fotSubImgs,
                                            parms=fotParms,
                                            obsUtc=dtStr)

            mpQuery.closeDb()
            resultFlag = True
        else:
            tmsgStr = "%s.fit resi image has %d tot objects, maybe wrong" % (
                oImgPre, totProps.shape[0])
            self.log.error(tmsgStr)
            #self.sendMsg(tmsgStr)
            resultFlag = False

        endtime = datetime.now()
        runTime = (endtime - starttime).seconds
        self.log.info("********** image diff total use %d seconds" % (runTime))

        return resultFlag
Esempio n. 2
0
    def doClassifyAndUpload(self,
                            subImgPath,
                            totFile,
                            fotFile,
                            fullImgPath,
                            newImg,
                            tmpImg,
                            resImg,
                            origName,
                            serverIP,
                            prob=0.01,
                            maxNEllip=0.6,
                            maxMEllip=0.5,
                            reverse=False):

        self.log.debug("start new thread classifyAndUpload %s" % (origName))

        starttime = datetime.now()
        self.pbb_threshold = prob
        try:
            nameBase = origName[:origName.index(".")]

            tParms1 = self.doClassifyFile(subImgPath, totFile)
            if tParms1.shape[0] > 0:
                tParms1 = tParms1[tParms1[:, 6] < maxNEllip]
                #tParms1 = tParms1[(tParms1[:,6]<maxMEllip) & (tParms1[:,17]>=prob)]
                if tParms1.shape[0] > 0:
                    tflags1 = np.ones((tParms1.shape[0], 1))  #OT FLAG
                    tParms1 = np.concatenate((tParms1, tflags1), axis=1)

            tParms2 = self.doClassifyFile(subImgPath, fotFile)
            if tParms2.shape[0] > 0:
                tParms2 = tParms2[(tParms2[:, 6] < maxMEllip)
                                  & (tParms2[:, 17] >= prob)]
                if tParms2.shape[0] > 0:
                    tflags2 = np.zeros((tParms2.shape[0], 1))  #OT FLAG
                    tParms2 = np.concatenate((tParms2, tflags2), axis=1)

            if tParms1.shape[0] > 0 and tParms2.shape[0] > 0 and tParms2.shape[
                    0] < 50:
                tParms = np.concatenate((tParms1, tParms2), axis=0)
            elif tParms1.shape[0] > 0:
                tParms = tParms1
            elif tParms2.shape[0] > 0 and tParms2.shape[0] < 25:
                tParms = tParms2
            else:
                tParms = np.array([])

            if tParms.shape[0] > 0:
                tSubImgs, tParms = getWindowImgs(fullImgPath, newImg, tmpImg,
                                                 resImg, tParms, 100)
                if tParms.shape[0] > 0:
                    self.log.info(
                        "after classified, %s total get %d sub images" %
                        (origName, tSubImgs.shape[0]))

                    i = 1
                    timgNames = []
                    for timg in tSubImgs:
                        objWid, tmpWid, resiWid = timg[0], timg[1], timg[2]

                        objWidz = zscale_image(objWid)
                        tmpWidz = zscale_image(tmpWid)
                        resiWidz = zscale_image(resiWid)
                        objWidz = scipy.ndimage.zoom(objWidz, 2, order=0)
                        tmpWidz = scipy.ndimage.zoom(tmpWidz, 2, order=0)
                        resiWidz = scipy.ndimage.zoom(resiWidz, 2, order=0)

                        objWidz = objWidz.reshape(objWidz.shape[0],
                                                  objWidz.shape[1],
                                                  1).repeat(3, 2)
                        tmpWidz = tmpWidz.reshape(tmpWidz.shape[0],
                                                  tmpWidz.shape[1],
                                                  1).repeat(3, 2)
                        resiWidz = resiWidz.reshape(resiWidz.shape[0],
                                                    resiWidz.shape[1],
                                                    1).repeat(3, 2)

                        shift00 = 3
                        cv2.circle(objWidz,
                                   (int(objWidz.shape[0] / 2) - shift00,
                                    int(objWidz.shape[1] / 2) - shift00),
                                   10, (0, 255, 0),
                                   thickness=2)
                        cv2.circle(tmpWidz,
                                   (int(tmpWidz.shape[0] / 2) - shift00,
                                    int(tmpWidz.shape[1] / 2) - shift00),
                                   10, (0, 255, 0),
                                   thickness=2)
                        cv2.circle(resiWidz,
                                   (int(resiWidz.shape[0] / 2) - shift00,
                                    int(resiWidz.shape[1] / 2) - shift00),
                                   10, (0, 255, 0),
                                   thickness=2)

                        spaceLine = np.ones(
                            (objWidz.shape[0], 5), dtype=np.uint8) * 255
                        spaceLine = spaceLine.reshape(spaceLine.shape[0],
                                                      spaceLine.shape[1],
                                                      1).repeat(3, 2)
                        sub2Con = np.concatenate(
                            (objWidz, spaceLine, tmpWidz, spaceLine, resiWidz),
                            axis=1)

                        tImgName = "%s_%05d.jpg" % (nameBase, i)
                        timgNames.append(tImgName)
                        savePath = "%s/%s" % (fullImgPath, tImgName)
                        Image.fromarray(sub2Con).save(savePath)
                        i = i + 1

                    catName = tImgName = "%s.cat" % (nameBase)
                    catPath = "%s/%s" % (fullImgPath, catName)
                    fp0 = open(catPath, 'w')
                    #fp0.write(self.theader)
                    i = 0
                    for td in tParms:
                        tstr = "%.4f,%.4f,%.2f,%.2f,%.2f,%.3f,%.3f,%.3f,%.2f,%.2f,%d,%.4f,%.4f,%.4f,%.4f,%f,%f,%.3f,%d,%s\n"%\
                            (td[0],td[1],td[2],td[3],td[4],td[5],td[6],td[7],td[8],td[9],td[10],td[11],td[12],td[13],
                             td[14],td[15],td[16],td[17],td[18], timgNames[i])
                        fp0.write(tstr)
                        i = i + 1
                    fp0.close()

                    self.doUpload(fullImgPath, [catName], 'diffot1', serverIP)
                    self.doUpload(fullImgPath, timgNames, 'diffot1img',
                                  serverIP)

            if tParms.shape[0] == 0:
                self.log.info("after classified, no OT candidate left")
            if tParms2.shape[0] >= 50:
                self.log.error(
                    "too more matched OT candidate, skip upload matched to db: after classified, %s total get %d matchend sub images"
                    % (origName, tParms2.shape[0]))
            os.system("rm -rf %s" % (fullImgPath))

        except Exception as e:
            self.log.error('classifyAndUpload error')
            self.log.error(str(e))
            tstr = traceback.format_exc()
            self.log.error(tstr)
        endtime = datetime.now()
        runTime = (endtime - starttime).seconds
        self.log.info("********** classifyAndUpload %s use %d seconds" %
                      (origName, runTime))
Esempio n. 3
0
    def doClassifyAndUpload(self,
                            subImgPath,
                            totFile,
                            fotFile,
                            fullImgPath,
                            newImg,
                            tmpImg,
                            resImg,
                            origName,
                            serverIP,
                            runName,
                            skyName,
                            tcatParm,
                            prob=0.0000001,
                            maxNEllip=0.6,
                            maxMEllip=0.5,
                            reverse=False):

        self.log.info("start new thread classifyAndUpload %s" % (origName))
        print("start new thread classifyAndUpload %s" % (origName))

        minFwhm = 1.5
        maxFwhm = 3.0

        starttime = datetime.now()
        self.pbb_threshold = prob
        try:
            nameBase = origName[:origName.index(".")]

            dateStr = nameBase.split('_')[3][:6]
            camName = nameBase[:4]
            cmbNum = 5
            #tidx = nameBase.index('_c')+2
            #cmbNum = nameBase[tidx:tidx+3] #'G021_tom_objt_190109T13531492_c005.fit'
            crossTaskName = "%s_%s_%s_C%03d" % (dateStr, camName, skyName,
                                                cmbNum)
            if runName != 'p1':
                crossTaskName = "%s_%s" % (crossTaskName, runName)
            #self.log.info("crossTaskName %s"%(crossTaskName))
            #print("crossTaskName %s"%(crossTaskName))

            tParms1t = np.array([])
            tParms2t = np.array([])

            tParms1, obsUtc1 = self.doClassifyFile(subImgPath, totFile)
            #print("doClassifyAndUpload 001")
            if tParms1.shape[0] > 0:
                tParms1t = tParms1[(tParms1[:, 6] < maxNEllip)
                                   & (tParms1[:, 9] < maxFwhm) &
                                   (tParms1[:, 9] > minFwhm)]
                #tParms1t = tParms1[(tParms1[:,6]<maxMEllip) & (tParms1[:,15]>=prob)]
                if tParms1t.shape[0] > 0:
                    tflags1t = np.ones((tParms1t.shape[0], 1))  #OT FLAG
                    tParms1t = np.concatenate((tParms1t, tflags1t), axis=1)

            #print("doClassifyAndUpload 002")
            tParms2, obsUtc2 = self.doClassifyFile(subImgPath, fotFile)
            if tParms2.shape[0] > 0:
                #tParms2t = tParms2[(tParms2[:,6]<maxMEllip) & (tParms2[:,15]>=prob)&(tParms2[:,9]<maxFwhm)&(tParms2[:,9]>minFwhm)]
                tParms2t = tParms2[(tParms2[:, 6] < maxMEllip)
                                   & (tParms2[:, 9] < maxFwhm) &
                                   (tParms2[:, 9] > minFwhm)]
                if tParms2t.shape[0] > 0:
                    tflags2t = np.zeros((tParms2t.shape[0], 1))  #OT FLAG
                    tParms2t = np.concatenate((tParms2t, tflags2t), axis=1)
            '''
            tParms3t = np.array([])
            badotFile = fotFile.replace('fotimg', 'badimg2')
            tParms3, obsUtc3 = self.doClassifyFile(subImgPath, badotFile)
            if tParms3.shape[0]>0:
                tParms3t = tParms3[(tParms3[:,6]<maxMEllip) & (tParms3[:,15]>=prob)]
                if tParms3t.shape[0]>0:
                    tflags3t = np.zeros((tParms3t.shape[0],1)) #OT FLAG 
                    tParms3t = np.concatenate((tParms3t, tflags3t), axis=1)
            '''

            print("classify result: %d tot(%d), %d fot(%d)" %
                  (tParms1t.shape[0], tParms1.shape[0], tParms2t.shape[0],
                   tParms2.shape[0]))

            #print("doClassifyAndUpload 003")
            if tParms1t.shape[0] > 0 and tParms1t.shape[
                    0] < 100 and tParms2t.shape[0] > 0 and tParms2t.shape[
                        0] < 100:
                tParms = np.concatenate((tParms1t, tParms2t), axis=0)
            elif tParms1t.shape[0] > 0 and tParms1t.shape[0] < 100:
                tParms = tParms1t
            elif tParms2t.shape[0] > 0 and tParms2t.shape[0] < 100:
                tParms = tParms2t
            else:
                tParms = np.array([])

            #print("doClassifyAndUpload 004")
            if tParms.shape[0] == 0:
                #print("doClassifyAndUpload 005")
                self.log.info("after classified, no OT candidate left")
            elif tParms.shape[0] > 0:
                #print("doClassifyAndUpload 006")
                tSubImgs, tParms = getWindowImgs(fullImgPath, newImg, tmpImg,
                                                 resImg, tParms, 100)
                if tParms.shape[0] > 0:
                    self.log.info(
                        "after classified, %s total get %d sub images" %
                        (origName, tSubImgs.shape[0]))
                    #print("after classified, %s total get %d sub images"%(origName, tSubImgs.shape[0]))

                    i = 1
                    timgNames = []
                    for timg in tSubImgs:
                        objWid, tmpWid, resiWid = timg[0], timg[1], timg[2]

                        objWidz = zscale_image(objWid)
                        tmpWidz = zscale_image(tmpWid)
                        resiWidz = zscale_image(resiWid)
                        objWidz = scipy.ndimage.zoom(objWidz, 2, order=0)
                        tmpWidz = scipy.ndimage.zoom(tmpWidz, 2, order=0)
                        resiWidz = scipy.ndimage.zoom(resiWidz, 2, order=0)

                        objWidz = objWidz.reshape(objWidz.shape[0],
                                                  objWidz.shape[1],
                                                  1).repeat(3, 2)
                        tmpWidz = tmpWidz.reshape(tmpWidz.shape[0],
                                                  tmpWidz.shape[1],
                                                  1).repeat(3, 2)
                        resiWidz = resiWidz.reshape(resiWidz.shape[0],
                                                    resiWidz.shape[1],
                                                    1).repeat(3, 2)

                        shift00 = 3
                        cv2.circle(objWidz,
                                   (int(objWidz.shape[0] / 2) - shift00,
                                    int(objWidz.shape[1] / 2) - shift00),
                                   10, (0, 255, 0),
                                   thickness=2)
                        cv2.circle(tmpWidz,
                                   (int(tmpWidz.shape[0] / 2) - shift00,
                                    int(tmpWidz.shape[1] / 2) - shift00),
                                   10, (0, 255, 0),
                                   thickness=2)
                        cv2.circle(resiWidz,
                                   (int(resiWidz.shape[0] / 2) - shift00,
                                    int(resiWidz.shape[1] / 2) - shift00),
                                   10, (0, 255, 0),
                                   thickness=2)

                        spaceLine = np.ones(
                            (objWidz.shape[0], 5), dtype=np.uint8) * 255
                        spaceLine = spaceLine.reshape(spaceLine.shape[0],
                                                      spaceLine.shape[1],
                                                      1).repeat(3, 2)
                        sub2Con = np.concatenate(
                            (objWidz, spaceLine, tmpWidz, spaceLine, resiWidz),
                            axis=1)

                        tstrs = tmpImg.split(
                            '_')  #G044_mon_objt_190128T10264248.fit
                        if len(tstrs) == 4:
                            tmpDateTime = tstrs[3]
                            cv2.putText(
                                sub2Con,  #numpy array on which text is written
                                tmpDateTime[:tmpDateTime.index('.')],  #text 
                                (objWidz.shape[1] + 25, objWidz.shape[0] -
                                 10),  #position at which writing has to start
                                cv2.FONT_HERSHEY_SIMPLEX,  #font family
                                0.5,  #font size
                                (0, 255, 0),  #font color
                                2)

                        tImgName = "%s_%05d.jpg" % (nameBase, i)
                        timgNames.append(tImgName)
                        savePath = "%s/%s" % (fullImgPath, tImgName)
                        Image.fromarray(sub2Con).save(savePath)
                        i = i + 1

                    catName = tImgName = "%s.cat" % (nameBase)
                    catPath = "%s/%s" % (fullImgPath, catName)
                    fp0 = open(catPath, 'w')
                    fp0.write(self.theader2)

                    t2oX = tcatParm[-2]
                    t2oY = tcatParm[-1]
                    tx = tParms[:, 0]
                    ty = tParms[:, 1]
                    ox = t2oX(tx, ty)
                    oy = t2oY(tx, ty)

                    i = 0
                    for td in tParms:
                        #print(td)
                        #print(td[15])
                        tstr = self.catFormate%\
                            (ox[i],oy[i],td[2],td[3],td[4],td[5],td[6],td[7],td[8],td[9],td[10],td[11],td[12],td[13],
                             td[14],td[15],td[16], timgNames[i], obsUtc1, td[0],td[1])
                        fp0.write(tstr)
                        i = i + 1
                    fp0.close()

                    self.doUpload(fullImgPath, [catName], 'crossOTList',
                                  serverIP, crossTaskName)
                    self.doUpload(fullImgPath, timgNames, 'crossOTStamp',
                                  serverIP, crossTaskName)

            if tParms1.shape[0] >= 100 or tParms2.shape[0] >= 100:
                self.log.error(
                    "too more unmatched or matched OT candidate, skip upload matched to db: after classified, %s total get %d matchend sub images"
                    % (origName, tParms2.shape[0]))
            os.system("rm -rf %s" % (fullImgPath))

        except Exception as e:
            self.log.error('classifyAndUpload error')
            self.log.error(str(e))
            tstr = traceback.format_exc()
            self.log.error(tstr)
        endtime = datetime.now()
        runTime = (endtime - starttime).seconds
        self.log.info("********** classifyAndUpload %s use %d seconds" %
                      (origName, runTime))
Esempio n. 4
0
    def diff(self, srcDir, destDir, tmpFit, objFits, reverse=False):

        try:
            if not os.path.exists(destDir):
                os.system("mkdir -p %s" % (destDir))

            self.regImg2DB(srcDir, tmpFit)

            os.system("rm -rf %s/*" % (self.templateDir))
            os.system("cp %s/%s %s/%s" %
                      (srcDir, tmpFit, self.templateDir, self.templateImg))
            self.makeTemplate()

            for i, imgName in enumerate(objFits):

                starttime = datetime.now()
                self.log.info("diff %d: %s" % (i, imgName))

                self.regImg2DB(srcDir, imgName)

                os.system("rm -rf %s/*" % (self.tmpDir))
                imgpre = imgName.split(".")[0]
                tobjFitsFullPath = "%s/%s.fit" % (srcDir, imgpre)
                if not os.path.exists(tobjFitsFullPath):
                    self.log.error("%s.fit not exist, stop" % (imgpre))
                    break

                if os.path.exists("%s/diffResi/%s.fit" % (destDir, imgpre)):
                    self.log.info("%s.fit already diffed, skip" % (imgpre))
                    continue

                os.system("cp %s/%s.fit %s/%s" %
                          (srcDir, imgpre, self.tmpDir, self.objectImg))
                os.system("cp %s/%s %s/%s" %
                          (self.templateDir, self.templateImg, self.tmpDir,
                           self.templateImg))
                os.system("cp %s/%s %s/%s" % (self.templateDir, self.badPixCat,
                                              self.tmpDir, self.badPixCat))

                self.objTmpResi, runSuccess = self.tools.runHotpants(
                    self.objectImg, self.templateImg, self.tmpDir)
                if not runSuccess:
                    self.log.info("%s.fit diff error..." % (imgpre))
                    continue

                if reverse:
                    os.system("cp %s/%s %s/diffResi/%s_r.fit" %
                              (self.tmpDir, self.objTmpResi, destDir, imgpre))
                else:
                    os.system("cp %s/%s %s/diffResi/%s.fit" %
                              (self.tmpDir, self.objTmpResi, destDir, imgpre))

                tgrid = 4
                tsize = 500
                tzoom = 2
                timg = getThumbnail(self.tmpDir,
                                    self.objTmpResi,
                                    stampSize=(tsize, tsize),
                                    grid=(tgrid, tgrid),
                                    innerSpace=1)
                timg = scipy.ndimage.zoom(timg, tzoom, order=0)
                if reverse:
                    preViewPath = "%s/preview/%s_resi_r.jpg" % (destDir,
                                                                imgpre)
                else:
                    preViewPath = "%s/preview/%s_resi.jpg" % (destDir, imgpre)
                Image.fromarray(timg).save(preViewPath)

                fpar = 'sex_diff.par'
                sexConf = [
                    '-DETECT_MINAREA', '3', '-DETECT_THRESH', '2.5',
                    '-ANALYSIS_THRESH', '2.5'
                ]
                self.objectImgCat = self.tools.runSextractor(
                    self.objectImg, self.tmpDir, self.tmpDir, fpar, sexConf)
                resiCat = self.tools.runSextractor(self.objTmpResi,
                                                   self.tmpDir, self.tmpDir,
                                                   fpar, sexConf)

                mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
                    self.tmpDir, resiCat, self.objectImgCat, 1)  #1 and 5
                badPixProps2 = np.loadtxt("%s/%s" % (self.tmpDir, nmhFile))
                mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
                    self.tmpDir, mchFile, self.badPixCat, 1)  #1 and 5

                badPixProps = np.loadtxt("%s/%s" %
                                         (self.tmpDir, self.badPixCat))
                resiCatTrans = self.getRaDec(nmhFile)
                objProps = np.loadtxt("%s/%s" % (self.tmpDir, resiCatTrans))
                tstr = "%s,  resi objs %d, orgBadPix %d, nmBad %d" % (
                    imgName, objProps.shape[0], badPixProps.shape[0],
                    badPixProps2.shape[0])
                self.log.info(tstr)

                #size = self.subImgSize
                size = 68
                if objProps.shape[0] < 20000 and objProps.shape[0] > 0:

                    totSubImgs, totParms = getWindowImgs(
                        self.tmpDir, self.objectImg, self.templateImg,
                        self.objTmpResi, objProps, size)
                    if totParms.shape[0] > 0:
                        if reverse:
                            fotpath = '%s/subImgs/%s_r.npz' % (destDir, imgpre)
                        else:
                            fotpath = '%s/subImgs/%s.npz' % (destDir, imgpre)
                        np.savez_compressed(fotpath,
                                            imgs=totSubImgs,
                                            parms=totParms)

                        self.classifyAndUpload(destDir,
                                               imgName,
                                               reverse=reverse)

                endtime = datetime.now()
                runTime = (endtime - starttime).seconds
                self.log.info("diff: %s use %d seconds" % (imgName, runTime))

                #break

        except Exception as e:
            print(str(e))
            tstr = traceback.format_exc()
            print(tstr)
Esempio n. 5
0
    def diffImage(self):

        starttime = datetime.now()

        resultFlag = True

        oImgPre = self.origObjectImg[:self.origObjectImg.index(".")]

        os.system("cp %s/%s %s/%s" % (self.templateDir, self.templateImg,
                                      self.tmpDir, self.templateImg))
        os.system(
            "cp %s/%s %s/%s" %
            (self.templateDir, self.badPixCat, self.tmpDir, self.badPixCat))
        os.system("cp %s/%s %s/%s" % (self.templateDir, self.templateImgCat,
                                      self.tmpDir, self.templateImgCat))

        #self.newImageName = self.tools.imageAlign(self.tmpDir, self.objectImg, self.transHG)

        self.objTmpResi, runSuccess = self.tools.runHotpants(
            self.newImageName, self.templateImg, self.tmpDir)
        if not runSuccess:
            return False

        os.system("cp %s/%s %s/%s.fit" %
                  (self.tmpDir, self.objTmpResi, self.resiFitsDir, oImgPre))

        tgrid = 1
        tsize = 4096
        tzoom = 1
        timg = getThumbnail(self.tmpDir,
                            self.objTmpResi,
                            stampSize=(tsize, tsize),
                            grid=(tgrid, tgrid),
                            innerSpace=1)
        #timg = scipy.ndimage.zoom(timg, tzoom, order=0)
        preViewPath = "%s/%s_resi.jpg" % (self.origPreViewDir, oImgPre)
        Image.fromarray(timg).save(preViewPath)
        '''
        return
        '''
        fpar = 'sex_diff.par'
        sexConf = [
            '-DETECT_MINAREA', '3', '-DETECT_THRESH', '2.5',
            '-ANALYSIS_THRESH', '2.5'
        ]
        resiCat = self.tools.runSextractor(self.objTmpResi, self.tmpDir,
                                           self.tmpDir, fpar, sexConf)
        mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
            self.tmpDir, resiCat, self.objectImgCatTrans, 1)  #1 and 5
        badPixProps2 = np.loadtxt("%s/%s" % (self.tmpDir, nmhFile))

        tdata1 = np.loadtxt("%s/%s" % (self.tmpDir, mchFile))
        tdata2 = np.loadtxt("%s/%s" % (self.tmpDir, self.objectImgCatTrans))
        tIdx1 = np.loadtxt("%s/%s" % (self.tmpDir, mchPair)).astype(np.int)
        tIdx1 = tIdx1 - 1
        origData = tdata2[tIdx1[:, 1]]

        if origData.shape[0] == tdata1.shape[0]:
            outCatName = "%s_orgpos.cat" % (mchFile[:mchFile.index(".")])
            outCatPath = "%s/%s" % (self.tmpDir, outCatName)
            tstr = ""
            i = 0
            for td in tdata1:
                tstr += "%.4f %.4f %.2f %.2f %.2f %.3f %.3f %.3f %.2f %.2f %d %.4f %.4f %.4f %.4f\n"%\
                   (td[0],td[1],td[2],td[3],td[4],td[5],td[6],td[7],td[8],td[9],td[10],origData[i][11], origData[i][12], origData[i][13], origData[i][14])
                i = i + 1
            fp0 = open(outCatPath, 'w')
            fp0.write(tstr)
            fp0.close()
            mchFile = outCatName
        else:
            self.log.error("add orig pos error")
        '''
        self.tools.runSelfMatch(self.tmpDir, resiCat, 1) #debug: get ds9 reg file
        tdata = np.loadtxt("%s/%s"%(self.tmpDir, resiCat))
        self.log.info("resi image star %d"%(tdata.shape[0]))
        '''
        ''' '''
        mchRadius = 15  #15 10
        mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
            self.tmpDir, mchFile, self.templateImgCat, mchRadius)
        fotProps = np.loadtxt("%s/%s" % (self.tmpDir, mchFile))

        mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
            self.tmpDir, nmhFile, self.badPixCat, 1)  #1 and 5
        os.system("cp %s/%s %s/%s" %
                  (self.tmpDir, nmhFile, self.resiCatDir, "%s.cat" %
                   (oImgPre)))

        totProps = np.loadtxt("%s/%s" % (self.tmpDir, nmhFile))
        #badPixProps = np.loadtxt("%s/%s"%(self.tmpDir, self.badPixCat))
        badPixProps = np.array([])
        tstr = "orgBadPix %d, nmBad %d, match %d, noMatch %d" % (
            badPixProps.shape[0], badPixProps2.shape[0], fotProps.shape[0],
            totProps.shape[0])
        self.log.info(tstr)

        #size = self.subImgSize
        size = 68
        if totProps.shape[0] < 500 and totProps.shape[0] > 0:

            totSubImgs, totParms = getWindowImgs(self.tmpDir,
                                                 self.newImageName,
                                                 self.templateImg,
                                                 self.objTmpResi, totProps,
                                                 size)
            if totParms.shape[0] > 0:
                tXY = totParms[:, 0:2]
                tRaDec = self.wcs.all_pix2world(tXY, 1)
                totParms = np.concatenate((totParms, tRaDec), axis=1)
                fotpath = '%s/%s_totimg.npz' % (self.destDir, oImgPre)
                np.savez_compressed(fotpath, imgs=totSubImgs, parms=totParms)

                resiImgs = []
                for timg in totSubImgs:
                    resiImgs.append(timg[2])

                preViewPath = "%s/%s_tot.jpg" % (self.preViewDir, oImgPre)
                #if not os.path.exists(preViewPath):
                psfView = genPSFView(resiImgs)
                Image.fromarray(psfView).save(preViewPath)

            if fotProps.shape[0] > 0 and fotProps.shape[0] < 2000:
                fotSubImgs, fotParms = getWindowImgs(self.tmpDir,
                                                     self.newImageName,
                                                     self.templateImg,
                                                     self.objTmpResi, fotProps,
                                                     size)
                if fotParms.shape[0] > 0:
                    tXY = fotParms[:, 0:2]
                    tRaDec = self.wcs.all_pix2world(tXY, 1)
                    fotParms = np.concatenate((fotParms, tRaDec), axis=1)
                    fotpath = '%s/%s_fotimg.npz' % (self.destDir, oImgPre)
                    np.savez_compressed(fotpath,
                                        imgs=fotSubImgs,
                                        parms=fotParms)
            '''
            if badPixProps.shape[0]>0:
                badSubImgs, badParms = getWindowImgs(self.tmpDir, self.newImageName, self.templateImg, self.objTmpResi, badPixProps, size)
                if badParms.shape[0]>0:
                    fotpath = '%s/%s_badimg.npz'%(self.destDir, oImgPre)
                    np.savez_compressed(fotpath, imgs=badSubImgs, parms=badParms)
            
            if badPixProps2.shape[0]>0:
                badSubImgs, badParms = getWindowImgs(self.tmpDir, self.newImageName, self.templateImg, self.objTmpResi, badPixProps2, size)
                if badParms.shape[0]>0:
                    fotpath = '%s/%s_badimg2.npz'%(self.destDir, oImgPre)
                    np.savez_compressed(fotpath, imgs=badSubImgs, parms=badParms)
            '''
            resultFlag = True
        else:
            tmsgStr = "%s.fit resi image has %d tot objects, maybe wrong" % (
                oImgPre, totProps.shape[0])
            self.log.error(tmsgStr)
            #self.sendMsg(tmsgStr)
            resultFlag = False
        '''
        tgrid = 4
        tsize = 500
        tzoom = 1
        timg = getThumbnail(self.tmpDir, self.objTmpResi, stampSize=(tsize,tsize), grid=(tgrid, tgrid), innerSpace = 1)
        #timg = scipy.ndimage.zoom(timg, tzoom, order=0)
        preViewPath = "%s/%s_resi.jpg"%(self.origPreViewDir, oImgPre)
        Image.fromarray(timg).save(preViewPath)
        '''
        '''
        timg = getThumbnail(self.tmpDir, self.newImageName, stampSize=(tsize,tsize), grid=(tgrid, tgrid), innerSpace = 1)
        timg = scipy.ndimage.zoom(timg, tzoom, order=0)
        preViewPath = "%s/%s_obj.jpg"%(self.origPreViewDir, oImgPre)
        Image.fromarray(timg).save(preViewPath)
        timg = getThumbnail(self.tmpDir, self.templateImg, stampSize=(tsize,tsize), grid=(tgrid, tgrid), innerSpace = 1)
        timg = scipy.ndimage.zoom(timg, tzoom, order=0)
        preViewPath = "%s/%s_tmp.jpg"%(self.origPreViewDir, oImgPre)
        Image.fromarray(timg).save(preViewPath)
        '''
        endtime = datetime.now()
        runTime = (endtime - starttime).seconds
        self.log.info("********** image diff total use %d seconds" % (runTime))

        return resultFlag
Esempio n. 6
0
    def diffImage(self, imgName, tmplParms):

        starttime = datetime.now()

        resultFlag = True
        oImgPre = imgName.split(".")[0]

        os.system("rm -rf %s/*" % (self.diff))

        status = tmplParms[0]
        #tmplImgName = tmplParms[1][-1][0]
        tmplImgName = tmplParms[1][0][0]
        tmplImgPre = tmplImgName.split(".")[0]
        if status == '1':
            ttmplPath = self.tmplAlignDir
        else:
            ttmplPath = self.tmplDiffDir

        ofitPath = "%s/%s.fit" % (self.doDiffTmpl, tmplImgPre)
        if not os.path.exists(ofitPath):
            os.system("rm -rf %s/*" % (self.doDiffTmpl))
            os.system("cp %s/%s.fit %s/%s.fit" %
                      (ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre))
            os.system("cp %s/%s.cat %s/%s.cat" %
                      (ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre))
            os.system("cp %s/%s_badpix.cat %s/%s_badpix.cat" %
                      (ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre))

        os.system("cp %s/%s.fit %s/ti.fit" %
                  (self.doDiffTmpl, tmplImgPre, self.diff))
        os.system("cp %s/%s.cat %s/ti.cat" %
                  (self.doDiffTmpl, tmplImgPre, self.diff))
        os.system("cp %s/%s_badpix.cat %s/ti_badpix.cat" %
                  (self.doDiffTmpl, tmplImgPre, self.diff))

        os.system("cp %s/%s.fit %s/oi.fit" % (self.cmbDir, oImgPre, self.diff))
        os.system("cp %s/%s.cat %s/oi.cat" %
                  (self.cmbCatDir, oImgPre, self.diff))

        theader = fits.getheader("%s/oi.fit" % (self.diff))
        dateObs = theader['DATE-OBS']
        timeObs = theader['TIME-OBS']
        dtStr = "%sT%s" % (dateObs, timeObs)

        objTmpResi, runSuccess = self.tools.runHotpants(
            'oi.fit', 'ti.fit', self.diff)
        if not runSuccess:
            self.log.error("diffImage failure: %s" % (imgName))
            return False

        fpar = 'sex_diff.par'
        #sexConf=['-DETECT_MINAREA','3','-DETECT_THRESH','2.5','-ANALYSIS_THRESH','2.5']
        sexConf = [
            '-DETECT_MINAREA', '5', '-DETECT_THRESH', '2.5',
            '-ANALYSIS_THRESH', '2.5'
        ]
        resiCat, isSuccess = self.tools.runSextractor(objTmpResi, self.diff,
                                                      self.diff, fpar, sexConf)
        if not isSuccess:
            self.log.error("diffImage runSextractor failure")
            return isSuccess

        os.system("cp %s/%s %s/%s_resi.fit" %
                  (self.diff, objTmpResi, self.diffImgDir, oImgPre))
        os.system("cp %s/%s %s/%s_resi.cat" %
                  (self.diff, resiCat, self.diffCatDir, oImgPre))

        mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
            self.diff, resiCat, 'oi.cat', 1)  #1 and 5

        badPix2Path = "%s/%s" % (self.diff, nmhFile)
        if os.path.exists(badPix2Path):
            badPixProps2 = np.loadtxt(badPix2Path)
        else:
            badPixProps2 = np.array([])

        mchRadius = 15  #15 10
        mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
            self.diff, mchFile, 'ti.cat', mchRadius)

        fotPath = "%s/%s" % (self.diff, mchFile)
        if os.path.exists(fotPath):
            fotProps = np.loadtxt(fotPath)
        else:
            fotProps = np.array([])

        mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
            self.diff, nmhFile, 'ti_badpix.cat', 1)  #1 and 5
        os.system("cp %s/%s %s/%s_tot.cat" %
                  (self.diff, nmhFile, self.diffCatDir, oImgPre))

        totPath = "%s/%s" % (self.diff, nmhFile)
        if os.path.exists(totPath):
            totProps = np.loadtxt(totPath)
        else:
            totProps = np.array([])

        badPixPath = "%s/ti_badpix.cat" % (self.diff)
        if os.path.exists(badPixPath):
            badPixProps = np.loadtxt(badPixPath)
        else:
            badPixProps = np.array([])

        #badPixProps = np.array([])
        tstr = "orgBadPix %d, nmBad %d, match %d, noMatch %d" % (
            badPixProps.shape[0], badPixProps2.shape[0], fotProps.shape[0],
            totProps.shape[0])
        self.log.info(tstr)
        print(tstr)

        #size = self.subImgSize
        size = 68
        if totProps.shape[0] < 500 and totProps.shape[0] > 0:

            wcsPath = "%s/%s.wcs" % (ttmplPath, tmplImgPre)
            wcs = WCS(wcsPath)

            totSubImgs, totParms = getWindowImgs(self.diff, 'oi.fit', 'ti.fit',
                                                 objTmpResi, totProps, size)
            if totParms.shape[0] > 0:
                tXY = totParms[:, 0:2]
                #tdates = np.repeat(dtStr,tXY.shape[0]).reshape((tXY.shape[0],1))
                tRaDec = wcs.all_pix2world(tXY, 1)
                totParms = np.concatenate((totParms, tRaDec), axis=1)
                fotpath = '%s/%s_totimg.npz' % (self.destDir, oImgPre)
                np.savez_compressed(fotpath,
                                    imgs=totSubImgs,
                                    parms=totParms,
                                    obsUtc=dtStr)

                resiImgs = []
                for timg in totSubImgs:
                    resiImgs.append(timg[2])

                preViewPath = "%s/%s_tot.jpg" % (self.subImgViewDir, oImgPre)
                #if not os.path.exists(preViewPath):
                psfView = genPSFView(resiImgs)
                Image.fromarray(psfView).save(preViewPath)

            if fotProps.shape[0] > 0 and fotProps.shape[0] < 3000:
                fotSubImgs, fotParms = getWindowImgs(self.diff, 'oi.fit',
                                                     'ti.fit', objTmpResi,
                                                     fotProps, size)
                if fotParms.shape[0] > 0:
                    tXY = fotParms[:, 0:2]
                    #tdates = np.repeat(dtStr,tXY.shape[0]).reshape((tXY.shape[0],1))
                    tRaDec = wcs.all_pix2world(tXY, 1)
                    fotParms = np.concatenate((fotParms, tRaDec), axis=1)
                    fotpath = '%s/%s_fotimg.npz' % (self.destDir, oImgPre)
                    np.savez_compressed(fotpath,
                                        imgs=fotSubImgs,
                                        parms=fotParms,
                                        obsUtc=dtStr)
            '''        
            if badPixProps.shape[0]>0:
                badSubImgs, badParms = getWindowImgs(self.diff, 'oi.fit', 'ti.fit', objTmpResi, badPixProps, size)
                if badParms.shape[0]>0:
                    fotpath = '%s/%s_badimg.npz'%(self.destDir, oImgPre)
                    np.savez_compressed(fotpath, imgs=badSubImgs, parms=badParms)
            
            if badPixProps2.shape[0]>0:
                badSubImgs, badParms = getWindowImgs(self.diff, 'oi.fit', 'ti.fit', objTmpResi, badPixProps2, size)
                if badParms.shape[0]>0:
                    tXY = badParms[:,0:2]
                    tRaDec = wcs.all_pix2world(tXY, 1)
                    badParms = np.concatenate((badParms, tRaDec), axis=1)
                    fotpath = '%s/%s_badimg2.npz'%(self.destDir, oImgPre)
                    np.savez_compressed(fotpath, imgs=badSubImgs, parms=badParms, obsUtc=dtStr)
            '''

            resultFlag = True
        else:
            tmsgStr = "%s.fit resi image has %d tot objects, maybe wrong" % (
                oImgPre, totProps.shape[0])
            self.log.error(tmsgStr)
            #self.sendMsg(tmsgStr)
            resultFlag = False
        ''' '''
        tgrid = 4
        tsize = 500
        tzoom = 1
        timg = getThumbnail(self.diff,
                            objTmpResi,
                            stampSize=(tsize, tsize),
                            grid=(tgrid, tgrid),
                            innerSpace=1)
        timg = scipy.ndimage.zoom(timg, tzoom, order=0)
        preViewPath = "%s/%s_resi.jpg" % (self.origPreViewDir, oImgPre)
        Image.fromarray(timg).save(preViewPath)
        '''
        '''
        timg = getThumbnail(self.diff,
                            'oi.fit',
                            stampSize=(tsize, tsize),
                            grid=(tgrid, tgrid),
                            innerSpace=1)
        timg = scipy.ndimage.zoom(timg, tzoom, order=0)
        preViewPath = "%s/%s_obj.jpg" % (self.origPreViewDir, oImgPre)
        Image.fromarray(timg).save(preViewPath)
        timg = getThumbnail(self.diff,
                            'ti.fit',
                            stampSize=(tsize, tsize),
                            grid=(tgrid, tgrid),
                            innerSpace=1)
        timg = scipy.ndimage.zoom(timg, tzoom, order=0)
        preViewPath = "%s/%s_tmp.jpg" % (self.origPreViewDir, oImgPre)
        Image.fromarray(timg).save(preViewPath)

        endtime = datetime.now()
        runTime = (endtime - starttime).seconds
        self.log.info("********** image diff total use %d seconds" % (runTime))

        return resultFlag
Esempio n. 7
0
    def diffImage(self):

        starttime = datetime.now()

        resultFlag = True

        oImgPre = self.origObjectImg[:self.origObjectImg.index(".")]

        os.system("cp %s/%s %s/%s" % (self.templateDir, self.templateImg,
                                      self.tmpDir, self.templateImg))
        os.system(
            "cp %s/%s %s/%s" %
            (self.templateDir, self.badPixCat, self.tmpDir, self.badPixCat))
        os.system("cp %s/%s %s/%s" % (self.templateDir, self.templateImgCat,
                                      self.tmpDir, self.templateImgCat))

        #self.newImageName = self.tools.imageAlign(self.tmpDir, self.objectImg, self.transHG)

        self.objTmpResi = self.tools.runHotpants(self.newImageName,
                                                 self.templateImg, self.tmpDir)

        fpar = 'sex_diff.par'
        sexConf = [
            '-DETECT_MINAREA', '3', '-DETECT_THRESH', '2.5',
            '-ANALYSIS_THRESH', '2.5'
        ]
        resiCat = self.tools.runSextractor(self.objTmpResi, self.tmpDir,
                                           self.tmpDir, fpar, sexConf)
        '''
        self.tools.runSelfMatch(self.tmpDir, resiCat, 1) #debug: get ds9 reg file
        tdata = np.loadtxt("%s/%s"%(self.tmpDir, resiCat))
        self.log.info("resi image star %d"%(tdata.shape[0]))
        '''
        ''' '''
        mchRadius = 15  #15 10
        mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
            self.tmpDir, resiCat, self.templateImgCat, mchRadius)
        fotProps = np.loadtxt("%s/%s" % (self.tmpDir, mchFile))

        mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
            self.tmpDir, nmhFile, self.badPixCat, 1)  #1 and 5
        os.system("cp %s/%s %s/%s" %
                  (self.tmpDir, nmhFile, self.resiCatDir, "%s.cat" %
                   (oImgPre)))

        totProps = np.loadtxt("%s/%s" % (self.tmpDir, nmhFile))
        #badPixProps = np.loadtxt("%s/%s"%(self.tmpDir, mchFile))
        badPixProps = np.loadtxt("%s/%s" % (self.tmpDir, self.badPixCat))
        tstr = "badPix %d, match %d, noMatch %d" % (
            badPixProps.shape[0], fotProps.shape[0], totProps.shape[0])
        self.log.info(tstr)

        size = self.subImgSize
        if totProps.shape[0] < 500 and totProps.shape[0] > 0:

            totSubImgs, totParms = getWindowImgs(self.tmpDir,
                                                 self.newImageName,
                                                 self.templateImg,
                                                 self.objTmpResi, totProps,
                                                 size)
            tXY = totParms[:, 0:2]
            tRaDec = self.wcs.all_pix2world(tXY, 1)
            totParms = np.concatenate((totParms, tRaDec), axis=1)
            fotpath = '%s/%s_totimg.npz' % (self.destDir, oImgPre)
            np.savez_compressed(fotpath, imgs=totSubImgs, parms=totParms)

            resiImgs = []
            for timg in totSubImgs:
                resiImgs.append(timg[2])

            preViewPath = "%s/%s_tot.jpg" % (self.preViewDir, oImgPre)
            #if not os.path.exists(preViewPath):
            psfView = genPSFView(resiImgs)
            Image.fromarray(psfView).save(preViewPath)

            if fotProps.shape[0] > 0 and fotProps.shape[0] < 2000:
                fotSubImgs, fotParms = getWindowImgs(self.tmpDir,
                                                     self.newImageName,
                                                     self.templateImg,
                                                     self.objTmpResi, fotProps,
                                                     size)
                tXY = fotParms[:, 0:2]
                tRaDec = self.wcs.all_pix2world(tXY, 1)
                fotParms = np.concatenate((fotParms, tRaDec), axis=1)
                fotpath = '%s/%s_fotimg.npz' % (self.destDir, oImgPre)
                np.savez_compressed(fotpath, imgs=fotSubImgs, parms=fotParms)

            if badPixProps.shape[0] > 0:
                badSubImgs, badParms = getWindowImgs(self.tmpDir,
                                                     self.newImageName,
                                                     self.templateImg,
                                                     self.objTmpResi,
                                                     badPixProps, size)
                fotpath = '%s/%s_badimg.npz' % (self.destDir, oImgPre)
                np.savez_compressed(fotpath, imgs=badSubImgs, parms=badParms)

            resultFlag = True
        else:
            tmsgStr = "%s.fit resi image has %d tot objects, maybe wrong" % (
                oImgPre, totProps.shape[0])
            self.log.error(tmsgStr)
            self.tools.sendTriggerMsg(tmsgStr)
            resultFlag = False

        tgrid = 4
        tsize = 500
        tzoom = 1
        timg = getThumbnail(self.tmpDir,
                            self.objTmpResi,
                            stampSize=(tsize, tsize),
                            grid=(tgrid, tgrid),
                            innerSpace=1)
        #timg = scipy.ndimage.zoom(timg, tzoom, order=0)
        preViewPath = "%s/%s_resi.jpg" % (self.origPreViewDir, oImgPre)
        Image.fromarray(timg).save(preViewPath)
        '''
        timg = getThumbnail(self.tmpDir, self.newImageName, stampSize=(tsize,tsize), grid=(tgrid, tgrid), innerSpace = 1)
        timg = scipy.ndimage.zoom(timg, tzoom, order=0)
        preViewPath = "%s/%s_obj.jpg"%(self.origPreViewDir, oImgPre)
        Image.fromarray(timg).save(preViewPath)
        timg = getThumbnail(self.tmpDir, self.templateImg, stampSize=(tsize,tsize), grid=(tgrid, tgrid), innerSpace = 1)
        timg = scipy.ndimage.zoom(timg, tzoom, order=0)
        preViewPath = "%s/%s_tmp.jpg"%(self.origPreViewDir, oImgPre)
        Image.fromarray(timg).save(preViewPath)
        '''
        endtime = datetime.now()
        runTime = (endtime - starttime).seconds
        self.log.info("********** image diff total use %d seconds" % (runTime))

        return resultFlag