def test_critical(self): msg = "unit_test_critical_message" Alogging.critical(msg) found = False r = open(self.logfile, 'r') for line in r.readlines(): if msg in line: if(self.verbose): print "\nFound message > ", line found = True r.close() break self.assertTrue(found)
def fitgauss1Dm(xdat, ydat, usePeak=False, dx=-1.0): """ gaussfit helper function this will get a reasonable gauss even if you only have 2 points assumes evenly spaced data in xdat, so it can extract a width. It can be used like fitgauss1D(), but uses the first three moments of the distribution to "match" that of a gauss. area preserving if you wish. If you set usePeak, it will pick the highest value in your data. Warning: if you must fit negative profiles, be sure to rescale before you come into this routine. """ if len(xdat) == 1: # special case, it will need dx if dx < 0.0: logging.critical( "Cannot determine gaussian of delta function if width not given" ) raise return (ydat[0], xdat[0], dx) sum0 = sum1 = sum2 = peak = 0.0 # the mean is given as the weighted mean of x for x, y in zip(xdat, ydat): if y > peak: peak = y sum0 = sum0 + y sum1 = sum1 + y * x xmean = sum1 / sum0 # re-center the data for a moment-2 calculation # @todo pos/neg xdat0 = xdat - xmean for x, y in zip(xdat0, ydat): sum2 = sum2 + y * x * x sigma = math.sqrt(abs(sum2 / sum0)) # equate the area under the histogram with the area under a gauss # to get the peak of the "matched" gauss dx = abs(xdat[1] - xdat[0]) # @todo use optional "dx=None" ? if not usePeak: # pick the area preserving one, vs. the "real" peak # The area preserving one seems to be about 18% higher # but with a tight correllation peak = (sum0 * dx) / math.sqrt(2 * sigma * math.pi) fwhm = 2.35482 * sigma return (peak, xmean, fwhm)
def fitgauss1Dm(xdat, ydat, usePeak = False, dx = -1.0): """ gaussfit helper function this will get a reasonable gauss even if you only have 2 points assumes evenly spaced data in xdat, so it can extract a width. It can be used like fitgauss1D(), but uses the first three moments of the distribution to "match" that of a gauss. area preserving if you wish. If you set usePeak, it will pick the highest value in your data. Warning: if you must fit negative profiles, be sure to rescale before you come into this routine. """ if len(xdat) == 1: # special case, it will need dx if dx < 0.0: logging.critical("Cannot determine gaussian of delta function if width not given") raise return (ydat[0], xdat[0], dx) sum0 = sum1 = sum2 = peak = 0.0 # the mean is given as the weighted mean of x for x,y in zip(xdat,ydat): if y>peak: peak = y sum0 = sum0 + y sum1 = sum1 + y*x xmean = sum1/sum0 # re-center the data for a moment-2 calculation # @todo pos/neg xdat0 = xdat - xmean for x,y in zip(xdat0,ydat): sum2 = sum2 + y*x*x sigma = math.sqrt(abs(sum2/sum0)) # equate the area under the histogram with the area under a gauss # to get the peak of the "matched" gauss dx = abs(xdat[1]-xdat[0]) # @todo use optional "dx=None" ? if not usePeak: # pick the area preserving one, vs. the "real" peak # The area preserving one seems to be about 18% higher # but with a tight correllation peak = (sum0 * dx) / math.sqrt(2*sigma*math.pi) fwhm = 2.35482 * sigma return (peak, xmean, fwhm)
def run(self): # self._summary = {} # prepare to make a summary here dt = utils.Dtime("Ingest") # timer for debugging do_cbeam = True # enforce a common beam # pb = self.getkey('pb') do_pb = len(pb) > 0 use_pb = self.getkey("usepb") # create_mask = self.getkey('mask') # create a new mask ? box = self.getkey("box") # corners in Z, XY or XYZ edge = self.getkey("edge") # number of edge channels to remove restfreq = self.getkey("restfreq") # < 0 means not activated # smooth= could become deprecated, and/or include a decimation option to make it useful # again, Smooth_AT() does this also , at the cost of an extra cube to store smooth = self.getkey("smooth") # # vlsr = self.getkey("vlsr") # see also LineID, where this could be given again # first place a fits file in the admit project directory (symlink) # this is a bit involved, depending on if an absolute or relative path was # give to Ingest_AT(file=) fitsfile = self.getkey('file') if fitsfile[0] != os.sep: fitsfile = os.path.abspath(os.getcwd() + os.sep + fitsfile) logging.debug('FILE=%s' % fitsfile) if fitsfile[0] != os.sep: raise Exception,"Bad file=%s, expected absolute name",fitsfile # now determine if it could have been a CASA (or MIRIAD) image already # which we'll assume if it's a directory; this is natively supported by CASA # but there are tools where if you pass it a FITS or MIRIAD # MIRIAD is not recommended for serious work, especially big files, since there # is a performance penalty due to tiling. file_is_casa = casautil.iscasa(fitsfile) loc = fitsfile.rfind(os.sep) # find the '/' ffile0 = fitsfile[loc+1:] # basename.fits basename = self.getkey('basename') # (new) basename allowed (allow no dots?) if len(basename) == 0: basename = ffile0[:ffile0.rfind('.')] # basename logging.info("basename=%s" % basename) target = self.dir(ffile0) if not os.path.exists(target) : cmd = 'ln -s "%s" "%s"' % (fitsfile, target) logging.debug("CMD: %s" % cmd) os.system(cmd) readonly = False if file_is_casa: logging.debug("Assuming input %s is a CASA (or MIRIAD) image" % ffile0) bdpfile = self.mkext(basename,"im") if bdpfile == ffile0: logging.warning("No selections allowed on CASA image, since no alias was given") readonly = True b1 = SpwCube_BDP(bdpfile) self.addoutput(b1) b1.setkey("image", Image(images={bt.CASA:bdpfile})) # @todo b2 and PB? else: # construct the output name and construct the BDP based on the CASA image name # this also takes care of the behind the scenes alias= substitution bdpfile = self.mkext(basename,"im") if bdpfile == basename: raise Exception,"basename and bdpfile are the same, Ingest_AT needs a fix for this" b1 = SpwCube_BDP(bdpfile) self.addoutput(b1) if do_pb: print "doing the PB" bdpfile2 = self.mkext(basename,"pb") b2 = Image_BDP(bdpfile2) self.addoutput(b2) # @todo we should also set readonly=True if no box, no mask etc. and still an alias # that way it will speed up and not make a copy of the image ? # fni and fno are full (abspath) filenames, ready for CASA # fni is the same as fitsfile fni = self.dir(ffile0) fno = self.dir(bdpfile) if do_pb: fno2 = self.dir(bdpfile2) dt.tag("start") if file_is_casa: taskinit.ia.open(fni) else: if do_pb and use_pb: # @todo this needs a fix for the path for pb, only works if abs path is given # impbcor(im.fits,pb.fits,out.im,overwrite=True,mode='m') if False: # this may seem like a nice shortcut, to have the fits->casa conversion be done # internally in impbcor, but it's a terrible performance for big cubes. (tiling?) # we keep this code here, perhaps at some future time (mpi?) this performs better # @todo fno2 impbcor(fni,pb,fno,overwrite=True,mode='m') dt.tag("impbcor-1") else: # the better way is to convert FITS->CASA and then call impbcor() # the CPU savings are big, but I/O overhead can still be substantial taskinit.ia.fromfits('_pbcor',fni,overwrite=True) taskinit.ia.fromfits('_pb',pb,overwrite=True) dt.tag("impbcor-1f") if False: impbcor('_pbcor','_pb',fno,overwrite=True,mode='m') # @todo fno2 utils.remove('_pbcor') utils.remove('_pb') dt.tag("impbcor-2") else: # immath appears to be even faster (2x in CPU) # https://bugs.nrao.edu/browse/CAS-8299 # @todo this needs to be confirmed that impbcor is now good to go (r36078) casa.immath(['_pbcor','_pb'],'evalexpr',fno,'IM0*IM1') dt.tag("immath") if True: # use the mean of all channels... faster may be to use the middle plane # barf; edge channels can be with fewer subfields in a mosaic taskinit.ia.open('_pb') taskinit.ia.summary() ia1=taskinit.ia.moments(moments=[-1],drop=True,outfile=fno2) ia1.done() taskinit.ia.close() dt.tag("moments") utils.remove('_pbcor') utils.remove('_pb') dt.tag("impbcor-3") elif do_pb and not use_pb: # cheat case: PB was given, but not meant to be used # not implemented yet print "cheat case dummy PB not implemented yet" else: # no PB given if True: # re-running this was more consistently faster in wall clock time # note that zeroblanks=True will still keep the mask logging.debug("casa::ia.fromfits(%s) -> %s" % (fni,bdpfile)) taskinit.ia.fromfits(fno,fni,overwrite=True) #taskinit.ia.fromfits(fno,fni,overwrite=True,zeroblanks=True) dt.tag("fromfits") else: # not working to extend 3D yet, but this would solve the impv() 3D problem logging.debug("casa::importfits(%s) -> %s" % (fni,bdpfile)) #casa.importfits(fni,fno,defaultaxes=True,defaultaxesvalues=[None,None,None,'I']) # possible bug: zeroblanks=True has no effect? casa.importfits(fni,fno,zeroblanks=True) dt.tag("importfits") taskinit.ia.open(fno) if len(smooth) > 0: # smooth here, but Smooth_AT is another option # here we only allow pixel smoothing # spatial: gauss # spectral: boxcar/hanning (check for flux conservation) # is the boxcar wrong, not centered, but edged? # @todo CASA BUG: this will loose the object name (and maybe more?) from header, so VLSR lookup fails fnos = fno + '.smooth' taskinit.ia.convolve2d(outfile=fnos, overwrite=True, pa='0deg', major='%gpix' % smooth[0], minor='%gpix' % smooth[1], type='gaussian') taskinit.ia.close() srcname = casa.imhead(fno,mode="get",hdkey="object") # work around CASA bug #@todo use safer ia.rename() here. # https://casa.nrao.edu/docs/CasaRef/image.rename.html utils.rename(fnos,fno) casa.imhead(fno,mode="put",hdkey="object",hdvalue=srcname) # work around CASA bug dt.tag("convolve2d") if len(smooth) > 2 and smooth[2] > 0: if smooth[2] == 1: # @todo only 1 channel option specsmooth(fno,fnos,axis=2,function='hanning',dmethod="") else: # @todo may have the wrong center specsmooth(fno,fnos,axis=2,function='boxcar',dmethod="",width=smooth[2]) #@todo use safer ia.rename() here. # https://casa.nrao.edu/docs/CasaRef/image.rename.html utils.rename(fnos,fno) dt.tag("specsmooth") taskinit.ia.open(fno) s = taskinit.ia.summary() if len(s['shape']) != 4: logging.warning("Adding dummy STOKES-I axis") fnot = fno + '_4' taskinit.ia.adddegaxes(stokes='I',outfile=fnot) taskinit.ia.close() #@todo use safer ia.rename() here. # https://casa.nrao.edu/docs/CasaRef/image.rename.html utils.rename(fnot,fno) taskinit.ia.open(fno) dt.tag("adddegaxes") else: logging.info("SHAPE: %s" % str(s['shape'])) s = taskinit.ia.summary() dt.tag("summary-0") if s['hasmask'] and create_mask: logging.warning("no extra mask created because input image already had one") create_mask = False # if a box= or edge= was given, only a subset of the cube needs to be ingested # this however complicates PB correction later on if len(box) > 0 or len(edge) > 0: if readonly: raise Exception,"Cannot use box= or edge=, data is read-only, or use an basename/alias" if len(edge) == 1: edge.append(edge[0]) nx = s['shape'][0] ny = s['shape'][1] nz = s['shape'][2] logging.info("box=%s edge=%s processing with SHAPE: %s" % (str(box),str(edge),str(s['shape']))) if len(box) == 2: # select zrange if len(edge)>0: raise Exception,"Cannot use edge= when box=[z1,z2] is used" r1 = taskinit.rg.box([0,0,box[0]] , [nx-1,ny-1,box[1]]) elif len(box) == 4: if len(edge) == 0: # select just an XY box r1 = taskinit.rg.box([box[0],box[1]] , [box[2],box[3]]) elif len(edge) == 2: # select an XY box, but remove some edge channels r1 = taskinit.rg.box([box[0],box[1],edge[0]] , [box[2],box[3],nz-edge[1]-1]) else: raise Exception,"Bad edge= for len(box)=4" elif len(box) == 6: # select an XYZ box r1 = taskinit.rg.box([box[0],box[1],box[2]] , [box[3],box[4],box[5]]) elif len(edge) == 2: # remove some edge channels, but keep the whole XY box r1 = taskinit.rg.box([0,0,edge[0]] , [nx-1,ny-1,nz-edge[1]-1]) else: raise Exception,"box=%s illegal" % box logging.debug("BOX/EDGE selection: %s %s" % (str(r1['blc']),str(r1['trc']))) #if taskinit.ia.isopen(): taskinit.ia.close() logging.info("SUBIMAGE") subimage = taskinit.ia.subimage(region=r1,outfile=fno+'.box',overwrite=True) taskinit.ia.close() taskinit.ia.done() subimage.rename(fno,overwrite=True) subimage.close() subimage.done() taskinit.ia.open(fno) dt.tag("subimage-1") else: # the whole cube is passed onto ADMIT if readonly and create_mask: raise Exception,"Cannot use mask=True, data read-only, or use an alias" if file_is_casa and not readonly: # @todo a miriad file - which should be read only - will also create a useless copy here if no alias used taskinit.ia.subimage(overwrite=True,outfile=fno) taskinit.ia.close() taskinit.ia.open(fno) dt.tag("subimage-0") if create_mask: if readonly: raise Exception,"Cannot create mask, data read-only, or use an alias" # also check out the 'fromfits::zeroblanks = False' # calcmask() will overwrite any previous pixelmask #taskinit.ia.calcmask('mask("%s") && "%s" != 0.0' % (fno,fno)) taskinit.ia.calcmask('"%s" != 0.0' % fno) dt.tag("mask") s = taskinit.ia.summary() dt.tag("summary-1") # do a fast statistics (no median or robust) s0 = taskinit.ia.statistics() dt.tag("statistics") if len(s0['npts']) == 0: raise Exception,"No statistics possible, are there valid data in this cube?" # There may be multiple beams per plane so we can't # rely on the BEAM's 'major', 'minor', 'positionangle' being present. # ia.commonbeam() is guaranteed to return beam parameters # if present if do_cbeam and s.has_key('perplanebeams'): # report on the beam extremities, need to loop over all, # first and last don't need to be extremes.... n = s['perplanebeams']['nChannels'] ab0 = '*0' bb0 = s['perplanebeams']['beams'][ab0]['*0'] bmaj0 = bb0['major']['value'] bmin0 = bb0['minor']['value'] beamd = 0.0 for i in range(n): ab1 = '*%d' % i bb1 = s['perplanebeams']['beams'][ab1]['*0'] bmaj1 = bb1['major']['value'] bmin1 = bb1['minor']['value'] beamd = max(beamd,abs(bmaj0-bmaj1),abs(bmin0-bmin1)) logging.warning("MAX-BEAMSPREAD %f" % (beamd)) # if True: logging.info("Applying a commonbeam from the median beam accross the band") # imhead is a bit slow; alternatively use ia.summary() at the half point for setrestoringbeam() h = casa.imhead(fno,mode='list') b = h['perplanebeams']['median area beam'] taskinit.ia.setrestoringbeam(remove=True) taskinit.ia.setrestoringbeam(beam=b) commonbeam = taskinit.ia.commonbeam() else: # @todo : this will be VERY slow - code not finished, needs renaming etc. # this is however formally the better solution logging.warning("commmonbeam code not finished") cb = taskinit.ia.commonbeam() taskinit.ia.convolve2d(outfile='junk-common.im', major=cb['major'], minor=cb['minor'], pa=cb['pa'], targetres=True, overwrite=True) dt.tag('convolve2d') commonbeam = {} else: try: commonbeam = taskinit.ia.commonbeam() except: nppb = 4.0 logging.warning("No synthesized beam found, faking one to prevent downstream problems: nppb=%f" % nppb) s = taskinit.ia.summary() cdelt2 = abs(s['incr'][0]) * 180.0/math.pi*3600.0 bmaj = nppb * cdelt2 # use a nominal 4 points per (round) beam bmin = nppb * cdelt2 bpa = 0.0 taskinit.ia.setrestoringbeam(major='%farcsec' % bmaj, minor='%farcsec' % bmin, pa='%fdeg' % bpa) commonbeam = {} logging.info("COMMONBEAM[%d] %s" % (len(commonbeam),str(commonbeam))) first_point = taskinit.ia.getchunk(blc=[0,0,0,0],trc=[0,0,0,0],dropdeg=True) logging.debug("DATA0*: %s" % str(first_point)) taskinit.ia.close() logging.info('BASICS: [shape] npts min max: %s %d %f %f' % (s['shape'],s0['npts'][0],s0['min'][0],s0['max'][0])) logging.info('S/N (all data): %f' % (s0['max'][0]/s0['rms'][0])) npix = 1 nx = s['shape'][0] ny = s['shape'][1] nz = s['shape'][2] for n in s['shape']: npix = npix * n ngood = int(s0['npts'][0]) fgood = (1.0*ngood)/npix logging.info('GOOD PIXELS: %d/%d (%f%% good or %f%% bad)' % (ngood,npix,100.0*fgood,100.0*(1 - fgood))) if s['hasmask']: logging.warning('MASKS: %s' % (str(s['masks']))) if not file_is_casa: b1.setkey("image", Image(images={bt.CASA:bdpfile})) if do_pb: b2.setkey("image", Image(images={bt.CASA:bdpfile2})) # cube sanity: needs to be either 4D or 2D. But p-p-v cube # alternative: ia.subimage(dropdeg = True) # see also: https://bugs.nrao.edu/browse/CAS-5406 shape = s['shape'] if len(shape)>3: if shape[3]>1: # @todo this happens when you ingest a fits or casa image which is ra-dec-pol-freq if nz > 1: msg = 'Ingest_AT: cannot deal with real 4D cubes yet' logging.critical(msg) raise Exception,msg else: # @todo this is not working yet when the input was a casa image, but ok when fits. go figure. fnot = fno + ".trans" if True: # this works #@todo use safer ia.rename() here. # https://casa.nrao.edu/docs/CasaRef/image.rename.html utils.rename(fno,fnot) imtrans(fnot,fno,"0132") utils.remove(fnot) else: # this does not work, what the heck imtrans(fno,fnot,"0132") #@todo use safer ia.rename() here. # https://casa.nrao.edu/docs/CasaRef/image.rename.html utils.rename(fnot,fno) nz = s['shape'][3] # get a new summary 's' taskinit.ia.open(fno) s = taskinit.ia.summary() taskinit.ia.close() logging.warning("Using imtrans, with nz=%d, to fix axis ordering" % nz) dt.tag("imtrans4") # @todo ensure first two axes are position, followed by frequency elif len(shape)==3: # the current importfits() can do defaultaxes=True,defaultaxesvalues=['', '', '', 'I'] # but then appears to return a ra-dec-pol-freq cube # this branch probably never happens, since ia.fromfits() will # properly convert a 3D cube to 4D now !! # NO: when NAXIS=3 but various AXIS4's are present, that works. But not if it's pure 3D # @todo box= logging.warning("patching up a 3D to 4D cube") raise Exception,"SHOULD NEVER GET HERE" fnot = fno + ".trans" casa.importfits(fni,fnot,defaultaxes=True,defaultaxesvalues=['', '', '', 'I']) utils.remove(fno) # ieck imtrans(fnot,fno,"0132") utils.remove(fnot) dt.tag("imtrans3") logging.regression('CUBE: %g %g %g %d %d %d %f' % (s0['min'],s0['max'],s0['rms'],nx,ny,nz,100.0*(1 - fgood))) # if the cube has only 1 plane (e.g. continuum) , create a visual (png or so) # for 3D cubes, rely on something like CubeSum if nz == 1: implot = ImPlot(pmode=self._plot_mode,ptype=self._plot_type,abspath=self.dir()) implot.plotter(rasterfile=bdpfile,figname=bdpfile) # @todo needs to be registered for the BDP, right now we only have the plot # ia.summary() doesn't have this easily available, so run the more expensive imhead() h = casa.imhead(fno,mode='list') telescope = h['telescope'] # work around CASA's PIPELINE bug/feature? if 'OBJECT' is blank, try 'FIELD' srcname = h['object'] if srcname == ' ': logging.warning('FIELD used for OBJECT') srcname = casa.imhead(fno,mode='get',hdkey='field') if srcname == False: # if no FIELD either, we're doomed. yes, this did happen. srcname = 'Unknown' casa.imhead(fno,mode="put",hdkey="object",hdvalue=srcname) h['object'] = srcname logging.info('TELESCOPE: %s' % telescope) logging.info('OBJECT: %s' % srcname) logging.info('REFFREQTYPE: %s' % h['reffreqtype']) if h['reffreqtype'].find('TOPO')>=0: msg = 'Ingest_AT: cannot deal with cubes with TOPOCENTRIC frequencies yet - winging it' logging.warning(msg) #raise Exception,msg # Ensure beam parameters are available if there are multiple beams # If there is just one beam, then we are just overwriting the header # variables with their identical values. if len(commonbeam) != 0: h['beammajor'] = commonbeam['major'] h['beamminor'] = commonbeam['minor'] h['beampa'] = commonbeam['pa'] # cheat add some things that need to be passed to summary.... h['badpixel'] = 1.0-fgood if vlsr < -999998.0: vlsr = admit.VLSR().vlsr(h['object'].upper()) h['vlsr'] = vlsr logging.info("VLSR = %f (from source catalog)" % vlsr) taskargs = "file=" + fitsfile if create_mask == True: taskargs = taskargs + " mask=True" if len(box) > 0: taskargs = taskargs + " " + str(box) if len(edge) > 0: taskargs = taskargs + " " + str(edge) r2d = 57.29577951308232 logging.info("RA Axis 1: %f %f %f" % (h['crval1']*r2d,h['cdelt1']*r2d*3600.0,h['crpix1'])) logging.info("DEC Axis 2: %f %f %f" % (h['crval2']*r2d,h['cdelt2']*r2d*3600.0,h['crpix2'])) if nz > 1: # @todo check if this is really a freq axis (for ALMA it is, but...) t3 = h['ctype3'] df = h['cdelt3'] fc = h['crval3'] + (0.5*(float(shape[2])-1)-h['crpix3'])*df # center freq; 0 based pixels if h.has_key('restfreq'): fr = float(h['restfreq'][0]) else: fr = fc fw = df*float(shape[2]) dv = -df/fr*utils.c logging.info("Freq Axis 3: %g %g %g" % (h['crval3']/1e9,h['cdelt3']/1e9,h['crpix3'])) logging.info("Cube Axis 3: type=%s velocity increment=%f km/s @ fc=%f fw=%f GHz" % (t3,dv,fc/1e9,fw/1e9)) # @todo sort out this restfreq/vlsr # report 'reffreqtype', 'restfreq' 'telescope' # if the fits file has ALTRVAL/ALTRPIX, this is lost in CASA? # but if you do fits->casa->fits , it's back in fits (with some obvious single precision loss of digits) # @todo ZSOURCE is the proposed VLSR slot in the fits header, but this has frame issues (it's also optical) # # Another method to get the vlsr is to override the restfreq (f0) with an AT keyword # and the 'restfreq' from the header (f) is then used to compute the vlsr: v = c (1 - f/f0) # if shape[2] > 1 and h.has_key('restfreq'): logging.info("RESTFREQ: %g %g %g" % (fr/1e9,h['restfreq'][0]/1e9,restfreq)) if shape[2] > 1: # v_radio of the center of the window w.r.t. restfreq c = utils.c # 299792.458 km/s vlsrc = c*(1-fc/fr) # @todo rel frame? vlsrw = dv*float(shape[2]) if restfreq > 0: vlsrf = c*(1-fr/restfreq/1e9) h['vlsr'] = vlsrf else: vlsrf = 0.0 logging.info("VLSRc = %f VLSRw = %f VLSRf = %f VLSR = %f" % (vlsrc, vlsrw, vlsrf, vlsr)) if h['vlsr'] == 0.0: # @todo! This fails if vlsr actually is zero. Need another magic number h['vlsr'] = vlsrc logging.warning("Warning: No VLSR found, substituting VLSRc = %f" % vlsrc) else: msg = 'Ingest_AT: missing RESTFREQ' print msg # @todo LINTRN is the ALMA keyword that designates the expected line transition in a spw self._summarize(fitsfile, bdpfile, h, shape, taskargs) dt.tag("done") dt.end()