def clearcal( vis=None, field=None, spw=None, intent=None, addmodel=None, ): casalog.origin('clearcal') # Do the trivial parallelization if ParallelTaskHelper.isParallelMS(vis): helper = ParallelTaskHelper('clearcal', locals()) helper.go() return # Local versions of the tools tblocal = tbtool() cblocal = cbtool() mslocal = mstool() try: # we will initialize scr cols only if we don't create them doinit = False if (type(vis) == str) & os.path.exists(vis): tblocal.open(vis) doinit = tblocal.colnames().count('CORRECTED_DATA') > 0 tblocal.close() # We ignore selection if creating the scratch columns if not doinit: casalog.post( 'Need to create scratch columns; ignoring selection.') cblocal.setvi(old=True, quiet=False) # Old VI for now cblocal.open(vis, addmodel=addmodel) else: raise Exception, \ 'Visibility data set not found - please verify the name' # If necessary (scr col not just created), initialize scr cols if doinit: cblocal.selectvis(field=field, spw=spw, intent=intent) cblocal.initcalset(1) cblocal.close() # Write history to the MS param_names = clearcal.func_code.co_varnames[:clearcal.func_code. co_argcount] param_vals = [eval(p) for p in param_names] casalog.post('Updating the history in the output', 'DEBUG1') write_history(mslocal, vis, 'clearcal', param_names, param_vals, casalog) except Exception, instance: print '*** Error ***', instance
def delmod(vis=None,otf=None,field=None,scr=None): casalog.origin('delmod') # Do the trivial parallelization if ParallelTaskHelper.isParallelMS(vis): helper = ParallelTaskHelper('delmod', locals()) helper.go() return #Python script try: # only if vis exists... if ((type(vis)==str) & (os.path.exists(vis))): # ... and we are asked to do something... # open without adding anything! cb.open(vis,addcorr=False,addmodel=False) cb.delmod(otf=otf,field=field,scr=scr) cb.close() else: raise Exception, 'Visibility data set not found - please verify the name' #write history ms.open(vis,nomodify=False) ms.writehistory(message='taskname = delmod',origin='delmod') ms.writehistory(message='vis = "'+str(vis)+'"',origin='delmod') ms.writehistory(message='otf = "'+str(otf)+'"',origin='delmod') ms.writehistory(message='scr = "'+str(scr)+'"',origin='delmod') ms.close() except Exception, instance: print '*** Error ***',instance
def setjy(vis=None, field=None, spw=None, selectdata=None, timerange=None, scan=None, intent=None, observation=None, scalebychan=None, standard=None, model=None, modimage=None, listmodels=None, fluxdensity=None, spix=None, reffreq=None, polindex=None, polangle=None, rotmeas=None, fluxdict=None, useephemdir=None, interpolation=None, usescratch=None, ismms=None): """Fills the model column for flux density calibrators.""" casalog.origin('setjy') casalog.post("standard="+standard,'DEBUG1') mylocals = locals() if not listmodels: # listmmodels=T does not require vis sh = SetjyHelper(vis) rstat = sh.resetModelCol() # Take care of the trivial parallelization if ( not listmodels and ParallelTaskHelper.isParallelMS(vis) and usescratch): # jagonzal: We actually operate in parallel when usescratch=True because only # in this case there is a good trade-off between the parallelization overhead # and speed up due to the load involved with MODEL_DATA column creation # Create the default MODEL columns in all sub-MSs to avoid corruption of the MMS # when there are NULL MS selections # # TT: Use ismms is used to change behavior of some of the execption handling # for MMS case. It is a hidden task parameter only modified when input vis # is identified as MMS via SetjyHelper.resetModel(). #sh = SetjyHelper(vis) #rstat = sh.resetModelCol() if rstat: ismms=rstat mylocals['ismms']=ismms #print "mylocals now=",mylocals helper = ParallelTaskHelper('setjy', mylocals) helper._consolidateOutput = False #helper._consolidateOutput = True try: retval = helper.go() # Remove the subMS names from the returned dictionary #print "remove subms names ...retval=",retval if (any(isinstance(v,dict) for v in retval.itervalues())): for subMS in retval: dict_i = retval[subMS] if isinstance(dict_i,dict): retval = dict_i break else: casalog.post("Error in parallel processing of MMS",'SEVERE') retval = False except Exception, instance: retval = False else: casalog.post("Could not initialize MODEL columns in sub-MSs", 'SEVERE') retval = False
def initweights(vis=None, wtmode=None, tsystable=None, gainfield=None, interp=None, spwmap=None, dowtsp=None): casalog.origin('initweights') # Do the trivial parallelization if ParallelTaskHelper.isParallelMS(vis): helper = ParallelTaskHelper('initweights', locals()) helper.go() return #Python script try: mycb = cbtool() # only if vis exists... if ((type(vis) == str) & (os.path.exists(vis))): if wtmode.upper().find("TSYS") > -1: if not os.path.exists(tsystable): raise Exception, 'Tsys calibration table %s not found' % tsystable if len(spwmap) == 0: spwmap = [-1] if interp == "": interp = "linear" # ... and we are asked to do something... # open without adding anything! mycb.open(vis, compress=False, addcorr=False, addmodel=False) mycb.initweights(wtmode=wtmode, dowtsp=dowtsp, tsystable=tsystable, gainfield=gainfield, interp=interp, spwmap=spwmap) mycb.close() else: raise Exception, 'Visibility data set not found - please verify the name' #write history ms.open(vis, nomodify=False) ms.writehistory(message='taskname = initweights', origin='initweights') ms.writehistory(message='vis = "' + str(vis) + '"', origin='initweights') ms.writehistory(message='wtmode = "' + wtmode + '"', origin='initweights') ms.writehistory(message='dowtsp = "' + str(dowtsp) + '"', origin='initweights') ms.close() except Exception, instance: print '*** Error ***', instance
def uvcontsub(vis, field, fitspw, excludechans, combine, solint, fitorder, spw, want_cont): if ParallelTaskHelper.isParallelMS(vis): helper = ParallelTaskHelper('uvcontsub', locals()) helper._consolidateOutput = False retVar = helper.go() # Gather the list of continuum subtraction-SubMSs cont_subMS_list = [] contsub_subMS_list = [] for subMS in retVar: if retVar[subMS]: cont_subMS_list.append(subMS + ".cont") contsub_subMS_list.append(subMS + ".contsub") if len(cont_subMS_list) <= 0: casalog.post("No continuum-subtracted sub-MSs for concatenation", "SEVERE") return False # We have to sort the list because otherwise it # depends on the time the engines dispatches their sub-MSs cont_subMS_list.sort() contsub_subMS_list.sort() # deal with the pointing table auxfile = "uvcontsub_aux2_" + str(time.time()) pnrows = 0 try: mytb.open(vis + '/POINTING') pnrows = mytb.nrows() mytb.close() if (pnrows > 0): shutil.copytree(os.path.realpath(vis + '/POINTING'), auxfile) except Exception, instance: casalog.post( "Error handling POINTING table %s: %s" % (vis + '/POINTING', str(instance)), 'SEVERE') if want_cont: try: virtualconcat(concatvis=helper._arg['vis'] + ".cont", vis=cont_subMS_list, copypointing=False) except Exception, instance: casalog.post( "Error concatenating continuum sub-MSs %s: %s" % (str(cont_subMS_list), str(instance)), 'SEVERE')
def initweights(vis=None,wtmode=None,tsystable=None,gainfield=None,interp=None,spwmap=None,dowtsp=None): casalog.origin('initweights') # Do the trivial parallelization if ParallelTaskHelper.isMPIEnabled() and ParallelTaskHelper.isParallelMS(vis): tsystable = ParallelTaskHelper.findAbsPath(tsystable) helper = ParallelTaskHelper('initweights', locals()) helper.go() # Write history to MS. try: param_names = initweights.func_code.co_varnames[:initweights.func_code.co_argcount] param_vals = [eval(p) for p in param_names] casalog.post('Updating the history in the output', 'DEBUG1') write_history(ms, vis, 'initweights', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') return
def initweights(vis=None, wtmode=None, dowtsp=None): casalog.origin('initweights') # Do the trivial parallelization if ParallelTaskHelper.isParallelMS(vis): helper = ParallelTaskHelper('initweights', locals()) helper.go() return #Python script try: mycb = cbtool() # only if vis exists... if ((type(vis) == str) & (os.path.exists(vis))): # ... and we are asked to do something... # open without adding anything! mycb.open(vis, compress=False, addcorr=False, addmodel=False) mycb.initweights(wtmode=wtmode, dowtsp=dowtsp) mycb.close() else: raise Exception, 'Visibility data set not found - please verify the name' #write history ms.open(vis, nomodify=False) ms.writehistory(message='taskname = initweights', origin='initweights') ms.writehistory(message='vis = "' + str(vis) + '"', origin='initweights') ms.writehistory(message='wtmode = "' + wtmode + '"', origin='initweights') ms.writehistory(message='dowtsp = "' + str(dowtsp) + '"', origin='initweights') ms.close() except Exception, instance: print '*** Error ***', instance
def delmod(vis=None, otf=None, field=None, scr=None): casalog.origin('delmod') # Do the trivial parallelization if ParallelTaskHelper.isParallelMS(vis): helper = ParallelTaskHelper('delmod', locals()) helper.go() return #Python script try: # only if vis exists... if ((type(vis) == str) & (os.path.exists(vis))): # ... and we are asked to do something... # open without adding anything! _cb.open(vis, addcorr=False, addmodel=False) _cb.delmod(otf=otf, field=field, scr=scr) _cb.close() else: raise Exception, 'Visibility data set not found - please verify the name' # Write history to MS try: param_names = delmod.func_code.co_varnames[:delmod.func_code. co_argcount] param_vals = [eval(p) for p in param_names] write_history(mstool(), vis, 'delmod', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') except Exception, instance: print '*** Error ***', instance
def oldsplit(vis, outputvis, datacolumn, field, spw, width, antenna, timebin, timerange, scan, intent, array, uvrange, correlation, observation, combine, keepflags, keepmms): """Create a visibility subset from an existing visibility set: Keyword arguments: vis -- Name of input visibility file (MS) default: none; example: vis='ngc5921.ms' outputvis -- Name of output visibility file (MS) default: none; example: outputvis='ngc5921_src.ms' datacolumn -- Which data column to split out default='corrected'; example: datacolumn='data' Options: 'data', 'corrected', 'model', 'all', 'float_data', 'lag_data', 'float_data,data', and 'lag_data,data'. note: 'all' = whichever of the above that are present. field -- Field name default: field = '' means use all sources field = 1 # will get field_id=1 (if you give it an integer, it will retrieve the source with that index) field = '1328+307' specifies source '1328+307'. Minimum match can be used, egs field = '13*' will retrieve '1328+307' if it is unique or exists. Source names with imbedded blanks cannot be included. spw -- Spectral window index identifier default=-1 (all); example: spw=1 antenna -- antenna names default '' (all), antenna = '3 & 7' gives one baseline with antennaid = 3,7. timebin -- Interval width for time averaging. default: '0s' or '-1s' (no averaging) example: timebin='30s' timerange -- Time range default='' means all times. examples: timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss' timerange='< YYYY/MM/DD/HH:MM:SS.sss' timerange='> YYYY/MM/DD/HH:MM:SS.sss' timerange='< ddd/HH:MM:SS.sss' timerange='> ddd/HH:MM:SS.sss' scan -- Scan numbers to select. default '' (all). intent -- Scan intents to select. default '' (all). array -- (Sub)array IDs to select. default '' (all). uvrange -- uv distance range to select. default '' (all). correlation -- Select correlations, e.g. 'rr, ll' or ['XY', 'YX']. default '' (all). observation -- Select by observation ID(s). default '' (all). combine -- Data descriptors that time averaging can ignore: scan, and/or state Default '' (none) keepflags -- Keep flagged data, if possible Default True keepmms -- If the input is a multi-MS, make the output one, too. (experimental) Default: False """ casalog.origin('oldsplit') mylocals = locals() rval = True try: if (keepmms and ParallelTaskHelper.isParallelMS(vis)): if (timebin!='0s' and timebin!='-1s'): casalog.post('Averaging over time with keepmms=True may lead to results different\n' +' from those obtained with keepmms=False due to different binning.', 'WARN') myms = mstool() myms.open(vis) mses = myms.getreferencedtables() myms.close() mses.sort() nfail = 0 if os.path.exists(outputvis): raise ValueError, "Output MS %s already exists - will not overwrite." % outputvis tempout = outputvis+str(time.time()) os.mkdir(tempout) successfulmses = [] mastersubms = '' masterptab = '' emptyptab = tempout+'/EMPTY_POINTING' nochangeinpointing = (str(antenna)+str(timerange)=='') if nochangeinpointing: # resulting pointing table is the same for all # -> replace by empty table if it is a link and won't be modified anyway # and put back original into the master after split # find the master for m in mses: theptab = m+'/POINTING' if not os.path.islink(theptab): #print "is master ", theptab mastersubms = m masterptab = m+'/POINTING' # save time by not copying the POINTING table len(mses) times myttb = tbtool() myttb.open(masterptab) tmpp = myttb.copy(newtablename=emptyptab, norows=True) myttb.close() del myttb tmpp.close() del tmpp break mytb = tbtool() # prepare the input MMS for processing replaced = [] outputviss = [] theptabs = [] for m in mses: # make sure the SORTED_TABLE keywords are disabled mytb.open(m, nomodify=False) if 'SORTED_TABLE' in mytb.keywordnames(): tobedel = mytb.getkeyword('SORTED_TABLE').split(' ')[1] mytb.removekeyword('SORTED_TABLE') os.system('rm -rf '+tobedel) mytb.close() # deal with the POINTING table theptab = m+'/POINTING' theptabs.append(theptab) if nochangeinpointing and os.path.islink(theptab): #print "is link ", theptab os.remove(theptab) shutil.copytree(emptyptab, theptab) replaced.append(True) else: replaced.append(False) # run oldsplit outputviss.append(os.path.abspath(tempout+'/'+os.path.basename(m))) # end for # send off the jobs print 'Running split_core ... ' helper = ParallelTaskHelper('oldsplit', mylocals) helper.override_arg('outputvis',outputviss) helper._consolidateOutput = False goretval = helper.go() for i in xrange(len(mses)): m = mses[i] # deal with the POINTING table if replaced[i]: # restore link shutil.rmtree(theptabs[i], ignore_errors=True) os.symlink('../'+os.path.basename(mastersubms)+'/POINTING', theptabs[i]) # (link in target will be created my makeMMS) # accumulate list of successful splits if not goretval[m]: nfail+=1 else: successfulmses.append(outputviss[i]) if nfail>0: # there were unsuccessful splits if len(successfulmses)==0: casalog.post('Split failed in all subMSs.', 'WARN') rval=False else: casalog.post('*** Summary: there were failures in '+str(nfail)+' SUBMSs:', 'WARN') casalog.post('*** (these are harmless if they are caused by selection):', 'WARN') for m in mses: if not goretval[m]: casalog.post(os.path.basename(m)+': '+str(goretval[m]), 'WARN') else: casalog.post(os.path.basename(m)+': '+str(goretval[m]), 'NORMAL') casalog.post('Will construct MMS from subMSs with successful selection ...', 'NORMAL') if nochangeinpointing: # need to take care of POINTING table # in case the master subms did not make it if not (tempout+'/'+os.path.basename(mastersubms) in successfulmses): # old master subms was not selected. # copy the original masterptab into the new master shutil.rmtree(successfulmses[0]+'/POINTING') shutil.copytree(masterptab, successfulmses[0]+'/POINTING') if rval: # construct new MMS from the output if(width==1 and str(field)+str(spw)+str(antenna)+str(timerange)+str(scan)+str(intent)\ +str(array)+str(uvrange)+str(correlation)+str(observation)==''): ph.makeMMS(outputvis, successfulmses) else: myms.open(successfulmses[0], nomodify=False) auxfile = "split_aux_"+str(time.time()) for i in xrange(1,len(successfulmses)): myms.virtconcatenate(successfulmses[i], auxfile, '1Hz', '10mas', True) myms.close() os.remove(auxfile) ph.makeMMS(outputvis, successfulmses, True, ['POINTING']) shutil.rmtree(tempout, ignore_errors=True) else: # do not output an MMS rval = split_core(vis, outputvis, datacolumn, field, spw, width, antenna, timebin, timerange, scan, intent, array, uvrange, correlation, observation, combine, keepflags) except Exception, instance: casalog.post("*** Error: %s" % (instance), 'SEVERE') rval = False
def applycal( vis=None, field=None, spw=None, intent=None, selectdata=None, timerange=None, uvrange=None, antenna=None, scan=None, observation=None, msselect=None, docallib=None, callib=None, gaintable=None, gainfield=None, interp=None, spwmap=None, calwt=None, parang=None, applymode=None, flagbackup=None, ): # Python script casalog.origin('applycal') # Take care of the trivial parallelization if ParallelTaskHelper.isParallelMS(vis): # Back up the flags, if requested (and if necessary) if flagbackup and applymode != 'calonly' and applymode != 'trial': fh.backupFlags(aflocal=None, msfile=vis, prename='applycal') flagbackup = False # To be safe convert file names to absolute paths. gaintable = ParallelTaskHelper.findAbsPath(gaintable) helper = ParallelTaskHelper('applycal', locals()) ret = helper.go() if ParallelTaskHelper.getAsyncMode(): return ret else: return try: mycb = cbtool() if (type(vis) == str) & os.path.exists(vis): # add CORRECTED_DATA column mycb.open(filename=vis, compress=False, addcorr=True, addmodel=False) else: raise Exception, \ 'Visibility data set not found - please verify the name' # enforce default if unspecified if applymode == '': applymode = 'calflag' # Back up the flags, if requested (and if necessary) if flagbackup and applymode != 'calonly' and applymode \ != 'trial': fh.backupFlags(aflocal=None, msfile=vis, prename='applycal') # Do data selection according to selectdata if selectdata: # pass all data selection parameters in as specified mycb.selectvis( time=timerange, spw=spw, scan=scan, field=field, intent=intent, observation=str(observation), baseline=antenna, uvrange=uvrange, chanmode='none', msselect=msselect, ) else: # selectdata=F, so time,scan,baseline,uvrange,msselect='' # using spw and field specifications only mycb.selectvis( time='', spw=spw, scan='', field=field, intent=intent, observation='', baseline='', uvrange='', chanmode='none', msselect='', ) # Arrange applies.... if docallib: # by cal library from file # parsing using c++ parser thiscallib=mycb.parsecallibfile(callib) mycb.setcallib(thiscallib) else: # by traditional parameters ngaintab = 0 if gaintable != ['']: ngaintab = len(gaintable) ncalwt = len(calwt) if ncalwt == 1: calwt = [calwt[0] for i in range(ngaintab)] ngainfld = len(gainfield) nspwmap = len(spwmap) ninterp = len(interp) # handle list of list issues with spwmap if nspwmap > 0: if type(spwmap[0]) != list: # first element not a list, only one spwmap specified # make it a list of list spwmap = [spwmap] nspwmap = 1 for igt in range(ngaintab): if gaintable[igt] != '': # field selection is null unless specified thisgainfield = '' if igt < ngainfld: thisgainfield = gainfield[igt] # spwmap is null unless specifed thisspwmap = [-1] if igt < nspwmap: thisspwmap = spwmap[igt] # interp is 'linear' unless specified thisinterp = 'linear' if igt < ninterp: if interp[igt] == '': interp[igt] = thisinterp thisinterp = interp[igt] mycb.setapply( t=0.0, table=gaintable[igt], field=thisgainfield, calwt=calwt[igt], spwmap=thisspwmap, interp=thisinterp, ) # ...and now the specialized terms # Apply parallactic angle, if requested if parang: mycb.setapply(type='P') mycb.correct(applymode) # report what the flags did reportflags(mycb.activityrec()) mycb.close() # write history try: param_names = \ applycal.func_code.co_varnames[:applycal.func_code.co_argcount] param_vals = [eval(p) for p in param_names] write_history( mstool(), vis, 'applycal', param_names, param_vals, casalog, ) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % instance, 'WARN') except Exception, instance: print '*** Error ***', instance mycb.close() casalog.post("Error in applycal: %s" % str(instance), "SEVERE") raise Exception, "Error in applycal: "+str(instance)