def setjy(vis=None, field=None, spw=None, selectdata=None, timerange=None, scan=None, intent=None, observation=None, scalebychan=None, standard=None, model=None, modimage=None, listmodels=None, fluxdensity=None, spix=None, reffreq=None, polindex=None, polangle=None, rotmeas=None, fluxdict=None, useephemdir=None, interpolation=None, usescratch=None, ismms=None): """Fills the model column for flux density calibrators.""" casalog.origin('setjy') casalog.post("standard="+standard,'DEBUG1') mylocals = locals() if not listmodels: # listmmodels=T does not require vis sh = SetjyHelper(vis) rstat = sh.resetModelCol() # Take care of the trivial parallelization if ( not listmodels and ParallelTaskHelper.isParallelMS(vis) and usescratch): # jagonzal: We actually operate in parallel when usescratch=True because only # in this case there is a good trade-off between the parallelization overhead # and speed up due to the load involved with MODEL_DATA column creation # Create the default MODEL columns in all sub-MSs to avoid corruption of the MMS # when there are NULL MS selections # # TT: Use ismms is used to change behavior of some of the execption handling # for MMS case. It is a hidden task parameter only modified when input vis # is identified as MMS via SetjyHelper.resetModel(). #sh = SetjyHelper(vis) #rstat = sh.resetModelCol() if rstat: ismms=rstat mylocals['ismms']=ismms #print "mylocals now=",mylocals helper = ParallelTaskHelper('setjy', mylocals) helper._consolidateOutput = False #helper._consolidateOutput = True try: retval = helper.go() # Remove the subMS names from the returned dictionary #print "remove subms names ...retval=",retval if (any(isinstance(v,dict) for v in retval.itervalues())): for subMS in retval: dict_i = retval[subMS] if isinstance(dict_i,dict): retval = dict_i break else: casalog.post("Error in parallel processing of MMS",'SEVERE') retval = False except Exception, instance: retval = False else: casalog.post("Could not initialize MODEL columns in sub-MSs", 'SEVERE') retval = False
def uvcontsub(vis, field, fitspw, excludechans, combine, solint, fitorder, spw, want_cont): if ParallelTaskHelper.isParallelMS(vis): helper = ParallelTaskHelper('uvcontsub', locals()) helper._consolidateOutput = False retVar = helper.go() # Gather the list of continuum subtraction-SubMSs cont_subMS_list = [] contsub_subMS_list = [] for subMS in retVar: if retVar[subMS]: cont_subMS_list.append(subMS + ".cont") contsub_subMS_list.append(subMS + ".contsub") if len(cont_subMS_list) <= 0: casalog.post("No continuum-subtracted sub-MSs for concatenation", "SEVERE") return False # We have to sort the list because otherwise it # depends on the time the engines dispatches their sub-MSs cont_subMS_list.sort() contsub_subMS_list.sort() # deal with the pointing table auxfile = "uvcontsub_aux2_" + str(time.time()) pnrows = 0 try: mytb.open(vis + '/POINTING') pnrows = mytb.nrows() mytb.close() if (pnrows > 0): shutil.copytree(os.path.realpath(vis + '/POINTING'), auxfile) except Exception, instance: casalog.post( "Error handling POINTING table %s: %s" % (vis + '/POINTING', str(instance)), 'SEVERE') if want_cont: try: virtualconcat(concatvis=helper._arg['vis'] + ".cont", vis=cont_subMS_list, copypointing=False) except Exception, instance: casalog.post( "Error concatenating continuum sub-MSs %s: %s" % (str(cont_subMS_list), str(instance)), 'SEVERE')
def uvcontsub(vis, field, fitspw, excludechans, combine, solint, fitorder, spw, want_cont): if ParallelTaskHelper.isParallelMS(vis): helper = ParallelTaskHelper('uvcontsub', locals()) helper._consolidateOutput = False retVar = helper.go() # Gather the list of continuum subtraction-SubMSs cont_subMS_list = [] contsub_subMS_list = [] for subMS in retVar: if retVar[subMS]: cont_subMS_list.append(subMS + ".cont") contsub_subMS_list.append(subMS + ".contsub") if len(cont_subMS_list) <= 0: casalog.post("No continuum-subtracted sub-MSs for concatenation","SEVERE") return False # We have to sort the list because otherwise it # depends on the time the engines dispatches their sub-MSs cont_subMS_list.sort() contsub_subMS_list.sort() # deal with the pointing table auxfile = "uvcontsub_aux2_"+str(time.time()) pnrows = 0 try: mytb.open(vis+'/POINTING') pnrows = mytb.nrows() mytb.close() if(pnrows>0): shutil.copytree(os.path.realpath(vis+'/POINTING'), auxfile) except Exception, instance: casalog.post("Error handling POINTING table %s: %s" % (vis+'/POINTING',str(instance)),'SEVERE') if want_cont: try: virtualconcat(concatvis=helper._arg['vis'] + ".cont",vis=cont_subMS_list, copypointing=False) except Exception, instance: casalog.post("Error concatenating continuum sub-MSs %s: %s" % (str(cont_subMS_list),str(instance)),'SEVERE')
def oldsplit(vis, outputvis, datacolumn, field, spw, width, antenna, timebin, timerange, scan, intent, array, uvrange, correlation, observation, combine, keepflags, keepmms): """Create a visibility subset from an existing visibility set: Keyword arguments: vis -- Name of input visibility file (MS) default: none; example: vis='ngc5921.ms' outputvis -- Name of output visibility file (MS) default: none; example: outputvis='ngc5921_src.ms' datacolumn -- Which data column to split out default='corrected'; example: datacolumn='data' Options: 'data', 'corrected', 'model', 'all', 'float_data', 'lag_data', 'float_data,data', and 'lag_data,data'. note: 'all' = whichever of the above that are present. field -- Field name default: field = '' means use all sources field = 1 # will get field_id=1 (if you give it an integer, it will retrieve the source with that index) field = '1328+307' specifies source '1328+307'. Minimum match can be used, egs field = '13*' will retrieve '1328+307' if it is unique or exists. Source names with imbedded blanks cannot be included. spw -- Spectral window index identifier default=-1 (all); example: spw=1 antenna -- antenna names default '' (all), antenna = '3 & 7' gives one baseline with antennaid = 3,7. timebin -- Interval width for time averaging. default: '0s' or '-1s' (no averaging) example: timebin='30s' timerange -- Time range default='' means all times. examples: timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss' timerange='< YYYY/MM/DD/HH:MM:SS.sss' timerange='> YYYY/MM/DD/HH:MM:SS.sss' timerange='< ddd/HH:MM:SS.sss' timerange='> ddd/HH:MM:SS.sss' scan -- Scan numbers to select. default '' (all). intent -- Scan intents to select. default '' (all). array -- (Sub)array IDs to select. default '' (all). uvrange -- uv distance range to select. default '' (all). correlation -- Select correlations, e.g. 'rr, ll' or ['XY', 'YX']. default '' (all). observation -- Select by observation ID(s). default '' (all). combine -- Data descriptors that time averaging can ignore: scan, and/or state Default '' (none) keepflags -- Keep flagged data, if possible Default True keepmms -- If the input is a multi-MS, make the output one, too. (experimental) Default: False """ casalog.origin('oldsplit') mylocals = locals() rval = True try: if (keepmms and ParallelTaskHelper.isParallelMS(vis)): if (timebin!='0s' and timebin!='-1s'): casalog.post('Averaging over time with keepmms=True may lead to results different\n' +' from those obtained with keepmms=False due to different binning.', 'WARN') myms = mstool() myms.open(vis) mses = myms.getreferencedtables() myms.close() mses.sort() nfail = 0 if os.path.exists(outputvis): raise ValueError, "Output MS %s already exists - will not overwrite." % outputvis tempout = outputvis+str(time.time()) os.mkdir(tempout) successfulmses = [] mastersubms = '' masterptab = '' emptyptab = tempout+'/EMPTY_POINTING' nochangeinpointing = (str(antenna)+str(timerange)=='') if nochangeinpointing: # resulting pointing table is the same for all # -> replace by empty table if it is a link and won't be modified anyway # and put back original into the master after split # find the master for m in mses: theptab = m+'/POINTING' if not os.path.islink(theptab): #print "is master ", theptab mastersubms = m masterptab = m+'/POINTING' # save time by not copying the POINTING table len(mses) times myttb = tbtool() myttb.open(masterptab) tmpp = myttb.copy(newtablename=emptyptab, norows=True) myttb.close() del myttb tmpp.close() del tmpp break mytb = tbtool() # prepare the input MMS for processing replaced = [] outputviss = [] theptabs = [] for m in mses: # make sure the SORTED_TABLE keywords are disabled mytb.open(m, nomodify=False) if 'SORTED_TABLE' in mytb.keywordnames(): tobedel = mytb.getkeyword('SORTED_TABLE').split(' ')[1] mytb.removekeyword('SORTED_TABLE') os.system('rm -rf '+tobedel) mytb.close() # deal with the POINTING table theptab = m+'/POINTING' theptabs.append(theptab) if nochangeinpointing and os.path.islink(theptab): #print "is link ", theptab os.remove(theptab) shutil.copytree(emptyptab, theptab) replaced.append(True) else: replaced.append(False) # run oldsplit outputviss.append(os.path.abspath(tempout+'/'+os.path.basename(m))) # end for # send off the jobs print 'Running split_core ... ' helper = ParallelTaskHelper('oldsplit', mylocals) helper.override_arg('outputvis',outputviss) helper._consolidateOutput = False goretval = helper.go() for i in xrange(len(mses)): m = mses[i] # deal with the POINTING table if replaced[i]: # restore link shutil.rmtree(theptabs[i], ignore_errors=True) os.symlink('../'+os.path.basename(mastersubms)+'/POINTING', theptabs[i]) # (link in target will be created my makeMMS) # accumulate list of successful splits if not goretval[m]: nfail+=1 else: successfulmses.append(outputviss[i]) if nfail>0: # there were unsuccessful splits if len(successfulmses)==0: casalog.post('Split failed in all subMSs.', 'WARN') rval=False else: casalog.post('*** Summary: there were failures in '+str(nfail)+' SUBMSs:', 'WARN') casalog.post('*** (these are harmless if they are caused by selection):', 'WARN') for m in mses: if not goretval[m]: casalog.post(os.path.basename(m)+': '+str(goretval[m]), 'WARN') else: casalog.post(os.path.basename(m)+': '+str(goretval[m]), 'NORMAL') casalog.post('Will construct MMS from subMSs with successful selection ...', 'NORMAL') if nochangeinpointing: # need to take care of POINTING table # in case the master subms did not make it if not (tempout+'/'+os.path.basename(mastersubms) in successfulmses): # old master subms was not selected. # copy the original masterptab into the new master shutil.rmtree(successfulmses[0]+'/POINTING') shutil.copytree(masterptab, successfulmses[0]+'/POINTING') if rval: # construct new MMS from the output if(width==1 and str(field)+str(spw)+str(antenna)+str(timerange)+str(scan)+str(intent)\ +str(array)+str(uvrange)+str(correlation)+str(observation)==''): ph.makeMMS(outputvis, successfulmses) else: myms.open(successfulmses[0], nomodify=False) auxfile = "split_aux_"+str(time.time()) for i in xrange(1,len(successfulmses)): myms.virtconcatenate(successfulmses[i], auxfile, '1Hz', '10mas', True) myms.close() os.remove(auxfile) ph.makeMMS(outputvis, successfulmses, True, ['POINTING']) shutil.rmtree(tempout, ignore_errors=True) else: # do not output an MMS rval = split_core(vis, outputvis, datacolumn, field, spw, width, antenna, timebin, timerange, scan, intent, array, uvrange, correlation, observation, combine, keepflags) except Exception, instance: casalog.post("*** Error: %s" % (instance), 'SEVERE') rval = False