def setjy(vis=None, field=None, spw=None,
selectdata=None, timerange=None, scan=None, intent=None, observation=None,
scalebychan=None, standard=None, model=None, modimage=None,
listmodels=None, fluxdensity=None, spix=None, reffreq=None, polindex=None,
polangle=None, rotmeas=None, fluxdict=None,
useephemdir=None, interpolation=None, usescratch=None, ismms=None):
"""Fills the model column for flux density calibrators."""
casalog.origin('setjy')
casalog.post("standard="+standard,'DEBUG1')
mylocals = locals()
if not listmodels: # listmmodels=T does not require vis
sh = SetjyHelper(vis)
rstat = sh.resetModelCol()
# Take care of the trivial parallelization
if ( not listmodels and ParallelTaskHelper.isParallelMS(vis) and usescratch):
# jagonzal: We actually operate in parallel when usescratch=True because only
# in this case there is a good trade-off between the parallelization overhead
# and speed up due to the load involved with MODEL_DATA column creation
# Create the default MODEL columns in all sub-MSs to avoid corruption of the MMS
# when there are NULL MS selections
#
# TT: Use ismms is used to change behavior of some of the execption handling
# for MMS case. It is a hidden task parameter only modified when input vis
# is identified as MMS via SetjyHelper.resetModel().
#sh = SetjyHelper(vis)
#rstat = sh.resetModelCol()
if rstat:
ismms=rstat
mylocals['ismms']=ismms
#print "mylocals now=",mylocals
helper = ParallelTaskHelper('setjy', mylocals)
helper._consolidateOutput = False
#helper._consolidateOutput = True
try:
retval = helper.go()
# Remove the subMS names from the returned dictionary
#print "remove subms names ...retval=",retval
if (any(isinstance(v,dict) for v in retval.itervalues())):
for subMS in retval:
dict_i = retval[subMS]
if isinstance(dict_i,dict):
retval = dict_i
break
else:
casalog.post("Error in parallel processing of MMS",'SEVERE')
retval = False
except Exception, instance:
retval = False
else:
casalog.post("Could not initialize MODEL columns in sub-MSs", 'SEVERE')
retval = False
def setjy(vis=None, field=None, spw=None,
selectdata=None, timerange=None, scan=None, intent=None, observation=None,
scalebychan=None, standard=None, model=None, modimage=None,
listmodels=None, fluxdensity=None, spix=None, reffreq=None, polindex=None,
polangle=None, rotmeas=None, fluxdict=None,
useephemdir=None, interpolation=None, usescratch=None, ismms=None):
"""Fills the model column for flux density calibrators."""
casalog.origin('setjy')
casalog.post("standard="+standard,'DEBUG1')
mylocals = locals()
if not listmodels: # listmmodels=T does not require vis
sh = SetjyHelper(vis)
rstat = sh.resetModelCol()
# Take care of the trivial parallelization
if ( not listmodels and ParallelTaskHelper.isParallelMS(vis) and usescratch):
# jagonzal: We actually operate in parallel when usescratch=True because only
# in this case there is a good trade-off between the parallelization overhead
# and speed up due to the load involved with MODEL_DATA column creation
# Create the default MODEL columns in all sub-MSs to avoid corruption of the MMS
# when there are NULL MS selections
#
# TT: Use ismms is used to change behavior of some of the execption handling
# for MMS case. It is a hidden task parameter only modified when input vis
# is identified as MMS via SetjyHelper.resetModel().
#sh = SetjyHelper(vis)
#rstat = sh.resetModelCol()
if rstat:
ismms=rstat
mylocals['ismms']=ismms
#print "mylocals now=",mylocals
helper = ParallelTaskHelper('setjy', mylocals)
helper._consolidateOutput = False
#helper._consolidateOutput = True
try:
retval = helper.go()
# Remove the subMS names from the returned dictionary
#print "remove subms names ...retval=",retval
if (any(isinstance(v,dict) for v in retval.itervalues())):
for subMS in retval:
dict_i = retval[subMS]
if isinstance(dict_i,dict):
retval = dict_i
break
else:
casalog.post("Error in parallel processing of MMS",'SEVERE')
retval = False
except Exception, instance:
retval = False
else:
casalog.post("Could not initialize MODEL columns in sub-MSs", 'SEVERE')
retval = False
def uvcontsub(vis, field, fitspw, excludechans, combine, solint, fitorder, spw,
want_cont):
if ParallelTaskHelper.isParallelMS(vis):
helper = ParallelTaskHelper('uvcontsub', locals())
helper._consolidateOutput = False
retVar = helper.go()
# Gather the list of continuum subtraction-SubMSs
cont_subMS_list = []
contsub_subMS_list = []
for subMS in retVar:
if retVar[subMS]:
cont_subMS_list.append(subMS + ".cont")
contsub_subMS_list.append(subMS + ".contsub")
if len(cont_subMS_list) <= 0:
casalog.post("No continuum-subtracted sub-MSs for concatenation",
"SEVERE")
return False
# We have to sort the list because otherwise it
# depends on the time the engines dispatches their sub-MSs
cont_subMS_list.sort()
contsub_subMS_list.sort()
# deal with the pointing table
auxfile = "uvcontsub_aux2_" + str(time.time())
pnrows = 0
try:
mytb.open(vis + '/POINTING')
pnrows = mytb.nrows()
mytb.close()
if (pnrows > 0):
shutil.copytree(os.path.realpath(vis + '/POINTING'), auxfile)
except Exception, instance:
casalog.post(
"Error handling POINTING table %s: %s" %
(vis + '/POINTING', str(instance)), 'SEVERE')
if want_cont:
try:
virtualconcat(concatvis=helper._arg['vis'] + ".cont",
vis=cont_subMS_list,
copypointing=False)
except Exception, instance:
casalog.post(
"Error concatenating continuum sub-MSs %s: %s" %
(str(cont_subMS_list), str(instance)), 'SEVERE')
def uvcontsub(vis, field, fitspw, excludechans, combine, solint, fitorder, spw, want_cont):
if ParallelTaskHelper.isParallelMS(vis):
helper = ParallelTaskHelper('uvcontsub', locals())
helper._consolidateOutput = False
retVar = helper.go()
# Gather the list of continuum subtraction-SubMSs
cont_subMS_list = []
contsub_subMS_list = []
for subMS in retVar:
if retVar[subMS]:
cont_subMS_list.append(subMS + ".cont")
contsub_subMS_list.append(subMS + ".contsub")
if len(cont_subMS_list) <= 0:
casalog.post("No continuum-subtracted sub-MSs for concatenation","SEVERE")
return False
# We have to sort the list because otherwise it
# depends on the time the engines dispatches their sub-MSs
cont_subMS_list.sort()
contsub_subMS_list.sort()
# deal with the pointing table
auxfile = "uvcontsub_aux2_"+str(time.time())
pnrows = 0
try:
mytb.open(vis+'/POINTING')
pnrows = mytb.nrows()
mytb.close()
if(pnrows>0):
shutil.copytree(os.path.realpath(vis+'/POINTING'), auxfile)
except Exception, instance:
casalog.post("Error handling POINTING table %s: %s" %
(vis+'/POINTING',str(instance)),'SEVERE')
if want_cont:
try:
virtualconcat(concatvis=helper._arg['vis'] + ".cont",vis=cont_subMS_list,
copypointing=False)
except Exception, instance:
casalog.post("Error concatenating continuum sub-MSs %s: %s" %
(str(cont_subMS_list),str(instance)),'SEVERE')
def oldsplit(vis, outputvis, datacolumn, field, spw, width, antenna,
timebin, timerange, scan, intent, array, uvrange,
correlation, observation, combine, keepflags, keepmms):
"""Create a visibility subset from an existing visibility set:
Keyword arguments:
vis -- Name of input visibility file (MS)
default: none; example: vis='ngc5921.ms'
outputvis -- Name of output visibility file (MS)
default: none; example: outputvis='ngc5921_src.ms'
datacolumn -- Which data column to split out
default='corrected'; example: datacolumn='data'
Options: 'data', 'corrected', 'model', 'all',
'float_data', 'lag_data', 'float_data,data', and
'lag_data,data'.
note: 'all' = whichever of the above that are present.
field -- Field name
default: field = '' means use all sources
field = 1 # will get field_id=1 (if you give it an
integer, it will retrieve the source with that index)
field = '1328+307' specifies source '1328+307'.
Minimum match can be used, egs field = '13*' will
retrieve '1328+307' if it is unique or exists.
Source names with imbedded blanks cannot be included.
spw -- Spectral window index identifier
default=-1 (all); example: spw=1
antenna -- antenna names
default '' (all),
antenna = '3 & 7' gives one baseline with antennaid = 3,7.
timebin -- Interval width for time averaging.
default: '0s' or '-1s' (no averaging)
example: timebin='30s'
timerange -- Time range
default='' means all times. examples:
timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
timerange='< YYYY/MM/DD/HH:MM:SS.sss'
timerange='> YYYY/MM/DD/HH:MM:SS.sss'
timerange='< ddd/HH:MM:SS.sss'
timerange='> ddd/HH:MM:SS.sss'
scan -- Scan numbers to select.
default '' (all).
intent -- Scan intents to select.
default '' (all).
array -- (Sub)array IDs to select.
default '' (all).
uvrange -- uv distance range to select.
default '' (all).
correlation -- Select correlations, e.g. 'rr, ll' or ['XY', 'YX'].
default '' (all).
observation -- Select by observation ID(s).
default '' (all).
combine -- Data descriptors that time averaging can ignore:
scan, and/or state
Default '' (none)
keepflags -- Keep flagged data, if possible
Default True
keepmms -- If the input is a multi-MS, make the output one, too. (experimental)
Default: False
"""
casalog.origin('oldsplit')
mylocals = locals()
rval = True
try:
if (keepmms and ParallelTaskHelper.isParallelMS(vis)):
if (timebin!='0s' and timebin!='-1s'):
casalog.post('Averaging over time with keepmms=True may lead to results different\n'
+' from those obtained with keepmms=False due to different binning.', 'WARN')
myms = mstool()
myms.open(vis)
mses = myms.getreferencedtables()
myms.close()
mses.sort()
nfail = 0
if os.path.exists(outputvis):
raise ValueError, "Output MS %s already exists - will not overwrite." % outputvis
tempout = outputvis+str(time.time())
os.mkdir(tempout)
successfulmses = []
mastersubms = ''
masterptab = ''
emptyptab = tempout+'/EMPTY_POINTING'
nochangeinpointing = (str(antenna)+str(timerange)=='')
if nochangeinpointing:
# resulting pointing table is the same for all
# -> replace by empty table if it is a link and won't be modified anyway
# and put back original into the master after split
# find the master
for m in mses:
theptab = m+'/POINTING'
if not os.path.islink(theptab):
#print "is master ", theptab
mastersubms = m
masterptab = m+'/POINTING'
# save time by not copying the POINTING table len(mses) times
myttb = tbtool()
myttb.open(masterptab)
tmpp = myttb.copy(newtablename=emptyptab, norows=True)
myttb.close()
del myttb
tmpp.close()
del tmpp
break
mytb = tbtool()
# prepare the input MMS for processing
replaced = []
outputviss = []
theptabs = []
for m in mses:
# make sure the SORTED_TABLE keywords are disabled
mytb.open(m, nomodify=False)
if 'SORTED_TABLE' in mytb.keywordnames():
tobedel = mytb.getkeyword('SORTED_TABLE').split(' ')[1]
mytb.removekeyword('SORTED_TABLE')
os.system('rm -rf '+tobedel)
mytb.close()
# deal with the POINTING table
theptab = m+'/POINTING'
theptabs.append(theptab)
if nochangeinpointing and os.path.islink(theptab):
#print "is link ", theptab
os.remove(theptab)
shutil.copytree(emptyptab, theptab)
replaced.append(True)
else:
replaced.append(False)
# run oldsplit
outputviss.append(os.path.abspath(tempout+'/'+os.path.basename(m)))
# end for
# send off the jobs
print 'Running split_core ... '
helper = ParallelTaskHelper('oldsplit', mylocals)
helper.override_arg('outputvis',outputviss)
helper._consolidateOutput = False
goretval = helper.go()
for i in xrange(len(mses)):
m = mses[i]
# deal with the POINTING table
if replaced[i]:
# restore link
shutil.rmtree(theptabs[i], ignore_errors=True)
os.symlink('../'+os.path.basename(mastersubms)+'/POINTING', theptabs[i])
# (link in target will be created my makeMMS)
# accumulate list of successful splits
if not goretval[m]:
nfail+=1
else:
successfulmses.append(outputviss[i])
if nfail>0: # there were unsuccessful splits
if len(successfulmses)==0:
casalog.post('Split failed in all subMSs.', 'WARN')
rval=False
else:
casalog.post('*** Summary: there were failures in '+str(nfail)+' SUBMSs:', 'WARN')
casalog.post('*** (these are harmless if they are caused by selection):', 'WARN')
for m in mses:
if not goretval[m]:
casalog.post(os.path.basename(m)+': '+str(goretval[m]), 'WARN')
else:
casalog.post(os.path.basename(m)+': '+str(goretval[m]), 'NORMAL')
casalog.post('Will construct MMS from subMSs with successful selection ...', 'NORMAL')
if nochangeinpointing: # need to take care of POINTING table
# in case the master subms did not make it
if not (tempout+'/'+os.path.basename(mastersubms) in successfulmses):
# old master subms was not selected.
# copy the original masterptab into the new master
shutil.rmtree(successfulmses[0]+'/POINTING')
shutil.copytree(masterptab, successfulmses[0]+'/POINTING')
if rval: # construct new MMS from the output
if(width==1 and str(field)+str(spw)+str(antenna)+str(timerange)+str(scan)+str(intent)\
+str(array)+str(uvrange)+str(correlation)+str(observation)==''):
ph.makeMMS(outputvis, successfulmses)
else:
myms.open(successfulmses[0], nomodify=False)
auxfile = "split_aux_"+str(time.time())
for i in xrange(1,len(successfulmses)):
myms.virtconcatenate(successfulmses[i], auxfile, '1Hz', '10mas', True)
myms.close()
os.remove(auxfile)
ph.makeMMS(outputvis, successfulmses, True, ['POINTING'])
shutil.rmtree(tempout, ignore_errors=True)
else: # do not output an MMS
rval = split_core(vis, outputvis, datacolumn, field, spw, width, antenna,
timebin, timerange, scan, intent, array, uvrange,
correlation, observation, combine, keepflags)
except Exception, instance:
casalog.post("*** Error: %s" % (instance), 'SEVERE')
rval = False
