def oldsplit(vis, outputvis, datacolumn, field, spw, width, antenna,
timebin, timerange, scan, intent, array, uvrange,
correlation, observation, combine, keepflags, keepmms):
"""Create a visibility subset from an existing visibility set:
Keyword arguments:
vis -- Name of input visibility file (MS)
default: none; example: vis='ngc5921.ms'
outputvis -- Name of output visibility file (MS)
default: none; example: outputvis='ngc5921_src.ms'
datacolumn -- Which data column to split out
default='corrected'; example: datacolumn='data'
Options: 'data', 'corrected', 'model', 'all',
'float_data', 'lag_data', 'float_data,data', and
'lag_data,data'.
note: 'all' = whichever of the above that are present.
field -- Field name
default: field = '' means use all sources
field = 1 # will get field_id=1 (if you give it an
integer, it will retrieve the source with that index)
field = '1328+307' specifies source '1328+307'.
Minimum match can be used, egs field = '13*' will
retrieve '1328+307' if it is unique or exists.
Source names with imbedded blanks cannot be included.
spw -- Spectral window index identifier
default=-1 (all); example: spw=1
antenna -- antenna names
default '' (all),
antenna = '3 & 7' gives one baseline with antennaid = 3,7.
timebin -- Interval width for time averaging.
default: '0s' or '-1s' (no averaging)
example: timebin='30s'
timerange -- Time range
default='' means all times. examples:
timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
timerange='< YYYY/MM/DD/HH:MM:SS.sss'
timerange='> YYYY/MM/DD/HH:MM:SS.sss'
timerange='< ddd/HH:MM:SS.sss'
timerange='> ddd/HH:MM:SS.sss'
scan -- Scan numbers to select.
default '' (all).
intent -- Scan intents to select.
default '' (all).
array -- (Sub)array IDs to select.
default '' (all).
uvrange -- uv distance range to select.
default '' (all).
correlation -- Select correlations, e.g. 'rr, ll' or ['XY', 'YX'].
default '' (all).
observation -- Select by observation ID(s).
default '' (all).
combine -- Data descriptors that time averaging can ignore:
scan, and/or state
Default '' (none)
keepflags -- Keep flagged data, if possible
Default True
keepmms -- If the input is a multi-MS, make the output one, too. (experimental)
Default: False
"""
casalog.origin('oldsplit')
mylocals = locals()
rval = True
try:
if (keepmms and ParallelTaskHelper.isParallelMS(vis)):
if (timebin!='0s' and timebin!='-1s'):
casalog.post('Averaging over time with keepmms=True may lead to results different\n'
+' from those obtained with keepmms=False due to different binning.', 'WARN')
myms = mstool()
myms.open(vis)
mses = myms.getreferencedtables()
myms.close()
mses.sort()
nfail = 0
if os.path.exists(outputvis):
raise ValueError, "Output MS %s already exists - will not overwrite." % outputvis
tempout = outputvis+str(time.time())
os.mkdir(tempout)
successfulmses = []
mastersubms = ''
masterptab = ''
emptyptab = tempout+'/EMPTY_POINTING'
nochangeinpointing = (str(antenna)+str(timerange)=='')
if nochangeinpointing:
# resulting pointing table is the same for all
# -> replace by empty table if it is a link and won't be modified anyway
# and put back original into the master after split
# find the master
for m in mses:
theptab = m+'/POINTING'
if not os.path.islink(theptab):
#print "is master ", theptab
mastersubms = m
masterptab = m+'/POINTING'
# save time by not copying the POINTING table len(mses) times
myttb = tbtool()
myttb.open(masterptab)
tmpp = myttb.copy(newtablename=emptyptab, norows=True)
myttb.close()
del myttb
tmpp.close()
del tmpp
break
mytb = tbtool()
# prepare the input MMS for processing
replaced = []
outputviss = []
theptabs = []
for m in mses:
# make sure the SORTED_TABLE keywords are disabled
mytb.open(m, nomodify=False)
if 'SORTED_TABLE' in mytb.keywordnames():
tobedel = mytb.getkeyword('SORTED_TABLE').split(' ')[1]
mytb.removekeyword('SORTED_TABLE')
os.system('rm -rf '+tobedel)
mytb.close()
# deal with the POINTING table
theptab = m+'/POINTING'
theptabs.append(theptab)
if nochangeinpointing and os.path.islink(theptab):
#print "is link ", theptab
os.remove(theptab)
shutil.copytree(emptyptab, theptab)
replaced.append(True)
else:
replaced.append(False)
# run oldsplit
outputviss.append(os.path.abspath(tempout+'/'+os.path.basename(m)))
# end for
# send off the jobs
print 'Running split_core ... '
helper = ParallelTaskHelper('oldsplit', mylocals)
helper.override_arg('outputvis',outputviss)
helper._consolidateOutput = False
goretval = helper.go()
for i in xrange(len(mses)):
m = mses[i]
# deal with the POINTING table
if replaced[i]:
# restore link
shutil.rmtree(theptabs[i], ignore_errors=True)
os.symlink('../'+os.path.basename(mastersubms)+'/POINTING', theptabs[i])
# (link in target will be created my makeMMS)
# accumulate list of successful splits
if not goretval[m]:
nfail+=1
else:
successfulmses.append(outputviss[i])
if nfail>0: # there were unsuccessful splits
if len(successfulmses)==0:
casalog.post('Split failed in all subMSs.', 'WARN')
rval=False
else:
casalog.post('*** Summary: there were failures in '+str(nfail)+' SUBMSs:', 'WARN')
casalog.post('*** (these are harmless if they are caused by selection):', 'WARN')
for m in mses:
if not goretval[m]:
casalog.post(os.path.basename(m)+': '+str(goretval[m]), 'WARN')
else:
casalog.post(os.path.basename(m)+': '+str(goretval[m]), 'NORMAL')
casalog.post('Will construct MMS from subMSs with successful selection ...', 'NORMAL')
if nochangeinpointing: # need to take care of POINTING table
# in case the master subms did not make it
if not (tempout+'/'+os.path.basename(mastersubms) in successfulmses):
# old master subms was not selected.
# copy the original masterptab into the new master
shutil.rmtree(successfulmses[0]+'/POINTING')
shutil.copytree(masterptab, successfulmses[0]+'/POINTING')
if rval: # construct new MMS from the output
if(width==1 and str(field)+str(spw)+str(antenna)+str(timerange)+str(scan)+str(intent)\
+str(array)+str(uvrange)+str(correlation)+str(observation)==''):
ph.makeMMS(outputvis, successfulmses)
else:
myms.open(successfulmses[0], nomodify=False)
auxfile = "split_aux_"+str(time.time())
for i in xrange(1,len(successfulmses)):
myms.virtconcatenate(successfulmses[i], auxfile, '1Hz', '10mas', True)
myms.close()
os.remove(auxfile)
ph.makeMMS(outputvis, successfulmses, True, ['POINTING'])
shutil.rmtree(tempout, ignore_errors=True)
else: # do not output an MMS
rval = split_core(vis, outputvis, datacolumn, field, spw, width, antenna,
timebin, timerange, scan, intent, array, uvrange,
correlation, observation, combine, keepflags)
except Exception, instance:
casalog.post("*** Error: %s" % (instance), 'SEVERE')
rval = False
def virtualconcat(vislist,concatvis,freqtol,dirtol,respectname,
visweightscale,keepcopy,copypointing):
"""
Concatenate visibility data sets creating a multi-MS.
Combine the input datasets into a multi-MS.
NOTE: The input datasets are moved into the multi-MS and may be modified
to account for subtable reference changes.
If none of the input MSs have any scratch columns, none are created.
Otherwise scratch columns are created and initialized in those MSs
which don't have a complete set.
Keyword arguments:
vis -- Name of input visibility files (MS)
default: none; example: vis=['ngc5921-1.ms', 'ngc5921-2.ms']
concatvis -- Name of the output visibility file
default: none; example: concatvis='src2.ms'
freqtol -- Frequency shift tolerance for considering data as the same spwid
default: '' means always combine
example: freqtol='10MHz' will not combine spwid unless they are
within 10 MHz
dirtol -- Direction shift tolerance for considering data as the same field
default: ;; means always combine
example: dirtol='1.arcsec' will not combine data for a field unless
their phase center is less than 1 arcsec.
respectname -- If true, fields with a different name are not merged even if their
direction agrees (within dirtol)
default: True
visweightscale -- list of the weight scales to be applied to the individual MSs
default: [] (don't modify weights, equivalent to setting scale to 1 for each MS)
keepcopy -- If true, a copy of the input MSs is kept in their original place.
default: false
copypointing -- If true, the POINTING table information will be present in the output.
If false, the result is an empty POINTING table.
default: True
"""
###
#Python script
tempdir = ''
originalvis = vislist
try:
casalog.origin('virtualconcat')
t = tbtool()
m = mstool()
#break the reference between vis and vislist as we modify vis
if(type(vislist)==str):
vis=[vislist]
else:
vis=list(vislist)
#dto. for concavis
theconcatvis = concatvis
doweightscale = False
if(len(visweightscale)>0):
if (len(visweightscale) != len(vis)):
raise Exception, 'parameter visweightscale must have same number of elements as parameter vis'
for factor in visweightscale:
if factor<0.:
raise Exception, 'parameter visweightscale must only contain positive numbers'
elif factor!=1.:
doweightscale=True
if((type(concatvis)!=str) or (len(concatvis.split()) < 1)):
raise Exception, 'Parameter concatvis is invalid.'
if(vis.count(concatvis) > 0):
raise Exception, 'Parameter concatvis must not be equal to one of the members of parameter vis.'
if(os.path.exists(concatvis)):
raise Exception, 'The output MMS must not yet exist.'
# process the input MSs in chronological order
sortedvis = []
sortedvisweightscale = []
namestuples = []
for name in vis:
t.open(name)
times = t.getcol('TIME')
t.close()
times.sort()
if doweightscale:
namestuples.append( (times[0], name, visweightscale[vis.index(name)]) )
else:
namestuples.append( (times[0], name, 0) )
sorted_namestuples = sorted(namestuples, key=lambda msname: msname[0])
for i in range(0,len(vis)):
sortedvis.append(sorted_namestuples[i][1])
sortedvisweightscale.append(sorted_namestuples[i][2])
if not vis == sortedvis:
casalog.post('The list of input MSs is not in chronological order and will need to be sorted.' , 'INFO')
casalog.post('The chronological order in which the concatenation will take place is:' , 'INFO')
for name in sortedvis:
casalog.post(' MJD '+str(qa.splitdate(qa.quantity(sorted_namestuples[sortedvis.index(name)][0],'s'))['mjd'])+': '+name, 'INFO')
if doweightscale:
casalog.post('In this new order, the weights are:'+str(sortedvisweightscale) , 'INFO')
# replace the original vis and visweightscale by the sorted ones (with concatvis removed if it exists)
vis = sortedvis
visweightscale = sortedvisweightscale
# if there are MMSs among the input, make their constituents the new input
mmslist = []
ismaster = []
for elvis in vis:
ismaster.append(True) # may be revised later
if(ParallelTaskHelper.isParallelMS(elvis)):
mmslist.append(elvis)
if len(mmslist)>0:
casalog.post('*** The following input measurement sets are multi-MSs', 'INFO')
for mname in mmslist:
casalog.post('*** '+mname, 'INFO')
oldvis = vis
oldvisweightscale = visweightscale
vis = []
visweightscale = []
ismaster = [] # reset ismaster
i = 0
for elvis in oldvis:
if elvis in mmslist: # append the subMSs individually
m.open(elvis)
mses = m.getreferencedtables()
m.close()
mses.sort()
mastername = os.path.basename(os.path.dirname(os.path.realpath(elvis+'/ANTENNA')))
for mname in mses:
#print 'subms: ', mname
vis.append(mname)
if doweightscale:
visweightscale.append(oldvisweightscale[i])
if os.path.basename(mname) == mastername:
ismaster.append(True)
else:
ismaster.append(False)
else:
vis.append(elvis)
if doweightscale:
visweightscale.append(oldvisweightscale[i])
ismaster.append(True)
i += 1
if keepcopy:
casalog.post('*** keepcopy==True: creating copy of input MSs to keep ...' , 'INFO')
tempdir = 'concat_tmp_'+str(time.time())
os.mkdir(tempdir)
for elvis in originalvis:
shutil.move(elvis,tempdir) # keep timestamps and permissions
shutil.copytree(tempdir+'/'+elvis, elvis, True) # symlinks=True
casalog.post('Concatenating ...' , 'INFO')
if not copypointing: # delete the rows of all pointing tables
casalog.post('*** copypointing==False: resulting MMS will have empty POINTING table.', 'INFO')
tmptabname = 'TMPPOINTING'+str(time.time())
tmptabname2 = 'TMPPOINTING2'+str(time.time())
shutil.rmtree(tmptabname, ignore_errors=True)
shutil.rmtree(tmptabname2, ignore_errors=True)
shutil.move(vis[0]+'/POINTING', tmptabname)
t.open(tmptabname)
if(t.nrows()>0):
ttab = t.copy(newtablename=tmptabname2, deep=False, valuecopy=True, norows=True)
ttab.close()
t.close()
shutil.rmtree(tmptabname, ignore_errors=True)
else: # the POINTING table is already empty
t.close()
casalog.post('*** Input POINTING table was already empty.', 'INFO')
shutil.move(tmptabname, tmptabname2)
for i in range(len(vis)): # replace the POINTING tables by the empty one
os.system('rm -rf '+vis[i]+'/POINTING')
shutil.copytree(tmptabname2, vis[i]+'/POINTING')
shutil.rmtree(tmptabname2, ignore_errors=True)
if(len(vis) >0): # (note: in case len is 1, we only copy, essentially)
theconcatvis = vis[0]
if(len(vis)==1):
shutil.copytree(vis[0], concatvis, True)
vis.remove(vis[0])
# Determine if scratch columns should be considered at all
# by checking if any of the MSs has them.
considerscrcols = False
needscrcols = []
if ((type(theconcatvis)==str) and (os.path.exists(theconcatvis))):
# check if all scratch columns are present
t.open(theconcatvis)
if(t.colnames().count('CORRECTED_DATA')==1
or t.colnames().count('MODEL_DATA')==1):
considerscrcols = True # there are scratch columns
needscrcols.append(t.colnames().count('CORRECTED_DATA')==0
or t.colnames().count('MODEL_DATA')==0)
t.close()
else:
raise Exception, 'Visibility data set '+theconcatvis+' not found - please verify the name'
for elvis in vis : ###Oh no Elvis does not exist Mr Bill
if(not os.path.exists(elvis)):
raise Exception, 'Visibility data set '+elvis+' not found - please verify the name'
# check if all scratch columns are present
t.open(elvis)
if(t.colnames().count('CORRECTED_DATA')==1
or t.colnames().count('MODEL_DATA')==1):
considerscrcols = True # there are scratch columns
needscrcols.append(t.colnames().count('CORRECTED_DATA')==0
or t.colnames().count('MODEL_DATA')==0)
t.close()
# start actual work, file existence has already been checked
if(considerscrcols and needscrcols[0]):
# create scratch cols
casalog.post('creating scratch columns in '+theconcatvis , 'INFO')
cb.open(theconcatvis) # calibrator-open creates scratch columns
cb.close()
# scale the weights of the first MS in the chain
if doweightscale:
wscale = visweightscale[0]
if(wscale==1.):
casalog.post('Will leave the weights for this MS unchanged.', 'INFO')
else:
casalog.post('Scaling weights for first MS by factor '+str(wscale), 'INFO')
t.open(theconcatvis, nomodify=False)
for colname in [ 'WEIGHT', 'WEIGHT_SPECTRUM']:
if (colname in t.colnames()) and (t.iscelldefined(colname,0)):
for j in xrange(0,t.nrows()):
a = t.getcell(colname, j)
a *= wscale
t.putcell(colname, j, a)
t.close()
m.open(theconcatvis,nomodify=False)
mmsmembers = [theconcatvis]
auxfile = 'concat_aux_'+str(time.time())
i = 0
for elvis in vis :
i = i + 1
mmsmembers.append(elvis)
casalog.post('adding '+elvis+' to multi-MS '+concatvis, 'INFO')
wscale = 1.
if doweightscale:
wscale = visweightscale[i]
if(wscale==1.):
casalog.post('Will leave the weights for this MS unchanged.', 'INFO')
else:
casalog.post('Will scale weights for this MS by factor '+str(wscale) , 'INFO')
if(considerscrcols and needscrcols[i]):
# create scratch cols
casalog.post('creating scratch columns for '+elvis, 'INFO')
cb.open(elvis) # calibrator-open creates scratch columns
cb.close()
m.virtconcatenate(msfile=elvis,
auxfilename=auxfile,
freqtol=freqtol,dirtol=dirtol,respectname=respectname,
weightscale=wscale)
#end for
os.remove(auxfile)
m.writehistory(message='taskname=virtualconcat',origin='virtualconcat')
m.writehistory(message='vis = "'+str(vis)+'"',origin='virtualconcat')
m.writehistory(message='concatvis = "'+str(concatvis)+'"',origin='virtualconcat')
m.writehistory(message='freqtol = "'+str(freqtol)+'"',origin='virtualconcat')
m.writehistory(message='dirtol = "'+str(dirtol)+'"',origin='virtualconcat')
m.writehistory(message='respectname = "'+str(respectname)+'"',origin='virtualconcat')
m.writehistory(message='visweightscale = "'+str(visweightscale)+'"',origin='virtualconcat')
m.close()
# concatenate the POINTING tables
masterptable = mmsmembers[0]+'/POINTING'
ptablemembers = []
if os.path.exists(masterptable) and copypointing:
casalog.post('Concatenating the POINTING tables ...', 'INFO')
i = 0
for i in xrange(len(mmsmembers)):
ptable = mmsmembers[i]+'/POINTING'
if ismaster[i] and os.path.exists(ptable):
casalog.post(' '+ptable, 'INFO')
shutil.move(ptable, ptable+str(i))
ptablemembers.append(ptable+str(i))
#end for
t.createmultitable(masterptable, ptablemembers, 'SUBTBS')
# endif
ph.makeMMS(concatvis, mmsmembers,
True, # copy subtables from first to all other members
['POINTING', 'SYSCAL', 'SYSPOWER']) # excluding tables which will be linked
# remove the remaining "hulls" of the emptied input MMSs (if there are any)
for elvis in mmslist:
shutil.rmtree(elvis)
if keepcopy:
for elvis in originalvis:
shutil.move(tempdir+'/'+elvis, elvis)
os.rmdir(tempdir)
except Exception, instance:
print '*** Error ***',instance
if keepcopy and tempdir!='':
print "Restoring original MSs ..."
for elvis in originalvis:
if os.path.exists(tempdir+'/'+elvis):
shutil.rmtree(elvis)
shutil.move(tempdir+'/'+elvis, elvis)
os.rmdir(tempdir)
raise Exception, instance
def postExecution(self):
'''
This overrides the post execution portion of the task helper
in this case we probably need to generate the output reference
ms.
'''
if self._arg['createmms']:
casalog.post("Finalizing MMS structure")
# restore POINTING and SYSCAL
if self.pwriteaccess and not self.pointingisempty:
print "restoring POINTING"
os.system('rm -rf '+self.ptab) # remove empty copy
os.system('mv '+self.dataDir+'/POINTING '+self.ptab)
if self.swriteaccess and not self.syscalisempty:
print "restoring SYSCAL"
os.system('rm -rf '+self.stab) # remove empty copy
os.system('mv '+self.dataDir+'/SYSCAL '+self.stab)
# jagonzal (CAS-4287): Add a cluster-less mode to by-pass parallel processing for MMSs as requested
if (ParallelTaskHelper.getBypassParallelProcessing()==1):
outputList = self._sequential_return_list
self._sequential_return_list = {}
else:
outputList = self._jobQueue.getOutputJobs()
# We created a data directory and many SubMSs,
# now build the reference MS
if self._arg['calmsselection'] in ['auto','manual']:
# A Cal MS was created in the data directory, see if it was
# successful, if so build a multi-MS
if os.path.exists(self._arg['calmsname']):
raise ValueError, "Output MS already exists"
self._msTool.createmultims(self._arg['calmsname'],
[self.dataDir +'/%s.cal.ms'%self.outputBase])
subMSList = []
if (ParallelTaskHelper.getBypassParallelProcessing()==1):
for subMS in outputList:
subMSList.append(subMS)
else:
for job in outputList:
if job.status == 'done':
subMSList.append(job.getCommandArguments()['outputvis'])
subMSList.sort()
if len(subMSList) == 0:
casalog.post("Error: no subMSs were created.", 'WARN')
return False
mastersubms = subMSList[0]
subtabs_to_omit = []
# deal with POINTING table
if not self.pointingisempty:
shutil.rmtree(mastersubms+'/POINTING', ignore_errors=True)
shutil.copytree(self.ptab, mastersubms+'/POINTING') # master subms gets a full copy of the original
if self.makepointinglinks:
for i in xrange(1,len(subMSList)):
theptab = subMSList[i]+'/POINTING'
shutil.rmtree(theptab, ignore_errors=True)
os.symlink('../'+os.path.basename(mastersubms)+'/POINTING', theptab)
# (link in target will be created my makeMMS)
subtabs_to_omit.append('POINTING')
# deal with SYSCAL table
if not self.syscalisempty:
shutil.rmtree(mastersubms+'/SYSCAL', ignore_errors=True)
shutil.copytree(self.stab, mastersubms+'/SYSCAL') # master subms gets a full copy of the original
if self.makesyscallinks:
for i in xrange(1,len(subMSList)):
thestab = subMSList[i]+'/SYSCAL'
shutil.rmtree(thestab, ignore_errors=True)
os.symlink('../'+os.path.basename(mastersubms)+'/SYSCAL', thestab)
# (link in target will be created my makeMMS)
subtabs_to_omit.append('SYSCAL')
ph.makeMMS(self._arg['outputvis'], subMSList,
True, # copy subtables
subtabs_to_omit # omitting these
)
thesubmscontainingdir = os.path.dirname(subMSList[0].rstrip('/'))
os.rmdir(thesubmscontainingdir)
return True
def virtualconcat(vislist, concatvis, freqtol, dirtol, respectname,
visweightscale, keepcopy, copypointing):
"""
Concatenate visibility data sets creating a Multi-MS.
Combine the input datasets into a Multi-MS.
NOTE: The input datasets are moved into the Multi-MS and may be modified
to account for subtable reference changes.
If none of the input MSs have any scratch columns, none are created.
Otherwise scratch columns are created and initialized in those MSs
which don't have a complete set.
Keyword arguments:
vis -- Name of input visibility files (MS)
default: none; example: vis=['ngc5921-1.ms', 'ngc5921-2.ms']
concatvis -- Name of the output visibility file
default: none; example: concatvis='src2.ms'
freqtol -- Frequency shift tolerance for considering data as the same spwid
default: '' means always combine
example: freqtol='10MHz' will not combine spwid unless they are
within 10 MHz
dirtol -- Direction shift tolerance for considering data as the same field
default: ;; means always combine
example: dirtol='1.arcsec' will not combine data for a field unless
their phase center is less than 1 arcsec.
respectname -- If true, fields with a different name are not merged even if their
direction agrees (within dirtol)
default: True
visweightscale -- list of the weight scales to be applied to the individual MSs
default: [] (don't modify weights, equivalent to setting scale to 1 for each MS)
keepcopy -- If true, a copy of the input MSs is kept in their original place.
default: false
copypointing -- If true, the POINTING table information will be present in the output.
If false, the result is an empty POINTING table.
default: True
"""
###
#Python script
tempdir = ''
originalvis = vislist
try:
casalog.origin('virtualconcat')
t = tbtool()
m = mstool()
#break the reference between vis and vislist as we modify vis
if (type(vislist) == str):
vis = [vislist]
else:
vis = list(vislist)
#dto. for concavis
theconcatvis = concatvis
doweightscale = False
if (len(visweightscale) > 0):
if (len(visweightscale) != len(vis)):
raise Exception, 'parameter visweightscale must have same number of elements as parameter vis'
for factor in visweightscale:
if factor < 0.:
raise Exception, 'parameter visweightscale must only contain positive numbers'
elif factor != 1.:
doweightscale = True
if ((type(concatvis) != str) or (len(concatvis.split()) < 1)):
raise Exception, 'Parameter concatvis is invalid.'
if (vis.count(concatvis) > 0):
raise Exception, 'Parameter concatvis must not be equal to one of the members of parameter vis.'
if (os.path.exists(concatvis)):
raise Exception, 'The output MMS must not yet exist.'
# process the input MSs in chronological order
sortedvis = []
sortedvisweightscale = []
namestuples = []
for name in vis:
t.open(name)
times = t.getcol('TIME')
t.close()
times.sort()
if doweightscale:
namestuples.append(
(times[0], name, visweightscale[vis.index(name)]))
else:
namestuples.append((times[0], name, 0))
sorted_namestuples = sorted(namestuples, key=lambda msname: msname[0])
for i in range(0, len(vis)):
sortedvis.append(sorted_namestuples[i][1])
sortedvisweightscale.append(sorted_namestuples[i][2])
if not vis == sortedvis:
casalog.post(
'The list of input MSs is not in chronological order and will need to be sorted.',
'INFO')
casalog.post(
'The chronological order in which the concatenation will take place is:',
'INFO')
for name in sortedvis:
casalog.post(
' MJD ' + str(
qa.splitdate(
qa.quantity(
sorted_namestuples[sortedvis.index(name)][0],
's'))['mjd']) + ': ' + name, 'INFO')
if doweightscale:
casalog.post(
'In this new order, the weights are:' +
str(sortedvisweightscale), 'INFO')
# replace the original vis and visweightscale by the sorted ones (with concatvis removed if it exists)
vis = sortedvis
visweightscale = sortedvisweightscale
# if there are MMSs among the input, make their constituents the new input
mmslist = []
ismaster = []
for elvis in vis:
ismaster.append(True) # may be revised later
if (ParallelTaskHelper.isParallelMS(elvis)):
mmslist.append(elvis)
if len(mmslist) > 0:
casalog.post(
'*** The following input measurement sets are multi-MSs',
'INFO')
for mname in mmslist:
casalog.post('*** ' + mname, 'INFO')
oldvis = vis
oldvisweightscale = visweightscale
vis = []
visweightscale = []
ismaster = [] # reset ismaster
i = 0
for elvis in oldvis:
if elvis in mmslist: # append the subMSs individually
m.open(elvis)
mses = m.getreferencedtables()
m.close()
mses.sort()
mastername = os.path.basename(
os.path.dirname(os.path.realpath(elvis + '/ANTENNA')))
for mname in mses:
#print 'subms: ', mname
vis.append(mname)
if doweightscale:
visweightscale.append(oldvisweightscale[i])
if os.path.basename(mname) == mastername:
ismaster.append(True)
else:
ismaster.append(False)
else:
vis.append(elvis)
if doweightscale:
visweightscale.append(oldvisweightscale[i])
ismaster.append(True)
i += 1
if keepcopy:
casalog.post(
'*** keepcopy==True: creating copy of input MSs to keep ...',
'INFO')
tempdir = 'concat_tmp_' + str(time.time())
os.mkdir(tempdir)
for elvis in originalvis:
shutil.move(elvis, tempdir) # keep timestamps and permissions
shutil.copytree(tempdir + '/' + elvis, elvis,
True) # symlinks=True
casalog.post('Concatenating ...', 'INFO')
if not copypointing: # delete the rows of all pointing tables
casalog.post(
'*** copypointing==False: resulting MMS will have empty POINTING table.',
'INFO')
tmptabname = 'TMPPOINTING' + str(time.time())
tmptabname2 = 'TMPPOINTING2' + str(time.time())
shutil.rmtree(tmptabname, ignore_errors=True)
shutil.rmtree(tmptabname2, ignore_errors=True)
shutil.move(vis[0] + '/POINTING', tmptabname)
t.open(tmptabname)
if (t.nrows() > 0):
ttab = t.copy(newtablename=tmptabname2,
deep=False,
valuecopy=True,
norows=True)
ttab.close()
t.close()
shutil.rmtree(tmptabname, ignore_errors=True)
else: # the POINTING table is already empty
t.close()
casalog.post('*** Input POINTING table was already empty.',
'INFO')
shutil.move(tmptabname, tmptabname2)
for i in range(
len(vis)): # replace the POINTING tables by the empty one
os.system('rm -rf ' + vis[i] + '/POINTING')
shutil.copytree(tmptabname2, vis[i] + '/POINTING')
shutil.rmtree(tmptabname2, ignore_errors=True)
if (len(vis) >
0): # (note: in case len is 1, we only copy, essentially)
theconcatvis = vis[0]
if (len(vis) == 1):
shutil.copytree(vis[0], concatvis, True)
vis.remove(vis[0])
# Determine if scratch columns should be considered at all
# by checking if any of the MSs has them.
considerscrcols = False
needscrcols = []
if ((type(theconcatvis) == str) and (os.path.exists(theconcatvis))):
# check if all scratch columns are present
t.open(theconcatvis)
if (t.colnames().count('CORRECTED_DATA') == 1
or t.colnames().count('MODEL_DATA') == 1):
considerscrcols = True # there are scratch columns
needscrcols.append(t.colnames().count('CORRECTED_DATA') == 0
or t.colnames().count('MODEL_DATA') == 0)
t.close()
else:
raise Exception, 'Visibility data set ' + theconcatvis + ' not found - please verify the name'
for elvis in vis: ###Oh no Elvis does not exist Mr Bill
if (not os.path.exists(elvis)):
raise Exception, 'Visibility data set ' + elvis + ' not found - please verify the name'
# check if all scratch columns are present
t.open(elvis)
if (t.colnames().count('CORRECTED_DATA') == 1
or t.colnames().count('MODEL_DATA') == 1):
considerscrcols = True # there are scratch columns
needscrcols.append(t.colnames().count('CORRECTED_DATA') == 0
or t.colnames().count('MODEL_DATA') == 0)
t.close()
# start actual work, file existence has already been checked
if (considerscrcols and needscrcols[0]):
# create scratch cols
casalog.post('creating scratch columns in ' + theconcatvis, 'INFO')
_cb.open(theconcatvis) # calibrator-open creates scratch columns
_cb.close()
# scale the weights of the first MS in the chain
if doweightscale:
wscale = visweightscale[0]
if (wscale == 1.):
casalog.post('Will leave the weights for this MS unchanged.',
'INFO')
else:
casalog.post(
'Scaling weights for first MS by factor ' + str(wscale),
'INFO')
t.open(theconcatvis, nomodify=False)
for colname in ['WEIGHT', 'WEIGHT_SPECTRUM']:
if (colname in t.colnames()) and (t.iscelldefined(
colname, 0)):
for j in xrange(0, t.nrows()):
a = t.getcell(colname, j)
a *= wscale
t.putcell(colname, j, a)
t.close()
m.open(theconcatvis, nomodify=False)
mmsmembers = [theconcatvis]
auxfile = 'concat_aux_' + str(time.time())
i = 0
for elvis in vis:
i = i + 1
mmsmembers.append(elvis)
casalog.post('adding ' + elvis + ' to multi-MS ' + concatvis,
'INFO')
wscale = 1.
if doweightscale:
wscale = visweightscale[i]
if (wscale == 1.):
casalog.post(
'Will leave the weights for this MS unchanged.',
'INFO')
else:
casalog.post(
'Will scale weights for this MS by factor ' +
str(wscale), 'INFO')
if (considerscrcols and needscrcols[i]):
# create scratch cols
casalog.post('creating scratch columns for ' + elvis, 'INFO')
_cb.open(elvis) # calibrator-open creates scratch columns
_cb.close()
m.virtconcatenate(msfile=elvis,
auxfilename=auxfile,
freqtol=freqtol,
dirtol=dirtol,
respectname=respectname,
weightscale=wscale)
#end for
os.remove(auxfile)
m.writehistory(message='taskname=virtualconcat',
origin='virtualconcat')
m.writehistory(message='vis = "' + str(vis) + '"',
origin='virtualconcat')
m.writehistory(message='concatvis = "' + str(concatvis) + '"',
origin='virtualconcat')
m.writehistory(message='freqtol = "' + str(freqtol) + '"',
origin='virtualconcat')
m.writehistory(message='dirtol = "' + str(dirtol) + '"',
origin='virtualconcat')
m.writehistory(message='respectname = "' + str(respectname) + '"',
origin='virtualconcat')
m.writehistory(message='visweightscale = "' + str(visweightscale) +
'"',
origin='virtualconcat')
m.close()
# concatenate the POINTING tables
masterptable = mmsmembers[0] + '/POINTING'
ptablemembers = []
if os.path.exists(masterptable) and copypointing:
casalog.post('Concatenating the POINTING tables ...', 'INFO')
i = 0
for i in xrange(len(mmsmembers)):
ptable = mmsmembers[i] + '/POINTING'
if ismaster[i] and os.path.exists(ptable):
casalog.post(' ' + ptable, 'INFO')
shutil.move(ptable, ptable + str(i))
ptablemembers.append(ptable + str(i))
#end for
t.createmultitable(masterptable, ptablemembers, 'SUBTBS')
# endif
# Get all available subtables
thesubtables = ph.getSubtables(mmsmembers[0])
# Remove the SOURCE and HISTORY tables, which will be the only copied.
# All other sub-tables will be linked to first subms
thesubtables.remove('SOURCE')
thesubtables.remove('HISTORY')
subtabs_to_omit = thesubtables
ph.makeMMS(
concatvis,
mmsmembers,
True, # copy subtables from first to all other members
subtabs_to_omit) # excluding tables which will be linked
# remove the remaining "hulls" of the emptied input MMSs (if there are any)
for elvis in mmslist:
shutil.rmtree(elvis)
if keepcopy:
for elvis in originalvis:
shutil.move(tempdir + '/' + elvis, elvis)
os.rmdir(tempdir)
except Exception, instance:
print '*** Error ***', instance
if keepcopy and tempdir != '':
print "Restoring original MSs ..."
for elvis in originalvis:
if os.path.exists(tempdir + '/' + elvis):
shutil.rmtree(elvis)
shutil.move(tempdir + '/' + elvis, elvis)
os.rmdir(tempdir)
raise Exception, instance