def oldsplit(vis, outputvis, datacolumn, field, spw, width, antenna, timebin, timerange, scan, intent, array, uvrange, correlation, observation, combine, keepflags, keepmms): """Create a visibility subset from an existing visibility set: Keyword arguments: vis -- Name of input visibility file (MS) default: none; example: vis='ngc5921.ms' outputvis -- Name of output visibility file (MS) default: none; example: outputvis='ngc5921_src.ms' datacolumn -- Which data column to split out default='corrected'; example: datacolumn='data' Options: 'data', 'corrected', 'model', 'all', 'float_data', 'lag_data', 'float_data,data', and 'lag_data,data'. note: 'all' = whichever of the above that are present. field -- Field name default: field = '' means use all sources field = 1 # will get field_id=1 (if you give it an integer, it will retrieve the source with that index) field = '1328+307' specifies source '1328+307'. Minimum match can be used, egs field = '13*' will retrieve '1328+307' if it is unique or exists. Source names with imbedded blanks cannot be included. spw -- Spectral window index identifier default=-1 (all); example: spw=1 antenna -- antenna names default '' (all), antenna = '3 & 7' gives one baseline with antennaid = 3,7. timebin -- Interval width for time averaging. default: '0s' or '-1s' (no averaging) example: timebin='30s' timerange -- Time range default='' means all times. examples: timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss' timerange='< YYYY/MM/DD/HH:MM:SS.sss' timerange='> YYYY/MM/DD/HH:MM:SS.sss' timerange='< ddd/HH:MM:SS.sss' timerange='> ddd/HH:MM:SS.sss' scan -- Scan numbers to select. default '' (all). intent -- Scan intents to select. default '' (all). array -- (Sub)array IDs to select. default '' (all). uvrange -- uv distance range to select. default '' (all). correlation -- Select correlations, e.g. 'rr, ll' or ['XY', 'YX']. default '' (all). observation -- Select by observation ID(s). default '' (all). combine -- Data descriptors that time averaging can ignore: scan, and/or state Default '' (none) keepflags -- Keep flagged data, if possible Default True keepmms -- If the input is a multi-MS, make the output one, too. (experimental) Default: False """ casalog.origin('oldsplit') mylocals = locals() rval = True try: if (keepmms and ParallelTaskHelper.isParallelMS(vis)): if (timebin!='0s' and timebin!='-1s'): casalog.post('Averaging over time with keepmms=True may lead to results different\n' +' from those obtained with keepmms=False due to different binning.', 'WARN') myms = mstool() myms.open(vis) mses = myms.getreferencedtables() myms.close() mses.sort() nfail = 0 if os.path.exists(outputvis): raise ValueError, "Output MS %s already exists - will not overwrite." % outputvis tempout = outputvis+str(time.time()) os.mkdir(tempout) successfulmses = [] mastersubms = '' masterptab = '' emptyptab = tempout+'/EMPTY_POINTING' nochangeinpointing = (str(antenna)+str(timerange)=='') if nochangeinpointing: # resulting pointing table is the same for all # -> replace by empty table if it is a link and won't be modified anyway # and put back original into the master after split # find the master for m in mses: theptab = m+'/POINTING' if not os.path.islink(theptab): #print "is master ", theptab mastersubms = m masterptab = m+'/POINTING' # save time by not copying the POINTING table len(mses) times myttb = tbtool() myttb.open(masterptab) tmpp = myttb.copy(newtablename=emptyptab, norows=True) myttb.close() del myttb tmpp.close() del tmpp break mytb = tbtool() # prepare the input MMS for processing replaced = [] outputviss = [] theptabs = [] for m in mses: # make sure the SORTED_TABLE keywords are disabled mytb.open(m, nomodify=False) if 'SORTED_TABLE' in mytb.keywordnames(): tobedel = mytb.getkeyword('SORTED_TABLE').split(' ')[1] mytb.removekeyword('SORTED_TABLE') os.system('rm -rf '+tobedel) mytb.close() # deal with the POINTING table theptab = m+'/POINTING' theptabs.append(theptab) if nochangeinpointing and os.path.islink(theptab): #print "is link ", theptab os.remove(theptab) shutil.copytree(emptyptab, theptab) replaced.append(True) else: replaced.append(False) # run oldsplit outputviss.append(os.path.abspath(tempout+'/'+os.path.basename(m))) # end for # send off the jobs print 'Running split_core ... ' helper = ParallelTaskHelper('oldsplit', mylocals) helper.override_arg('outputvis',outputviss) helper._consolidateOutput = False goretval = helper.go() for i in xrange(len(mses)): m = mses[i] # deal with the POINTING table if replaced[i]: # restore link shutil.rmtree(theptabs[i], ignore_errors=True) os.symlink('../'+os.path.basename(mastersubms)+'/POINTING', theptabs[i]) # (link in target will be created my makeMMS) # accumulate list of successful splits if not goretval[m]: nfail+=1 else: successfulmses.append(outputviss[i]) if nfail>0: # there were unsuccessful splits if len(successfulmses)==0: casalog.post('Split failed in all subMSs.', 'WARN') rval=False else: casalog.post('*** Summary: there were failures in '+str(nfail)+' SUBMSs:', 'WARN') casalog.post('*** (these are harmless if they are caused by selection):', 'WARN') for m in mses: if not goretval[m]: casalog.post(os.path.basename(m)+': '+str(goretval[m]), 'WARN') else: casalog.post(os.path.basename(m)+': '+str(goretval[m]), 'NORMAL') casalog.post('Will construct MMS from subMSs with successful selection ...', 'NORMAL') if nochangeinpointing: # need to take care of POINTING table # in case the master subms did not make it if not (tempout+'/'+os.path.basename(mastersubms) in successfulmses): # old master subms was not selected. # copy the original masterptab into the new master shutil.rmtree(successfulmses[0]+'/POINTING') shutil.copytree(masterptab, successfulmses[0]+'/POINTING') if rval: # construct new MMS from the output if(width==1 and str(field)+str(spw)+str(antenna)+str(timerange)+str(scan)+str(intent)\ +str(array)+str(uvrange)+str(correlation)+str(observation)==''): ph.makeMMS(outputvis, successfulmses) else: myms.open(successfulmses[0], nomodify=False) auxfile = "split_aux_"+str(time.time()) for i in xrange(1,len(successfulmses)): myms.virtconcatenate(successfulmses[i], auxfile, '1Hz', '10mas', True) myms.close() os.remove(auxfile) ph.makeMMS(outputvis, successfulmses, True, ['POINTING']) shutil.rmtree(tempout, ignore_errors=True) else: # do not output an MMS rval = split_core(vis, outputvis, datacolumn, field, spw, width, antenna, timebin, timerange, scan, intent, array, uvrange, correlation, observation, combine, keepflags) except Exception, instance: casalog.post("*** Error: %s" % (instance), 'SEVERE') rval = False
def virtualconcat(vislist,concatvis,freqtol,dirtol,respectname, visweightscale,keepcopy,copypointing): """ Concatenate visibility data sets creating a multi-MS. Combine the input datasets into a multi-MS. NOTE: The input datasets are moved into the multi-MS and may be modified to account for subtable reference changes. If none of the input MSs have any scratch columns, none are created. Otherwise scratch columns are created and initialized in those MSs which don't have a complete set. Keyword arguments: vis -- Name of input visibility files (MS) default: none; example: vis=['ngc5921-1.ms', 'ngc5921-2.ms'] concatvis -- Name of the output visibility file default: none; example: concatvis='src2.ms' freqtol -- Frequency shift tolerance for considering data as the same spwid default: '' means always combine example: freqtol='10MHz' will not combine spwid unless they are within 10 MHz dirtol -- Direction shift tolerance for considering data as the same field default: ;; means always combine example: dirtol='1.arcsec' will not combine data for a field unless their phase center is less than 1 arcsec. respectname -- If true, fields with a different name are not merged even if their direction agrees (within dirtol) default: True visweightscale -- list of the weight scales to be applied to the individual MSs default: [] (don't modify weights, equivalent to setting scale to 1 for each MS) keepcopy -- If true, a copy of the input MSs is kept in their original place. default: false copypointing -- If true, the POINTING table information will be present in the output. If false, the result is an empty POINTING table. default: True """ ### #Python script tempdir = '' originalvis = vislist try: casalog.origin('virtualconcat') t = tbtool() m = mstool() #break the reference between vis and vislist as we modify vis if(type(vislist)==str): vis=[vislist] else: vis=list(vislist) #dto. for concavis theconcatvis = concatvis doweightscale = False if(len(visweightscale)>0): if (len(visweightscale) != len(vis)): raise Exception, 'parameter visweightscale must have same number of elements as parameter vis' for factor in visweightscale: if factor<0.: raise Exception, 'parameter visweightscale must only contain positive numbers' elif factor!=1.: doweightscale=True if((type(concatvis)!=str) or (len(concatvis.split()) < 1)): raise Exception, 'Parameter concatvis is invalid.' if(vis.count(concatvis) > 0): raise Exception, 'Parameter concatvis must not be equal to one of the members of parameter vis.' if(os.path.exists(concatvis)): raise Exception, 'The output MMS must not yet exist.' # process the input MSs in chronological order sortedvis = [] sortedvisweightscale = [] namestuples = [] for name in vis: t.open(name) times = t.getcol('TIME') t.close() times.sort() if doweightscale: namestuples.append( (times[0], name, visweightscale[vis.index(name)]) ) else: namestuples.append( (times[0], name, 0) ) sorted_namestuples = sorted(namestuples, key=lambda msname: msname[0]) for i in range(0,len(vis)): sortedvis.append(sorted_namestuples[i][1]) sortedvisweightscale.append(sorted_namestuples[i][2]) if not vis == sortedvis: casalog.post('The list of input MSs is not in chronological order and will need to be sorted.' , 'INFO') casalog.post('The chronological order in which the concatenation will take place is:' , 'INFO') for name in sortedvis: casalog.post(' MJD '+str(qa.splitdate(qa.quantity(sorted_namestuples[sortedvis.index(name)][0],'s'))['mjd'])+': '+name, 'INFO') if doweightscale: casalog.post('In this new order, the weights are:'+str(sortedvisweightscale) , 'INFO') # replace the original vis and visweightscale by the sorted ones (with concatvis removed if it exists) vis = sortedvis visweightscale = sortedvisweightscale # if there are MMSs among the input, make their constituents the new input mmslist = [] ismaster = [] for elvis in vis: ismaster.append(True) # may be revised later if(ParallelTaskHelper.isParallelMS(elvis)): mmslist.append(elvis) if len(mmslist)>0: casalog.post('*** The following input measurement sets are multi-MSs', 'INFO') for mname in mmslist: casalog.post('*** '+mname, 'INFO') oldvis = vis oldvisweightscale = visweightscale vis = [] visweightscale = [] ismaster = [] # reset ismaster i = 0 for elvis in oldvis: if elvis in mmslist: # append the subMSs individually m.open(elvis) mses = m.getreferencedtables() m.close() mses.sort() mastername = os.path.basename(os.path.dirname(os.path.realpath(elvis+'/ANTENNA'))) for mname in mses: #print 'subms: ', mname vis.append(mname) if doweightscale: visweightscale.append(oldvisweightscale[i]) if os.path.basename(mname) == mastername: ismaster.append(True) else: ismaster.append(False) else: vis.append(elvis) if doweightscale: visweightscale.append(oldvisweightscale[i]) ismaster.append(True) i += 1 if keepcopy: casalog.post('*** keepcopy==True: creating copy of input MSs to keep ...' , 'INFO') tempdir = 'concat_tmp_'+str(time.time()) os.mkdir(tempdir) for elvis in originalvis: shutil.move(elvis,tempdir) # keep timestamps and permissions shutil.copytree(tempdir+'/'+elvis, elvis, True) # symlinks=True casalog.post('Concatenating ...' , 'INFO') if not copypointing: # delete the rows of all pointing tables casalog.post('*** copypointing==False: resulting MMS will have empty POINTING table.', 'INFO') tmptabname = 'TMPPOINTING'+str(time.time()) tmptabname2 = 'TMPPOINTING2'+str(time.time()) shutil.rmtree(tmptabname, ignore_errors=True) shutil.rmtree(tmptabname2, ignore_errors=True) shutil.move(vis[0]+'/POINTING', tmptabname) t.open(tmptabname) if(t.nrows()>0): ttab = t.copy(newtablename=tmptabname2, deep=False, valuecopy=True, norows=True) ttab.close() t.close() shutil.rmtree(tmptabname, ignore_errors=True) else: # the POINTING table is already empty t.close() casalog.post('*** Input POINTING table was already empty.', 'INFO') shutil.move(tmptabname, tmptabname2) for i in range(len(vis)): # replace the POINTING tables by the empty one os.system('rm -rf '+vis[i]+'/POINTING') shutil.copytree(tmptabname2, vis[i]+'/POINTING') shutil.rmtree(tmptabname2, ignore_errors=True) if(len(vis) >0): # (note: in case len is 1, we only copy, essentially) theconcatvis = vis[0] if(len(vis)==1): shutil.copytree(vis[0], concatvis, True) vis.remove(vis[0]) # Determine if scratch columns should be considered at all # by checking if any of the MSs has them. considerscrcols = False needscrcols = [] if ((type(theconcatvis)==str) and (os.path.exists(theconcatvis))): # check if all scratch columns are present t.open(theconcatvis) if(t.colnames().count('CORRECTED_DATA')==1 or t.colnames().count('MODEL_DATA')==1): considerscrcols = True # there are scratch columns needscrcols.append(t.colnames().count('CORRECTED_DATA')==0 or t.colnames().count('MODEL_DATA')==0) t.close() else: raise Exception, 'Visibility data set '+theconcatvis+' not found - please verify the name' for elvis in vis : ###Oh no Elvis does not exist Mr Bill if(not os.path.exists(elvis)): raise Exception, 'Visibility data set '+elvis+' not found - please verify the name' # check if all scratch columns are present t.open(elvis) if(t.colnames().count('CORRECTED_DATA')==1 or t.colnames().count('MODEL_DATA')==1): considerscrcols = True # there are scratch columns needscrcols.append(t.colnames().count('CORRECTED_DATA')==0 or t.colnames().count('MODEL_DATA')==0) t.close() # start actual work, file existence has already been checked if(considerscrcols and needscrcols[0]): # create scratch cols casalog.post('creating scratch columns in '+theconcatvis , 'INFO') cb.open(theconcatvis) # calibrator-open creates scratch columns cb.close() # scale the weights of the first MS in the chain if doweightscale: wscale = visweightscale[0] if(wscale==1.): casalog.post('Will leave the weights for this MS unchanged.', 'INFO') else: casalog.post('Scaling weights for first MS by factor '+str(wscale), 'INFO') t.open(theconcatvis, nomodify=False) for colname in [ 'WEIGHT', 'WEIGHT_SPECTRUM']: if (colname in t.colnames()) and (t.iscelldefined(colname,0)): for j in xrange(0,t.nrows()): a = t.getcell(colname, j) a *= wscale t.putcell(colname, j, a) t.close() m.open(theconcatvis,nomodify=False) mmsmembers = [theconcatvis] auxfile = 'concat_aux_'+str(time.time()) i = 0 for elvis in vis : i = i + 1 mmsmembers.append(elvis) casalog.post('adding '+elvis+' to multi-MS '+concatvis, 'INFO') wscale = 1. if doweightscale: wscale = visweightscale[i] if(wscale==1.): casalog.post('Will leave the weights for this MS unchanged.', 'INFO') else: casalog.post('Will scale weights for this MS by factor '+str(wscale) , 'INFO') if(considerscrcols and needscrcols[i]): # create scratch cols casalog.post('creating scratch columns for '+elvis, 'INFO') cb.open(elvis) # calibrator-open creates scratch columns cb.close() m.virtconcatenate(msfile=elvis, auxfilename=auxfile, freqtol=freqtol,dirtol=dirtol,respectname=respectname, weightscale=wscale) #end for os.remove(auxfile) m.writehistory(message='taskname=virtualconcat',origin='virtualconcat') m.writehistory(message='vis = "'+str(vis)+'"',origin='virtualconcat') m.writehistory(message='concatvis = "'+str(concatvis)+'"',origin='virtualconcat') m.writehistory(message='freqtol = "'+str(freqtol)+'"',origin='virtualconcat') m.writehistory(message='dirtol = "'+str(dirtol)+'"',origin='virtualconcat') m.writehistory(message='respectname = "'+str(respectname)+'"',origin='virtualconcat') m.writehistory(message='visweightscale = "'+str(visweightscale)+'"',origin='virtualconcat') m.close() # concatenate the POINTING tables masterptable = mmsmembers[0]+'/POINTING' ptablemembers = [] if os.path.exists(masterptable) and copypointing: casalog.post('Concatenating the POINTING tables ...', 'INFO') i = 0 for i in xrange(len(mmsmembers)): ptable = mmsmembers[i]+'/POINTING' if ismaster[i] and os.path.exists(ptable): casalog.post(' '+ptable, 'INFO') shutil.move(ptable, ptable+str(i)) ptablemembers.append(ptable+str(i)) #end for t.createmultitable(masterptable, ptablemembers, 'SUBTBS') # endif ph.makeMMS(concatvis, mmsmembers, True, # copy subtables from first to all other members ['POINTING', 'SYSCAL', 'SYSPOWER']) # excluding tables which will be linked # remove the remaining "hulls" of the emptied input MMSs (if there are any) for elvis in mmslist: shutil.rmtree(elvis) if keepcopy: for elvis in originalvis: shutil.move(tempdir+'/'+elvis, elvis) os.rmdir(tempdir) except Exception, instance: print '*** Error ***',instance if keepcopy and tempdir!='': print "Restoring original MSs ..." for elvis in originalvis: if os.path.exists(tempdir+'/'+elvis): shutil.rmtree(elvis) shutil.move(tempdir+'/'+elvis, elvis) os.rmdir(tempdir) raise Exception, instance
def postExecution(self): ''' This overrides the post execution portion of the task helper in this case we probably need to generate the output reference ms. ''' if self._arg['createmms']: casalog.post("Finalizing MMS structure") # restore POINTING and SYSCAL if self.pwriteaccess and not self.pointingisempty: print "restoring POINTING" os.system('rm -rf '+self.ptab) # remove empty copy os.system('mv '+self.dataDir+'/POINTING '+self.ptab) if self.swriteaccess and not self.syscalisempty: print "restoring SYSCAL" os.system('rm -rf '+self.stab) # remove empty copy os.system('mv '+self.dataDir+'/SYSCAL '+self.stab) # jagonzal (CAS-4287): Add a cluster-less mode to by-pass parallel processing for MMSs as requested if (ParallelTaskHelper.getBypassParallelProcessing()==1): outputList = self._sequential_return_list self._sequential_return_list = {} else: outputList = self._jobQueue.getOutputJobs() # We created a data directory and many SubMSs, # now build the reference MS if self._arg['calmsselection'] in ['auto','manual']: # A Cal MS was created in the data directory, see if it was # successful, if so build a multi-MS if os.path.exists(self._arg['calmsname']): raise ValueError, "Output MS already exists" self._msTool.createmultims(self._arg['calmsname'], [self.dataDir +'/%s.cal.ms'%self.outputBase]) subMSList = [] if (ParallelTaskHelper.getBypassParallelProcessing()==1): for subMS in outputList: subMSList.append(subMS) else: for job in outputList: if job.status == 'done': subMSList.append(job.getCommandArguments()['outputvis']) subMSList.sort() if len(subMSList) == 0: casalog.post("Error: no subMSs were created.", 'WARN') return False mastersubms = subMSList[0] subtabs_to_omit = [] # deal with POINTING table if not self.pointingisempty: shutil.rmtree(mastersubms+'/POINTING', ignore_errors=True) shutil.copytree(self.ptab, mastersubms+'/POINTING') # master subms gets a full copy of the original if self.makepointinglinks: for i in xrange(1,len(subMSList)): theptab = subMSList[i]+'/POINTING' shutil.rmtree(theptab, ignore_errors=True) os.symlink('../'+os.path.basename(mastersubms)+'/POINTING', theptab) # (link in target will be created my makeMMS) subtabs_to_omit.append('POINTING') # deal with SYSCAL table if not self.syscalisempty: shutil.rmtree(mastersubms+'/SYSCAL', ignore_errors=True) shutil.copytree(self.stab, mastersubms+'/SYSCAL') # master subms gets a full copy of the original if self.makesyscallinks: for i in xrange(1,len(subMSList)): thestab = subMSList[i]+'/SYSCAL' shutil.rmtree(thestab, ignore_errors=True) os.symlink('../'+os.path.basename(mastersubms)+'/SYSCAL', thestab) # (link in target will be created my makeMMS) subtabs_to_omit.append('SYSCAL') ph.makeMMS(self._arg['outputvis'], subMSList, True, # copy subtables subtabs_to_omit # omitting these ) thesubmscontainingdir = os.path.dirname(subMSList[0].rstrip('/')) os.rmdir(thesubmscontainingdir) return True
def virtualconcat(vislist, concatvis, freqtol, dirtol, respectname, visweightscale, keepcopy, copypointing): """ Concatenate visibility data sets creating a Multi-MS. Combine the input datasets into a Multi-MS. NOTE: The input datasets are moved into the Multi-MS and may be modified to account for subtable reference changes. If none of the input MSs have any scratch columns, none are created. Otherwise scratch columns are created and initialized in those MSs which don't have a complete set. Keyword arguments: vis -- Name of input visibility files (MS) default: none; example: vis=['ngc5921-1.ms', 'ngc5921-2.ms'] concatvis -- Name of the output visibility file default: none; example: concatvis='src2.ms' freqtol -- Frequency shift tolerance for considering data as the same spwid default: '' means always combine example: freqtol='10MHz' will not combine spwid unless they are within 10 MHz dirtol -- Direction shift tolerance for considering data as the same field default: ;; means always combine example: dirtol='1.arcsec' will not combine data for a field unless their phase center is less than 1 arcsec. respectname -- If true, fields with a different name are not merged even if their direction agrees (within dirtol) default: True visweightscale -- list of the weight scales to be applied to the individual MSs default: [] (don't modify weights, equivalent to setting scale to 1 for each MS) keepcopy -- If true, a copy of the input MSs is kept in their original place. default: false copypointing -- If true, the POINTING table information will be present in the output. If false, the result is an empty POINTING table. default: True """ ### #Python script tempdir = '' originalvis = vislist try: casalog.origin('virtualconcat') t = tbtool() m = mstool() #break the reference between vis and vislist as we modify vis if (type(vislist) == str): vis = [vislist] else: vis = list(vislist) #dto. for concavis theconcatvis = concatvis doweightscale = False if (len(visweightscale) > 0): if (len(visweightscale) != len(vis)): raise Exception, 'parameter visweightscale must have same number of elements as parameter vis' for factor in visweightscale: if factor < 0.: raise Exception, 'parameter visweightscale must only contain positive numbers' elif factor != 1.: doweightscale = True if ((type(concatvis) != str) or (len(concatvis.split()) < 1)): raise Exception, 'Parameter concatvis is invalid.' if (vis.count(concatvis) > 0): raise Exception, 'Parameter concatvis must not be equal to one of the members of parameter vis.' if (os.path.exists(concatvis)): raise Exception, 'The output MMS must not yet exist.' # process the input MSs in chronological order sortedvis = [] sortedvisweightscale = [] namestuples = [] for name in vis: t.open(name) times = t.getcol('TIME') t.close() times.sort() if doweightscale: namestuples.append( (times[0], name, visweightscale[vis.index(name)])) else: namestuples.append((times[0], name, 0)) sorted_namestuples = sorted(namestuples, key=lambda msname: msname[0]) for i in range(0, len(vis)): sortedvis.append(sorted_namestuples[i][1]) sortedvisweightscale.append(sorted_namestuples[i][2]) if not vis == sortedvis: casalog.post( 'The list of input MSs is not in chronological order and will need to be sorted.', 'INFO') casalog.post( 'The chronological order in which the concatenation will take place is:', 'INFO') for name in sortedvis: casalog.post( ' MJD ' + str( qa.splitdate( qa.quantity( sorted_namestuples[sortedvis.index(name)][0], 's'))['mjd']) + ': ' + name, 'INFO') if doweightscale: casalog.post( 'In this new order, the weights are:' + str(sortedvisweightscale), 'INFO') # replace the original vis and visweightscale by the sorted ones (with concatvis removed if it exists) vis = sortedvis visweightscale = sortedvisweightscale # if there are MMSs among the input, make their constituents the new input mmslist = [] ismaster = [] for elvis in vis: ismaster.append(True) # may be revised later if (ParallelTaskHelper.isParallelMS(elvis)): mmslist.append(elvis) if len(mmslist) > 0: casalog.post( '*** The following input measurement sets are multi-MSs', 'INFO') for mname in mmslist: casalog.post('*** ' + mname, 'INFO') oldvis = vis oldvisweightscale = visweightscale vis = [] visweightscale = [] ismaster = [] # reset ismaster i = 0 for elvis in oldvis: if elvis in mmslist: # append the subMSs individually m.open(elvis) mses = m.getreferencedtables() m.close() mses.sort() mastername = os.path.basename( os.path.dirname(os.path.realpath(elvis + '/ANTENNA'))) for mname in mses: #print 'subms: ', mname vis.append(mname) if doweightscale: visweightscale.append(oldvisweightscale[i]) if os.path.basename(mname) == mastername: ismaster.append(True) else: ismaster.append(False) else: vis.append(elvis) if doweightscale: visweightscale.append(oldvisweightscale[i]) ismaster.append(True) i += 1 if keepcopy: casalog.post( '*** keepcopy==True: creating copy of input MSs to keep ...', 'INFO') tempdir = 'concat_tmp_' + str(time.time()) os.mkdir(tempdir) for elvis in originalvis: shutil.move(elvis, tempdir) # keep timestamps and permissions shutil.copytree(tempdir + '/' + elvis, elvis, True) # symlinks=True casalog.post('Concatenating ...', 'INFO') if not copypointing: # delete the rows of all pointing tables casalog.post( '*** copypointing==False: resulting MMS will have empty POINTING table.', 'INFO') tmptabname = 'TMPPOINTING' + str(time.time()) tmptabname2 = 'TMPPOINTING2' + str(time.time()) shutil.rmtree(tmptabname, ignore_errors=True) shutil.rmtree(tmptabname2, ignore_errors=True) shutil.move(vis[0] + '/POINTING', tmptabname) t.open(tmptabname) if (t.nrows() > 0): ttab = t.copy(newtablename=tmptabname2, deep=False, valuecopy=True, norows=True) ttab.close() t.close() shutil.rmtree(tmptabname, ignore_errors=True) else: # the POINTING table is already empty t.close() casalog.post('*** Input POINTING table was already empty.', 'INFO') shutil.move(tmptabname, tmptabname2) for i in range( len(vis)): # replace the POINTING tables by the empty one os.system('rm -rf ' + vis[i] + '/POINTING') shutil.copytree(tmptabname2, vis[i] + '/POINTING') shutil.rmtree(tmptabname2, ignore_errors=True) if (len(vis) > 0): # (note: in case len is 1, we only copy, essentially) theconcatvis = vis[0] if (len(vis) == 1): shutil.copytree(vis[0], concatvis, True) vis.remove(vis[0]) # Determine if scratch columns should be considered at all # by checking if any of the MSs has them. considerscrcols = False needscrcols = [] if ((type(theconcatvis) == str) and (os.path.exists(theconcatvis))): # check if all scratch columns are present t.open(theconcatvis) if (t.colnames().count('CORRECTED_DATA') == 1 or t.colnames().count('MODEL_DATA') == 1): considerscrcols = True # there are scratch columns needscrcols.append(t.colnames().count('CORRECTED_DATA') == 0 or t.colnames().count('MODEL_DATA') == 0) t.close() else: raise Exception, 'Visibility data set ' + theconcatvis + ' not found - please verify the name' for elvis in vis: ###Oh no Elvis does not exist Mr Bill if (not os.path.exists(elvis)): raise Exception, 'Visibility data set ' + elvis + ' not found - please verify the name' # check if all scratch columns are present t.open(elvis) if (t.colnames().count('CORRECTED_DATA') == 1 or t.colnames().count('MODEL_DATA') == 1): considerscrcols = True # there are scratch columns needscrcols.append(t.colnames().count('CORRECTED_DATA') == 0 or t.colnames().count('MODEL_DATA') == 0) t.close() # start actual work, file existence has already been checked if (considerscrcols and needscrcols[0]): # create scratch cols casalog.post('creating scratch columns in ' + theconcatvis, 'INFO') _cb.open(theconcatvis) # calibrator-open creates scratch columns _cb.close() # scale the weights of the first MS in the chain if doweightscale: wscale = visweightscale[0] if (wscale == 1.): casalog.post('Will leave the weights for this MS unchanged.', 'INFO') else: casalog.post( 'Scaling weights for first MS by factor ' + str(wscale), 'INFO') t.open(theconcatvis, nomodify=False) for colname in ['WEIGHT', 'WEIGHT_SPECTRUM']: if (colname in t.colnames()) and (t.iscelldefined( colname, 0)): for j in xrange(0, t.nrows()): a = t.getcell(colname, j) a *= wscale t.putcell(colname, j, a) t.close() m.open(theconcatvis, nomodify=False) mmsmembers = [theconcatvis] auxfile = 'concat_aux_' + str(time.time()) i = 0 for elvis in vis: i = i + 1 mmsmembers.append(elvis) casalog.post('adding ' + elvis + ' to multi-MS ' + concatvis, 'INFO') wscale = 1. if doweightscale: wscale = visweightscale[i] if (wscale == 1.): casalog.post( 'Will leave the weights for this MS unchanged.', 'INFO') else: casalog.post( 'Will scale weights for this MS by factor ' + str(wscale), 'INFO') if (considerscrcols and needscrcols[i]): # create scratch cols casalog.post('creating scratch columns for ' + elvis, 'INFO') _cb.open(elvis) # calibrator-open creates scratch columns _cb.close() m.virtconcatenate(msfile=elvis, auxfilename=auxfile, freqtol=freqtol, dirtol=dirtol, respectname=respectname, weightscale=wscale) #end for os.remove(auxfile) m.writehistory(message='taskname=virtualconcat', origin='virtualconcat') m.writehistory(message='vis = "' + str(vis) + '"', origin='virtualconcat') m.writehistory(message='concatvis = "' + str(concatvis) + '"', origin='virtualconcat') m.writehistory(message='freqtol = "' + str(freqtol) + '"', origin='virtualconcat') m.writehistory(message='dirtol = "' + str(dirtol) + '"', origin='virtualconcat') m.writehistory(message='respectname = "' + str(respectname) + '"', origin='virtualconcat') m.writehistory(message='visweightscale = "' + str(visweightscale) + '"', origin='virtualconcat') m.close() # concatenate the POINTING tables masterptable = mmsmembers[0] + '/POINTING' ptablemembers = [] if os.path.exists(masterptable) and copypointing: casalog.post('Concatenating the POINTING tables ...', 'INFO') i = 0 for i in xrange(len(mmsmembers)): ptable = mmsmembers[i] + '/POINTING' if ismaster[i] and os.path.exists(ptable): casalog.post(' ' + ptable, 'INFO') shutil.move(ptable, ptable + str(i)) ptablemembers.append(ptable + str(i)) #end for t.createmultitable(masterptable, ptablemembers, 'SUBTBS') # endif # Get all available subtables thesubtables = ph.getSubtables(mmsmembers[0]) # Remove the SOURCE and HISTORY tables, which will be the only copied. # All other sub-tables will be linked to first subms thesubtables.remove('SOURCE') thesubtables.remove('HISTORY') subtabs_to_omit = thesubtables ph.makeMMS( concatvis, mmsmembers, True, # copy subtables from first to all other members subtabs_to_omit) # excluding tables which will be linked # remove the remaining "hulls" of the emptied input MMSs (if there are any) for elvis in mmslist: shutil.rmtree(elvis) if keepcopy: for elvis in originalvis: shutil.move(tempdir + '/' + elvis, elvis) os.rmdir(tempdir) except Exception, instance: print '*** Error ***', instance if keepcopy and tempdir != '': print "Restoring original MSs ..." for elvis in originalvis: if os.path.exists(tempdir + '/' + elvis): shutil.rmtree(elvis) shutil.move(tempdir + '/' + elvis, elvis) os.rmdir(tempdir) raise Exception, instance