def sdsmooth(infile=None, datacolumn=None, antenna=None, field=None, spw=None, timerange=None, scan=None, pol=None, intent=None, reindex=None, kernel=None, kwidth=None, outfile=None, overwrite=None): casalog.origin('sdsmooth') try: if len(outfile) == 0: errmsg = 'outfile is empty.' raise_exception(errmsg) if (os.path.exists(outfile)) and (not overwrite): errmsg = outfile+' exists.' raise_exception(errmsg) sdms.open(infile) sdms.set_selection(spw=spw, field=field, antenna=antenna, timerange=timerange, scan=scan, polarization=pol, intent=intent, reindex=reindex) sdms.smooth(type=kernel, width=kwidth, datacolumn=datacolumn, outfile=outfile) # Write to HISTORY of outfile MS param_names = sdsmooth.func_code.co_varnames[:sdsmooth.func_code.co_argcount] param_vals = [eval(p) for p in param_names] write_history(ms, outfile, 'sdsmooth', param_names, param_vals, casalog) except Exception, instance: raise Exception, instance
def clearcal( vis=None, field=None, spw=None, intent=None, addmodel=None, ): casalog.origin('clearcal') # Do the trivial parallelization if ParallelTaskHelper.isParallelMS(vis): helper = ParallelTaskHelper('clearcal', locals()) helper.go() return # Local versions of the tools tblocal = tbtool() cblocal = cbtool() mslocal = mstool() try: # we will initialize scr cols only if we don't create them doinit = False if (type(vis) == str) & os.path.exists(vis): tblocal.open(vis) doinit = tblocal.colnames().count('CORRECTED_DATA') > 0 tblocal.close() # We ignore selection if creating the scratch columns if not doinit: casalog.post('Need to create scratch columns; ignoring selection.' ) cblocal.setvi(old=True,quiet=False); # Old VI for now cblocal.open(vis, addmodel=addmodel) else: raise Exception, \ 'Visibility data set not found - please verify the name' # If necessary (scr col not just created), initialize scr cols if doinit: cblocal.selectvis(field=field, spw=spw, intent=intent) cblocal.initcalset(1) cblocal.close() # Write history to the MS param_names = clearcal.func_code.co_varnames[:clearcal.func_code.co_argcount] param_vals = [eval(p) for p in param_names] casalog.post('Updating the history in the output', 'DEBUG1') write_history(mslocal, vis, 'clearcal', param_names, param_vals, casalog) except Exception, instance: print '*** Error ***', instance
def importuvfits(fitsfile, vis, antnamescheme=None): """ Convert a UVFITS file to a CASA visibility data set (MS): Keyword arguments: fitsfile -- Name of input UV FITS file default = none; example='3C273XC1.fits' vis -- Name of output visibility file (MS) default = none; example: vis='3C273XC1.ms' antnamescheme -- Naming scheme for VLA/JVLA/CARMA antennas default = old; old: Antenna name is a number, '04' This option exists for backwards compatibility but can lead to ambiguous results when antenna indices are used for data selection. new: Antenna name is not a number, 'VA04' or 'EA04' With this scheme, data selection via antenna names and indices is non-ambiguous. async -- Run asynchronously default = false; do not run asychronously """ myms = mstool() try: try: casalog.origin('importuvfits') casalog.post("") myms.fromfits(vis, fitsfile, antnamescheme=antnamescheme) myms.close() except Exception, instance: casalog.post( str( '*** Error ***') + str(instance), 'SEVERE') raise # Write the args to HISTORY. try: param_names = \ importuvfits.func_code.co_varnames[:importuvfits.func_code.co_argcount] param_vals = [eval(p) for p in param_names] write_history( myms, vis, 'importuvfits', param_names, param_vals, casalog ) except Exception, instance: casalog.post("Failed to updated HISTORY table", 'WARN')
def importnro(infile=None, outputvis=None, overwrite=None, parallel=None): """ """ casalog.origin('importnro') status = True try: outputvis_temp = outputvis + '-backup-' + datetime.datetime.now( ).strftime('%Y%m%d-%H%M%S') if os.path.exists(outputvis): if overwrite: os.rename(outputvis, outputvis_temp) else: raise RuntimeError('%s exists.' % (outputvis)) if not is_nostar(infile): raise RuntimeError('%s is not a valid NOSTAR data.' % (infile)) status = mysdms.importnro(infile, outputvis, parallel) if status: # initialize weights using cb tool mycb.open(outputvis, compress=False, addcorr=False, addmodel=False) mycb.initweights(wtmode='nyq') if os.path.exists(outputvis_temp): shutil.rmtree(outputvis_temp) else: if os.path.exists(outputvis): shutil.rmtree(outputvis) if os.path.exists(outputvis_temp): os.rename(outputvis_temp, outputvis) raise RuntimeError('import failed.') # Write parameters to HISTORY table of MS param_names = importnro.func_code.co_varnames[:importnro.func_code. co_argcount] param_vals = [eval(p) for p in param_names] write_history(myms, outputvis, 'importnro', param_names, param_vals, casalog) return status except Exception, instance: casalog.post('*** Error *** ' + str(instance), 'SEVERE') raise instance
def uvsub(vis=None,reverse=False): """Subtract model from the corrected visibility data uvsub(vis='ngc5921.ms', reverse=false) This function subtracts model visibility data from corrected visibility data leaving the residuals in the corrected data column. If the parameter 'reverse' is set true, the process is reversed. Keyword arguments: vis -- Name of input visibility file (MS) default: none; example: vis='ngc5921.ms' reverse -- Reverse the operation (add rather than subtract) default: false; example: reverse=true uvsub(vis='ngc5921.ms', reverse=false) """ #Python script # try: ms = mstool() casalog.origin('uvsub') if ((type(vis)==str) & (os.path.exists(vis))): ms.open(thems=vis,nomodify=False) else: raise Exception, 'Visibility data set not found - please verify the name' return ms.uvsub(reverse) ms.close # Write history to MS try: param_names = uvsub.func_code.co_varnames[:uvsub.func_code.co_argcount] param_vals = [eval(p) for p in param_names] write_history(mstool(), vis, 'uvsub', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') return
def initweights(vis=None,wtmode=None,tsystable=None,gainfield=None,interp=None,spwmap=None,dowtsp=None): casalog.origin('initweights') # Do the trivial parallelization if ParallelTaskHelper.isMPIEnabled() and ParallelTaskHelper.isParallelMS(vis): tsystable = ParallelTaskHelper.findAbsPath(tsystable) helper = ParallelTaskHelper('initweights', locals()) helper.go() # Write history to MS. try: param_names = initweights.func_code.co_varnames[:initweights.func_code.co_argcount] param_vals = [eval(p) for p in param_names] casalog.post('Updating the history in the output', 'DEBUG1') write_history(ms, vis, 'initweights', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') return
def delmod(vis=None, otf=None, field=None, scr=None): casalog.origin('delmod') # Do the trivial parallelization if ParallelTaskHelper.isParallelMS(vis): helper = ParallelTaskHelper('delmod', locals()) helper.go() return #Python script try: # only if vis exists... if ((type(vis) == str) & (os.path.exists(vis))): # ... and we are asked to do something... # open without adding anything! _cb.open(vis, addcorr=False, addmodel=False) _cb.delmod(otf=otf, field=field, scr=scr) _cb.close() else: raise Exception, 'Visibility data set not found - please verify the name' # Write history to MS try: param_names = delmod.func_code.co_varnames[:delmod.func_code. co_argcount] param_vals = [eval(p) for p in param_names] write_history(mstool(), vis, 'delmod', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') except Exception, instance: print '*** Error ***', instance
def importfitsidi(fitsidifile, vis, constobsid=None, scanreindexgap_s=None, specframe=None): """Convert FITS-IDI visibility file into a CASA visibility file (MS). Keyword arguments: fitsidifile -- Name(s) of input FITS IDI file(s) default: None; example='3C273XC1.IDI' or ['3C273XC1.IDI1', '3C273XC1.IDI2'] vis -- Name of output visibility file (MS) default: None; example: vis='3C273XC1.ms' constobsid -- If True a constant obs id == 0 of is given to all input files default = False (new obs id for each input file) scanreindexgap_s -- if > 0., a new scan is started whenever the gap between two integrations is > the given value (seconds) or when a new field starts or when the ARRAY_ID changes. default = 0. (no reindexing) specframe -- this frame will be used to set the spectral reference frame for all spectral windows in the output MS default = GEO (geocentric), other options: TOPO, LSRK, BARY NOTE: if specframe is set to TOPO, the reference location will be taken from the Observatories table in the CASA data repository for the given name of the observatory. You can edit that table and add new rows. """ #Python script retval = True try: casalog.origin('importfitsidi') casalog.post("") myms = mstool() mytb = tbtool() if type(specframe) == str and not specframe == '': myspecframe = specframe.upper() else: myspecframe = 'GEO' refframes = { 'REST': 0, 'LSRK': 1, 'LSRD': 2, 'BARY': 3, 'GEO': 4, 'TOPO': 5 } if not refframes.has_key(myspecframe): raise Exception, 'Value ' + myspecframe + ' of parameter specframe invalid. Possible values are REST, LSRK, LSRD, BARY, GEO, TOPO' if (type(fitsidifile) == str): casalog.post('### Reading file ' + fitsidifile, 'INFO') myms.fromfitsidi(vis, fitsidifile) myms.close() elif (type(fitsidifile) == list): clist = fitsidifile casalog.post('### Reading file ' + clist[0], 'INFO') myms.fromfitsidi(vis, clist[0]) myms.close() clist.pop(0) tname = vis + '_importfitsidi_tmp_' shutil.rmtree(tname, ignore_errors=True) for fidifile in clist: casalog.post('### Reading file ' + fidifile, 'INFO') myms.fromfitsidi(tname, fidifile) myms.close() myms.open(vis, nomodify=False) myms.concatenate(msfile=tname, freqtol='', dirtol='') myms.close() shutil.rmtree(tname, ignore_errors=True) else: raise Exception, 'Parameter fitsidifile should be of type str or list' if (constobsid): mytb.open(vis + '/OBSERVATION', nomodify=False) nobs = mytb.nrows() cando = True if nobs > 1: casalog.post( 'Trying to keep obsid constant == 0 for all input files', 'INFO') # check if all observations are from the same telescope; if not warn and leave as is tels = mytb.getcol('TELESCOPE_NAME') for i in range(1, nobs): if tels[i] != tels[0]: cando = False if cando: # get min and max time and write them into the first row; casalog.post('Adjusting OBSERVATION table', 'INFO') timeranges = mytb.getcol('TIME_RANGE') ttr = timeranges.transpose() newmin = min(ttr[0]) newmax = max(ttr[1]) mytb.putcell('TIME_RANGE', 0, [newmin, newmax]) # delete the other rows mytb.removerows(range(1, nobs)) else: casalog.post( 'The input files stem from different telescopes. Need to give different obs id.', 'WARN') mytb.close() if cando: # give the same obs id == 0 to the entire output MS casalog.post('Setting observation ID of all integrations to 0', 'INFO') mytb.open(vis, nomodify=False) for i in xrange(0, mytb.nrows()): mytb.putcell('OBSERVATION_ID', i, 0) mytb.close() else: # don't want constant obs id if (type(fitsidifile) == list and len(fitsidifile) > 1): casalog.post( 'Incrementing observation ID for each input file ...', 'INFO') if (scanreindexgap_s > 0.): # reindex the scan column mytb.open(vis, nomodify=False) times = mytb.getcol('TIME') fields = mytb.getcol('FIELD_ID') arrayids = mytb.getcol('ARRAY_ID') scannumbers = mytb.getcol('SCAN_NUMBER') timesorted = np.argsort(np.array(times)) scannumber = 0 scannumber_field = len(fields) * [0] prevtime = len(fields) * [0] prevarrayid = arrayids[timesorted[0]] for i in xrange(0, mytb.nrows()): ii = timesorted[i] timenow = times[ii] fieldnow = fields[ii] arrayidnow = arrayids[ii] if (timenow-prevtime[fieldnow] > scanreindexgap_s) \ or (arrayidnow != prevarrayid): scannumber += 1 scannumber_field[fieldnow] = scannumber casalog.post("Starting new scan "+str(scannumber)+" at "+str(timenow)\ +", field "+str(fieldnow)+", array_id "+str(arrayidnow), 'INFO') scannumbers[ii] = scannumber_field[fieldnow] prevtime[fieldnow] = timenow prevarrayid = arrayidnow mytb.putcol('SCAN_NUMBER', scannumbers) mytb.close() if refframes.has_key(myspecframe): casalog.post( 'Setting reference frame for all spectral windows to ' + myspecframe, 'INFO') if myspecframe == 'TOPO': casalog.post( 'NOTE: reference position for TOPO frame will be the observatory location', 'WARN') mytb.open(vis + '/SPECTRAL_WINDOW', nomodify=False) refcol = mytb.getcol('MEAS_FREQ_REF') refcol = [refframes[myspecframe]] * len(refcol) mytb.putcol('MEAS_FREQ_REF', refcol) mytb.close() # write history try: param_names = importfitsidi.func_code.co_varnames[:importfitsidi. func_code. co_argcount] param_vals = [eval(p) for p in param_names] retval &= write_history(myms, vis, 'importfitsidi', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') except Exception, instance: print '*** Error ***', instance shutil.rmtree(vis + '_importfitsidi_tmp_', ignore_errors=True) raise Exception, instance
def importatca(files=None, vis=None, options=None, spw=None, nscans=None, lowfreq=None, highfreq=None, fields=None, edge=8): """Convert an RPFITS file into a CASA visibility file (MS). The conversion of the RPFITS format into a measurement set. This version has been tested for both old ATCA and CABB data. ................ Keyword arguments: files -- Name of input RPFITS file(s) default: none; example: file='2010-01-02_1234.c999' .... vis -- Output ms name, note a postfix (.ms) is NOT appended to this name default: none .... options -- Processing options, comma separated list birdie - flag parts of spectrum known to be bad reweight - (pre-CABB) reweight lag spectrum to avoid ringing noautoflag - don't apply automatic flags (e.g. pointing scans) noxycorr - don't apply xyphase correction fastmosaic - use for large mosaics to speed up data access hires - turn time binned data into fast sampled data notsys - undo online Tsys calibration noac - don't load autocorrelations .... spw -- specify the input spectral windows to use. For CABB the order is first continuum, 2nd continuum, then any zooms for first band, followed by zooms for 2nd band. Pre-CABB data just has 0 and 1. The output may have more spectral windows if there are frequency changes. ........ default: all .... nscans -- Number of scans to skip followed by number of scans to read .... default: 0,0 (read all) .... lowfreq -- Lowest reference frequency to select .... default: 0 (all) .... highfreq -- highest reference frequency to select .... default: 0 (all) .... fields -- List of field names to select ........ default: all .... edge -- Percentage of edge channels to flag. For combined zooms, this specifies the percentage for a single zoom ........ default: 8 (flags 4% of channels at lower and upper edge) """ # Python script myaf = casac.atcafiller() try: try: casalog.origin('importatca') # ----------------------------------------- # beginning of importatca implementation # ----------------------------------------- myaf.open(vis, files, options) firstscan = 0 lastscan = 9999 if (nscans != None): if len(nscans) > 0: firstscan = nscans[0] if len(nscans) > 1: lastscan = nscans[1] myaf.select(firstscan, lastscan, spw, lowfreq, highfreq, fields, edge) myaf.fill() except Exception, e: print e casalog.post("Failed to import atca rpfits file(s) %s" % files) raise # Write the args to HISTORY. try: mslocal = mstool() param_names = importatca.func_code.co_varnames[:importatca. func_code. co_argcount] param_vals = [eval(p) for p in param_names] write_history(mslocal, vis, 'importatca', param_names, param_vals, casalog) except Exception, instance: casalog.post("Failed to updated HISTORY", 'WARN')
def uvcontsub3(vis, fitspw, combine, fitorder, field, spw, scan, intent, correlation, observation): """Extract the line(s) of an MS.""" retval = True casalog.origin('uvcontsub3') myms = mstool() mytb = tbtool() # This one is redundant - it is already checked at the XML level. if not ((type(vis) == str) and os.path.isdir(vis)): casalog.post('Visibility data set not found - please verify the name', 'SEVERE') return False outputvis = vis + '.contsub' if os.path.exists(outputvis): casalog.post("Output MS " + outputvis + " already exists - will not overwrite.", 'SEVERE') return False if combine and combine.lower() != 'spw': casalog.post("uvcontsub3 deliberately does not support combination by", 'SEVERE') casalog.post("anything except spw.", 'SEVERE') return False # MSStateGram is picky ('CALIBRATE_WVR.REFERENCE, OBSERVE_TARGET_ON_SOURCE' # doesn't work, but 'CALIBRATE_WVR.REFERENCE,OBSERVE_TARGET_ON_SOURCE' # does), and I don't want to mess with bison now. A .upper() might be a # good idea too, but the MS def'n v.2 does not say whether OBS_MODE should # be case-insensitive. intent = intent.replace(', ', ',') if type(spw) == list: spw = ','.join([str(s) for s in spw]) elif type(spw) == int: spw = str(spw) ## if ':' in spw: ## casalog.post("uvcontsub3 does not yet support selection by channel for the output", ## 'SEVERE') ## casalog.post("Meanwhile, use split to select the desired channels", 'WARN') ## return False if ';' in spw: casalog.post("uvcontsub3 does not yet support writing multiple channel groups per output spw", 'SEVERE') return False mytb.open(vis + '/SPECTRAL_WINDOW') allspw = '0~' + str(mytb.nrows() - 1) mytb.close() if 'spw' not in combine: spwmfitspw = subtract_spws(spw, fitspw) if spwmfitspw == 'UNKNOWN': spwmfitspw = subtract_spws(allspw, fitspw) if spwmfitspw: raise Exception, "combine must include 'spw' when the fit is being applied to spws outside fitspw." if type(correlation) == list: correlation = ', '.join(correlation) correlation = correlation.upper() mytb.open(vis, nomodify=True) if 'CORRECTED_DATA' in mytb.colnames(): datacolumn = 'CORRECTED_DATA' else: # DON'T remind the user that split before uvcontsub wastes time - # scratch columns will eventually go away. datacolumn = 'DATA' mytb.close() myms.open(vis, nomodify=True) if not myms.contsub(outputms=outputvis, fitspw=fitspw, fitorder=fitorder, combine=combine, spw=spw, unionspw=join_spws(fitspw, spw), field=field, scan=scan, intent=intent, correlation=correlation, obs=str(observation), whichcol=datacolumn): myms.close() return False myms.close() # Write history to output MS, not the input ms. try: param_names = uvcontsub3.func_code.co_varnames[:uvcontsub3.func_code.co_argcount] param_vals = [eval(p) for p in param_names] retval &= write_history(myms, outputvis, 'uvcontsub3', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN')
def applycal( vis=None, field=None, spw=None, intent=None, selectdata=None, timerange=None, uvrange=None, antenna=None, scan=None, observation=None, msselect=None, docallib=None, callib=None, gaintable=None, gainfield=None, interp=None, spwmap=None, calwt=None, parang=None, applymode=None, flagbackup=None, ): # Python script casalog.origin('applycal') # Take care of the trivial parallelization if ParallelTaskHelper.isParallelMS(vis): # Back up the flags, if requested (and if necessary) if flagbackup and applymode != 'calonly' and applymode != 'trial': fh.backupFlags(aflocal=None, msfile=vis, prename='applycal') flagbackup = False # To be safe convert file names to absolute paths. gaintable = ParallelTaskHelper.findAbsPath(gaintable) helper = ParallelTaskHelper('applycal', locals()) ret = helper.go() if ParallelTaskHelper.getAsyncMode(): return ret else: return try: mycb = cbtool() if (type(vis) == str) & os.path.exists(vis): # add CORRECTED_DATA column mycb.open(filename=vis, compress=False, addcorr=True, addmodel=False) else: raise Exception, \ 'Visibility data set not found - please verify the name' # enforce default if unspecified if applymode == '': applymode = 'calflag' # Back up the flags, if requested (and if necessary) if flagbackup and applymode != 'calonly' and applymode \ != 'trial': fh.backupFlags(aflocal=None, msfile=vis, prename='applycal') # Do data selection according to selectdata if selectdata: # pass all data selection parameters in as specified mycb.selectvis( time=timerange, spw=spw, scan=scan, field=field, intent=intent, observation=str(observation), baseline=antenna, uvrange=uvrange, chanmode='none', msselect=msselect, ) else: # selectdata=F, so time,scan,baseline,uvrange,msselect='' # using spw and field specifications only mycb.selectvis( time='', spw=spw, scan='', field=field, intent=intent, observation='', baseline='', uvrange='', chanmode='none', msselect='', ) # Arrange applies.... if docallib: # by cal library from file # parsing using c++ parser thiscallib=mycb.parsecallibfile(callib) mycb.setcallib(thiscallib) else: # by traditional parameters ngaintab = 0 if gaintable != ['']: ngaintab = len(gaintable) ncalwt = len(calwt) if ncalwt == 1: calwt = [calwt[0] for i in range(ngaintab)] ngainfld = len(gainfield) nspwmap = len(spwmap) ninterp = len(interp) # handle list of list issues with spwmap if nspwmap > 0: if type(spwmap[0]) != list: # first element not a list, only one spwmap specified # make it a list of list spwmap = [spwmap] nspwmap = 1 for igt in range(ngaintab): if gaintable[igt] != '': # field selection is null unless specified thisgainfield = '' if igt < ngainfld: thisgainfield = gainfield[igt] # spwmap is null unless specifed thisspwmap = [-1] if igt < nspwmap: thisspwmap = spwmap[igt] # interp is 'linear' unless specified thisinterp = 'linear' if igt < ninterp: if interp[igt] == '': interp[igt] = thisinterp thisinterp = interp[igt] mycb.setapply( t=0.0, table=gaintable[igt], field=thisgainfield, calwt=calwt[igt], spwmap=thisspwmap, interp=thisinterp, ) # ...and now the specialized terms # Apply parallactic angle, if requested if parang: mycb.setapply(type='P') mycb.correct(applymode) # report what the flags did reportflags(mycb.activityrec()) mycb.close() # write history try: param_names = \ applycal.func_code.co_varnames[:applycal.func_code.co_argcount] param_vals = [eval(p) for p in param_names] write_history( mstool(), vis, 'applycal', param_names, param_vals, casalog, ) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % instance, 'WARN') except Exception, instance: print '*** Error ***', instance mycb.close() casalog.post("Error in applycal: %s" % str(instance), "SEVERE") raise Exception, "Error in applycal: "+str(instance)
def split_core(vis, outputvis, datacolumn, field, spw, width, antenna, timebin, timerange, scan, intent, array, uvrange, correlation, observation, combine, keepflags): retval = True if not outputvis or outputvis.isspace(): raise ValueError, 'Please specify outputvis' myms = mstool() mytb = None if ((type(vis) == str) & (os.path.exists(vis))): myms.open(vis, nomodify=True) else: raise ValueError, 'Visibility data set not found - please verify the name' if os.path.exists(outputvis): myms.close() raise ValueError, "Output MS %s already exists - will not overwrite." % outputvis if (os.path.exists(outputvis + ".flagversions")): myms.close() raise ValueError, "The flagversions \"%s.flagversions\" for the output MS already exist. Please delete." % outputvis # No longer needed. When did it get put in? Note that the default # spw='*' in myms.split ends up as '' since the default type for a variant # is BOOLVEC. (Of course!) Therefore both split and myms.split must # work properly when spw=''. #if(spw == ''): # spw = '*' if (type(antenna) == list): antenna = ', '.join([str(ant) for ant in antenna]) ## Accept digits without units ...assume seconds timebin = qa.convert(qa.quantity(timebin), 's')['value'] timebin = str(timebin) + 's' if timebin == '0s': timebin = '-1s' # MSStateGram is picky ('CALIBRATE_WVR.REFERENCE, OBSERVE_TARGET_ON_SOURCE' # doesn't work, but 'CALIBRATE_WVR.REFERENCE,OBSERVE_TARGET_ON_SOURCE' # does), and I don't want to mess with bison now. A .upper() might be a # good idea too, but the MS def'n v.2 does not say whether OBS_MODE should # be case-insensitive. intent = intent.replace(', ', ',') if '^' in spw: casalog.post( "The interpretation of ^n in split's spw strings has changed from 'average n' to 'skip n' channels!", 'WARN') casalog.post("Watch for Slicer errors", 'WARN') if type(width) == str: try: if (width.isdigit()): width = [string.atoi(width)] elif (width.count('[') == 1 and width.count(']') == 1): width = width.replace('[', '') width = width.replace(']', '') splitwidth = width.split(',') width = [] for ws in splitwidth: if (ws.isdigit()): width.append(string.atoi(ws)) else: width = [1] except: raise TypeError, 'parameter width is invalid...using 1' if type(correlation) == list: correlation = ', '.join(correlation) correlation = correlation.upper() if hasattr(combine, '__iter__'): combine = ', '.join(combine) if type(spw) == list: spw = ','.join([str(s) for s in spw]) elif type(spw) == int: spw = str(spw) do_chan_mod = spw.find('^') > -1 # '0:2~11^1' would be pointless. if not do_chan_mod: # ...look in width. if type(width) == int and width > 1: do_chan_mod = True elif hasattr(width, '__iter__'): for w in width: if w > 1: do_chan_mod = True break do_both_chan_and_time_mod = (do_chan_mod and string.atof(timebin[:-1]) > 0.0) if do_both_chan_and_time_mod: # Do channel averaging first because it might be included in the spw # string. import tempfile # We want the directory outputvis is in, not /tmp, because /tmp # might not have enough space. # outputvis is itself a directory, so strip off a trailing slash if # it is present. # I don't know if giving tempfile an absolute directory is necessary - # dir='' is effectively '.' in Ubuntu. workingdir = os.path.abspath(os.path.dirname(outputvis.rstrip('/'))) cavms = tempfile.mkdtemp(suffix=outputvis, dir=workingdir) casalog.post('Channel averaging to ' + cavms) if not myms.split(outputms=cavms, field=field, spw=spw, step=width, baseline=antenna, subarray=array, timebin='', time=timerange, whichcol=datacolumn, scan=scan, uvrange=uvrange, combine=combine, correlation=correlation, intent=intent, obs=str(observation)): myms.close() if os.path.isdir(cavms): import shutil shutil.rmtree(cavms) return False # The selection was already made, so blank them before time averaging. field = '' spw = '' width = [1] antenna = '' array = '' timerange = '' datacolumn = 'all' scan = '' intent = '' uvrange = '' observation = '' myms.close() myms.open(cavms) casalog.post('Starting time averaging') if keepflags: taqlstr = '' else: taqlstr = 'NOT (FLAG_ROW OR ALL(FLAG))' if not myms.split(outputms=outputvis, field=field, spw=spw, step=width, baseline=antenna, subarray=array, timebin=timebin, time=timerange, whichcol=datacolumn, scan=scan, uvrange=uvrange, combine=combine, correlation=correlation, taql=taqlstr, intent=intent, obs=str(observation)): myms.close() return False myms.close() if do_both_chan_and_time_mod: import shutil shutil.rmtree(cavms) # Write history to output MS, not the input ms. try: param_names = split_core.func_code.co_varnames[:split_core.func_code. co_argcount] param_vals = [eval(p) for p in param_names] retval &= write_history(myms, outputvis, 'oldsplit', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN')
casalog.post('Starting import ...', 'NORMAL') importuvfits(fitsfile, vis) except Exception, instance: casalog.post(str(instance), 'SEVERE') return retValue # mytb, myms, myfg = gentools(['tb', 'ms', 'fg']) mytb, myms = gentools(['tb', 'ms']) aflocal = casac.agentflagger() # Write history try: param_names = importgmrt.func_code.co_varnames[:importgmrt.func_code. co_argcount] param_vals = [eval(p) for p in param_names] ok &= write_history(myms, vis, 'importgmrt', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') # CASA's importuvfits doesn't understand the GMRT antenna # names very well and/or the FITS files tend to put the # antenna names in a place that CASA doesn't look for them. # Luckily the information seems to be in the "station" # information. So let's fix up the Name column. casalog.post('Correcting GMRT Antenna names.', 'NORMAL1') try: mytb.open(vis + '/ANTENNA', nomodify=False) stations = mytb.getcol('STATION') names = [] for idx in range(0, len(stations)): names.append(stations[idx].split(':')[0])
# Open the MS, select the data and configure the output mtlocal.open() # Run the tool casalog.post('Apply Hanning smoothing on data') mtlocal.run() mtlocal.done() except Exception, instance: mtlocal.done() casalog.post('%s' % instance, 'ERROR') return False # Write history to output MS, not the input ms. try: param_names = hanningsmooth.func_code.co_varnames[:hanningsmooth. func_code. co_argcount] param_vals = [eval(p) for p in param_names] casalog.post('Updating the history in the output', 'DEBUG1') write_history(mslocal, outputvis, 'hanningsmooth', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') return False mslocal = None return True
def importasdm(asdm=None, vis=None, createmms=None, separationaxis=None, numsubms=None, corr_mode=None, srt=None, time_sampling=None, ocorr_mode=None, compression=None, lazy=None, asis=None, wvr_corrected_data=None, scans=None, ignore_time=None, process_syspower=None, process_caldevice=None, process_pointing=None, process_flags=None, tbuff=None, applyflags=None, savecmds=None, outfile=None, flagbackup=None, verbose=None, overwrite=None, showversion=None, useversion=None, bdfflags=None, with_pointing_correction=None, remove_ref_undef=None, convert_ephem2geo=None, polyephem_tabtimestep=None): """Convert an ALMA Science Data Model observation into a CASA visibility file (MS) or single-dish data format (Scantable). The conversion of the ALMA SDM archive format into a measurement set. This version is under development and is geared to handling many spectral windows of different shapes. Keyword arguments: asdm -- Name of input ASDM file (directory) default: none; example: asdm='ExecBlock3' vis -- Root ms or scantable name, note a prefix (.ms or .asap) is NOT appended to this name default: none createmms -- Create a Multi-MS default: False corr_mode -- correlation mode to be considered on input. Could be one or more of the following, ao, co, ac, or all default: all srt -- spectral resolution type. Could be one or more of the following, fr, ca, bw, or all default: all time_sampling -- specifies the time sampling, INTEGRATION and/or SUBINTEGRATION. could be one or more of the following i, si, or all. default: all ocorr_mode -- output data for correlation mode AUTO_ONLY (ao) or CROSS_ONLY (co) or CROSS_AND_AUTO (ca) default: ca compression -- produces comrpressed columns in the resulting measurement set. default: False lazy -- Make the MS DATA column read the ASDM Binary data directly (faster import, smaller MS) default: False asis -- creates verbatim copies of the ASDM tables in the output measurement set. The value given to this option must be a list of table names separated by space characters; the wildcard character '*' is allowed in table names. wvr_corrected_data -- specifies which values are considered in the ASDM binary data to fill the DATA column in the MAIN table of the MS. Expected values for this option are 'no' for the uncorrected data (this is the default), 'yes' for the corrected data and 'both' for corrected and uncorrected data. In the latter case, two measurement sets are created, one containing the uncorrected data and the other one, whose name is suffixed by '-wvr-corrected', containing the corrected data. scans -- processes only the scans specified in the option's value. This value is a semicolon separated list of scan specifications. A scan specification consists in an exec bock index followed by the character ':' followed by a comma separated list of scan indexes or scan index ranges. A scan index is relative to the exec block it belongs to. Scan indexes are 1-based while exec blocks's are 0-based. "0:1" or "2:2~6" or "0:1,1:2~6,8;2:,3:24~30" "1,2" are valid values for the option. "3:" alone will be interpreted as 'all the scans of the exec block#3'. An scan index or a scan index range not preceded by an exec block index will be interpreted as 'all the scans with such indexes in all the exec blocks'. By default all the scans are considered. ignore_time -- All the rows of the tables Feed, History, Pointing, Source, SysCal, CalDevice, SysPower, and Weather are processed independently of the time range of the selected exec block / scan. process_syspower -- The SysPower table is processed if and only if this parameter is set to True. default: True process_caldevice -- The CalDevice table is processed if and only if this parameter is set to True. default: True process_pointing -- The Pointing table is processed if and only if this parameter is set to True. If the parameter is set to False the resulting MS will have an empty POINTING table. default: True process_flags -- Process the online flags and save them to the FLAG_CMD sub-table. default: True >>> process_flags expandable parameter tbuff -- Time padding buffer (in seconds). default: 0.0 applyflags -- Apply the online flags to the MS. default: False savecmds -- Save the online flags to an ASCII file. default: False outfile -- Filename to save the online flags. default: '' flagbackup -- Backup the FLAG column in the .flagversions. default: True verbose -- produce log output as asdm2MS is being run. overwrite -- Over write an existing MS. showversion -- report the version of the asdm2MS being used. useversion -- Selects the version of asdm2MS to be used (presently only \'v3\' is available). default: v3 bdfflags -- Set the MS FLAG column according to the ASDM _binary_ flags default: false with_pointing_correction -- add (ASDM::Pointing::encoder - ASDM::Pointing::pointingDirection) to the value to be written in MS::Pointing::direction default: false remove_ref_undef -- if set to True then apply fixspwbackport on the resulting MSes. convert_ephem2geo -- if True, convert any attached ephemerides to the GEO reference frame polyephem_tabtimestep -- Timestep (days) for the tabulation of polynomial ephemerides. A value <= 0 disables tabulation. Presently, VLA data can contain polynomial ephemerides. ALMA data uses tabulated values. default: 0. """ # Python script # make agentflagger tool local aflocal = casac.agentflagger() # make table tool local tblocal = casac.table() try: casalog.origin('importasdm') viso = '' visoc = '' # for the wvr corrected version, if needed if len(vis) > 0: viso = vis tmps = vis.rstrip('.ms') if tmps == vis: visoc = vis + '-wvr-corrected' else: visoc = tmps + '-wvr-corrected.ms' else: viso = asdm.rstrip("/") + '.ms' visoc = asdm.rstrip("/") + '-wvr-corrected.ms' vis = asdm.rstrip("/") useversion = 'v3' theexecutable = 'asdm2MS' execute_string = theexecutable + ' --icm "' + corr_mode \ + '" --isrt "' + srt + '" --its "' + time_sampling \ + '" --ocm "' + ocorr_mode + '" --wvr-corrected-data "' \ + wvr_corrected_data + '" --asis "' + asis \ + '" --logfile "' + casalog.logfile() + '"' if len(scans) > 0: execute_string = execute_string + ' --scans ' + scans if ignore_time: execute_string = execute_string + ' --ignore-time' if useversion == 'v3': if not process_syspower: execute_string = execute_string + ' --no-syspower' if not process_caldevice: execute_string = execute_string + ' --no-caldevice' if not process_pointing: execute_string = execute_string + ' --no-pointing' if compression: execute_string = execute_string + ' --compression' elif lazy: execute_string = execute_string + ' --lazy' if verbose: execute_string = execute_string + ' --verbose' # if not overwrite and os.path.exists(viso): # raise Exception, \ # 'You have specified an existing MS and have indicated you do not wish to overwrite it' # Compression if compression: # viso = viso + '.compressed' viso = viso.rstrip('.ms') + '.compressed.ms' visoc = visoc.rstrip('.ms') + '.compressed.ms' vistoproc = [] # the output MSs to post-process if wvr_corrected_data == 'no' or wvr_corrected_data == 'both': vistoproc.append(viso) if (wvr_corrected_data == 'yes' or wvr_corrected_data == 'both'): vistoproc.append(visoc) for ff in vistoproc: if not overwrite and os.path.exists(ff): raise Exception, \ 'You have specified an existing MS and have indicated you do not wish to overwrite it: %s'%ff # If viso+".flagversions" then process differently depending on the value of overwrite.. # if flagbackup: for myviso in vistoproc: dotFlagversion = myviso + '.flagversions' if os.path.exists(dotFlagversion): if overwrite: casalog.post( "Found '" + dotFlagversion + "' . It'll be deleted before running the filler.") os.system('rm -rf %s' % dotFlagversion) else: casalog.post("Found '%s' but can't overwrite it." % dotFlagversion) raise Exception, "Found '%s' but can't overwrite it." \ % dotFlagversion # Make outfile always a list if isinstance(outfile, str): if outfile == '': outfile = [] else: noutfile = [outfile] outfile = noutfile if savecmds: if len(outfile) == 0: # Create default names for the online flags for myviso in vistoproc: outfile.append(myviso.replace('.ms', '_cmd.txt')) elif len(outfile) != len(vistoproc): casalog.post( 'List of outfile names does not match list of MSs', 'WARN') casalog.post('Will save online flags to temporary filenames', 'WARN') outfile = [] for myviso in vistoproc: online_file = myviso.replace('.ms', '_TEMP_cmd.txt') outfile.append(online_file) if not overwrite: for of in outfile: if os.path.exists(of): raise Exception, "Cannot overwrite online flags file '%s'; overwrite is set to False." % of execute_string = execute_string + ' ' + asdm + ' ' + viso if showversion: casalog.post( "You set option \'showversion\' to True. Will just output the version information and then terminate.", 'WARN') execute_string = theexecutable + ' --revision' if with_pointing_correction: execute_string = execute_string + ' --with-pointing-correction' if (polyephem_tabtimestep != None) and (type(polyephem_tabtimestep) == int or type(polyephem_tabtimestep) == float): if polyephem_tabtimestep > 0: casalog.post( 'Will tabulate all attached polynomial ephemerides with a time step of ' + str(polyephem_tabtimestep) + ' days.') if polyephem_tabtimestep > 1.: casalog.post( 'A tabulation timestep of <= 1 days is recommended.', 'WARN') execute_string = execute_string + ' --polyephem-tabtimestep ' + str( polyephem_tabtimestep) casalog.post('Running ' + theexecutable + ' standalone invoked as:') # print execute_string casalog.post(execute_string) exitcode = os.system(execute_string) if exitcode != 0: if not showversion: casalog.post( theexecutable + ' terminated with exit code ' + str(exitcode), 'SEVERE') raise Exception, \ 'ASDM conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.' if showversion: return # # Possibly remove the name of the measurement set expected to contain the corrected data from the list of of produced measurement # sets if it appears the filler did not find any corrected data. # if not os.path.exists(visoc): vistoproc = [myviso for myviso in vistoproc if myviso != visoc] # # Do we apply fixspwbackport if remove_ref_undef: casalog.post( 'remove_ref_undef=True: fixspwbackport will be applied ...') for myviso in vistoproc: cmd = 'fixspwbackport ' + myviso casalog.post('Running fixspwbackport standalone invoked as:') casalog.post(cmd) cmdexitcode = os.system(cmd) if cmdexitcode != 0: casalog.post( cmd + ' terminated with exit code ' + str(cmdexitcode), 'SEVERE') raise Exception, 'fixspwbackport error.' # Binary Flag processing if bdfflags: casalog.post( 'Parameter bdfflags==True: flags from the ASDM binary data will be used to set the MS flags ...' ) bdffexecutable = 'bdflags2MS ' bdffexecstring_base = bdffexecutable + ' -f ALL' + ' --ocm "' + ocorr_mode \ + '" --logfile "' + casalog.logfile() + '"' if len(scans) > 0: bdffexecstring_base = bdffexecstring_base + ' --scans ' + scans if lazy and not compression: bdffexecstring_base = bdffexecstring_base + ' --lazy=true' for myviso in vistoproc: if myviso.find("wvr-corrected") != -1: options = " --wvr-corrected=True " else: options = " " bdffexecstring = bdffexecstring_base + options + asdm + ' ' + myviso casalog.post('Running ' + bdffexecutable + ' standalone invoked as:') casalog.post(bdffexecstring) bdffexitcode = os.system(bdffexecstring) if bdffexitcode != 0: casalog.post( bdffexecutable + ' terminated with exit code ' + str(bdffexitcode), 'SEVERE') raise Exception, \ 'ASDM binary flags conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.' theephemfields = ce.findattachedephemfields(myviso, field='*') if len(theephemfields) > 0: # until asdm2MS does this internally: recalc the UVW coordinates for ephem fields imt = imtool() imt.open(myviso, usescratch=False) imt.calcuvw(theephemfields, refcode='J2000', reuse=False) imt.close() if convert_ephem2geo: for myviso in vistoproc: ce.convert2geo(myviso, '*') # convert any attached ephemerides to GEO if len(theephemfields) > 0: # also set the direction column in the SOURCE table tblocal.open(myviso + '/FIELD', nomodify=False) sourceids = tblocal.getcol('SOURCE_ID') ftimes = tblocal.getcol('TIME') ftimekw = tblocal.getcolkeywords('TIME') tmpa = tblocal.getcol('PHASE_DIR') origphasedir = tmpa affectedsids = [] thesamplefields = [] for fld in theephemfields: # determine all source ids used by the ephem fields if not (sourceids[fld] in affectedsids): # this source id wasn't handled yet affectedsids.append(sourceids[fld]) thesamplefields.append(fld) # need to temporarily change the offset (not all mosaics have an element at (0,0)) tmpa[0][0][fld] = 0. tmpa[1][0][fld] = 0. #endif #endfor tblocal.putcol('PHASE_DIR', tmpa) tblocal.close() tblocal.open(myviso + '/SOURCE') sourceposref = tblocal.getcolkeywords( 'DIRECTION')['MEASINFO']['Ref'] tblocal.close() directions = [] msmdlocal = casac.msmetadata() msmdlocal.open(myviso) for fld in thesamplefields: thedirmeas = msmdlocal.phasecenter(fld) if thedirmeas['refer'] != sourceposref: casalog.post( 'Ephemeris is in ' + thedirmeas['refer'] + ' instead of ' + sourceposref + ' frame.\nEntry in SOURCE table will be converted to ' + sourceposref, 'WARN') melocal = metool() melocal.doframe(thedirmeas) thedirmeas = melocal.measure(thedirmeas, sourceposref) directions.append( [thedirmeas['m0']['value'], thedirmeas['m1']['value']]) thetime = me.epoch(v0=str(ftimes[fld]) + 's', rf=ftimekw['MEASINFO']['Ref']) casalog.post("Will set SOURCE direction for SOURCE_ID " + str(sourceids[fld]) + " to ephemeris phase center for time " + str(thetime['m0']['value']) + " " + thetime['m0']['unit'] + " " + thetime['refer']) #endfor msmdlocal.close() # restore original PHASE_DIR tblocal.open(myviso + '/FIELD', nomodify=False) tblocal.putcol('PHASE_DIR', origphasedir) tblocal.close() # write source directions tblocal.open(myviso + '/SOURCE', nomodify=False) ssourceids = tblocal.getcol('SOURCE_ID') sdirs = tblocal.getcol('DIRECTION') for row in xrange(0, len(ssourceids)): for i in xrange(0, len(affectedsids)): if ssourceids[row] == affectedsids[i]: sdirs[0][row] = directions[i][0] sdirs[1][row] = directions[i][1] break #endfor #endfor tblocal.putcol('DIRECTION', sdirs) # write back corrected directions tblocal.close() #end if ##############################################################################################3 # CAS-7369 - Create an output Multi-MS (MMS) if createmms: # Get the default parameters of partition from tasks import partition fpars = partition.parameters for mypar in fpars.keys(): fpars[mypar] = partition.itsdefault(mypar) # Call the cluster for each MS for myviso in vistoproc: casalog.origin('importasdm') # Move original MS to tempdir tempname = myviso + '.temp.ms' outputmms = myviso shutil.move(myviso, tempname) # Get the proper column datacolumn = 'DATA' dcols = ['DATA', 'FLOAT_DATA'] for dc in dcols: if len(th.getColDesc(tempname, dc)) > 0: datacolumn = dc break fpars['datacolumn'] = datacolumn casalog.post('Will create a Multi-MS for: ' + myviso) fpars['vis'] = tempname fpars['flagbackup'] = False fpars['outputvis'] = outputmms fpars['separationaxis'] = separationaxis fpars['numsubms'] = numsubms # Run partition only at the MPIServers pdh = ParallelDataHelper('partition', fpars) # Get a cluster pdh.setupCluster(thistask='partition') try: pdh.go() # Remove original MS shutil.rmtree(tempname) except Exception, instance: # Restore MS in case of error in MMS creation shutil.move(tempname, myviso) casalog.post('%s' % instance, 'ERROR') return False casalog.origin('importasdm') # Create a .flagversions for the MS or MMS if flagbackup: for myviso in vistoproc: if os.path.exists(myviso): aflocal.open(myviso) aflocal.saveflagversion( 'Original', comment='Original flags at import into CASA', merge='save') aflocal.done() # Importasdm Flag Parsing if os.access(asdm + '/Flag.xml', os.F_OK): # Find Flag.xml casalog.post('Found Flag.xml in SDM') # Find Antenna.xml if os.access(asdm + '/Antenna.xml', os.F_OK): casalog.post('Found Antenna.xml in SDM') else: raise Exception, 'Failed to find Antenna.xml in SDM' # Find SpectralWindow.xml if os.access(asdm + '/SpectralWindow.xml', os.F_OK): casalog.post('Found SpectralWindow.xml in SDM') else: raise Exception, \ 'Failed to find SpectralWindow.xml in SDM' # # Parse Flag.xml into flag dictionary # if process_flags: flagcmds = fh.parseXML(asdm, float(tbuff)) onlinekeys = flagcmds.keys() nflags = onlinekeys.__len__() # Apply flags to the MS if nflags > 0: idx = 0 for myviso in vistoproc: if applyflags: # Open the MS and attach it to the tool aflocal.open(myviso) # Select the data aflocal.selectdata() # Setup the agent's parameters fh.parseAgents(aflocal, flagcmds, [], True, True, '') # Initialize the agents aflocal.init() # Run the tool aflocal.run(True, True) casalog.post('Applied %s flag commands to %s' % (nflags, myviso)) # Destroy the tool and de-attach the MS aflocal.done() # Save to FLAG_CMD table. APPLIED is set to True. fh.writeFlagCommands(myviso, flagcmds, True, '', '', True) else: casalog.post( 'Will not apply flags to %s (apply_flags=False), use flagcmd to apply' % myviso) # Write to FLAG_CMD, APPLIED is set to False fh.writeFlagCommands(myviso, flagcmds, False, '', '', True) # Save the flag cmds to an ASCII file if savecmds: # Save to standard filename fh.writeFlagCommands(myviso, flagcmds, False, '', outfile[idx], False) casalog.post('Saved %s flag commands to %s' % (nflags, outfile[idx])) idx += 1 else: casalog.post('There are no flag commands to process') else: casalog.post('There is no Flag.xml in ASDM', 'WARN') # Write parameters to HISTORY table of MS mslocal = mstool() param_names = importasdm.func_code.co_varnames[:importasdm.func_code. co_argcount] param_vals = [eval(p) for p in param_names] for myviso in vistoproc: write_history(mslocal, myviso, 'importasdm', param_names, param_vals, casalog) return
def fluxscale(vis=None, caltable=None, fluxtable=None, reference=None, transfer=None, listfile=None, append=None, refspwmap=None, gainthreshold=None, antenna=None, timerange=None, scan=None, incremental=None, fitorder=None, display=None): """Bootstrap the flux density scale from standard calibrators: After running gaincal on standard flux density calibrators (with or without a model), and other calibrators with unknown flux densities, fluxscale will determine the flux density of the unknowns calibrators that are most consistent with the standard calibrator antenna gains. Keyword arguments: vis -- Name of input visibility file default: none; example: vis='ngc5921.ms' caltable -- Name of input calibration table default: none; example: caltable='ngc5921.gcal' This cal table was obtained from task gaincal. fluxtable -- Name of output, flux-scaled calibration table default: none; example: fluxtable='ngc5921.gcal2' The gains in this table have been adjusted by the derived flux density each calibrator. The MODEL_DATA column has NOT been updated for the flux density of the calibrator. Use setjy to do this if it is a point source. reference -- Reference field name(s) The names of the fields with a known flux densities or visibilties that have been placed in the MODEL column by setjy or ft for a model not in the CASA system. The syntax is similar to field. Hence field index or names can be used. default: none; example: reference='1328+307' transfer -- Transfer field name(s) The names of the fields with unknown flux densities. These should be point-like calibrator sources The syntax is similar to field. Hence source index or names can be used. default: '' = all sources in caltable that are not specified as reference sources. Do not include unknown target sources example: transfer='1445+099, 3C84'; transfer = '0,4' NOTE: All fields in reference and transfer must have solutions in the caltable. listfile -- Fit listfile name The list file contains the flux density, flux density error, S/N, and number of solutions (all antennas and feeds) for each spectral window. NOTE: The nominal spectral window frequencies will be included in the future. default: '' = no fit listfile will be created. append -- Append fluxscaled solutions to the fluxtable. default: False; (will overwrite if already existing) example: append=True refspwmap -- Vector of spectral windows enablings scaling across spectral windows default: [-1]==> none. Example with 4 spectral windows: if the reference fields were observed only in spw=1 & 3, and the transfer fields were observed in all 4 spws (0,1,2,3), specify refspwmap=[1,1,3,3]. This will ensure that transfer fields observed in spws 0,1,2,3 will be referenced to reference field solutions only in spw 1 or 3. gainthreshold -- % deviation threshold from the median gain to be used flux scaling derivation antenna -- Select antennas to be used to derive flux scaling timerange -- Select timerage to be used to derive flux scaling with given antenna selection scan -- Select scans to be used to derived flux scaling with given antenna selection incremental -- Create an incremental caltable containing only the gain correction factors. default: False; (older behavior, produces flux scaled gain table) fitorder -- the order of spectral fitting when solutions for multiple spws are available default: 1 display -- display statistics (histogram) of derived correction factors default: false """ try: casalog.origin('fluxscale') mycb = cbtool() mycb.open(filename=vis, compress=False, addcorr=False, addmodel=False) output = mycb.fluxscale(tablein=caltable, tableout=fluxtable, reference=reference, transfer=transfer, listfile=listfile, append=append, refspwmap=refspwmap, gainthreshold=gainthreshold, antenna=antenna, timerange=timerange, scan=scan, incremental=incremental, fitorder=fitorder, display=display) mycb.close() #write history try: param_names = fluxscale.func_code.co_varnames[:fluxscale.func_code. co_argcount] param_vals = [eval(p) for p in param_names] write_history(mstool(), vis, 'fluxscale', param_names, param_vals, casalog) writeResultsHistory(mstool(), vis, casalog, output) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') except Exception, instance: print '*** Error ***', instance mycb.close() raise Exception, instance
def concat(vislist,concatvis,freqtol,dirtol,respectname,timesort,copypointing, visweightscale, forcesingleephemfield): """concatenate visibility datasets The list of data sets given in the vis argument are chronologically concatenated into an output data set in concatvis, i.e. the data sets in vis are first ordered by the time of their earliest integration and then concatenated. If there are fields whose direction agrees within the direction tolerance (parameter dirtol), the actual direction in the resulting, merged output field will be the one from the chronologically first input MS. If concatvis already exists (e.g., it is the same as the first input data set), then the other input data sets will be appended to the concatvis data set. There is no limit to the number of input data sets. If none of the input data sets have any scratch columns (model and corrected columns), none are created in the concatvis. Otherwise these columns are created on output and initialized to their default value (1 in model column, data in corrected column) for those data with no input columns. Spectral windows for each data set with the same chanelization, and within a specified frequency tolerance of another data set will be combined into one spectral window. A field position in one data set that is within a specified direction tolerance of another field position in any other data set will be combined into one field. The field names need not be the same---only their position is used. Each appended dataset is assigned a new observation id (provided the entries in the observation table are indeed different). Keyword arguments: vis -- Name of input visibility files to be combined default: none; example: vis = ['src2.ms','ngc5921.ms','ngc315.ms'] concatvis -- Name of visibility file that will contain the concatenated data note: if this file exits on disk then the input files are added to this file. Otherwise the new file contains the concatenated data. Be careful here when concatenating to an existing file. default: none; example: concatvis='src2.ms' example: concatvis='outvis.ms' other examples: concat(vis=['src2.ms','ngc5921.ms'], concatvis='src2.ms') will concatenate 'ngc5921.ms' into 'src2.ms', and the original src2.ms is lost concat(vis=['src2.ms','ngc5921.ms'], concatvis='out.ms') will concatenate 'ngc5921.ms' and 'src2.ms' into a file named 'out.ms'; the original 'ngc5921.ms' and 'src2.ms' are untouched. concat(vis=['v1.ms','v2.ms'], concatvis = 'vall.ms') then concat(vis=['v3.ms','v4.ms'], concatvis = 'vall.ms') vall.ms will contains v1.ms+v2.ms+v3.ms+v4.ms Note: run flagmanager to save flags in the concatvis freqtol -- Frequency shift tolerance for considering data to be in the same spwid. The number of channels must also be the same. default: '' == 1 Hz example: freqtol='10MHz' will not combine spwid unless they are within 10 MHz. Note: This option is useful to combine spectral windows with very slight frequency differences caused by Doppler tracking, for example. dirtol -- Direction shift tolerance for considering data as the same field default: '' == 1 mas (milliarcsec) example: dirtol='1.arcsec' will not combine data for a field unless their phase center differ by less than 1 arcsec. If the field names are different in the input data sets, the name in the output data set will be the first relevant data set in the list. respectname -- If true, fields with a different name are not merged even if their direction agrees (within dirtol) default: False timesort -- If true, the output visibility table will be sorted in time. default: false. Data in order as read in. example: timesort=true Note: There is no constraint on data that is simultaneously observed for more than one field; for example multi-source correlation of VLBA data. copypointing -- Make a proper copy of the POINTING subtable (can be time consuming). If False, the result is an empty POINTING table. default: True visweightscale -- The weights of the individual MSs will be scaled in the concatenated output MS by the factors in this list. Useful for handling heterogeneous arrays. Use plotms to inspect the "Wt" column as a reference for determining the scaling factors. See the cookbook for more details. example: [1.,3.,3.] - scale the weights of the second and third MS by a factor 3. default: [] (empty list) - no scaling forcesingleephemfield -- By default, concat will only merge two ephemeris fields if the first ephemeris covers the time range of the second. Otherwise, two separate fields with separate ephemerides are placed in the output MS. In order to override this behaviour and make concat merge the non-overlapping or only partially overlapping input ephemerides, the name or id of the field in question needs to be placed into the list in parameter 'forcesingleephemfield'. example: ['Neptune'] - will make sure that there is only one joint ephemeris for field Neptune in the output MS default: '' - standard treatment of all ephemeris fields """ ### #Python script try: casalog.origin('concat') t = tbtool() m = mstool() #break the reference between vis and vislist as we modify vis if(type(vislist)==str): vis=[vislist] else: vis=list(vislist) #dto. for concavis theconcatvis = concatvis # warn if there are MMSs mmslist = [] for elvis in vis : ###Oh no Elvis does not exist Mr Bill if(ParallelTaskHelper.isParallelMS(elvis)): mmslist.append(elvis) if len(mmslist)>0: if (vis[0] == mmslist[0]): casalog.post('*** The first input MS is a multi-MS to which no row can be added. Cannot proceed.', 'WARN') casalog.post('*** Please use virtualconcat or convert the first input MS to a normal MS using split.', 'WARN') raise Exception, 'Cannot append to a multi-MS. Please use virtualconcat.' casalog.post('*** The following input measurement sets are multi-MSs', 'INFO') for mname in mmslist: casalog.post('*** '+mname, 'INFO') casalog.post('*** Use virtualconcat to produce a single multi-MS from several multi-MSs.', 'INFO') doweightscale = False if(len(visweightscale)>0): if (len(visweightscale) != len(vis)): raise Exception, 'parameter visweightscale must have same number of elements as parameter vis' for factor in visweightscale: if factor<0.: raise Exception, 'parameter visweightscale must only contain positive numbers' elif factor!=1.: doweightscale=True # process the input MSs in chronological order sortedvis = [] sortedvisweightscale = [] namestuples = [] for name in vis: t.open(name) times = t.getcol('TIME') t.close() times.sort() if doweightscale: namestuples.append( (times[0], name, visweightscale[vis.index(name)]) ) else: namestuples.append( (times[0], name, 0) ) sorted_namestuples = sorted(namestuples, key=lambda msname: msname[0]) for i in range(0,len(vis)): sortedvis.append(sorted_namestuples[i][1]) sortedvisweightscale.append(sorted_namestuples[i][2]) if((type(concatvis)!=str) or (len(concatvis.split()) < 1)): raise Exception, 'parameter concatvis is invalid' existingconcatvis = False if(vis.count(concatvis) > 0): existingconcatvis = True cvisindex = sortedvis.index(concatvis) if not sorted_namestuples[cvisindex][0] == sorted_namestuples[0][0]: raise Exception, 'If concatvis is set to the name of an existing MS in vis, it must be the chronologically first.'+\ '\n I.e. in this case you should set concatvis to '+sortedvis[0] sortedvis.pop(cvisindex) if doweightscale: vwscale = sortedvisweightscale[cvisindex] sortedvisweightscale.pop(cvisindex) sortedvisweightscale = [vwscale] + sortedvisweightscale # move the corresponding weight to the front if not vis == sortedvis: casalog.post('The list of input MSs is not in chronological order and will need to be sorted.' , 'INFO') casalog.post('The chronological order in which the concatenation will take place is:' , 'INFO') if existingconcatvis: casalog.post(' MJD '+str(qa.splitdate(qa.quantity(sorted_namestuples[0][0],'s'))['mjd'])+': '+concatvis, 'INFO') for name in sortedvis: casalog.post(' MJD '+str(qa.splitdate(qa.quantity(sorted_namestuples[sortedvis.index(name)][0],'s'))['mjd'])+': '+name, 'INFO') if doweightscale: casalog.post('In this new order, the weights are:'+str(sortedvisweightscale) , 'INFO') # replace the original vis and visweightscale by the sorted ones (with concatvis removed if it exists) vis = sortedvis visweightscale = sortedvisweightscale if(os.path.exists(concatvis)): casalog.post('Will be concatenating into the existing ms '+concatvis , 'WARN') if doweightscale and not existingconcatvis: visweightscale = [1.]+visweightscale # set the weight for this existing MS to 1. casalog.post('The weights for this existing MS will be left unchanged.' , 'WARN') else: if(len(vis) >0): # (note: in case len is 1, we only copy, essentially) casalog.post('copying '+vis[0]+' to '+theconcatvis , 'INFO') shutil.copytree(vis[0], theconcatvis) # note that the resulting copy is writable even if the original was read-only vis.pop(0) # don't need to pop visweightscale here! if not copypointing: # remove the rows from the POINTING table of the first MS casalog.post('*** copypointing==False: resulting MS will have empty POINTING table.', 'INFO') tmptabname = 'TMPPOINTING'+str(time.time()) shutil.rmtree(tmptabname, ignore_errors=True) shutil.move(theconcatvis+'/POINTING', tmptabname) t.open(tmptabname) if(t.nrows()>0): ttab = t.copy(newtablename=theconcatvis+'/POINTING', deep=False, valuecopy=True, norows=True) ttab.close() t.close() shutil.rmtree(tmptabname, ignore_errors=True) else: # the POINTING table is already empty casalog.post('*** Input POINTING table was already empty.', 'INFO') shutil.move(tmptabname, theconcatvis+'/POINTING') t.close() # handle the ephemeris concatenation if not forcesingleephemfield=='': from recipes.ephemerides import concatephem if type(forcesingleephemfield)==str or type(forcesingleephemfield)==int: forcesingleephemfield = [forcesingleephemfield] if not type(forcesingleephemfield) == list: raise Exception, 'Type of parameter forcesingleephemfield must be str, int, or list' themss = [theconcatvis] for x in vis: themss.append(x) for ephemfield in forcesingleephemfield: if not type(ephemfield)==int: ephemfield = str(ephemfield) casalog.post('*** Forcing single ephemeris for field '+str(ephemfield), 'INFO') thetabs = concatephem.findephems(themss, ephemfield) if thetabs != [] and not ('' in thetabs): tmptab = os.path.basename(thetabs[0])+'.concattmp' targettab = theconcatvis+'/FIELD/'+os.path.basename(thetabs[0]) if not os.path.exists(targettab): raise Exception, 'Internal ERROR: ephemeris '+targettab+' does not exist' concatephem.concatephem(thetabs, tmptab) if os.path.exists(tmptab): os.system('rm -rf '+targettab) os.system('mv '+tmptab+' '+targettab) else: casalog.post('ERROR while forcing single ephemeris for field '+str(ephemfield), 'SEVERE') raise Exception, 'Concatenation of ephemerides for field '+str(ephemfield)+' failed.' else: casalog.post('ERROR while forcing single ephemeris for field '+str(ephemfield), 'SEVERE') raise Exception, 'Cannot find ephemerides for field '+str(ephemfield)+' in all input MSs.' # Determine if scratch columns should be considered at all # by checking if any of the MSs has them. considerscrcols = False considercorr = False considermodel = False needscrcols = [] needmodel = [] needcorr = [] if ((type(theconcatvis)==str) and (os.path.exists(theconcatvis))): # check if all scratch columns are present t.open(theconcatvis) if (t.colnames().count('MODEL_DATA')==1): considermodel = True if(t.colnames().count('CORRECTED_DATA')==1): considercorr = True needscrcols.append(t.colnames().count('CORRECTED_DATA')==0 or t.colnames().count('MODEL_DATA')==0) needmodel.append(t.colnames().count('MODEL_DATA')==0) needcorr.append(t.colnames().count('CORRECTED_DATA')==0) t.close() else: raise Exception, 'Visibility data set '+theconcatvis+' not found - please verify the name' for elvis in vis : ###Oh no Elvis does not exist Mr Bill if(not os.path.exists(elvis)): raise Exception, 'Visibility data set '+elvis+' not found - please verify the name' # check if all scratch columns are present t.open(elvis) if (t.colnames().count('MODEL_DATA')==1): considermodel = True if(t.colnames().count('CORRECTED_DATA')==1): considercorr = True needscrcols.append(t.colnames().count('CORRECTED_DATA')==0 or t.colnames().count('MODEL_DATA')==0) needmodel.append(t.colnames().count('MODEL_DATA')==0) needcorr.append(t.colnames().count('CORRECTED_DATA')==0) t.close() considerscrcols = (considercorr or considermodel) # there are scratch columns # start actual work, file existence has already been checked i = 0 if(considerscrcols and needscrcols[i]): # create scratch cols casalog.post('creating scratch columns in '+theconcatvis , 'INFO') _cb.open(theconcatvis, addcorr=(considercorr and needcorr[i]), addmodel=(considermodel and needmodel[i])) # calibrator-open creates scratch columns _cb.close() # scale the weights and sigma of the first MS in the chain if doweightscale: wscale = visweightscale[i] if(wscale==1.): casalog.post('Will leave the weights for this MS unchanged.', 'INFO') else: casalog.post('Scaling weights for first MS by factor '+str(wscale), 'INFO') t.open(theconcatvis, nomodify=False) for colname in [ 'WEIGHT', 'WEIGHT_SPECTRUM']: if (colname in t.colnames()) and (t.iscelldefined(colname,0)): for j in xrange(0,t.nrows()): a = t.getcell(colname, j) a *= wscale t.putcell(colname, j, a) for colname in ['SIGMA']: if (wscale > 0. and colname in t.colnames()) and (t.iscelldefined(colname,0)): sscale = 1./sqrt(wscale) for j in xrange(0,t.nrows()): a = t.getcell(colname, j) a *= sscale t.putcell(colname, j, a) t.close() # determine handling switch value handlingswitch = 0 if not copypointing: handlingswitch = 2 m.open(theconcatvis,nomodify=False) mmsmembers = [theconcatvis] for elvis in vis : i = i + 1 destms = "" casalog.post('concatenating '+elvis+' into '+theconcatvis , 'INFO') wscale = 1. if doweightscale: wscale = visweightscale[i] if(wscale==1.): casalog.post('Will leave the weights for this MS unchanged.', 'INFO') else: casalog.post('Will scale weights for this MS by factor '+str(wscale) , 'INFO') if(considerscrcols and needscrcols[i]): if(ParallelTaskHelper.isParallelMS(elvis)): raise Exception, 'Cannot create scratch columns in a multi-MS. Use virtualconcat.' else: # create scratch cols casalog.post('creating scratch columns for '+elvis+' (original MS unchanged)', 'INFO') tempname = elvis+'_with_scrcols' shutil.rmtree(tempname, ignore_errors=True) shutil.copytree(elvis, tempname) _cb.open(tempname, addcorr=(considercorr and needcorr[i]), addmodel=(considermodel and needmodel[i])) # calibrator-open creates scratch columns _cb.close() # concatenate copy instead of original file m.concatenate(msfile=tempname,freqtol=freqtol,dirtol=dirtol,respectname=respectname, weightscale=wscale,handling=handlingswitch, destmsfile=destms) shutil.rmtree(tempname, ignore_errors=True) else: m.concatenate(msfile=elvis,freqtol=freqtol,dirtol=dirtol,respectname=respectname, weightscale=wscale,handling=handlingswitch, destmsfile=destms) if timesort: casalog.post('Sorting main table by TIME ...', 'INFO') m.timesort() # Write history to output MS, not the input ms. try: param_names = concat.func_code.co_varnames[:concat.func_code.co_argcount] param_vals = [eval(p) for p in param_names] write_history(mstool(), concatvis, 'concat', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') m.close() return True
def cvel(vis, outputvis, passall, field, spw, selectdata, antenna, timerange, scan, array, mode, nchan, start, width, interpolation, phasecenter, restfreq, outframe, veltype, hanning): """ regrid an MS to a new spectral window / channel structure or frame vis -- Name of input visibility file default: none; example: vis='ngc5921.ms' outputvis -- Name of output measurement set (required) default: none; example: vis='ngc5921-regridded.ms' passall -- if False, data not meeting the selection on field or spw is omitted; if True, data not meeting the selection is passed through without modification default: False; example: field='NGC5921' passall=False : only data from NGC5921 is included in output MS, no data from other fields (e.g. 1331+305) is included passall=True : data from NGC5921 is transformed by cvel, all other fields are passed through unchanged field -- Select fields in MS. Use field id(s) or field name(s). ['go listobs' to obtain the list id's or names] default: ''= all fields If field string is a non-negative integer, it is assumed to be a field index otherwise, it is assumed to be a field name field='0~2'; field ids 0,1,2 field='0,4,5~7'; field ids 0,4,5,6,7 field='3C286,3C295'; field named 3C286 and 3C295 field = '3,4C*'; field id 3, all names starting with 4C spw --Select spectral window/channels NOTE: This selects the data passed as the INPUT to mode default: ''=all spectral windows and channels spw='0~2,4'; spectral windows 0,1,2,4 (all channels) spw='0:5~61'; spw 0, channels 5 to 61 spw='<2'; spectral windows less than 2 (i.e. 0,1) spw='0,10,3:3~45'; spw 0,10 all channels, spw 3, channels 3 to 45. spw='0~2:2~6'; spw 0,1,2 with channels 2 through 6 in each. spw='0:0~10;15~60'; spectral window 0 with channels 0-10,15-60 spw='0:0~10,1:20~30,2:1;2;3'; spw 0, channels 0-10, spw 1, channels 20-30, and spw 2, channels, 1,2 and 3 selectdata -- Other data selection parameters default: True selectdata=True expandable parameters antenna -- Select data based on antenna/baseline default: '' (all) If antenna string is a non-negative integer, it is assumed to be an antenna index, otherwise, it is considered an antenna name. antenna='5&6'; baseline between antenna index 5 and index 6. antenna='VA05&VA06'; baseline between VLA antenna 5 and 6. antenna='5&6;7&8'; baselines 5-6 and 7-8 antenna='5'; all baselines with antenna index 5 antenna='05'; all baselines with antenna number 05 (VLA old name) antenna='5,6,9'; all baselines with antennas 5,6,9 index numbers timerange -- Select data based on time range: default = '' (all); examples, timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss' Note: if YYYY/MM/DD is missing date defaults to first day in data set timerange='09:14:0~09:54:0' picks 40 min on first day timerange= '25:00:00~27:30:00' picks 1 hr to 3 hr 30min on NEXT day timerange='09:44:00' pick data within one integration of time timerange='>10:24:00' data after this time scan -- Scan number range. default: '' (all) example: scan='1~5' Check 'go listobs' to insure the scan numbers are in order. array -- Select data by (sub)array indices default: '' (all); example: array='0~2'; arrays 0 to 2 mode -- Frequency Specification: NOTE: See examples below: default: 'channel' mode = 'channel'; Use with nchan, start, width to specify output spw. See examples below mode = 'velocity', means channels are specified in velocity. mode = 'frequency', means channels are specified in frequency. mode expandable parameters Start, width are given in units of channels, frequency or velocity as indicated by mode nchan -- Number of channels in output spw default: -1 = all channels; example: nchan=3 start -- Start input channel (relative-0) default=0; example: start=5 width -- Output channel width in units of the input channel width (>1 indicates channel averaging) default=1; example: width=4 interpolation -- Interpolation method default = 'linear' examples: spw = '0,1'; mode = 'channel' will produce a single spw containing all channels in spw 0 and 1 spw='0:5~28^2'; mode = 'channel' will produce a single spw made with channels (5,7,9,...,25,27) spw = '0'; mode = 'channel': nchan=3; start=5; width=4 will produce an spw with 3 output channels new channel 1 contains data from channels (5+6+7+8) new channel 2 contains data from channels (9+10+11+12) new channel 3 contains data from channels (13+14+15+16) spw = '0:0~63^3'; mode='channel'; nchan=21; start = 0; width = 1 will produce an spw with 21 channels new channel 1 contains data from channel 0 new channel 2 contains data from channel 2 new channel 21 contains data from channel 61 spw = '0:0~40^2'; mode = 'channel'; nchan = 3; start = 5; width = 4 will produce an spw with three output channels new channel 1 contains channels (5,7) new channel 2 contains channels (13,15) new channel 3 contains channels (21,23) phasecenter -- direction measure or fieldid for the mosaic center default: '' => first field selected ; example: phasecenter=6 or phasecenter='J2000 19h30m00 -40d00m00' restfreq -- Specify rest frequency to use for output image default='' Occasionally it is necessary to set this (for example some VLA spectral line data). For example for NH_3 (1,1) put restfreq='23.694496GHz' outframe -- output reference frame (not case-sensitive) possible values: LSRK, LSRD, BARY, GALACTO, LGROUP, CMB, GEO, TOPO or SOURCE default='' (keep original reference frame) ; example: outframe='BARY' veltype -- definition of velocity (in mode) default = 'radio' hanning -- if true, Hanning smooth frequency channel data before regridding to remove Gibbs ringing default = False """ #Python script # make ms and tb tool local ms = casac.ms() tb = casac.table() try: casalog.origin('cvel') if not ((type(vis) == str) & (os.path.exists(vis))): raise Exception, 'Visibility data set not found - please verify the name' if (outputvis == ""): raise Exception, "Must provide output data set name in parameter outputvis." if os.path.exists(outputvis): raise Exception, "Output MS %s already exists - will not overwrite." % outputvis if (os.path.exists(outputvis + ".flagversions")): raise Exception, "The flagversions \"%s.flagversions\" for the output MS already exist. Please delete." % outputvis # Handle selectdata explicitly # (avoid hidden globals) if (selectdata == False): timerange = '' array = '' antenna = '' scan = '' if (type(antenna) == list): antenna = ', '.join([str(ant) for ant in antenna]) if (field == ''): field = '*' if (spw == ''): spw = '*' if (passall and spw == '*' and field == '*'): # all spws and fields selected, nothing to pass through passall = False doselection = True if (field == '*' and spw == '*' and (not selectdata or (selectdata and antenna == '' and timerange == '' and scan == '' and array == ''))): doselection = False # open input MS ms.open(vis) if (type(phasecenter) == str): ### blank means take field 0 if (phasecenter == ''): phasecenter = ms.msseltoindex(vis, field=field)['field'][0] else: tmppc = phasecenter try: if (len(ms.msseltoindex(vis, field=phasecenter)['field']) > 0): tmppc = ms.msseltoindex(vis, field=phasecenter)['field'][0] ##succesful must be string like '0' or 'NGC*' except Exception, instance: ##failed must be a string 'J2000 18h00m00 10d00m00' tmppc = phasecenter phasecenter = tmppc newphasecenter = phasecenter phasecentername = phasecenter if not (type(phasecenter) == str): # first check that this field will be in the output MS if not (phasecenter in ms.msseltoindex(vis, field=field)['field']): message = "Field id " + str(phasecenter) message += " was selected as phasecenter but is not among the fields selected for output: " message += str(ms.msseltoindex(vis, field=field)['field']) ms.close() raise Exception, message tb.open(vis + "/FIELD") try: # get the name for later display phasecentername = tb.getcell( 'NAME', phasecenter) + " (original field " + str(phasecenter) tb.close() # phase center id will change if we select on fields, # the name column is only optionally filled and not necessarily unique. # But ms.msseltoindex(vis,field=field)['field'] returns the old field ids # in the order in which they will occur in the new field table. # => need to get index from there as new phase center ID newphasecenter = (ms.msseltoindex( vis, field=field)['field']).tolist().index(phasecenter) phasecentername += ", new field " + str(newphasecenter) + ")" except: tb.close() message = "phasecenter field id " + str( phasecenter) + " is not valid" ms.close() raise Exception, message if (mode == 'frequency'): ## reset the default values if (start == 0): start = "" if (width == 1): width = "" ##check that start and width have units if they are non-empty if not (start == ""): if (qa.quantity(start)['unit'].find('Hz') < 0): ms.close() raise TypeError, "start parameter is not a valid frequency quantity " % start if (not (width == "") and (qa.quantity(width)['unit'].find('Hz') < 0)): ms.close() raise TypeError, "width parameter %s is not a valid frequency quantity " % width elif (mode == 'velocity'): ## reset the default values if (start == 0): start = "" if (width == 1): width = "" ##check that start and width have units if they are non-empty if not (start == ""): if (qa.quantity(start)['unit'].find('m/s') < 0): ms.close() raise TypeError, "start parameter %s is not a valid velocity quantity " % start if (not (width == "") and (qa.quantity(width)['unit'].find('m/s') < 0)): ms.close() raise TypeError, "width parameter %s is not a valid velocity quantity " % width elif (mode == 'channel' or mode == 'channel_b'): if ((type(width) != int) or (type(start) != int)): ms.close() raise TypeError, "start and width have to be integers with mode = %s" % mode ## check if preaveraging is necessary dopreaverage = False preavwidth = [1] thespwsel = ms.msseltoindex(vis=vis, spw=spw)['spw'] thefieldsel = ms.msseltoindex(vis=vis, field=field)['field'] outgrid = ms.cvelfreqs(spwids=thespwsel, fieldids=thefieldsel, mode=mode, nchan=nchan, start=start, width=width, phasec=newphasecenter, restfreq=restfreq, outframe=outframe, veltype=veltype, verbose=False) if (len(outgrid) > 1): tmpavwidth = [] for thespw in thespwsel: outgridw1 = ms.cvelfreqs( spwids=[thespw], fieldids=thefieldsel, mode='channel', nchan=-1, start=0, width=1, # native width phasec=newphasecenter, restfreq=restfreq, outframe=outframe, veltype=veltype, verbose=False) if (len(outgridw1) > 1): widthratio = abs((outgrid[1] - outgrid[0]) / (outgridw1[1] - outgridw1[0])) if (widthratio >= 2.0): # do preaverage tmpavwidth.append(int(widthratio + 0.001)) dopreaverage = True else: tmpavwidth.append(1) if dopreaverage: preavwidth = tmpavwidth ms.close() # if in channel mode and preaveraging, if (dopreaverage and (mode == 'channel' or mode == 'channel_b')): if (max(preavwidth) == 1): dopreaverage = False else: # switch to frequency mode mode = 'frequency' if (width > 0): start = str(outgrid[0] / 1E6) + 'MHz' width = str((outgrid[1] - outgrid[0])) + 'Hz' else: start = str(outgrid[len(outgrid) - 1] / 1E6) + 'MHz' width = str(-(outgrid[1] - outgrid[0])) + 'Hz' casalog.post( "After pre-averaging, will switch to frequency mode with start=" + start + ", width = " + width, 'INFO') if (dopreaverage and hanning and max(preavwidth) > 2): casalog.post( "NOTE: You have requested Hanning smoothing and at the same time you have chosen\n" + "a large width parameter which makes pre-averaging necessary.\n" + "The Hanning-smoothing may be redundant in this context.\n", 'WARN') if dopreaverage: # Past this point we know we are going to 'dopreaverage' # CAS-9798 raise RuntimeError( 'ERROR: cvel does not regrid properly for channel ' 'widths > or = 2x the native channel width, please use ' 'mstransform, clean, or tclean for larger regridding. ' 'All earlier versions of CASA also have this issue.') # determine parameter "datacolumn" tb.open(vis) allcols = tb.colnames() tb.close() dpresent = ('DATA' in allcols) mpresent = ('MODEL_DATA' in allcols) cpresent = ('CORRECTED_DATA' in allcols) if (dpresent and cpresent): datacolumn = 'all' elif (dpresent and not cpresent): datacolumn = 'data' elif (cpresent and not dpresent): datacolumn = 'corrected_data' elif (mpresent and not cpresent and not dpresent): datacolumn = 'model_data' else: raise Exception, "Neither DATA nor CORRECTED_DATA nor MODEL_DATA column present. Cannot proceed." if (doselection and not dopreaverage): casalog.post("Creating selected SubMS ...", 'INFO') ms.open(vis) ms.split(outputms=outputvis, field=field, spw=spw, step=[1], baseline=antenna, subarray=array, timebin='-1s', time=timerange, whichcol=datacolumn, scan=scan, uvrange="") ms.close() elif (dopreaverage and not doselection): if (hanning): casalog.post("Creating working copy for Hanning-smoothing ...", 'INFO') shutil.rmtree(outputvis + 'TMP', ignore_errors=True) shutil.copytree(vis, outputvis + 'TMP') ms.open(outputvis + 'TMP', nomodify=False) ms.hanningsmooth(datacolumn=datacolumn) ms.close() hanning = False ms.open(outputvis + 'TMP') else: ms.open(vis) casalog.post( "Creating preaveraged SubMS using widths " + str(preavwidth), 'INFO') ms.split(outputms=outputvis, whichcol=datacolumn, step=preavwidth) ms.close() elif (doselection and dopreaverage): if (hanning): casalog.post( "Creating selected working copy for Hanning-smoothing ...", 'INFO') shutil.rmtree(outputvis + 'TMP', ignore_errors=True) ms.open(vis) ms.split(outputms=outputvis + 'TMP', field=field, spw=spw, step=[1], baseline=antenna, subarray=array, timebin='-1s', time=timerange, whichcol=datacolumn, scan=scan, uvrange="") ms.close() ms.open(outputvis + 'TMP', nomodify=False) ms.hanningsmooth(datacolumn=datacolumn) ms.close() hanning = False ms.open(outputvis + 'TMP') casalog.post( "Creating preaveraged SubMS using widths " + str(preavwidth), 'INFO') ms.split(outputms=outputvis, whichcol=datacolumn, step=preavwidth) ms.close() else: casalog.post( "Creating selected, preaveraged SubMS using widths " + str(preavwidth), 'INFO') ms.open(vis) ms.split(outputms=outputvis, field=field, spw=spw, step=preavwidth, baseline=antenna, subarray=array, timebin='-1s', time=timerange, whichcol=datacolumn, scan=scan, uvrange="") ms.close() else: # no selection or preaveraging necessary, just copy casalog.post("Creating working copy ...", 'INFO') shutil.rmtree(outputvis, ignore_errors=True) shutil.copytree(vis, outputvis) # Combine and if necessary regrid it ms.open(outputvis, nomodify=False) message = "Using " + phasecentername + " as common direction for the output reference frame." casalog.post(message, 'INFO') if not ms.cvel(mode=mode, nchan=nchan, start=start, width=width, interp=interpolation, phasec=newphasecenter, restfreq=restfreq, outframe=outframe, veltype=veltype, hanning=hanning): ms.close() raise Exception, "Error in regridding step ..." ms.close() # deal with the passall option temp_suffix = ".deselected" if (passall): # determine range of fields fieldsel = ms.msseltoindex(vis=vis, field=field)['field'] tb.open(vis + "/FIELD") nfields = tb.nrows() tb.close() fielddesel = "" for i in xrange(0, nfields): if not (i in fieldsel): if not (fielddesel == ""): fielddesel += "," fielddesel += str(i) # determine range of SPWs spwsel = ms.msseltoindex(vis=vis, spw=spw)['spw'] tb.open(vis + "/SPECTRAL_WINDOW") nspws = tb.nrows() tb.close() spwdesel = "" for i in xrange(0, nspws): if not (i in spwsel): if not (spwdesel == ""): spwdesel += "," spwdesel += str(i) if not (fielddesel == "" and spwdesel == ""): # split out the data not selected by the conditions on field and spw # from the original MS and join it to the output MS # need to do this in two steps # I) field == "*" and deselected spws if not (spwdesel == ""): ms.open(vis) casalog.post("Passing through data with", 'INFO') casalog.post(" spws: " + spwdesel, 'INFO') ms.split(outputms=outputvis + temp_suffix, field='*', spw=spwdesel, step=[1], baseline=antenna, subarray=array, timebin='-1s', time=timerange, whichcol=datacolumn, scan=scan, uvrange="") ms.close() # join with the deselected part ms.open(outputvis, nomodify=False) rval = ms.concatenate(msfile=outputvis + temp_suffix) ms.close() shutil.rmtree(outputvis + temp_suffix) if not rval: raise Exception, "Error in attaching passed-through data ..." # II) deselected fields and selected spws if not (fielddesel == ""): ms.open(vis) casalog.post("Passing through data with", 'INFO') casalog.post(" fields: " + fielddesel, 'INFO') casalog.post(" spws: " + spw, 'INFO') ms.split(outputms=outputvis + temp_suffix, field=fielddesel, spw=spw, step=[1], baseline=antenna, subarray=array, timebin='-1s', time=timerange, whichcol=datacolumn, scan=scan, uvrange="") ms.close() # join with the deselected part ms.open(outputvis, nomodify=False) rval = ms.concatenate(msfile=outputvis + temp_suffix) ms.close() shutil.rmtree(outputvis + temp_suffix) if not rval: raise Exception, "Error in attaching passed-through data ..." # Write history to output MS try: param_names = cvel.func_code.co_varnames[:cvel.func_code. co_argcount] param_vals = [eval(p) for p in param_names] write_history(mstool(), outputvis, 'cvel', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN')
def sdbaseline(infile=None, datacolumn=None, antenna=None, field=None, spw=None, timerange=None, scan=None, pol=None, intent=None, reindex=None, maskmode=None, thresh=None, avg_limit=None, minwidth=None, edge=None, blmode=None, dosubtract=None, blformat=None, bloutput=None, bltable=None, blfunc=None, order=None, npiece=None, applyfft=None, fftmethod=None, fftthresh=None, addwn=None, rejwn=None, clipthresh=None, clipniter=None, blparam=None, verbose=None, showprogress=None, minnrow=None, outfile=None, overwrite=None): casalog.origin('sdbaseline') try: if not os.path.exists(infile): raise Exception("infile='" + str(infile) + "' does not exist.") if (outfile == '') or not isinstance(outfile, str): #print("type=%s, value=%s" % (type(outfile), str(outfile))) #raise ValueError, "outfile name is empty." outfile = infile.rstrip('/') + '_bs' print("outfile is empty or non-string. set to '" + outfile + "'") if os.path.exists(outfile) and not overwrite: raise Exception("outfile='%s' exists, and cannot overwrite it." % (outfile)) if (maskmode == 'interact'): raise ValueError, "maskmode='%s' is not supported yet" % maskmode if (blfunc == 'variable' and not os.path.exists(blparam)): raise ValueError, "input file '%s' does not exists" % blparam if (spw == ''): spw = '*' if (blmode == 'apply'): if not os.path.exists(bltable): raise ValueError, "file specified in bltable '%s' does not exist." % bltable sorttab_info = remove_sorted_table_keyword(infile) if overwrite and os.path.exists(outfile) and (infile != outfile): os.system('rm -rf %s' % outfile) selection = ms.msseltoindex(vis=infile, spw=spw, field=field, baseline=antenna, time=timerange, scan=scan) sdms.open(infile) sdms.set_selection(spw=sdutil.get_spwids(selection), field=field, antenna=antenna, timerange=timerange, scan=scan, polarization=pol, intent=intent, reindex=reindex) sdms.apply_baseline_table(bltable=bltable, datacolumn=datacolumn, spw=spw, outfile=outfile) sdms.close() restore_sorted_table_keyword(infile, sorttab_info) elif (blmode == 'fit'): if(blfunc == 'sinusoid'): addwn = sdutil.parse_wavenumber_param(addwn) rejwn = sdutil.parse_wavenumber_param(rejwn) check_fftthresh(fftthresh) blformat, bloutput = prepare_for_blformat_bloutput(infile, blformat, bloutput, overwrite) output_bloutput_text_header(blformat, bloutput, blfunc, maskmode, infile, outfile) if (blfunc == 'variable'): sorttab_info = remove_sorted_table_keyword(infile) if overwrite and os.path.exists(outfile) and (infile != outfile): os.system('rm -rf %s' % outfile) selection = ms.msseltoindex(vis=infile, spw=spw, field=field, baseline=antenna, time=timerange, scan=scan) sdms.open(infile) sdms.set_selection(spw=sdutil.get_spwids(selection), field=field, antenna=antenna, timerange=timerange, scan=scan, polarization=pol, intent=intent, reindex=reindex) params, func = prepare_for_baselining(blfunc=blfunc, datacolumn=datacolumn, outfile=outfile, bloutput=','.join(bloutput), dosubtract=dosubtract, spw=spw, pol=pol, linefinding=(maskmode=='auto'), threshold=thresh, avg_limit=avg_limit, minwidth=minwidth, edge=edge, order=order, npiece=npiece, applyfft=applyfft, fftmethod=fftmethod, fftthresh=fftthresh, addwn=addwn, rejwn=rejwn, clip_threshold_sigma=clipthresh, num_fitting_max=clipniter+1, blparam=blparam, verbose=verbose) func(**params) sdms.close() if (blfunc == 'variable'): restore_sorted_table_keyword(infile, sorttab_info) # Write history to outfile param_names = sdbaseline.func_code.co_varnames[:sdbaseline.func_code.co_argcount] param_vals = [eval(p) for p in param_names] write_history(ms, outfile, 'sdbaseline', param_names, param_vals, casalog) except Exception, instance: raise Exception, instance
fh.writeFlagCommands(vis, flagcmd, writeflags, cmdreason, outfile, False) # Destroy the tool aflocal.done() retval = True # Write history to the MS. Only for modes that write to the MS if not iscal: if mode != 'summary' and action == 'apply': try: param_names = flagdata.func_code.co_varnames[:flagdata. func_code. co_argcount] param_vals = [eval(p) for p in param_names] retval &= write_history(mslocal, vis, 'flagdata', param_names, param_vals, casalog) except Exception, instance: casalog.post( "*** Error \'%s\' updating HISTORY" % (instance), 'WARN') # Pull out the 'summary' reports of summary_stats_list. if mode == 'summary' or mode == 'list': ordered_summary_list = OrderedDict(summary_stats_list) nreps = ordered_summary_list['nreport'] if nreps > 0: for rep in range(0, nreps): repname = "report" + str(rep) if ordered_summary_list[repname]['type'] != "summary": ordered_summary_list.pop(repname)
def importasap(infile=None, outputvis=None, flagbackup=None, overwrite=None, parallel=None): """ """ casalog.origin('importasap') try: if infile is None or outputvis is None: raise RuntimeError( 'Error: infile and outputvis must be specified.') # default value if flagbackup is None: flagbackup = True if overwrite is None: overwrite = False # basic check if os.path.exists(outputvis) and not overwrite: raise RuntimeError('%s exists.' % (outputvis)) if not is_scantable(infile): raise RuntimeError('%s is not a valid Scantable.' % (infile)) # import status = mysdms.importasap(infile, outputvis, parallel) if status == True: # flagversions file must be deleted flagversions = outputvis.rstrip('/') + '.flagversions' if os.path.exists(flagversions): os.system('rm -rf %s' % (flagversions)) # initialize weights using cb tool mycb.open(outputvis, compress=False, addcorr=False, addmodel=False) mycb.initweights(wtmode='nyq') # create flagbackup file if user requests it if flagbackup == True: aflocal = casac.agentflagger() aflocal.open(outputvis) aflocal.saveflagversion( 'Original', comment= 'Original flags at import into CASA using importasap', merge='save') aflocal.done() # Write history to output MS param_names = importasap.func_code.co_varnames[:importasap.func_code. co_argcount] param_vals = [eval(p) for p in param_names] write_history(myms, outputvis, 'importasap', param_names, param_vals, casalog) return status except Exception, instance: casalog.post('*** Error *** ' + str(instance), 'SEVERE') raise instance
raise Exception, 'Tsys calibration table %s not found' % tsystable if len(spwmap)==0: spwmap=[-1] if interp=="": interp="linear" # ... and we are asked to do something... # open without adding anything! mycb.open(vis,compress=False,addcorr=False,addmodel=False) mycb.initweights(wtmode=wtmode,dowtsp=dowtsp,tsystable=tsystable,gainfield=gainfield,interp=interp,spwmap=spwmap) mycb.close() else: raise Exception, 'Visibility data set not found - please verify the name' # Write history to MS. # When running in parallel, history will be written in the parallel section above # normal MSs should write the history here if ParallelTaskHelper.isMPIClient(): try: param_names = initweights.func_code.co_varnames[:initweights.func_code.co_argcount] param_vals = [eval(p) for p in param_names] casalog.post('Updating the history in the output', 'DEBUG1') write_history(myms, vis, 'initweights', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') except Exception, instance: print '*** Error ***',instance
def sdcal(infile=None, calmode='tsys', fraction='10%', noff=-1, width=0.5, elongated=False, applytable='',interp='', spwmap={}, outfile='', overwrite=False, field='', spw='', scan='',intent=''): """ Externally specify calibration solutions of various types """ # Composite mode: compute calibration table and calibrate if ',' in calmode: handle_composite_mode(locals()) return # Single mode: either calibrate or compute calibration table try: # Parameters check if calmode == 'tsys': if scan != '': raise UserWarning, "Scan input must be ''(=all) in calmode='tsys'." if spw != '': raise UserWarning, "Spw input must be ''(=all) in calmode='tsys'." if isinstance(infile,str) and os.path.exists(infile): # check if CORRECTED_DATA is necessary addcorr = calmode == 'apply' cb.setvi(old=True) cb.open(filename=infile,compress=False,addcorr=addcorr,addmodel=False) cb.selectvis(spw=spw, scan=scan, field=field) else: raise Exception, 'Infile data set not found - please verify the name' if not isinstance(calmode,str): raise Exception, "Calmode must be a string" if calmode.lower() not in ['tsys', 'ps', 'otfraster', 'otf', 'apply']: raise Exception, "Calmode must be either 'ps' or 'otfraster' or 'otf' or 'tsys' or 'apply'." if (not overwrite) and os.path.exists(outfile): raise RuntimeError, "overwrite is False and output file exists: {}".format(outfile) # Calibration if calmode == 'apply': # Calibrate using existing tables # single calibration table if isinstance(applytable, str): _table_list = [applytable] # multiple calibration tables if isinstance(applytable, list) or isinstance(applytable, numpy.ndarray): _table_list = applytable # no calibration table if len(_table_list) == 0: raise Exception, 'Applytable name must be specified.' # check calibration table files for table in _table_list: # empty string if len(table) == 0: raise Exception, 'Applytable name must be specified.' # unexisting table if not os.path.exists(table): raise Exception, "Table doesn't exist: {}".format(table) # warning on usage difference with asap.sdcal2 if (outfile != ''): warning_msg = '\n'.join([ 'The outfile you specified will NOT be created.', "Calibrated data will be stored in a new 'CORRECTED_DATA' column", 'inserted in the main table of the input MS file.' ]) casalog.post(warning_msg,priority="WARN") if(type(spwmap)!=types.ListType and (type(spwmap)!=types.DictType)): raise Exception, 'Spwmap type must be list or dictionary.' if (type(spwmap)==types.DictType): MS = infile tb.open(MS+'/SPECTRAL_WINDOW') total_spwID=tb.nrows() tb.close() spwmap_dict = spwmap spwmap_list = range(total_spwID) for key, value in spwmap_dict.items(): for v in value: if v in spwmap_list: index = spwmap_list.index(v) spwmap_list[index]=int(key) spwmap = spwmap_list # Setup calibrator for _table in _table_list: caltype = inspect_caltype(_table) if caltype == 'UNKNOWN': raise RuntimeError('Applytable \'%s\' is not a caltable format'%(_table)) elif caltype == 'B TSYS': cb.setapply(table=_table, spwmap=spwmap, interp=interp, calwt=True) else: # no spw mapping is needed for sky calibration cb.setapply(table=_table, interp=interp, calwt=True) # Calibrate cb.correct(applymode='calflag') # Write to HISTORY table of MS param_names = sdcal.func_code.co_varnames[:sdcal.func_code.co_argcount] param_vals = [eval(p) for p in param_names] write_history(myms, infile, 'sdcal', param_names, param_vals, casalog) else: # Compute calibration table # Reconciliating 'Python world' calmode with 'C++ world' calmode # cpp_calmode[python_calmode] = c++_calmode cpp_calmode = { 'tsys': 'tsys', 'ps': 'sdsky_ps', 'otfraster': 'sdsky_raster', 'otf': 'sdsky_otf' } if len(outfile) == 0: raise RuntimeError, 'Output file name must be specified.' if calmode == 'tsys': cb.specifycal(caltable=outfile,time="",spw=spw,caltype=cpp_calmode[calmode]) else: fraction_numeric = to_numeric_fraction(fraction) if noff <= 0 and fraction_numeric >= 0.5: raise ValueError, 'Too many edge points. fraction must be < 0.5.' # Setup calibrator cb.selectvis(spw=spw, scan=scan, field=field, intent=intent) cb.setsolve(type=cpp_calmode[calmode], table=outfile, fraction=fraction_numeric, numedge=noff) # Compute calibration table cb.solve() except UserWarning, instance: print '*** Warning ***',instance
shutil.rmtree(csvis) return False else: # This takes almost 30s/GB. (lustre, 8/2011) casalog.post('Copying ' + vis + ' to ' + csvis + ' with cp.') copy_tree(vis, csvis) # It is less confusing if we write the history now that the "root" MS # is made, but before cb adds its messages. # # Not a dict, because we want to maintain the order. param_names = uvcontsub.func_code.co_varnames[:uvcontsub.func_code. co_argcount] param_vals = [eval(p) for p in param_names] write_history(myms, csvis, 'uvcontsub', param_names, param_vals, casalog) if (type(csvis) == str) and os.path.isdir(csvis): mycb.setvi(old=True, quiet=False) # old VI, for now mycb.open(csvis) else: raise Exception, 'Visibility data set not found - please verify the name' # select the data for continuum subtraction mycb.reset() mycb.selectvis(spw=myfitspw) # Set up the solve amuellertab = tempfile.mkdtemp(prefix='Temp_contsub.tab', dir=workingdir)
casalog.post( 'FLAG_CMD table contains spw selection by name. Will not update it!', 'DEBUG') mytb.close() except Exception, instance: if isopen: mytb.close() mslocal = None mytb = None casalog.post("*** Error \'%s\' updating FLAG_CMD" % (instance), 'SEVERE') return False mytb = None # Write history to output MS, not the input ms. try: param_names = split.func_code.co_varnames[:split.func_code.co_argcount] param_vals = [eval(p) for p in param_names] write_history(mslocal, outputvis, 'split', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') return False mslocal = None return True
def conjugatevis(vis, spwlist=[], outputvis="", overwrite=False): """: Change the sign of the phases in all visibility columns Keyword arguments: vis -- Name of input visibility file default: none; example='3C273XC1.ms' spwlist -- Select spectral window default: [] all spws will be conjugated; example: spw=[1,2] outputvis -- name of output visibility file default: 'conjugated_'+vis; example= 'conjugated.ms' overwrite -- Overwrite the outputvis if it exists default=False; example: overwrite=True """ #Python script tb = tbtool() try: casalog.origin('conjugatevis') myddlist = [] tb.open(vis + '/SPECTRAL_WINDOW') maxspw = tb.nrows() - 1 tb.close() if (type(spwlist) == type(1)): spwlist = [spwlist] elif (spwlist == None or spwlist == [] or spwlist == ""): spwlist = [] casalog.post("Will conjugate visibilities for all spws.", 'INFO') if not spwlist == []: try: tb.open(vis + '/DATA_DESCRIPTION') for k in spwlist: if (k < -1 or k > maxspw): raise Exception, "Error: max valid spw id is " + str( maxspw) raise else: for j in range(0, tb.nrows()): if (tb.getcell("SPECTRAL_WINDOW_ID", j) == k and not (j in myddlist)): myddlist = myddlist + [j] #end for k tb.close() casalog.post('DATA_DESC_IDs to process: ' + str(myddlist), 'INFO') except: raise Exception, 'Error reading DATA_DESCRIPTION table' #endif outname = 'conjugatedvis_' + vis if not outputvis == "": if ((type(outputvis) != str) or (len(outputvis.split()) < 1)): raise Exception, 'parameter outputvis is invalid' outname = outputvis if not overwrite and os.path.exists(outname): raise Exception, 'outputvis ' + outname + ' exists and you did not permit overwrite' os.system('rm -rf ' + outname) os.system('cp -R ' + vis + ' ' + outname) tb.open(outname, nomodify=False) if tb.iswritable(): if (spwlist == None): for colname in ['DATA', 'CORRECTED_DATA', 'FLOAT_DATA']: if colname in tb.colnames(): casalog.post('Conjugating ' + str(colname), 'INFO') for i in xrange(0, tb.nrows()): a = tb.getcell(colname, i) a = a.conjugate() tb.putcell(colname, i, a) else: for colname in ['DATA', 'CORRECTED_DATA', 'FLOAT_DATA']: if colname in tb.colnames(): casalog.post('Conjugating ' + str(colname), 'INFO') for i in xrange(0, tb.nrows()): if (tb.getcell("DATA_DESC_ID", i) in myddlist): a = tb.getcell(colname, i) a = a.conjugate() tb.putcell(colname, i, a) #endif tb.flush() tb.close() casalog.post('Created ' + str(outname), 'INFO') else: tb.close() casalog.post('Cannot write to output MS ' + str(outname), 'WARN') # Write history to output MS try: param_names = conjugatevis.func_code.co_varnames[:conjugatevis. func_code. co_argcount] param_vals = [eval(p) for p in param_names] write_history(mstool(), outputvis, 'conjugatevis', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') except Exception, instance: tb.close() print '*** Error ***', instance raise Exception, instance
def importmiriad( mirfile=None, vis=None, tsys=None, spw=None, vel=None, linecal=None, wide=None, debug=None, ): """Convert a Miriad visibility file into a CASA visibility file (MS). The conversion of the Miriad visibility format into a measurement set. This version has been tested for both ATNF and CARMA Miriad files. ................ Keyword arguments: mirfile -- Name of input Miriad visibility file (directory) default: none; example: mirfile='mydata.uv' .... vis -- Output ms name default: mirfile name with suffix replaced by '.ms' .... tsys -- Set True to use the system temperature to set the visibility weights default: False .... spw -- specify the spectral windows to use ........ default: all .... vel -- Velocity system to use: LSRK, LSRD or TOPO .... default: TOPO for ATCA, LSRK for CARMA .... linecal -- (CARMA only) apply CARMA linecal on the fly? .... default: False .... wide -- (CARMA only) specify the window averages to use ........ default: all ........ .... debug -- specify level of debug messages (0,1,2,3) default: 0 (=None) """ # Python script mymf = casac.miriadfiller() try: try: casalog.origin('importmiriad') # -----------------------------------------' # beginning of importmiriad implementation # ----------------------------------------- mymf.fill(vis, mirfile, tsys, spw, vel, linecal, wide, debug) except Exception, e: print e casalog.post("Failed to import miriad file %s" % mirfile) raise # Write the args to HISTORY. try: mslocal = mstool() param_names = \ importmiriad.func_code.co_varnames[:importmiriad.func_code.co_argcount] param_vals = [eval(p) for p in param_names] write_history(mslocal, vis, 'importmiriad', param_names, param_vals, casalog) except Exception, instance: casalog.post("Failed to updated HISTORY", 'WARN')
antnamescheme=antnamescheme, keepblanks=keepblanks, evlabands=evlabands) i += 1 else: raise Exception, 'Archive file not found - please verify the name' except Exception, instance: print '*** Error importing %s to %s:' % (archivefiles, vis) raise Exception, instance # Write history try: param_names = importvla.func_code.co_varnames[:importvla.func_code. co_argcount] param_vals = [eval(p) for p in param_names] ok &= write_history(mstool(), vis, 'importvla', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') # write initial flag version try: aflocal = casac.agentflagger() ok &= aflocal.open(vis) ok &= aflocal.saveflagversion( 'Original', comment='Original flags at import into CASA', merge='replace') ok &= aflocal.done() except Exception, instance: print '*** Error writing initial flag version of %s:' % vis raise Exception, instance
mytb.close() except Exception, instance: if isopen: mytb.close() mslocal = None mytb = None casalog.post("*** Error \'%s\' updating FLAG_CMD" % (instance), 'SEVERE') return False mytb = None # Write history to output MS, not the input ms. try: param_names = mstransform.func_code.co_varnames[:mstransform.func_code.co_argcount] param_vals = [eval(p) for p in param_names] write_history(mslocal, outputvis, 'mstransform', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') return False mslocal = None return True
def virtualconcat(vislist, concatvis, freqtol, dirtol, respectname, visweightscale, keepcopy, copypointing): """ Concatenate visibility data sets creating a Multi-MS. Combine the input datasets into a Multi-MS. NOTE: The input datasets are moved into the Multi-MS and may be modified to account for subtable reference changes. If none of the input MSs have any scratch columns, none are created. Otherwise scratch columns are created and initialized in those MSs which don't have a complete set. Keyword arguments: vis -- Name of input visibility files (MS) default: none; example: vis=['ngc5921-1.ms', 'ngc5921-2.ms'] concatvis -- Name of the output visibility file default: none; example: concatvis='src2.ms' freqtol -- Frequency shift tolerance for considering data as the same spwid default: '' means always combine example: freqtol='10MHz' will not combine spwid unless they are within 10 MHz dirtol -- Direction shift tolerance for considering data as the same field default: ;; means always combine example: dirtol='1.arcsec' will not combine data for a field unless their phase center is less than 1 arcsec. respectname -- If true, fields with a different name are not merged even if their direction agrees (within dirtol) default: True visweightscale -- list of the weight scales to be applied to the individual MSs default: [] (don't modify weights, equivalent to setting scale to 1 for each MS) keepcopy -- If true, a copy of the input MSs is kept in their original place. default: false copypointing -- If true, the POINTING table information will be present in the output. If false, the result is an empty POINTING table. default: True """ ### #Python script tempdir = '' originalvis = vislist try: casalog.origin('virtualconcat') t = tbtool() m = mstool() #break the reference between vis and vislist as we modify vis if (type(vislist) == str): vis = [vislist] else: vis = list(vislist) #dto. for concavis theconcatvis = concatvis doweightscale = False if (len(visweightscale) > 0): if (len(visweightscale) != len(vis)): raise Exception, 'parameter visweightscale must have same number of elements as parameter vis' for factor in visweightscale: if factor < 0.: raise Exception, 'parameter visweightscale must only contain positive numbers' elif factor != 1.: doweightscale = True if ((type(concatvis) != str) or (len(concatvis.split()) < 1)): raise Exception, 'Parameter concatvis is invalid.' if (vis.count(concatvis) > 0): raise Exception, 'Parameter concatvis must not be equal to one of the members of parameter vis.' if (os.path.exists(concatvis)): raise Exception, 'The output MMS must not yet exist.' # process the input MSs in chronological order sortedvis = [] sortedvisweightscale = [] namestuples = [] for name in vis: t.open(name) times = t.getcol('TIME') t.close() times.sort() if doweightscale: namestuples.append( (times[0], name, visweightscale[vis.index(name)])) else: namestuples.append((times[0], name, 0)) sorted_namestuples = sorted(namestuples, key=lambda msname: msname[0]) for i in range(0, len(vis)): sortedvis.append(sorted_namestuples[i][1]) sortedvisweightscale.append(sorted_namestuples[i][2]) if not vis == sortedvis: casalog.post( 'The list of input MSs is not in chronological order and will need to be sorted.', 'INFO') casalog.post( 'The chronological order in which the concatenation will take place is:', 'INFO') for name in sortedvis: casalog.post( ' MJD ' + str( qa.splitdate( qa.quantity( sorted_namestuples[sortedvis.index(name)][0], 's'))['mjd']) + ': ' + name, 'INFO') if doweightscale: casalog.post( 'In this new order, the weights are:' + str(sortedvisweightscale), 'INFO') # replace the original vis and visweightscale by the sorted ones (with concatvis removed if it exists) vis = sortedvis visweightscale = sortedvisweightscale # if there are MMSs among the input, make their constituents the new input mmslist = [] ismaster = [] for elvis in vis: ismaster.append(True) # may be revised later if (ParallelTaskHelper.isParallelMS(elvis)): mmslist.append(elvis) if len(mmslist) > 0: casalog.post( '*** The following input measurement sets are multi-MSs', 'INFO') for mname in mmslist: casalog.post('*** ' + mname, 'INFO') oldvis = vis oldvisweightscale = visweightscale vis = [] visweightscale = [] ismaster = [] # reset ismaster i = 0 for elvis in oldvis: if elvis in mmslist: # append the subMSs individually m.open(elvis) mses = m.getreferencedtables() m.close() mses.sort() mastername = os.path.basename( os.path.dirname(os.path.realpath(elvis + '/ANTENNA'))) for mname in mses: #print 'subms: ', mname vis.append(mname) if doweightscale: visweightscale.append(oldvisweightscale[i]) if os.path.basename(mname) == mastername: ismaster.append(True) else: ismaster.append(False) else: vis.append(elvis) if doweightscale: visweightscale.append(oldvisweightscale[i]) ismaster.append(True) i += 1 if keepcopy: casalog.post( '*** keepcopy==True: creating copy of input MSs to keep ...', 'INFO') tempdir = 'concat_tmp_' + str(time.time()) os.mkdir(tempdir) for elvis in originalvis: shutil.move(elvis, tempdir) # keep timestamps and permissions shutil.copytree(tempdir + '/' + elvis, elvis, True) # symlinks=True casalog.post('Concatenating ...', 'INFO') if not copypointing: # delete the rows of all pointing tables casalog.post( '*** copypointing==False: resulting MMS will have empty POINTING table.', 'INFO') tmptabname = 'TMPPOINTING' + str(time.time()) tmptabname2 = 'TMPPOINTING2' + str(time.time()) shutil.rmtree(tmptabname, ignore_errors=True) shutil.rmtree(tmptabname2, ignore_errors=True) shutil.move(vis[0] + '/POINTING', tmptabname) t.open(tmptabname) if (t.nrows() > 0): ttab = t.copy(newtablename=tmptabname2, deep=False, valuecopy=True, norows=True) ttab.close() t.close() shutil.rmtree(tmptabname, ignore_errors=True) else: # the POINTING table is already empty t.close() casalog.post('*** Input POINTING table was already empty.', 'INFO') shutil.move(tmptabname, tmptabname2) for i in range( len(vis)): # replace the POINTING tables by the empty one os.system('rm -rf ' + vis[i] + '/POINTING') shutil.copytree(tmptabname2, vis[i] + '/POINTING') shutil.rmtree(tmptabname2, ignore_errors=True) if (len(vis) > 0): # (note: in case len is 1, we only copy, essentially) theconcatvis = vis[0] if (len(vis) == 1): shutil.copytree(vis[0], concatvis, True) vis.remove(vis[0]) # Determine if scratch columns should be considered at all # by checking if any of the MSs has them. considerscrcols = False needscrcols = [] if ((type(theconcatvis) == str) and (os.path.exists(theconcatvis))): # check if all scratch columns are present t.open(theconcatvis) if (t.colnames().count('CORRECTED_DATA') == 1 or t.colnames().count('MODEL_DATA') == 1): considerscrcols = True # there are scratch columns needscrcols.append(t.colnames().count('CORRECTED_DATA') == 0 or t.colnames().count('MODEL_DATA') == 0) t.close() else: raise Exception, 'Visibility data set ' + theconcatvis + ' not found - please verify the name' for elvis in vis: ###Oh no Elvis does not exist Mr Bill if (not os.path.exists(elvis)): raise Exception, 'Visibility data set ' + elvis + ' not found - please verify the name' # check if all scratch columns are present t.open(elvis) if (t.colnames().count('CORRECTED_DATA') == 1 or t.colnames().count('MODEL_DATA') == 1): considerscrcols = True # there are scratch columns needscrcols.append(t.colnames().count('CORRECTED_DATA') == 0 or t.colnames().count('MODEL_DATA') == 0) t.close() # start actual work, file existence has already been checked if (considerscrcols and needscrcols[0]): # create scratch cols casalog.post('creating scratch columns in ' + theconcatvis, 'INFO') _cb.open(theconcatvis) # calibrator-open creates scratch columns _cb.close() # scale the weights of the first MS in the chain if doweightscale: wscale = visweightscale[0] if (wscale == 1.): casalog.post('Will leave the weights for this MS unchanged.', 'INFO') else: casalog.post( 'Scaling weights for first MS by factor ' + str(wscale), 'INFO') t.open(theconcatvis, nomodify=False) for colname in ['WEIGHT', 'WEIGHT_SPECTRUM']: if (colname in t.colnames()) and (t.iscelldefined( colname, 0)): for j in xrange(0, t.nrows()): a = t.getcell(colname, j) a *= wscale t.putcell(colname, j, a) t.close() m.open(theconcatvis, nomodify=False) mmsmembers = [theconcatvis] auxfile = 'concat_aux_' + str(time.time()) i = 0 for elvis in vis: i = i + 1 mmsmembers.append(elvis) casalog.post('adding ' + elvis + ' to multi-MS ' + concatvis, 'INFO') wscale = 1. if doweightscale: wscale = visweightscale[i] if (wscale == 1.): casalog.post( 'Will leave the weights for this MS unchanged.', 'INFO') else: casalog.post( 'Will scale weights for this MS by factor ' + str(wscale), 'INFO') if (considerscrcols and needscrcols[i]): # create scratch cols casalog.post('creating scratch columns for ' + elvis, 'INFO') _cb.open(elvis) # calibrator-open creates scratch columns _cb.close() m.virtconcatenate(msfile=elvis, auxfilename=auxfile, freqtol=freqtol, dirtol=dirtol, respectname=respectname, weightscale=wscale) #end for os.remove(auxfile) m.close() # Write history to MS try: param_names = virtualconcat.func_code.co_varnames[:virtualconcat. func_code. co_argcount] param_vals = [eval(p) for p in param_names] write_history(mstool(), theconcatvis, 'virtualconcat', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') # concatenate the POINTING tables masterptable = mmsmembers[0] + '/POINTING' ptablemembers = [] if os.path.exists(masterptable) and copypointing: casalog.post('Concatenating the POINTING tables ...', 'INFO') i = 0 for i in xrange(len(mmsmembers)): ptable = mmsmembers[i] + '/POINTING' if ismaster[i] and os.path.exists(ptable): casalog.post(' ' + ptable, 'INFO') shutil.move(ptable, ptable + str(i)) ptablemembers.append(ptable + str(i)) #end for t.createmultitable(masterptable, ptablemembers, 'SUBTBS') # endif # Get all available subtables thesubtables = ph.getSubtables(mmsmembers[0]) # Remove the SOURCE and HISTORY tables, which will be the only copied. # All other sub-tables will be linked to first subms thesubtables.remove('SOURCE') thesubtables.remove('HISTORY') subtabs_to_omit = thesubtables ph.makeMMS( concatvis, mmsmembers, True, # copy subtables from first to all other members subtabs_to_omit) # excluding tables which will be linked # remove the remaining "hulls" of the emptied input MMSs (if there are any) for elvis in mmslist: shutil.rmtree(elvis) if keepcopy: for elvis in originalvis: shutil.move(tempdir + '/' + elvis, elvis) os.rmdir(tempdir)