Esempio n. 1
0
def get_scantimes(ms):

    # Open the MS

    tbHandle = casac.table()
    tbHandle.open(ms)

    # Get the time and scan columns

    times = tbHandle.getcol('TIME')
    scans = tbHandle.getcol('SCAN_NUMBER')

    # Assign each time to the proper scan

    d = dict()

    for r in range(tbHandle.nrows()):
        if not d.has_key(scans[r]): d[scans[r]] = list()
        d[scans[r]].append(times[r])

    tbHandle.close()
    del tbHandle

    # Get the minimum and maximum time for each scan

    scanTimes = dict()

    for k in d.keys():
        scanTimes[k] = dict()
        scanTimes[k]['min'] = numpy.min(d[k])
        scanTimes[k]['max'] = numpy.max(d[k])

    # Return the minimum and maximum time for each scan

    return scanTimes
Esempio n. 2
0
def pixelmask2cleanmask(imagename='',
                        maskname='mask0',
                        maskimage='',
                        usemasked=False):
    """
    convert pixel(T/F) mask (in a CASA image) to a mask image (1/0)
    used for clean
    imagename - input imagename that contain a mask to be used
    maskname - mask name in the image (default: mask0)
    maskimage - output mask image name
    usemasked - if True use masked region as a valid region
    """
    ia = casac.image()
    ia.open(imagename)
    masks = ia.maskhandler('get')
    ia.close()

    inmaskname = ''
    if type(masks) != list:
        masks = [masks]
    for msk in masks:
        if maskname == msk:
            inmaskname = msk
            break
    if inmaskname == '':
        raise Exception, "mask %s does not exist. Available masks are: %s" % (
            maskname, masks)

    tb = casac.table()
    tb.open(imagename + '/' + maskname)
    dat0 = tb.getcol('PagedArray')
    tb.close()

    #os.system('cp -r %s %s' % (imagename, maskimage))
    shutil.copytree(imagename, maskimage)
    ia.open(maskimage)
    # to unset mask
    ia.maskhandler('set', [''])
    # make all valid
    if (usemasked):
        ia.set(1)
    else:
        ia.set(0)
    ia.close()
    #
    tb.open(maskimage, nomodify=False)
    imd = tb.getcol('map')
    # maybe shape check here
    #by default use True part of bool mask
    masked = 1
    if (usemasked): masked = 0
    imd[dat0] = masked
    tb.putcol('map', imd)
    tb.close()
Esempio n. 3
0
def cl2skycat(componentlist='', skycat=''):
    """
    Converts a componentlist dictionary or componnent list file on disk
    to  a skycatalog to overlay on image in the viewer
    
    """
    qa = casac.quanta()
    cl = casac.componentlist()
    tb = casac.table()
    if (type(componentlist) == str):
        cl.open(componentlist)
    elif (type(componentlist) == dict):
        cl.purge()
        cl.fromrecord(componentlist)
    if (cl.length() == 0):
        print "no components found"
        return

    des = {}
    des['Type'] = {'valueType': 'string'}
    des['Long'] = {'valueType': 'double'}
    des['Lat'] = {'valueType': 'double'}
    des['COMP_ID'] = {'valueType': 'string'}
    des['RA'] = {'valueType': 'string'}
    des['DEC'] = {'valueType': 'string'}
    des['FluxValue'] = {'valueType': 'double'}
    tb.create(tablename=skycat, tabledesc=des, nrow=cl.length())
    eltype = []
    nam = []
    RA = []
    DEC = []
    lati = np.zeros((cl.length(), ))
    longi = np.zeros((cl.length(), ))
    fluxval = np.zeros((cl.length(), ))
    for k in range(cl.length()):
        longi[k] = qa.convert(cl.getrefdir(k)['m0'], 'deg')['value']
        lati[k] = qa.convert(cl.getrefdir(k)['m1'], 'deg')['value']
        fluxval[k] = cl.getfluxvalue(k)[0]
        RA.append(qa.time(cl.getrefdir(k)['m0'], prec=10))
        DEC.append(qa.angle(cl.getrefdir(k)['m1'], prec=10))
        eltype.append(cl.getrefdir(k)['refer'])
        nam.append(str(k))
    tb.putcol('Type', eltype)
    tb.putcol('RA', RA)
    tb.putcol('DEC', DEC)
    tb.putcol('COMP_ID', nam)
    tb.putcol('Long', longi)
    tb.putcol('Lat', lati)
    tb.putcol('FluxValue', fluxval)
    tb.putcolkeyword(columnname='Long', keyword='UNIT', value='deg')
    tb.putcolkeyword(columnname='Lat', keyword='UNIT', value='deg')
    tb.putinfo({'type': 'Skycatalog'})
    tb.done()
Esempio n. 4
0
def get_polbasis(ms):

    # Open the MS

    tbHandle = casac.table()
    tbHandle.open(ms + '/FEED')

    # Get the POLARIZATION_TYPE column

    pt = tbHandle.getcol('POLARIZATION_TYPE')

    # Determine the polarization basis

    if pt[0, :][0] == 'R' or pt[1, :][0] == 'R':
        polBasis = 'CIRCULAR'
    else:
        polBasis = 'LINEAR'

    # Return the polarization basis

    return polBasis
Esempio n. 5
0
def get_polbasis(ms):

    # Open the MS

    tbHandle = casac.table()
    tbHandle.open(ms + "/FEED")

    # Get the POLARIZATION_TYPE column

    pt = tbHandle.getcol("POLARIZATION_TYPE")

    # Determine the polarization basis

    if pt[0, :][0] == "R" or pt[1, :][0] == "R":
        polBasis = "CIRCULAR"
    else:
        polBasis = "LINEAR"

        # Return the polarization basis

    return polBasis
Esempio n. 6
0
def addCorrectedData(msname):
    """ Add CORRECTED_DATA column to an MS.

    """
    tb = casac.table()

    # Open the MS
    tb.open(msname, nomodify=False)
    cnames = tb.colnames()
    # Get the description of the DATA column.
    try:
        cdesc = tb.getcoldesc('DATA')
    except:
        raise ValueError('Column DATA does not exist')
    hasTiled = False
    # Determine if the DATA storage specification is tiled.
    hasTiled = False
    try:
        dminfo = None
        for rec in tb.getdminfo("DATA"):
            if recs[rec]['COLUMNS'][0] == 'DATA':
                dminfo = recs[rec]
        if dminfo['TYPE'][:5] == 'Tiled':
            hasTiled = True
    except:
        hasTiled = False
    # Use TiledShapeStMan if needed.
    if not hasTiled:
        dminfo = {'TYPE': 'TiledShapeStMan', 'SPEC': {'DEFAULTTILESHAPE':[4,32,128]}}
    # Add the columns (if not existing). Use the description of the DATA column.
    if 'CORRECTED_DATA' in cnames:
        print("Column CORRECTED_DATA not added; it already exists")
    else:
        dminfo['NAME'] = 'correcteddata'
        cdesc['comment'] = 'The corrected data column'
        tb.addcols({'CORRECTED_DATA': cdesc}, dminfo)
    # Flush the table to make sure it is written.
    tb.flush()
Esempio n. 7
0
def get_scantimes(ms):

    # Open the MS

    tbHandle = casac.table()
    tbHandle.open(ms)

    # Get the time and scan columns

    times = tbHandle.getcol("TIME")
    scans = tbHandle.getcol("SCAN_NUMBER")

    # Assign each time to the proper scan

    d = dict()

    for r in range(tbHandle.nrows()):
        if not d.has_key(scans[r]):
            d[scans[r]] = list()
        d[scans[r]].append(times[r])

    tbHandle.close()
    del tbHandle

    # Get the minimum and maximum time for each scan

    scanTimes = dict()

    for k in d.keys():
        scanTimes[k] = dict()
        scanTimes[k]["min"] = numpy.min(d[k])
        scanTimes[k]["max"] = numpy.max(d[k])

        # Return the minimum and maximum time for each scan

    return scanTimes
Esempio n. 8
0
from casac import casac
import utils

tb = casac.table()


def formatTable(tname):
    tb.open(tname)
    times = [utils.render_time(t) for t in tb.getcol('TIME')]
    ants = tb.getcol('ANTENNA1')
    swids = tb.getcol('SPECTRAL_WINDOW_ID')
    flags = list(tb.getcol('FLAG').squeeze().transpose().all(axis=1))
    params = list(tb.getcol("FPARAM").squeeze())
    fs = "{:20} {:3} {:2} " + 6 * "{:>15.8g} "
    s = "\n".join(
        fs.format(*r[1:]) for r in zip(flags, times, ants, swids, *params)
        if not r[0])
    tb.close()
    return s
Esempio n. 9
0
# jagonzal: Import tasks but don't load task manager and dbus
os.environ['CASA_ENGINE']="YES"
from tasks import *

casalog = casac.logsink()
casalog.setglobal(True)
quanta = casac.quanta()
measures = casac.measures()
imager = casac.imager()
calibrater = casac.calibrater()
ms = casac.ms()
tableplot = casac.tableplot()
msplot = casac.msplot()
calplot = casac.calplot()
table = casac.table()
#flagger = casac.flagger()
agentflagger = casac.agentflagger()
image = casac.image()
imagepol = casac.imagepol()
simulator = casac.simulator()
componentlist = casac.componentlist()
coordsys = casac.coordsys()
regionmanager = casac.regionmanager()
spectralline = casac.spectralline()
utils = casac.utils()
deconvolver = casac.deconvolver()
vpmanager = casac.vpmanager()
vlafillertask = casac.vlafillertask()
atmosphere = casac.atmosphere()
cu = casac.utils()
Esempio n. 10
0
def convert_gaincurve(antab, gc, min_elevation=0.0, max_elevation=90.0):
    tb = casatools.table()

    outfp = tempfile.NamedTemporaryFile('w')

    t = time.strptime("2000y01d00h00m00s", "%Yy%jd%Hh%Mm%Ss")
    btime = time.mktime(t) + 40587.0 * 86400
    t = time.strptime("2100y01d00h00m00s", "%Yy%jd%Hh%Mm%Ss")
    etime = time.mktime(t) + 40587.0 * 86400

    keys = StringIO()
    fp = open(antab, 'r')
    for line in fp:
        if line.startswith('!'):
            continue
        keys.write(line)
        if line.strip().endswith('/'):
            keys.seek(0)
            gain = key.read_keyfile(keys)
            # Standard ANTAB
            if gain and gain[0] and gain[0][0][0] == 'GAIN':
                antenna = find_antenna(gain[0], keyin_keys)
                gain = dict(gain[0])
                try:
                    bfreq = gain['FREQ'][0] * 1e6
                except:
                    bfreq = 0
                    pass
                try:
                    efreq = gain['FREQ'][1] * 1e6
                except:
                    efreq = 1e12
                    pass
                gain_common(gain, antenna, "C", bfreq, efreq, btime, etime,
                            min_elevation, max_elevation, outfp)
            # VLBA gains file
            elif gain and gain[0] and gain[0][0][0] == 'BAND':
                antenna = gain[0][8][0]
                gain = dict(gain[0])
                timerange = gain['TIMERANG']
                btime = parse_timerange(timerange[0:])
                etime = parse_timerange(timerange[4:])
                band = gain['BAND']
                try:
                    freq = vlba_freqs[band]
                    bfreq = freq[0] * 1e9
                    efreq = freq[1] * 1e9
                except:
                    freq = gain['FREQ']
                    bfreq = freq - freq / 4
                    efreq = freq + freq / 4
                    pass
                gain_common(gain, antenna, band, bfreq, efreq, btime, etime,
                            min_elevation, max_elevation, outfp)
            elif gain and gain[0] and gain[0][0][0] == 'TSYS':
                skip_values(fp)
                pass
            keys = StringIO()
            continue
        continue

    outfp.flush()

    tb.fromascii(gc, asciifile=outfp.name, sep=' ',
                 columnnames=columnnames, datatypes=datatypes)

    outfp.close()
    fp.close()
    return
Esempio n. 11
0
def importasdm(
    asdm=None,
    vis=None,
    createmms=None,
    separationaxis=None,
    numsubms=None,    
    corr_mode=None,
    srt=None,
    time_sampling=None,
    ocorr_mode=None,
    compression=None,
    lazy=None,
    asis=None,
    wvr_corrected_data=None,
    scans=None,
    ignore_time=None,
    process_syspower=None,
    process_caldevice=None,
    process_pointing=None,
    process_flags=None,
    tbuff=None,
    applyflags=None,
    savecmds=None,
    outfile=None,
    flagbackup=None,
    verbose=None,
    overwrite=None,
    showversion=None,
    useversion=None,
    bdfflags=None,
    with_pointing_correction=None,
    remove_ref_undef=None,
    convert_ephem2geo=None,
    polyephem_tabtimestep=None
    ):
    """Convert an ALMA Science Data Model observation into a CASA visibility file (MS) or single-dish data format (Scantable).
           The conversion of the ALMA SDM archive format into a measurement set.  This version
           is under development and is geared to handling many spectral windows of different
           shapes.

           Keyword arguments:
           asdm -- Name of input ASDM file (directory)
               default: none; example: asdm='ExecBlock3'

       vis       -- Root ms or scantable name, note a prefix (.ms or .asap) is NOT appended to this name
           default: none
           
       createmms  -- Create a Multi-MS
           default: False
           
       corr_mode -- correlation mode to be considered on input. Could
            be one or more of the following, ao, co, ac, or all
           default: all

       srt       -- spectral resolution type. Could be one or more of
                    the following, fr, ca, bw, or all
           default: all

       time_sampling -- specifies the time sampling, INTEGRATION and/or
                            SUBINTEGRATION. could be one or more of the following
                            i, si, or all.
           default: all

       ocorr_mode    -- output data for correlation mode AUTO_ONLY 
                            (ao) or CROSS_ONLY (co) or CROSS_AND_AUTO (ca)
           default: ca

      compression  -- produces comrpressed columns in the resulting measurement set.
                 default: False

       lazy         -- Make the MS DATA column read the ASDM Binary data directly
                       (faster import, smaller MS)
                 default: False

       asis         --  creates verbatim copies of the ASDM tables in 
                        the output measurement set. The value given to
                    this option must be a list of table names separated
                    by space characters; the wildcard character '*' is 
                            allowed in table names.

       wvr_corrected_data -- specifies wich values are considered in the 
                      ASDM binary data to fill the DATA column in 
                      the MAIN table of the MS. Expected values for 
                      this option are 'no' for the uncorrected data 
                      (this is the default), 'yes' for the corrected
                      data and 'both' for corrected and uncorrected 
                      data. In the latter case, two measurement sets
                      are created, one containing the uncorrected 
                      data and the other one, whose name is suffixed
                      by '-wvr-corrected', containing the corrected 
                      data.

       scans --  processes only the scans specified in the option's value. This value is a semicolon 
                 separated list of scan specifications. A scan specification consists in an exec bock index 
                 followed by the character ':' followed by a comma separated list of scan indexes or scan 
                 index ranges. A scan index is relative to the exec block it belongs to. Scan indexes are 
                 1-based while exec blocks's are 0-based. "0:1" or "2:2~6" or "0:1,1:2~6,8;2:,3:24~30" "1,2" 
                 are valid values for the option. "3:" alone will be interpreted as 'all the scans of the 
                 exec block#3'. An scan index or a scan index range not preceded by an exec block index will
                 be interpreted as 'all the scans with such indexes in all the exec blocks'.  By default 
                 all the scans are considered.

       ignore_time -- All the rows of the tables Feed, History, Pointing, Source, SysCal, CalDevice, SysPower,
                      and Weather are processed independently of the time range of the selected exec block / scan.

       process_syspower -- The SysPower table is processed if and only if this parameter is set to True.
              default: True

       process_caldevice -- The CalDevice table is processed if and only if this parameter is set to True.
              default: True

       process_pointing -- The Pointing table is processed if and only if this parameter is set to True.
                       If the parameter is set to False the resulting MS will have an empty POINTING table.
              default: True

       process_flags -- Process the online flags and save them to the FLAG_CMD sub-table.
              default: True

            >>> process_flags expandable parameter
                 tbuff -- Time padding buffer (in seconds).
                    default: 0.0

                 applyflags -- Apply the online flags to the MS.
                    default: False

                 savecmds -- Save the online flags to an ASCII file.
                    default: False
                    
                 outfile -- Filename to save the online flags.
                    default: ''

       flagbackup -- Backup the FLAG column in the .flagversions.
              default: True

       verbose     -- produce log output as asdm2MS is being run.

       overwrite -- Over write an existing MS.

       showversion -- report the version of the asdm2MS being used.

       useversion -- Selects the version of asdm2MS to be used (presently only \'v3\' is available).
                     default: v3
                     
       bdfflags -- Set the MS FLAG column according to the ASDM _binary_ flags
                   default: false

       with_pointing_correction -- add (ASDM::Pointing::encoder - ASDM::Pointing::pointingDirection)
                 to the value to be written in MS::Pointing::direction 
                   default: false

       remove_ref_undef -- if set to True then apply fixspwbackport on the resulting MSes.

       convert_ephem2geo -- if True, convert any attached ephemerides to the GEO reference frame

       polyephem_tabtimestep -- Timestep (days) for the tabulation of polynomial ephemerides. A value <= 0 disables tabulation.
                   Presently, VLA data can contain polynomial ephemerides. ALMA data uses tabulated values.
                   default: 0.          

        """

    # Python script
    
    # make agentflagger tool local
    aflocal = casac.agentflagger()

    # make table tool local
    tblocal = casac.table()

    try:
        casalog.origin('importasdm')
        viso = ''
        visoc = ''  # for the wvr corrected version, if needed
        if len(vis) > 0:
            viso = vis
            tmps = vis.rstrip('.ms')
            if tmps == vis:
                visoc = vis + '-wvr-corrected'
            else:
                visoc = tmps + '-wvr-corrected.ms'
        else:
            viso = asdm.rstrip("/") + '.ms'
            visoc = asdm.rstrip("/") + '-wvr-corrected.ms'
            vis = asdm.rstrip("/")



        useversion = 'v3'
        theexecutable = 'asdm2MS'

        execute_string = theexecutable + ' --icm "' + corr_mode \
            + '" --isrt "' + srt + '" --its "' + time_sampling \
            + '" --ocm "' + ocorr_mode + '" --wvr-corrected-data "' \
            + wvr_corrected_data + '" --asis "' + asis \
            + '" --logfile "' + casalog.logfile() + '"'

        if len(scans) > 0:
            execute_string = execute_string + ' --scans ' + scans
        if ignore_time:
            execute_string = execute_string + ' --ignore-time'
        if useversion == 'v3':
            if not process_syspower:
                execute_string = execute_string + ' --no-syspower'
            if not process_caldevice:
                execute_string = execute_string + ' --no-caldevice'
            if not process_pointing:
                execute_string = execute_string + ' --no-pointing'

        if compression:
            execute_string = execute_string + ' --compression'
        elif lazy:
            execute_string = execute_string + ' --lazy'
            
        if verbose:
            execute_string = execute_string + ' --verbose'
#         if not overwrite and os.path.exists(viso):
#             raise Exception, \
#                 'You have specified an existing MS and have indicated you do not wish to overwrite it'

        # Compression
        if compression:
                   # viso = viso + '.compressed'
            viso = viso.rstrip('.ms') + '.compressed.ms'
            visoc = visoc.rstrip('.ms') + '.compressed.ms'

        vistoproc = [] # the output MSs to post-process
        if wvr_corrected_data == 'no' or wvr_corrected_data == 'both':
            vistoproc.append(viso)
        if (wvr_corrected_data == 'yes' or wvr_corrected_data == 'both') : 
            vistoproc.append(visoc)

        for ff in vistoproc:
            if not overwrite and os.path.exists(ff):
                raise Exception, \
                    'You have specified an existing MS and have indicated you do not wish to overwrite it: %s'%ff

        # If viso+".flagversions" then process differently depending on the value of overwrite..
        #
        if flagbackup:
            for myviso in vistoproc:
                dotFlagversion = myviso + '.flagversions'
                if os.path.exists(dotFlagversion):
                    if overwrite:
                        casalog.post("Found '" + dotFlagversion
                                     + "' . It'll be deleted before running the filler."
                                     )
                        os.system('rm -rf %s' % dotFlagversion)
                    else:
                        casalog.post("Found '%s' but can't overwrite it."
                                     % dotFlagversion)
                        raise Exception, "Found '%s' but can't overwrite it." \
                            % dotFlagversion
               
        # Make outfile always a list             
        if isinstance(outfile, str):
            if outfile == '': 
                outfile = []
            else:
                noutfile = [outfile]
                outfile = noutfile
            
        if savecmds:
            if len(outfile) == 0:
                # Create default names for the online flags
                for myviso in vistoproc:
                    outfile.append(myviso.replace('.ms','_cmd.txt'))
            elif len(outfile) != len(vistoproc):
                casalog.post('List of outfile names does not match list of MSs','WARN')
                casalog.post('Will save online flags to temporary filenames', 'WARN')
                outfile = []
                for myviso in vistoproc:
                    online_file = myviso.replace('.ms','_TEMP_cmd.txt')
                    outfile.append(online_file)
                                     
            if not overwrite:
                for of in outfile:
                    if os.path.exists(of):
                        raise Exception, "Cannot overwrite online flags file '%s'; overwrite is set to False."% of
                
            
        execute_string = execute_string + ' ' + asdm + ' ' + viso

        if showversion:
            casalog.post("You set option \'showversion\' to True. Will just output the version information and then terminate."
                         , 'WARN')
            execute_string = theexecutable + ' --revision'

        if with_pointing_correction:
            execute_string = execute_string + ' --with-pointing-correction'

        if (polyephem_tabtimestep!=None) and (type(polyephem_tabtimestep)==int or type(polyephem_tabtimestep)==float):
            if polyephem_tabtimestep>0:
                casalog.post('Will tabulate all attached polynomial ephemerides with a time step of '
                             +str(polyephem_tabtimestep)+' days.')
                if polyephem_tabtimestep>1.:
                    casalog.post('A tabulation timestep of <= 1 days is recommended.', 'WARN')
                execute_string = execute_string + ' --polyephem-tabtimestep '+str(polyephem_tabtimestep)

        casalog.post('Running ' + theexecutable
                     + ' standalone invoked as:')
        # print execute_string
        casalog.post(execute_string)
        exitcode = os.system(execute_string)
        if exitcode != 0:
            if not showversion:
                casalog.post(theexecutable
                             + ' terminated with exit code '
                             + str(exitcode), 'SEVERE')
                raise Exception, \
                    'ASDM conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'

        if showversion:
            return
        
        #
        # Possibly remove the name of the measurement set expected to contain the corrected data from the list of of produced measurement
        # sets if it appears the filler did not find any corrected data.
        #
        if not os.path.exists(visoc):
            vistoproc = [myviso for myviso in vistoproc if myviso != visoc]

        # CAS-7369. HISTORY should be written after createmms is tested
        #
        # Populate the HISTORY table of the MS with information about the context in which it's been created
        #
        try: 
            mslocal = mstool() 
            param_names = importasdm.func_code.co_varnames[:importasdm.func_code.co_argcount] 
            param_vals = [eval(p) for p in param_names]

            for myviso in vistoproc:
                write_history(mslocal, myviso, 'importasdm', param_names, 
                              param_vals, casalog) 

        except Exception, instance: 
            casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 
                         'WARN')
            return False 

        if mslocal:
            mslocal = None 
            
        # 
        # Do we apply fixspwbackport
        if remove_ref_undef :
            casalog.post('remove_ref_undef=True: fixspwbackport will be applied ...')
            
            for myviso in vistoproc:
                cmd = 'fixspwbackport ' + myviso
                casalog.post('Running fixspwbackport standalone invoked as:')
                casalog.post(cmd)
                cmdexitcode = os.system(cmd)

                if cmdexitcode != 0:
                    casalog.post(cmd
                                 + ' terminated with exit code '
                                 + str(cmdexitcode), 'SEVERE')
                    raise Exception, 'fixspwbackport error.'

        # Binary Flag processing
        if bdfflags:
            
            casalog.post('Parameter bdfflags==True: flags from the ASDM binary data will be used to set the MS flags ...')
            
            bdffexecutable = 'bdflags2MS '
            bdffexecstring_base = bdffexecutable + ' -f ALL' + ' --ocm "' + ocorr_mode \
            + '" --logfile "' + casalog.logfile() + '"'
 
            if len(scans) > 0:
                bdffexecstring_base = bdffexecstring_base + ' --scans ' + scans

            if lazy and not compression:
                bdffexecstring_base = bdffexecstring_base + ' --lazy=true'

            for myviso in vistoproc:
                if myviso.find("wvr-corrected") != -1:
                    options = " --wvr-corrected=True " 
                else:
                    options = " "

                bdffexecstring = bdffexecstring_base + options + asdm + ' ' + myviso

                casalog.post('Running '+bdffexecutable+' standalone invoked as:')
                casalog.post(bdffexecstring)

                bdffexitcode = os.system(bdffexecstring)
                if bdffexitcode != 0:
                    casalog.post(bdffexecutable
                                 + ' terminated with exit code '
                                 + str(bdffexitcode), 'SEVERE')
                    raise Exception, \
                          'ASDM binary flags conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'


        theephemfields = ce.findattachedephemfields(myviso,field='*')
        if len(theephemfields)>0: 
            # until asdm2MS does this internally: recalc the UVW coordinates for ephem fields
            imt = imtool()
            imt.open(myviso, usescratch=False)
            imt.calcuvw(theephemfields, refcode='J2000', reuse=False)
            imt.close()

        if convert_ephem2geo:
            for myviso in vistoproc:
                ce.convert2geo(myviso, '*') # convert any attached ephemerides to GEO
        
        if len(theephemfields)>0: 
            # also set the direction column in the SOURCE table
            tblocal.open(myviso+'/FIELD', nomodify=False)
            sourceids = tblocal.getcol('SOURCE_ID')
            ftimes = tblocal.getcol('TIME')
            ftimekw = tblocal.getcolkeywords('TIME')
            tmpa = tblocal.getcol('PHASE_DIR')
            origphasedir = tmpa

            affectedsids = []
            thesamplefields = []
            for fld in theephemfields: # determine all source ids used by the ephem fields
                if not (sourceids[fld] in affectedsids): # this source id wasn't handled yet
                    affectedsids.append(sourceids[fld])
                    thesamplefields.append(fld)
                    # need to temporarily change the offset (not all mosaics have an element at (0,0))
                    tmpa[0][0][fld]=0.
                    tmpa[1][0][fld]=0.
                #endif
            #endfor
            tblocal.putcol('PHASE_DIR', tmpa)
            tblocal.close()

            tblocal.open(myviso+'/SOURCE')
            sourceposref = tblocal.getcolkeywords('DIRECTION')['MEASINFO']['Ref']
            tblocal.close()

            directions = []
            msmdlocal = casac.msmetadata()
            msmdlocal.open(myviso)
            
            for fld in thesamplefields:
                thedirmeas = msmdlocal.phasecenter(fld)
                if thedirmeas['refer']!=sourceposref:
                    casalog.post('Ephemeris is in '+thedirmeas['refer']+' instead of '+sourceposref
                                 +' frame.\nEntry in SOURCE table will be converted to '+sourceposref, 'WARN')
                    melocal = metool()
                    melocal.doframe(thedirmeas)
                    thedirmeas = melocal.measure(thedirmeas, sourceposref)

                directions.append([thedirmeas['m0']['value'], thedirmeas['m1']['value']])
                thetime = me.epoch(v0=str(ftimes[fld])+'s', rf=ftimekw['MEASINFO']['Ref'])
                casalog.post("Will set SOURCE direction for SOURCE_ID "+str(sourceids[fld])
                             +" to ephemeris phase center for time "+str(thetime['m0']['value'])+" "+thetime['m0']['unit']+" "+thetime['refer']) 
            #endfor
            msmdlocal.close()
             
            # restore original PHASE_DIR
            tblocal.open(myviso+'/FIELD', nomodify=False)
            tblocal.putcol('PHASE_DIR', origphasedir)
            tblocal.close()

            # write source directions
            tblocal.open(myviso+'/SOURCE', nomodify=False)
            ssourceids = tblocal.getcol('SOURCE_ID')
            sdirs = tblocal.getcol('DIRECTION')
            for row in xrange(0,len(ssourceids)):
                for i in xrange(0,len(affectedsids)):
                    if ssourceids[row]==affectedsids[i]:
                        sdirs[0][row] = directions[i][0]
                        sdirs[1][row] = directions[i][1]
                        break
                #endfor
            #endfor
            tblocal.putcol('DIRECTION', sdirs) # write back corrected directions
            tblocal.close()
                
        #end if        

        ##############################################################################################3
        # CAS-7369 - Create an output Multi-MS (MMS)
        if createmms:
            # Get the default parameters of partition
            from tasks import partition
            fpars = partition.parameters
            for mypar in fpars.keys():
                fpars[mypar] = partition.itsdefault(mypar)
                
            # Call the cluster for each MS
            for myviso in vistoproc:
                casalog.origin('importasdm')
                
                # Move original MS to tempdir
                tempname = myviso+'.temp.ms'
                outputmms = myviso
                shutil.move(myviso, tempname)
                
                # Get the proper column
                datacolumn = 'DATA'
                dcols = ['DATA', 'FLOAT_DATA']
                for dc in dcols:
                    if len(th.getColDesc(tempname, dc)) > 0:
                        datacolumn = dc
                        break
                    
                fpars['datacolumn'] = datacolumn
                    
                casalog.post('Will create a Multi-MS for: '+myviso)
                
                fpars['vis'] =  tempname
                fpars['flagbackup'] =  False 
                fpars['outputvis'] = outputmms
                fpars['separationaxis'] = separationaxis
                fpars['numsubms'] = numsubms
                pdh = ParallelDataHelper('partition', fpars) 
            
                # Get a cluster
                pdh.setupCluster(thistask='partition')
                try:
                    pdh.go()
                    
                    # Remove original MS
                    shutil.rmtree(tempname)

                except Exception, instance:
                    # Restore MS in case of error in MMS creation
                    shutil.move(tempname, myviso)
                    casalog.post('%s'%instance,'ERROR')
                    return False
                
            casalog.origin('importasdm')
Esempio n. 12
0
def caltab_convert2(caltabold, ms, pType, caltabnew=''):

    # Check the inputs

    if not os.path.exists(caltabold):
        raise IOError('Invalid old-format caltable.')

    if not os.path.exists(ms):
        raise IOError('Invalid visibility file (MS).')

    pTypeTemp = pType.strip().lower()
    if (pTypeTemp != 'complex' and pTypeTemp != 'float'):
        raise Exception('Invalid parameter type ("complex" or "float").')

    if caltabnew == '': caltabnew = caltabold + '.new'
    if os.path.exists(caltabnew):
        raise IOError('New-format caltable already exists.')

    # Open the old-format caltable and get the number of rows

    tbOld = casac.table()
    tbOld.open(caltabold)
    nRow = tbOld.nrows()

    # Get spwid map from old CAL_DESC subtable
    tbCD = casac.table()
    tbCD.open(caltabold + '/CAL_DESC')
    spwmap = tbCD.getcol('SPECTRAL_WINDOW_ID')[0, :]
    tbCD.close()

    # Create the empty new-format caltable with the correct number of rows

    tbNew = casac.table()
    tbNew.create(caltabnew, desc_new(pTypeTemp))

    tbNew.addrows(nRow)

    # Transfer most column information from the old-format caltable to the
    # new-format caltable.  NB: The REF_ANT column in the old-format
    # caltable (analogous to the ANTENNA2 column in the new-format caltable)
    # is messed up, which means that I fill ANTENNA2 with -1.

    tbNew.putcol('ANTENNA1', tbOld.getcol('ANTENNA1'))
    tbNew.putcol('ANTENNA2', -1.0 * numpy.ones(nRow, dtype='int'))
    tbNew.putcol('FIELD_ID', tbOld.getcol('FIELD_ID'))
    tbNew.putcol('FLAG', tbOld.getcol('FLAG'))
    tbNew.putcol('INTERVAL', tbOld.getcol('INTERVAL'))
    tbNew.putcol('SNR', tbOld.getcol('SNR'))
    # Map CAL_DESC_ID to SPECTRAL_WINDOW_ID:
    tbNew.putcol('SPECTRAL_WINDOW_ID', spwmap[tbOld.getcol('CAL_DESC_ID')])
    tbNew.putcol('TIME', tbOld.getcol('TIME'))
    tbNew.putcol('WEIGHT', tbOld.getcol('FIT_WEIGHT'))

    # Transfer the parameter column from the old-format caltable to the
    # new-format caltable, taking the type into account.  No parameter
    # errors are present in the old-format caltable, so they are set to
    # 1.0 (+j0.0).

    param = tbOld.getcol('GAIN')
    if pTypeTemp == 'float':
        param = param.real
        tbNew.putcol('FPARAM', param)
    else:
        tbNew.putcol('CPARAM', param)

    tbNew.putcol('PARAMERR', -1.0 * numpy.ones(param.shape, dtype='float'))

    # Determine the scans and put them into the new-format caltable

    scanTimes = get_scantimes(ms)
    keys = scanTimes.keys()

    times = tbOld.getcol('TIME')
    scans = numpy.zeros(0, dtype='int')

    for t in times.tolist():

        flag = False

        for i in range(len(keys)):
            k = keys[i]
            if t >= scanTimes[k]['min'] and t < scanTimes[k]['max']:
                flag = True
                scans = numpy.append(scans, k)
                break
            if i == len(keys) - 1: break
            l = keys[i + 1]
            if t >= scanTimes[k]['max'] and t < scanTimes[l]['min']:
                flag = True
                scans = numpy.append(scans, l)
                break

        if not flag: scans = numpy.append(scans, -1)

    tbNew.putcol('SCAN_NUMBER', scans)

    # Copy the appropriate subcaltables from the MS to the new-format
    # caltable

    arg = 'cp -r ' + ms + '/ANTENNA ' + caltabnew
    os.system(arg)

    arg = 'cp -r ' + ms + '/FIELD ' + caltabnew
    os.system(arg)

    #	arg = 'cp -r ' + caltabold + '/CAL_HISTORY ' + caltabnew + '/HISTORY'
    #	os.system( arg )

    tbHis = casac.table()
    tbHis.open(ms + '/HISTORY')
    tbHis.copy(newtablename=caltabnew + '/HISTORY',
               deep=True,
               valuecopy=True,
               norows=True)
    tbHis.close()
    del tbHis

    arg = 'cp -r ' + ms + '/SPECTRAL_WINDOW ' + caltabnew \
     + '/SPECTRAL_WINDOW'
    os.system(arg)

    # Add the info and keywords to the main table of the new-format caltable
    tabinfo = tbOld.info()

    tbNew.putinfo(tabinfo)

    if (pTypeTemp == 'complex'):
        tbNew.putkeyword('ParType', 'Complex')
    if (pTypeTemp == 'float'):
        tbNew.putkeyword('ParType', 'Float')

    tbNew.putkeyword('MSName', ms)
    tbNew.putkeyword('VisCal', tabinfo['subType'])

    polBasis = get_polbasis(ms)
    tbNew.putkeyword('PolBasis', polBasis)

    tbNew.putkeyword('ANTENNA', 'Table: ' + caltabnew + '/ANTENNA')
    tbNew.putkeyword('FIELD', 'Table: ' + caltabnew + '/FIELD')
    tbNew.putkeyword('HISTORY', 'Table: ' + caltabnew + '/HISTORY')
    tbNew.putkeyword('SPECTRAL_WINDOW',
                     'Table: ' + caltabnew + '/SPECTRAL_WINDOW')

    # Add the column keywords to the main table of the new-format caltable

    colList = ['INTERVAL', 'TIME']

    for col in colList:
        colKeys = tbOld.getcolkeywords(col)
        if colKeys != {}: tbNew.putcolkeywords(col, colKeys)

    # Close the old- and new- format caltables

    tbOld.close()
    del tbOld

    tbNew.close()
    del tbNew

    # Get the channel ranges from the CAL_DESC subtable of the old-format
    # caltable

    tbDesc = casac.table()
    tbDesc.open(caltabold + '/CAL_DESC')

    nDesc = tbDesc.nrows()
    rDesc = range(nDesc)

    spwMap = tbDesc.getcol('SPECTRAL_WINDOW_ID')[0]

    chanMin = numpy.zeros(nDesc, dtype='int')
    chanMax = numpy.zeros(nDesc, dtype='int')

    for d in rDesc:
        chanRange = tbDesc.getcol('CHAN_RANGE', startrow=d, nrow=1)
        chanMin[d] = numpy.min(chanRange[0, :, :, :][0, :, 0])
        chanMax[d] = numpy.max(chanRange[1, :, :, :][0, :, 0])

    if numpy.all(chanMax - chanMin + 1 == 1):
        gain = True
    else:
        gain = False

    tbDesc.close()
    del tbDesc

    # Modify the columns of the new-format caltable according to the
    # channel ranges

    tbSPW = casac.table()
    tbSPW.open(caltabnew + '/SPECTRAL_WINDOW', nomodify=False)

    for d in rDesc:

        s = int(spwMap[d])

        nChan = int(chanMax[d] - chanMin[d] + 1)
        tbSPW.putcell('NUM_CHAN', s, nChan)

        chanFreq = tbSPW.getcell('CHAN_FREQ', s)
        if gain:
            chanFreq = numpy.median(chanFreq)
        else:
            chanFreq = chanFreq[chanMin[d]:chanMax[d] + 1]
        tbSPW.putcell('CHAN_FREQ', s, chanFreq)

        chanWidth = tbSPW.getcell('CHAN_WIDTH', s)
        if gain:
            chanWidth = numpy.sum(chanWidth)
        else:
            chanWidth = chanWidth[chanMin[d]:chanMax[d] + 1]
        tbSPW.putcell('CHAN_WIDTH', s, chanWidth)
        tbSPW.putcell('EFFECTIVE_BW', s, chanWidth)
        tbSPW.putcell('RESOLUTION', s, chanWidth)

        totalBandwidth = numpy.sum(chanWidth)
        tbSPW.putcell('TOTAL_BANDWIDTH', s, totalBandwidth)

    tbSPW.close()
    del tbSPW

    # Return True

    return True
Esempio n. 13
0
def field2skycat(msname='', skycat='', fieldpattern=''):
    """
    Converts field table in ms to skycatalog
    allow overlay on image in the viewer
    """
    enumTypes = [
        'J2000', 'JMEAN', 'JTRUE', 'APP', 'B1950', 'B1950_VLA', 'BMEAN',
        'BTRUE', 'GALACTIC', 'HADEC', 'AZEL', 'AZELSW', 'AZELGEO', 'AZELSWGEO',
        'JNAT', 'ECLIPTIC', 'MECLIPTIC', 'TECLIPTIC', 'SUPERGAL', 'ITRF',
        'TOPO', 'ICRS'
    ]

    qa = casac.quanta()
    tb = casac.table()
    tb.open(msname + '/FIELD')
    dir = tb.getcol('PHASE_DIR')
    nam = tb.getcol('NAME')
    nfield = tb.nrows()
    n = 0
    for k in range(nfield):
        if (re.match(fieldpattern, nam[k]) != None):
            n = n + 1

    eltype = []
    if (tb.getcolkeyword('PHASE_DIR', 'MEASINFO').has_key('VarRefCol')):
        typeid = tb.getcol(
            tb.getcolkeyword('PHASE_DIR', 'MEASINFO')['VarRefCol'])
        for k in range(nfield):
            if (re.match(fieldpattern, nam[k]) != None):
                eltype.append(enumTypes[typeid[k]])
    else:
        eltype = [tb.getcolkeyword('PHASE_DIR', 'MEASINFO')['Ref']] * nfield
    unitra = tb.getcolkeyword('PHASE_DIR', 'QuantumUnits')[0]
    unitdec = tb.getcolkeyword('PHASE_DIR', 'QuantumUnits')[1]
    tb.done()
    des = {}
    des['Type'] = {'valueType': 'string'}
    des['Long'] = {'valueType': 'double'}
    des['Lat'] = {'valueType': 'double'}
    des['FIELD_NAME'] = {'valueType': 'string'}
    des['FIELD_ID'] = {'valueType': 'string'}
    des['RA'] = {'valueType': 'string'}
    des['DEC'] = {'valueType': 'string'}

    tb.create(tablename=skycat, tabledesc=des, nrow=n)
    tb.putcol('Type', eltype)
    lati = np.zeros((n, ))
    longi = np.zeros((n, ))
    RA = []
    DEC = []
    fid = []
    fieldname = []
    n = 0
    for k in range(nfield):
        if (re.match(fieldpattern, nam[k]) != None):
            longi[n] = qa.convert(qa.quantity(dir[0, 0, k], unitra),
                                  'deg')['value']
            lati[n] = qa.convert(qa.quantity(dir[1, 0, k], unitdec),
                                 'deg')['value']
            RA.append(qa.time(qa.quantity(dir[0, 0, k], unitra), prec=10))
            DEC.append(qa.angle(qa.quantity(dir[1, 0, 0], unitdec), prec=10))
            fid.append(str(k))
            fieldname.append(nam[k])
            n = n + 1
    tb.putcol('RA', RA)
    tb.putcol('DEC', DEC)
    #    tb.putcol('FIELD_NAME', nam)
    tb.putcol('FIELD_NAME', fieldname)
    tb.putcol('FIELD_ID', fid)
    tb.putcol('Long', longi)
    tb.putcol('Lat', lati)
    tb.putcolkeyword(columnname='Long', keyword='UNIT', value='deg')
    tb.putcolkeyword(columnname='Lat', keyword='UNIT', value='deg')
    tb.putinfo({'type': 'Skycatalog'})
    tb.done()
Esempio n. 14
0
def compVarColTables(referencetab, testtab, varcol, tolerance=0.):
    '''Compare a variable column of two tables.
       referencetab  --> a reference table
       testtab       --> a table to verify
       varcol        --> the name of a variable column (str)
       Returns True or False.
    '''
    
    retval = True
    tb2 = casac.table()

    tb.open(referencetab)
    cnames = tb.colnames()

    tb2.open(testtab)
    col = varcol
    if tb.isvarcol(col) and tb2.isvarcol(col):
        try:
            # First check
            if tb.nrows() != tb2.nrows():
                print 'Length of %s differ from %s, %s!=%s'%(referencetab,testtab,len(rk),len(tk))
                retval = False
            else:
                for therow in xrange(tb.nrows()):
            
                    rdata = tb.getcell(col,therow)
                    tdata = tb2.getcell(col,therow)

#                    if not (rdata==tdata).all():
                    if not rdata.all()==tdata.all():
                        if (tolerance>0.):
                            differs=False
                            for j in range(0,len(rdata)):
###                                if (type(rdata[j])==float or type(rdata[j])==int):
                                if ((isinstance(rdata[j],float)) or (isinstance(rdata[j],int))):
                                    if (abs(rdata[j]-tdata[j]) > tolerance*abs(rdata[j]+tdata[j])):
#                                        print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab
#                                        print therow, j
#                                        print rdata[j]
#                                        print tdata[j]
                                        differs = True
###                                elif (type(rdata[j])==list or type(rdata[j])==np.ndarray):
                                elif (isinstance(rdata[j],list)) or (isinstance(rdata[j],np.ndarray)):
                                    for k in range(0,len(rdata[j])):
                                        if (abs(rdata[j][k]-tdata[j][k]) > tolerance*abs(rdata[j][k]+tdata[j][k])):
#                                            print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab
#                                            print therow, j, k
#                                            print rdata[j][k]
#                                            print tdata[j][k]
                                            differs = True
                                if differs:
                                    print 'ERROR: Column %s of %s and %s do not agree within tolerance %s'%(col,referencetab, testtab, tolerance)
                                    retval = False
                                    break
                        else:
                            print 'ERROR: Column %s of %s and %s do not agree.'%(col,referencetab, testtab)
                            print 'ERROR: First row to differ is row=%s'%therow
                            retval = False
                            break
        finally:
            tb.close()
            tb2.close()
    
    else:
        print 'Columns are not varcolumns.'
        retval = False

    if retval:
        print 'Column %s of %s and %s agree'%(col,referencetab, testtab)
        
    return retval
Esempio n. 15
0
def compTables(referencetab, testtab, excludecols, tolerance=0.001, mode="percentage", startrow = 0, nrow = -1, rowincr = 1):

    """
    compTables - compare two CASA tables
    
       referencetab - the table which is assumed to be correct

       testtab - the table which is to be compared to referencetab

       excludecols - list of column names which are to be ignored

       tolerance - permitted fractional difference (default 0.001 = 0.1 percent)

       mode - comparison is made as "percentage", "absolute", "phaseabsdeg" (for complex numbers = difference of the phases in degrees)  
    """

    rval = True

    tb2 = casac.table()

    tb.open(referencetab)
    cnames = tb.colnames()

    tb2.open(testtab)

    try:
        for c in cnames:
            if c in excludecols:
                continue
            
            print "\nTesting column " + c 
            
            a = 0
            try:
                a = tb.getcol(c,startrow=startrow,nrow=nrow,rowincr=rowincr)
            except:
                rval = False
                print 'Error accessing column ', c, ' in table ', referencetab
                print sys.exc_info()[0]
                break

            b = 0
            try:
                b = tb2.getcol(c,startrow=startrow,nrow=nrow,rowincr=rowincr)
            except:
                rval = False
                print 'Error accessing column ', c, ' in table ', testtab
                print sys.exc_info()[0]
                break

            if not (len(a)==len(b)):
                print 'Column ',c,' has different length in tables ', referencetab, ' and ', testtab
                print a
                print b
                rval = False
                break
            else:
                differs = False
                if not (a==b).all():
                    for i in range(0,len(a)):
                        if (isinstance(a[i],float)):
                            if ((mode=="percentage") and (abs(a[i]-b[i]) > tolerance*abs(a[i]))) or ((mode=="absolute") and (abs(a[i]-b[i]) > tolerance)):
                                print "Column " + c + " differs"
                                print "Row=" + str(i)
                                print "Reference file value: " + str(a[i])
                                print "Input file value: " + str(b[i])
                                if (mode=="percentage"):
                                    print "Tolerance is {0}%; observed difference was {1} %".format (tolerance * 100, 100*abs(a[i]-b[i])/abs(a[i]))
                                else:
                                    print "Absolute tolerance is {0}; observed difference: {1}".format (tolerance, (abs(a[i]-b[i])))
                                differs = True
                                rval = False
                                break
                        elif (isinstance(a[i],int) or isinstance(a[i],np.int32)):
                            if (abs(a[i]-b[i]) > 0):
                                print "Column " + c + " differs"
                                print "Row=" + str(i)
                                print "Reference file value: " + str(a[i])
                                print "Input file value: " + str(b[i])
                                if (mode=="percentage"):
                                    print "tolerance in % should be " + str(100*abs(a[i]-b[i])/abs(a[i]))
                                else:
                                    print "absolute tolerance should be " + str(abs(a[i]-b[i]))
                                differs = True
                                rval = False
                                break
                        elif (isinstance(a[i],str) or isinstance(a[i],np.bool_)):
                            if not (a[i]==b[i]):
                                print "Column " + c + " differs"
                                print "Row=" + str(i)
                                print "Reference file value: " + str(a[i])
                                print "Input file value: " + str(b[i])
                                if (mode=="percentage"):   
                                    print "tolerance in % should be " + str(100*abs(a[i]-b[i])/abs(a[i]))
                                else:
                                    print "absolute tolerance should be " + str(abs(a[i]-b[i]))
                                differs = True
                                rval = False
                                break
                        elif (isinstance(a[i],list)) or (isinstance(a[i],np.ndarray)):
                            for j in range(0,len(a[i])):
                                if differs: break
                                if ((isinstance(a[i][j],float)) or (isinstance(a[i][j],int))):
                                    if ((mode=="percentage") and (abs(a[i][j]-b[i][j]) > tolerance*abs(a[i][j]))) or ((mode=="absolute") and (abs(a[i][j]-b[i][j]) > tolerance)):
                                        print "Column " + c + " differs"
                                        print "(Row,Element)=(" + str(j) + "," + str(i) + ")"
                                        print "Reference file value: " + str(a[i][j])
                                        print "Input file value: " + str(b[i][j])
                                        if (mode=="percentage"):
                                            print "Tolerance in % should be " + str(100*abs(a[i][j]-b[i][j])/abs(a[i][j]))
                                        else:
                                            print "Absolute tolerance should be " + str(abs(a[i][j]-b[i][j]))
                                        differs = True
                                        rval = False
                                        break
                                elif (isinstance(a[i][j],list)) or (isinstance(a[i][j],np.ndarray)):
                                    for k in range(0,len(a[i][j])):
                                        if differs: break
                                        if ( ((mode=="percentage") and (abs(a[i][j][k]-b[i][j][k]) > tolerance*abs(a[i][j][k]))) \
                                                 or ((mode=="absolute") and (abs(a[i][j][k]-b[i][j][k]) > tolerance)) \
                                                 or ((mode=="phaseabsdeg") and (phasediffabsdeg(a[i][j][k],b[i][j][k])>tolerance)) \
                                                 ):
                                            print "Column " + c + " differs"
                                            print "(Row,Channel,Corr)=(" + str(k) + "," + str(j) + "," + str(i) + ")"
                                            print "Reference file value: " + str(a[i][j][k])
                                            print "Input file value: " + str(b[i][j][k])
                                            if (mode=="percentage"):
                                                print "Tolerance in % should be " + str(100*abs(a[i][j][k]-b[i][j][k])/abs(a[i][j][k]))
                                            elif (mode=="absolute"):
                                                print "Absolute tolerance should be " + str(abs(a[i][j][k]-b[i][j][k]))                     
                                            elif (mode=="phaseabsdeg"):
                                                print "Phase tolerance in degrees should be " + str(phasediffabsdeg(a[i][j][k],b[i][j][k]))
                                            else:
                                                print "Unknown comparison mode: ",mode
                                            differs = True
                                            rval = False
                                            break                                          
                                            
                        else:
                            print "Unknown data type: ",type(a[i])
                            differs = True
                            rval = False
                            break
                
                if not differs: print "Column " + c + " PASSED" 
    finally:
        tb.close()
        tb2.close()

    return rval
Esempio n. 16
0
def compVarColTables(referencetab, testtab, varcol, tolerance=0.):
    '''Compare a variable column of two tables.
       referencetab  --> a reference table
       testtab       --> a table to verify
       varcol        --> the name of a variable column (str)
       Returns True or False.
    '''

    retval = True
    tb2 = casac.table()

    tb.open(referencetab)
    cnames = tb.colnames()

    tb2.open(testtab)
    col = varcol
    if tb.isvarcol(col) and tb2.isvarcol(col):
        try:
            # First check
            if tb.nrows() != tb2.nrows():
                print 'Length of %s differ from %s, %s!=%s' % (
                    referencetab, testtab, len(rk), len(tk))
                retval = False
            else:
                for therow in xrange(tb.nrows()):

                    rdata = tb.getcell(col, therow)
                    tdata = tb2.getcell(col, therow)

                    #                    if not (rdata==tdata).all():
                    if not rdata.all() == tdata.all():
                        if (tolerance > 0.):
                            differs = False
                            for j in range(0, len(rdata)):
                                ###                                if (type(rdata[j])==float or type(rdata[j])==int):
                                if ((isinstance(rdata[j], float))
                                        or (isinstance(rdata[j], int))):
                                    if (abs(rdata[j] - tdata[j]) > tolerance *
                                            abs(rdata[j] + tdata[j])):
                                        #                                        print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab
                                        #                                        print therow, j
                                        #                                        print rdata[j]
                                        #                                        print tdata[j]
                                        differs = True


###                                elif (type(rdata[j])==list or type(rdata[j])==np.ndarray):
                                elif (isinstance(rdata[j],
                                                 list)) or (isinstance(
                                                     rdata[j], np.ndarray)):
                                    for k in range(0, len(rdata[j])):
                                        if (abs(rdata[j][k] - tdata[j][k]) >
                                                tolerance * abs(rdata[j][k] +
                                                                tdata[j][k])):
                                            #                                            print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab
                                            #                                            print therow, j, k
                                            #                                            print rdata[j][k]
                                            #                                            print tdata[j][k]
                                            differs = True
                                if differs:
                                    print 'ERROR: Column %s of %s and %s do not agree within tolerance %s' % (
                                        col, referencetab, testtab, tolerance)
                                    retval = False
                                    break
                        else:
                            print 'ERROR: Column %s of %s and %s do not agree.' % (
                                col, referencetab, testtab)
                            print 'ERROR: First row to differ is row=%s' % therow
                            retval = False
                            break
        finally:
            tb.close()
            tb2.close()

    else:
        print 'Columns are not varcolumns.'
        retval = False

    if retval:
        print 'Column %s of %s and %s agree' % (col, referencetab, testtab)

    return retval
Esempio n. 17
0
def importasdm(asdm=None,
               vis=None,
               createmms=None,
               separationaxis=None,
               numsubms=None,
               corr_mode=None,
               srt=None,
               time_sampling=None,
               ocorr_mode=None,
               compression=None,
               lazy=None,
               asis=None,
               wvr_corrected_data=None,
               scans=None,
               ignore_time=None,
               process_syspower=None,
               process_caldevice=None,
               process_pointing=None,
               process_flags=None,
               tbuff=None,
               applyflags=None,
               savecmds=None,
               outfile=None,
               flagbackup=None,
               verbose=None,
               overwrite=None,
               showversion=None,
               useversion=None,
               bdfflags=None,
               with_pointing_correction=None,
               remove_ref_undef=None,
               convert_ephem2geo=None,
               polyephem_tabtimestep=None):
    """Convert an ALMA Science Data Model observation into a CASA visibility file (MS) or single-dish data format (Scantable).
           The conversion of the ALMA SDM archive format into a measurement set.  This version
           is under development and is geared to handling many spectral windows of different
           shapes.

           Keyword arguments:
           asdm -- Name of input ASDM file (directory)
               default: none; example: asdm='ExecBlock3'

       vis       -- Root ms or scantable name, note a prefix (.ms or .asap) is NOT appended to this name
           default: none
           
       createmms  -- Create a Multi-MS
           default: False
           
       corr_mode -- correlation mode to be considered on input. Could
            be one or more of the following, ao, co, ac, or all
           default: all

       srt       -- spectral resolution type. Could be one or more of
                    the following, fr, ca, bw, or all
           default: all

       time_sampling -- specifies the time sampling, INTEGRATION and/or
                            SUBINTEGRATION. could be one or more of the following
                            i, si, or all.
           default: all

       ocorr_mode    -- output data for correlation mode AUTO_ONLY 
                            (ao) or CROSS_ONLY (co) or CROSS_AND_AUTO (ca)
           default: ca

      compression  -- produces comrpressed columns in the resulting measurement set.
                 default: False

       lazy         -- Make the MS DATA column read the ASDM Binary data directly
                       (faster import, smaller MS)
                 default: False

       asis         --  creates verbatim copies of the ASDM tables in 
                        the output measurement set. The value given to
                    this option must be a list of table names separated
                    by space characters; the wildcard character '*' is 
                            allowed in table names.

       wvr_corrected_data -- specifies wich values are considered in the 
                      ASDM binary data to fill the DATA column in 
                      the MAIN table of the MS. Expected values for 
                      this option are 'no' for the uncorrected data 
                      (this is the default), 'yes' for the corrected
                      data and 'both' for corrected and uncorrected 
                      data. In the latter case, two measurement sets
                      are created, one containing the uncorrected 
                      data and the other one, whose name is suffixed
                      by '-wvr-corrected', containing the corrected 
                      data.

       scans --  processes only the scans specified in the option's value. This value is a semicolon 
                 separated list of scan specifications. A scan specification consists in an exec bock index 
                 followed by the character ':' followed by a comma separated list of scan indexes or scan 
                 index ranges. A scan index is relative to the exec block it belongs to. Scan indexes are 
                 1-based while exec blocks's are 0-based. "0:1" or "2:2~6" or "0:1,1:2~6,8;2:,3:24~30" "1,2" 
                 are valid values for the option. "3:" alone will be interpreted as 'all the scans of the 
                 exec block#3'. An scan index or a scan index range not preceded by an exec block index will
                 be interpreted as 'all the scans with such indexes in all the exec blocks'.  By default 
                 all the scans are considered.

       ignore_time -- All the rows of the tables Feed, History, Pointing, Source, SysCal, CalDevice, SysPower,
                      and Weather are processed independently of the time range of the selected exec block / scan.

       process_syspower -- The SysPower table is processed if and only if this parameter is set to True.
              default: True

       process_caldevice -- The CalDevice table is processed if and only if this parameter is set to True.
              default: True

       process_pointing -- The Pointing table is processed if and only if this parameter is set to True.
                       If the parameter is set to False the resulting MS will have an empty POINTING table.
              default: True

       process_flags -- Process the online flags and save them to the FLAG_CMD sub-table.
              default: True

            &gt;&gt;&gt; process_flags expandable parameter
                 tbuff -- Time padding buffer (in seconds).
                    default: 0.0

                 applyflags -- Apply the online flags to the MS.
                    default: False

                 savecmds -- Save the online flags to an ASCII file.
                    default: False
                    
                 outfile -- Filename to save the online flags.
                    default: ''

       flagbackup -- Backup the FLAG column in the .flagversions.
              default: True

       verbose     -- produce log output as asdm2MS is being run.

       overwrite -- Over write an existing MS.

       showversion -- report the version of the asdm2MS being used.

       useversion -- Selects the version of asdm2MS to be used (presently only \'v3\' is available).
                     default: v3
                     
       bdfflags -- Set the MS FLAG column according to the ASDM _binary_ flags
                   default: false

       with_pointing_correction -- add (ASDM::Pointing::encoder - ASDM::Pointing::pointingDirection)
                 to the value to be written in MS::Pointing::direction 
                   default: false

       remove_ref_undef -- if set to True then apply fixspwbackport on the resulting MSes.

       convert_ephem2geo -- if True, convert any attached ephemerides to the GEO reference frame

       polyephem_tabtimestep -- Timestep (days) for the tabulation of polynomial ephemerides. A value <= 0 disables tabulation.
                   Presently, VLA data can contain polynomial ephemerides. ALMA data uses tabulated values.
                   default: 0.          

        """

    # Python script

    # make agentflagger tool local
    aflocal = casac.agentflagger()

    # make table tool local
    tblocal = casac.table()

    try:
        casalog.origin('importasdm')
        viso = ''
        visoc = ''  # for the wvr corrected version, if needed
        if len(vis) > 0:
            viso = vis
            tmps = vis.rstrip('.ms')
            if tmps == vis:
                visoc = vis + '-wvr-corrected'
            else:
                visoc = tmps + '-wvr-corrected.ms'
        else:
            viso = asdm.rstrip("/") + '.ms'
            visoc = asdm.rstrip("/") + '-wvr-corrected.ms'
            vis = asdm.rstrip("/")

        useversion = 'v3'
        theexecutable = 'asdm2MS'

        execute_string = theexecutable + ' --icm "' + corr_mode \
            + '" --isrt "' + srt + '" --its "' + time_sampling \
            + '" --ocm "' + ocorr_mode + '" --wvr-corrected-data "' \
            + wvr_corrected_data + '" --asis "' + asis \
            + '" --logfile "' + casalog.logfile() + '"'

        if len(scans) > 0:
            execute_string = execute_string + ' --scans ' + scans
        if ignore_time:
            execute_string = execute_string + ' --ignore-time'
        if useversion == 'v3':
            if not process_syspower:
                execute_string = execute_string + ' --no-syspower'
            if not process_caldevice:
                execute_string = execute_string + ' --no-caldevice'
            if not process_pointing:
                execute_string = execute_string + ' --no-pointing'

        if compression:
            execute_string = execute_string + ' --compression'
        elif lazy:
            execute_string = execute_string + ' --lazy'

        if verbose:
            execute_string = execute_string + ' --verbose'
#         if not overwrite and os.path.exists(viso):
#             raise Exception, \
#                 'You have specified an existing MS and have indicated you do not wish to overwrite it'

# Compression
        if compression:
            # viso = viso + '.compressed'
            viso = viso.rstrip('.ms') + '.compressed.ms'
            visoc = visoc.rstrip('.ms') + '.compressed.ms'

        vistoproc = []  # the output MSs to post-process
        if wvr_corrected_data == 'no' or wvr_corrected_data == 'both':
            vistoproc.append(viso)
        if (wvr_corrected_data == 'yes' or wvr_corrected_data == 'both'):
            vistoproc.append(visoc)

        for ff in vistoproc:
            if not overwrite and os.path.exists(ff):
                raise Exception, \
                    'You have specified an existing MS and have indicated you do not wish to overwrite it: %s'%ff

        # If viso+".flagversions" then process differently depending on the value of overwrite..
        #
        if flagbackup:
            for myviso in vistoproc:
                dotFlagversion = myviso + '.flagversions'
                if os.path.exists(dotFlagversion):
                    if overwrite:
                        casalog.post(
                            "Found '" + dotFlagversion +
                            "' . It'll be deleted before running the filler.")
                        os.system('rm -rf %s' % dotFlagversion)
                    else:
                        casalog.post("Found '%s' but can't overwrite it." %
                                     dotFlagversion)
                        raise Exception, "Found '%s' but can't overwrite it." \
                            % dotFlagversion

        # Make outfile always a list
        if isinstance(outfile, str):
            if outfile == '':
                outfile = []
            else:
                noutfile = [outfile]
                outfile = noutfile

        if savecmds:
            if len(outfile) == 0:
                # Create default names for the online flags
                for myviso in vistoproc:
                    outfile.append(myviso.replace('.ms', '_cmd.txt'))
            elif len(outfile) != len(vistoproc):
                casalog.post(
                    'List of outfile names does not match list of MSs', 'WARN')
                casalog.post('Will save online flags to temporary filenames',
                             'WARN')
                outfile = []
                for myviso in vistoproc:
                    online_file = myviso.replace('.ms', '_TEMP_cmd.txt')
                    outfile.append(online_file)

            if not overwrite:
                for of in outfile:
                    if os.path.exists(of):
                        raise Exception, "Cannot overwrite online flags file '%s'; overwrite is set to False." % of

        execute_string = execute_string + ' ' + asdm + ' ' + viso

        if showversion:
            casalog.post(
                "You set option \'showversion\' to True. Will just output the version information and then terminate.",
                'WARN')
            execute_string = theexecutable + ' --revision'

        if with_pointing_correction:
            execute_string = execute_string + ' --with-pointing-correction'

        if (polyephem_tabtimestep !=
                None) and (type(polyephem_tabtimestep) == int
                           or type(polyephem_tabtimestep) == float):
            if polyephem_tabtimestep > 0:
                casalog.post(
                    'Will tabulate all attached polynomial ephemerides with a time step of '
                    + str(polyephem_tabtimestep) + ' days.')
                if polyephem_tabtimestep > 1.:
                    casalog.post(
                        'A tabulation timestep of <= 1 days is recommended.',
                        'WARN')
                execute_string = execute_string + ' --polyephem-tabtimestep ' + str(
                    polyephem_tabtimestep)

        casalog.post('Running ' + theexecutable + ' standalone invoked as:')
        # print execute_string
        casalog.post(execute_string)
        exitcode = os.system(execute_string)
        if exitcode != 0:
            if not showversion:
                casalog.post(
                    theexecutable + ' terminated with exit code ' +
                    str(exitcode), 'SEVERE')
                raise Exception, \
                    'ASDM conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'

        if showversion:
            return

        #
        # Possibly remove the name of the measurement set expected to contain the corrected data from the list of of produced measurement
        # sets if it appears the filler did not find any corrected data.
        #
        if not os.path.exists(visoc):
            vistoproc = [myviso for myviso in vistoproc if myviso != visoc]

        # CAS-7369. HISTORY should be written after createmms is tested
        #
        # Populate the HISTORY table of the MS with information about the context in which it's been created
        #
        try:
            mslocal = mstool()
            param_names = importasdm.func_code.co_varnames[:importasdm.
                                                           func_code.
                                                           co_argcount]
            param_vals = [eval(p) for p in param_names]

            for myviso in vistoproc:
                write_history(mslocal, myviso, 'importasdm', param_names,
                              param_vals, casalog)

        except Exception, instance:
            casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
                         'WARN')
            return False

        if mslocal:
            mslocal = None

        #
        # Do we apply fixspwbackport
        if remove_ref_undef:
            casalog.post(
                'remove_ref_undef=True: fixspwbackport will be applied ...')

            for myviso in vistoproc:
                cmd = 'fixspwbackport ' + myviso
                casalog.post('Running fixspwbackport standalone invoked as:')
                casalog.post(cmd)
                cmdexitcode = os.system(cmd)

                if cmdexitcode != 0:
                    casalog.post(
                        cmd + ' terminated with exit code ' + str(cmdexitcode),
                        'SEVERE')
                    raise Exception, 'fixspwbackport error.'

        # Binary Flag processing
        if bdfflags:

            casalog.post(
                'Parameter bdfflags==True: flags from the ASDM binary data will be used to set the MS flags ...'
            )

            bdffexecutable = 'bdflags2MS '
            bdffexecstring_base = bdffexecutable + ' -f ALL' + ' --ocm "' + ocorr_mode \
            + '" --logfile "' + casalog.logfile() + '"'

            if len(scans) > 0:
                bdffexecstring_base = bdffexecstring_base + ' --scans ' + scans

            if lazy and not compression:
                bdffexecstring_base = bdffexecstring_base + ' --lazy=true'

            for myviso in vistoproc:
                if myviso.find("wvr-corrected") != -1:
                    options = " --wvr-corrected=True "
                else:
                    options = " "

                bdffexecstring = bdffexecstring_base + options + asdm + ' ' + myviso

                casalog.post('Running ' + bdffexecutable +
                             ' standalone invoked as:')
                casalog.post(bdffexecstring)

                bdffexitcode = os.system(bdffexecstring)
                if bdffexitcode != 0:
                    casalog.post(
                        bdffexecutable + ' terminated with exit code ' +
                        str(bdffexitcode), 'SEVERE')
                    raise Exception, \
                          'ASDM binary flags conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'

        theephemfields = ce.findattachedephemfields(myviso, field='*')
        if len(theephemfields) > 0:
            # until asdm2MS does this internally: recalc the UVW coordinates for ephem fields
            imt = imtool()
            imt.open(myviso, usescratch=False)
            imt.calcuvw(theephemfields, refcode='J2000', reuse=False)
            imt.close()

        if convert_ephem2geo:
            for myviso in vistoproc:
                ce.convert2geo(myviso,
                               '*')  # convert any attached ephemerides to GEO

        if len(theephemfields) > 0:
            # also set the direction column in the SOURCE table
            tblocal.open(myviso + '/FIELD', nomodify=False)
            sourceids = tblocal.getcol('SOURCE_ID')
            ftimes = tblocal.getcol('TIME')
            ftimekw = tblocal.getcolkeywords('TIME')
            tmpa = tblocal.getcol('PHASE_DIR')
            origphasedir = tmpa

            affectedsids = []
            thesamplefields = []
            for fld in theephemfields:  # determine all source ids used by the ephem fields
                if not (sourceids[fld]
                        in affectedsids):  # this source id wasn't handled yet
                    affectedsids.append(sourceids[fld])
                    thesamplefields.append(fld)
                    # need to temporarily change the offset (not all mosaics have an element at (0,0))
                    tmpa[0][0][fld] = 0.
                    tmpa[1][0][fld] = 0.
                #endif
            #endfor
            tblocal.putcol('PHASE_DIR', tmpa)
            tblocal.close()

            tblocal.open(myviso + '/SOURCE')
            sourceposref = tblocal.getcolkeywords(
                'DIRECTION')['MEASINFO']['Ref']
            tblocal.close()

            directions = []
            msmdlocal = casac.msmetadata()
            msmdlocal.open(myviso)

            for fld in thesamplefields:
                thedirmeas = msmdlocal.phasecenter(fld)
                if thedirmeas['refer'] != sourceposref:
                    casalog.post(
                        'Ephemeris is in ' + thedirmeas['refer'] +
                        ' instead of ' + sourceposref +
                        ' frame.\nEntry in SOURCE table will be converted to '
                        + sourceposref, 'WARN')
                    melocal = metool()
                    melocal.doframe(thedirmeas)
                    thedirmeas = melocal.measure(thedirmeas, sourceposref)

                directions.append(
                    [thedirmeas['m0']['value'], thedirmeas['m1']['value']])
                thetime = me.epoch(v0=str(ftimes[fld]) + 's',
                                   rf=ftimekw['MEASINFO']['Ref'])
                casalog.post("Will set SOURCE direction for SOURCE_ID " +
                             str(sourceids[fld]) +
                             " to ephemeris phase center for time " +
                             str(thetime['m0']['value']) + " " +
                             thetime['m0']['unit'] + " " + thetime['refer'])
            #endfor
            msmdlocal.close()

            # restore original PHASE_DIR
            tblocal.open(myviso + '/FIELD', nomodify=False)
            tblocal.putcol('PHASE_DIR', origphasedir)
            tblocal.close()

            # write source directions
            tblocal.open(myviso + '/SOURCE', nomodify=False)
            ssourceids = tblocal.getcol('SOURCE_ID')
            sdirs = tblocal.getcol('DIRECTION')
            for row in xrange(0, len(ssourceids)):
                for i in xrange(0, len(affectedsids)):
                    if ssourceids[row] == affectedsids[i]:
                        sdirs[0][row] = directions[i][0]
                        sdirs[1][row] = directions[i][1]
                        break
                #endfor
            #endfor
            tblocal.putcol('DIRECTION',
                           sdirs)  # write back corrected directions
            tblocal.close()

        #end if

        ##############################################################################################3
        # CAS-7369 - Create an output Multi-MS (MMS)
        if createmms:
            # Get the default parameters of partition
            from tasks import partition
            fpars = partition.parameters
            for mypar in fpars.keys():
                fpars[mypar] = partition.itsdefault(mypar)

            # Call the cluster for each MS
            for myviso in vistoproc:
                casalog.origin('importasdm')

                # Move original MS to tempdir
                tempname = myviso + '.temp.ms'
                outputmms = myviso
                shutil.move(myviso, tempname)

                # Get the proper column
                datacolumn = 'DATA'
                dcols = ['DATA', 'FLOAT_DATA']
                for dc in dcols:
                    if len(th.getColDesc(tempname, dc)) > 0:
                        datacolumn = dc
                        break

                fpars['datacolumn'] = datacolumn

                casalog.post('Will create a Multi-MS for: ' + myviso)

                fpars['vis'] = tempname
                fpars['flagbackup'] = False
                fpars['outputvis'] = outputmms
                fpars['separationaxis'] = separationaxis
                fpars['numsubms'] = numsubms
                pdh = ParallelDataHelper('partition', fpars)

                # Get a cluster
                pdh.setupCluster(thistask='partition')
                try:
                    pdh.go()

                    # Remove original MS
                    shutil.rmtree(tempname)

                except Exception, instance:
                    # Restore MS in case of error in MMS creation
                    shutil.move(tempname, myviso)
                    casalog.post('%s' % instance, 'ERROR')
                    return False

            casalog.origin('importasdm')
Esempio n. 18
0
def caltab_convert2(caltabold, ms, pType, caltabnew=""):

    # Check the inputs

    if not os.path.exists(caltabold):
        raise IOError("Invalid old-format caltable.")

    if not os.path.exists(ms):
        raise IOError("Invalid visibility file (MS).")

    pTypeTemp = pType.strip().lower()
    if pTypeTemp != "complex" and pTypeTemp != "float":
        raise Exception('Invalid parameter type ("complex" or "float").')

    if caltabnew == "":
        caltabnew = caltabold + ".new"
    if os.path.exists(caltabnew):
        raise IOError("New-format caltable already exists.")

        # Open the old-format caltable and get the number of rows

    tbOld = casac.table()
    tbOld.open(caltabold)
    nRow = tbOld.nrows()

    # Get spwid map from old CAL_DESC subtable
    tbCD = casac.table()
    tbCD.open(caltabold + "/CAL_DESC")
    spwmap = tbCD.getcol("SPECTRAL_WINDOW_ID")[0, :]
    tbCD.close()

    # Create the empty new-format caltable with the correct number of rows

    tbNew = casac.table()
    tbNew.create(caltabnew, desc_new(pTypeTemp))

    tbNew.addrows(nRow)

    # Transfer most column information from the old-format caltable to the
    # new-format caltable.  NB: The REF_ANT column in the old-format
    # caltable (analogous to the ANTENNA2 column in the new-format caltable)
    # is messed up, which means that I fill ANTENNA2 with -1.

    tbNew.putcol("ANTENNA1", tbOld.getcol("ANTENNA1"))
    tbNew.putcol("ANTENNA2", -1.0 * numpy.ones(nRow, dtype="int"))
    tbNew.putcol("FIELD_ID", tbOld.getcol("FIELD_ID"))
    tbNew.putcol("FLAG", tbOld.getcol("FLAG"))
    tbNew.putcol("INTERVAL", tbOld.getcol("INTERVAL"))
    tbNew.putcol("SNR", tbOld.getcol("SNR"))
    # Map CAL_DESC_ID to SPECTRAL_WINDOW_ID:
    tbNew.putcol("SPECTRAL_WINDOW_ID", spwmap[tbOld.getcol("CAL_DESC_ID")])
    tbNew.putcol("TIME", tbOld.getcol("TIME"))
    tbNew.putcol("WEIGHT", tbOld.getcol("FIT_WEIGHT"))

    # Transfer the parameter column from the old-format caltable to the
    # new-format caltable, taking the type into account.  No parameter
    # errors are present in the old-format caltable, so they are set to
    # 1.0 (+j0.0).

    param = tbOld.getcol("GAIN")
    if pTypeTemp == "float":
        param = param.real
        tbNew.putcol("FPARAM", param)
    else:
        tbNew.putcol("CPARAM", param)

    tbNew.putcol("PARAMERR", -1.0 * numpy.ones(param.shape, dtype="float"))

    # Determine the scans and put them into the new-format caltable

    scanTimes = get_scantimes(ms)
    keys = scanTimes.keys()

    times = tbOld.getcol("TIME")
    scans = numpy.zeros(0, dtype="int")

    for t in times.tolist():

        flag = False

        for i in range(len(keys)):
            k = keys[i]
            if t >= scanTimes[k]["min"] and t < scanTimes[k]["max"]:
                flag = True
                scans = numpy.append(scans, k)
                break
            if i == len(keys) - 1:
                break
            l = keys[i + 1]
            if t >= scanTimes[k]["max"] and t < scanTimes[l]["min"]:
                flag = True
                scans = numpy.append(scans, l)
                break

        if not flag:
            scans = numpy.append(scans, -1)

    tbNew.putcol("SCAN_NUMBER", scans)

    # Copy the appropriate subcaltables from the MS to the new-format
    # caltable

    arg = "cp -r " + ms + "/ANTENNA " + caltabnew
    os.system(arg)

    arg = "cp -r " + ms + "/FIELD " + caltabnew
    os.system(arg)

    # 	arg = 'cp -r ' + caltabold + '/CAL_HISTORY ' + caltabnew + '/HISTORY'
    # 	os.system( arg )

    tbHis = casac.table()
    tbHis.open(ms + "/HISTORY")
    tbHis.copy(newtablename=caltabnew + "/HISTORY", deep=True, valuecopy=True, norows=True)
    tbHis.close()
    del tbHis

    arg = "cp -r " + ms + "/SPECTRAL_WINDOW " + caltabnew + "/SPECTRAL_WINDOW"
    os.system(arg)

    # Add the info and keywords to the main table of the new-format caltable
    tabinfo = tbOld.info()

    tbNew.putinfo(tabinfo)

    if pTypeTemp == "complex":
        tbNew.putkeyword("ParType", "Complex")
    if pTypeTemp == "float":
        tbNew.putkeyword("ParType", "Float")

    tbNew.putkeyword("MSName", ms)
    tbNew.putkeyword("VisCal", tabinfo["subType"])

    polBasis = get_polbasis(ms)
    tbNew.putkeyword("PolBasis", polBasis)

    tbNew.putkeyword("ANTENNA", "Table: " + caltabnew + "/ANTENNA")
    tbNew.putkeyword("FIELD", "Table: " + caltabnew + "/FIELD")
    tbNew.putkeyword("HISTORY", "Table: " + caltabnew + "/HISTORY")
    tbNew.putkeyword("SPECTRAL_WINDOW", "Table: " + caltabnew + "/SPECTRAL_WINDOW")

    # Add the column keywords to the main table of the new-format caltable

    colList = ["INTERVAL", "TIME"]

    for col in colList:
        colKeys = tbOld.getcolkeywords(col)
        if colKeys != {}:
            tbNew.putcolkeywords(col, colKeys)

        # Close the old- and new- format caltables

    tbOld.close()
    del tbOld

    tbNew.close()
    del tbNew

    # Get the channel ranges from the CAL_DESC subtable of the old-format
    # caltable

    tbDesc = casac.table()
    tbDesc.open(caltabold + "/CAL_DESC")

    nDesc = tbDesc.nrows()
    rDesc = range(nDesc)

    spwMap = tbDesc.getcol("SPECTRAL_WINDOW_ID")[0]

    chanMin = numpy.zeros(nDesc, dtype="int")
    chanMax = numpy.zeros(nDesc, dtype="int")

    for d in rDesc:
        chanRange = tbDesc.getcol("CHAN_RANGE", startrow=d, nrow=1)
        chanMin[d] = numpy.min(chanRange[0, :, :, :][0, :, 0])
        chanMax[d] = numpy.max(chanRange[1, :, :, :][0, :, 0])

    if numpy.all(chanMax - chanMin + 1 == 1):
        gain = True
    else:
        gain = False

    tbDesc.close()
    del tbDesc

    # Modify the columns of the new-format caltable according to the
    # channel ranges

    tbSPW = casac.table()
    tbSPW.open(caltabnew + "/SPECTRAL_WINDOW", nomodify=False)

    for d in rDesc:

        s = int(spwMap[d])

        nChan = int(chanMax[d] - chanMin[d] + 1)
        tbSPW.putcell("NUM_CHAN", s, nChan)

        chanFreq = tbSPW.getcell("CHAN_FREQ", s)
        if gain:
            chanFreq = numpy.median(chanFreq)
        else:
            chanFreq = chanFreq[chanMin[d] : chanMax[d] + 1]
        tbSPW.putcell("CHAN_FREQ", s, chanFreq)

        chanWidth = tbSPW.getcell("CHAN_WIDTH", s)
        if gain:
            chanWidth = numpy.sum(chanWidth)
        else:
            chanWidth = chanWidth[chanMin[d] : chanMax[d] + 1]
        tbSPW.putcell("CHAN_WIDTH", s, chanWidth)
        tbSPW.putcell("EFFECTIVE_BW", s, chanWidth)
        tbSPW.putcell("RESOLUTION", s, chanWidth)

        totalBandwidth = numpy.sum(chanWidth)
        tbSPW.putcell("TOTAL_BANDWIDTH", s, totalBandwidth)

    tbSPW.close()
    del tbSPW

    # Return True

    return True
Esempio n. 19
0
def compTables(referencetab,
               testtab,
               excludecols,
               tolerance=0.001,
               mode="percentage",
               startrow=0,
               nrow=-1,
               rowincr=1):
    """
    compTables - compare two CASA tables
    
       referencetab - the table which is assumed to be correct

       testtab - the table which is to be compared to referencetab

       excludecols - list of column names which are to be ignored

       tolerance - permitted fractional difference (default 0.001 = 0.1 percent)

       mode - comparison is made as "percentage", "absolute", "phaseabsdeg" (for complex numbers = difference of the phases in degrees)  
    """

    rval = True

    tb2 = casac.table()

    tb.open(referencetab)
    cnames = tb.colnames()

    tb2.open(testtab)

    try:
        for c in cnames:
            if c in excludecols:
                continue

            print "\nTesting column " + c

            a = 0
            try:
                a = tb.getcol(c, startrow=startrow, nrow=nrow, rowincr=rowincr)
            except:
                rval = False
                print 'Error accessing column ', c, ' in table ', referencetab
                print sys.exc_info()[0]
                break

            b = 0
            try:
                b = tb2.getcol(c,
                               startrow=startrow,
                               nrow=nrow,
                               rowincr=rowincr)
            except:
                rval = False
                print 'Error accessing column ', c, ' in table ', testtab
                print sys.exc_info()[0]
                break

            if not (len(a) == len(b)):
                print 'Column ', c, ' has different length in tables ', referencetab, ' and ', testtab
                print a
                print b
                rval = False
                break
            else:
                differs = False
                if not (a == b).all():
                    for i in range(0, len(a)):
                        if (isinstance(a[i], float)):
                            if ((mode == "percentage") and
                                (abs(a[i] - b[i]) > tolerance * abs(a[i]))
                                ) or ((mode == "absolute") and
                                      (abs(a[i] - b[i]) > tolerance)):
                                print "Column " + c + " differs"
                                print "Row=" + str(i)
                                print "Reference file value: " + str(a[i])
                                print "Input file value: " + str(b[i])
                                if (mode == "percentage"):
                                    print "Tolerance is {0}%; observed difference was {1} %".format(
                                        tolerance * 100,
                                        100 * abs(a[i] - b[i]) / abs(a[i]))
                                else:
                                    print "Absolute tolerance is {0}; observed difference: {1}".format(
                                        tolerance, (abs(a[i] - b[i])))
                                differs = True
                                rval = False
                                break
                        elif (isinstance(a[i], int)
                              or isinstance(a[i], np.int32)):
                            if (abs(a[i] - b[i]) > 0):
                                print "Column " + c + " differs"
                                print "Row=" + str(i)
                                print "Reference file value: " + str(a[i])
                                print "Input file value: " + str(b[i])
                                if (mode == "percentage"):
                                    print "tolerance in % should be " + str(
                                        100 * abs(a[i] - b[i]) / abs(a[i]))
                                else:
                                    print "absolute tolerance should be " + str(
                                        abs(a[i] - b[i]))
                                differs = True
                                rval = False
                                break
                        elif (isinstance(a[i], str)
                              or isinstance(a[i], np.bool_)):
                            if not (a[i] == b[i]):
                                print "Column " + c + " differs"
                                print "Row=" + str(i)
                                print "Reference file value: " + str(a[i])
                                print "Input file value: " + str(b[i])
                                if (mode == "percentage"):
                                    print "tolerance in % should be " + str(
                                        100 * abs(a[i] - b[i]) / abs(a[i]))
                                else:
                                    print "absolute tolerance should be " + str(
                                        abs(a[i] - b[i]))
                                differs = True
                                rval = False
                                break
                        elif (isinstance(a[i], list)) or (isinstance(
                                a[i], np.ndarray)):
                            for j in range(0, len(a[i])):
                                if differs: break
                                if ((isinstance(a[i][j], float))
                                        or (isinstance(a[i][j], int))):
                                    if ((mode == "percentage") and
                                        (abs(a[i][j] - b[i][j]) >
                                         tolerance * abs(a[i][j]))) or (
                                             (mode == "absolute") and
                                             (abs(a[i][j] - b[i][j]) >
                                              tolerance)):
                                        print "Column " + c + " differs"
                                        print "(Row,Element)=(" + str(
                                            j) + "," + str(i) + ")"
                                        print "Reference file value: " + str(
                                            a[i][j])
                                        print "Input file value: " + str(
                                            b[i][j])
                                        if (mode == "percentage"):
                                            print "Tolerance in % should be " + str(
                                                100 * abs(a[i][j] - b[i][j]) /
                                                abs(a[i][j]))
                                        else:
                                            print "Absolute tolerance should be " + str(
                                                abs(a[i][j] - b[i][j]))
                                        differs = True
                                        rval = False
                                        break
                                elif (isinstance(a[i][j],
                                                 list)) or (isinstance(
                                                     a[i][j], np.ndarray)):
                                    it = xrange(0, len(a[i][j]))
                                    if mode == "percentage":
                                        diff = np.abs(
                                            np.subtract(a[i][j], b[i][j])
                                        ) > tolerance * np.abs(a[i][j])
                                        it = np.where(diff)[0]
                                    elif (mode == "absolute"):
                                        diff = np.abs(
                                            np.subtract(a[i][j],
                                                        b[i][j])) > tolerance
                                        it = np.where(diff)[0]
                                    for k in it:
                                        if differs: break
                                        if ( ((mode=="percentage") and (abs(a[i][j][k]-b[i][j][k]) > tolerance*abs(a[i][j][k]))) \
                                                 or ((mode=="absolute") and (abs(a[i][j][k]-b[i][j][k]) > tolerance)) \
                                                 or ((mode=="phaseabsdeg") and (phasediffabsdeg(a[i][j][k],b[i][j][k])>tolerance)) \
                                                 ):
                                            print "Column " + c + " differs"
                                            print "(Row,Channel,Corr)=(" + str(
                                                k) + "," + str(j) + "," + str(
                                                    i) + ")"
                                            print "Reference file value: " + str(
                                                a[i][j][k])
                                            print "Input file value: " + str(
                                                b[i][j][k])
                                            if (mode == "percentage"):
                                                print "Tolerance in % should be " + str(
                                                    100 * abs(a[i][j][k] -
                                                              b[i][j][k]) /
                                                    abs(a[i][j][k]))
                                            elif (mode == "absolute"):
                                                print "Absolute tolerance should be " + str(
                                                    abs(a[i][j][k] -
                                                        b[i][j][k]))
                                            elif (mode == "phaseabsdeg"):
                                                print "Phase tolerance in degrees should be " + str(
                                                    phasediffabsdeg(
                                                        a[i][j][k],
                                                        b[i][j][k]))
                                            else:
                                                print "Unknown comparison mode: ", mode
                                            differs = True
                                            rval = False
                                            break

                        else:
                            print "Unknown data type: ", type(a[i])
                            differs = True
                            rval = False
                            break

                if not differs: print "Column " + c + " PASSED"
    finally:
        tb.close()
        tb2.close()

    return rval
Esempio n. 20
0
def importasdm(
    asdm=None,
    vis=None,
    createmms=None,
    separationaxis=None,
    numsubms=None,    
    singledish=None,
    antenna=None,
    corr_mode=None,
    srt=None,
    time_sampling=None,
    ocorr_mode=None,
    compression=None,
    lazy=None,
    asis=None,
    wvr_corrected_data=None,
    scans=None,
    ignore_time=None,
    process_syspower=None,
    process_caldevice=None,
    process_pointing=None,
    process_flags=None,
    tbuff=None,
    applyflags=None,
    savecmds=None,
    outfile=None,
    flagbackup=None,
    verbose=None,
    overwrite=None,
    showversion=None,
    useversion=None,
    bdfflags=None,
    with_pointing_correction=None,
    remove_ref_undef=None,
    convert_ephem2geo=None
    ):
    """Convert an ALMA Science Data Model observation into a CASA visibility file (MS) or single-dish data format (Scantable).
           The conversion of the ALMA SDM archive format into a measurement set.  This version
           is under development and is geared to handling many spectral windows of different
           shapes.

           Keyword arguments:
           asdm -- Name of input ASDM file (directory)
               default: none; example: asdm='ExecBlock3'

       vis       -- Root ms or scantable name, note a prefix (.ms or .asap) is NOT appended to this name
           default: none
           
       createmms  -- Create a Multi-MS
           default: False
           
       singledish   -- Set True to write data as single-dish format (Scantable)
               default: False singledish expandable parameter
                 antenna -- antenna name or id.
 
       corr_mode -- correlation mode to be considered on input. Could
            be one or more of the following, ao, co, ac, or all
           default: all

       srt       -- spectral resolution type. Could be one or more of
                    the following, fr, ca, bw, or all
           default: all

       time_sampling -- specifies the time sampling, INTEGRATION and/or
                            SUBINTEGRATION. could be one or more of the following
                            i, si, or all.
           default: all

       ocorr_mode    -- output data for correlation mode AUTO_ONLY 
                            (ao) or CROSS_ONLY (co) or CROSS_AND_AUTO (ca)
           default: ca

      compression  -- produces comrpressed columns in the resulting measurement set.
                 default: False

       lazy         -- Make the MS DATA column read the ASDM Binary data directly
                       (faster import, smaller MS)
                 default: False

       asis         --  creates verbatim copies of the ASDM tables in 
                        the output measurement set. The value given to
                    this option must be a list of table names separated
                    by space characters; the wildcard character '*' is 
                            allowed in table names.

       wvr_corrected_data -- specifies wich values are considered in the 
                      ASDM binary data to fill the DATA column in 
                      the MAIN table of the MS. Expected values for 
                      this option are 'no' for the uncorrected data 
                      (this is the default), 'yes' for the corrected
                      data and 'both' for corrected and uncorrected 
                      data. In the latter case, two measurement sets
                      are created, one containing the uncorrected 
                      data and the other one, whose name is suffixed
                      by '-wvr-corrected', containing the corrected 
                      data.

       scans --  processes only the scans specified in the option's value. This value is a semicolon 
                 separated list of scan specifications. A scan specification consists in an exec bock index 
                 followed by the character ':' followed by a comma separated list of scan indexes or scan 
                 index ranges. A scan index is relative to the exec block it belongs to. Scan indexes are 
                 1-based while exec blocks's are 0-based. "0:1" or "2:2~6" or "0:1,1:2~6,8;2:,3:24~30" "1,2" 
                 are valid values for the option. "3:" alone will be interpreted as 'all the scans of the 
                 exec block#3'. An scan index or a scan index range not preceded by an exec block index will
                 be interpreted as 'all the scans with such indexes in all the exec blocks'.  By default 
                 all the scans are considered.

       ignore_time -- All the rows of the tables Feed, History, Pointing, Source, SysCal, CalDevice, SysPower,
                      and Weather are processed independently of the time range of the selected exec block / scan.

       process_syspower -- The SysPower table is processed if and only if this parameter is set to True.
              default: True

       process_caldevice -- The CalDevice table is processed if and only if this parameter is set to True.
              default: True

       process_pointing -- The Pointing table is processed if and only if this parameter is set to True.
                       If the parameter is set to False the resulting MS will have an empty POINTING table.
              default: True

      process_flags -- Process the online flags and save them to the FLAG_CMD sub-table.
              default: True

            &gt;&gt;&gt; process_flags expandable parameter
                 tbuff -- Time padding buffer (in seconds).
                    default: 0.0

                 applyflags -- Apply the online flags to the MS.
                    default: False

                 savecmds -- Save the online flags to an ASCII file.
                    default: False
                    
                 outfile -- Filename to save the online flags.
                    default: ''

       flagbackup -- Backup the FLAG column in the .flagversions.
              default: True

       verbose     -- produce log output as asdm2MS is being run.

       overwrite -- Over write an existing MS.

       showversion -- report the version of the asdm2MS being used.

       useversion -- Selects the version of asdm2MS to be used (presently only \'v3\' is available).
                     default: v3
                     
      bdfflags -- Set the MS FLAG column according to the ASDM _binary_ flags
                   default: false

      with_pointing_correction -- add (ASDM::Pointing::encoder - ASDM::Pointing::pointingDirection)
                 to the value to be written in MS::Pointing::direction 
                   default: false

      remove_ref_undef -- if set to True then apply fixspwbackport on the resulting MSes.

      convert_ephem2geo -- if True, convert any attached ephemerides to the GEO reference frame
           
        """

    # Python script
    
    # make agentflagger tool local
    aflocal = casac.agentflagger()

    # make table tool local
    tblocal = casac.table()

    try:
        casalog.origin('importasdm')
        viso = ''
        visoc = ''  # for the wvr corrected version, if needed
                # -----------------------------------------
                # beginning of importasdm_sd implementation
                # -----------------------------------------
        if singledish:
            theexecutable = 'asdm2ASAP'
                        # if useversion == 'v2':
                        #        theexecutable = 'oldasdm2ASAP'
            if compression:
                casalog.post('compression=True has no effect for single-dish format.')
                                
            cmd = 'which %s > /dev/null 2>&1' % theexecutable
            ret = os.system(cmd)
            if ret == 0:
                import commands
                casalog.post('found %s' % theexecutable)
                if showversion:
                    execute_string = theexecutable + ' --help'
                else:
                    execute_string = theexecutable + ' -asdm ' + asdm
                    if len(vis) != 0:
                        execute_string += ' -asap ' + vis.rstrip('/')
                    execute_string += ' -antenna ' + str(antenna) \
                        + ' -apc ' + wvr_corrected_data \
                        + ' -time-sampling ' + time_sampling.lower() \
                        + ' -overwrite ' + str(overwrite)
                    if corr_mode == 'all':
                        execute_string += \
                            ' -corr-mode ao,ca -ocorr-mode ao'
                    else:
                        execute_string += ' -corr-mode ' \
                            + corr_mode.replace(' ', ',') \
                            + ' -ocorr-mode ao'
                    if srt == 'all':
                        execute_string += ' -srt ' + srt
                    else:
                        execute_string += ' -srt ' + srt.replace(' ',
                                ',')
                    execute_string += ' -logfile ' + casalog.logfile()
                casalog.post('execute_string is')
                casalog.post('   ' + execute_string)
                ret = os.system(execute_string)
                if ret != 0 and not showversion:
                    casalog.post(theexecutable
                                 + ' terminated with exit code '
                                 + str(ret), 'SEVERE')
                    # raise Exception, "ASDM conversion error, please check if it is a valid ASDM and/or useversion='%s' is consistent with input ASDM."%(useversion)
                    raise Exception, \
                        'ASDM conversion error, please check if it is a valid ASDM.'
            else:
                casalog.post('You have to build ASAP to be able to create single-dish data.'
                             , 'SEVERE')
                        # implementation of asis option using tb.fromASDM
            if asis != '':
                import commands
                asdmTables = commands.getoutput('ls %s/*.xml'
                        % asdm).split('\n')
                asdmTabNames = []
                for tab in asdmTables:
                    asdmTabNames.append(tab.split('/')[-1].rstrip('.xml'
                            ))
                if asis == '*':
                    targetTables = asdmTables
                    targetTabNames = asdmTabNames
                else:
                    targetTables = []
                    targetTabNames = []
                    tmpTabNames = asis.split()
                    for i in xrange(len(tmpTabNames)):
                        tab = tmpTabNames[i]
                        try:
                            targetTables.append(asdmTables[asdmTabNames.index(tab)])
                            targetTabNames.append(tab)
                        except:
                            pass
                outTabNames = []
                outTables = []
                for tab in targetTabNames:
                    out = 'ASDM_' + tab.upper()
                    outTabNames.append(out)
                    outTables.append(vis + '/' + out)
                
                tblocal.open(vis, nomodify=False)
                wtb = casac.table()
                for i in xrange(len(outTables)):
                    wtb.fromASDM(outTables[i], targetTables[i])
                    tblocal.putkeyword(outTabNames[i], 'Table: %s'
                                  % outTables[i])
                    tblocal.flush()
                tblocal.close()
            return
                # -----------------------------------
                # end of importasdm_sd implementation
                # -----------------------------------
        if len(vis) > 0:
            viso = vis
            tmps = vis.rstrip('.ms')
            if tmps == vis:
                visoc = vis + '-wvr-corrected'
            else:
                visoc = tmps + '-wvr-corrected.ms'
            if singledish:
                viso = vis.rstrip('/') + '.importasdm.tmp.ms'
        else:
            viso = asdm.rstrip("/") + '.ms'
            visoc = asdm.rstrip("/") + '-wvr-corrected.ms'
            vis = asdm.rstrip("/")
            if singledish:
                viso = asdm.rstrip('/') + '.importasdm.tmp.ms'
                vis = asdm.rstrip('/') + '.asap'



        useversion = 'v3'
        theexecutable = 'asdm2MS'

        execute_string = theexecutable + ' --icm "' + corr_mode \
            + '" --isrt "' + srt + '" --its "' + time_sampling \
            + '" --ocm "' + ocorr_mode + '" --wvr-corrected-data "' \
            + wvr_corrected_data + '" --asis "' + asis \
            + '" --logfile "' + casalog.logfile() + '"'

        if len(scans) > 0:
            execute_string = execute_string + ' --scans ' + scans
        if ignore_time:
            execute_string = execute_string + ' --ignore-time'
        if useversion == 'v3':
            if not process_syspower:
                execute_string = execute_string + ' --no-syspower'
            if not process_caldevice:
                execute_string = execute_string + ' --no-caldevice'
            if not process_pointing:
                execute_string = execute_string + ' --no-pointing'

        if compression:
            execute_string = execute_string + ' --compression'
        elif lazy:
            execute_string = execute_string + ' --lazy'
            
        if verbose:
            execute_string = execute_string + ' --verbose'
#         if not overwrite and os.path.exists(viso):
#             raise Exception, \
#                 'You have specified an existing MS and have indicated you do not wish to overwrite it'

        # Compression
        if compression:
                   # viso = viso + '.compressed'
            viso = viso.rstrip('.ms') + '.compressed.ms'
            visoc = visoc.rstrip('.ms') + '.compressed.ms'

        vistoproc = [] # the output MSs to post-process
        if wvr_corrected_data == 'no' or wvr_corrected_data == 'both':
            vistoproc.append(viso)
        if (wvr_corrected_data == 'yes' or wvr_corrected_data == 'both') : 
            vistoproc.append(visoc)

        for ff in vistoproc:
            if not overwrite and os.path.exists(ff):
                raise Exception, \
                    'You have specified an existing MS and have indicated you do not wish to overwrite it: %s'%ff

        # If viso+".flagversions" then process differently depending on the value of overwrite..
        #
        if flagbackup:
            for myviso in vistoproc:
                dotFlagversion = myviso + '.flagversions'
                if os.path.exists(dotFlagversion):
                    if overwrite:
                        casalog.post("Found '" + dotFlagversion
                                     + "' . It'll be deleted before running the filler."
                                     )
                        os.system('rm -rf %s' % dotFlagversion)
                    else:
                        casalog.post("Found '%s' but can't overwrite it."
                                     % dotFlagversion)
                        raise Exception, "Found '%s' but can't overwrite it." \
                            % dotFlagversion
               
        # Make outfile always a list             
        if isinstance(outfile, str):
            if outfile == '': 
                outfile = []
            else:
                noutfile = [outfile]
                outfile = noutfile
            
        if savecmds:
            if len(outfile) == 0:
                # Create default names for the online flags
                for myviso in vistoproc:
                    outfile.append(myviso.replace('.ms','_cmd.txt'))
            elif len(outfile) != len(vistoproc):
                casalog.post('List of outfile names does not match list of MSs','WARN')
                casalog.post('Will save online flags to temporary filenames', 'WARN')
                outfile = []
                for myviso in vistoproc:
                    online_file = myviso.replace('.ms','_TEMP_cmd.txt')
                    outfile.append(online_file)
                                     
            if not overwrite:
                for of in outfile:
                    if os.path.exists(of):
                        raise Exception, "Cannot overwrite online flags file '%s'; overwrite is set to False."% of
                
            
        execute_string = execute_string + ' ' + asdm + ' ' + viso

        if showversion:
            casalog.post("You set option \'showversion\' to True. Will just output the version information and then terminate."
                         , 'WARN')
            execute_string = theexecutable + ' --revision'

        if with_pointing_correction:
            execute_string = execute_string + ' --with-pointing-correction'

        casalog.post('Running ' + theexecutable
                     + ' standalone invoked as:')
        # print execute_string
        casalog.post(execute_string)
        exitcode = os.system(execute_string)
        if exitcode != 0:
            if not showversion:
                casalog.post(theexecutable
                             + ' terminated with exit code '
                             + str(exitcode), 'SEVERE')
                raise Exception, \
                    'ASDM conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'

        if showversion:
            return
        
        #
        # Possibly remove the element the name of the measurement set expected to contain the corrected data from the list of of produced measurement
        # sets if it appears the filler did not find any corrected data.
        #
        if not os.path.exists(visoc):
            vistoproc = [myviso for myviso in vistoproc if myviso != visoc]

        # CAS-7369. HISTORY should be written after createmms is tested
        #
        # Populate the HISTORY table of the MS with information about the context in which it's been created
        #
        try: 
            mslocal = mstool() 
            param_names = importasdm.func_code.co_varnames[:importasdm.func_code.co_argcount] 
            param_vals = [eval(p) for p in param_names]

            for myviso in vistoproc:
                write_history(mslocal, myviso, 'importasdm', param_names, 
                              param_vals, casalog) 

        except Exception, instance: 
            casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 
                         'WARN')
            return False 

        if mslocal:
            mslocal = None 
            
        # 
        # Do we apply fixspwbackport
        if remove_ref_undef :
            casalog.post('remove_ref_undef=True: fixspwbackport will be applied ...')
            
            for myviso in vistoproc:
                cmd = 'fixspwbackport ' + myviso
                casalog.post('Running fixspwbackport standalone invoked as:')
                casalog.post(cmd)
                cmdexitcode = os.system(cmd)

                if cmdexitcode != 0:
                    casalog.post(cmd
                                 + ' terminated with exit code '
                                 + str(cmdexitcode), 'SEVERE')
                    raise Exception, 'fixspwbackport error.'

        # Binary Flag processing
        if bdfflags:
            
            casalog.post('Parameter bdfflags==True: flags from the ASDM binary data will be used to set the MS flags ...')
            
            bdffexecutable = 'bdflags2MS '
            bdffexecstring_base = bdffexecutable+' -f ALL'
            if len(scans) > 0:
                bdffexecstring_base = bdffexecstring_base + ' --scans ' + scans

            for myviso in vistoproc:
                if myviso.find("wvr-corrected") != -1:
                    options = " --wvr-corrected=True " 
                else:
                    options = " "

                bdffexecstring = bdffexecstring_base + options + asdm + ' ' + myviso

                casalog.post('Running '+bdffexecutable+' standalone invoked as:')
                casalog.post(bdffexecstring)

                bdffexitcode = os.system(bdffexecstring)
                if bdffexitcode != 0:
                    casalog.post(bdffexecutable
                                 + ' terminated with exit code '
                                 + str(bdffexitcode), 'SEVERE')
                    raise Exception, \
                          'ASDM binary flags conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'

        for myviso in vistoproc:
            if os.path.exists(myviso) and flagbackup==True:
                aflocal.open(myviso)
                aflocal.saveflagversion('Original',
                        comment='Original flags at import into CASA',
                        merge='save')
                aflocal.done()
                
        # Importasdm Flag Parsing
        if os.access(asdm + '/Flag.xml', os.F_OK):
            # Find Flag.xml
            casalog.post('Found Flag.xml in SDM')
            
            # Find Antenna.xml
            if os.access(asdm + '/Antenna.xml', os.F_OK):
                casalog.post('Found Antenna.xml in SDM')

            else:
                raise Exception, 'Failed to find Antenna.xml in SDM'
            
            # Find SpectralWindow.xml
            if os.access(asdm + '/SpectralWindow.xml', os.F_OK):
                casalog.post('Found SpectralWindow.xml in SDM')

            else:
                raise Exception, \
                    'Failed to find SpectralWindow.xml in SDM'
                    
            #
            # Parse Flag.xml into flag dictionary
            #
            if process_flags:
                flagcmds = fh.parseXML(asdm, float(tbuff))
                onlinekeys = flagcmds.keys()
                nflags = onlinekeys.__len__()
                                
                # Apply flags to the MS
                if nflags > 0:
                    idx = 0
                    for myviso in vistoproc:
                        if applyflags:
                            # Open the MS and attach it to the tool
                            aflocal.open(myviso)
                            # Select the data
                            aflocal.selectdata()
                            # Setup the agent's parameters
                            fh.parseAgents(aflocal, flagcmds, [], True, True, '')
                            # Initialize the agents
                            aflocal.init()
                            # Run the tool
                            aflocal.run(True, True)
                            casalog.post('Applied %s flag commands to %s'%(nflags,myviso))
                            # Destroy the tool and de-attach the MS
                            aflocal.done()
                            # Save to FLAG_CMD table. APPLIED is set to True.
                            fh.writeFlagCommands(myviso, flagcmds, True, '', '', True)       
                        else:
                            casalog.post('Will not apply flags to %s (apply_flags=False), use flagcmd to apply'%myviso)

                            # Write to FLAG_CMD, APPLIED is set to False
                            fh.writeFlagCommands(myviso, flagcmds, False, '', '', True)
                    
                        # Save the flag cmds to an ASCII file
                        if savecmds:
                            # Save to standard filename
                            fh.writeFlagCommands(myviso, flagcmds, False, '', outfile[idx], False)
                            casalog.post('Saved %s flag commands to %s'%(nflags,outfile[idx]))
                            idx += 1
                    
                else:
                    casalog.post('There are no flag commands to process')
                
        else:
            casalog.post('There is no Flag.xml in ASDM', 'WARN')

        import recipes.ephemerides.convertephem as ce
        
        theephemfields = ce.findattachedephemfields(myviso,field='*')
        if len(theephemfields)>0: # temporary fix until asdm2MS does this internally: recalc the UVW coordinates for ephem fields
            imt = imtool()
            imt.open(myviso, usescratch=False)
            imt.calcuvw(theephemfields, refcode='J2000', reuse=False)
            imt.close()

        if convert_ephem2geo:
            for myviso in vistoproc:
                ce.convert2geo(myviso, '*') # convert any attached ephemerides to GEO
        
        # CAS-7369 - Create an output Multi-MS (MMS)
        if createmms:
            # Get the default parameters of partition
            from tasks import partition
            fpars = partition.parameters
            for mypar in fpars.keys():
                fpars[mypar] = partition.itsdefault(mypar)
                
            # Call the cluster for each MS
            for myviso in vistoproc:
                casalog.origin('importasdm')
                outputmms = myviso.replace('.ms','.mms')
                
                # Get the proper column
                datacolumn = 'DATA'
                dcols = ['DATA', 'FLOAT_DATA']
                for dc in dcols:
                    if len(th.getColDesc(myviso, dc)) > 0:
                        datacolumn = dc
                        break
                    
                fpars['datacolumn'] = datacolumn
                    
                casalog.post('Will create a Multi-MS for: '+myviso)
                
                fpars['vis'] =  myviso
                fpars['flagbackup'] =  False 
                fpars['outputvis'] = outputmms
                fpars['separationaxis'] = separationaxis
                fpars['numsubms'] = numsubms
                pdh = ParallelDataHelper('partition', fpars) 
            
                # Get a cluster
                pdh.setupCluster(thistask='partition')
                try:
                    pdh.go()
                    
                    # Rename MMS to MS 
                    shutil.rmtree(myviso)
                    shutil.move(outputmms, myviso)

                except Exception, instance:
                    casalog.post('%s'%instance,'ERROR')
                    return False
                
            casalog.origin('importasdm')
            return