def runTest(self):

        self.modelName = "ThemiTest"
        self.homeDir = os.path.realpath(os.path.join(os.path.dirname(__file__), "testEnvironment"))

        # Rename first observation file to have todays date.
        imomoDir = os.path.join(self.homeDir, "app", self.modelName, "data", "raw", "imomo")
        date = jdutil.date_to_jd(
            int(datetime.now().strftime("%Y")), int(datetime.now().strftime("%m")), float(datetime.now().strftime("%d"))
        )
        date = jdutil.jd_to_mjd(date)
        # Convert to matlab datenum
        date = date + 678942
        datenum = "%d" % date
        filename = os.path.join(imomoDir, (datenum + "DLoad.csv"))
        shutil.copy(os.path.join(imomoDir, "736239DLoad.csv"), os.path.join(imomoDir, filename))

        # Call to observationsAvailable.
        self.bol = observationsAvailable(self.homeDir, self.modelName)

        # AssertTrue
        self.assertEqual(self.bol, True)

        # Test false case.
        shutil.copy(os.path.join(imomoDir, "736238DLoad.csv"), os.path.join(imomoDir, filename))

        # Call to observationsAvailable.
        self.bol = observationsAvailable(self.homeDir, self.modelName)

        # AssertTrue
        self.assertEqual(self.bol, False)
def observationsAvailable(homeDirectory, modelName):
    """
    Check if observations are available for the current time step.
    
    - Figure out todays date in matlab datenum format to get the filename
    - If there is a file with todays measurements read it in and figure out if
    there is discharge data available.
    
    If there are discharge observations available return true, else return false.
    """
    date = jdutil.date_to_jd(
        int(datetime.now().strftime("%Y")), int(datetime.now().strftime("%m")), float(datetime.now().strftime("%d"))
    )
    date = jdutil.jd_to_mjd(date)
    # Convert to matlab datenum
    date = date + 678942
    datenum = "%d" % date
    obsDir = os.path.join(homeDirectory, "app", modelName, "data", "raw", "imomo")
    filename = os.path.join(obsDir, (datenum + "DLoad.csv"))
    if os.path.exists(filename):
        # Scan line by line until the second entry (variable ID) of a line matches 25
        # (discharge).
        for line in open(filename):
            if re.search("^\d+\.*\d*,25,.+", line):
                return True
        return False  # No match while reading the lines.
    else:  # Return False if there is no new data.
        return False
示例#3
0
def groundStationSec(groundList, startTime, passSec, interNum):
    year, month, day, hour, minute, sec = startTime
    jd = jdutil.date_to_jd(year, month, day)
    mjd = jdutil.jd_to_mjd(jd)
    print jd, mjd
    J = len(groundList[0])
    silde = []
    groundSecList = []
    groundfun = range(J)
    index = 0
    found = False
    while not found:
        if int(groundList[index][0]) == int(mjd):
            found = True
        else:
            index += 1
    for j in range(1, J):
        x = [
            float(groundList[index + i - interNum][0])
            for i in range(2 * interNum + 1)
        ]
        fx = [
            groundList[index + ii - interNum][j]
            for ii in range(2 * interNum + 1)
        ]
        groundfun[j] = lagInterpolation.get_Lxfunc(x, fx)
    for i in range(passSec):
        time = mjd + hour / 24. + minute / (24 * 60.) + (sec + i) / (24 * 60 *
                                                                     60.)
        silde.append(i)
        for j in range(1, J):
            silde.append(groundfun[j](time))
        groundSecList.append(silde[i * J:])
    print 'ground station position interpolation calculated by second !'
    return groundSecList
def import_eovsa(filename, outpath=None, nocreatms=False, nowritems=False):
    uv = aipy.miriad.UV(filename)
    if uv['source'].lower() == 'sun':
        outpath = outpath + 'sun/'
        if not os.path.exists(outpath):
            os.mkdir(outpath)
    else:
        outpath = outpath + 'calibrator/'
        if not os.path.exists(outpath):
            os.mkdir(outpath)
    uv.rewind()

    start_time = 0  # The start and stop times are referenced to ref_time_jd in second
    end_time = 600
    delta_time = 1
    time_steps = (end_time - start_time) / delta_time
    time0 = time.time()

    if 'antlist' in uv.vartable:
        ants = uv['antlist']
        antlist = map(int, ants.split())
    else:
        antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]

    good_idx = np.arange(len(uv['sfreq']))

    ref_time_jd = uv['time']
    ref_time_mjd = jdutil.jd_to_mjd(
        ref_time_jd) * 24. * 3600. + 0.5 * delta_time
    nf = len(good_idx)
    freq = uv['sfreq'][good_idx]
    npol = uv['npol']
    nants = uv['nants']
    sdf = uv['sdf']
    project = uv['proj']
    source_id = uv['source']
    ra, dec = uv['ra'], uv['dec']
    scan_id = uv['scanid']
    nbl = nants * (nants - 1) / 2
    bl2ord = bl_list2(nants)
    npairs = nbl + nants
    flag = np.ones((npol, nf, time_steps, npairs), dtype=bool)
    out = np.zeros((npol, nf, time_steps, npairs),
                   dtype=np.complex64)  # Cross-correlations
    uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float)
    bandedge = get_band_edge(freq)
    nband = len(bandedge) - 1

    uv.rewind()
    l = -1
    for preamble, data in uv.all():
        uvw, t, (i0, j0) = preamble
        i = antlist.index(i0 + 1)
        j = antlist.index(j0 + 1)
        if i > j:
            # Reverse order of indices
            j = antlist.index(i0 + 1)
            i = antlist.index(j0 + 1)
        # Assumes uv['pol'] is one of -5, -6, -7, -8
        k = -5 - uv['pol']
        l += 1
        out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data
        flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask
        if i != j:
            if k == 3:
                uvwarray[:, l / (npairs * npol),
                         bl2ord[i0, j0]] = uvw * constants.speed_of_light / 1e9

    nrows = time_steps * npairs
    out = out.reshape(npol, nf, nrows)
    flag = flag.reshape(npol, nf, nrows)
    uvwarray = uvwarray.reshape(3, nrows)
    uvwarray = np.tile(uvwarray, (1, nband))
    sigma = np.ones((4, nrows), dtype=np.float) + 1
    sigma = np.tile(sigma, (1, nband))

    print 'IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(
        filename, (time.time() - time0))

    # nocreatms=True
    # nowritems=True
    # nocreatms=False
    # nowritems=False
    msname = list(filename.split('/')[-1])
    msname.insert(11, 'T')
    msname = outpath + source_id.upper() + '_' + ''.join(
        msname[3:]) + '-10m.1s.ms'

    if not nocreatms:
        if os.path.exists(msname):
            os.system("rm -fr %s" % msname)
        """ Creates an empty measurement set using CASA simulate (sm) tool. """
        sm.open(msname)

        enu = np.reshape(uv['antpos'],
                         (16, 3)) * constants.speed_of_light / 1e9
        refpos_wgs84 = me.position('wgs84', '-118.286952892965deg',
                                   '37.2331698901026deg', '1207.1339m')
        lon, lat, rad = [
            me.measure(refpos_wgs84, 'itrf')[x]['value']
            for x in 'm0', 'm1', 'm2'
        ]
        # 3x3 transform matrix. Each row is a normal vector, i.e. the rows are (dE,dN,dU)
        # ----------- local xyz ------------
        xform = np.array([[0, -np.sin(lat), np.cos(lat)], [1, 0, 0],
                          [0, np.cos(lat), np.sin(lat)]])
        xyz = enu.dot(xform)  # + xyz0[np.newaxis,:]

        # ----------- global xyz ------------
        # xyz0 = rad*np.array([np.cos(lat)*np.cos(lon),np.cos(lat)*np.sin(lon),np.sin(lat)])
        # # 3x3 transform matrix. Each row is a normal vector, i.e. the rows are (dE,dN,dU)
        # xform = np.array([
        #     [-np.sin(lon),np.cos(lon),0],
        #     [-np.cos(lon)*np.sin(lat),-np.sin(lon)*np.sin(lat),np.cos(lat)],
        #     [np.cos(lat)*np.cos(lon),np.cos(lat)*np.sin(lon),np.sin(lat)]
        # ])
        # xyz = xyz0[np.newaxis,:] + enu.dot(xform)

        dishdiam = np.full(uv['nants'], 2.1)
        dishdiam[-3:-1] = 27
        dishdiam[-1] = 2.1
        station = uv['telescop']
        mount = ['ALT-AZ'] * uv['nants']
        for l in [8, 9, 10, 12, 13, 14]:
            mount[l] = 'EQUATORIAL'
        sm.setconfig(telescopename=station,
                     x=np.asarray(xyz)[:, 0],
                     y=np.asarray(xyz)[:, 1],
                     z=np.asarray(xyz)[:, 2],
                     dishdiameter=dishdiam,
                     mount=mount,
                     antname=['eo' + "{0:02d}".format(l) for l in antlist],
                     padname=station,
                     coordsystem='local',
                     referencelocation=refpos_wgs84)

        sm.setfield(sourcename=source_id,
                    sourcedirection=me.direction(
                        'J2000', '{:22.19f}'.format(uv['obsra']) + 'rad',
                        '{:22.19f}'.format(uv['obsdec']) + 'rad'))
        sm.setfeed(mode='perfect X Y')

        ref_time = me.epoch(
            'tai', '{:20.13f}'.format(jdutil.jd_to_mjd(ref_time_jd)) + 'd')

        sm.settimes(integrationtime='1s',
                    usehourangle=False,
                    referencetime=ref_time)

        for l, bdedge in enumerate(bandedge[:-1]):
            nchannels = (bandedge[l + 1] - bandedge[l])
            stokes = 'XX YY XY YX'
            df = sdf[bandedge[l]]
            st_freq = freq[bandedge[l]]

            sm.setspwindow(spwname='band%02d' % (l + 1),
                           freq='{:22.19f}'.format(st_freq) + 'GHz',
                           deltafreq='{:22.19f}'.format(df) + 'GHz',
                           freqresolution='{:22.19f}'.format(df) + 'GHz',
                           nchannels=nchannels,
                           stokes=stokes)

        nband = len(bandedge) - 1
        for bdid in range(nband):
            sm.observe(source_id,
                       'band%02d' % (bdid + 1),
                       starttime=start_time,
                       stoptime=end_time,
                       project=project,
                       state_obs_mode='')

        if sm.done():
            print 'Empty MS {0} created in --- {1:10.2f} seconds ---'.format(
                msname, (time.time() - time0))
        else:
            raise RuntimeError('Failed to create MS. Look at the log file. '
                               'Double check you settings.')
    else:
        print '----------------------------------------'
        print 'copying standard MS to {0}'.format(msname,
                                                  (time.time() - time0))
        print '----------------------------------------'
        os.system("rm -fr %s" % msname)
        os.system("cp -r " + " %s" % ms_std + " %s" % msname)
        print 'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format(
            msname, (time.time() - time0))

    if not nowritems:
        tb.open(msname, nomodify=False)
        print '----------------------------------------'
        print "Updating the main table of" '%s' % msname
        print '----------------------------------------'
        for l, bdedge in enumerate(bandedge[:-1]):
            time1 = time.time()
            nchannels = (bandedge[l + 1] - bandedge[l])
            for row in range(nrows):
                tb.putcell('DATA', (row + l * nrows),
                           out[:, bandedge[l]:bandedge[l + 1], row])
                tb.putcell('FLAG', (row + l * nrows),
                           flag[:, bandedge[l]:bandedge[l + 1], row])
            print '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format(
                (l + 1),
                time.time() - time1)
        tb.putcol('UVW', uvwarray)
        tb.putcol('SIGMA', sigma)
        tb.putcol('WEIGHT', 1.0 / sigma**2)
        timearr = np.arange((time_steps), dtype=np.float)
        timearr = timearr.reshape(1, time_steps, 1)
        timearr = np.tile(timearr, (nband, 1, npairs))
        timearr = timearr.reshape(nband * npairs * time_steps) + ref_time_mjd
        tb.putcol('TIME', timearr)
        tb.putcol('TIME_CENTROID', timearr)
        colnames = tb.colnames()
        cols2rm = ["MODEL_DATA", "CORRECTED_DATA"]
        for l in range(len(cols2rm)):
            if cols2rm[l] in colnames:
                tb.removecols(cols2rm[l])
        tb.close()

        print '----------------------------------------'
        print "Updating the OBSERVATION table of" '%s' % msname
        print '----------------------------------------'
        tb.open(msname + '/OBSERVATION', nomodify=False)
        tb.putcol(
            'TIME_RANGE',
            np.asarray([
                ref_time_mjd - 0.5 * delta_time,
                ref_time_mjd + end_time - 0.5 * delta_time
            ]).reshape(2, 1))
        tb.putcol('OBSERVER', ['EOVSA team'])
        tb.close()

        print '----------------------------------------'
        print "Updating the POINTING table of" '%s' % msname
        print '----------------------------------------'
        tb.open(msname + '/POINTING', nomodify=False)
        timearr = np.arange((time_steps),
                            dtype=np.float).reshape(1, time_steps, 1)
        timearr = np.tile(timearr, (nband, 1, nants))
        timearr = timearr.reshape(nband * time_steps * nants) + ref_time_mjd
        tb.putcol('TIME', timearr)
        tb.putcol('TIME_ORIGIN', timearr - 0.5 * delta_time)
        direction = tb.getcol('DIRECTION')
        direction[0, 0, :] = ra
        direction[1, 0, :] = dec
        tb.putcol('DIRECTION', direction)
        target = tb.getcol('TARGET')
        target[0, 0, :] = ra
        target[1, 0, :] = dec
        tb.putcol('TARGET', target)
        tb.close()

        print '----------------------------------------'
        print "Updating the SOURCE table of" '%s' % msname
        print '----------------------------------------'
        tb.open(msname + '/SOURCE', nomodify=False)
        radec = tb.getcol('DIRECTION')
        radec[0], radec[1] = ra, dec
        tb.putcol('DIRECTION', radec)
        name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
        tb.putcol('NAME', name)
        tb.close()

        print '----------------------------------------'
        print "Updating the DATA_DESCRIPTION table of" '%s' % msname
        print '----------------------------------------'
        tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False)
        pol_id = tb.getcol('POLARIZATION_ID')
        pol_id *= 0
        tb.putcol('POLARIZATION_ID', pol_id)
        tb.close()

        print '----------------------------------------'
        print "Updating the POLARIZATION table of" '%s' % msname
        print '----------------------------------------'
        tb.open(msname + '/POLARIZATION/', nomodify=False)
        tb.removerows(rownrs=np.arange(1, nband, dtype=int))
        tb.close()

        print '----------------------------------------'
        print "Updating the FIELD table of" '%s' % msname
        print '----------------------------------------'
        tb.open(msname + '/FIELD/', nomodify=False)
        delay_dir = tb.getcol('DELAY_DIR')
        delay_dir[0], delay_dir[1] = ra, dec
        tb.putcol('DELAY_DIR', delay_dir)
        phase_dir = tb.getcol('PHASE_DIR')
        phase_dir[0], phase_dir[1] = ra, dec
        tb.putcol('PHASE_DIR', phase_dir)
        reference_dir = tb.getcol('REFERENCE_DIR')
        reference_dir[0], reference_dir[1] = ra, dec
        tb.putcol('REFERENCE_DIR', reference_dir)
        name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
        tb.putcol('NAME', name)
        tb.close()

        # FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME

        del out, flag, uvwarray, uv, timearr, sigma
        gc.collect()  #

    print("finished in --- %s seconds ---" % (time.time() - time0))
    return
示例#5
0
def convert_fitacf_data(date, in_fname, radar_info):
    day = in_fname.split('.')[0].split('/')[-1]
    month = day[:-2] 
    
    # Keep track of fitACF files that have multiple beam definitions in a
    # monthly log file
    multiBeamLogDir = date.strftime(helper.FIT_NET_LOG_DIR) + month
    multiBeamLogfile = '{dir}/multi_beam_defs_{m}.log'.format(dir = multiBeamLogDir, m = month)

    # Store conversion info like returns outside FOV, missing slist, etc 
    # for each conversion
    conversionLogDir = '{dir}/{d}'.format(dir = multiBeamLogDir, d = day)
    fName = in_fname.split('/')[-1]
    conversionLogfile = '{dir}/{fit}_to_nc.log'.format(dir = conversionLogDir, fit = fName)

    # Define the name of the file holding the list of rawACFs used to 
    # create the fitACF
    rawacfListFilename = '.'.join(in_fname.split('.')[:-1]) + '.rawacfList.txt'

    SDarn_read = pydarn.SuperDARNRead(in_fname)
    data = SDarn_read.read_fitacf()
    bmdata = {
        'rsep': [],
        'frang': [],
    }
    for rec in data:
        for k, v in bmdata.items():
            bmdata[k].append(rec[k])
        if 'slist' in rec.keys():
            if radar_info['maxrg'] < rec['slist'].max():
                radar_info['maxrg'] = rec['slist'].max() + 5
    
    for k, v in bmdata.items():
        val = np.unique(v)
        if len(val) > 1:        
            os.makedirs(conversionLogDir, exist_ok=True)
            os.makedirs(multiBeamLogDir, exist_ok=True)
            
            # Log the multiple beams error in the monthly mutli beam def log
            logText = '{fitacfFullFile} has {numBeamDefs} beam definitions - skipping file conversion.\n'.format(fitacfFullFile = in_fname, numBeamDefs = len(val))
            
            with open(multiBeamLogfile, "a+") as fp: 
                fp.write(logText)

            # Log the multiple beams error in this fitACF's conversion log
            with open(conversionLogfile, "a+") as fp: 
                fp.write(logText)

            return MULTIPLE_BEAM_DEFS_ERROR_CODE, MULTIPLE_BEAM_DEFS_ERROR_CODE
        
        bmdata[k] = int(val)

    # Define FOV
    fov = radFov.fov(
        frang=bmdata['frang'], rsep=bmdata['rsep'], site=None, nbeams=int(radar_info['maxbeams']),
        ngates=int(radar_info['maxrg']), bmsep=radar_info['beamsep'], recrise=radar_info['risetime'], siteLat=radar_info['glat'],
        siteLon=radar_info['glon'], siteBore=radar_info['boresight'], siteAlt=radar_info['alt'], siteYear=date.year,
        elevation=None, altitude=300., hop=None, model='IS',
        coords='geo', date_time=date, coord_alt=0., fov_dir='front',
    )

    # Define fields 
    short_flds = 'tfreq', 'noise.sky', 'cp',
    fov_flds = 'mjd', 'beam', 'range', 'lat', 'lon', 
    data_flds = 'p_l', 'v', 'v_e', 'gflg', 
    elv_flds = 'elv', 'elv_low', 'elv_high',

    # Figure out if we have elevation information
    elv_exists = True
    for rec in data:
        if 'elv' not in rec.keys():
            elv_exists = False
    if elv_exists:
        data_flds += elv_flds

    # Set up data storage
    out = {}
    for fld in (fov_flds + data_flds + short_flds):
        out[fld] = []
   
    # Run through each beam record and store 
    for rec in data:
        time = dt.datetime(rec['time.yr'], rec['time.mo'], rec['time.dy'], rec['time.hr'], rec['time.mt'], rec['time.sc'])
        # slist is the list of range gates with backscatter
        if 'slist' not in rec.keys():
            os.makedirs(conversionLogDir, exist_ok=True)
            logText = 'Could not find slist in record {recordTime} - skipping\n'.format(recordTime = time.strftime('%Y-%m-%d %H:%M:%S'))
            with open(conversionLogfile, "a+") as fp: 
                fp.write(logText)

            continue

        # Can't deal with returns outside of FOV
        if rec['slist'].max() >= fov.slantRCenter.shape[1]:
            os.makedirs(conversionLogDir, exist_ok=True)

            # Log returns outside of FOV
            logText = 'Record {recordTime} found to have a max slist of {maxSList} - skipping record/n'.format(recordTime = time.strftime('%Y-%m-%d %H:%M:%S'), maxSList = rec['slist'].max())
            with open(conversionLogfile, "a+") as fp: 
                fp.write(logText)

            continue

        time = dt.datetime(rec['time.yr'], rec['time.mo'], rec['time.dy'], rec['time.hr'], rec['time.mt'], rec['time.sc'])
        one_obj = np.ones(len(rec['slist'])) 
        mjd = jdutil.jd_to_mjd(jdutil.datetime_to_jd(time))
        bmnum = one_obj * rec['bmnum']
        fovi = fov.beams == rec['bmnum']
        out['mjd'] += (one_obj * mjd).tolist()
        out['beam'] += bmnum.tolist()
        out['range'] += fov.slantRCenter[fovi, rec['slist']].tolist()
        out['lat'] += fov.latCenter[fovi, rec['slist']].tolist()
        out['lon'] += fov.lonCenter[fovi, rec['slist']].tolist()

        for fld in data_flds:
            out[fld] += rec[fld].tolist()
        for fld in short_flds:  # expand out to size
            out[fld] += (one_obj * rec[fld]).tolist()

    # Convert to numpy arrays 
    for k, v in out.items():
        out[k] = np.array(v)

    # Calculate beam azimuths assuming 20 degrees elevation
    beam_off = radar_info['beamsep'] * (fov.beams - (radar_info['maxbeams'] - 1) / 2.0)
    el = 15.
    brng = np.zeros(beam_off.shape)
    for ind, beam_off_elzero in enumerate(beam_off):
        brng[ind] = radFov.calcAzOffBore(el, beam_off_elzero, fov_dir=fov.fov_dir) + radar_info['boresight']

    # Pull the fit version out of the fitACF filename
    fit_version = '.'.join(in_fname.split('.')[-3:-1])

    # Load the list of rawacf files used to create the fitacf and netcdf    
    with open(rawacfListFilename, "rb") as fp:
        rawacf_source_files = pickle.load(fp)

    # Once the list of rawacf source files has been loaded, delete the file used to
    # temporarily store that information
    os.system('rm {rawacfListFile}'.format(rawacfListFile = rawacfListFilename))
    
    hdr = {
        'lat': radar_info['glat'],
        'lon': radar_info['glon'],
        'alt': radar_info['alt'],
        'rsep': bmdata['rsep'],
        'maxrg': radar_info['maxrg'],
        'bmsep': radar_info['beamsep'],
        'boresight': radar_info['boresight'],
        'beams': fov.beams,
        'brng_at_15deg_el': brng,
        'fitacf_version': fit_version,
        'rawacf_source': rawacf_source_files
    }
    return out, hdr
  ## Write RRMDA.oda
  try:
    DOMTree = xml.dom.minidom.parse("RRMDA.oda")
    openDaApplication = DOMTree.documentElement
    restartInFile = openDaApplication.getElementsByTagName("restartInFile")
    restartInFileName = restartInFile[0].childNodes[0].nodeValue
    restartOutFile = openDaApplication.getElementsByTagName("restartOutFilePrefix")
    restartOutFilePrefix = restartOutFile[0].childNodes[0].nodeValue
    logger.info('Parsing RRMDA.oda')
    logger.debug('restartOutFilePrefix = %s',restartOutFilePrefix)
    # Change fileName
    date = jdutil.date_to_jd(int(datetime.now().strftime('%Y')),
                             int(datetime.now().strftime('%m')),
                             float(datetime.now().strftime('%d')))
    date = jdutil.jd_to_mjd(date)
    # Convert to matlab datenum
    date = date + 678942
    datenum = "%d" % date
    logger.debug('datenum of today = %d', date)
    restartInFileName = restartOutFilePrefix + datenum + '.zip'
    logger.debug('new restartInFileName = %s',restartInFileName)
    restartInFile[0].childNodes[0].replaceWholeText(restartInFileName)
    file = open('RRMDA.oda','w')
    DOMTree.writexml(file)
    file.close()
  except Exception, e:
    logger.error('Problem parsing and writing RRMDA.oda. Error: %s.',e)
    return 1

  # Write Observations.xml
示例#7
0
def idb2ms(trange,
           modelms=None,
           outpath=None,
           nocreatms=True,
           nowritems=False):
    ''' This is the main routine to convert the EOVSA IDB files to CASA measurement set.
        Parameters
        ----------
        trange:  Finding the IDB files within the given time range. If trange is not a Time() object,
        assume that it is the list of files to read.    
    '''

    if type(trange) == Time:
        filelist = ri.get_trange_files(trange)
    else:
        # If input type is not Time, assume that it is the list of files to read
        filelist = trange

    if not modelms:
        modelms = '/home/user/sjyu/20160531/ms/sun/SUN/SUN_20160531T142234-10m.1s.ms'
    try:
        for f in filelist:
            os.path.exists(f)
    except ValueError:
        print("Some files in filelist are invalid. Aborting...")
    if not outpath:
        # use current directory
        outpath = './'

    for filename in filelist:
        uv = aipy.miriad.UV(filename)

        start_time = 0  # The start and stop times are referenced to ref_time_jd in second
        end_time = 600
        delta_time = 1
        time_steps = (end_time - start_time) / delta_time
        time0 = time.time()

        if 'antlist' in uv.vartable:
            ants = uv['antlist']
            antlist = map(int, ants.split())
        else:
            antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]

        good_idx = np.arange(len(uv['sfreq']))

        ref_time_jd = uv['time']
        ref_time_mjd = jdutil.jd_to_mjd(
            ref_time_jd) * 24. * 3600. + 0.5 * delta_time
        nf = len(good_idx)
        freq = uv['sfreq'][good_idx]
        npol = uv['npol']
        nants = uv['nants']
        sdf = uv['sdf']
        project = uv['proj']
        source_id = uv['source']
        ra, dec = uv['ra'], uv['dec']
        scan_id = uv['scanid']
        nbl = nants * (nants - 1) / 2
        bl2ord = bl_list2(nants)
        npairs = nbl + nants
        flag = np.ones((npol, nf, time_steps, npairs), dtype=bool)
        out = np.zeros((npol, nf, time_steps, npairs),
                       dtype=np.complex64)  # Cross-correlations
        uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float)
        bandedge = get_band_edge(freq)
        nband = len(bandedge) - 1

        uv.rewind()
        l = -1
        for preamble, data in uv.all():
            uvw, t, (i0, j0) = preamble
            i = antlist.index(i0 + 1)
            j = antlist.index(j0 + 1)
            if i > j:
                # Reverse order of indices
                j = antlist.index(i0 + 1)
                i = antlist.index(j0 + 1)
            # Assumes uv['pol'] is one of -5, -6, -7, -8
            k = -5 - uv['pol']
            l += 1
            out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data
            flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask
            if i != j:
                if k == 3:
                    uvwarray[:, l / (npairs * npol),
                             bl2ord[i0,
                                    j0]] = uvw * constants.speed_of_light / 1e9

        nrows = time_steps * npairs
        out = out.reshape(npol, nf, nrows)
        flag = flag.reshape(npol, nf, nrows)
        uvwarray = uvwarray.reshape(3, nrows).swapaxes(0, 1)
        uvwarray = np.tile(uvwarray, (nband, 1))
        sigma = np.ones((nrows, 4), dtype=np.float)
        sigma = np.tile(sigma, (nband, 1))

        print 'IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(
            filename, (time.time() - time0))

        msname = outpath + filename.split('/')[-1] + '.ms'

        if not nocreatms:
            print 'Empty MS {0} created in --- {1:10.2f} seconds ---'.format(
                msname, (time.time() - time0))
        else:
            os.system("rm -fr %s" % msname)
            os.system("cp -r " + " %s" % modelms + " %s" % msname)
            print 'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format(
                msname, (time.time() - time0))

        if not nowritems:
            print '----------------------------------------'
            print "Updating the main table of" '%s' % msname
            print '----------------------------------------'
            tabl = tb(msname, readonly=False)
            for l, bdedge in enumerate(bandedge[:-1]):
                time1 = time.time()
                nchannels = (bandedge[l + 1] - bandedge[l])
                for row in range(nrows):
                    tabl.putcell(
                        'DATA', (row + l * nrows),
                        out[:, bandedge[l]:bandedge[l + 1],
                            row].swapaxes(0, 1))
                    tabl.putcell(
                        'FLAG', (row + l * nrows),
                        flag[:, bandedge[l]:bandedge[l + 1],
                             row].swapaxes(0, 1))
                print '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format(
                    (l + 1),
                    time.time() - time1)
            tabl.putcol('UVW', uvwarray)
            tabl.putcol('SIGMA', sigma)
            tabl.putcol('WEIGHT', 1.0 / sigma**2)
            timearr = np.arange((time_steps), dtype=np.float)
            timearr = timearr.reshape(1, time_steps, 1)
            timearr = np.tile(timearr, (nband, 1, npairs))
            timearr = timearr.reshape(
                nband * npairs * time_steps) + ref_time_mjd
            tabl.putcol('TIME', timearr)
            tabl.putcol('TIME_CENTROID', timearr)
            tabl.close()

            print '----------------------------------------'
            print "Updating the OBSERVATION table of" '%s' % msname
            print '----------------------------------------'
            tabl = tb(msname + '/OBSERVATION', readonly=False)
            tabl.putcol(
                'TIME_RANGE',
                np.asarray([
                    ref_time_mjd - 0.5 * delta_time,
                    ref_time_mjd + end_time - 0.5 * delta_time
                ]).reshape(2, 1).swapaxes(0, 1))
            tabl.putcol('OBSERVER', ['EOVSA team'])
            tabl.close()

            print '----------------------------------------'
            print "Updating the POINTING table of" '%s' % msname
            print '----------------------------------------'
            tabl = tb(msname + '/POINTING', readonly=False)
            timearr = np.arange((time_steps),
                                dtype=np.float).reshape(1, time_steps, 1)
            timearr = np.tile(timearr, (nband, 1, nants))
            timearr = timearr.reshape(
                nband * time_steps * nants) + ref_time_mjd
            tabl.putcol('TIME', timearr)
            tabl.putcol('TIME_ORIGIN', timearr - 0.5 * delta_time)
            direction = tabl.getcol('DIRECTION')
            direction[:, 0, 0] = ra
            direction[:, 0, 1] = dec
            tabl.putcol('DIRECTION', direction)
            target = tabl.getcol('TARGET')
            target[:, 0, 0] = ra
            target[:, 0, 1] = dec
            tabl.putcol('TARGET', target)
            tabl.close()

            print '----------------------------------------'
            print "Updating the SOURCE table of" '%s' % msname
            print '----------------------------------------'
            tabl = tb(msname + '/SOURCE', readonly=False)
            radec = tabl.getcol('DIRECTION')
            radecshape = radec.shape
            radecflatten = radec.flatten()
            radecflatten[0], radecflatten[1] = ra, dec
            tabl.putcol('DIRECTION', radecflatten.reshape(radecshape))
            name = [source_id]
            tabl.putcol('NAME', name)
            tabl.close()

            print '----------------------------------------'
            print "Updating the DATA_DESCRIPTION table of" '%s' % msname
            print '----------------------------------------'
            tabl = tb(msname + '/DATA_DESCRIPTION/', readonly=False)
            pol_id = tabl.getcol('POLARIZATION_ID')
            pol_id *= 0
            tabl.putcol('POLARIZATION_ID', pol_id)
            tabl.close()

            if not nocreatms:
                print '----------------------------------------'
                print "Updating the POLARIZATION table of" '%s' % msname
                print '----------------------------------------'
                tabl = tb(msname + '/POLARIZATION/', readonly=False)
                tabl.removerows(rownrs=np.arange(1, nband, dtype=int))
                tabl.close()

            print '----------------------------------------'
            print "Updating the FIELD table of" '%s' % msname
            print '----------------------------------------'
            tabl = tb(msname + '/FIELD/', readonly=False)
            radec = tabl.getcol('DELAY_DIR')
            radecshape = radec.shape
            radecflatten = radec.flatten()
            radecflatten[0], radecflatten[1] = ra, dec
            tabl.putcol('DELAY_DIR', radecflatten.reshape(radecshape))

            radec = tabl.getcol('PHASE_DIR')
            radecshape = radec.shape
            radecflatten = radec.flatten()
            radecflatten[0], radecflatten[1] = ra, dec
            tabl.putcol('PHASE_DIR', radecflatten.reshape(radecshape))

            radec = tabl.getcol('REFERENCE_DIR')
            radecshape = radec.shape
            radecflatten = radec.flatten()
            radecflatten[0], radecflatten[1] = ra, dec
            tabl.putcol('REFERENCE_DIR', radecflatten.reshape(radecshape))

            name = np.array([source_id],
                            dtype='|S{0}'.format(len(source_id) + 1))
            tabl.putcol('NAME', name)
            tabl.close()

        print 'finished in --- {0:10.2f} seconds ---'.format(time.time() -
                                                             time0)
示例#8
0
def idb2ms(filelist,outpath=None,ms_std=None,nowritems=False):

''' This is the main routine to convert the EOVSA IDB files to CASA measurement set.
    Parameters
    ----------
    filelist: list of input IDB files ['IDB1','IDB2',...]
    outpath: output path
    ms_std: a standard CASA measurement set as a template

    Example
    ----------
    import importeovsa_ipy
    filelist=['/data1/eovsa/fits/IDB/20160524/IDB20160524000518','/data1/eovsa/fits/IDB/20160524/IDB20160524004752']
    ms_std = '/home/user/sjyu/20160531/ms/sun/SUN/SUN_20160531T142234-10m.1s.ms'
    outpath='/home/user/sjyu/20160531/ms/'
    importeovsa_ipy.idb2ms(filelist,outpath=yourpath,ms_std=your_standard_ms)
'''
    # filelist=glob.glob('/Volumes/MyPassport/EOVSA/20160531/data/sun/IDB20160531003520')
    # filelist=[filelist[31]]
    # filelist=filelist[31:34]
    # filelist=['/data1/eovsa/fits/IDB/20160531/IDB20160531144234']

    nocreatms = True
    nowritems = False


    if len(filelist) == 0:
        print 'eovsa_idb2ms: No files input'
        return None

    # Be sure that files exist, and has all of the appropriate elements
    filelist_test, ok_filelist, bad_filelist = valid_miriad_dataset(filelist)
    if len(ok_filelist) == 0:
        print 'eovsa_idb2ms: No valid files input'
        return None

    if ms_std:
        ms_std = '/home/user/sjyu/20160531/ms/sun/SUN/SUN_20160531T142234-10m.1s.ms'

    if outpath:
        outpath='/home/user/sjyu/20160531/ms/'        

    for filename in filelist:     
        uv = aipy.miriad.UV(filename)
        if uv['source'].lower() == 'sun':
            outpath=outpath+'sun/'
            if not os.path.exists(outpath):
                os.mkdir(outpath)            
        else:
            outpath=outpath+'calibrator/' 
            if not os.path.exists(outpath):
                os.mkdir(outpath)               
        uv.rewind()

        start_time = 0 # The start and stop times are referenced to ref_time_jd in second
        end_time=600
        delta_time = 1
        time_steps = (end_time-start_time)/delta_time
        time0 = time.time()

        if 'antlist' in uv.vartable:
            ants = uv['antlist']
            antlist = map(int, ants.split())
        else:
            antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]    

        good_idx = np.arange(len(uv['sfreq']))

        ref_time_jd = uv['time']
        ref_time_mjd = jdutil.jd_to_mjd(ref_time_jd)*24.*3600.+0.5*delta_time
        nf = len(good_idx)
        freq = uv['sfreq'][good_idx]
        npol = uv['npol']
        nants = uv['nants']
        sdf=uv['sdf']
        project= uv['proj']
        source_id = uv['source']
        ra,dec = uv['ra'], uv['dec']
        scan_id = uv['scanid']
        nbl = nants*(nants-1)/2
        bl2ord = bl_list2(nants)
        npairs = nbl+nants
        flag = np.ones((npol,nf,time_steps,npairs), dtype=bool)
        out = np.zeros((npol,nf,time_steps,npairs),dtype=np.complex64)  # Cross-correlations
        uvwarray = np.zeros((3,time_steps,npairs),dtype=np.float)
        bandedge = get_band_edge(freq)
        nband = len(bandedge)-1

        uv.rewind()
        l = -1
        for preamble, data in uv.all():
            uvw, t, (i0,j0) = preamble
            i = antlist.index(i0+1)
            j = antlist.index(j0+1)
            if i > j:
                # Reverse order of indices
                j = antlist.index(i0+1)
                i = antlist.index(j0+1)
            # Assumes uv['pol'] is one of -5, -6, -7, -8
            k = -5 - uv['pol']
            l += 1
            out[k,:,l/(npairs*npol),bl2ord[i0,j0]] = data.data
            flag[k,:,l/(npairs*npol),bl2ord[i0,j0]] = data.mask
            if i!=j:
                if k == 3: 
                    uvwarray[:,l/(npairs*npol),bl2ord[i0,j0]] = uvw*constants.speed_of_light/1e9

        nrows = time_steps*npairs
        out=out.reshape(npol,nf,nrows)
        flag=flag.reshape(npol,nf,nrows)
        uvwarray = uvwarray.reshape(3,nrows).swapaxes(0,1)
        uvwarray = np.tile(uvwarray,(nband,1))
        sigma = np.ones((nrows,4),dtype=np.float)
        sigma = np.tile(sigma,(nband,1))

        print 'IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(filename,(time.time() - time0))


        # nocreatms=True
        # nowritems=True
        # nocreatms=False
        # nowritems=False
        msname = list(filename.split('/')[-1])
        msname.insert(11,'T')
        msname = outpath+source_id.upper()+'_'+''.join(msname[3:])+'-10m.1s.ms'



        if not nocreatms:
            print 'Empty MS {0} created in --- {1:10.2f} seconds ---'.format(msname,(time.time() - time0))
        else:
            os.system("rm -fr %s"%msname)
            os.system("cp -r "+" %s"%ms_std+" %s"%msname)     
            print 'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format(msname,(time.time() - time0))        



        if not nowritems:
            print '----------------------------------------'
            print "Updating the main table of" '%s'%msname
            print '----------------------------------------'
            tabl=tb(msname,readonly=False)
            for l,bdedge in enumerate(bandedge[:-1]):
                time1 = time.time()
                nchannels = (bandedge[l+1]-bandedge[l])
                for row in range(nrows):
                    tabl.putcell('DATA',(row+l*nrows),out[:,bandedge[l]:bandedge[l+1],row].swapaxes(0,1))
                    tabl.putcell('FLAG',(row+l*nrows),flag[:,bandedge[l]:bandedge[l+1],row].swapaxes(0,1))
                print '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format((l+1),time.time() - time1)
            tabl.putcol('UVW',uvwarray)
            tabl.putcol('SIGMA',sigma) 
            tabl.putcol('WEIGHT',1.0/sigma**2) 
            timearr = np.arange((time_steps),dtype=np.float)
            timearr=timearr.reshape(1,time_steps,1)
            timearr=np.tile(timearr,(nband,1,npairs))
            timearr=timearr.reshape(nband*npairs*time_steps)+ref_time_mjd
            tabl.putcol('TIME',timearr)
            tabl.putcol('TIME_CENTROID',timearr)
            tabl.close()

            print '----------------------------------------'
            print "Updating the OBSERVATION table of" '%s'%msname
            print '----------------------------------------'
            tabl=tb(msname+'/OBSERVATION',readonly=False)
            tabl.putcol('TIME_RANGE',np.asarray([ref_time_mjd-0.5*delta_time,ref_time_mjd+end_time-0.5*delta_time]).reshape(2,1).swapaxes(0,1))
            tabl.putcol('OBSERVER',['EOVSA team'])
            tabl.close()

            print '----------------------------------------'
            print "Updating the POINTING table of" '%s'%msname
            print '----------------------------------------'
            tabl=tb(msname+'/POINTING',readonly=False)
            timearr=np.arange((time_steps),dtype=np.float).reshape(1,time_steps,1)
            timearr=np.tile(timearr,(nband,1,nants))
            timearr=timearr.reshape(nband*time_steps*nants)+ref_time_mjd        
            tabl.putcol('TIME',timearr)
            tabl.putcol('TIME_ORIGIN',timearr-0.5*delta_time)
            direction=tabl.getcol('DIRECTION')
            direction[:,0,0]=ra
            direction[:,0,1]=dec
            tabl.putcol('DIRECTION',direction)
            target=tabl.getcol('TARGET')
            target[:,0,0]=ra
            target[:,0,1]=dec        
            tabl.putcol('TARGET',target)
            tabl.close()

            print '----------------------------------------'
            print "Updating the SOURCE table of" '%s'%msname
            print '----------------------------------------'
            tabl=tb(msname+'/SOURCE',readonly=False)
            radec=tabl.getcol('DIRECTION')
            radecshape=radec.shape
            radecflatten=radec.flatten()
            radecflatten[0],radecflatten[1]=ra,dec
            tabl.putcol('DIRECTION',radecflatten.reshape(radecshape))        
            name=[source_id]
            tabl.putcol('NAME',name)
            tabl.close()  


            print '----------------------------------------'
            print "Updating the DATA_DESCRIPTION table of" '%s'%msname
            print '----------------------------------------'
            tabl=tb(msname+'/DATA_DESCRIPTION/',readonly=False)  
            pol_id=tabl.getcol('POLARIZATION_ID')
            pol_id*=0
            tabl.putcol('POLARIZATION_ID',pol_id)
            tabl.close()

            if not nocreatms:
                print '----------------------------------------'
                print "Updating the POLARIZATION table of" '%s'%msname
                print '----------------------------------------'        
                tabl=tb(msname+'/POLARIZATION/',readonly=False)  
                tabl.removerows(rownrs=np.arange(1,nband,dtype=int))
                tabl.close()         

            print '----------------------------------------'
            print "Updating the FIELD table of" '%s'%msname
            print '----------------------------------------'        
            tabl=tb(msname+'/FIELD/',readonly=False) 
            radec=tabl.getcol('DELAY_DIR')
            radecshape=radec.shape
            radecflatten=radec.flatten()
            radecflatten[0],radecflatten[1]=ra,dec
            tabl.putcol('DELAY_DIR',radecflatten.reshape(radecshape))

            radec=tabl.getcol('PHASE_DIR')
            radecshape=radec.shape
            radecflatten=radec.flatten()
            radecflatten[0],radecflatten[1]=ra,dec
            tabl.putcol('PHASE_DIR',radecflatten.reshape(radecshape))

            radec=tabl.getcol('REFERENCE_DIR')
            radecshape=radec.shape
            radecflatten=radec.flatten()
            radecflatten[0],radecflatten[1]=ra,dec
            tabl.putcol('REFERENCE_DIR',radecflatten.reshape(radecshape))

            name=np.array([source_id],dtype='|S{0}'.format(len(source_id)+1))
            tabl.putcol('NAME',name)     
            tabl.close()   

        print 'finished in --- {0:10.2f} seconds ---'.format(time.time() - time0)