def runTest(self): self.modelName = "ThemiTest" self.homeDir = os.path.realpath(os.path.join(os.path.dirname(__file__), "testEnvironment")) # Rename first observation file to have todays date. imomoDir = os.path.join(self.homeDir, "app", self.modelName, "data", "raw", "imomo") date = jdutil.date_to_jd( int(datetime.now().strftime("%Y")), int(datetime.now().strftime("%m")), float(datetime.now().strftime("%d")) ) date = jdutil.jd_to_mjd(date) # Convert to matlab datenum date = date + 678942 datenum = "%d" % date filename = os.path.join(imomoDir, (datenum + "DLoad.csv")) shutil.copy(os.path.join(imomoDir, "736239DLoad.csv"), os.path.join(imomoDir, filename)) # Call to observationsAvailable. self.bol = observationsAvailable(self.homeDir, self.modelName) # AssertTrue self.assertEqual(self.bol, True) # Test false case. shutil.copy(os.path.join(imomoDir, "736238DLoad.csv"), os.path.join(imomoDir, filename)) # Call to observationsAvailable. self.bol = observationsAvailable(self.homeDir, self.modelName) # AssertTrue self.assertEqual(self.bol, False)
def observationsAvailable(homeDirectory, modelName): """ Check if observations are available for the current time step. - Figure out todays date in matlab datenum format to get the filename - If there is a file with todays measurements read it in and figure out if there is discharge data available. If there are discharge observations available return true, else return false. """ date = jdutil.date_to_jd( int(datetime.now().strftime("%Y")), int(datetime.now().strftime("%m")), float(datetime.now().strftime("%d")) ) date = jdutil.jd_to_mjd(date) # Convert to matlab datenum date = date + 678942 datenum = "%d" % date obsDir = os.path.join(homeDirectory, "app", modelName, "data", "raw", "imomo") filename = os.path.join(obsDir, (datenum + "DLoad.csv")) if os.path.exists(filename): # Scan line by line until the second entry (variable ID) of a line matches 25 # (discharge). for line in open(filename): if re.search("^\d+\.*\d*,25,.+", line): return True return False # No match while reading the lines. else: # Return False if there is no new data. return False
def groundStationSec(groundList, startTime, passSec, interNum): year, month, day, hour, minute, sec = startTime jd = jdutil.date_to_jd(year, month, day) mjd = jdutil.jd_to_mjd(jd) print jd, mjd J = len(groundList[0]) silde = [] groundSecList = [] groundfun = range(J) index = 0 found = False while not found: if int(groundList[index][0]) == int(mjd): found = True else: index += 1 for j in range(1, J): x = [ float(groundList[index + i - interNum][0]) for i in range(2 * interNum + 1) ] fx = [ groundList[index + ii - interNum][j] for ii in range(2 * interNum + 1) ] groundfun[j] = lagInterpolation.get_Lxfunc(x, fx) for i in range(passSec): time = mjd + hour / 24. + minute / (24 * 60.) + (sec + i) / (24 * 60 * 60.) silde.append(i) for j in range(1, J): silde.append(groundfun[j](time)) groundSecList.append(silde[i * J:]) print 'ground station position interpolation calculated by second !' return groundSecList
def import_eovsa(filename, outpath=None, nocreatms=False, nowritems=False): uv = aipy.miriad.UV(filename) if uv['source'].lower() == 'sun': outpath = outpath + 'sun/' if not os.path.exists(outpath): os.mkdir(outpath) else: outpath = outpath + 'calibrator/' if not os.path.exists(outpath): os.mkdir(outpath) uv.rewind() start_time = 0 # The start and stop times are referenced to ref_time_jd in second end_time = 600 delta_time = 1 time_steps = (end_time - start_time) / delta_time time0 = time.time() if 'antlist' in uv.vartable: ants = uv['antlist'] antlist = map(int, ants.split()) else: antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] good_idx = np.arange(len(uv['sfreq'])) ref_time_jd = uv['time'] ref_time_mjd = jdutil.jd_to_mjd( ref_time_jd) * 24. * 3600. + 0.5 * delta_time nf = len(good_idx) freq = uv['sfreq'][good_idx] npol = uv['npol'] nants = uv['nants'] sdf = uv['sdf'] project = uv['proj'] source_id = uv['source'] ra, dec = uv['ra'], uv['dec'] scan_id = uv['scanid'] nbl = nants * (nants - 1) / 2 bl2ord = bl_list2(nants) npairs = nbl + nants flag = np.ones((npol, nf, time_steps, npairs), dtype=bool) out = np.zeros((npol, nf, time_steps, npairs), dtype=np.complex64) # Cross-correlations uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float) bandedge = get_band_edge(freq) nband = len(bandedge) - 1 uv.rewind() l = -1 for preamble, data in uv.all(): uvw, t, (i0, j0) = preamble i = antlist.index(i0 + 1) j = antlist.index(j0 + 1) if i > j: # Reverse order of indices j = antlist.index(i0 + 1) i = antlist.index(j0 + 1) # Assumes uv['pol'] is one of -5, -6, -7, -8 k = -5 - uv['pol'] l += 1 out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask if i != j: if k == 3: uvwarray[:, l / (npairs * npol), bl2ord[i0, j0]] = uvw * constants.speed_of_light / 1e9 nrows = time_steps * npairs out = out.reshape(npol, nf, nrows) flag = flag.reshape(npol, nf, nrows) uvwarray = uvwarray.reshape(3, nrows) uvwarray = np.tile(uvwarray, (1, nband)) sigma = np.ones((4, nrows), dtype=np.float) + 1 sigma = np.tile(sigma, (1, nband)) print 'IDB File {0} is readed in --- {1:10.2f} seconds ---'.format( filename, (time.time() - time0)) # nocreatms=True # nowritems=True # nocreatms=False # nowritems=False msname = list(filename.split('/')[-1]) msname.insert(11, 'T') msname = outpath + source_id.upper() + '_' + ''.join( msname[3:]) + '-10m.1s.ms' if not nocreatms: if os.path.exists(msname): os.system("rm -fr %s" % msname) """ Creates an empty measurement set using CASA simulate (sm) tool. """ sm.open(msname) enu = np.reshape(uv['antpos'], (16, 3)) * constants.speed_of_light / 1e9 refpos_wgs84 = me.position('wgs84', '-118.286952892965deg', '37.2331698901026deg', '1207.1339m') lon, lat, rad = [ me.measure(refpos_wgs84, 'itrf')[x]['value'] for x in 'm0', 'm1', 'm2' ] # 3x3 transform matrix. Each row is a normal vector, i.e. the rows are (dE,dN,dU) # ----------- local xyz ------------ xform = np.array([[0, -np.sin(lat), np.cos(lat)], [1, 0, 0], [0, np.cos(lat), np.sin(lat)]]) xyz = enu.dot(xform) # + xyz0[np.newaxis,:] # ----------- global xyz ------------ # xyz0 = rad*np.array([np.cos(lat)*np.cos(lon),np.cos(lat)*np.sin(lon),np.sin(lat)]) # # 3x3 transform matrix. Each row is a normal vector, i.e. the rows are (dE,dN,dU) # xform = np.array([ # [-np.sin(lon),np.cos(lon),0], # [-np.cos(lon)*np.sin(lat),-np.sin(lon)*np.sin(lat),np.cos(lat)], # [np.cos(lat)*np.cos(lon),np.cos(lat)*np.sin(lon),np.sin(lat)] # ]) # xyz = xyz0[np.newaxis,:] + enu.dot(xform) dishdiam = np.full(uv['nants'], 2.1) dishdiam[-3:-1] = 27 dishdiam[-1] = 2.1 station = uv['telescop'] mount = ['ALT-AZ'] * uv['nants'] for l in [8, 9, 10, 12, 13, 14]: mount[l] = 'EQUATORIAL' sm.setconfig(telescopename=station, x=np.asarray(xyz)[:, 0], y=np.asarray(xyz)[:, 1], z=np.asarray(xyz)[:, 2], dishdiameter=dishdiam, mount=mount, antname=['eo' + "{0:02d}".format(l) for l in antlist], padname=station, coordsystem='local', referencelocation=refpos_wgs84) sm.setfield(sourcename=source_id, sourcedirection=me.direction( 'J2000', '{:22.19f}'.format(uv['obsra']) + 'rad', '{:22.19f}'.format(uv['obsdec']) + 'rad')) sm.setfeed(mode='perfect X Y') ref_time = me.epoch( 'tai', '{:20.13f}'.format(jdutil.jd_to_mjd(ref_time_jd)) + 'd') sm.settimes(integrationtime='1s', usehourangle=False, referencetime=ref_time) for l, bdedge in enumerate(bandedge[:-1]): nchannels = (bandedge[l + 1] - bandedge[l]) stokes = 'XX YY XY YX' df = sdf[bandedge[l]] st_freq = freq[bandedge[l]] sm.setspwindow(spwname='band%02d' % (l + 1), freq='{:22.19f}'.format(st_freq) + 'GHz', deltafreq='{:22.19f}'.format(df) + 'GHz', freqresolution='{:22.19f}'.format(df) + 'GHz', nchannels=nchannels, stokes=stokes) nband = len(bandedge) - 1 for bdid in range(nband): sm.observe(source_id, 'band%02d' % (bdid + 1), starttime=start_time, stoptime=end_time, project=project, state_obs_mode='') if sm.done(): print 'Empty MS {0} created in --- {1:10.2f} seconds ---'.format( msname, (time.time() - time0)) else: raise RuntimeError('Failed to create MS. Look at the log file. ' 'Double check you settings.') else: print '----------------------------------------' print 'copying standard MS to {0}'.format(msname, (time.time() - time0)) print '----------------------------------------' os.system("rm -fr %s" % msname) os.system("cp -r " + " %s" % ms_std + " %s" % msname) print 'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format( msname, (time.time() - time0)) if not nowritems: tb.open(msname, nomodify=False) print '----------------------------------------' print "Updating the main table of" '%s' % msname print '----------------------------------------' for l, bdedge in enumerate(bandedge[:-1]): time1 = time.time() nchannels = (bandedge[l + 1] - bandedge[l]) for row in range(nrows): tb.putcell('DATA', (row + l * nrows), out[:, bandedge[l]:bandedge[l + 1], row]) tb.putcell('FLAG', (row + l * nrows), flag[:, bandedge[l]:bandedge[l + 1], row]) print '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format( (l + 1), time.time() - time1) tb.putcol('UVW', uvwarray) tb.putcol('SIGMA', sigma) tb.putcol('WEIGHT', 1.0 / sigma**2) timearr = np.arange((time_steps), dtype=np.float) timearr = timearr.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, npairs)) timearr = timearr.reshape(nband * npairs * time_steps) + ref_time_mjd tb.putcol('TIME', timearr) tb.putcol('TIME_CENTROID', timearr) colnames = tb.colnames() cols2rm = ["MODEL_DATA", "CORRECTED_DATA"] for l in range(len(cols2rm)): if cols2rm[l] in colnames: tb.removecols(cols2rm[l]) tb.close() print '----------------------------------------' print "Updating the OBSERVATION table of" '%s' % msname print '----------------------------------------' tb.open(msname + '/OBSERVATION', nomodify=False) tb.putcol( 'TIME_RANGE', np.asarray([ ref_time_mjd - 0.5 * delta_time, ref_time_mjd + end_time - 0.5 * delta_time ]).reshape(2, 1)) tb.putcol('OBSERVER', ['EOVSA team']) tb.close() print '----------------------------------------' print "Updating the POINTING table of" '%s' % msname print '----------------------------------------' tb.open(msname + '/POINTING', nomodify=False) timearr = np.arange((time_steps), dtype=np.float).reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, nants)) timearr = timearr.reshape(nband * time_steps * nants) + ref_time_mjd tb.putcol('TIME', timearr) tb.putcol('TIME_ORIGIN', timearr - 0.5 * delta_time) direction = tb.getcol('DIRECTION') direction[0, 0, :] = ra direction[1, 0, :] = dec tb.putcol('DIRECTION', direction) target = tb.getcol('TARGET') target[0, 0, :] = ra target[1, 0, :] = dec tb.putcol('TARGET', target) tb.close() print '----------------------------------------' print "Updating the SOURCE table of" '%s' % msname print '----------------------------------------' tb.open(msname + '/SOURCE', nomodify=False) radec = tb.getcol('DIRECTION') radec[0], radec[1] = ra, dec tb.putcol('DIRECTION', radec) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() print '----------------------------------------' print "Updating the DATA_DESCRIPTION table of" '%s' % msname print '----------------------------------------' tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False) pol_id = tb.getcol('POLARIZATION_ID') pol_id *= 0 tb.putcol('POLARIZATION_ID', pol_id) tb.close() print '----------------------------------------' print "Updating the POLARIZATION table of" '%s' % msname print '----------------------------------------' tb.open(msname + '/POLARIZATION/', nomodify=False) tb.removerows(rownrs=np.arange(1, nband, dtype=int)) tb.close() print '----------------------------------------' print "Updating the FIELD table of" '%s' % msname print '----------------------------------------' tb.open(msname + '/FIELD/', nomodify=False) delay_dir = tb.getcol('DELAY_DIR') delay_dir[0], delay_dir[1] = ra, dec tb.putcol('DELAY_DIR', delay_dir) phase_dir = tb.getcol('PHASE_DIR') phase_dir[0], phase_dir[1] = ra, dec tb.putcol('PHASE_DIR', phase_dir) reference_dir = tb.getcol('REFERENCE_DIR') reference_dir[0], reference_dir[1] = ra, dec tb.putcol('REFERENCE_DIR', reference_dir) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() # FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME del out, flag, uvwarray, uv, timearr, sigma gc.collect() # print("finished in --- %s seconds ---" % (time.time() - time0)) return
def convert_fitacf_data(date, in_fname, radar_info): day = in_fname.split('.')[0].split('/')[-1] month = day[:-2] # Keep track of fitACF files that have multiple beam definitions in a # monthly log file multiBeamLogDir = date.strftime(helper.FIT_NET_LOG_DIR) + month multiBeamLogfile = '{dir}/multi_beam_defs_{m}.log'.format(dir = multiBeamLogDir, m = month) # Store conversion info like returns outside FOV, missing slist, etc # for each conversion conversionLogDir = '{dir}/{d}'.format(dir = multiBeamLogDir, d = day) fName = in_fname.split('/')[-1] conversionLogfile = '{dir}/{fit}_to_nc.log'.format(dir = conversionLogDir, fit = fName) # Define the name of the file holding the list of rawACFs used to # create the fitACF rawacfListFilename = '.'.join(in_fname.split('.')[:-1]) + '.rawacfList.txt' SDarn_read = pydarn.SuperDARNRead(in_fname) data = SDarn_read.read_fitacf() bmdata = { 'rsep': [], 'frang': [], } for rec in data: for k, v in bmdata.items(): bmdata[k].append(rec[k]) if 'slist' in rec.keys(): if radar_info['maxrg'] < rec['slist'].max(): radar_info['maxrg'] = rec['slist'].max() + 5 for k, v in bmdata.items(): val = np.unique(v) if len(val) > 1: os.makedirs(conversionLogDir, exist_ok=True) os.makedirs(multiBeamLogDir, exist_ok=True) # Log the multiple beams error in the monthly mutli beam def log logText = '{fitacfFullFile} has {numBeamDefs} beam definitions - skipping file conversion.\n'.format(fitacfFullFile = in_fname, numBeamDefs = len(val)) with open(multiBeamLogfile, "a+") as fp: fp.write(logText) # Log the multiple beams error in this fitACF's conversion log with open(conversionLogfile, "a+") as fp: fp.write(logText) return MULTIPLE_BEAM_DEFS_ERROR_CODE, MULTIPLE_BEAM_DEFS_ERROR_CODE bmdata[k] = int(val) # Define FOV fov = radFov.fov( frang=bmdata['frang'], rsep=bmdata['rsep'], site=None, nbeams=int(radar_info['maxbeams']), ngates=int(radar_info['maxrg']), bmsep=radar_info['beamsep'], recrise=radar_info['risetime'], siteLat=radar_info['glat'], siteLon=radar_info['glon'], siteBore=radar_info['boresight'], siteAlt=radar_info['alt'], siteYear=date.year, elevation=None, altitude=300., hop=None, model='IS', coords='geo', date_time=date, coord_alt=0., fov_dir='front', ) # Define fields short_flds = 'tfreq', 'noise.sky', 'cp', fov_flds = 'mjd', 'beam', 'range', 'lat', 'lon', data_flds = 'p_l', 'v', 'v_e', 'gflg', elv_flds = 'elv', 'elv_low', 'elv_high', # Figure out if we have elevation information elv_exists = True for rec in data: if 'elv' not in rec.keys(): elv_exists = False if elv_exists: data_flds += elv_flds # Set up data storage out = {} for fld in (fov_flds + data_flds + short_flds): out[fld] = [] # Run through each beam record and store for rec in data: time = dt.datetime(rec['time.yr'], rec['time.mo'], rec['time.dy'], rec['time.hr'], rec['time.mt'], rec['time.sc']) # slist is the list of range gates with backscatter if 'slist' not in rec.keys(): os.makedirs(conversionLogDir, exist_ok=True) logText = 'Could not find slist in record {recordTime} - skipping\n'.format(recordTime = time.strftime('%Y-%m-%d %H:%M:%S')) with open(conversionLogfile, "a+") as fp: fp.write(logText) continue # Can't deal with returns outside of FOV if rec['slist'].max() >= fov.slantRCenter.shape[1]: os.makedirs(conversionLogDir, exist_ok=True) # Log returns outside of FOV logText = 'Record {recordTime} found to have a max slist of {maxSList} - skipping record/n'.format(recordTime = time.strftime('%Y-%m-%d %H:%M:%S'), maxSList = rec['slist'].max()) with open(conversionLogfile, "a+") as fp: fp.write(logText) continue time = dt.datetime(rec['time.yr'], rec['time.mo'], rec['time.dy'], rec['time.hr'], rec['time.mt'], rec['time.sc']) one_obj = np.ones(len(rec['slist'])) mjd = jdutil.jd_to_mjd(jdutil.datetime_to_jd(time)) bmnum = one_obj * rec['bmnum'] fovi = fov.beams == rec['bmnum'] out['mjd'] += (one_obj * mjd).tolist() out['beam'] += bmnum.tolist() out['range'] += fov.slantRCenter[fovi, rec['slist']].tolist() out['lat'] += fov.latCenter[fovi, rec['slist']].tolist() out['lon'] += fov.lonCenter[fovi, rec['slist']].tolist() for fld in data_flds: out[fld] += rec[fld].tolist() for fld in short_flds: # expand out to size out[fld] += (one_obj * rec[fld]).tolist() # Convert to numpy arrays for k, v in out.items(): out[k] = np.array(v) # Calculate beam azimuths assuming 20 degrees elevation beam_off = radar_info['beamsep'] * (fov.beams - (radar_info['maxbeams'] - 1) / 2.0) el = 15. brng = np.zeros(beam_off.shape) for ind, beam_off_elzero in enumerate(beam_off): brng[ind] = radFov.calcAzOffBore(el, beam_off_elzero, fov_dir=fov.fov_dir) + radar_info['boresight'] # Pull the fit version out of the fitACF filename fit_version = '.'.join(in_fname.split('.')[-3:-1]) # Load the list of rawacf files used to create the fitacf and netcdf with open(rawacfListFilename, "rb") as fp: rawacf_source_files = pickle.load(fp) # Once the list of rawacf source files has been loaded, delete the file used to # temporarily store that information os.system('rm {rawacfListFile}'.format(rawacfListFile = rawacfListFilename)) hdr = { 'lat': radar_info['glat'], 'lon': radar_info['glon'], 'alt': radar_info['alt'], 'rsep': bmdata['rsep'], 'maxrg': radar_info['maxrg'], 'bmsep': radar_info['beamsep'], 'boresight': radar_info['boresight'], 'beams': fov.beams, 'brng_at_15deg_el': brng, 'fitacf_version': fit_version, 'rawacf_source': rawacf_source_files } return out, hdr
## Write RRMDA.oda try: DOMTree = xml.dom.minidom.parse("RRMDA.oda") openDaApplication = DOMTree.documentElement restartInFile = openDaApplication.getElementsByTagName("restartInFile") restartInFileName = restartInFile[0].childNodes[0].nodeValue restartOutFile = openDaApplication.getElementsByTagName("restartOutFilePrefix") restartOutFilePrefix = restartOutFile[0].childNodes[0].nodeValue logger.info('Parsing RRMDA.oda') logger.debug('restartOutFilePrefix = %s',restartOutFilePrefix) # Change fileName date = jdutil.date_to_jd(int(datetime.now().strftime('%Y')), int(datetime.now().strftime('%m')), float(datetime.now().strftime('%d'))) date = jdutil.jd_to_mjd(date) # Convert to matlab datenum date = date + 678942 datenum = "%d" % date logger.debug('datenum of today = %d', date) restartInFileName = restartOutFilePrefix + datenum + '.zip' logger.debug('new restartInFileName = %s',restartInFileName) restartInFile[0].childNodes[0].replaceWholeText(restartInFileName) file = open('RRMDA.oda','w') DOMTree.writexml(file) file.close() except Exception, e: logger.error('Problem parsing and writing RRMDA.oda. Error: %s.',e) return 1 # Write Observations.xml
def idb2ms(trange, modelms=None, outpath=None, nocreatms=True, nowritems=False): ''' This is the main routine to convert the EOVSA IDB files to CASA measurement set. Parameters ---------- trange: Finding the IDB files within the given time range. If trange is not a Time() object, assume that it is the list of files to read. ''' if type(trange) == Time: filelist = ri.get_trange_files(trange) else: # If input type is not Time, assume that it is the list of files to read filelist = trange if not modelms: modelms = '/home/user/sjyu/20160531/ms/sun/SUN/SUN_20160531T142234-10m.1s.ms' try: for f in filelist: os.path.exists(f) except ValueError: print("Some files in filelist are invalid. Aborting...") if not outpath: # use current directory outpath = './' for filename in filelist: uv = aipy.miriad.UV(filename) start_time = 0 # The start and stop times are referenced to ref_time_jd in second end_time = 600 delta_time = 1 time_steps = (end_time - start_time) / delta_time time0 = time.time() if 'antlist' in uv.vartable: ants = uv['antlist'] antlist = map(int, ants.split()) else: antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] good_idx = np.arange(len(uv['sfreq'])) ref_time_jd = uv['time'] ref_time_mjd = jdutil.jd_to_mjd( ref_time_jd) * 24. * 3600. + 0.5 * delta_time nf = len(good_idx) freq = uv['sfreq'][good_idx] npol = uv['npol'] nants = uv['nants'] sdf = uv['sdf'] project = uv['proj'] source_id = uv['source'] ra, dec = uv['ra'], uv['dec'] scan_id = uv['scanid'] nbl = nants * (nants - 1) / 2 bl2ord = bl_list2(nants) npairs = nbl + nants flag = np.ones((npol, nf, time_steps, npairs), dtype=bool) out = np.zeros((npol, nf, time_steps, npairs), dtype=np.complex64) # Cross-correlations uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float) bandedge = get_band_edge(freq) nband = len(bandedge) - 1 uv.rewind() l = -1 for preamble, data in uv.all(): uvw, t, (i0, j0) = preamble i = antlist.index(i0 + 1) j = antlist.index(j0 + 1) if i > j: # Reverse order of indices j = antlist.index(i0 + 1) i = antlist.index(j0 + 1) # Assumes uv['pol'] is one of -5, -6, -7, -8 k = -5 - uv['pol'] l += 1 out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask if i != j: if k == 3: uvwarray[:, l / (npairs * npol), bl2ord[i0, j0]] = uvw * constants.speed_of_light / 1e9 nrows = time_steps * npairs out = out.reshape(npol, nf, nrows) flag = flag.reshape(npol, nf, nrows) uvwarray = uvwarray.reshape(3, nrows).swapaxes(0, 1) uvwarray = np.tile(uvwarray, (nband, 1)) sigma = np.ones((nrows, 4), dtype=np.float) sigma = np.tile(sigma, (nband, 1)) print 'IDB File {0} is readed in --- {1:10.2f} seconds ---'.format( filename, (time.time() - time0)) msname = outpath + filename.split('/')[-1] + '.ms' if not nocreatms: print 'Empty MS {0} created in --- {1:10.2f} seconds ---'.format( msname, (time.time() - time0)) else: os.system("rm -fr %s" % msname) os.system("cp -r " + " %s" % modelms + " %s" % msname) print 'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format( msname, (time.time() - time0)) if not nowritems: print '----------------------------------------' print "Updating the main table of" '%s' % msname print '----------------------------------------' tabl = tb(msname, readonly=False) for l, bdedge in enumerate(bandedge[:-1]): time1 = time.time() nchannels = (bandedge[l + 1] - bandedge[l]) for row in range(nrows): tabl.putcell( 'DATA', (row + l * nrows), out[:, bandedge[l]:bandedge[l + 1], row].swapaxes(0, 1)) tabl.putcell( 'FLAG', (row + l * nrows), flag[:, bandedge[l]:bandedge[l + 1], row].swapaxes(0, 1)) print '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format( (l + 1), time.time() - time1) tabl.putcol('UVW', uvwarray) tabl.putcol('SIGMA', sigma) tabl.putcol('WEIGHT', 1.0 / sigma**2) timearr = np.arange((time_steps), dtype=np.float) timearr = timearr.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, npairs)) timearr = timearr.reshape( nband * npairs * time_steps) + ref_time_mjd tabl.putcol('TIME', timearr) tabl.putcol('TIME_CENTROID', timearr) tabl.close() print '----------------------------------------' print "Updating the OBSERVATION table of" '%s' % msname print '----------------------------------------' tabl = tb(msname + '/OBSERVATION', readonly=False) tabl.putcol( 'TIME_RANGE', np.asarray([ ref_time_mjd - 0.5 * delta_time, ref_time_mjd + end_time - 0.5 * delta_time ]).reshape(2, 1).swapaxes(0, 1)) tabl.putcol('OBSERVER', ['EOVSA team']) tabl.close() print '----------------------------------------' print "Updating the POINTING table of" '%s' % msname print '----------------------------------------' tabl = tb(msname + '/POINTING', readonly=False) timearr = np.arange((time_steps), dtype=np.float).reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, nants)) timearr = timearr.reshape( nband * time_steps * nants) + ref_time_mjd tabl.putcol('TIME', timearr) tabl.putcol('TIME_ORIGIN', timearr - 0.5 * delta_time) direction = tabl.getcol('DIRECTION') direction[:, 0, 0] = ra direction[:, 0, 1] = dec tabl.putcol('DIRECTION', direction) target = tabl.getcol('TARGET') target[:, 0, 0] = ra target[:, 0, 1] = dec tabl.putcol('TARGET', target) tabl.close() print '----------------------------------------' print "Updating the SOURCE table of" '%s' % msname print '----------------------------------------' tabl = tb(msname + '/SOURCE', readonly=False) radec = tabl.getcol('DIRECTION') radecshape = radec.shape radecflatten = radec.flatten() radecflatten[0], radecflatten[1] = ra, dec tabl.putcol('DIRECTION', radecflatten.reshape(radecshape)) name = [source_id] tabl.putcol('NAME', name) tabl.close() print '----------------------------------------' print "Updating the DATA_DESCRIPTION table of" '%s' % msname print '----------------------------------------' tabl = tb(msname + '/DATA_DESCRIPTION/', readonly=False) pol_id = tabl.getcol('POLARIZATION_ID') pol_id *= 0 tabl.putcol('POLARIZATION_ID', pol_id) tabl.close() if not nocreatms: print '----------------------------------------' print "Updating the POLARIZATION table of" '%s' % msname print '----------------------------------------' tabl = tb(msname + '/POLARIZATION/', readonly=False) tabl.removerows(rownrs=np.arange(1, nband, dtype=int)) tabl.close() print '----------------------------------------' print "Updating the FIELD table of" '%s' % msname print '----------------------------------------' tabl = tb(msname + '/FIELD/', readonly=False) radec = tabl.getcol('DELAY_DIR') radecshape = radec.shape radecflatten = radec.flatten() radecflatten[0], radecflatten[1] = ra, dec tabl.putcol('DELAY_DIR', radecflatten.reshape(radecshape)) radec = tabl.getcol('PHASE_DIR') radecshape = radec.shape radecflatten = radec.flatten() radecflatten[0], radecflatten[1] = ra, dec tabl.putcol('PHASE_DIR', radecflatten.reshape(radecshape)) radec = tabl.getcol('REFERENCE_DIR') radecshape = radec.shape radecflatten = radec.flatten() radecflatten[0], radecflatten[1] = ra, dec tabl.putcol('REFERENCE_DIR', radecflatten.reshape(radecshape)) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tabl.putcol('NAME', name) tabl.close() print 'finished in --- {0:10.2f} seconds ---'.format(time.time() - time0)
def idb2ms(filelist,outpath=None,ms_std=None,nowritems=False): ''' This is the main routine to convert the EOVSA IDB files to CASA measurement set. Parameters ---------- filelist: list of input IDB files ['IDB1','IDB2',...] outpath: output path ms_std: a standard CASA measurement set as a template Example ---------- import importeovsa_ipy filelist=['/data1/eovsa/fits/IDB/20160524/IDB20160524000518','/data1/eovsa/fits/IDB/20160524/IDB20160524004752'] ms_std = '/home/user/sjyu/20160531/ms/sun/SUN/SUN_20160531T142234-10m.1s.ms' outpath='/home/user/sjyu/20160531/ms/' importeovsa_ipy.idb2ms(filelist,outpath=yourpath,ms_std=your_standard_ms) ''' # filelist=glob.glob('/Volumes/MyPassport/EOVSA/20160531/data/sun/IDB20160531003520') # filelist=[filelist[31]] # filelist=filelist[31:34] # filelist=['/data1/eovsa/fits/IDB/20160531/IDB20160531144234'] nocreatms = True nowritems = False if len(filelist) == 0: print 'eovsa_idb2ms: No files input' return None # Be sure that files exist, and has all of the appropriate elements filelist_test, ok_filelist, bad_filelist = valid_miriad_dataset(filelist) if len(ok_filelist) == 0: print 'eovsa_idb2ms: No valid files input' return None if ms_std: ms_std = '/home/user/sjyu/20160531/ms/sun/SUN/SUN_20160531T142234-10m.1s.ms' if outpath: outpath='/home/user/sjyu/20160531/ms/' for filename in filelist: uv = aipy.miriad.UV(filename) if uv['source'].lower() == 'sun': outpath=outpath+'sun/' if not os.path.exists(outpath): os.mkdir(outpath) else: outpath=outpath+'calibrator/' if not os.path.exists(outpath): os.mkdir(outpath) uv.rewind() start_time = 0 # The start and stop times are referenced to ref_time_jd in second end_time=600 delta_time = 1 time_steps = (end_time-start_time)/delta_time time0 = time.time() if 'antlist' in uv.vartable: ants = uv['antlist'] antlist = map(int, ants.split()) else: antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] good_idx = np.arange(len(uv['sfreq'])) ref_time_jd = uv['time'] ref_time_mjd = jdutil.jd_to_mjd(ref_time_jd)*24.*3600.+0.5*delta_time nf = len(good_idx) freq = uv['sfreq'][good_idx] npol = uv['npol'] nants = uv['nants'] sdf=uv['sdf'] project= uv['proj'] source_id = uv['source'] ra,dec = uv['ra'], uv['dec'] scan_id = uv['scanid'] nbl = nants*(nants-1)/2 bl2ord = bl_list2(nants) npairs = nbl+nants flag = np.ones((npol,nf,time_steps,npairs), dtype=bool) out = np.zeros((npol,nf,time_steps,npairs),dtype=np.complex64) # Cross-correlations uvwarray = np.zeros((3,time_steps,npairs),dtype=np.float) bandedge = get_band_edge(freq) nband = len(bandedge)-1 uv.rewind() l = -1 for preamble, data in uv.all(): uvw, t, (i0,j0) = preamble i = antlist.index(i0+1) j = antlist.index(j0+1) if i > j: # Reverse order of indices j = antlist.index(i0+1) i = antlist.index(j0+1) # Assumes uv['pol'] is one of -5, -6, -7, -8 k = -5 - uv['pol'] l += 1 out[k,:,l/(npairs*npol),bl2ord[i0,j0]] = data.data flag[k,:,l/(npairs*npol),bl2ord[i0,j0]] = data.mask if i!=j: if k == 3: uvwarray[:,l/(npairs*npol),bl2ord[i0,j0]] = uvw*constants.speed_of_light/1e9 nrows = time_steps*npairs out=out.reshape(npol,nf,nrows) flag=flag.reshape(npol,nf,nrows) uvwarray = uvwarray.reshape(3,nrows).swapaxes(0,1) uvwarray = np.tile(uvwarray,(nband,1)) sigma = np.ones((nrows,4),dtype=np.float) sigma = np.tile(sigma,(nband,1)) print 'IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(filename,(time.time() - time0)) # nocreatms=True # nowritems=True # nocreatms=False # nowritems=False msname = list(filename.split('/')[-1]) msname.insert(11,'T') msname = outpath+source_id.upper()+'_'+''.join(msname[3:])+'-10m.1s.ms' if not nocreatms: print 'Empty MS {0} created in --- {1:10.2f} seconds ---'.format(msname,(time.time() - time0)) else: os.system("rm -fr %s"%msname) os.system("cp -r "+" %s"%ms_std+" %s"%msname) print 'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format(msname,(time.time() - time0)) if not nowritems: print '----------------------------------------' print "Updating the main table of" '%s'%msname print '----------------------------------------' tabl=tb(msname,readonly=False) for l,bdedge in enumerate(bandedge[:-1]): time1 = time.time() nchannels = (bandedge[l+1]-bandedge[l]) for row in range(nrows): tabl.putcell('DATA',(row+l*nrows),out[:,bandedge[l]:bandedge[l+1],row].swapaxes(0,1)) tabl.putcell('FLAG',(row+l*nrows),flag[:,bandedge[l]:bandedge[l+1],row].swapaxes(0,1)) print '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format((l+1),time.time() - time1) tabl.putcol('UVW',uvwarray) tabl.putcol('SIGMA',sigma) tabl.putcol('WEIGHT',1.0/sigma**2) timearr = np.arange((time_steps),dtype=np.float) timearr=timearr.reshape(1,time_steps,1) timearr=np.tile(timearr,(nband,1,npairs)) timearr=timearr.reshape(nband*npairs*time_steps)+ref_time_mjd tabl.putcol('TIME',timearr) tabl.putcol('TIME_CENTROID',timearr) tabl.close() print '----------------------------------------' print "Updating the OBSERVATION table of" '%s'%msname print '----------------------------------------' tabl=tb(msname+'/OBSERVATION',readonly=False) tabl.putcol('TIME_RANGE',np.asarray([ref_time_mjd-0.5*delta_time,ref_time_mjd+end_time-0.5*delta_time]).reshape(2,1).swapaxes(0,1)) tabl.putcol('OBSERVER',['EOVSA team']) tabl.close() print '----------------------------------------' print "Updating the POINTING table of" '%s'%msname print '----------------------------------------' tabl=tb(msname+'/POINTING',readonly=False) timearr=np.arange((time_steps),dtype=np.float).reshape(1,time_steps,1) timearr=np.tile(timearr,(nband,1,nants)) timearr=timearr.reshape(nband*time_steps*nants)+ref_time_mjd tabl.putcol('TIME',timearr) tabl.putcol('TIME_ORIGIN',timearr-0.5*delta_time) direction=tabl.getcol('DIRECTION') direction[:,0,0]=ra direction[:,0,1]=dec tabl.putcol('DIRECTION',direction) target=tabl.getcol('TARGET') target[:,0,0]=ra target[:,0,1]=dec tabl.putcol('TARGET',target) tabl.close() print '----------------------------------------' print "Updating the SOURCE table of" '%s'%msname print '----------------------------------------' tabl=tb(msname+'/SOURCE',readonly=False) radec=tabl.getcol('DIRECTION') radecshape=radec.shape radecflatten=radec.flatten() radecflatten[0],radecflatten[1]=ra,dec tabl.putcol('DIRECTION',radecflatten.reshape(radecshape)) name=[source_id] tabl.putcol('NAME',name) tabl.close() print '----------------------------------------' print "Updating the DATA_DESCRIPTION table of" '%s'%msname print '----------------------------------------' tabl=tb(msname+'/DATA_DESCRIPTION/',readonly=False) pol_id=tabl.getcol('POLARIZATION_ID') pol_id*=0 tabl.putcol('POLARIZATION_ID',pol_id) tabl.close() if not nocreatms: print '----------------------------------------' print "Updating the POLARIZATION table of" '%s'%msname print '----------------------------------------' tabl=tb(msname+'/POLARIZATION/',readonly=False) tabl.removerows(rownrs=np.arange(1,nband,dtype=int)) tabl.close() print '----------------------------------------' print "Updating the FIELD table of" '%s'%msname print '----------------------------------------' tabl=tb(msname+'/FIELD/',readonly=False) radec=tabl.getcol('DELAY_DIR') radecshape=radec.shape radecflatten=radec.flatten() radecflatten[0],radecflatten[1]=ra,dec tabl.putcol('DELAY_DIR',radecflatten.reshape(radecshape)) radec=tabl.getcol('PHASE_DIR') radecshape=radec.shape radecflatten=radec.flatten() radecflatten[0],radecflatten[1]=ra,dec tabl.putcol('PHASE_DIR',radecflatten.reshape(radecshape)) radec=tabl.getcol('REFERENCE_DIR') radecshape=radec.shape radecflatten=radec.flatten() radecflatten[0],radecflatten[1]=ra,dec tabl.putcol('REFERENCE_DIR',radecflatten.reshape(radecshape)) name=np.array([source_id],dtype='|S{0}'.format(len(source_id)+1)) tabl.putcol('NAME',name) tabl.close() print 'finished in --- {0:10.2f} seconds ---'.format(time.time() - time0)