def msclearhistory(msfile): from taskinit import tb tb.open(msfile + '/HISTORY', nomodify=False) nrows = tb.nrows() if nrows > 0: tb.removerows(range(nrows)) tb.close()
def concateovsa(msname, msfiles, visprefix='./', doclearcal=True, keep_orig_ms=False, cols2rm=["MODEL_DATA", "CORRECTED_DATA"]): concatvis = visprefix + msname msfiles_ = [] for idx, ll in enumerate(msfiles): if str(ll).endswith('/'): msfiles[idx] = str(ll)[:-1] if doclearcal: print 'Warning: Corrected column in the input ms file will be cleared!!!' for ll in msfiles: clearcal(vis=str(ll), addmodel=True) else: try: tmpdir = visprefix + '/tmp_ms/' if not os.path.exists(tmpdir): os.makedirs(tmpdir) for ll in msfiles: msfile_ = tmpdir + os.path.basename(str(ll)) msfiles_.append(msfile_) split(vis=str(ll), outputvis=msfile_, datacolumn='corrected') clearcal(vis=msfile_, addmodel=True) except: print 'Warning: Corrected column not found in the input ms file.' msfiles_ = msfiles if msfiles_: concat(vis=msfiles_, concatvis=concatvis, timesort=True) else: concat(vis=msfiles, concatvis=concatvis, timesort=True) # Change all observation ids to be the same (zero) tb.open(concatvis + '/OBSERVATION', nomodify=False) nobs = tb.nrows() tim0 = tb.getcell('TIME_RANGE', 0)[0] tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1] tb.removerows([i + 1 for i in range(nobs - 1)]) tb.putcell('TIME_RANGE', 0, [tim0, tim1]) tb.close() tb.open(concatvis, nomodify=False) obsid = tb.getcol('OBSERVATION_ID') newobsid = np.zeros(len(obsid), dtype='int') tb.putcol('OBSERVATION_ID', newobsid) colnames = tb.colnames() for l in range(len(cols2rm)): if cols2rm[l] in colnames: try: tb.removecols(cols2rm[l]) except: pass tb.close() if msfiles_ != [] and msfiles_ != msfiles: for ll in msfiles_: os.system('rm -rf {}'.format(ll)) if not keep_orig_ms: for ll in msfiles: os.system('rm -rf {}'.format(ll))
def ms_clearhistory(msfile): from taskinit import tb tb_history = msfile + '/HISTORY' os.system('cp -r {0} {0}_bk'.format(tb_history)) tb.open(tb_history, nomodify=False) nrows = tb.nrows() if nrows > 0: tb.removerows(range(nrows)) tb.close()
def msclearhistory(msfile): '''Clears history in the a measurement sets file :param msfile: string The name of a measurement sets file :return: ''' tb.open(msfile + '/HISTORY', nomodify=False) nrows = tb.nrows() if nrows > 0: tb.removerows(range(nrows)) tb.close()
def imreg(vis=None, ephem=None, msinfo=None, imagefile=None, timerange=None, reftime=None, fitsfile=None, beamfile=None, offsetfile=None, toTb=None, sclfactor=1.0, verbose=False, p_ang=False, overwrite=True, usephacenter=True, deletehistory=False, subregion=[], docompress=False): ''' main routine to register CASA images Required Inputs: vis: STRING. CASA measurement set from which the image is derived imagefile: STRING or LIST. name of the input CASA image timerange: STRING or LIST. timerange used to generate the CASA image, must have the same length as the input images. Each element should be in CASA standard time format, e.g., '2012/03/03/12:00:00~2012/03/03/13:00:00' Optional Inputs: msinfo: DICTIONARY. CASA MS information, output from read_msinfo. If not provided, generate one from the supplied vis ephem: DICTIONARY. solar ephem, output from read_horizons. If not provided, query JPL Horizons based on time info of the vis (internet connection required) fitsfile: STRING or LIST. name of the output registered fits files reftime: STRING or LIST. Each element should be in CASA standard time format, e.g., '2012/03/03/12:00:00' offsetfile: optionally provide an offset with a series of solar x and y offsets with timestamps toTb: Bool. Convert the default Jy/beam to brightness temperature? sclfactor: scale the image values up by its value (to compensate VLA 20 dB attenuator) verbose: Bool. Show more diagnostic info if True. usephacenter: Bool -- if True, correct for the RA and DEC in the ms file based on solar empheris. Otherwise assume the phasecenter is correctly pointed to the solar disk center (EOVSA case) subregion: Region selection. See 'help par.region' for details. Usage: >>> from suncasa.utils import helioimage2fits as hf >>> hf.imreg(vis='mydata.ms', imagefile='myimage.image', fitsfile='myimage.fits', timerange='2017/08/21/20:21:10~2017/08/21/20:21:18') The output fits file is 'myimage.fits' History: BC (sometime in 2014): function was first wrote, followed by a number of edits by BC and SY BC (2019-07-16): Added checks for stokes parameter. Verified that for converting from Jy/beam to brightness temperature, the convention of 2*k_b*T should always be used. I.e., for unpolarized source, stokes I, RR, LL, XX, YY, etc. in the output CASA images from (t)clean should all have same values of radio intensity (in Jy/beam) and brightness temperature (in K). ''' if deletehistory: ms_clearhistory(vis) if not imagefile: raise ValueError('Please specify input image') if not timerange: raise ValueError('Please specify timerange of the input image') if type(imagefile) == str: imagefile = [imagefile] if type(timerange) == str: timerange = [timerange] if not fitsfile: fitsfile = [img + '.fits' for img in imagefile] if type(fitsfile) == str: fitsfile = [fitsfile] nimg = len(imagefile) if len(timerange) != nimg: raise ValueError( 'Number of input images does not equal to number of timeranges!') if len(fitsfile) != nimg: raise ValueError( 'Number of input images does not equal to number of output fits files!' ) nimg = len(imagefile) if verbose: print(str(nimg) + ' images to process...') if reftime: # use as reference time to find solar disk RA and DEC to register the image, but not the actual timerange associated with the image if type(reftime) == str: reftime = [reftime] * nimg if len(reftime) != nimg: raise ValueError( 'Number of reference times does not match that of input images!' ) helio = ephem_to_helio(vis, ephem=ephem, msinfo=msinfo, reftime=reftime, usephacenter=usephacenter) else: # use the supplied timerange to register the image helio = ephem_to_helio(vis, ephem=ephem, msinfo=msinfo, reftime=timerange, usephacenter=usephacenter) if toTb: (bmajs, bmins, bpas, beamunits, bpaunits) = getbeam(imagefile=imagefile, beamfile=beamfile) for n, img in enumerate(imagefile): if verbose: print('processing image #' + str(n) + ' ' + img) fitsf = fitsfile[n] timeran = timerange[n] # obtain duration of the image as FITS header exptime try: [tbg0, tend0] = timeran.split('~') tbg_d = qa.getvalue(qa.convert(qa.totime(tbg0), 'd'))[0] tend_d = qa.getvalue(qa.convert(qa.totime(tend0), 'd'))[0] tdur_s = (tend_d - tbg_d) * 3600. * 24. dateobs = qa.time(qa.quantity(tbg_d, 'd'), form='fits', prec=10)[0] except: print('Error in converting the input timerange: ' + str(timeran) + '. Proceeding to the next image...') continue hel = helio[n] if not os.path.exists(img): warnings.warn('{} does not existed!'.format(img)) else: if os.path.exists(fitsf) and not overwrite: raise ValueError( 'Specified fits file already exists and overwrite is set to False. Aborting...' ) else: p0 = hel['p0'] tb.open(img + '/logtable', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() ia.open(img) imr = ia.rotate(pa=str(-p0) + 'deg') if subregion is not []: imr = imr.subimage(region=subregion) imr.tofits(fitsf, history=False, overwrite=overwrite) imr.close() imsum = ia.summary() ia.close() ia.done() # construct the standard fits header # RA and DEC of the reference pixel crpix1 and crpix2 (imra, imdec) = (imsum['refval'][0], imsum['refval'][1]) # find out the difference of the image center to the CASA phase center # RA and DEC difference in arcseconds ddec = degrees((imdec - hel['dec_fld'])) * 3600. dra = degrees((imra - hel['ra_fld']) * cos(hel['dec_fld'])) * 3600. # Convert into image heliocentric offsets prad = -radians(hel['p0']) dx = (-dra) * cos(prad) - ddec * sin(prad) dy = (-dra) * sin(prad) + ddec * cos(prad) if offsetfile: try: offset = np.load(offsetfile) except: raise ValueError( 'The specified offsetfile does not exist!') reftimes_d = offset['reftimes_d'] xoffs = offset['xoffs'] yoffs = offset['yoffs'] timg_d = hel['reftime'] ind = bisect.bisect_left(reftimes_d, timg_d) xoff = xoffs[ind - 1] yoff = yoffs[ind - 1] else: xoff = hel['refx'] yoff = hel['refy'] if verbose: print( 'offset of image phase center to visibility phase center (arcsec): dx={0:.2f}, dy={1:.2f}' .format(dx, dy)) print( 'offset of visibility phase center to solar disk center (arcsec): dx={0:.2f}, dy={1:.2f}' .format(xoff, yoff)) (crval1, crval2) = (xoff + dx, yoff + dy) # update the fits header to heliocentric coordinates hdu = pyfits.open(fitsf, mode='update') hdu[0].verify('fix') header = hdu[0].header dshape = hdu[0].data.shape ndim = hdu[0].data.ndim (cdelt1, cdelt2) = (-header['cdelt1'] * 3600., header['cdelt2'] * 3600. ) # Original CDELT1, 2 are for RA and DEC in degrees header['cdelt1'] = cdelt1 header['cdelt2'] = cdelt2 header['cunit1'] = 'arcsec' header['cunit2'] = 'arcsec' header['crval1'] = crval1 header['crval2'] = crval2 header['ctype1'] = 'HPLN-TAN' header['ctype2'] = 'HPLT-TAN' header['date-obs'] = dateobs # begin time of the image if not p_ang: hel['p0'] = 0 try: # this works for pyfits version of CASA 4.7.0 but not CASA 4.6.0 if tdur_s: header.set('exptime', tdur_s) else: header.set('exptime', 1.) header.set('p_angle', hel['p0']) header.set('hgln_obs', 0.) header.set('rsun_ref', sun.constants.radius.value) if sunpyver <= 1: header.set( 'dsun_obs', sun.sunearth_distance(Time(dateobs)).to(u.meter).value) header.set( 'rsun_obs', sun.solar_semidiameter_angular_size( Time(dateobs)).value) header.set( 'hglt_obs', sun.heliographic_solar_center(Time(dateobs))[1].value) else: header.set( 'dsun_obs', sun.earth_distance(Time(dateobs)).to(u.meter).value) header.set('rsun_obs', sun.angular_radius(Time(dateobs)).value) header.set('hglt_obs', sun.L0(Time(dateobs)).value) except: # this works for astropy.io.fits if tdur_s: header.append(('exptime', tdur_s)) else: header.append(('exptime', 1.)) header.append(('p_angle', hel['p0'])) header.append(('hgln_obs', 0.)) header.append(('rsun_ref', sun.constants.radius.value)) if sunpyver <= 1: header.append( ('dsun_obs', sun.sunearth_distance(Time(dateobs)).to( u.meter).value)) header.append(('rsun_obs', sun.solar_semidiameter_angular_size( Time(dateobs)).value)) header.append(('hglt_obs', sun.heliographic_solar_center( Time(dateobs))[1].value)) else: header.append( ('dsun_obs', sun.earth_distance(Time(dateobs)).to(u.meter).value)) header.append( ('rsun_obs', sun.angular_radius(Time(dateobs)).value)) header.append(('hglt_obs', sun.L0(Time(dateobs)).value)) # check if stokes parameter exist exist_stokes = False stokes_mapper = { 'I': 1, 'Q': 2, 'U': 3, 'V': 4, 'RR': -1, 'LL': -2, 'RL': -3, 'LR': -4, 'XX': -5, 'YY': -6, 'XY': -7, 'YX': -8 } if 'CRVAL3' in header.keys(): if header['CTYPE3'] == 'STOKES': stokenum = header['CRVAL3'] exist_stokes = True if 'CRVAL4' in header.keys(): if header['CTYPE4'] == 'STOKES': stokenum = header['CRVAL4'] exist_stokes = True if exist_stokes: if stokenum in stokes_mapper.values(): stokesstr = list(stokes_mapper.keys())[list( stokes_mapper.values()).index(stokenum)] else: print('Stokes parameter {0:d} not recognized'.format( stokenum)) if verbose: print('This image is in Stokes ' + stokesstr) else: print( 'STOKES Information does not seem to exist! Assuming Stokes I' ) stokenum = 1 # intensity units to brightness temperature if toTb: # get restoring beam info bmaj = bmajs[n] bmin = bmins[n] beamunit = beamunits[n] data = hdu[ 0].data # remember the data order is reversed due to the FITS convension keys = list(header.keys()) values = list(header.values()) # which axis is frequency? faxis = keys[values.index('FREQ')][-1] faxis_ind = ndim - int(faxis) # find out the polarization of this image k_b = qa.constants('k')['value'] c_l = qa.constants('c')['value'] # Always use 2*kb for all polarizations const = 2. * k_b / c_l**2 if header['BUNIT'].lower() == 'jy/beam': header['BUNIT'] = 'K' header['BTYPE'] = 'Brightness Temperature' for i in range(dshape[faxis_ind]): nu = header['CRVAL' + faxis] + header['CDELT' + faxis] * ( i + 1 - header['CRPIX' + faxis]) if header['CUNIT' + faxis] == 'KHz': nu *= 1e3 if header['CUNIT' + faxis] == 'MHz': nu *= 1e6 if header['CUNIT' + faxis] == 'GHz': nu *= 1e9 if len(bmaj) > 1: # multiple (per-plane) beams bmajtmp = bmaj[i] bmintmp = bmin[i] else: # one single beam bmajtmp = bmaj[0] bmintmp = bmin[0] if beamunit == 'arcsec': bmaj0 = np.radians(bmajtmp / 3600.) bmin0 = np.radians(bmintmp / 3600.) if beamunit == 'arcmin': bmaj0 = np.radians(bmajtmp / 60.) bmin0 = np.radians(bmintmp / 60.) if beamunit == 'deg': bmaj0 = np.radians(bmajtmp) bmin0 = np.radians(bmintmp) if beamunit == 'rad': bmaj0 = bmajtmp bmin0 = bmintmp beam_area = bmaj0 * bmin0 * np.pi / (4. * log(2.)) factor = const * nu**2 # SI unit jy_to_si = 1e-26 # print(nu/1e9, beam_area, factor) factor2 = sclfactor # if sclfactor: # factor2 = 100. if faxis == '3': data[:, i, :, :] *= jy_to_si / beam_area / factor * factor2 if faxis == '4': data[ i, :, :, :] *= jy_to_si / beam_area / factor * factor2 header = fu.headerfix(header) hdu.flush() hdu.close() if ndim - np.count_nonzero(np.array(dshape) == 1) > 3: docompress = False ''' Caveat: only 1D, 2D, or 3D images are currently supported by the astropy fits compression. If a n-dimensional image data array does not have at least n-3 single-dimensional entries, force docompress to be False ''' print( 'warning: The fits data contains more than 3 non squeezable dimensions. Skipping fits compression..' ) if docompress: fitsftmp = fitsf + ".tmp.fits" os.system("mv {} {}".format(fitsf, fitsftmp)) hdu = pyfits.open(fitsftmp) hdu[0].verify('fix') header = hdu[0].header data = hdu[0].data fu.write_compressed_image_fits(fitsf, data, header, compression_type='RICE_1', quantize_level=4.0) os.system("rm -rf {}".format(fitsftmp)) if deletehistory: ms_restorehistory(vis) return fitsfile
def concateovsa(vis, concatvis, datacolumn='corrected', keep_orig_ms=True, cols2rm="model,corrected", freqtol="", dirtol="", respectname=False, timesort=True, copypointing=True, visweightscale=[], forcesingleephemfield=""): if concatvis[-1] == os.path.sep: concatvis = concatvis[:-1] if os.path.sep not in concatvis: visprefix = './' else: visprefix = os.path.dirname(concatvis) + os.path.sep msfiles = vis msfiles_ = [] for idx, ll in enumerate(msfiles): if str(ll).endswith('/'): msfiles[idx] = str(ll)[:-1] datacolumn = datacolumn.lower() if datacolumn == 'data': print 'DATA columns will be concatenated.' for ll in msfiles: clearcal(vis=str(ll), addmodel=True) elif datacolumn == 'corrected': # try: print 'CORRECTED columns will be concatenated.' tmpdir = os.path.join(visprefix, 'tmp_ms') + os.path.sep if not os.path.exists(tmpdir): os.makedirs(tmpdir) for ll in msfiles: msfile_ = os.path.join(tmpdir, os.path.basename(str(ll))) msfiles_.append(msfile_) split(vis=str(ll), outputvis=msfile_, datacolumn='corrected') clearcal(vis=msfile_, addmodel=True) else: raise ValueError('Please set datacolumn to be "data" or "corrected"!') if msfiles_: concat(vis=msfiles_, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort, copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield) os.system('rm -rf {}'.format(tmpdir)) else: concat(vis=msfiles, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort, copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield) # Change all observation ids to be the same (zero) tb.open(concatvis + '/OBSERVATION', nomodify=False) nobs = tb.nrows() tim0 = tb.getcell('TIME_RANGE', 0)[0] tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1] tb.removerows([i + 1 for i in range(nobs - 1)]) tb.putcell('TIME_RANGE', 0, [tim0, tim1]) tb.close() tb.open(concatvis + '/DATA_DESCRIPTION', nomodify=False) nrows = tb.nrows() pol_id = tb.getcol('POLARIZATION_ID') tb.removerows(np.where(pol_id != 0)[0]) tb.close() tb.open(concatvis, nomodify=False) dd_id = tb.getcol('DATA_DESC_ID') idx_dd_id, = np.where(dd_id >= nrows / 2) dd_id[idx_dd_id] = dd_id[idx_dd_id] - nrows / 2 tb.putcol('DATA_DESC_ID', dd_id) tb.close() tb.open(concatvis + '/FIELD', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() tb.open(concatvis + '/SOURCE', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() tb.open(concatvis, nomodify=False) obsid = tb.getcol('OBSERVATION_ID') newobsid = np.zeros(len(obsid), dtype='int') tb.putcol('OBSERVATION_ID', newobsid) fldid = tb.getcol('FIELD_ID') newfldid = np.zeros(len(fldid), dtype='int') tb.putcol('FIELD_ID', newfldid) colnames = tb.colnames() cols2rm = cols2rm.upper() cols2rm = cols2rm.split(',') for l in range(len(cols2rm)): col = cols2rm[l] + '_DATA' if col in colnames: try: tb.removecols(col) print 'Column {} removed.'.format(col) except: pass tb.close() if msfiles_ != [] and msfiles_ != msfiles: for ll in msfiles_: os.system('rm -rf {}'.format(ll)) if not keep_orig_ms: for ll in msfiles: os.system('rm -rf {}'.format(ll))
def pimporteovsa(idbfiles, ncpu=8, timebin=None, width=None, visprefix=None, nocreatms=False, doconcat=False, modelms=''): casalog.origin('pimporteovsa') # # Initialize the helper class # pdh = ParallelDataHelper("importeovsa", locals()) # # # Validate input and output parameters # try: # pdh.setupIO() # except Exception, instance: # casalog.post('%s' % instance, 'ERROR') # return False # import glob # idbfiles = glob.glob('IDB*_unrot')[:4] # ncpu = 4 # timebin = None # width = None # visprefix = None # nocreatms = False # doconcat = False # modelms = '' # if not (type(ncpu) is int): # casalog.post('ncpu should be an integer') # ncpu = 8 if type(idbfiles) == Time: filelist = ri.get_trange_files(idbfiles) else: # If input type is not Time, assume that it is the list of files to read filelist = idbfiles if type(filelist) == str: filelist = [filelist] for f in filelist: if not os.path.exists(f): casalog.post("Some files in filelist are invalid. Aborting...") # return False if not visprefix: visprefix = './' if not timebin: timebin = '0s' if not width: width = 1 if not modelms: if nocreatms: filename = filelist[0] modelms = ipe.creatms(filename, visprefix) else: if not os.path.exists(modelms): if nocreatms: filename = filelist[0] modelms = ipe.creatms(filename, visprefix) iterable = range(len(filelist)) imppart = partial(importeovsa_iter, filelist, timebin, width, visprefix, nocreatms, modelms) t0 = time.time() casalog.post('Perform importeovsa in parallel ...') pool = mp.Pool(ncpu) res = pool.map(imppart, iterable) pool.close() pool.join() t1 = time.time() timelapse = t1 - t0 print 'It took %f secs to complete' % timelapse results = pd.DataFrame({'succeeded': [], 'msfile': [], 'durtim': []}) for r in res: results = results.append( pd.DataFrame({ 'succeeded': [r[0]], 'msfile': [r[1]], 'durtim': [r[2]] })) if doconcat: msname = list(os.path.basename(filelist[0])) concatvis = visprefix + ''.join(msname) + '-{:d}m.ms'.format( int(results['durtim'].sum())) msfile = results.loc[results['succeeded'] == True, :].msfile.tolist() concat(vis=msfile, concatvis=concatvis, timesort=True) # Change all observation ids to be the same (zero) tb.open(concatvis + '/OBSERVATION', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() tb.open(concatvis, nomodify=False) obsid = tb.getcol('OBSERVATION_ID') newobsid = np.zeros(len(obsid), dtype='int') tb.putcol('OBSERVATION_ID', newobsid) tb.close() for ll in msfile: os.system('rm -rf {}'.format(ll)) return True
def importeovsa_iter(filelist, timebin, width, visprefix, nocreatms, modelms, fileidx): from taskinit import tb, casalog filename = filelist[fileidx] uv = aipy.miriad.UV(filename) # if filename.split('/')[-1][0:3] == 'UDB': # uv_str = uv_hex_rm(uv) # else: # uv_str = uv try: uv.select('antennae', 0, 1, include=True) uv.select('polarization', -5, -5, include=True) times = [] uv.rewind() for preamble, data in uv.all(): uvw, t, (i, j) = preamble times.append(t) uv.select('clear', -1, -1, include=True) times = ipe.jd2mjds(np.asarray(times)) inttime = np.median((times - np.roll(times, 1))[1:]) / 60 time_steps = len(times) durtim = int((times[-1] - times[0]) / 60 + inttime) time0 = time.time() if 'antlist' in uv.vartable: ants = uv['antlist'].replace('\x00', '') antlist = map(int, ants.split()) else: antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] good_idx = np.where(uv['sfreq'] > 0)[0] nf = len(good_idx) npol = uv['npol'] nants = uv['nants'] source_id = uv['source'].replace('\x00', '') sfreq = uv['sfreq'][good_idx] sdf = uv['sdf'][good_idx] ra, dec = uv['ra'], uv['dec'] nbl = nants * (nants - 1) / 2 bl2ord = ipe.bl_list2(nants) npairs = nbl + nants flag = np.ones((npol, nf, time_steps, npairs), dtype=bool) out = np.zeros((npol, nf, time_steps, npairs), dtype=np.complex64) # Cross-correlations uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float) chan_band = ipe.get_band(sfreq=sfreq, sdf=sdf) nband = len(chan_band) uv.rewind() l = -1 for preamble, data in uv.all(): uvw, t, (i0, j0) = preamble i = antlist.index(i0 + 1) j = antlist.index(j0 + 1) if i > j: # Reverse order of indices j = antlist.index(i0 + 1) i = antlist.index(j0 + 1) # Assumes uv['pol'] is one of -5, -6, -7, -8 k = -5 - uv['pol'] l += 1 out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask if i != j: if k == 3: uvwarray[:, l / (npairs * npol), bl2ord[ i0, j0]] = -uvw * constants.speed_of_light / 1e9 nrows = time_steps * npairs out = out.reshape(npol, nf, nrows) flag = flag.reshape(npol, nf, nrows) uvwarray = uvwarray.reshape(3, nrows) uvwarray = np.tile(uvwarray, (1, nband)) sigma = np.ones((4, nrows), dtype=np.float) + 1 sigma = np.tile(sigma, (1, nband)) casalog.post( 'IDB File {0} is readed in --- {1:10.2f} seconds ---'.format( filename, (time.time() - time0))) msname = list(filename.split('/')[-1]) msname = visprefix + ''.join(msname) + '-{:d}m.ms'.format( int((times[-1] - times[0]) / 60 + inttime)) if not nocreatms: modelms = ipe.creatms(filename, visprefix) os.system('mv {} {}'.format(modelms, msname)) else: casalog.post('----------------------------------------') casalog.post('copying standard MS to {0}'.format( msname, (time.time() - time0))) casalog.post('----------------------------------------') os.system("rm -fr %s" % msname) os.system("cp -r " + " %s" % modelms + " %s" % msname) casalog.post( 'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'. format(msname, (time.time() - time0))) tb.open(msname, nomodify=False) casalog.post('----------------------------------------') casalog.post("Updating the main table of" '%s' % msname) casalog.post('----------------------------------------') for l, cband in enumerate(chan_band): time1 = time.time() nchannels = len(cband['cidx']) for row in range(nrows): tb.putcell('DATA', (row + l * nrows), out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) tb.putcell( 'FLAG', (row + l * nrows), flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) casalog.post( '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'. format((l + 1), time.time() - time1)) tb.putcol('UVW', uvwarray) tb.putcol('SIGMA', sigma) tb.putcol('WEIGHT', 1.0 / sigma**2) timearr = times timearr = timearr.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, npairs)) timearr = timearr.reshape(nband * npairs * time_steps) tb.putcol('TIME', timearr) tb.putcol('TIME_CENTROID', timearr) scan_id = tb.getcol('SCAN_NUMBER') scan_id *= 0 tb.putcol('SCAN_NUMBER', scan_id) colnames = tb.colnames() cols2rm = ["MODEL_DATA", "CORRECTED_DATA"] for l in range(len(cols2rm)): if cols2rm[l] in colnames: tb.removecols(cols2rm[l]) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the OBSERVATION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/OBSERVATION', nomodify=False) tb.putcol( 'TIME_RANGE', np.asarray([times[0] - 0.5 * inttime, times[-1] + 0.5 * inttime]).reshape(2, 1)) tb.putcol('OBSERVER', ['EOVSA team']) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the POINTING table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/POINTING', nomodify=False) timearr = times.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, nants)) timearr = timearr.reshape(nband * time_steps * nants) tb.putcol('TIME', timearr) tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time) direction = tb.getcol('DIRECTION') direction[0, 0, :] = ra direction[1, 0, :] = dec tb.putcol('DIRECTION', direction) target = tb.getcol('TARGET') target[0, 0, :] = ra target[1, 0, :] = dec tb.putcol('TARGET', target) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the SOURCE table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/SOURCE', nomodify=False) radec = tb.getcol('DIRECTION') radec[0], radec[1] = ra, dec tb.putcol('DIRECTION', radec) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False) pol_id = tb.getcol('POLARIZATION_ID') pol_id *= 0 tb.putcol('POLARIZATION_ID', pol_id) # spw_id = tb.getcol('SPECTRAL_WINDOW_ID') # spw_id *= 0 # tb.putcol('SPECTRAL_WINDOW_ID', spw_id) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the POLARIZATION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/POLARIZATION/', nomodify=False) tb.removerows(rownrs=np.arange(1, nband, dtype=int)) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the FIELD table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/FIELD/', nomodify=False) delay_dir = tb.getcol('DELAY_DIR') delay_dir[0], delay_dir[1] = ra, dec tb.putcol('DELAY_DIR', delay_dir) phase_dir = tb.getcol('PHASE_DIR') phase_dir[0], phase_dir[1] = ra, dec tb.putcol('PHASE_DIR', phase_dir) reference_dir = tb.getcol('REFERENCE_DIR') reference_dir[0], reference_dir[1] = ra, dec tb.putcol('REFERENCE_DIR', reference_dir) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() # FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME # del out, flag, uvwarray, uv, timearr, sigma # gc.collect() # if not (timebin == '0s' and width == 1): split(vis=msname, outputvis=msname + '.split', datacolumn='data', timebin=timebin, width=width, keepflags=False) os.system('rm -rf {}'.format(msname)) msfile = msname + '.split' else: msfile = msname casalog.post("finished in --- %s seconds ---" % (time.time() - time0)) return [True, msfile, durtim] except: print 'error in processing idb file: ' + filename return [False, '', 0]