def makenoise (): # make noise images addnoise(); imager.make_image(channelize=1,dirty_image="$OUTFILE.noisecube.fits",npix=256,wprojplanes=0,stokes="I",column="MODEL_DATA"); imager.make_image(dirty_image="$OUTFILE.noise.fits",npix=256,wprojplanes=0,stokes="I",column="MODEL_DATA"); noise = pyfits.open(II("$OUTFILE.noise.fits"))[0].data.std(); info(">>> maximum noise value is %.2f uJy"%(noise*1e+6));
def image(msname='$MS', lsmname='$LSM', use_imager='LWIMAGER', restore=False, options={}, **kw): """ Images MS""" imager.cellsize = '1arcsec' if restore: #ms.copycol() # Copy content of DATA to CORRECTED_DATA if making clean map options['data'] = 'CORRECTED_DATA' # make sure to image corrected data if NCHAN > 1: start = 1 step = CHANNELIZE or NCHAN temp = dict(nchan=NCHAN, img_nchan=(NCHAN - 1) / step, chanstart=0, img_chanstart=1, chanstep=1, img_chanstep=step) options.update(temp) imager.make_image(restore=restore, column=COLUMN, restore_lsm=False, **options)
def simulate_imaging_bd_3c147 (hiresms=None, loresms=None, inputcolumn="DATA", outputcolumn="CORRECTED_DATA", dtime=None, dfreq=None): """ The function make use of baseline dependent averaging on the 3C147 real data observed on the 2013/01/27/01:00:28.5, at declination 49.51.07.23356, ra 05:42:36.137916 NB: We have removed the fictive baseline of index 7 """ # make an instance of the class containing the method for bd-averaging mshi = MSResampler.MSResampler(hiresms+"/", column=inputcolumn) # BD-averaging, giving the integration time of the shortest baseline, dtime and # the number of uv-frequency bins dfreq to average psh=2 qsh=8 arrays = mshi.bd_averaging (dtime,dfreq,psh,qsh) # arrays is of size (p,q,datacom,flagrowpq,weightpq) # take the number of time bins of the longest baseline and make low res timeslots MSResampler.save_visibility_arrays (loresms,arrays,column=outputcolumn) imager.npix= 2048#1024#2048 imager.cellsize = "2arcsec" imager.stokes = "I" imager.weight = "natural" imager.wprojplanes = 128 #cleaning options imager.niter = 1000 imager.threshold = "5mJy" imager.CLEAN_ALGORITHM = "csclean" imager.make_image(msname = loresms, column = outputcolumn, restore = True, dirty = False, weight = "natural");
def makenoise(): # make noise images addnoise() imager.make_image(channelize=1, dirty_image="$OUTFILE.noisecube.fits", npix=256, wprojplanes=0, stokes="I", column="MODEL_DATA") imager.make_image(dirty_image="$OUTFILE.noise.fits", npix=256, wprojplanes=0, stokes="I", column="MODEL_DATA") noise = pyfits.open(II("$OUTFILE.noise.fits"))[0].data.std() info(">>> maximum noise value is %.2f uJy" % (noise * 1e+6))
def sim_now(): radius1 = np.arange(0., 100., 10.) #radius1 = np.arange(0.,3,0.3) Flux_array1 = np.zeros(len(radius1)) lores1 = "MeerKATlores.MS_p0_p0" imager.cellsize = "10.arcsec" imager.npix = 512 imager.niter = 0 imager.CLEAN_ALGORITHM = "csclean" for k in range(len(radius1)): options = {} options['gridded_sky.grid_m0'] = radius1[k] options['ms_sel.msname'] = lores1 mqt.run("turbo-sim.py", job="_tdl_job_1_simulate_MS", config="tdlconf.profiles1", section="Sim_source_radius", options=options) center_min = -45 * 60 + radius1[k] center_deg = math.ceil(center_min / 60) center_min = abs(center_min - center_deg * 60) imager.make_image(msname=lores1, column='CORRECTED_DATA', phasecenter="j2000,0h0m,%dd%dm" % (center_deg, center_min), restore=False, dirty=True, restore_lsm=False, weight="natural") f1 = np.max(pyfits.open(imager.DIRTY_IMAGE)[0].data[0][0]) Flux_array1[k] = f1 np.save("DATA/Flux1", Flux_array1) np.save("DATA/radius1", radius1)
def simcube (cube,nchan=None,npix=4096,cellsize=".5arcsec",niter=100000,padding=1.5,threshold=".2mJy",predict=True,dirty=False,restore=False,noise=0,resume=False,label=None,column='DATA',channelize=1,wprojplanes=0): if label: v.LABEL = label tab = ms.ms(subtable='SPECTRAL_WINDOW') nchan = nchan or tab.getcol('NUM_CHAN')[0] ms.CHANRANGE = 0,nchan-1; imager.wprojplanes = wprojplanes; imager.IMAGE_CHANNELIZE = 1; if predict: imager.predict_vis(image=cube,padding=padding,copy=False,column=column); if noise > 0: simnoise(addToCol=column,noise=noise) if dirty: info('Weights are%s'%WEIGHTS) for weight in WEIGHTS.split(':'): opts,weight_txt,quals = get_weight_opts(weight) restore.update(opts) dirty_image = II("${OUTFILE}-$weight.dirty.fits") model_image = II("${OUTFILE}-$weight.model.fits") residual_image = II("${OUTFILE}-$weight.residual.fits") restored_image = II("${OUTFILE}-$weight.restored.fits") imager.make_image(dirty=dirty,restore=restore,channelize=channelize,column=column if restore==False else "CORRECTED_DATA",dirty_image=dirty_image,model_image=model_image,residual_image=residual_image,restored_image=restored_image,**opts);
def compute_psf_and_noise (make_psf=True,noise=0,noise_map=True,scale_noise=1.0,add_noise=False,rowchunk=1000000,dirty=True,measure_sdl=False,use_old_noise_map=False,**kw): info("weights are",*WEIGHTS.split(":")) for weight in WEIGHTS.split(":"): opts,weight_txt,quals = get_weight_opts(weight); opts.update(kw) # make PSF image if make_psf: psfimage = II('$OUTFILE-$weight-psf.fits') psf_opts = opts if DBL_PSF: try : psf_opts['npix'] = opts['npix']*2 except KeyError : psf_opts['npix'] = imager.npix * 2 info(' >>>>>>>>>> %d -- %d '%(imager.npix,psf_opts['npix'])) imager.make_image(dirty=dict(data="psf"),dirty_image=psfimage,**psf_opts); # make PSF cross-sections and measure FWHMs rx,ry = measure_psf(psfimage, arcsec_size = 10, #2.5**math.ceil(math.log(5*5)/math.log(5)), savefig = psfimage.replace('.fits','.png'), title =os.path.basename(OUTFILE).split('_')[0]) info(">>> $weight: PSF FWHM $rx by $ry") _writestat("psf_fwhm",(rx,ry,(rx+ry)/2),*quals); if noise_map: info(' >>> Making noise map') if add_noise: simnoise(rowchunk=rowchunk,scale_noise=scale_noise,noise=noise) noiseimage = II('$OUTFILE-$weight-noise.fits') opts.update(kw) make_noise_map = False if use_old_noise_map: if os.path.exists(noiseimage): info('Using existing noise map: $noiseimage') else: make_noise_map = True if make_noise_map: imager.make_image(dirty=dirty,column='MODEL_DATA',dirty_image=noiseimage,**opts); noise = pyfits.open(noiseimage)[0].data.std(); info(">>> rms pixel noise (%s) is %g uJy"%(weight_txt,noise*1e+6)); _writestat("pixnoise",noise,*quals); if measure_sdl: r0 = int(SIDELOBES_R0_ARCSEC/SIDELOBES_CELL_ARCSEC); r1 = int(SIDELOBES_R1_ARCSEC/SIDELOBES_CELL_ARCSEC); npix = r1*2; bigpsf = II('$OUTFILE-$weight-bigpsf.fits') imager.make_image(dirty_image=bigpsf,dirty=dict(data="psf",cellsize="%farcsec"%SIDELOBES_CELL_ARCSEC,npix=npix),**opts); data = pyfits.open(bigpsf)[0].data[0,0,...]; radius = numpy.arange(npix)-r1 radius = numpy.sqrt(radius[numpy.newaxis,:]**2+radius[:,numpy.newaxis]**2) mask = (radius<=r1)&(radius>=r0) rms = data[mask].std(); data = None; r0,r1 = r0/3600.,r1/3600.; info(">>> rms far sidelobes (%s) is %g (%.2f<=r<=%.2fdeg"%(weight_txt,rms,r0,r1)); _writestat("sidelobe_radius",(r0,r1),*quals); _writestat("sidelobes",rms,*quals); return noise
def main(): parser = argparse.ArgumentParser() parser.add_argument("min_x", type=int, help="The X coordinate of the bottom left of the map.") parser.add_argument("min_z", type=int, help="The Z coordinate of the bottom left of the map.") parser.add_argument("max_x", type=int, help="The width of the map (along X)") parser.add_argument("max_z", type=int, help="The height of the map (along Z)") parser.add_argument("output_file", help="The .ppm file to output the image to") parser.add_argument("data_directory", help="The directory where region files are stored") parser.add_argument("map_type", default='minecraft', help='The type of map to generate' ) #, choices=['heightmap', 'minecraft']) args = parser.parse_args() x_start = math.floor(args.min_x / 16 / 32) z_start = math.floor(args.min_z / 16 / 32) x_end = math.floor(args.max_x / 16 / 32) z_end = math.floor(args.max_z / 16 / 32) if args.map_type not in ['minecraft', 'heightmap' ] and args.map_type[:2] != 'y=': args.map_type = 'minecraft' progress = 0 image = Image.new('RGB', (args.max_x - args.min_x, args.max_z - args.min_z)) # image = [[None for z in range(args.max_x - args.min_x)] for x in range(args.max_z - args.min_z)] for x in range(x_start, x_end + 1): r_x = x * 512 for z in range(z_start, z_end + 1): progress += 1 status = f", region {progress} of {(x_end - x_start + 1) * (z_end - z_start + 1)} " r_z = z * 512 loaded_chunks = {} try: loaded_chunks.update( chunks.read_mca( os.path.join(args.data_directory, f"r.{x}.{z-1}.mca"))) except FileNotFoundError: pass try: loaded_chunks.update( chunks.read_mca( os.path.join(args.data_directory, f"r.{x}.{z}.mca"))) except FileNotFoundError: continue length = 512 if args.min_z > r_z: length -= (args.min_z - r_z) if args.max_z < r_z + 512: length -= (r_z + 512 - args.max_z) width = 512 if args.min_x > r_x: width -= (args.min_x - r_x) if args.max_x < r_x + 512: width -= (r_x + 512 - args.max_x) image_chunk = imager.make_image( loaded_chunks, (width, length), (max(args.min_x, r_x), max(args.min_z, r_z)), mode=args.map_type, status=status) orig_x = max(args.min_x, r_x) orig_z = max(args.min_z, r_z) if len(image_chunk) > 0: for a in range(len(image_chunk[0])): for b in range(len(image_chunk)): # image[orig_z + b - args.min_z][orig_x + a - args.min_x] = image_chunk[b][a]; image.putpixel( (orig_x + a - args.min_x, orig_z + b - args.min_z), image_chunk[b][a]) # print(image) image.save(args.output_file)
def cal_ms(): """cal_ms: runs a calibration loop over the current MS""" # initialize the calibration "step" counter and "label". These are used # to auto-generate output filenames; cal.stefcal() below automatically increments # cal.STEP v.STEP = 0 # set the superglobal LSM variable. See explanation for "v." in the text v.LSM = LSM0 # info(), warn() and abort() are Pyxis functions for writing output to the log info("########## solving for G with initial LSM") # do one stefcal run with the initial model, produce corrected residuals, # do simple flagging on the residuals. See cal.stefcal() for full docs. stefcal.stefcal(stefcal_reset_all=True, output="CORR_RES", restore=False, flag_threshold=(1, .5)) # If we got to this point, then assume things are rolling along fine. # Copy the recipe and config to the destination directory for future reference # (very useful when modifying recipes, so you can keep track of what settings # are responsible for what output!) xo.sh("cp pyxis*.py pyxis*.conf ${mqt.TDLCONFIG} $DESTDIR", shell=True) info( "########## remaking data image, running source finder and updating model" ) # apply previous stefcal solutions to produce a corrected data image. # Make a restored image with 2000 CLEAN iterations. stefcal.stefcal(apply_only=True, output="CORR_DATA", plotvis=False, restore=dict(niter=2000), restore_lsm=False) ## now run pybdsm on the restored image, write output to a new LSM file lsm.pybdsm_search(threshold=5, output=LSM1) # the new sky model becomes the "current" LSM v.LSM = LSM1 # note that in principle this step is not needed (may as well go on directly # to the dE solutions, below), but it is instructive, as it will produce an intermediate-stage # image. But to save time, set restore=False to skip making clean images info("########## recalibrating with new model") stefcal.stefcal(stefcal_reset_all=True, output="CORR_RES", restore=False, flag_threshold=(1, .5)) # if we have a "reference" sky model configured, use it to set dE tags on # sources in our new sky model. Sources with dE tags will have DDE solutions. # The cal.transfer_tags() function sets the specified tag on any source # near enough to a reference model source with that tag. if LSMREF: lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC) info("########## recalibrating with dEs") # re-run stefcal with the new model (and with direction dependent # solutions on dE-tagged sources, if any). Make a corrected residual # image, run CLEAN on it, restore the LSM sources back into the # image stefcal.stefcal(stefcal_reset_all=True, diffgains=True, output="CORR_RES", restore=dict(niter=1000), restore_lsm=True, flag_threshold=(1, .5)) # no reference LSM? then still need to make a restored image from the step above else: imager.make_image(dirty=False, restore=dict(niter=1000), restore_lsm=True) # the resulting image filename is in cal.FULLREST_IMAGE, # add it to our image list file IMAGE_LIST.add("${imager.FULLREST_IMAGE}")
def make_images(): for w, rob in ("natural", 0), ("uniform", 0), ("robust", 0), ("robust", 1): imager.make_image(weight=w, robust=rob, dirty_image="$OUTFILE-$w$rob.fits")
def stefcal(msname="$MS", section="$STEFCAL_SECTION", diffgains=None, apply_only=False, gain_apply_only=False, diffgain_apply_only=False, ifrgain_apply_only=False, diffgain_intervals=None, diffgain_smoothing=None, flag_threshold=None, output="CORR_RES", plotvis="${ms.PLOTVIS}", dirty=True, restore=False, restore_lsm=True, label=None, args=[], options={}, **kws): """Generic function to run a stefcal job. 'section' TDL config file section 'label' will be assigned to the global LABEL for purposes of file naming 'apply_only' if true, will only apply saved solutions 'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE" 'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only 'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above, or T1,T2 for --above T1 --fm-above T2 'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful) 'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help) 'dirty','restore' 'restore_lsm' image output visibilities (passed to imager.make_image above as is) 'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs), can be used to supply extra TDL options extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus overriding settings in the TDL config file. Useful arguments of this kind are e.g.: stefcal_reset_all=True to remove prior gains solutions. """ msname, section, lsm, label, plotvis = interpolate_locals( "msname section lsm label plotvis") makedir(v.DESTDIR) # increment step counter and assign global label if label is not None: v.LABEL = str(label) if type(v.STEP) is int and STEFCAL_STEP_INCR: v.STEP += STEFCAL_STEP_INCR # setup stefcal options and run info("Running stefcal ${step <STEP} ${(<LABEL>)}") # setup args args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} ms_sel.output_column=$STEFCAL_OUTPUT_COLUMN stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s""" % ((1 if diffgains else 0), STEFCAL_TDLOPTS) ] if diffgains: if diffgains is True: diffgains = "=dE" args0.append( "de_subset.subset_enabled=1 de_subset.source_subset=$diffgains") opts = { 'do_output': output, 'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save", 'stefcal_gain1.mode': "apply" if apply_only or gain_apply_only else "solve-save", 'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save", 'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only else "solve-save", 'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save", 'stefcal_gain.table': STEFCAL_GAIN, 'stefcal_gain1.table': STEFCAL_GAIN1, 'stefcal_diffgain.table': STEFCAL_DIFFGAIN, 'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1, 'stefcal_ifr_gain_table': STEFCAL_IFRGAIN, 'stefcal_visualize': False } timesmooth, freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or ( 0, 0) timeint, freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0, 0) opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint opts['stefcal_diffgain.timesmooth'] = timesmooth opts['stefcal_diffgain.freqsmooth'] = freqsmooth # add user-defined args args0 += list(args) opts.update(options) opts.update(kws) # run the job mqt.run(STEFCAL_SCRIPT, STEFCAL_JOBNAME, section=section, args=args0, options=opts) # copy gains if not apply_only: if os.path.exists(STEFCAL_GAIN): std.copy(STEFCAL_GAIN, STEFCAL_GAIN_SAVE) if os.path.exists(STEFCAL_GAIN1): std.copy(STEFCAL_GAIN1, STEFCAL_GAIN1_SAVE) if os.path.exists(STEFCAL_DIFFGAIN): std.copy(STEFCAL_DIFFGAIN, STEFCAL_DIFFGAIN_SAVE) if os.path.exists(STEFCAL_IFRGAIN): std.copy(STEFCAL_IFRGAIN, STEFCAL_IFRGAIN_SAVE) # post-calibration flagging if flag_threshold: if isinstance(flag_threshold, (list, tuple)): t0, t1 = flag_threshold else: t0, t1 = flag_threshold, None ms.flagms("--above %g" % t0, "-f threshold -c") if t1: ms.flagms("--fm-above %g" % t1, "-f fmthreshold -c") # plot residuals if plotvis: info("Plotting visibilities ($plotvis)") ms.PLOTVIS = plotvis ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png") # make images imager.make_image(msname, column=STEFCAL_OUTPUT_COLUMN, dirty=dirty, restore=restore, restore_lsm=restore_lsm)
def cal_ms (): """cal_ms: runs a calibration loop over the current MS""" # initialize the calibration "step" counter and "label". These are used # to auto-generate output filenames; cal.stefcal() below automatically increments # cal.STEP v.STEP = 0 # set the superglobal LSM variable. See explanation for "v." in the text v.LSM = LSM0 # info(), warn() and abort() are Pyxis functions for writing output to the log info("########## solving for G with initial LSM") # do one stefcal run with the initial model, produce corrected residuals, # do simple flagging on the residuals. See cal.stefcal() for full docs. stefcal.stefcal(stefcal_reset_all=True,output="CORR_RES",restore=False,flag_threshold=(1,.5)) # If we got to this point, then assume things are rolling along fine. # Copy the recipe and config to the destination directory for future reference # (very useful when modifying recipes, so you can keep track of what settings # are responsible for what output!) xo.sh("cp pyxis*.py pyxis*.conf ${mqt.TDLCONFIG} $DESTDIR",shell=True) info("########## remaking data image, running source finder and updating model") # apply previous stefcal solutions to produce a corrected data image. # Make a restored image with 2000 CLEAN iterations. stefcal.stefcal(apply_only=True,output="CORR_DATA",plotvis=False, restore=dict(niter=2000),restore_lsm=False) ## now run pybdsm on the restored image, write output to a new LSM file lsm.pybdsm_search(threshold=5,output=LSM1) # the new sky model becomes the "current" LSM v.LSM = LSM1 # note that in principle this step is not needed (may as well go on directly # to the dE solutions, below), but it is instructive, as it will produce an intermediate-stage # image. But to save time, set restore=False to skip making clean images info("########## recalibrating with new model") stefcal.stefcal(stefcal_reset_all=True,output="CORR_RES",restore=False,flag_threshold=(1,.5)); # if we have a "reference" sky model configured, use it to set dE tags on # sources in our new sky model. Sources with dE tags will have DDE solutions. # The cal.transfer_tags() function sets the specified tag on any source # near enough to a reference model source with that tag. if LSMREF: lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC) info("########## recalibrating with dEs") # re-run stefcal with the new model (and with direction dependent # solutions on dE-tagged sources, if any). Make a corrected residual # image, run CLEAN on it, restore the LSM sources back into the # image stefcal.stefcal(stefcal_reset_all=True,diffgains=True, output="CORR_RES",restore=dict(niter=1000), restore_lsm=True,flag_threshold=(1,.5)) # no reference LSM? then still need to make a restored image from the step above else: imager.make_image(dirty=False,restore=dict(niter=1000),restore_lsm=True); # the resulting image filename is in cal.FULLREST_IMAGE, # add it to our image list file IMAGE_LIST.add("${imager.FULLREST_IMAGE}");
def runall(): #alpha_v = 5 #Power low distribution parameter num_sources_v = 2 #how many sources num_cal_sources_v = 1#how many sources in calibration model #fov_v = 3 #degrees #the field of view in degrees #only do stefcal or not skip_LM = True dist = True point_sources = np.array([(1,0,0),(0.2,(1*np.pi)/180,(np.pi*0)/180)]) #sets the pointsources in the sky #contain all the pointsources #point_sources = np.zeros((num_sources_v,3)) #generate flux and positions #point_sources[:,0] = generate_flux(a = alpha_v,num_sources = num_sources_v,plot=False) #point_sources[:,1] = generate_pos(fov = fov_v,num_sources=num_sources_v) #point_sources[:,2] = generate_pos(fov = fov_v,num_sources=num_sources_v) ra0,dec0 = get_field_center() #generate true and calibration sky models from point_sources meqskymodel(point_sources,num_cal_sources=num_cal_sources_v) #simulate complete skymodel --- store in CORRECTED_DATA #sim_function(cal=False) #simulate calibration skymodel --- store in DATA #sim_function(cal=True) #set up imager image_settings() options = image_advanced_settings() #make images of complete sky and calibrated sky model #imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False) #v.CALORNOT = "cal_model" #imager.make_image(column="DATA",dirty=options,restore=False) #v.CALORNOT = '' #determine residual CORRECTED_DATA-DATA and image #residual() #v.CALORNOT = "res" #imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False) #v.CALORNOT = '' #perform LM calibration if not (skip_LM): sim_function(cal=False) cal_function() v.CALORNOT = "cal_app_LM" imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False) v.CALORNOT = '' residual() v.CALORNOT = "cal_res_LM" imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False) v.CALORNOT = '' #perform STEFcal calibration if not (dist): sim_function(cal=False) cal_function(type_cal="STEF") v.CALORNOT = "cal_app_STEF" imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False) v.CALORNOT = '' residual() v.CALORNOT = "cal_res_STEF" imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False) v.CALORNOT = '' else: #perform STEFcal calibration (distilation) sim_function(cal=True,whole=True) sim_function(cal=False) cal_function(type_cal="STEF") v.CALORNOT = "cal_app_STEF" imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False) v.CALORNOT = '' v.CALORNOT = "cal_whole_STEF" imager.make_image(column="DATA",dirty=options,restore=False) v.CALORNOT = '' residual() v.CALORNOT = "cal_res_dist_STEF" imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False) v.CALORNOT = ''
def makecube (npix=512,stokes="I"): imager.make_image(channelize=1,dirty_image="$OUTFILE.cube.fits",npix=npix,wprojplanes=0,stokes=stokes);
def stefcal ( msname="$MS",section="$STEFCAL_SECTION", diffgains=None, apply_only=False, gain_apply_only=False, diffgain_apply_only=False, ifrgain_apply_only=False, diffgain_intervals=None,diffgain_smoothing=None, flag_threshold=None, output="CORR_RES", plotvis="${ms.PLOTVIS}", dirty=True,restore=False,restore_lsm=True, label=None, args=[],options={}, **kws): """Generic function to run a stefcal job. 'section' TDL config file section 'label' will be assigned to the global LABEL for purposes of file naming 'apply_only' if true, will only apply saved solutions 'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE" 'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only 'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above, or T1,T2 for --above T1 --fm-above T2 'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful) 'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help) 'dirty','restore' 'restore_lsm' image output visibilities (passed to imager.make_image above as is) 'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs), can be used to supply extra TDL options extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus overriding settings in the TDL config file. Useful arguments of this kind are e.g.: stefcal_reset_all=True to remove prior gains solutions. """ msname,section,lsm,label,plotvis = interpolate_locals("msname section lsm label plotvis"); makedir(v.DESTDIR); # increment step counter and assign global label if label is not None: v.LABEL = str(label); if type(v.STEP) is int and STEFCAL_STEP_INCR: v.STEP += STEFCAL_STEP_INCR; # setup stefcal options and run info("Running stefcal ${step <STEP} ${(<LABEL>)}"); # setup args args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} ms_sel.output_column=$STEFCAL_OUTPUT_COLUMN stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s"""% ((1 if diffgains else 0),STEFCAL_TDLOPTS) ]; if diffgains: if diffgains is True: diffgains = "=dE"; args0.append("de_subset.subset_enabled=1 de_subset.source_subset=$diffgains"); opts = { 'do_output': output, 'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save", 'stefcal_gain1.mode': "apply" if apply_only or gain_apply_only else "solve-save", 'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save", 'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only else "solve-save", 'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save", 'stefcal_gain.table': STEFCAL_GAIN, 'stefcal_gain1.table': STEFCAL_GAIN1, 'stefcal_diffgain.table': STEFCAL_DIFFGAIN, 'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1, 'stefcal_ifr_gain_table': STEFCAL_IFRGAIN, 'stefcal_visualize': False } timesmooth,freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (0,0); timeint,freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,0); opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint; opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint; opts['stefcal_diffgain.timesmooth'] = timesmooth; opts['stefcal_diffgain.freqsmooth'] = freqsmooth; # add user-defined args args0 += list(args); opts.update(options); opts.update(kws); # run the job mqt.run(STEFCAL_SCRIPT,STEFCAL_JOBNAME,section=section,args=args0,options=opts); # copy gains if not apply_only: if os.path.exists(STEFCAL_GAIN): std.copy(STEFCAL_GAIN,STEFCAL_GAIN_SAVE); if os.path.exists(STEFCAL_GAIN1): std.copy(STEFCAL_GAIN1,STEFCAL_GAIN1_SAVE); if os.path.exists(STEFCAL_DIFFGAIN): std.copy(STEFCAL_DIFFGAIN,STEFCAL_DIFFGAIN_SAVE); if os.path.exists(STEFCAL_IFRGAIN): std.copy(STEFCAL_IFRGAIN,STEFCAL_IFRGAIN_SAVE); # post-calibration flagging if flag_threshold: if isinstance(flag_threshold,(list,tuple)): t0,t1 = flag_threshold; else: t0,t1 = flag_threshold,None; ms.flagms("--above %g"%t0,"-f threshold -c"); if t1: ms.flagms("--fm-above %g"%t1,"-f fmthreshold -c"); # plot residuals if plotvis: info("Plotting visibilities ($plotvis)"); ms.PLOTVIS = plotvis; ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png"); # make images imager.make_image(msname,column=STEFCAL_OUTPUT_COLUMN,dirty=dirty,restore=restore,restore_lsm=restore_lsm);
dots = dots.splitlines() folds = folds.splitlines() paper = set() for dot in dots: x, y = dot.split(',') x, y = int(x), int(y) paper.add((x, y)) for fold in folds: axis, line = re.findall(r'fold along ([xy])=(\d+)', fold)[0] line = int(line) match axis: case "x": paper = {dot if dot[0] < line else (2*line - dot[0], dot[1]) for dot in paper} case "y": paper = {dot if dot[1] < line else (dot[0], 2*line - dot[1]) for dot in paper} max_x = max(paper, key=lambda x: x[0])[0] max_y = max(paper, key=lambda y: y[1])[1] grid = np.zeros((max_y+1, max_x+1)) for dot in paper: grid[dot[1], dot[0]] = 1 imager.make_image(grid, palette, name="code", resize=20)
def send_all(): """ Send all new raids from username to the designated chat in case of error inform the admin. """ all_results = get_all_results() if len(all_results) == 0: print("No new results right now.") return if SECRETS.CHAT['messenger']['ACTIVATE']: sett = SECRETS.CHAT['messenger'] try: print("Logging in...") ua = sett['user_agent'] client = Client(sett['username'], sett['password'], user_agent=ua) except Exception as e: print("Could not login!") print(traceback.format_exc()) return False try: for result in all_results: if result['type'] == 'spawn' and \ result["pokemon_id"] not in SECRETS.POKEMON['default'] and \ result["iv"] not in ["45"]: continue print(result["message"]) make_image(result) client.sendLocalImage("./img/tmp_raid.png", result["message"], **sett['chat']) except Exception as e: exc = traceback.format_exc() print("GoMap Messenger failed") print(exc) client.sendMessage("GoMap Messenger sender failed. Sorry.", **sett['admin']) client.sendMessage(exc, **sett['admin']) elif SECRETS.CHAT['discord']['ACTIVATE']: try: sett = SECRETS.CHAT['discord'] for result in all_results: make_image(result) try: img_link = upload_image() except UploadError as e: exc = traceback.format_exc() print(exc) txt = "{}: An error while uploading image:\n{}\n{}".format( datetime.datetime.now().isoformat(), str(e), e.extra_data.content) msg = Webhook( sett['error_webhook'], msg=txt, ) msg.post() img_link = None and "https://i.imgur.com/ZdjcRWW.png" urls = [sett['raid_webhook']] tags = "" if result['type'] == 'spawn': urls = [] for key, pokemon in SECRETS.POKEMON.items(): if result["pokemon_id"] in pokemon: urls.append(sett['pokemon_webhook'][key]) if result["iv"].isdigit() and int(result["iv"]) >= 43: urls.append(sett['highiv_webhook']) for pkm, val in sett["special_ranks"].items(): if result["pokemon_id"] in pkm: tags += " <@&{}>".format(val["role_id"]) for name, role_id in result['districts']: tags += " <@&{}>".format(role_id) if tags: tags = "\n" + tags for url in urls: msg = Webhook(url, msg=result['message'] + tags, image=img_link) msg.post() except Exception as e: exc = traceback.format_exc() print(exc) txt = "{}: An error while sending results:\n{}".format( datetime.datetime.now().isoformat(), exc) msg = Webhook( sett['error_webhook'], msg=txt, ) msg.post()
def makecube(npix=512, stokes="I"): imager.make_image(channelize=1, dirty_image="$OUTFILE.cube.fits", npix=npix, wprojplanes=0, stokes=stokes)
def jointcal(goto_step=1, last_step=10, lsmbase=None, STEPS=None): """Calibration for joint C and D-config data""" info( ">>>>>>>>>>>>> output directory is $OUTDIR. Please set OUTDIR explicitly to override" ) # setup LSM filenames based on the full MS # note that these get interpolated once and for all here (and the _Template definitions above # get cancelled due to the explicit assignment here). The reason for doing it like this # is because I don't want these names to be changing due to the templates every time the # MS changes in a per(MS) call. v.FULLMS = MS LSM1 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm.lsm.html") LSM2 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2.lsm.html") LSM3 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2+cc.lsm.html") LSM_CCMODEL = II("$DESTDIR/$LSMBASE$SUFFIX+ccmodel.fits") saveconf() stefcal.STEFCAL_DIFFGAIN_SMOOTHING = DE_SMOOTHING if not DE_INTERVALS else None stefcal.STEFCAL_DIFFGAIN_INTERVALS = DE_INTERVALS if DE_INTERVALS and TILE: stefcal.STEFCAL_TDLOPTS = "ms_sel.tile_size=%d" % (TILE * DE_INTERVALS[0]) # make MS list from sub-MSs import glob v.MS_List = glob.glob(MS + "/SUBMSS/*MS") info("MS list is $MS_List") if not MS_List: abort("No sub-MSs found") imager.npix = NPIX = 4096 imager.cellsize = "2arcsec" imager.wprojplanes = 128 imager.CLEAN_ALGORITHM = "csclean" v.LSMREF = "${MS:BASE}.refmodel.lsm.html" THRESH_PIX, THRESH_ISL = (50, 10), (15, 5) CLEAN_THRESH = ".4mJy", ".1mJy", ".05mJy" stefcal.STEFCAL_STEP_INCR = 0 # precvent stefcal from auto-incrementing v.STEP: we set the step counter explicitly here if STEPS is None: STEPS = list(numpy.arange(goto_step, last_step + .1, .5)) STEPS = map(float, STEPS) if STEPS[0] != 1: info("########## restarting calibration from step %.1f" % STEPS[0]) if lsmbase: LSMBASE = lsmbase ## initial calibration if 1. in STEPS: info("########## step 1: solving for G with initial LSM") v.LSM, v.STEP = LSM0, 1 per_ms(jointcal_g) if 1.5 in STEPS: info("########## step 1.5: making joint image") v.LSM, v.STEP = LSM0, 1 v.MS = FULLMS # initial model is total flux only, made from a 2x size image to catch distant sources imager.make_image(dirty=False, stokes="I", restore=dict(npix=NPIX * 2, threshold=CLEAN_THRESH[0], wprojplanes=128), restore_lsm=False) info("########## running source finder and updating model") ## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT ### NB: select on radius to exclude any artefacts picked up around 3C147 itself lsm.pybdsm_search(thresh_pix=THRESH_PIX[0], thresh_isl=THRESH_ISL[0], select="r.gt.30s", pol=False) ### merge new sources into sky model, give it a new name ($LSM1) lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f") # if 2. in STEPS: # info("########## step 2: repeating G solution"); # v.LSM,v.STEP = LSM1,2 # v.MS = FULLMS # per_ms(jointcal_g); if 2. in STEPS: info("########## step 2: initial dE solution") v.LSM, v.STEP = LSM1, 2 v.MS = FULLMS # now, set dE tags on sources lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC) per_ms(jointcal_de_reset) if 3. in STEPS: info("########## step 3: re-solving for G to apply IFR solutions") v.LSM, v.STEP = LSM1, 3 v.MS = FULLMS per_ms(jointcal_de_apply) info("########## running source finder and updating model") v.MS = FULLMS imager.make_image(dirty=False, stokes="IV", restore=dict(npix=NPIX, threshold=CLEAN_THRESH[1], wprojplanes=128), restore_lsm=False) ## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT ### NB: select on radius to exclude any artefacts picked up around 3C147 itself lsm.pybdsm_search(thresh_pix=THRESH_PIX[1], thresh_isl=THRESH_ISL[1], select="r.gt.30s") ### merge new sources into sky model, give it a new name ($LSM1) lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM2 --rename -f") if 4. in STEPS: info( "########## step 4: solving for G+dE with updated LSM (initial+pybdsm^2)" ) v.MS = FULLMS v.LSM, v.STEP = LSM2, 4 lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC) per_ms(jointcal_de) v.MS = FULLMS imager.make_image(dirty=False, stokes="IV", restore=dict(npix=NPIX, threshold=CLEAN_THRESH[1], wprojplanes=128), restore_lsm=False) info("########## adding clean components to LSM") ff = pyfits.open(imager.MODEL_IMAGE) dd = ff[0].data dd *= 1.0769 # scale up to compensate for selfcal flux suppression # dd[dd<0] = 0; # remove negative components ff.writeto(LSM_CCMODEL, clobber=True) # add model image to LSM lsm.tigger_convert("$LSM $LSM3 --add-brick=ccmodel:$LSM_CCMODEL:2 -f") if 5. in STEPS: info("########## step 5: re-running DD solutions") v.MS = FULLMS v.LSM, v.STEP = LSM3, 5 per_ms(jointcal_de_final) if 5.5 in STEPS: info("########## step 5.5: making joint image") v.MS = FULLMS v.LSM, v.STEP = LSM3, 5 imager.make_image(dirty=False, stokes="IQUV", restore=dict(npix=NPIX, threshold=CLEAN_THRESH[2], wprojplanes=128), restore_lsm=True) if 6. in STEPS: info("########## step 6: noise sim") per_ms(lambda: makecube(stokes="IQUV")) v.LSM, v.STEP = LSM3, 5 v.MS = FULLMS makecube(stokes="IQUV") makenoise()
def jointcal (goto_step=1,last_step=10,lsmbase=None,STEPS=None): """Calibration for joint C and D-config data""" info(">>>>>>>>>>>>> output directory is $OUTDIR. Please set OUTDIR explicitly to override"); # setup LSM filenames based on the full MS # note that these get interpolated once and for all here (and the _Template definitions above # get cancelled due to the explicit assignment here). The reason for doing it like this # is because I don't want these names to be changing due to the templates every time the # MS changes in a per(MS) call. v.FULLMS = MS LSM1 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm.lsm.html"); LSM2 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2.lsm.html"); LSM3 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2+cc.lsm.html"); LSM_CCMODEL = II("$DESTDIR/$LSMBASE$SUFFIX+ccmodel.fits"); saveconf() stefcal.STEFCAL_DIFFGAIN_SMOOTHING = DE_SMOOTHING if not DE_INTERVALS else None; stefcal.STEFCAL_DIFFGAIN_INTERVALS = DE_INTERVALS; if DE_INTERVALS and TILE: stefcal.STEFCAL_TDLOPTS = "ms_sel.tile_size=%d"%(TILE*DE_INTERVALS[0]); # make MS list from sub-MSs import glob v.MS_List = glob.glob(MS+"/SUBMSS/*MS"); info("MS list is $MS_List"); if not MS_List: abort("No sub-MSs found"); imager.npix = NPIX = 4096 imager.cellsize = "2arcsec" imager.wprojplanes = 128 imager.CLEAN_ALGORITHM = "csclean" v.LSMREF = "${MS:BASE}.refmodel.lsm.html" THRESH_PIX,THRESH_ISL = (50,10),(15,5) CLEAN_THRESH = ".4mJy",".1mJy",".05mJy" stefcal.STEFCAL_STEP_INCR = 0 # precvent stefcal from auto-incrementing v.STEP: we set the step counter explicitly here if STEPS is None: STEPS = list(numpy.arange(goto_step,last_step+.1,.5)); STEPS = map(float,STEPS); if STEPS[0] != 1: info("########## restarting calibration from step %.1f"%STEPS[0]); if lsmbase: LSMBASE = lsmbase; ## initial calibration if 1. in STEPS: info("########## step 1: solving for G with initial LSM"); v.LSM,v.STEP = LSM0,1 per_ms(jointcal_g); if 1.5 in STEPS: info("########## step 1.5: making joint image"); v.LSM,v.STEP = LSM0,1 v.MS = FULLMS # initial model is total flux only, made from a 2x size image to catch distant sources imager.make_image(dirty=False,stokes="I",restore=dict(npix=NPIX*2,threshold=CLEAN_THRESH[0],wprojplanes=128),restore_lsm=False); info("########## running source finder and updating model"); ## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT ### NB: select on radius to exclude any artefacts picked up around 3C147 itself lsm.pybdsm_search(thresh_pix=THRESH_PIX[0],thresh_isl=THRESH_ISL[0],select="r.gt.30s",pol=False); ### merge new sources into sky model, give it a new name ($LSM1) lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f"); # if 2. in STEPS: # info("########## step 2: repeating G solution"); # v.LSM,v.STEP = LSM1,2 # v.MS = FULLMS # per_ms(jointcal_g); if 2. in STEPS: info("########## step 2: initial dE solution"); v.LSM,v.STEP = LSM1,2 v.MS = FULLMS # now, set dE tags on sources lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC); per_ms(jointcal_de_reset); if 3. in STEPS: info("########## step 3: re-solving for G to apply IFR solutions"); v.LSM,v.STEP = LSM1,3 v.MS = FULLMS per_ms(jointcal_de_apply); info("########## running source finder and updating model"); v.MS = FULLMS imager.make_image(dirty=False,stokes="IV",restore=dict(npix=NPIX,threshold=CLEAN_THRESH[1],wprojplanes=128),restore_lsm=False); ## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT ### NB: select on radius to exclude any artefacts picked up around 3C147 itself lsm.pybdsm_search(thresh_pix=THRESH_PIX[1],thresh_isl=THRESH_ISL[1],select="r.gt.30s"); ### merge new sources into sky model, give it a new name ($LSM1) lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM2 --rename -f"); if 4. in STEPS: info("########## step 4: solving for G+dE with updated LSM (initial+pybdsm^2)"); v.MS = FULLMS v.LSM,v.STEP = LSM2,4 lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC); per_ms(jointcal_de); v.MS = FULLMS imager.make_image(dirty=False,stokes="IV",restore=dict(npix=NPIX,threshold=CLEAN_THRESH[1],wprojplanes=128),restore_lsm=False); info("########## adding clean components to LSM"); ff = pyfits.open(imager.MODEL_IMAGE); dd = ff[0].data; dd *= 1.0769 # scale up to compensate for selfcal flux suppression # dd[dd<0] = 0; # remove negative components ff.writeto(LSM_CCMODEL,clobber=True); # add model image to LSM lsm.tigger_convert("$LSM $LSM3 --add-brick=ccmodel:$LSM_CCMODEL:2 -f"); if 5. in STEPS: info("########## step 5: re-running DD solutions"); v.MS = FULLMS v.LSM,v.STEP = LSM3,5 per_ms(jointcal_de_final); if 5.5 in STEPS: info("########## step 5.5: making joint image"); v.MS = FULLMS v.LSM,v.STEP = LSM3,5 imager.make_image(dirty=False,stokes="IQUV",restore=dict(npix=NPIX,threshold=CLEAN_THRESH[2],wprojplanes=128),restore_lsm=True); if 6. in STEPS: info("########## step 6: noise sim"); per_ms(lambda:makecube(stokes="IQUV")); v.LSM,v.STEP = LSM3,5 v.MS = FULLMS; makecube(stokes="IQUV"); makenoise();
def make_images (): for w,rob in ("natural",0),("uniform",0),("robust",0),("robust",1): imager.make_image(weight=w,robust=rob,dirty_image="$OUTFILE-$w$rob.fits");