def makenoise ():
# make noise images
addnoise();
imager.make_image(channelize=1,dirty_image="$OUTFILE.noisecube.fits",npix=256,wprojplanes=0,stokes="I",column="MODEL_DATA");
imager.make_image(dirty_image="$OUTFILE.noise.fits",npix=256,wprojplanes=0,stokes="I",column="MODEL_DATA");
noise = pyfits.open(II("$OUTFILE.noise.fits"))[0].data.std();
info(">>> maximum noise value is %.2f uJy"%(noise*1e+6));
def image(msname='$MS',
lsmname='$LSM',
use_imager='LWIMAGER',
restore=False,
options={},
**kw):
""" Images MS"""
imager.cellsize = '1arcsec'
if restore:
#ms.copycol() # Copy content of DATA to CORRECTED_DATA if making clean map
options['data'] = 'CORRECTED_DATA' # make sure to image corrected data
if NCHAN > 1:
start = 1
step = CHANNELIZE or NCHAN
temp = dict(nchan=NCHAN,
img_nchan=(NCHAN - 1) / step,
chanstart=0,
img_chanstart=1,
chanstep=1,
img_chanstep=step)
options.update(temp)
imager.make_image(restore=restore,
column=COLUMN,
restore_lsm=False,
**options)
def simulate_imaging_bd_3c147 (hiresms=None, loresms=None, inputcolumn="DATA", outputcolumn="CORRECTED_DATA", dtime=None, dfreq=None):
"""
The function make use of baseline dependent averaging on the 3C147 real data observed on the
2013/01/27/01:00:28.5, at declination 49.51.07.23356, ra 05:42:36.137916
NB: We have removed the fictive baseline of index 7
"""
# make an instance of the class containing the method for bd-averaging
mshi = MSResampler.MSResampler(hiresms+"/", column=inputcolumn)
# BD-averaging, giving the integration time of the shortest baseline, dtime and
# the number of uv-frequency bins dfreq to average
psh=2
qsh=8
arrays = mshi.bd_averaging (dtime,dfreq,psh,qsh)
# arrays is of size (p,q,datacom,flagrowpq,weightpq)
# take the number of time bins of the longest baseline and make low res timeslots
MSResampler.save_visibility_arrays (loresms,arrays,column=outputcolumn)
imager.npix= 2048#1024#2048
imager.cellsize = "2arcsec"
imager.stokes = "I"
imager.weight = "natural"
imager.wprojplanes = 128
#cleaning options
imager.niter = 1000
imager.threshold = "5mJy"
imager.CLEAN_ALGORITHM = "csclean"
imager.make_image(msname = loresms, column = outputcolumn, restore = True, dirty = False, weight = "natural");
def makenoise():
# make noise images
addnoise()
imager.make_image(channelize=1,
dirty_image="$OUTFILE.noisecube.fits",
npix=256,
wprojplanes=0,
stokes="I",
column="MODEL_DATA")
imager.make_image(dirty_image="$OUTFILE.noise.fits",
npix=256,
wprojplanes=0,
stokes="I",
column="MODEL_DATA")
noise = pyfits.open(II("$OUTFILE.noise.fits"))[0].data.std()
info(">>> maximum noise value is %.2f uJy" % (noise * 1e+6))
def sim_now():
radius1 = np.arange(0., 100., 10.)
#radius1 = np.arange(0.,3,0.3)
Flux_array1 = np.zeros(len(radius1))
lores1 = "MeerKATlores.MS_p0_p0"
imager.cellsize = "10.arcsec"
imager.npix = 512
imager.niter = 0
imager.CLEAN_ALGORITHM = "csclean"
for k in range(len(radius1)):
options = {}
options['gridded_sky.grid_m0'] = radius1[k]
options['ms_sel.msname'] = lores1
mqt.run("turbo-sim.py",
job="_tdl_job_1_simulate_MS",
config="tdlconf.profiles1",
section="Sim_source_radius",
options=options)
center_min = -45 * 60 + radius1[k]
center_deg = math.ceil(center_min / 60)
center_min = abs(center_min - center_deg * 60)
imager.make_image(msname=lores1,
column='CORRECTED_DATA',
phasecenter="j2000,0h0m,%dd%dm" %
(center_deg, center_min),
restore=False,
dirty=True,
restore_lsm=False,
weight="natural")
f1 = np.max(pyfits.open(imager.DIRTY_IMAGE)[0].data[0][0])
Flux_array1[k] = f1
np.save("DATA/Flux1", Flux_array1)
np.save("DATA/radius1", radius1)
def simcube (cube,nchan=None,npix=4096,cellsize=".5arcsec",niter=100000,padding=1.5,threshold=".2mJy",predict=True,dirty=False,restore=False,noise=0,resume=False,label=None,column='DATA',channelize=1,wprojplanes=0):
if label: v.LABEL = label
tab = ms.ms(subtable='SPECTRAL_WINDOW')
nchan = nchan or tab.getcol('NUM_CHAN')[0]
ms.CHANRANGE = 0,nchan-1;
imager.wprojplanes = wprojplanes;
imager.IMAGE_CHANNELIZE = 1;
if predict:
imager.predict_vis(image=cube,padding=padding,copy=False,column=column);
if noise > 0:
simnoise(addToCol=column,noise=noise)
if dirty:
info('Weights are%s'%WEIGHTS)
for weight in WEIGHTS.split(':'):
opts,weight_txt,quals = get_weight_opts(weight)
restore.update(opts)
dirty_image = II("${OUTFILE}-$weight.dirty.fits")
model_image = II("${OUTFILE}-$weight.model.fits")
residual_image = II("${OUTFILE}-$weight.residual.fits")
restored_image = II("${OUTFILE}-$weight.restored.fits")
imager.make_image(dirty=dirty,restore=restore,channelize=channelize,column=column if restore==False else "CORRECTED_DATA",dirty_image=dirty_image,model_image=model_image,residual_image=residual_image,restored_image=restored_image,**opts);
def compute_psf_and_noise (make_psf=True,noise=0,noise_map=True,scale_noise=1.0,add_noise=False,rowchunk=1000000,dirty=True,measure_sdl=False,use_old_noise_map=False,**kw):
info("weights are",*WEIGHTS.split(":"))
for weight in WEIGHTS.split(":"):
opts,weight_txt,quals = get_weight_opts(weight);
opts.update(kw)
# make PSF image
if make_psf:
psfimage = II('$OUTFILE-$weight-psf.fits')
psf_opts = opts
if DBL_PSF:
try : psf_opts['npix'] = opts['npix']*2
except KeyError : psf_opts['npix'] = imager.npix * 2
info(' >>>>>>>>>> %d -- %d '%(imager.npix,psf_opts['npix']))
imager.make_image(dirty=dict(data="psf"),dirty_image=psfimage,**psf_opts);
# make PSF cross-sections and measure FWHMs
rx,ry = measure_psf(psfimage,
arcsec_size = 10, #2.5**math.ceil(math.log(5*5)/math.log(5)),
savefig = psfimage.replace('.fits','.png'),
title =os.path.basename(OUTFILE).split('_')[0])
info(">>> $weight: PSF FWHM $rx by $ry")
_writestat("psf_fwhm",(rx,ry,(rx+ry)/2),*quals);
if noise_map:
info(' >>> Making noise map')
if add_noise:
simnoise(rowchunk=rowchunk,scale_noise=scale_noise,noise=noise)
noiseimage = II('$OUTFILE-$weight-noise.fits')
opts.update(kw)
make_noise_map = False
if use_old_noise_map:
if os.path.exists(noiseimage):
info('Using existing noise map: $noiseimage')
else: make_noise_map = True
if make_noise_map:
imager.make_image(dirty=dirty,column='MODEL_DATA',dirty_image=noiseimage,**opts);
noise = pyfits.open(noiseimage)[0].data.std();
info(">>> rms pixel noise (%s) is %g uJy"%(weight_txt,noise*1e+6));
_writestat("pixnoise",noise,*quals);
if measure_sdl:
r0 = int(SIDELOBES_R0_ARCSEC/SIDELOBES_CELL_ARCSEC);
r1 = int(SIDELOBES_R1_ARCSEC/SIDELOBES_CELL_ARCSEC);
npix = r1*2;
bigpsf = II('$OUTFILE-$weight-bigpsf.fits')
imager.make_image(dirty_image=bigpsf,dirty=dict(data="psf",cellsize="%farcsec"%SIDELOBES_CELL_ARCSEC,npix=npix),**opts);
data = pyfits.open(bigpsf)[0].data[0,0,...];
radius = numpy.arange(npix)-r1
radius = numpy.sqrt(radius[numpy.newaxis,:]**2+radius[:,numpy.newaxis]**2)
mask = (radius<=r1)&(radius>=r0)
rms = data[mask].std();
data = None;
r0,r1 = r0/3600.,r1/3600.;
info(">>> rms far sidelobes (%s) is %g (%.2f<=r<=%.2fdeg"%(weight_txt,rms,r0,r1));
_writestat("sidelobe_radius",(r0,r1),*quals);
_writestat("sidelobes",rms,*quals);
return noise
def main():
parser = argparse.ArgumentParser()
parser.add_argument("min_x",
type=int,
help="The X coordinate of the bottom left of the map.")
parser.add_argument("min_z",
type=int,
help="The Z coordinate of the bottom left of the map.")
parser.add_argument("max_x",
type=int,
help="The width of the map (along X)")
parser.add_argument("max_z",
type=int,
help="The height of the map (along Z)")
parser.add_argument("output_file",
help="The .ppm file to output the image to")
parser.add_argument("data_directory",
help="The directory where region files are stored")
parser.add_argument("map_type",
default='minecraft',
help='The type of map to generate'
) #, choices=['heightmap', 'minecraft'])
args = parser.parse_args()
x_start = math.floor(args.min_x / 16 / 32)
z_start = math.floor(args.min_z / 16 / 32)
x_end = math.floor(args.max_x / 16 / 32)
z_end = math.floor(args.max_z / 16 / 32)
if args.map_type not in ['minecraft', 'heightmap'
] and args.map_type[:2] != 'y=':
args.map_type = 'minecraft'
progress = 0
image = Image.new('RGB',
(args.max_x - args.min_x, args.max_z - args.min_z))
# image = [[None for z in range(args.max_x - args.min_x)] for x in range(args.max_z - args.min_z)]
for x in range(x_start, x_end + 1):
r_x = x * 512
for z in range(z_start, z_end + 1):
progress += 1
status = f", region {progress} of {(x_end - x_start + 1) * (z_end - z_start + 1)} "
r_z = z * 512
loaded_chunks = {}
try:
loaded_chunks.update(
chunks.read_mca(
os.path.join(args.data_directory, f"r.{x}.{z-1}.mca")))
except FileNotFoundError:
pass
try:
loaded_chunks.update(
chunks.read_mca(
os.path.join(args.data_directory, f"r.{x}.{z}.mca")))
except FileNotFoundError:
continue
length = 512
if args.min_z > r_z:
length -= (args.min_z - r_z)
if args.max_z < r_z + 512:
length -= (r_z + 512 - args.max_z)
width = 512
if args.min_x > r_x:
width -= (args.min_x - r_x)
if args.max_x < r_x + 512:
width -= (r_x + 512 - args.max_x)
image_chunk = imager.make_image(
loaded_chunks, (width, length),
(max(args.min_x, r_x), max(args.min_z, r_z)),
mode=args.map_type,
status=status)
orig_x = max(args.min_x, r_x)
orig_z = max(args.min_z, r_z)
if len(image_chunk) > 0:
for a in range(len(image_chunk[0])):
for b in range(len(image_chunk)):
# image[orig_z + b - args.min_z][orig_x + a - args.min_x] = image_chunk[b][a];
image.putpixel(
(orig_x + a - args.min_x, orig_z + b - args.min_z),
image_chunk[b][a])
# print(image)
image.save(args.output_file)
def cal_ms():
"""cal_ms: runs a calibration loop over the current MS"""
# initialize the calibration "step" counter and "label". These are used
# to auto-generate output filenames; cal.stefcal() below automatically increments
# cal.STEP
v.STEP = 0
# set the superglobal LSM variable. See explanation for "v." in the text
v.LSM = LSM0
# info(), warn() and abort() are Pyxis functions for writing output to the log
info("########## solving for G with initial LSM")
# do one stefcal run with the initial model, produce corrected residuals,
# do simple flagging on the residuals. See cal.stefcal() for full docs.
stefcal.stefcal(stefcal_reset_all=True,
output="CORR_RES",
restore=False,
flag_threshold=(1, .5))
# If we got to this point, then assume things are rolling along fine.
# Copy the recipe and config to the destination directory for future reference
# (very useful when modifying recipes, so you can keep track of what settings
# are responsible for what output!)
xo.sh("cp pyxis*.py pyxis*.conf ${mqt.TDLCONFIG} $DESTDIR", shell=True)
info(
"########## remaking data image, running source finder and updating model"
)
# apply previous stefcal solutions to produce a corrected data image.
# Make a restored image with 2000 CLEAN iterations.
stefcal.stefcal(apply_only=True,
output="CORR_DATA",
plotvis=False,
restore=dict(niter=2000),
restore_lsm=False)
## now run pybdsm on the restored image, write output to a new LSM file
lsm.pybdsm_search(threshold=5, output=LSM1)
# the new sky model becomes the "current" LSM
v.LSM = LSM1
# note that in principle this step is not needed (may as well go on directly
# to the dE solutions, below), but it is instructive, as it will produce an intermediate-stage
# image. But to save time, set restore=False to skip making clean images
info("########## recalibrating with new model")
stefcal.stefcal(stefcal_reset_all=True,
output="CORR_RES",
restore=False,
flag_threshold=(1, .5))
# if we have a "reference" sky model configured, use it to set dE tags on
# sources in our new sky model. Sources with dE tags will have DDE solutions.
# The cal.transfer_tags() function sets the specified tag on any source
# near enough to a reference model source with that tag.
if LSMREF:
lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC)
info("########## recalibrating with dEs")
# re-run stefcal with the new model (and with direction dependent
# solutions on dE-tagged sources, if any). Make a corrected residual
# image, run CLEAN on it, restore the LSM sources back into the
# image
stefcal.stefcal(stefcal_reset_all=True,
diffgains=True,
output="CORR_RES",
restore=dict(niter=1000),
restore_lsm=True,
flag_threshold=(1, .5))
# no reference LSM? then still need to make a restored image from the step above
else:
imager.make_image(dirty=False,
restore=dict(niter=1000),
restore_lsm=True)
# the resulting image filename is in cal.FULLREST_IMAGE,
# add it to our image list file
IMAGE_LIST.add("${imager.FULLREST_IMAGE}")
def make_images():
for w, rob in ("natural", 0), ("uniform", 0), ("robust", 0), ("robust", 1):
imager.make_image(weight=w,
robust=rob,
dirty_image="$OUTFILE-$w$rob.fits")
def stefcal(msname="$MS",
section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
gain_apply_only=False,
diffgain_apply_only=False,
ifrgain_apply_only=False,
diffgain_intervals=None,
diffgain_smoothing=None,
flag_threshold=None,
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,
restore=False,
restore_lsm=True,
label=None,
args=[],
options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname, section, lsm, label, plotvis = interpolate_locals(
"msname section lsm label plotvis")
makedir(v.DESTDIR)
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label)
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}")
# setup args
args0 = [
"""${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.output_column=$STEFCAL_OUTPUT_COLUMN
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s""" %
((1 if diffgains else 0), STEFCAL_TDLOPTS)
]
if diffgains:
if diffgains is True:
diffgains = "=dE"
args0.append(
"de_subset.subset_enabled=1 de_subset.source_subset=$diffgains")
opts = {
'do_output':
output,
'stefcal_gain.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_diffgain.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_ifr_gain_mode':
"apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_gain.table':
STEFCAL_GAIN,
'stefcal_gain1.table':
STEFCAL_GAIN1,
'stefcal_diffgain.table':
STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table':
STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table':
STEFCAL_IFRGAIN,
'stefcal_visualize':
False
}
timesmooth, freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (
0, 0)
timeint, freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,
0)
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint
opts['stefcal_diffgain.timesmooth'] = timesmooth
opts['stefcal_diffgain.freqsmooth'] = freqsmooth
# add user-defined args
args0 += list(args)
opts.update(options)
opts.update(kws)
# run the job
mqt.run(STEFCAL_SCRIPT,
STEFCAL_JOBNAME,
section=section,
args=args0,
options=opts)
# copy gains
if not apply_only:
if os.path.exists(STEFCAL_GAIN):
std.copy(STEFCAL_GAIN, STEFCAL_GAIN_SAVE)
if os.path.exists(STEFCAL_GAIN1):
std.copy(STEFCAL_GAIN1, STEFCAL_GAIN1_SAVE)
if os.path.exists(STEFCAL_DIFFGAIN):
std.copy(STEFCAL_DIFFGAIN, STEFCAL_DIFFGAIN_SAVE)
if os.path.exists(STEFCAL_IFRGAIN):
std.copy(STEFCAL_IFRGAIN, STEFCAL_IFRGAIN_SAVE)
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold, (list, tuple)):
t0, t1 = flag_threshold
else:
t0, t1 = flag_threshold, None
ms.flagms("--above %g" % t0, "-f threshold -c")
if t1:
ms.flagms("--fm-above %g" % t1, "-f fmthreshold -c")
# plot residuals
if plotvis:
info("Plotting visibilities ($plotvis)")
ms.PLOTVIS = plotvis
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png")
# make images
imager.make_image(msname,
column=STEFCAL_OUTPUT_COLUMN,
dirty=dirty,
restore=restore,
restore_lsm=restore_lsm)
def cal_ms ():
"""cal_ms: runs a calibration loop over the current MS"""
# initialize the calibration "step" counter and "label". These are used
# to auto-generate output filenames; cal.stefcal() below automatically increments
# cal.STEP
v.STEP = 0
# set the superglobal LSM variable. See explanation for "v." in the text
v.LSM = LSM0
# info(), warn() and abort() are Pyxis functions for writing output to the log
info("########## solving for G with initial LSM")
# do one stefcal run with the initial model, produce corrected residuals,
# do simple flagging on the residuals. See cal.stefcal() for full docs.
stefcal.stefcal(stefcal_reset_all=True,output="CORR_RES",restore=False,flag_threshold=(1,.5))
# If we got to this point, then assume things are rolling along fine.
# Copy the recipe and config to the destination directory for future reference
# (very useful when modifying recipes, so you can keep track of what settings
# are responsible for what output!)
xo.sh("cp pyxis*.py pyxis*.conf ${mqt.TDLCONFIG} $DESTDIR",shell=True)
info("########## remaking data image, running source finder and updating model")
# apply previous stefcal solutions to produce a corrected data image.
# Make a restored image with 2000 CLEAN iterations.
stefcal.stefcal(apply_only=True,output="CORR_DATA",plotvis=False,
restore=dict(niter=2000),restore_lsm=False)
## now run pybdsm on the restored image, write output to a new LSM file
lsm.pybdsm_search(threshold=5,output=LSM1)
# the new sky model becomes the "current" LSM
v.LSM = LSM1
# note that in principle this step is not needed (may as well go on directly
# to the dE solutions, below), but it is instructive, as it will produce an intermediate-stage
# image. But to save time, set restore=False to skip making clean images
info("########## recalibrating with new model")
stefcal.stefcal(stefcal_reset_all=True,output="CORR_RES",restore=False,flag_threshold=(1,.5));
# if we have a "reference" sky model configured, use it to set dE tags on
# sources in our new sky model. Sources with dE tags will have DDE solutions.
# The cal.transfer_tags() function sets the specified tag on any source
# near enough to a reference model source with that tag.
if LSMREF:
lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC)
info("########## recalibrating with dEs")
# re-run stefcal with the new model (and with direction dependent
# solutions on dE-tagged sources, if any). Make a corrected residual
# image, run CLEAN on it, restore the LSM sources back into the
# image
stefcal.stefcal(stefcal_reset_all=True,diffgains=True,
output="CORR_RES",restore=dict(niter=1000),
restore_lsm=True,flag_threshold=(1,.5))
# no reference LSM? then still need to make a restored image from the step above
else:
imager.make_image(dirty=False,restore=dict(niter=1000),restore_lsm=True);
# the resulting image filename is in cal.FULLREST_IMAGE,
# add it to our image list file
IMAGE_LIST.add("${imager.FULLREST_IMAGE}");
def runall():
#alpha_v = 5 #Power low distribution parameter
num_sources_v = 2 #how many sources
num_cal_sources_v = 1#how many sources in calibration model
#fov_v = 3 #degrees #the field of view in degrees
#only do stefcal or not
skip_LM = True
dist = True
point_sources = np.array([(1,0,0),(0.2,(1*np.pi)/180,(np.pi*0)/180)]) #sets the pointsources in the sky
#contain all the pointsources
#point_sources = np.zeros((num_sources_v,3))
#generate flux and positions
#point_sources[:,0] = generate_flux(a = alpha_v,num_sources = num_sources_v,plot=False)
#point_sources[:,1] = generate_pos(fov = fov_v,num_sources=num_sources_v)
#point_sources[:,2] = generate_pos(fov = fov_v,num_sources=num_sources_v)
ra0,dec0 = get_field_center()
#generate true and calibration sky models from point_sources
meqskymodel(point_sources,num_cal_sources=num_cal_sources_v)
#simulate complete skymodel --- store in CORRECTED_DATA
#sim_function(cal=False)
#simulate calibration skymodel --- store in DATA
#sim_function(cal=True)
#set up imager
image_settings()
options = image_advanced_settings()
#make images of complete sky and calibrated sky model
#imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False)
#v.CALORNOT = "cal_model"
#imager.make_image(column="DATA",dirty=options,restore=False)
#v.CALORNOT = ''
#determine residual CORRECTED_DATA-DATA and image
#residual()
#v.CALORNOT = "res"
#imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False)
#v.CALORNOT = ''
#perform LM calibration
if not (skip_LM):
sim_function(cal=False)
cal_function()
v.CALORNOT = "cal_app_LM"
imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False)
v.CALORNOT = ''
residual()
v.CALORNOT = "cal_res_LM"
imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False)
v.CALORNOT = ''
#perform STEFcal calibration
if not (dist):
sim_function(cal=False)
cal_function(type_cal="STEF")
v.CALORNOT = "cal_app_STEF"
imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False)
v.CALORNOT = ''
residual()
v.CALORNOT = "cal_res_STEF"
imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False)
v.CALORNOT = ''
else: #perform STEFcal calibration (distilation)
sim_function(cal=True,whole=True)
sim_function(cal=False)
cal_function(type_cal="STEF")
v.CALORNOT = "cal_app_STEF"
imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False)
v.CALORNOT = ''
v.CALORNOT = "cal_whole_STEF"
imager.make_image(column="DATA",dirty=options,restore=False)
v.CALORNOT = ''
residual()
v.CALORNOT = "cal_res_dist_STEF"
imager.make_image(column="CORRECTED_DATA",dirty=options,restore=False)
v.CALORNOT = ''
def makecube (npix=512,stokes="I"): imager.make_image(channelize=1,dirty_image="$OUTFILE.cube.fits",npix=npix,wprojplanes=0,stokes=stokes);
def stefcal ( msname="$MS",section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
gain_apply_only=False,
diffgain_apply_only=False,
ifrgain_apply_only=False,
diffgain_intervals=None,diffgain_smoothing=None,
flag_threshold=None,
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,restore=False,restore_lsm=True,
label=None,
args=[],options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname,section,lsm,label,plotvis = interpolate_locals("msname section lsm label plotvis");
makedir(v.DESTDIR);
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label);
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR;
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}");
# setup args
args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.output_column=$STEFCAL_OUTPUT_COLUMN
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s"""%
((1 if diffgains else 0),STEFCAL_TDLOPTS) ];
if diffgains:
if diffgains is True:
diffgains = "=dE";
args0.append("de_subset.subset_enabled=1 de_subset.source_subset=$diffgains");
opts = {
'do_output': output,
'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_gain.table': STEFCAL_GAIN,
'stefcal_gain1.table': STEFCAL_GAIN1,
'stefcal_diffgain.table': STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table': STEFCAL_IFRGAIN,
'stefcal_visualize': False
}
timesmooth,freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (0,0);
timeint,freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,0);
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint;
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint;
opts['stefcal_diffgain.timesmooth'] = timesmooth;
opts['stefcal_diffgain.freqsmooth'] = freqsmooth;
# add user-defined args
args0 += list(args);
opts.update(options);
opts.update(kws);
# run the job
mqt.run(STEFCAL_SCRIPT,STEFCAL_JOBNAME,section=section,args=args0,options=opts);
# copy gains
if not apply_only:
if os.path.exists(STEFCAL_GAIN):
std.copy(STEFCAL_GAIN,STEFCAL_GAIN_SAVE);
if os.path.exists(STEFCAL_GAIN1):
std.copy(STEFCAL_GAIN1,STEFCAL_GAIN1_SAVE);
if os.path.exists(STEFCAL_DIFFGAIN):
std.copy(STEFCAL_DIFFGAIN,STEFCAL_DIFFGAIN_SAVE);
if os.path.exists(STEFCAL_IFRGAIN):
std.copy(STEFCAL_IFRGAIN,STEFCAL_IFRGAIN_SAVE);
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold,(list,tuple)):
t0,t1 = flag_threshold;
else:
t0,t1 = flag_threshold,None;
ms.flagms("--above %g"%t0,"-f threshold -c");
if t1:
ms.flagms("--fm-above %g"%t1,"-f fmthreshold -c");
# plot residuals
if plotvis:
info("Plotting visibilities ($plotvis)");
ms.PLOTVIS = plotvis;
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png");
# make images
imager.make_image(msname,column=STEFCAL_OUTPUT_COLUMN,dirty=dirty,restore=restore,restore_lsm=restore_lsm);
dots = dots.splitlines()
folds = folds.splitlines()
paper = set()
for dot in dots:
x, y = dot.split(',')
x, y = int(x), int(y)
paper.add((x, y))
for fold in folds:
axis, line = re.findall(r'fold along ([xy])=(\d+)', fold)[0]
line = int(line)
match axis:
case "x":
paper = {dot if dot[0] < line
else (2*line - dot[0], dot[1])
for dot in paper}
case "y":
paper = {dot if dot[1] < line
else (dot[0], 2*line - dot[1])
for dot in paper}
max_x = max(paper, key=lambda x: x[0])[0]
max_y = max(paper, key=lambda y: y[1])[1]
grid = np.zeros((max_y+1, max_x+1))
for dot in paper:
grid[dot[1], dot[0]] = 1
imager.make_image(grid, palette, name="code", resize=20)
def send_all():
""" Send all new raids from username to the designated chat in case of error
inform the admin.
"""
all_results = get_all_results()
if len(all_results) == 0:
print("No new results right now.")
return
if SECRETS.CHAT['messenger']['ACTIVATE']:
sett = SECRETS.CHAT['messenger']
try:
print("Logging in...")
ua = sett['user_agent']
client = Client(sett['username'], sett['password'], user_agent=ua)
except Exception as e:
print("Could not login!")
print(traceback.format_exc())
return False
try:
for result in all_results:
if result['type'] == 'spawn' and \
result["pokemon_id"] not in SECRETS.POKEMON['default'] and \
result["iv"] not in ["45"]:
continue
print(result["message"])
make_image(result)
client.sendLocalImage("./img/tmp_raid.png", result["message"],
**sett['chat'])
except Exception as e:
exc = traceback.format_exc()
print("GoMap Messenger failed")
print(exc)
client.sendMessage("GoMap Messenger sender failed. Sorry.",
**sett['admin'])
client.sendMessage(exc, **sett['admin'])
elif SECRETS.CHAT['discord']['ACTIVATE']:
try:
sett = SECRETS.CHAT['discord']
for result in all_results:
make_image(result)
try:
img_link = upload_image()
except UploadError as e:
exc = traceback.format_exc()
print(exc)
txt = "{}: An error while uploading image:\n{}\n{}".format(
datetime.datetime.now().isoformat(), str(e),
e.extra_data.content)
msg = Webhook(
sett['error_webhook'],
msg=txt,
)
msg.post()
img_link = None and "https://i.imgur.com/ZdjcRWW.png"
urls = [sett['raid_webhook']]
tags = ""
if result['type'] == 'spawn':
urls = []
for key, pokemon in SECRETS.POKEMON.items():
if result["pokemon_id"] in pokemon:
urls.append(sett['pokemon_webhook'][key])
if result["iv"].isdigit() and int(result["iv"]) >= 43:
urls.append(sett['highiv_webhook'])
for pkm, val in sett["special_ranks"].items():
if result["pokemon_id"] in pkm:
tags += " <@&{}>".format(val["role_id"])
for name, role_id in result['districts']:
tags += " <@&{}>".format(role_id)
if tags:
tags = "\n" + tags
for url in urls:
msg = Webhook(url,
msg=result['message'] + tags,
image=img_link)
msg.post()
except Exception as e:
exc = traceback.format_exc()
print(exc)
txt = "{}: An error while sending results:\n{}".format(
datetime.datetime.now().isoformat(), exc)
msg = Webhook(
sett['error_webhook'],
msg=txt,
)
msg.post()
def makecube(npix=512, stokes="I"):
imager.make_image(channelize=1,
dirty_image="$OUTFILE.cube.fits",
npix=npix,
wprojplanes=0,
stokes=stokes)
def jointcal(goto_step=1, last_step=10, lsmbase=None, STEPS=None):
"""Calibration for joint C and D-config data"""
info(
">>>>>>>>>>>>> output directory is $OUTDIR. Please set OUTDIR explicitly to override"
)
# setup LSM filenames based on the full MS
# note that these get interpolated once and for all here (and the _Template definitions above
# get cancelled due to the explicit assignment here). The reason for doing it like this
# is because I don't want these names to be changing due to the templates every time the
# MS changes in a per(MS) call.
v.FULLMS = MS
LSM1 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm.lsm.html")
LSM2 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2.lsm.html")
LSM3 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2+cc.lsm.html")
LSM_CCMODEL = II("$DESTDIR/$LSMBASE$SUFFIX+ccmodel.fits")
saveconf()
stefcal.STEFCAL_DIFFGAIN_SMOOTHING = DE_SMOOTHING if not DE_INTERVALS else None
stefcal.STEFCAL_DIFFGAIN_INTERVALS = DE_INTERVALS
if DE_INTERVALS and TILE:
stefcal.STEFCAL_TDLOPTS = "ms_sel.tile_size=%d" % (TILE *
DE_INTERVALS[0])
# make MS list from sub-MSs
import glob
v.MS_List = glob.glob(MS + "/SUBMSS/*MS")
info("MS list is $MS_List")
if not MS_List:
abort("No sub-MSs found")
imager.npix = NPIX = 4096
imager.cellsize = "2arcsec"
imager.wprojplanes = 128
imager.CLEAN_ALGORITHM = "csclean"
v.LSMREF = "${MS:BASE}.refmodel.lsm.html"
THRESH_PIX, THRESH_ISL = (50, 10), (15, 5)
CLEAN_THRESH = ".4mJy", ".1mJy", ".05mJy"
stefcal.STEFCAL_STEP_INCR = 0 # precvent stefcal from auto-incrementing v.STEP: we set the step counter explicitly here
if STEPS is None:
STEPS = list(numpy.arange(goto_step, last_step + .1, .5))
STEPS = map(float, STEPS)
if STEPS[0] != 1:
info("########## restarting calibration from step %.1f" % STEPS[0])
if lsmbase:
LSMBASE = lsmbase
## initial calibration
if 1. in STEPS:
info("########## step 1: solving for G with initial LSM")
v.LSM, v.STEP = LSM0, 1
per_ms(jointcal_g)
if 1.5 in STEPS:
info("########## step 1.5: making joint image")
v.LSM, v.STEP = LSM0, 1
v.MS = FULLMS
# initial model is total flux only, made from a 2x size image to catch distant sources
imager.make_image(dirty=False,
stokes="I",
restore=dict(npix=NPIX * 2,
threshold=CLEAN_THRESH[0],
wprojplanes=128),
restore_lsm=False)
info("########## running source finder and updating model")
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
### NB: select on radius to exclude any artefacts picked up around 3C147 itself
lsm.pybdsm_search(thresh_pix=THRESH_PIX[0],
thresh_isl=THRESH_ISL[0],
select="r.gt.30s",
pol=False)
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f")
# if 2. in STEPS:
# info("########## step 2: repeating G solution");
# v.LSM,v.STEP = LSM1,2
# v.MS = FULLMS
# per_ms(jointcal_g);
if 2. in STEPS:
info("########## step 2: initial dE solution")
v.LSM, v.STEP = LSM1, 2
v.MS = FULLMS
# now, set dE tags on sources
lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC)
per_ms(jointcal_de_reset)
if 3. in STEPS:
info("########## step 3: re-solving for G to apply IFR solutions")
v.LSM, v.STEP = LSM1, 3
v.MS = FULLMS
per_ms(jointcal_de_apply)
info("########## running source finder and updating model")
v.MS = FULLMS
imager.make_image(dirty=False,
stokes="IV",
restore=dict(npix=NPIX,
threshold=CLEAN_THRESH[1],
wprojplanes=128),
restore_lsm=False)
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
### NB: select on radius to exclude any artefacts picked up around 3C147 itself
lsm.pybdsm_search(thresh_pix=THRESH_PIX[1],
thresh_isl=THRESH_ISL[1],
select="r.gt.30s")
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM2 --rename -f")
if 4. in STEPS:
info(
"########## step 4: solving for G+dE with updated LSM (initial+pybdsm^2)"
)
v.MS = FULLMS
v.LSM, v.STEP = LSM2, 4
lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC)
per_ms(jointcal_de)
v.MS = FULLMS
imager.make_image(dirty=False,
stokes="IV",
restore=dict(npix=NPIX,
threshold=CLEAN_THRESH[1],
wprojplanes=128),
restore_lsm=False)
info("########## adding clean components to LSM")
ff = pyfits.open(imager.MODEL_IMAGE)
dd = ff[0].data
dd *= 1.0769 # scale up to compensate for selfcal flux suppression
# dd[dd<0] = 0; # remove negative components
ff.writeto(LSM_CCMODEL, clobber=True)
# add model image to LSM
lsm.tigger_convert("$LSM $LSM3 --add-brick=ccmodel:$LSM_CCMODEL:2 -f")
if 5. in STEPS:
info("########## step 5: re-running DD solutions")
v.MS = FULLMS
v.LSM, v.STEP = LSM3, 5
per_ms(jointcal_de_final)
if 5.5 in STEPS:
info("########## step 5.5: making joint image")
v.MS = FULLMS
v.LSM, v.STEP = LSM3, 5
imager.make_image(dirty=False,
stokes="IQUV",
restore=dict(npix=NPIX,
threshold=CLEAN_THRESH[2],
wprojplanes=128),
restore_lsm=True)
if 6. in STEPS:
info("########## step 6: noise sim")
per_ms(lambda: makecube(stokes="IQUV"))
v.LSM, v.STEP = LSM3, 5
v.MS = FULLMS
makecube(stokes="IQUV")
makenoise()
def jointcal (goto_step=1,last_step=10,lsmbase=None,STEPS=None):
"""Calibration for joint C and D-config data"""
info(">>>>>>>>>>>>> output directory is $OUTDIR. Please set OUTDIR explicitly to override");
# setup LSM filenames based on the full MS
# note that these get interpolated once and for all here (and the _Template definitions above
# get cancelled due to the explicit assignment here). The reason for doing it like this
# is because I don't want these names to be changing due to the templates every time the
# MS changes in a per(MS) call.
v.FULLMS = MS
LSM1 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm.lsm.html");
LSM2 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2.lsm.html");
LSM3 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2+cc.lsm.html");
LSM_CCMODEL = II("$DESTDIR/$LSMBASE$SUFFIX+ccmodel.fits");
saveconf()
stefcal.STEFCAL_DIFFGAIN_SMOOTHING = DE_SMOOTHING if not DE_INTERVALS else None;
stefcal.STEFCAL_DIFFGAIN_INTERVALS = DE_INTERVALS;
if DE_INTERVALS and TILE:
stefcal.STEFCAL_TDLOPTS = "ms_sel.tile_size=%d"%(TILE*DE_INTERVALS[0]);
# make MS list from sub-MSs
import glob
v.MS_List = glob.glob(MS+"/SUBMSS/*MS");
info("MS list is $MS_List");
if not MS_List:
abort("No sub-MSs found");
imager.npix = NPIX = 4096
imager.cellsize = "2arcsec"
imager.wprojplanes = 128
imager.CLEAN_ALGORITHM = "csclean"
v.LSMREF = "${MS:BASE}.refmodel.lsm.html"
THRESH_PIX,THRESH_ISL = (50,10),(15,5)
CLEAN_THRESH = ".4mJy",".1mJy",".05mJy"
stefcal.STEFCAL_STEP_INCR = 0 # precvent stefcal from auto-incrementing v.STEP: we set the step counter explicitly here
if STEPS is None:
STEPS = list(numpy.arange(goto_step,last_step+.1,.5));
STEPS = map(float,STEPS);
if STEPS[0] != 1:
info("########## restarting calibration from step %.1f"%STEPS[0]);
if lsmbase:
LSMBASE = lsmbase;
## initial calibration
if 1. in STEPS:
info("########## step 1: solving for G with initial LSM");
v.LSM,v.STEP = LSM0,1
per_ms(jointcal_g);
if 1.5 in STEPS:
info("########## step 1.5: making joint image");
v.LSM,v.STEP = LSM0,1
v.MS = FULLMS
# initial model is total flux only, made from a 2x size image to catch distant sources
imager.make_image(dirty=False,stokes="I",restore=dict(npix=NPIX*2,threshold=CLEAN_THRESH[0],wprojplanes=128),restore_lsm=False);
info("########## running source finder and updating model");
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
### NB: select on radius to exclude any artefacts picked up around 3C147 itself
lsm.pybdsm_search(thresh_pix=THRESH_PIX[0],thresh_isl=THRESH_ISL[0],select="r.gt.30s",pol=False);
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f");
# if 2. in STEPS:
# info("########## step 2: repeating G solution");
# v.LSM,v.STEP = LSM1,2
# v.MS = FULLMS
# per_ms(jointcal_g);
if 2. in STEPS:
info("########## step 2: initial dE solution");
v.LSM,v.STEP = LSM1,2
v.MS = FULLMS
# now, set dE tags on sources
lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC);
per_ms(jointcal_de_reset);
if 3. in STEPS:
info("########## step 3: re-solving for G to apply IFR solutions");
v.LSM,v.STEP = LSM1,3
v.MS = FULLMS
per_ms(jointcal_de_apply);
info("########## running source finder and updating model");
v.MS = FULLMS
imager.make_image(dirty=False,stokes="IV",restore=dict(npix=NPIX,threshold=CLEAN_THRESH[1],wprojplanes=128),restore_lsm=False);
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
### NB: select on radius to exclude any artefacts picked up around 3C147 itself
lsm.pybdsm_search(thresh_pix=THRESH_PIX[1],thresh_isl=THRESH_ISL[1],select="r.gt.30s");
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM2 --rename -f");
if 4. in STEPS:
info("########## step 4: solving for G+dE with updated LSM (initial+pybdsm^2)");
v.MS = FULLMS
v.LSM,v.STEP = LSM2,4
lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC);
per_ms(jointcal_de);
v.MS = FULLMS
imager.make_image(dirty=False,stokes="IV",restore=dict(npix=NPIX,threshold=CLEAN_THRESH[1],wprojplanes=128),restore_lsm=False);
info("########## adding clean components to LSM");
ff = pyfits.open(imager.MODEL_IMAGE);
dd = ff[0].data;
dd *= 1.0769 # scale up to compensate for selfcal flux suppression
# dd[dd<0] = 0; # remove negative components
ff.writeto(LSM_CCMODEL,clobber=True);
# add model image to LSM
lsm.tigger_convert("$LSM $LSM3 --add-brick=ccmodel:$LSM_CCMODEL:2 -f");
if 5. in STEPS:
info("########## step 5: re-running DD solutions");
v.MS = FULLMS
v.LSM,v.STEP = LSM3,5
per_ms(jointcal_de_final);
if 5.5 in STEPS:
info("########## step 5.5: making joint image");
v.MS = FULLMS
v.LSM,v.STEP = LSM3,5
imager.make_image(dirty=False,stokes="IQUV",restore=dict(npix=NPIX,threshold=CLEAN_THRESH[2],wprojplanes=128),restore_lsm=True);
if 6. in STEPS:
info("########## step 6: noise sim");
per_ms(lambda:makecube(stokes="IQUV"));
v.LSM,v.STEP = LSM3,5
v.MS = FULLMS;
makecube(stokes="IQUV");
makenoise();
def make_images ():
for w,rob in ("natural",0),("uniform",0),("robust",0),("robust",1):
imager.make_image(weight=w,robust=rob,dirty_image="$OUTFILE-$w$rob.fits");
