def main(file_list, ms2uvfits_path='ms2uvfits'):
file_list = file_list.lstrip('[').rstrip(']').split(',')
print file_list
outstem = (file_list[0].split('.')[0])+'.ms'
pt.msconcat (file_list, outstem) # virtually concat files
os.system(ms2uvfits_path+' in='+outstem+' out='+outstem.replace('.ms','.fits')+' writesyscal=F')
return {'fits': outstem.replace('.ms','.fits') } # return name of final file created to main pipeline for further use
def _concat_timeslices(self, group_measurements_collected,
output_file_path):
"""
Msconcat to combine the time slices in a single ms:
It is a virtual ms, a ms with symbolic links to actual data is created!
"""
pt.msconcat(group_measurements_collected,
output_file_path, concatTime = True)
self.logger.debug("Concatenated the files: {0} into the single measure"
"mentset: {1}".format(
", ".join(group_measurements_collected), output_file_path))
def _concat_timeslices(self, group_measurements_collected,
output_file_path):
"""
Msconcat to combine the time slices in a single ms:
It is a virtual ms, a ms with symbolic links to actual data is created!
"""
pt.msconcat(group_measurements_collected,
output_file_path, concatTime = True)
self.logger.debug("Concatenated the files: {0} into the single measure"
"mentset: {1}".format(
", ".join(group_measurements_collected), output_file_path))
# ps.append(p)
#
#waitfunc(ps)
# BBS based direction independent phase calibration using GSM
ps = []
for s in range(nsb):
csb = sb + s
for i in range(lstart, lend + 1, 3):
cmdstr = 'calibrate-stand-alone -t 1 -f ' + wd + 'L' + str(
i
) + '_SB%03d' % csb + ending + ' ' + sd + 'phase_cal.bbs ' + gsmskymodel + ' > ' + wd + 'L' + str(
i) + '_SB%03d_phase_cal.log' % csb
p = Popen(cmdstr, shell=True)
ps.append(p)
waitfunc(ps)
# Create virtual concat
cmdstr = []
for s in range(nsb):
csb = sb + s
for i in range(lstart, lend + 1, 3):
cmdstr.append(wd + 'L' + str(i) + '_SB%03d' % csb + ending)
msconcat(cmdstr, wd + 'smartconcat_%03d' % sb + '_%03d' % (sb + nsb) + '.MS')
print('Calibration run complete')
def selfCalRunFuncImaging(self,stepProcess):
################################################################
# Concatenate in Time and Imaging
################################################################
if stepProcess == 0:
#Concatenate Calibrated Time chunks
listOfFiles = sorted(glob.glob("""%s*"""%(self.IterDir)))
print ''
print listOfFiles
print ''
pt.msconcat(listOfFiles, """%sAll_Iteration_number_%s"""%(self.IterDir,self.i), concatTime=True)
################################################################
#Determine background threshold
################################################################
threshold = 0
if self.i == 0:
# First Iteration threshold: 10xtheorical threshold ~ 5mJy
threshold = 0.05
else:
previousImagePath = '%sImage_%sarcsec_Iter%s.fits'%(self.ImagePathDir,self.pixsize[self.i-1],self.i-1)
# open a FITS file => Estimate the awimager threshold
fitsImage = pyfits.open(previousImagePath) #=> open fits image
scidata = fitsImage[0].data #=> load data
scidata = scidata.reshape((scidata.size,)) #=> flat the data in 1D array (before it was 4D array : freq,stokes, nx, ny)
scidata = scidata.ravel() #=> flat the data in 1D array (before it was 4D array : freq,stokes, nx, ny)
scidata = scidata[scidata<0] #=> select negative values (left part of the noise gaussian centered on 0)
sigma = math.sqrt(scidata.var()) #=> found sigma of that: squre root on the variance
threshold = sigma*5.0 #=> estimate the threshold which at 5 sigma)
################################################################
#Imaging
################################################################
# Using a mask for cleaning
if self.mask == 'yes':
# 0: Initial: needed for generate the mask
if stepProcess == 0:
if self.i < self.nbCycle:
if self.nbpixel[self.i]%2 ==1:
self.nbpixel[self.i] = self.nbpixel[self.i]+1
#Imaging now with the image
cmd_image="""awimager ms=%s image=%sTemporary_Image_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1.18 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i],self.i,self.robust[self.i],self.nbpixel[self.i],self.pixsize[self.i],self.nIteration, self.UVmin,self.UVmax[self.i],self.wmax[self.i],threshold)
print ''
print cmd_image
print ''
os.system(cmd_image)
if self.i == self.nbCycle:
if self.nbpixel[self.i-1]%2 ==1:
self.nbpixel[self.i-1] = self.nbpixel[self.i-1]+1
#Imaging now with the image
cmd_image="""awimager ms=%s image=%sTemporary_Final_Image_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1.18 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i-1],self.i,self.robust[self.i-1],self.nbpixel[self.i-1],self.pixsize[self.i-1],self.nIteration, self.UVmin,self.UVmax[self.i-1],self.wmax[self.i-1],threshold)
print ''
print cmd_image
print ''
os.system(cmd_image)
# 1: cleaning with mask
if stepProcess == 1:
if self.i < self.nbCycle:
if self.nbpixel[self.i]%2 ==1:
self.nbpixel[self.i] = self.nbpixel[self.i]+1
#Imaging now with the image
cmd_image="""awimager ms=%s image=%sImage_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1.18 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy mask=%s"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i],self.i,self.robust[self.i],self.nbpixel[self.i],self.pixsize[self.i],self.nIteration, self.UVmin,self.UVmax[self.i],self.wmax[self.i],threshold,"""%sMask_Iter%s.casa"""%(self.SkymodelPath,self.i+1))
print ''
print cmd_image
print ''
os.system(cmd_image)
if self.i == self.nbCycle:
if self.nbpixel[self.i-1]%2 ==1:
self.nbpixel[self.i-1] = self.nbpixel[self.i-1]+1
#Imaging now with the image
cmd_image="""awimager ms=%s image=%sFinal_Image_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1.18 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy mask=%s"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i-1],self.i,self.robust[self.i-1],self.nbpixel[self.i-1],self.pixsize[self.i-1],self.nIteration, self.UVmin,self.UVmax[self.i-1],self.wmax[self.i-1],threshold,"""%sFinal_Mask_Iter%s.casa"""%(self.SkymodelPath,self.i+1))
print ''
print cmd_image
print ''
os.system(cmd_image)
# Using no mask for cleaning
else:
if self.i < self.nbCycle:
if self.nbpixel[self.i]%2 ==1:
self.nbpixel[self.i] = self.nbpixel[self.i]+1
#Imaging now with the image
cmd_image="""awimager ms=%s image=%sImage_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1.18 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i],self.i,self.robust[self.i],self.nbpixel[self.i],self.pixsize[self.i],self.nIteration, self.UVmin,self.UVmax[self.i],self.wmax[self.i],threshold)
print ''
print cmd_image
print ''
os.system(cmd_image)
if self.i == self.nbCycle:
if self.nbpixel[self.i-1]%2 ==1:
self.nbpixel[self.i-1] = self.nbpixel[self.i-1]+1
#Imaging now with the image
cmd_image="""awimager ms=%s image=%sFinal_Image_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1.18 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i-1],self.i,self.robust[self.i-1],self.nbpixel[self.i-1],self.pixsize[self.i-1],self.nIteration, self.UVmin,self.UVmax[self.i-1],self.wmax[self.i-1],threshold)
print ''
print cmd_image
print ''
os.system(cmd_image)
def selfCalRunFuncImaging(self):
####################################################################
# Concatenate in Time and Imaging
####################################################################
#Concatenate Calibrated Time chunks
listOfFiles = sorted(glob.glob("""%s*_Iter%s"""%(self.IterDir,self.i)))
pt.msconcat(listOfFiles, """%sAll_Iteration_number_%s"""%(self.IterDir,self.i), concatTime=True)
#Determine background threshold
threshold = 0
if self.i == 0:
# First Iteration threshold: 10xtheorical threshold ~ 5mJy
threshold = 0.005
else:
previousImage2Convert = '%sImage_%sarcsec_Iter%s.restored'%(self.ImagePathDir,self.pixsize[self.i-1],self.i-1)
previousImagePath = '%sImage_%sarcsec_Iter%s.restored.fits'%(self.ImagePathDir,self.pixsize[self.i-1],self.i-1)
cmd = 'image2fits in=%s out=%s'%(previousImage2Convert,previousImagePath)
print ''
print cmd
print ''
os.system(cmd)
# open a FITS file
fitsImage = pyfits.open(previousImagePath)
scidata = fitsImage[0].data
dataRange = range(fitsImage[0].shape[2])
sortedData = range(fitsImage[0].shape[2]**2)
for i in dataRange:
for j in dataRange:
sortedData[i*fitsImage[0].shape[2]+j] = scidata[0,0,i,j]
sortedData = sorted(sortedData)
# Percent of faintest data to use to determine 5sigma value : use 5%
dataPercent = int(fitsImage[0].shape[2]*0.05)
fiveSigmaData = sum(sortedData[0:dataPercent])/dataPercent
threshold = abs(fiveSigmaData)/5.0*2.335/2.0
if self.i < self.nbCycle:
if self.nbpixel[self.i]%2 ==1:
self.nbpixel[self.i] = self.nbpixel[self.i]+1
#Imaging now with the image
cmd_image="""awimager data.ms=%s output.imagename=%sImage_%sarcsec_Iter%s weight.type=robust weight.robust=%s image.npix=%s image.cellsize=%sarcsec gridding.padding=1.18 clean.niter=%s operation=clean gridding.timewindow=300 data.uvrange=%s~%sklambda data.wmax=%s clean.threshold=%sJy"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i],self.i,self.robust[self.i],self.nbpixel[self.i],self.pixsize[self.i],self.nIteration, self.UVmin,self.UVmax[self.i],self.wmax[self.i],threshold)
print ''
print cmd_image
print ''
os.system(cmd_image)
if self.i == self.nbCycle:
if self.nbpixel[self.i-1]%2 ==1:
self.nbpixel[self.i-1] = self.nbpixel[self.i-1]+1
#Imaging now with the image
cmd_image="""awimager data.ms=%s output.imagename=%sFinal_Image_%sarcsec_Iter%s weight.type=robust weight.robust=%s image.npix=%s image.cellsize=%sarcsec gridding.padding=1.18 clean.niter=%s operation=clean gridding.timewindow=300 data.uvrange=%s~%sklambda data.wmax=%s clean.threshold=%sJy"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i-1],self.i,self.robust[self.i-1],self.nbpixel[self.i-1],self.pixsize[self.i-1],self.nIteration, self.UVmin,self.UVmax[self.i-1],self.wmax[self.i-1],threshold)
print ''
print cmd_image
print ''
os.system(cmd_image)
def main(ms_input,
ms_output,
min_length,
overhead=0.8,
filename=None,
mapfile_dir=None):
"""
Virtually concatenate subbands
Parameters
----------
ms_input : str
String from the list (map) of the calibrator MSs
ms_output : str
String from the outut concatenated MS
min_length : int
minimum amount of subbands to concatenate in frequency necessary to perform the wide-band flagging in the RAM. It data is too big aoflag will use indirect-read.
overhead : float
Only use this fraction of the available memory for deriving the amount of data to be concatenated.
filename: str
Name of output mapfile
mapfile_dir : str
Directory for output mapfile
"""
system_memory = getsystemmemory()
filelist = input2strlist_nomapfile(ms_input)
file_size = getfilesize(filelist[0])
overhead = float(overhead)
print("Detected available system memory is: " +
str(int(((system_memory / 1024. / 1024.) + 0.5))) + " GB")
if overhead * system_memory > global_limit:
system_memory = global_limit
overhead = 1.0
print(
"Number of files to concat will be limited to the global limit of: "
+ str(int(((global_limit / 1024. / 1024.) + 0.5))) + " GB")
max_space = overhead * system_memory / file_size
max_length = len(filelist) / int((len(filelist) / max_space) + 1.)
if max_length >= int(min_length):
memory = '-memory-read'
else:
memory = '-indirect-read'
max_length = len(filelist)
if max_length * file_size > global_limit:
max_space = global_limit / file_size
max_length = len(filelist) / int((len(filelist) / max_space) + 1.)
print(
"Number of files to concat was limited to the global limit of: "
+ str(int(((global_limit / 1024. / 1024.) + 0.5))) + " GB")
print(
"WARNING: The number of concatenated files will thus be lower than the min_length of: "
+ str(min_length))
print("Applying an overhead of: " + str(overhead))
print("The max_length value is: " + str(max_length))
set_ranges = list(numpy.arange(0, len(filelist) + 1, int(max_length)))
set_ranges[-1] = len(filelist)
map_out = DataMap([])
for i in numpy.arange(len(set_ranges) - 1):
f = ms_output + '_' + str(i)
pt.msconcat(filelist[set_ranges[i]:set_ranges[i + 1]], f)
map_out.data.append(DataProduct('localhost', f, False))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'concatmapfile': fileid, 'memory': memory}
return (result)
def __init__(self,obsDir,preprocessDir,preprocessImageDir,preprocessSkymodelDir,preprocessBBSDir,i,listFiles,Files,NbFiles,frequency,UVmin,nIteration,ra_target,dec_target,initNofAnnulusSources):
self.obsDir = obsDir
self.preprocessDir = preprocessDir
self.preprocessImageDir = preprocessImageDir
self.preprocessSkymodelDir = preprocessSkymodelDir
self.preprocessBBSDir = preprocessBBSDir
self.BBSParset = ''
self.i = i
self.listFiles = listFiles
self.Files = Files
self.NbFiles = NbFiles
self.frequency = frequency
self.UVmin = UVmin
self.nIteration = nIteration
self.ra_target = ra_target
self.dec_target = dec_target
self.initNofAnnulusSources = initNofAnnulusSources
#################################################
self.thresh_isl = 6
self.thresh_pix = 8
#################################################
# Create the initial directory
self.IterDir = self.preprocessDir+'Iter%s/'%(self.i)
cmd="""mkdir %s"""%(self.IterDir)
os.system(cmd)
#Copy data from observation directory
if self.i==0:
cmd=""" cp -r %s* %s"""%(self.obsDir,self.IterDir)
os.system(cmd)
print ''
print cmd
print ''
else:
cmd=""" cp -r %s %s"""%(self.preprocessDir+'Iter%s/*sub%s'%(self.i-1,self.i-1),self.IterDir)
print ''
print cmd
os.system(cmd)
print ''
# Concatenate Initial Time chunks
self.list0 = sorted(glob.glob(('%s*')%(self.IterDir)))
self.initialConcatMS = '%stimeConcatMS'%(self.IterDir)
if os.path.isdir(self.initialConcatMS) != True:
pt.msconcat(self.list0, self.initialConcatMS, concatTime=True)
elevationList = pt.taql('time calc mscal.azel1()[1] from [select from %s where ANTENNA1=1 orderby unique TIME,ANTENNA1]'%(self.initialConcatMS))
nbElevation = len(elevationList)
avgElevation = sum(elevationList)/nbElevation
#################################################
# Determine the fov for calibration step
if self.frequency > 1.9E8:
fov = float(fpformat.fix(1.5/(math.sin(avgElevation)**2),2))
self.thresh_isl = 3
self.thresh_pix = 5
else:
fov = float(fpformat.fix(5.0/(math.sin(avgElevation)**2),2))
##############################
#Extra param for preprocessing:
self.pixPerBeam = 4
self.pixsize = 22.5
self.UVmax = float(fpformat.fix((3E8/self.frequency)/(self.pixPerBeam*self.pixsize/3600.*3.14/180.)/(1E3*3E8/self.frequency),3))
self.wmax = float(fpformat.fix(self.UVmax*(3E8/self.frequency)*1E3,3))
self.nbpixel = int(fov*3600./self.pixsize)
class selfcal_bbs(LOFARnodeTCP):
"""
imager_bbs node performs a bbs run for each of measuremt sets supplied in
the mapfile at ms_list_path. Calibration is done on the sources in
the sourcedb in the mapfile sky_list_path. Solutions are stored in the
parmdb_list_path
1. Load the mapfiles
2. For each measurement set to calibrate start a subprocess
3. Check if the processes finished correctly
4. (added by Nicolas vilchez) concat in time the final MS
5. (added by N.Vilchez) copy time slices directory to a new one
"""
def run(self, bbs_executable, parset, ms_list_path, parmdb_list_path,
sky_list_path, concat_ms_path, major_cycle):
"""
selfcal_bbs functionality. Called by framework performing all the work
"""
self.logger.debug("Starting selfcal_bbs Node")
# *********************************************************************
# 1. Load mapfiles
# read in the mapfiles to data maps: The master recipe added the single
# path to a mapfilem which allows usage of default data methods
# (load_data_map)
# TODO: Datamap
ms_map = MultiDataMap.load(ms_list_path)
parmdb_map = MultiDataMap.load(parmdb_list_path)
sky_list = MultiDataMap.load(sky_list_path)
source_db = sky_list[0].file[0] # the sourcedb is the first file entry
try:
bbs_process_group = SubProcessGroup(self.logger,
self.resourceMonitor)
# *****************************************************************
# 2. start the bbs executable with data
# The data is located in multimaps. We need the first entry
# TODO: THis is not 'nice' usage of the multimap
for (measurement_set, parmdm) in zip(ms_map[0].file,
parmdb_map[0].file):
command = [
bbs_executable, "--sourcedb={0}".format(source_db),
"--parmdb={0}".format(parmdm), measurement_set, parset
]
self.logger.info("Executing bbs command: {0}".format(
" ".join(command)))
bbs_process_group.run(command)
# *****************************************************************
# 3. check status of the processes
if bbs_process_group.wait_for_finish() != None:
self.logger.error("Failed bbs run detected Aborting")
return 1
except OSError, exception:
self.logger.error("Failed to execute bbs: {0}".format(
str(exception)))
return 1
# *********************************************************************
# 4. Concat in time after bbs calibration your MSs using
# msconcat (pyrap.tables module) (added by N.Vilchez)
# this step has te be performed on this location. because the bbs run
# might add additional columns not present in the original ms
# and therefore not produced in the concat done in the prepare phase
# redmine issue #6021
pt.msconcat(ms_map[0].file, concat_ms_path, concatTime=True)
# *********************************************************************
# 5. copy time slives directory to a new one
# This is done for debugging purpose: The copy is not used for anything
# The actual selfcal steps are done in place
# (added by N.Vilchez)
# THe save location is created relative to the concat.ms
# we could also use the self.scratch_directory from the toplevel recipe
# this would need an aditional ingredient
# This is a 'debugging' step and should never ever cause a failure of \
# the pipeline
try:
working_dir = os.path.dirname(concat_ms_path)
time_slice_dir = os.path.join(working_dir, 'time_slices')
time_slice_copy_dir = os.path.join(
working_dir, 'time_slices_cycle_{0}'.format(major_cycle))
cmd = "cp -r {0} {1}".format(time_slice_dir, time_slice_copy_dir)
os.system(cmd)
except:
self.logger.warn(
"Debug copy of temporary files failed: continue operations")
pass # Do nothing
return 0
def selfCalRunFuncImaging(self):
#Concatenate Calibrated Time chunks
listOfFiles = sorted(
glob.glob("""%s*_Iter%s""" % (self.IterDir, self.i)))
pt.msconcat(listOfFiles,
"""%sAll_Iteration_number_%s""" % (self.IterDir, self.i),
concatTime=True)
#Determine background threshold
threshold = 0
if self.i == 0:
# First Iteration threshold: 10xtheorical threshold ~ 5mJy
threshold = 0.005
else:
previousImage2Convert = '%sImage_%sarcsec_Iter%s.restored' % (
self.ImagePathDir, self.pixsize[self.i - 1], self.i - 1)
previousImagePath = '%sImage_%sarcsec_Iter%s.restored.fits' % (
self.ImagePathDir, self.pixsize[self.i - 1], self.i - 1)
cmd = 'image2fits in=%s out=%s' % (previousImage2Convert,
previousImagePath)
print ''
print cmd
print ''
os.system(cmd)
# open a FITS file
fitsImage = pyfits.open(previousImagePath)
scidata = fitsImage[0].data
dataRange = range(fitsImage[0].shape[2])
sortedData = range(fitsImage[0].shape[2]**2)
for i in dataRange:
for j in dataRange:
sortedData[i * fitsImage[0].shape[2] + j] = scidata[0, 0,
i, j]
sortedData = sorted(sortedData)
# Percent of faintest data to use to determine 5sigma value : use 5%
dataPercent = int(fitsImage[0].shape[2] * 0.05)
fiveSigmaData = sum(sortedData[0:dataPercent]) / dataPercent
threshold = abs(fiveSigmaData) / 5.0 * 2.335 / 2.0
if self.i < self.nbCycle:
if self.nbpixel[self.i] % 2 == 1:
self.nbpixel[self.i] = self.nbpixel[self.i] + 1
#Imaging now with the image
cmd_image = """awimager ms=%s image=%sImage_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy""" % (
"""%sAll_Iteration_number_%s""" %
(self.IterDir, self.i), self.ImagePathDir,
self.pixsize[self.i], self.i, self.robust[self.i],
self.nbpixel[self.i], self.pixsize[self.i], self.nIteration,
self.UVmin, self.UVmax[self.i], self.wmax[self.i], threshold)
print ''
print cmd_image
print ''
os.system(cmd_image)
if self.i == self.nbCycle:
if self.nbpixel[self.i - 1] % 2 == 1:
self.nbpixel[self.i - 1] = self.nbpixel[self.i - 1] + 1
#Imaging now with the image
cmd_image = """awimager ms=%s image=%sFinal_Image_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy""" % (
"""%sAll_Iteration_number_%s""" % (self.IterDir, self.i),
self.ImagePathDir, self.pixsize[self.i - 1], self.i,
self.robust[self.i - 1], self.nbpixel[self.i - 1],
self.pixsize[self.i - 1], self.nIteration, self.UVmin,
self.UVmax[self.i - 1], self.wmax[self.i - 1], threshold)
print ''
print cmd_image
print ''
os.system(cmd_image)
def __init__(self, dataDir, preprocessDir, preprocessImageDir,
preprocessSkymodelDir, preprocessBBSDir, i, listFiles, Files,
NbFiles, frequency, UVmin, nIteration, ra_target, dec_target,
initNofAnnulusSources, annulusRadius, FOV):
################################################################
# Initialization
################################################################
self.dataDir = dataDir
self.preprocessDir = preprocessDir
self.preprocessImageDir = preprocessImageDir
self.preprocessSkymodelDir = preprocessSkymodelDir
self.preprocessBBSDir = preprocessBBSDir
self.BBSParset = ''
self.i = i
self.listFiles = listFiles
self.Files = Files
self.NbFiles = NbFiles
self.frequency = frequency
self.UVmin = UVmin
self.nIteration = nIteration
self.ra_target = ra_target
self.dec_target = dec_target
self.initNofAnnulusSources = initNofAnnulusSources
self.annulusRadius = annulusRadius
self.FOV = FOV
self.OriginalFOV = FOV
################################################################
# Default parameter for HBA and LBA obs => changes for High HBA
################################################################
self.thresh_isl = 6
self.thresh_pix = 8
################################################################
# Create the initial directory
################################################################
self.IterDir = self.preprocessDir + 'Iter%s/' % (self.i)
cmd = """mkdir %s""" % (self.IterDir)
os.system(cmd)
#Copy data from observation directory
if self.i == 0:
cmd = """ cp -r %s* %s""" % (self.dataDir, self.IterDir)
print ''
print cmd
os.system(cmd)
print ''
else:
cmd = """ cp -r %s %s""" % (self.preprocessDir + 'Iter%s/*sub%s' %
(self.i - 1, self.i - 1), self.IterDir)
print ''
print cmd
os.system(cmd)
print ''
# Concatenate Initial Time chunks
self.list0 = sorted(glob.glob(('%s*') % (self.IterDir)))
self.initialConcatMS = '%stimeConcatMS' % (self.IterDir)
if os.path.isdir(self.initialConcatMS) != True:
pt.msconcat(self.list0, self.initialConcatMS, concatTime=True)
################################################################
# Determine the fov for calibration step
################################################################
if self.frequency > 1.9E8:
print ''
print '########### PreProcessing INFOS ########################'
print 'This is a High HBA observation. It requires a small field of view, and so a small annulus subraction.'
print 'If FOV > 1.25 => FOV=1.25. If Annulus Radius >0.25 => Annulus Radius=0.25'
if self.FOV > 1.25:
self.FOV = 1.25
self.OriginalFOV = 1.25
if self.annulusRadius > 0.25:
self.annulusRadius = 0.25
print 'FOV = FOV (%s) + 2 x Annulus Radius(%s)' % (
self.FOV, self.annulusRadius)
self.FOV = self.OriginalFOV + 2 * self.annulusRadius
self.thresh_isl = 3
self.thresh_pix = 5
print 'Current FOV=%s degree and Annulus Radius=%s degree' % (
self.FOV, self.annulusRadius)
print 'Extraction Parameters changed for High HBA: thresh_isl=3 (instead 6); thresh_pix=5 (instead 8)'
print ''
print ''
else:
print ''
print '########### PreProcessing INFOS ########################'
print 'This is a LOFAR HBA or LBA observation.'
print 'If FOV > 15 => FOV=15. If Annulus Radius >5 => Annulus Radius=5'
print 'If FOV + 2 x Annulus Radius >10 => FOV=15 and Annulus Radius=2.5'
if self.FOV > 15:
self.FOV = 15
self.OriginalFOV = 15
if self.annulusRadius > 5:
self.annulusRadius = 5
if (self.FOV + self.annulusRadius) > 15:
self.FOV = 10
self.OriginalFOV = 10
self.annulusRadius = 2.5
print 'FOV = FOV (%s) + 2 x Annulus Radius(%s)' % (
self.FOV, self.annulusRadius)
self.FOV = self.FOV + 2 * self.annulusRadius
print 'Current TOTAL FOV=%s degree and Annulus Radius=%s degree' % (
self.FOV, self.annulusRadius)
print ''
################################################################
# Extra param for preprocessing:
################################################################
self.pixPerBeam = 4
self.pixsize = 15.0
self.UVmax = float(
fpformat.fix(
(3E8 / self.frequency) /
(self.pixPerBeam * self.pixsize / 3600. * 3.14 / 180.) /
(1E3 * 3E8 / self.frequency), 3))
self.wmax = float(
fpformat.fix(self.UVmax * (3E8 / self.frequency) * 1E3, 3))
################################################################
# Adapt Number of pixel to right value for FT
################################################################
multipleOfTwo = 2**(np.arange(15))
multipleOfThree = 3**(np.arange(10))
multipleOfFive = 5**(np.arange(6))
matrixOfGoodNumber = multipleOfTwo[:, np.newaxis,
np.newaxis] * multipleOfThree[
np.newaxis, :,
np.newaxis] * multipleOfFive[
np.newaxis, np.newaxis, :]
matrixFlat = matrixOfGoodNumber.ravel()
matrixFlatSorted = sorted(matrixFlat)
nbpixelTemp = int(self.FOV * 3600. / self.pixsize)
self.nbpixel = int(matrixFlatSorted[np.searchsorted(
matrixFlatSorted, nbpixelTemp)])
print ''
print 'To obtain quicker an optimized image, the nof pixel changed from %s to %s' % (
nbpixelTemp, self.nbpixel)
print 'It is for optimize the Fouriier Transform during the imaging process'
print ''
################################################################
# Check if UVmin is > UVmax
################################################################
if self.UVmin > self.UVmax:
print 'Selected UVmin (%s) > UVmax (%s) at Low resolution for preprocessing !!' % (
self.UVmin, self.UVmax)
print 'New UVmin = UVmax/2'
self.UVmin = float(fpformat.fix(self.UVmax / 2.0, 3))
print 'New UVmin = %s (in klambda)' % (self.UVmin)
print ''
print '########################################################'
print ''
print ''
def __init__(self,dataDir,preprocessDir,preprocessImageDir,preprocessSkymodelDir,preprocessBBSDir,i,listFiles,Files,NbFiles,frequency,UVmin,nIteration,ra_target,dec_target,initNofAnnulusSources,annulusRadius,FOV):
################################################################
# Initialization
################################################################
self.dataDir = dataDir
self.preprocessDir = preprocessDir
self.preprocessImageDir = preprocessImageDir
self.preprocessSkymodelDir = preprocessSkymodelDir
self.preprocessBBSDir = preprocessBBSDir
self.BBSParset = ''
self.i = i
self.listFiles = listFiles
self.Files = Files
self.NbFiles = NbFiles
self.frequency = frequency
self.UVmin = UVmin
self.nIteration = nIteration
self.ra_target = ra_target
self.dec_target = dec_target
self.initNofAnnulusSources = initNofAnnulusSources
self.annulusRadius = annulusRadius
self.FOV = FOV
self.OriginalFOV = FOV
################################################################
# Default parameter for HBA and LBA obs => changes for High HBA
################################################################
self.thresh_isl = 6
self.thresh_pix = 8
################################################################
# Create the initial directory
################################################################
self.IterDir = self.preprocessDir+'Iter%s/'%(self.i)
cmd="""mkdir %s"""%(self.IterDir)
os.system(cmd)
#Copy data from observation directory
if self.i==0:
cmd=""" cp -r %s* %s"""%(self.dataDir,self.IterDir)
print ''
print cmd
os.system(cmd)
print ''
else:
cmd=""" cp -r %s %s"""%(self.preprocessDir+'Iter%s/*sub%s'%(self.i-1,self.i-1),self.IterDir)
print ''
print cmd
os.system(cmd)
print ''
# Concatenate Initial Time chunks
self.list0 = sorted(glob.glob(('%s*')%(self.IterDir)))
self.initialConcatMS = '%stimeConcatMS'%(self.IterDir)
if os.path.isdir(self.initialConcatMS) != True:
pt.msconcat(self.list0, self.initialConcatMS, concatTime=True)
################################################################
# Determine the fov for calibration step
################################################################
if self.frequency > 1.9E8:
print ''
print '########### PreProcessing INFOS ########################'
print 'This is a High HBA observation. It requires a small field of view, and so a small annulus subraction.'
print 'If FOV > 1.25 => FOV=1.25. If Annulus Radius >0.25 => Annulus Radius=0.25'
if self.FOV > 1.25:
self.FOV = 1.25
self.OriginalFOV = 1.25
if self.annulusRadius > 0.25:
self.annulusRadius = 0.25
print 'FOV = FOV (%s) + 2 x Annulus Radius(%s)'%(self.FOV,self.annulusRadius)
self.FOV = self.OriginalFOV + 2*self.annulusRadius
self.thresh_isl = 3
self.thresh_pix = 5
print 'Current FOV=%s degree and Annulus Radius=%s degree'%(self.FOV,self.annulusRadius)
print 'Extraction Parameters changed for High HBA: thresh_isl=3 (instead 6); thresh_pix=5 (instead 8)'
print ''
print ''
else:
print ''
print '########### PreProcessing INFOS ########################'
print 'This is a LOFAR HBA or LBA observation.'
print 'If FOV > 15 => FOV=15. If Annulus Radius >5 => Annulus Radius=5'
print 'If FOV + 2 x Annulus Radius >10 => FOV=15 and Annulus Radius=2.5'
if self.FOV > 15:
self.FOV = 15
self.OriginalFOV = 15
if self.annulusRadius > 5:
self.annulusRadius = 5
if (self.FOV + self.annulusRadius) >15:
self.FOV = 10
self.OriginalFOV = 10
self.annulusRadius = 2.5
print 'FOV = FOV (%s) + 2 x Annulus Radius(%s)'%(self.FOV,self.annulusRadius)
self.FOV = self.FOV + 2*self.annulusRadius
print 'Current TOTAL FOV=%s degree and Annulus Radius=%s degree'%(self.FOV,self.annulusRadius)
print ''
################################################################
# Extra param for preprocessing:
################################################################
self.pixPerBeam = 4
self.pixsize = 15.0
self.UVmax = float(fpformat.fix((3E8/self.frequency)/(self.pixPerBeam*self.pixsize/3600.*3.14/180.)/(1E3*3E8/self.frequency),3))
self.wmax = float(fpformat.fix(self.UVmax*(3E8/self.frequency)*1E3,3))
################################################################
# Adapt Number of pixel to right value for FT
################################################################
multipleOfTwo = 2**(np.arange(15))
multipleOfThree = 3**(np.arange(10))
multipleOfFive = 5**(np.arange(6))
matrixOfGoodNumber = multipleOfTwo[:,np.newaxis,np.newaxis]*multipleOfThree[np.newaxis,:,np.newaxis]*multipleOfFive[np.newaxis,np.newaxis,:]
matrixFlat = matrixOfGoodNumber.ravel()
matrixFlatSorted = sorted(matrixFlat)
nbpixelTemp = int(self.FOV*3600./self.pixsize)
self.nbpixel = int(matrixFlatSorted[np.searchsorted(matrixFlatSorted,nbpixelTemp)])
print ''
print 'To obtain quicker an optimized image, the nof pixel changed from %s to %s'%(nbpixelTemp,self.nbpixel)
print 'It is for optimize the Fouriier Transform during the imaging process'
print ''
################################################################
# Check if UVmin is > UVmax
################################################################
if self.UVmin > self.UVmax:
print 'Selected UVmin (%s) > UVmax (%s) at Low resolution for preprocessing !!'%(self.UVmin,self.UVmax)
print 'New UVmin = UVmax/2'
self.UVmin = float(fpformat.fix(self.UVmax/2.0,3))
print 'New UVmin = %s (in klambda)'%(self.UVmin)
print ''
print '########################################################'
print ''
print ''
def __init__(self, obsDir, preprocessDir, preprocessImageDir,
preprocessSkymodelDir, preprocessBBSDir, i, listFiles, Files,
NbFiles, frequency, UVmin, nIteration, ra_target, dec_target,
initNofAnnulusSources):
self.obsDir = obsDir
self.preprocessDir = preprocessDir
self.preprocessImageDir = preprocessImageDir
self.preprocessSkymodelDir = preprocessSkymodelDir
self.preprocessBBSDir = preprocessBBSDir
self.BBSParset = ''
self.i = i
self.listFiles = listFiles
self.Files = Files
self.NbFiles = NbFiles
self.frequency = frequency
self.UVmin = UVmin
self.nIteration = nIteration
self.ra_target = ra_target
self.dec_target = dec_target
self.initNofAnnulusSources = initNofAnnulusSources
#################################################
self.thresh_isl = 6
self.thresh_pix = 8
#################################################
# Create the initial directory
self.IterDir = self.preprocessDir + 'Iter%s/' % (self.i)
cmd = """mkdir %s""" % (self.IterDir)
os.system(cmd)
#Copy data from observation directory
if self.i == 0:
cmd = """ cp -r %s* %s""" % (self.obsDir, self.IterDir)
os.system(cmd)
print ''
print cmd
print ''
else:
cmd = """ cp -r %s %s""" % (self.preprocessDir + 'Iter%s/*sub%s' %
(self.i - 1, self.i - 1), self.IterDir)
print ''
print cmd
os.system(cmd)
print ''
# Concatenate Initial Time chunks
self.list0 = sorted(glob.glob(('%s*') % (self.IterDir)))
self.initialConcatMS = '%stimeConcatMS' % (self.IterDir)
if os.path.isdir(self.initialConcatMS) != True:
pt.msconcat(self.list0, self.initialConcatMS, concatTime=True)
elevationList = pt.taql(
'time calc mscal.azel1()[1] from [select from %s where ANTENNA1=1 orderby unique TIME,ANTENNA1]'
% (self.initialConcatMS))
nbElevation = len(elevationList)
avgElevation = sum(elevationList) / nbElevation
#################################################
# Determine the fov for calibration step
if self.frequency > 1.9E8:
fov = float(fpformat.fix(1.5 / (math.sin(avgElevation)**2), 2))
self.thresh_isl = 3
self.thresh_pix = 5
else:
fov = float(fpformat.fix(5.0 / (math.sin(avgElevation)**2), 2))
##############################
#Extra param for preprocessing:
self.pixPerBeam = 4
self.pixsize = 22.5
self.UVmax = float(
fpformat.fix(
(3E8 / self.frequency) /
(self.pixPerBeam * self.pixsize / 3600. * 3.14 / 180.) /
(1E3 * 3E8 / self.frequency), 3))
self.wmax = float(
fpformat.fix(self.UVmax * (3E8 / self.frequency) * 1E3, 3))
self.nbpixel = int(fov * 3600. / self.pixsize)
