def main(file_list, ms2uvfits_path='ms2uvfits'): file_list = file_list.lstrip('[').rstrip(']').split(',') print file_list outstem = (file_list[0].split('.')[0])+'.ms' pt.msconcat (file_list, outstem) # virtually concat files os.system(ms2uvfits_path+' in='+outstem+' out='+outstem.replace('.ms','.fits')+' writesyscal=F') return {'fits': outstem.replace('.ms','.fits') } # return name of final file created to main pipeline for further use
def _concat_timeslices(self, group_measurements_collected, output_file_path): """ Msconcat to combine the time slices in a single ms: It is a virtual ms, a ms with symbolic links to actual data is created! """ pt.msconcat(group_measurements_collected, output_file_path, concatTime = True) self.logger.debug("Concatenated the files: {0} into the single measure" "mentset: {1}".format( ", ".join(group_measurements_collected), output_file_path))
# ps.append(p) # #waitfunc(ps) # BBS based direction independent phase calibration using GSM ps = [] for s in range(nsb): csb = sb + s for i in range(lstart, lend + 1, 3): cmdstr = 'calibrate-stand-alone -t 1 -f ' + wd + 'L' + str( i ) + '_SB%03d' % csb + ending + ' ' + sd + 'phase_cal.bbs ' + gsmskymodel + ' > ' + wd + 'L' + str( i) + '_SB%03d_phase_cal.log' % csb p = Popen(cmdstr, shell=True) ps.append(p) waitfunc(ps) # Create virtual concat cmdstr = [] for s in range(nsb): csb = sb + s for i in range(lstart, lend + 1, 3): cmdstr.append(wd + 'L' + str(i) + '_SB%03d' % csb + ending) msconcat(cmdstr, wd + 'smartconcat_%03d' % sb + '_%03d' % (sb + nsb) + '.MS') print('Calibration run complete')
def selfCalRunFuncImaging(self,stepProcess): ################################################################ # Concatenate in Time and Imaging ################################################################ if stepProcess == 0: #Concatenate Calibrated Time chunks listOfFiles = sorted(glob.glob("""%s*"""%(self.IterDir))) print '' print listOfFiles print '' pt.msconcat(listOfFiles, """%sAll_Iteration_number_%s"""%(self.IterDir,self.i), concatTime=True) ################################################################ #Determine background threshold ################################################################ threshold = 0 if self.i == 0: # First Iteration threshold: 10xtheorical threshold ~ 5mJy threshold = 0.05 else: previousImagePath = '%sImage_%sarcsec_Iter%s.fits'%(self.ImagePathDir,self.pixsize[self.i-1],self.i-1) # open a FITS file => Estimate the awimager threshold fitsImage = pyfits.open(previousImagePath) #=> open fits image scidata = fitsImage[0].data #=> load data scidata = scidata.reshape((scidata.size,)) #=> flat the data in 1D array (before it was 4D array : freq,stokes, nx, ny) scidata = scidata.ravel() #=> flat the data in 1D array (before it was 4D array : freq,stokes, nx, ny) scidata = scidata[scidata<0] #=> select negative values (left part of the noise gaussian centered on 0) sigma = math.sqrt(scidata.var()) #=> found sigma of that: squre root on the variance threshold = sigma*5.0 #=> estimate the threshold which at 5 sigma) ################################################################ #Imaging ################################################################ # Using a mask for cleaning if self.mask == 'yes': # 0: Initial: needed for generate the mask if stepProcess == 0: if self.i < self.nbCycle: if self.nbpixel[self.i]%2 ==1: self.nbpixel[self.i] = self.nbpixel[self.i]+1 #Imaging now with the image cmd_image="""awimager ms=%s image=%sTemporary_Image_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1.18 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i],self.i,self.robust[self.i],self.nbpixel[self.i],self.pixsize[self.i],self.nIteration, self.UVmin,self.UVmax[self.i],self.wmax[self.i],threshold) print '' print cmd_image print '' os.system(cmd_image) if self.i == self.nbCycle: if self.nbpixel[self.i-1]%2 ==1: self.nbpixel[self.i-1] = self.nbpixel[self.i-1]+1 #Imaging now with the image cmd_image="""awimager ms=%s image=%sTemporary_Final_Image_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1.18 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i-1],self.i,self.robust[self.i-1],self.nbpixel[self.i-1],self.pixsize[self.i-1],self.nIteration, self.UVmin,self.UVmax[self.i-1],self.wmax[self.i-1],threshold) print '' print cmd_image print '' os.system(cmd_image) # 1: cleaning with mask if stepProcess == 1: if self.i < self.nbCycle: if self.nbpixel[self.i]%2 ==1: self.nbpixel[self.i] = self.nbpixel[self.i]+1 #Imaging now with the image cmd_image="""awimager ms=%s image=%sImage_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1.18 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy mask=%s"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i],self.i,self.robust[self.i],self.nbpixel[self.i],self.pixsize[self.i],self.nIteration, self.UVmin,self.UVmax[self.i],self.wmax[self.i],threshold,"""%sMask_Iter%s.casa"""%(self.SkymodelPath,self.i+1)) print '' print cmd_image print '' os.system(cmd_image) if self.i == self.nbCycle: if self.nbpixel[self.i-1]%2 ==1: self.nbpixel[self.i-1] = self.nbpixel[self.i-1]+1 #Imaging now with the image cmd_image="""awimager ms=%s image=%sFinal_Image_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1.18 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy mask=%s"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i-1],self.i,self.robust[self.i-1],self.nbpixel[self.i-1],self.pixsize[self.i-1],self.nIteration, self.UVmin,self.UVmax[self.i-1],self.wmax[self.i-1],threshold,"""%sFinal_Mask_Iter%s.casa"""%(self.SkymodelPath,self.i+1)) print '' print cmd_image print '' os.system(cmd_image) # Using no mask for cleaning else: if self.i < self.nbCycle: if self.nbpixel[self.i]%2 ==1: self.nbpixel[self.i] = self.nbpixel[self.i]+1 #Imaging now with the image cmd_image="""awimager ms=%s image=%sImage_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1.18 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i],self.i,self.robust[self.i],self.nbpixel[self.i],self.pixsize[self.i],self.nIteration, self.UVmin,self.UVmax[self.i],self.wmax[self.i],threshold) print '' print cmd_image print '' os.system(cmd_image) if self.i == self.nbCycle: if self.nbpixel[self.i-1]%2 ==1: self.nbpixel[self.i-1] = self.nbpixel[self.i-1]+1 #Imaging now with the image cmd_image="""awimager ms=%s image=%sFinal_Image_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1.18 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i-1],self.i,self.robust[self.i-1],self.nbpixel[self.i-1],self.pixsize[self.i-1],self.nIteration, self.UVmin,self.UVmax[self.i-1],self.wmax[self.i-1],threshold) print '' print cmd_image print '' os.system(cmd_image)
def selfCalRunFuncImaging(self): #################################################################### # Concatenate in Time and Imaging #################################################################### #Concatenate Calibrated Time chunks listOfFiles = sorted(glob.glob("""%s*_Iter%s"""%(self.IterDir,self.i))) pt.msconcat(listOfFiles, """%sAll_Iteration_number_%s"""%(self.IterDir,self.i), concatTime=True) #Determine background threshold threshold = 0 if self.i == 0: # First Iteration threshold: 10xtheorical threshold ~ 5mJy threshold = 0.005 else: previousImage2Convert = '%sImage_%sarcsec_Iter%s.restored'%(self.ImagePathDir,self.pixsize[self.i-1],self.i-1) previousImagePath = '%sImage_%sarcsec_Iter%s.restored.fits'%(self.ImagePathDir,self.pixsize[self.i-1],self.i-1) cmd = 'image2fits in=%s out=%s'%(previousImage2Convert,previousImagePath) print '' print cmd print '' os.system(cmd) # open a FITS file fitsImage = pyfits.open(previousImagePath) scidata = fitsImage[0].data dataRange = range(fitsImage[0].shape[2]) sortedData = range(fitsImage[0].shape[2]**2) for i in dataRange: for j in dataRange: sortedData[i*fitsImage[0].shape[2]+j] = scidata[0,0,i,j] sortedData = sorted(sortedData) # Percent of faintest data to use to determine 5sigma value : use 5% dataPercent = int(fitsImage[0].shape[2]*0.05) fiveSigmaData = sum(sortedData[0:dataPercent])/dataPercent threshold = abs(fiveSigmaData)/5.0*2.335/2.0 if self.i < self.nbCycle: if self.nbpixel[self.i]%2 ==1: self.nbpixel[self.i] = self.nbpixel[self.i]+1 #Imaging now with the image cmd_image="""awimager data.ms=%s output.imagename=%sImage_%sarcsec_Iter%s weight.type=robust weight.robust=%s image.npix=%s image.cellsize=%sarcsec gridding.padding=1.18 clean.niter=%s operation=clean gridding.timewindow=300 data.uvrange=%s~%sklambda data.wmax=%s clean.threshold=%sJy"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i],self.i,self.robust[self.i],self.nbpixel[self.i],self.pixsize[self.i],self.nIteration, self.UVmin,self.UVmax[self.i],self.wmax[self.i],threshold) print '' print cmd_image print '' os.system(cmd_image) if self.i == self.nbCycle: if self.nbpixel[self.i-1]%2 ==1: self.nbpixel[self.i-1] = self.nbpixel[self.i-1]+1 #Imaging now with the image cmd_image="""awimager data.ms=%s output.imagename=%sFinal_Image_%sarcsec_Iter%s weight.type=robust weight.robust=%s image.npix=%s image.cellsize=%sarcsec gridding.padding=1.18 clean.niter=%s operation=clean gridding.timewindow=300 data.uvrange=%s~%sklambda data.wmax=%s clean.threshold=%sJy"""%("""%sAll_Iteration_number_%s"""%(self.IterDir,self.i),self.ImagePathDir,self.pixsize[self.i-1],self.i,self.robust[self.i-1],self.nbpixel[self.i-1],self.pixsize[self.i-1],self.nIteration, self.UVmin,self.UVmax[self.i-1],self.wmax[self.i-1],threshold) print '' print cmd_image print '' os.system(cmd_image)
def main(ms_input, ms_output, min_length, overhead=0.8, filename=None, mapfile_dir=None): """ Virtually concatenate subbands Parameters ---------- ms_input : str String from the list (map) of the calibrator MSs ms_output : str String from the outut concatenated MS min_length : int minimum amount of subbands to concatenate in frequency necessary to perform the wide-band flagging in the RAM. It data is too big aoflag will use indirect-read. overhead : float Only use this fraction of the available memory for deriving the amount of data to be concatenated. filename: str Name of output mapfile mapfile_dir : str Directory for output mapfile """ system_memory = getsystemmemory() filelist = input2strlist_nomapfile(ms_input) file_size = getfilesize(filelist[0]) overhead = float(overhead) print("Detected available system memory is: " + str(int(((system_memory / 1024. / 1024.) + 0.5))) + " GB") if overhead * system_memory > global_limit: system_memory = global_limit overhead = 1.0 print( "Number of files to concat will be limited to the global limit of: " + str(int(((global_limit / 1024. / 1024.) + 0.5))) + " GB") max_space = overhead * system_memory / file_size max_length = len(filelist) / int((len(filelist) / max_space) + 1.) if max_length >= int(min_length): memory = '-memory-read' else: memory = '-indirect-read' max_length = len(filelist) if max_length * file_size > global_limit: max_space = global_limit / file_size max_length = len(filelist) / int((len(filelist) / max_space) + 1.) print( "Number of files to concat was limited to the global limit of: " + str(int(((global_limit / 1024. / 1024.) + 0.5))) + " GB") print( "WARNING: The number of concatenated files will thus be lower than the min_length of: " + str(min_length)) print("Applying an overhead of: " + str(overhead)) print("The max_length value is: " + str(max_length)) set_ranges = list(numpy.arange(0, len(filelist) + 1, int(max_length))) set_ranges[-1] = len(filelist) map_out = DataMap([]) for i in numpy.arange(len(set_ranges) - 1): f = ms_output + '_' + str(i) pt.msconcat(filelist[set_ranges[i]:set_ranges[i + 1]], f) map_out.data.append(DataProduct('localhost', f, False)) fileid = os.path.join(mapfile_dir, filename) map_out.save(fileid) result = {'concatmapfile': fileid, 'memory': memory} return (result)
def __init__(self,obsDir,preprocessDir,preprocessImageDir,preprocessSkymodelDir,preprocessBBSDir,i,listFiles,Files,NbFiles,frequency,UVmin,nIteration,ra_target,dec_target,initNofAnnulusSources): self.obsDir = obsDir self.preprocessDir = preprocessDir self.preprocessImageDir = preprocessImageDir self.preprocessSkymodelDir = preprocessSkymodelDir self.preprocessBBSDir = preprocessBBSDir self.BBSParset = '' self.i = i self.listFiles = listFiles self.Files = Files self.NbFiles = NbFiles self.frequency = frequency self.UVmin = UVmin self.nIteration = nIteration self.ra_target = ra_target self.dec_target = dec_target self.initNofAnnulusSources = initNofAnnulusSources ################################################# self.thresh_isl = 6 self.thresh_pix = 8 ################################################# # Create the initial directory self.IterDir = self.preprocessDir+'Iter%s/'%(self.i) cmd="""mkdir %s"""%(self.IterDir) os.system(cmd) #Copy data from observation directory if self.i==0: cmd=""" cp -r %s* %s"""%(self.obsDir,self.IterDir) os.system(cmd) print '' print cmd print '' else: cmd=""" cp -r %s %s"""%(self.preprocessDir+'Iter%s/*sub%s'%(self.i-1,self.i-1),self.IterDir) print '' print cmd os.system(cmd) print '' # Concatenate Initial Time chunks self.list0 = sorted(glob.glob(('%s*')%(self.IterDir))) self.initialConcatMS = '%stimeConcatMS'%(self.IterDir) if os.path.isdir(self.initialConcatMS) != True: pt.msconcat(self.list0, self.initialConcatMS, concatTime=True) elevationList = pt.taql('time calc mscal.azel1()[1] from [select from %s where ANTENNA1=1 orderby unique TIME,ANTENNA1]'%(self.initialConcatMS)) nbElevation = len(elevationList) avgElevation = sum(elevationList)/nbElevation ################################################# # Determine the fov for calibration step if self.frequency > 1.9E8: fov = float(fpformat.fix(1.5/(math.sin(avgElevation)**2),2)) self.thresh_isl = 3 self.thresh_pix = 5 else: fov = float(fpformat.fix(5.0/(math.sin(avgElevation)**2),2)) ############################## #Extra param for preprocessing: self.pixPerBeam = 4 self.pixsize = 22.5 self.UVmax = float(fpformat.fix((3E8/self.frequency)/(self.pixPerBeam*self.pixsize/3600.*3.14/180.)/(1E3*3E8/self.frequency),3)) self.wmax = float(fpformat.fix(self.UVmax*(3E8/self.frequency)*1E3,3)) self.nbpixel = int(fov*3600./self.pixsize)
class selfcal_bbs(LOFARnodeTCP): """ imager_bbs node performs a bbs run for each of measuremt sets supplied in the mapfile at ms_list_path. Calibration is done on the sources in the sourcedb in the mapfile sky_list_path. Solutions are stored in the parmdb_list_path 1. Load the mapfiles 2. For each measurement set to calibrate start a subprocess 3. Check if the processes finished correctly 4. (added by Nicolas vilchez) concat in time the final MS 5. (added by N.Vilchez) copy time slices directory to a new one """ def run(self, bbs_executable, parset, ms_list_path, parmdb_list_path, sky_list_path, concat_ms_path, major_cycle): """ selfcal_bbs functionality. Called by framework performing all the work """ self.logger.debug("Starting selfcal_bbs Node") # ********************************************************************* # 1. Load mapfiles # read in the mapfiles to data maps: The master recipe added the single # path to a mapfilem which allows usage of default data methods # (load_data_map) # TODO: Datamap ms_map = MultiDataMap.load(ms_list_path) parmdb_map = MultiDataMap.load(parmdb_list_path) sky_list = MultiDataMap.load(sky_list_path) source_db = sky_list[0].file[0] # the sourcedb is the first file entry try: bbs_process_group = SubProcessGroup(self.logger, self.resourceMonitor) # ***************************************************************** # 2. start the bbs executable with data # The data is located in multimaps. We need the first entry # TODO: THis is not 'nice' usage of the multimap for (measurement_set, parmdm) in zip(ms_map[0].file, parmdb_map[0].file): command = [ bbs_executable, "--sourcedb={0}".format(source_db), "--parmdb={0}".format(parmdm), measurement_set, parset ] self.logger.info("Executing bbs command: {0}".format( " ".join(command))) bbs_process_group.run(command) # ***************************************************************** # 3. check status of the processes if bbs_process_group.wait_for_finish() != None: self.logger.error("Failed bbs run detected Aborting") return 1 except OSError, exception: self.logger.error("Failed to execute bbs: {0}".format( str(exception))) return 1 # ********************************************************************* # 4. Concat in time after bbs calibration your MSs using # msconcat (pyrap.tables module) (added by N.Vilchez) # this step has te be performed on this location. because the bbs run # might add additional columns not present in the original ms # and therefore not produced in the concat done in the prepare phase # redmine issue #6021 pt.msconcat(ms_map[0].file, concat_ms_path, concatTime=True) # ********************************************************************* # 5. copy time slives directory to a new one # This is done for debugging purpose: The copy is not used for anything # The actual selfcal steps are done in place # (added by N.Vilchez) # THe save location is created relative to the concat.ms # we could also use the self.scratch_directory from the toplevel recipe # this would need an aditional ingredient # This is a 'debugging' step and should never ever cause a failure of \ # the pipeline try: working_dir = os.path.dirname(concat_ms_path) time_slice_dir = os.path.join(working_dir, 'time_slices') time_slice_copy_dir = os.path.join( working_dir, 'time_slices_cycle_{0}'.format(major_cycle)) cmd = "cp -r {0} {1}".format(time_slice_dir, time_slice_copy_dir) os.system(cmd) except: self.logger.warn( "Debug copy of temporary files failed: continue operations") pass # Do nothing return 0
def selfCalRunFuncImaging(self): #Concatenate Calibrated Time chunks listOfFiles = sorted( glob.glob("""%s*_Iter%s""" % (self.IterDir, self.i))) pt.msconcat(listOfFiles, """%sAll_Iteration_number_%s""" % (self.IterDir, self.i), concatTime=True) #Determine background threshold threshold = 0 if self.i == 0: # First Iteration threshold: 10xtheorical threshold ~ 5mJy threshold = 0.005 else: previousImage2Convert = '%sImage_%sarcsec_Iter%s.restored' % ( self.ImagePathDir, self.pixsize[self.i - 1], self.i - 1) previousImagePath = '%sImage_%sarcsec_Iter%s.restored.fits' % ( self.ImagePathDir, self.pixsize[self.i - 1], self.i - 1) cmd = 'image2fits in=%s out=%s' % (previousImage2Convert, previousImagePath) print '' print cmd print '' os.system(cmd) # open a FITS file fitsImage = pyfits.open(previousImagePath) scidata = fitsImage[0].data dataRange = range(fitsImage[0].shape[2]) sortedData = range(fitsImage[0].shape[2]**2) for i in dataRange: for j in dataRange: sortedData[i * fitsImage[0].shape[2] + j] = scidata[0, 0, i, j] sortedData = sorted(sortedData) # Percent of faintest data to use to determine 5sigma value : use 5% dataPercent = int(fitsImage[0].shape[2] * 0.05) fiveSigmaData = sum(sortedData[0:dataPercent]) / dataPercent threshold = abs(fiveSigmaData) / 5.0 * 2.335 / 2.0 if self.i < self.nbCycle: if self.nbpixel[self.i] % 2 == 1: self.nbpixel[self.i] = self.nbpixel[self.i] + 1 #Imaging now with the image cmd_image = """awimager ms=%s image=%sImage_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy""" % ( """%sAll_Iteration_number_%s""" % (self.IterDir, self.i), self.ImagePathDir, self.pixsize[self.i], self.i, self.robust[self.i], self.nbpixel[self.i], self.pixsize[self.i], self.nIteration, self.UVmin, self.UVmax[self.i], self.wmax[self.i], threshold) print '' print cmd_image print '' os.system(cmd_image) if self.i == self.nbCycle: if self.nbpixel[self.i - 1] % 2 == 1: self.nbpixel[self.i - 1] = self.nbpixel[self.i - 1] + 1 #Imaging now with the image cmd_image = """awimager ms=%s image=%sFinal_Image_%sarcsec_Iter%s weight=briggs robust=%s npix=%s cellsize=%sarcsec data=CORRECTED_DATA padding=1 niter=%s stokes=I operation=mfclark timewindow=300 UVmin=%s UVmax=%s wmax=%s fits="" threshold=%sJy""" % ( """%sAll_Iteration_number_%s""" % (self.IterDir, self.i), self.ImagePathDir, self.pixsize[self.i - 1], self.i, self.robust[self.i - 1], self.nbpixel[self.i - 1], self.pixsize[self.i - 1], self.nIteration, self.UVmin, self.UVmax[self.i - 1], self.wmax[self.i - 1], threshold) print '' print cmd_image print '' os.system(cmd_image)
def __init__(self, dataDir, preprocessDir, preprocessImageDir, preprocessSkymodelDir, preprocessBBSDir, i, listFiles, Files, NbFiles, frequency, UVmin, nIteration, ra_target, dec_target, initNofAnnulusSources, annulusRadius, FOV): ################################################################ # Initialization ################################################################ self.dataDir = dataDir self.preprocessDir = preprocessDir self.preprocessImageDir = preprocessImageDir self.preprocessSkymodelDir = preprocessSkymodelDir self.preprocessBBSDir = preprocessBBSDir self.BBSParset = '' self.i = i self.listFiles = listFiles self.Files = Files self.NbFiles = NbFiles self.frequency = frequency self.UVmin = UVmin self.nIteration = nIteration self.ra_target = ra_target self.dec_target = dec_target self.initNofAnnulusSources = initNofAnnulusSources self.annulusRadius = annulusRadius self.FOV = FOV self.OriginalFOV = FOV ################################################################ # Default parameter for HBA and LBA obs => changes for High HBA ################################################################ self.thresh_isl = 6 self.thresh_pix = 8 ################################################################ # Create the initial directory ################################################################ self.IterDir = self.preprocessDir + 'Iter%s/' % (self.i) cmd = """mkdir %s""" % (self.IterDir) os.system(cmd) #Copy data from observation directory if self.i == 0: cmd = """ cp -r %s* %s""" % (self.dataDir, self.IterDir) print '' print cmd os.system(cmd) print '' else: cmd = """ cp -r %s %s""" % (self.preprocessDir + 'Iter%s/*sub%s' % (self.i - 1, self.i - 1), self.IterDir) print '' print cmd os.system(cmd) print '' # Concatenate Initial Time chunks self.list0 = sorted(glob.glob(('%s*') % (self.IterDir))) self.initialConcatMS = '%stimeConcatMS' % (self.IterDir) if os.path.isdir(self.initialConcatMS) != True: pt.msconcat(self.list0, self.initialConcatMS, concatTime=True) ################################################################ # Determine the fov for calibration step ################################################################ if self.frequency > 1.9E8: print '' print '########### PreProcessing INFOS ########################' print 'This is a High HBA observation. It requires a small field of view, and so a small annulus subraction.' print 'If FOV > 1.25 => FOV=1.25. If Annulus Radius >0.25 => Annulus Radius=0.25' if self.FOV > 1.25: self.FOV = 1.25 self.OriginalFOV = 1.25 if self.annulusRadius > 0.25: self.annulusRadius = 0.25 print 'FOV = FOV (%s) + 2 x Annulus Radius(%s)' % ( self.FOV, self.annulusRadius) self.FOV = self.OriginalFOV + 2 * self.annulusRadius self.thresh_isl = 3 self.thresh_pix = 5 print 'Current FOV=%s degree and Annulus Radius=%s degree' % ( self.FOV, self.annulusRadius) print 'Extraction Parameters changed for High HBA: thresh_isl=3 (instead 6); thresh_pix=5 (instead 8)' print '' print '' else: print '' print '########### PreProcessing INFOS ########################' print 'This is a LOFAR HBA or LBA observation.' print 'If FOV > 15 => FOV=15. If Annulus Radius >5 => Annulus Radius=5' print 'If FOV + 2 x Annulus Radius >10 => FOV=15 and Annulus Radius=2.5' if self.FOV > 15: self.FOV = 15 self.OriginalFOV = 15 if self.annulusRadius > 5: self.annulusRadius = 5 if (self.FOV + self.annulusRadius) > 15: self.FOV = 10 self.OriginalFOV = 10 self.annulusRadius = 2.5 print 'FOV = FOV (%s) + 2 x Annulus Radius(%s)' % ( self.FOV, self.annulusRadius) self.FOV = self.FOV + 2 * self.annulusRadius print 'Current TOTAL FOV=%s degree and Annulus Radius=%s degree' % ( self.FOV, self.annulusRadius) print '' ################################################################ # Extra param for preprocessing: ################################################################ self.pixPerBeam = 4 self.pixsize = 15.0 self.UVmax = float( fpformat.fix( (3E8 / self.frequency) / (self.pixPerBeam * self.pixsize / 3600. * 3.14 / 180.) / (1E3 * 3E8 / self.frequency), 3)) self.wmax = float( fpformat.fix(self.UVmax * (3E8 / self.frequency) * 1E3, 3)) ################################################################ # Adapt Number of pixel to right value for FT ################################################################ multipleOfTwo = 2**(np.arange(15)) multipleOfThree = 3**(np.arange(10)) multipleOfFive = 5**(np.arange(6)) matrixOfGoodNumber = multipleOfTwo[:, np.newaxis, np.newaxis] * multipleOfThree[ np.newaxis, :, np.newaxis] * multipleOfFive[ np.newaxis, np.newaxis, :] matrixFlat = matrixOfGoodNumber.ravel() matrixFlatSorted = sorted(matrixFlat) nbpixelTemp = int(self.FOV * 3600. / self.pixsize) self.nbpixel = int(matrixFlatSorted[np.searchsorted( matrixFlatSorted, nbpixelTemp)]) print '' print 'To obtain quicker an optimized image, the nof pixel changed from %s to %s' % ( nbpixelTemp, self.nbpixel) print 'It is for optimize the Fouriier Transform during the imaging process' print '' ################################################################ # Check if UVmin is > UVmax ################################################################ if self.UVmin > self.UVmax: print 'Selected UVmin (%s) > UVmax (%s) at Low resolution for preprocessing !!' % ( self.UVmin, self.UVmax) print 'New UVmin = UVmax/2' self.UVmin = float(fpformat.fix(self.UVmax / 2.0, 3)) print 'New UVmin = %s (in klambda)' % (self.UVmin) print '' print '########################################################' print '' print ''
def __init__(self,dataDir,preprocessDir,preprocessImageDir,preprocessSkymodelDir,preprocessBBSDir,i,listFiles,Files,NbFiles,frequency,UVmin,nIteration,ra_target,dec_target,initNofAnnulusSources,annulusRadius,FOV): ################################################################ # Initialization ################################################################ self.dataDir = dataDir self.preprocessDir = preprocessDir self.preprocessImageDir = preprocessImageDir self.preprocessSkymodelDir = preprocessSkymodelDir self.preprocessBBSDir = preprocessBBSDir self.BBSParset = '' self.i = i self.listFiles = listFiles self.Files = Files self.NbFiles = NbFiles self.frequency = frequency self.UVmin = UVmin self.nIteration = nIteration self.ra_target = ra_target self.dec_target = dec_target self.initNofAnnulusSources = initNofAnnulusSources self.annulusRadius = annulusRadius self.FOV = FOV self.OriginalFOV = FOV ################################################################ # Default parameter for HBA and LBA obs => changes for High HBA ################################################################ self.thresh_isl = 6 self.thresh_pix = 8 ################################################################ # Create the initial directory ################################################################ self.IterDir = self.preprocessDir+'Iter%s/'%(self.i) cmd="""mkdir %s"""%(self.IterDir) os.system(cmd) #Copy data from observation directory if self.i==0: cmd=""" cp -r %s* %s"""%(self.dataDir,self.IterDir) print '' print cmd os.system(cmd) print '' else: cmd=""" cp -r %s %s"""%(self.preprocessDir+'Iter%s/*sub%s'%(self.i-1,self.i-1),self.IterDir) print '' print cmd os.system(cmd) print '' # Concatenate Initial Time chunks self.list0 = sorted(glob.glob(('%s*')%(self.IterDir))) self.initialConcatMS = '%stimeConcatMS'%(self.IterDir) if os.path.isdir(self.initialConcatMS) != True: pt.msconcat(self.list0, self.initialConcatMS, concatTime=True) ################################################################ # Determine the fov for calibration step ################################################################ if self.frequency > 1.9E8: print '' print '########### PreProcessing INFOS ########################' print 'This is a High HBA observation. It requires a small field of view, and so a small annulus subraction.' print 'If FOV > 1.25 => FOV=1.25. If Annulus Radius >0.25 => Annulus Radius=0.25' if self.FOV > 1.25: self.FOV = 1.25 self.OriginalFOV = 1.25 if self.annulusRadius > 0.25: self.annulusRadius = 0.25 print 'FOV = FOV (%s) + 2 x Annulus Radius(%s)'%(self.FOV,self.annulusRadius) self.FOV = self.OriginalFOV + 2*self.annulusRadius self.thresh_isl = 3 self.thresh_pix = 5 print 'Current FOV=%s degree and Annulus Radius=%s degree'%(self.FOV,self.annulusRadius) print 'Extraction Parameters changed for High HBA: thresh_isl=3 (instead 6); thresh_pix=5 (instead 8)' print '' print '' else: print '' print '########### PreProcessing INFOS ########################' print 'This is a LOFAR HBA or LBA observation.' print 'If FOV > 15 => FOV=15. If Annulus Radius >5 => Annulus Radius=5' print 'If FOV + 2 x Annulus Radius >10 => FOV=15 and Annulus Radius=2.5' if self.FOV > 15: self.FOV = 15 self.OriginalFOV = 15 if self.annulusRadius > 5: self.annulusRadius = 5 if (self.FOV + self.annulusRadius) >15: self.FOV = 10 self.OriginalFOV = 10 self.annulusRadius = 2.5 print 'FOV = FOV (%s) + 2 x Annulus Radius(%s)'%(self.FOV,self.annulusRadius) self.FOV = self.FOV + 2*self.annulusRadius print 'Current TOTAL FOV=%s degree and Annulus Radius=%s degree'%(self.FOV,self.annulusRadius) print '' ################################################################ # Extra param for preprocessing: ################################################################ self.pixPerBeam = 4 self.pixsize = 15.0 self.UVmax = float(fpformat.fix((3E8/self.frequency)/(self.pixPerBeam*self.pixsize/3600.*3.14/180.)/(1E3*3E8/self.frequency),3)) self.wmax = float(fpformat.fix(self.UVmax*(3E8/self.frequency)*1E3,3)) ################################################################ # Adapt Number of pixel to right value for FT ################################################################ multipleOfTwo = 2**(np.arange(15)) multipleOfThree = 3**(np.arange(10)) multipleOfFive = 5**(np.arange(6)) matrixOfGoodNumber = multipleOfTwo[:,np.newaxis,np.newaxis]*multipleOfThree[np.newaxis,:,np.newaxis]*multipleOfFive[np.newaxis,np.newaxis,:] matrixFlat = matrixOfGoodNumber.ravel() matrixFlatSorted = sorted(matrixFlat) nbpixelTemp = int(self.FOV*3600./self.pixsize) self.nbpixel = int(matrixFlatSorted[np.searchsorted(matrixFlatSorted,nbpixelTemp)]) print '' print 'To obtain quicker an optimized image, the nof pixel changed from %s to %s'%(nbpixelTemp,self.nbpixel) print 'It is for optimize the Fouriier Transform during the imaging process' print '' ################################################################ # Check if UVmin is > UVmax ################################################################ if self.UVmin > self.UVmax: print 'Selected UVmin (%s) > UVmax (%s) at Low resolution for preprocessing !!'%(self.UVmin,self.UVmax) print 'New UVmin = UVmax/2' self.UVmin = float(fpformat.fix(self.UVmax/2.0,3)) print 'New UVmin = %s (in klambda)'%(self.UVmin) print '' print '########################################################' print '' print ''
def __init__(self, obsDir, preprocessDir, preprocessImageDir, preprocessSkymodelDir, preprocessBBSDir, i, listFiles, Files, NbFiles, frequency, UVmin, nIteration, ra_target, dec_target, initNofAnnulusSources): self.obsDir = obsDir self.preprocessDir = preprocessDir self.preprocessImageDir = preprocessImageDir self.preprocessSkymodelDir = preprocessSkymodelDir self.preprocessBBSDir = preprocessBBSDir self.BBSParset = '' self.i = i self.listFiles = listFiles self.Files = Files self.NbFiles = NbFiles self.frequency = frequency self.UVmin = UVmin self.nIteration = nIteration self.ra_target = ra_target self.dec_target = dec_target self.initNofAnnulusSources = initNofAnnulusSources ################################################# self.thresh_isl = 6 self.thresh_pix = 8 ################################################# # Create the initial directory self.IterDir = self.preprocessDir + 'Iter%s/' % (self.i) cmd = """mkdir %s""" % (self.IterDir) os.system(cmd) #Copy data from observation directory if self.i == 0: cmd = """ cp -r %s* %s""" % (self.obsDir, self.IterDir) os.system(cmd) print '' print cmd print '' else: cmd = """ cp -r %s %s""" % (self.preprocessDir + 'Iter%s/*sub%s' % (self.i - 1, self.i - 1), self.IterDir) print '' print cmd os.system(cmd) print '' # Concatenate Initial Time chunks self.list0 = sorted(glob.glob(('%s*') % (self.IterDir))) self.initialConcatMS = '%stimeConcatMS' % (self.IterDir) if os.path.isdir(self.initialConcatMS) != True: pt.msconcat(self.list0, self.initialConcatMS, concatTime=True) elevationList = pt.taql( 'time calc mscal.azel1()[1] from [select from %s where ANTENNA1=1 orderby unique TIME,ANTENNA1]' % (self.initialConcatMS)) nbElevation = len(elevationList) avgElevation = sum(elevationList) / nbElevation ################################################# # Determine the fov for calibration step if self.frequency > 1.9E8: fov = float(fpformat.fix(1.5 / (math.sin(avgElevation)**2), 2)) self.thresh_isl = 3 self.thresh_pix = 5 else: fov = float(fpformat.fix(5.0 / (math.sin(avgElevation)**2), 2)) ############################## #Extra param for preprocessing: self.pixPerBeam = 4 self.pixsize = 22.5 self.UVmax = float( fpformat.fix( (3E8 / self.frequency) / (self.pixPerBeam * self.pixsize / 3600. * 3.14 / 180.) / (1E3 * 3E8 / self.frequency), 3)) self.wmax = float( fpformat.fix(self.UVmax * (3E8 / self.frequency) * 1E3, 3)) self.nbpixel = int(fov * 3600. / self.pixsize)