class test_mpi4casa_plotms(unittest.TestCase): def setUp(self): self.vis = 'Four_ants_3C286.mms' setUpFile(self.vis,'vis') self.client = MPICommandClient() self.client.set_log_mode('redirect') self.client.start_services() # Prepare list of servers self.server_list = [] server_list = self.client.get_server_status() for server in server_list: if not server_list[server]['timeout']: self.server_list.append(server_list[server]['rank']) def tearDown(self): os.system('rm -rf ' + self.vis) def test_mpi4casa_plotms_concurrent(self): """Run plotms on the same MS from each server simulateneously""" # Change current working directory self.client.push_command_request("os.chdir('%s')" % os.getcwd(),True,self.server_list) # Farm plotms jobs command_request_id_list = [] for server in self.server_list: plotfile = 'test_mpi4casa_plotms_concurrent-%s.png' % str(server) cmd = "plotms('%s', avgchannel='8',avgtime='60',plotfile='%s',showgui=False)" % (self.vis,plotfile) command_request_id = self.client.push_command_request(cmd,False,server) command_request_id_list.append(command_request_id[0]) # Get response in block mode command_response_list = self.client.get_command_response(command_request_id_list,True,True) # Compare files for server_idx in range(0,len(self.server_list)): for server_idy in range(server_idx+1,len(self.server_list)): server_x = self.server_list[server_idx] server_y = self.server_list[server_idy] plotfile_server_idx = 'test_mpi4casa_plotms_concurrent-%s.png' % str(server_x) plotfile_server_idy = 'test_mpi4casa_plotms_concurrent-%s.png' % str(server_y) areEqual = filecmp.cmp(plotfile_server_idx,plotfile_server_idy) self.assertTrue(areEqual,"Plotfile generated by server %s is different from plotfile generated by server %s" % (str(server_x),str(server_y)))
def getResult(command_request_id_list,taskname): # Access MPICommandClietn singleton instance client = MPICommandClient() # Get response list command_response_list = client.get_command_response(command_request_id_list,True,True) # Format list in the form of vis dict ret_list = {} for command_response in command_response_list: vis = command_response['parameters']['vis'] ret_list[vis] = command_response['ret'] # Consolidate results and return ret = ParallelTaskHelper.consolidateResults(ret_list,taskname) return ret
def getResult(command_request_id_list, taskname): # Access MPICommandClietn singleton instance client = MPICommandClient() # Get response list command_response_list = client.get_command_response(command_request_id_list, True, True) # Format list in the form of vis dict ret_list = {} for command_response in command_response_list: vis = command_response["parameters"]["vis"] ret_list[vis] = command_response["ret"] # Consolidate results and return ret = ParallelTaskHelper.consolidateResults(ret_list, taskname) return ret
class test_MPICommandClient(unittest.TestCase): def setUp(self): self.client = MPICommandClient() self.client.set_log_mode('redirect') self.server_list = MPIEnvironment.mpi_server_rank_list() self.client.start_services() def test_exec_multiple_target_non_blocking_mode_str_params_successful(self): command_request_id_list = self.client.push_command_request("import time; time.sleep(3)",False,[self.server_list[0],self.server_list[1]]) # Try to get responses before time in non-blocking more command_response_list = self.client.get_command_response(command_request_id_list,False,True) # Get response in blocking mode command_response_list = self.client.get_command_response(command_request_id_list,True,True) # Analyze command response list contents self.assertEqual(len(command_response_list), 2, "Command response list should contain two elements") for command_response in command_response_list: self.assertEqual(command_response['successful'], True, "Command execution was not successful") self.assertEqual(command_response['traceback'], None, "Command execution trace-back should be None") self.assertEqual(command_response['status'], 'response received', "Command status should be 'response received'") self.assertEqual(command_response['ret'], None, "Command return variable should be None") def test_eval_multiple_target_blocking_mode_str_params_successful(self): command_response_list = self.client.push_command_request("1+1",True,[self.server_list[0],self.server_list[1]]) self.assertEqual(len(command_response_list), 2, "Command response list should contain two elements") for command_response in command_response_list: self.assertEqual(command_response['successful'], True, "Command execution was not successful") self.assertEqual(command_response['traceback'], None, "Command execution trace-back should be None") self.assertEqual(command_response['status'], 'response received', "Command status should be 'response received'") self.assertEqual(command_response['ret'], 2, "Command return variable should be 2") def test_eval_undefined_target_non_blocking_mode_dict_params_not_successful(self): command_request_id_list = self.client.push_command_request("pow(a,b)",False,None,{'a':'test','b':2}) # Try to get responses before time in non-blocking more command_response_list = self.client.get_command_response(command_request_id_list,False,True) # Get response in blocking mode command_response_list = self.client.get_command_response(command_request_id_list,True,True) # Analyze command response list contents self.assertEqual(len(command_response_list), 1, "Command response list should contain one element") self.assertEqual(command_response_list[0]['successful'], False, "Command execution was successful") self.assertEqual(command_response_list[0]['traceback'].find("TypeError:")>=0, True, "Trace-back should contain TypeError") self.assertEqual(command_response_list[0]['status'], 'response received', "Command status should be 'response received'") self.assertEqual(command_response_list[0]['ret'], None, "Command return variable should be None") def test_singleton_behaviour(self): # Delete current MPICommandClient singleton instance reference client_ref = self.client del client_ref # Create a new MPICommandClient singleton instance reference new_client_ref = MPICommandClient() # Execute some command command_response_list = new_client_ref.push_command_request("a+b",True,[self.server_list[0]],{'a':1,'b':1}) # Analyze command response list contents self.assertEqual(len(command_response_list), 1, "Command response list should contain one element") self.assertEqual(command_response_list[0]['successful'], True, "Command execution was not successful") self.assertEqual(command_response_list[0]['traceback'], None, "Command execution trace-back should be None") self.assertEqual(command_response_list[0]['status'], 'response received', "Command status should be 'response received'") self.assertEqual(command_response_list[0]['ret'], 2, "Command return variable should be 2")
class test_mpi4casa_flagdata(unittest.TestCase): def setUp(self): self.vis = "Four_ants_3C286.mms" setUpFile(self.vis,'vis') # Tmp files self.vis2 = self.vis + '.2' self.vis3 = self.vis + '.3' # Set up cluster self.client = MPICommandClient() self.client.set_log_mode('redirect') self.client.start_services() def tearDown(self): os.system('rm -rf ' + self.vis) # Remove tmp files os.system('rm -rf ' + self.vis2) os.system('rm -rf ' + self.vis3) def test_mpi4casa_flagdata_list_return(self): """Test support for MMS using flagdata in unflag+clip mode""" # Create list file text = "mode='unflag'\n"\ "mode='clip' clipminmax=[0,0.1]" filename = 'list_flagdata.txt' create_input(text, filename) # step 1: Do unflag+clip flagdata(vis=self.vis, mode='list', inpfile=filename) # step 2: Now do summary ret_dict = flagdata(vis=self.vis, mode='summary') # Check summary self.assertTrue(ret_dict['name']=='Summary') self.assertTrue(ret_dict['spw']['15']['flagged'] == 96284.0) self.assertTrue(ret_dict['spw']['0']['flagged'] == 129711.0) self.assertTrue(ret_dict['spw']['1']['flagged'] == 128551.0) self.assertTrue(ret_dict['spw']['2']['flagged'] == 125686.0) self.assertTrue(ret_dict['spw']['3']['flagged'] == 122862.0) self.assertTrue(ret_dict['spw']['4']['flagged'] == 109317.0) self.assertTrue(ret_dict['spw']['5']['flagged'] == 24481.0) self.assertTrue(ret_dict['spw']['6']['flagged'] == 0) self.assertTrue(ret_dict['spw']['7']['flagged'] == 0) self.assertTrue(ret_dict['spw']['8']['flagged'] == 0) self.assertTrue(ret_dict['spw']['9']['flagged'] == 27422.0) self.assertTrue(ret_dict['spw']['10']['flagged'] == 124638.0) self.assertTrue(ret_dict['spw']['11']['flagged'] == 137813.0) self.assertTrue(ret_dict['spw']['12']['flagged'] == 131896.0) self.assertTrue(ret_dict['spw']['13']['flagged'] == 125074.0) self.assertTrue(ret_dict['spw']['14']['flagged'] == 118039.0) def test_mpi4casa_flagdata_list_return_async(self): """Test flagdata summary in async mode""" # First run flagdata sequentially bypassParallelProcessing = ParallelTaskHelper.getBypassParallelProcessing() ParallelTaskHelper.bypassParallelProcessing(2) res = flagdata(vis=self.vis, mode='summary') ParallelTaskHelper.bypassParallelProcessing(bypassParallelProcessing) # Make a copy of the input MMS for each flagdata instance os.system("cp -r %s %s" % (self.vis,self.vis2)) os.system("cp -r %s %s" % (self.vis,self.vis3)) # Set async mode in ParallelTaskHelper ParallelTaskHelper.setAsyncMode(True) # Run applycal in MMS mode with the first set request_id_1 = flagdata(vis=self.vis, mode='summary') # Run applycal in MMS mode with the second set request_id_2 = flagdata(vis=self.vis2, mode='summary') # Run applycal in MMS mode with the third set request_id_3 = flagdata(vis=self.vis3, mode='summary') # Get response in block mode reques_id_list = request_id_1 + request_id_2 + request_id_3 command_response_list = self.client.get_command_response(reques_id_list,True,True) # Get result res1 = ParallelTaskHelper.getResult(request_id_1,'flagdata') res2 = ParallelTaskHelper.getResult(request_id_2,'flagdata') res3 = ParallelTaskHelper.getResult(request_id_3,'flagdata') # Unset async mode in ParallelTaskHelper ParallelTaskHelper.setAsyncMode(False) self.assertEqual(res1,res, "flagdata dictionary does not match for the first flagdata run") self.assertEqual(res2,res, "flagdata dictionary does not match for the second flagdata run") self.assertEqual(res3,res, "flagdata dictionary does not match for the third flagdata run") def test_mpi4casa_flagdata_list_return_multithreading(self): """Test flagdata summary in multithreading mode""" # First run flagdata sequentially bypassParallelProcessing = ParallelTaskHelper.getBypassParallelProcessing() ParallelTaskHelper.bypassParallelProcessing(2) res = flagdata(vis=self.vis, mode='summary') ParallelTaskHelper.bypassParallelProcessing(bypassParallelProcessing) # Make a copy of the input MMS for each flagdata instance os.system("cp -r %s %s" % (self.vis,self.vis2)) os.system("cp -r %s %s" % (self.vis,self.vis3)) ParallelTaskHelper.setMultithreadingMode(True) # Set up workers cmd1 = "flagdata(vis='%s', mode='summary')" % (self.vis) worker1 = ParallelTaskWorker(cmd1) cmd2 = "flagdata(vis='%s', mode='summary')" % (self.vis2) worker2 = ParallelTaskWorker(cmd2) cmd3 = "flagdata(vis='%s', mode='summary')" % (self.vis3) worker3 = ParallelTaskWorker(cmd3) # Spawn worker threads worker1.start() worker2.start() worker3.start() # Get resulting summary ict from each worker res1 = worker1.getResult() res2 = worker2.getResult() res3 = worker3.getResult() ParallelTaskHelper.setMultithreadingMode(False) # Compare return summary dicts with the one generated with a sequential run self.assertEqual(res1,res, "flagdata dictionary does not match for the first flagdata run") self.assertEqual(res2,res, "flagdata dictionary does not match for the second flagdata run") self.assertEqual(res3,res, "flagdata dictionary does not match for the third flagdata run")
def pieflag(vis, field, # data selection parameters refchanfile, fitorder_RR_LL, fitorder_RL_LR, scalethresh, SEFDfile, # scalethresh parameter plotSEFD, dynamicflag, chunktime, # dynamicflag parameters stdmax, maxoffset, staticflag, madmax, # staticflag parameter binsamples, extendflag, boxtime, # extendflag parameters boxthresh): # # Task pieflag # Flags bad data by comparing with clean channels in bandpass-calibrated data. # # Original reference: E. Middelberg, 2006, PASA, 23, 64 # Rewritten for use in CASA and updated to account for wideband # and SEFD effects by Christopher A. Hales 2014. # # Thanks to Kumar Golap, Justo Gonzalez, Jeff Kern, James Robnett, # Urvashi Rau, Sanjay Bhatnagar, and of course Enno Middelberg # for expert advice. Thanks to Emmanuel Momjian for providing # Jansky VLA SEFD data for L and X bands (EVLA Memos 152 and 166) # and to Bryan Butler for providing access to all other bands # from the Jansky VLA Exposure Calculator. # # Version 4.4 released 26 October 2016 # Tested with CASA 4.7.0 using Jansky VLA data # Available at: http://github.com/chrishales/pieflag # # Reference for this version: # C. A. Hales, E. Middelberg, 2014, Astrophysics Source Code Library, 1408.014 # http://adsabs.harvard.edu/abs/2014ascl.soft08014H # startTime = time.time() casalog.origin('pieflag') casalog.post('--> pieflag version 4.4') if (not staticflag) and (not dynamicflag): casalog.post('*** ERROR: You need to select static or dynamic flagging.', 'ERROR') casalog.post('*** ERROR: Exiting pieflag.', 'ERROR') return ms.open(vis) vis=ms.name() ms.close() useMPI = MPIEnvironment.is_mpi_enabled if useMPI: if vis.lower().endswith('.ms'): useMPI=False casalog.post('--> MS will be processed in serial mode.') elif ph.axisType(vis) == 'baseline': # client is ID 0 and will not perform parallel processing, servers start from ID 1 nthreads = MPIEnvironment.rank subms_path = vis+'/SUBMSS/' subms = filter(lambda x: os.path.isdir(os.path.join(subms_path, x)), os.listdir(subms_path)) if len(subms) != nthreads: casalog.post('*** ERROR: Mismatch, MMS tailored for '+str(len(subms))+' engines but '+\ 'CASA session tailored for '+str(nthreads)+' engines.', 'ERROR') casalog.post('*** ERROR: Exiting pieflag.', 'ERROR') return server_list = MPIEnvironment.mpi_server_rank_list() casalog.post('--> Initializing MPI parallel cluster with '+str(nthreads)+' engines.') client = MPICommandClient() client.start_services() # do some detective work to find appropriate path to push to clients syspaths = sys.path n = 0 for k in range(len(syspaths)): if os.path.isfile(syspaths[k]+'/mytasks.py'): for line in open(syspaths[k]+'/mytasks.py','r'): if re.search("task_location\['pieflag'\]",line): if n==0: n += 1 addpath = syspaths[k] elif syspaths[k] != addpath: n += 1 if n == 1: casalog.filter('WARN') #client.set_log_level('WARN') client.push_command_request("casalog.filter('WARN')",True,server_list) client.push_command_request("sys.path.append('"+addpath+"')",True,server_list) client.push_command_request('from task_pieflag import pieflag_getflagstats',True,server_list) client.push_command_request('from task_pieflag import pieflag_flag',True,server_list) casalog.filter('INFO') else: if n == 0: casalog.post('*** ERROR: pieflag mytasks.py installation not found in sys.path', 'ERROR') else: casalog.post('*** ERROR: Ambiguity, sys.path contains more than 1 pieflag installation', 'ERROR') casalog.post('*** (pieflag referenced in '+str(n)+' unique path/mytasks.py)', 'ERROR') casalog.post('*** ERROR: Exiting pieflag.', 'ERROR') return fcall1 = 'pieflag_getflagstats(vis,field,spw,npol,feedbasis)' fcall2 = 'pieflag_flag(vis,datacol,nthreads,field,vtbleLIST,inttime,nant,ddid,spw,refchan,nchan,npol,'+\ 'feedbasis,fitorderLIST,sefdLIST,staticflag,madmax,binsamples,dynamicflag,chunktime,stdmax,'+\ 'maxoffset,extendflag,boxtime,boxthresh)' else: casalog.post('*** ERROR: MMS is not partitioned by baseline. Cannot process.', 'ERROR') casalog.post('*** Use partition() to revert to MS then create baseline MMS.', 'ERROR') casalog.post('*** ERROR: Exiting pieflag.', 'ERROR') return else: if vis.lower().endswith('.mms'): casalog.post('*** ERROR: pieflag cannot handle MMS in non-MPI-enabled CASA session.', 'ERROR') casalog.post('*** ERROR: Exiting pieflag.', 'ERROR') return else: casalog.post('--> MS will be processed in serial mode.') tb.open(vis) if any('CORRECTED_DATA' in colnames for colnames in tb.colnames()): datacol='CORRECTED_DATA' else: datacol='DATA' tb.close() # load in reference channel details # OK, there are probably more elegant ways # of implementing the following code...meh refchandict=json.load(open(refchanfile)) spw=[] for i in refchandict.keys(): spw.append(int(i)) nspw=len(spw) # json doesn't seem to load in the spw order properly # The user might not have entered spw's in order either # so perform sort just in case # note: no need to perform sort on the string versions spw.sort() # now get reference channels in corresponding sorted order refchan=[] for i in range(nspw): refchan.append(refchandict[str(spw[i])]) # open MS and select relevant data ms.open(vis) ms.msselect({'field':str(field)}) # get integration time scan_summary = ms.getscansummary() ms.close() scan_list = [] for scan in scan_summary: if scan_summary[scan]['0']['FieldId'] == field: scan_list.append(int(scan)) inttime=scan_summary[str(scan_list[0])]['0']['IntegrationTime'] # get around potential floating point issues by rounding to nearest 1e-5 seconds if inttime != round(inttime,5): casalog.post('*** WARNING: It seems your integration time is specified to finer than 1e-5 seconds.','WARN') casalog.post('*** pieflag will assume this is a rounding error and carry on.','WARN') for i in range(len(scan_list)): if round(inttime,5) != round(scan_summary[str(scan_list[i])]['0']['IntegrationTime'],5): casalog.post('*** ERROR: Bummer, pieflag is not set up to handle '+\ 'changing integration times throughout your MS.', 'ERROR') casalog.post('*** ERROR: Exiting pieflag.','ERROR') return # get number of baselines tb.open(vis+'/ANTENNA') atble=tb.getcol('NAME') tb.close() nant=atble.shape[0] nbaselines=nant*(nant-1)/2 # channel to frequency (Hz) conversion tb.open(vis+'/SPECTRAL_WINDOW') vtble=tb.getcol('CHAN_FREQ') tb.close() # vtble format is vtble[channel][spw] # assume each spw has the same number of channels nchan=vtble.shape[0] # check that spw frequencies increase monotonically spwcheck=vtble[0,0] for s in range(1,len(vtble[0,:])): if vtble[0,s]<spwcheck: casalog.post("*** ERROR: Your spw's are not ordered with increasing frequency.",'ERROR') casalog.post('*** ERROR: Consider splitting your data and restarting pieflag. Exiting','ERROR') return spwcheck=vtble[0,s] # get number of polarizations, assume they don't change throughout observation # get details from the first user-selected spw within the first scan on target field # note: I won't assume that spw specifies data_desc_id in the main table, even # though in most cases it probably does. Probably overkill given the lack # of checks done elsewhere in this code... tb.open(vis+'/DATA_DESCRIPTION') temptb=tb.query('SPECTRAL_WINDOW_ID='+str(spw[0])) # while here, get the data_desc_id values that pair with spw number tempddid=tb.getcol('SPECTRAL_WINDOW_ID').tolist() ddid=[] for s in range(nspw): ddid.append(tempddid.index(spw[s])) tb.close() polid=temptb.getcell('POLARIZATION_ID') tb.open(vis+'/POLARIZATION') npol=tb.getcell('NUM_CORR',polid) poltype=tb.getcell('CORR_TYPE',polid) tb.close() if not (npol == 2 or npol == 4): casalog.post('*** ERROR: Your data contains '+str(npol)+' polarization products.','ERROR') casalog.post('*** ERROR: pieflag can only handle 2 (eg RR/LL) or 4 (eg RR/RL/LR/LL). Exiting.','ERROR') return # see stokes.h for details if poltype[0] == 5: # circular feedbasis = 1 elif poltype[0] == 9: #linear feedbasis = 0 else: casalog.post('*** ERROR: Your data uses an unsupported feed basis. Exiting','ERROR') return casalog.post('--> Some details about your data:') casalog.post(' data column to process = '+datacol) casalog.post(' integration time = '+str(inttime)+' sec') casalog.post(' number of baselines = '+str(nbaselines)) casalog.post(' spectral windows to process = '+str(spw)) casalog.post(' number of channels per spectral window = '+str(nchan)) if feedbasis: casalog.post(' feed basis = circular') else: casalog.post(' feed basis = linear') casalog.post(' number of polarization products to process = '+str(npol)) casalog.post('--> Statistics of pre-existing flags:') flag0 = np.zeros((nspw,2*npol+2)) for i in range(nspw): casalog.filter('WARN') if useMPI: for k in range(nthreads): param = {'vis':vis+'/SUBMSS/'+subms[k],'field':field,\ 'spw':spw[i],'npol':npol,'feedbasis':feedbasis} if k == 0: pid = client.push_command_request(fcall1,False,None,param) else: pid.append((client.push_command_request(fcall1,False,None,param))[0]) presults = client.get_command_response(pid,True) for k in range(nthreads): flag0[i] += presults[k]['ret'] else: flag0[i] = pieflag_getflagstats(vis,field,spw[i],npol,feedbasis) casalog.filter('INFO') RRs="{:.1f}".format(flag0[i][0]/flag0[i][1]*100.) LLs="{:.1f}".format(flag0[i][2]/flag0[i][3]*100.) TOTs="{:.1f}".format(flag0[i][4]/flag0[i][5]*100.) if npol == 2: if feedbasis: outstr=' flagged data in spw='+str(spw[i])+': RR='+RRs+'% LL='+LLs+'% total='+TOTs+'%' else: outstr=' flagged data in spw='+str(spw[i])+': XX='+RRs+'% YY='+LLs+'% total='+TOTs+'%' else: RLs="{:.1f}".format(flag0[i][6]/flag0[i][7]*100.) LRs="{:.1f}".format(flag0[i][8]/flag0[i][9]*100.) if feedbasis: outstr=' flagged data in spw='+str(spw[i])+': RR='+RRs+'% RL='+RLs+'% LR='+LRs+'% LL='+LLs+'% total='+TOTs+'%' else: outstr=' flagged data in spw='+str(spw[i])+': XX='+RRs+'% XY='+RLs+'% YX='+LRs+'% YY='+LLs+'% total='+TOTs+'%' casalog.post(outstr) # Check there are enough spectral windows to perform the fitting later on. If not, lower the order. if fitorder_RR_LL > nspw-1: if fitorder_RR_LL == 2: if feedbasis: casalog.post('*** WARNING: pieflag needs at least 3 spectral windows to fit for RR or LL spectral curvature.','WARN') else: casalog.post('*** WARNING: pieflag needs at least 3 spectral windows to fit for XX or YY spectral curvature.','WARN') else: if feedbasis: casalog.post('*** WARNING: pieflag needs at least 2 spectral windows to fit for RR or LL spectral index.','WARN') else: casalog.post('*** WARNING: pieflag needs at least 2 spectral windows to fit for XX or YY spectral index.','WARN') if nspw == 2: fitorder_RR_LL=1 else: fitorder_RR_LL=0 casalog.post('*** WARNING: fitorder_RR_LL has been reduced to '+str(int(fitorder_RR_LL))+ ' and','WARN') casalog.post('*** may be reduced further for some baselines if the','WARN') casalog.post('*** reference channel isn\'t available in all selected spw\'s.','WARN') if npol == 2: fitorder = np.zeros(2) fitorder[0] = fitorder_RR_LL fitorder[1] = fitorder_RR_LL elif npol == 4: if fitorder_RL_LR > nspw-1: if fitorder_RL_LR == 2: casalog.post('*** WARNING: pieflag needs at least 3 spectral windows to fit for RL or LR spectral curvature.','WARN') else: casalog.post('*** WARNING: pieflag needs at least 2 spectral windows to fit for RL or LR spectral index.','WARN') if nspw == 2: fitorder_RL_LR=1 else: fitorder_RL_LR=0 casalog.post('*** WARNING: fitorder_RL_LR has been reduced to '+str(int(fitorder_RL_LR))+' and','WARN') casalog.post('*** may be reduced further for some baselines if the','WARN') casalog.post('*** reference channel isn\'t available in all selected spw\'s.','WARN') fitorder = np.zeros(4) fitorder[0] = fitorder_RR_LL fitorder[1] = fitorder_RL_LR fitorder[2] = fitorder_RL_LR fitorder[3] = fitorder_RR_LL if scalethresh: # read in SEFD data and interpolate to get values at our channel frequencies casalog.post('--> Reading in SEFD and interpolating at channel frequencies...') sefdRAW=np.loadtxt(SEFDfile) sefd=np.zeros((nspw,nchan)) if not np.all(np.diff(sefdRAW[:,0]) >= 0): casalog.post('*** ERROR: Your SEFD file must be in order of increasing frequency.','ERROR') casalog.post('*** ERROR: Exiting pieflag.','ERROR') return for i in range(nspw): if (vtble[:,spw[i]].min() < sefdRAW[:,0].min()) or (vtble[:,spw[i]].max() > sefdRAW[:,0].max()): casalog.post('*** ERROR: pieflag cannot extrapolate your SEFD.','ERROR') casalog.post('*** ERROR: Provide new SEFD covering your entire frequency range.','ERROR') casalog.post('*** ERROR: Exiting pieflag.','ERROR') return sefdINTERP = interp1d(sefdRAW[:,0],sefdRAW[:,1]) for i in range(nspw): sefdREFCHAN = sefdINTERP(vtble[refchan[i]][spw[i]]) for j in range(nchan): # values in each spectral window will be relative to the reference channel value sefd[i][j] = sefdINTERP(vtble[j][spw[i]]) / sefdREFCHAN if plotSEFD: # clunky, but works, meh... sefdPLOT=np.zeros((nspw*nchan,3)) k=0 for i in range(nspw): sefdREFCHAN = sefdINTERP(vtble[refchan[i]][spw[i]]) for j in range(nchan): sefdPLOT[k][0] = vtble[j][spw[i]]/1.0e9 sefdPLOT[k][1] = sefd[i][j] * sefdREFCHAN sefdPLOT[k][2] = sefd[i][j] k += 1 f, (ax1, ax2) = plt.subplots(2,sharex=True) ax1.plot(sefdRAW[:,0]/1.0e9,sefdRAW[:,1],'b-',sefdPLOT[:,0],sefdPLOT[:,1],'r.',markersize=10) ax2.plot([sefdRAW[0,0]/1.0e9,sefdRAW[len(sefdRAW[:,0])-1,0]/1.0e9],[1.,1.],'c-',sefdPLOT[:,0],sefdPLOT[:,2],'r.',markersize=10) f.subplots_adjust(hspace=0) plt.setp([a.get_xticklabels() for a in f.axes[:-1]], visible=False) ax1.set_title('relative sensitivity assumed across your band,\nnormalized to the reference channel in each spw') ax1.legend(['raw input','interpolated']) ax1.set_ylabel('SEFD (arbitrary units)') ax2.set_xlabel('frequency (GHz)') ax2.set_ylabel('SEFD (normalized units per spw)') else: sefd=np.ones((nspw,nchan)) if not staticflag: madmax = 0 binsamples = 0 if not dynamicflag: chunktime = 0 stdmax = 0 maxoffset = 0 if not extendflag: boxtime = 0 boxthresh = 0 # forcibly remove all lock files #os.system('find '+vis+' -name "*lock" -print | xargs rm') if useMPI: casalog.post('--> pieflag will now flag your data using '+str(nthreads)+' parallel threads.') casalog.filter('WARN') for k in range(nthreads): param = {'vis':vis+'/SUBMSS/'+subms[k],'datacol':datacol,'nthreads':nthreads,'field':field, 'vtbleLIST':vtble.tolist(),'inttime':inttime,'nant':nant, 'ddid':ddid,'spw':spw,'refchan':refchan,'nchan':nchan,'npol':npol,'feedbasis':feedbasis, 'fitorderLIST':fitorder.tolist(),'sefdLIST':sefd.tolist(), 'staticflag':staticflag,'madmax':madmax,'binsamples':binsamples, 'dynamicflag':dynamicflag,'chunktime':chunktime,'stdmax':stdmax,'maxoffset':maxoffset, 'extendflag':extendflag,'boxtime':boxtime,'boxthresh':boxthresh} if k == 0: pid = client.push_command_request(fcall2,False,None,param) else: pid.append((client.push_command_request(fcall2,False,None,param))[0]) presults = client.get_command_response(pid,True) casalog.filter('INFO') else: casalog.post('--> pieflag will now flag your data in serial mode.') pieflag_flag(vis,datacol,1,field, vtble.tolist(),inttime,nant, ddid,spw,refchan,nchan,npol,feedbasis, fitorder.tolist(),sefd.tolist(), staticflag,madmax,binsamples, dynamicflag,chunktime,stdmax,maxoffset, extendflag,boxtime,boxthresh) # show updated flagging statistics casalog.post('--> Statistics of final flags (including pre-existing):') flag1 = np.zeros((nspw,2*npol+2)) for i in range(nspw): casalog.filter('WARN') if useMPI: for k in range(nthreads): param = {'vis':vis+'/SUBMSS/'+subms[k],'field':field,\ 'spw':spw[i],'npol':npol,'feedbasis':feedbasis} if k == 0: pid = client.push_command_request(fcall1,False,None,param) else: pid.append((client.push_command_request(fcall1,False,None,param))[0]) presults = client.get_command_response(pid,True) for k in range(nthreads): flag1[i] += presults[k]['ret'] else: flag1[i] = pieflag_getflagstats(vis,field,spw[i],npol,feedbasis) casalog.filter('INFO') RRs="{:.1f}".format(flag1[i][0]/flag1[i][1]*100.) LLs="{:.1f}".format(flag1[i][2]/flag1[i][3]*100.) TOTs="{:.1f}".format(flag1[i][4]/flag1[i][5]*100.) if npol == 2: if feedbasis: outstr=' flagged data in spw='+str(spw[i])+': RR='+RRs+'% LL='+LLs+'% total='+TOTs+'%' else: outstr=' flagged data in spw='+str(spw[i])+': XX='+RRs+'% YY='+LLs+'% total='+TOTs+'%' else: RLs="{:.1f}".format(flag1[i][6]/flag1[i][7]*100.) LRs="{:.1f}".format(flag1[i][8]/flag1[i][9]*100.) if feedbasis: outstr=' flagged data in spw='+str(spw[i])+': RR='+RRs+'% RL='+RLs+'% LR='+LRs+'% LL='+LLs+'% total='+TOTs+'%' else: outstr=' flagged data in spw='+str(spw[i])+': XX='+RRs+'% XY='+RLs+'% YX='+LRs+'% YY='+LLs+'% total='+TOTs+'%' casalog.post(outstr) casalog.post('--> Statistics of pieflag flags (excluding pre-existing):') for i in range(nspw): RRs="{:.1f}".format((flag1[i][0]-flag0[i][0])/flag0[i][1]*100.) LLs="{:.1f}".format((flag1[i][2]-flag0[i][2])/flag0[i][3]*100.) TOTs="{:.1f}".format((flag1[i][4]-flag0[i][4])/flag0[i][5]*100.) if npol == 2: if feedbasis: outstr=' data flagged in spw='+str(spw[i])+': RR='+RRs+'% LL='+LLs+'% total='+TOTs+'%' else: outstr=' data flagged in spw='+str(spw[i])+': XX='+RRs+'% YY='+LLs+'% total='+TOTs+'%' else: RLs="{:.1f}".format((flag1[i][6]-flag0[i][6])/flag0[i][7]*100.) LRs="{:.1f}".format((flag1[i][8]-flag0[i][8])/flag0[i][9]*100.) if feedbasis: outstr=' data flagged in spw='+str(spw[i])+': RR='+RRs+'% RL='+RLs+'% LR='+LRs+'% LL='+LLs+'% total='+TOTs+'%' else: outstr=' data flagged in spw='+str(spw[i])+': XX='+RRs+'% XY='+RLs+'% YX='+LRs+'% YY='+LLs+'% total='+TOTs+'%' casalog.post(outstr) # forcibly remove all lock files #os.system('find '+vis+' -name "*lock" -print | xargs rm') if useMPI: #client.set_log_level('INFO') client.push_command_request("casalog.filter('INFO')",True,server_list) t=time.time()-startTime casalog.post('--> pieflag run time: '+str(int(t//3600))+' hours '+\ str(int(t%3600//60))+' minutes '+str(int(t%60))+' seconds')