class test_mpi4casa_log_level(unittest.TestCase): def setUp(self): self.vis = "Four_ants_3C286.mms" setUpFile(self.vis,'vis') self.client = MPICommandClient() self.client.set_log_mode('redirect') self.client.start_services() # Prepare list of servers self.server_list = [] server_list = self.client.get_server_status() for server in server_list: if not server_list[server]['timeout']: self.server_list.append(server_list[server]['rank']) self.client.push_command_request("import os",True,self.server_list) def tearDown(self): os.system('rm -rf ' + self.vis) # Restore log file and level self.client.push_command_request("casalog.setlogfile(casa['files']['logfile'])",True,self.server_list) self.client.set_log_level("INFO") def test_mpi4casa_log_level_default_to_debug(self): """Test changing globally log level from default to debug """ # Change log level globally (test via MPIInterface as it internally uses MPICommandClient so both are tested) mpi_interface = MPIInterface() mpi_interface.set_log_level("DEBUG") # Use a separated log file per server to facilitate analysis for server in self.server_list: logfile = 'test_mpi4casa_log_level_debug-server-%s.log' % str(server) self.client.push_command_request("casalog.setlogfile('%s')" % (logfile),True,server) # Run flagdata flagdata(vis=self.vis, mode='summary') # Iterate trough log files to see if we find command handling msgs for server in self.server_list: # Get current working directory (we might be in the 'nosedir' subdirectory) cwd = self.client.push_command_request("os.getcwd()",True,server)[0]['ret'] logfile = '%s/test_mpi4casa_log_level_debug-server-%s.log' % (cwd,str(server)) content = open(logfile, 'r').read() if content.find('flagdata')>0: # Check only server with processed a flagdata sub-job self.assertEqual(content.find("MPICommandServer")<0, True, "MPICommandServer msgs should be filtered out")
class test_mpi4casa_plotms(unittest.TestCase): def setUp(self): self.vis = 'Four_ants_3C286.mms' setUpFile(self.vis,'vis') self.client = MPICommandClient() self.client.set_log_mode('redirect') self.client.start_services() # Prepare list of servers self.server_list = [] server_list = self.client.get_server_status() for server in server_list: if not server_list[server]['timeout']: self.server_list.append(server_list[server]['rank']) def tearDown(self): os.system('rm -rf ' + self.vis) def test_mpi4casa_plotms_concurrent(self): """Run plotms on the same MS from each server simulateneously""" # Change current working directory self.client.push_command_request("os.chdir('%s')" % os.getcwd(),True,self.server_list) # Farm plotms jobs command_request_id_list = [] for server in self.server_list: plotfile = 'test_mpi4casa_plotms_concurrent-%s.png' % str(server) cmd = "plotms('%s', avgchannel='8',avgtime='60',plotfile='%s',showgui=False)" % (self.vis,plotfile) command_request_id = self.client.push_command_request(cmd,False,server) command_request_id_list.append(command_request_id[0]) # Get response in block mode command_response_list = self.client.get_command_response(command_request_id_list,True,True) # Compare files for server_idx in range(0,len(self.server_list)): for server_idy in range(server_idx+1,len(self.server_list)): server_x = self.server_list[server_idx] server_y = self.server_list[server_idy] plotfile_server_idx = 'test_mpi4casa_plotms_concurrent-%s.png' % str(server_x) plotfile_server_idy = 'test_mpi4casa_plotms_concurrent-%s.png' % str(server_y) areEqual = filecmp.cmp(plotfile_server_idx,plotfile_server_idy) self.assertTrue(areEqual,"Plotfile generated by server %s is different from plotfile generated by server %s" % (str(server_x),str(server_y)))
class test_MPICommandServer(unittest.TestCase): def setUp(self): self.client = MPICommandClient() self.client.set_log_mode('unified') self.server_list = MPIEnvironment.mpi_server_rank_list() self.client.start_services() def test_server_cannot_be_instantiated(self): instantiated = False try: server = MPICommandServer() instantiated = True except Exception, instance: instantiated = False self.assertEqual(instantiated, False, "It should not be possible to instantiate MPICommandServer in the client")
class test_MPICommandClient(unittest.TestCase): def setUp(self): self.client = MPICommandClient() self.client.set_log_mode('redirect') self.server_list = MPIEnvironment.mpi_server_rank_list() self.client.start_services() def test_exec_multiple_target_non_blocking_mode_str_params_successful(self): command_request_id_list = self.client.push_command_request("import time; time.sleep(3)",False,[self.server_list[0],self.server_list[1]]) # Try to get responses before time in non-blocking more command_response_list = self.client.get_command_response(command_request_id_list,False,True) # Get response in blocking mode command_response_list = self.client.get_command_response(command_request_id_list,True,True) # Analyze command response list contents self.assertEqual(len(command_response_list), 2, "Command response list should contain two elements") for command_response in command_response_list: self.assertEqual(command_response['successful'], True, "Command execution was not successful") self.assertEqual(command_response['traceback'], None, "Command execution trace-back should be None") self.assertEqual(command_response['status'], 'response received', "Command status should be 'response received'") self.assertEqual(command_response['ret'], None, "Command return variable should be None") def test_eval_multiple_target_blocking_mode_str_params_successful(self): command_response_list = self.client.push_command_request("1+1",True,[self.server_list[0],self.server_list[1]]) self.assertEqual(len(command_response_list), 2, "Command response list should contain two elements") for command_response in command_response_list: self.assertEqual(command_response['successful'], True, "Command execution was not successful") self.assertEqual(command_response['traceback'], None, "Command execution trace-back should be None") self.assertEqual(command_response['status'], 'response received', "Command status should be 'response received'") self.assertEqual(command_response['ret'], 2, "Command return variable should be 2") def test_eval_undefined_target_non_blocking_mode_dict_params_not_successful(self): command_request_id_list = self.client.push_command_request("pow(a,b)",False,None,{'a':'test','b':2}) # Try to get responses before time in non-blocking more command_response_list = self.client.get_command_response(command_request_id_list,False,True) # Get response in blocking mode command_response_list = self.client.get_command_response(command_request_id_list,True,True) # Analyze command response list contents self.assertEqual(len(command_response_list), 1, "Command response list should contain one element") self.assertEqual(command_response_list[0]['successful'], False, "Command execution was successful") self.assertEqual(command_response_list[0]['traceback'].find("TypeError:")>=0, True, "Trace-back should contain TypeError") self.assertEqual(command_response_list[0]['status'], 'response received', "Command status should be 'response received'") self.assertEqual(command_response_list[0]['ret'], None, "Command return variable should be None") def test_singleton_behaviour(self): # Delete current MPICommandClient singleton instance reference client_ref = self.client del client_ref # Create a new MPICommandClient singleton instance reference new_client_ref = MPICommandClient() # Execute some command command_response_list = new_client_ref.push_command_request("a+b",True,[self.server_list[0]],{'a':1,'b':1}) # Analyze command response list contents self.assertEqual(len(command_response_list), 1, "Command response list should contain one element") self.assertEqual(command_response_list[0]['successful'], True, "Command execution was not successful") self.assertEqual(command_response_list[0]['traceback'], None, "Command execution trace-back should be None") self.assertEqual(command_response_list[0]['status'], 'response received', "Command status should be 'response received'") self.assertEqual(command_response_list[0]['ret'], 2, "Command return variable should be 2")
class test_mpi4casa_NullSelection(unittest.TestCase): def setUp(self): self.vis = "Four_ants_3C286.mms" setUpFile(self.vis,'vis') self.client = MPICommandClient() self.client.set_log_mode('redirect') self.client.start_services() # Prepare list of servers self.server_list = [] server_list = self.client.get_server_status() for server in server_list: if not server_list[server]['timeout']: self.server_list.append(server_list[server]['rank']) self.client.push_command_request("import os",True,self.server_list) def tearDown(self): os.system('rm -rf ' + self.vis) # Restore log file and filter self.client.push_command_request("casalog.setlogfile(casa['files']['logfile'])",True,self.server_list) def test_mpi4casa_NullSelection_entire_mms(self): """Test filter out NullSelection exceptions""" # First clear list of filter out msgs. and make sure that the MSSelectionNullSelection shows up for server in self.server_list: logfile = 'MSSelectionNullSelection-Not-Filtered.log-server-%s' % str(server) self.client.push_command_request("casalog.setlogfile('%s'); casalog.clearFilterMsgList()" % (logfile),True,server) # Run flagdata selecting a non-existing scan flagdata(vis=self.vis, scan='99') # Iterate trough log files to see if we find the exception for server in self.server_list: # Get current working directory (we might be in the 'nosedir' subdirectory) cwd = self.client.push_command_request("os.getcwd()",True,server)[0]['ret'] logfile = '%s/MSSelectionNullSelection-Not-Filtered.log-server-%s' % (cwd,str(server)) content = open(logfile, 'r').read() if content.find('flagdata')>0: # Check only server with processed a flagdata sub-job self.assertEqual(content.find("MSSelectionNullSelection")>0, True, "MSSelectionNullSelection should not be filtered out") # Now populate the list of msg to be filter out including MSSelectionNullSelection text = ['MSSelectionNullSelection','NeverHappens'] for server in self.server_list: logfile = 'MSSelectionNullSelection-Filtered.log-server-%s' % str(server) self.client.push_command_request("casalog.setlogfile('%s'); casalog.filterMsg(%s)" % (logfile,str(text)),True,server) # Run flagdata selecting a non-existing scan flagdata(vis=self.vis, scan='99') # Iterate trough log files to see if we find the exception for server in self.server_list: # Get current working directory (we might be in the 'nosedir' subdirectory) cwd = self.client.push_command_request("os.getcwd()",True,server)[0]['ret'] logfile = '%s/MSSelectionNullSelection-Filtered.log-server-%s' % (cwd,str(server)) content = open(logfile, 'r').read() if content.find('flagdata')>0: # Check only server with processed a flagdata sub-job self.assertEqual(content.find("MSSelectionNullSelection")<0, True, "MSSelectionNullSelection should be filtered out")
class test_mpi4casa_applycal(unittest.TestCase): def setUp(self): # Set-up MMS self.vis = "ngc5921.applycal.mms" self.vis_sorted = "ngc5921.applycal.sorted.mms" setUpFile(self.vis,'vis') # Set-up reference MMS self.ref = "ngc5921.applycal.ms" self.ref_sorted = "ngc5921.applycal.sorted.ms" setUpFile(self.ref,'ref') # Set-up auxiliary files self.aux = ["ngc5921.fluxscale", "ngc5921.gcal", "ngc5921.bcal"] setUpFile(self.aux ,'aux') # Repository caltables are pre-v4.1, and we # must update them _before_ applycal to avoid contention casalog.post("Updating pre-v4.1 caltables: %s" % str(self.aux),"WARN","test1_applycal_fluxscale_gcal_bcal") cblocal = cbtool() for oldct in self.aux: cblocal.updatecaltable(oldct) casalog.post("Pre-v4.1 caltables updated","INFO","test_mpi4casa_applycal") # Tmp files self.vis2 = self.vis + '.2' self.vis3 = self.vis + '.3' self.vis_sorted2 = self.vis_sorted + '.2' self.vis_sorted3 = self.vis_sorted + '.3' # Tmp aux files self.aux2 = [] self.aux3 = [] for file in self.aux: self.aux2.append(file + '.2') self.aux3.append(file + '.3') # Set up cluster self.client = MPICommandClient() self.client.set_log_mode('redirect') self.client.start_services() def tearDown(self): # Remove MMS os.system('rm -rf ' + self.vis) os.system('rm -rf ' + self.vis_sorted) # Remove ref MMS os.system('rm -rf ' + self.ref) os.system('rm -rf ' + self.ref_sorted) # Remove tmp files os.system('rm -rf ' + self.vis2) os.system('rm -rf ' + self.vis3) os.system('rm -rf ' + self.vis_sorted2) os.system('rm -rf ' + self.vis_sorted3) # Remove aux files for file in self.aux: os.system('rm -rf ' + file) # Remove tmp aux files for file in self.aux2: os.system('rm -rf ' + file) for file in self.aux3: os.system('rm -rf ' + file) def test1_applycal_fluxscale_gcal_bcal(self): """Test 1: Apply calibration using fluxscal gcal and bcal tables""" # Run applycal in MS mode applycal(vis=self.ref,field='',spw='',selectdata=False,gaintable=self.aux, gainfield=['nearest','nearest','0'], interp=['linear', 'linear','nearest'],spwmap=[]) # Run applycal in MMS mode applycal(vis=self.vis,field='',spw='',selectdata=False,gaintable=self.aux, gainfield=['nearest','nearest','0'], interp=['linear', 'linear','nearest'],spwmap=[]) # Sort file to properly match rows for comparison casalog.post("Sorting vis file: %s" % str(self.vis),"INFO","test1_applycal_fluxscale_gcal_bcal") sortFile(self.vis,self.vis_sorted) casalog.post("Sorting ref file: %s" % str(self.ref),"INFO","test1_applycal_fluxscale_gcal_bcal") sortFile(self.ref,self.ref_sorted) # Compare files compare = testhelper.compTables(self.ref_sorted,self.vis_sorted,['FLAG_CATEGORY']) self.assertTrue(compare)
class test_mpi4casa_flagdata(unittest.TestCase): def setUp(self): self.vis = "Four_ants_3C286.mms" setUpFile(self.vis,'vis') # Tmp files self.vis2 = self.vis + '.2' self.vis3 = self.vis + '.3' # Set up cluster self.client = MPICommandClient() self.client.set_log_mode('redirect') self.client.start_services() def tearDown(self): os.system('rm -rf ' + self.vis) # Remove tmp files os.system('rm -rf ' + self.vis2) os.system('rm -rf ' + self.vis3) def test_mpi4casa_flagdata_list_return(self): """Test support for MMS using flagdata in unflag+clip mode""" # Create list file text = "mode='unflag'\n"\ "mode='clip' clipminmax=[0,0.1]" filename = 'list_flagdata.txt' create_input(text, filename) # step 1: Do unflag+clip flagdata(vis=self.vis, mode='list', inpfile=filename) # step 2: Now do summary ret_dict = flagdata(vis=self.vis, mode='summary') # Check summary self.assertTrue(ret_dict['name']=='Summary') self.assertTrue(ret_dict['spw']['15']['flagged'] == 96284.0) self.assertTrue(ret_dict['spw']['0']['flagged'] == 129711.0) self.assertTrue(ret_dict['spw']['1']['flagged'] == 128551.0) self.assertTrue(ret_dict['spw']['2']['flagged'] == 125686.0) self.assertTrue(ret_dict['spw']['3']['flagged'] == 122862.0) self.assertTrue(ret_dict['spw']['4']['flagged'] == 109317.0) self.assertTrue(ret_dict['spw']['5']['flagged'] == 24481.0) self.assertTrue(ret_dict['spw']['6']['flagged'] == 0) self.assertTrue(ret_dict['spw']['7']['flagged'] == 0) self.assertTrue(ret_dict['spw']['8']['flagged'] == 0) self.assertTrue(ret_dict['spw']['9']['flagged'] == 27422.0) self.assertTrue(ret_dict['spw']['10']['flagged'] == 124638.0) self.assertTrue(ret_dict['spw']['11']['flagged'] == 137813.0) self.assertTrue(ret_dict['spw']['12']['flagged'] == 131896.0) self.assertTrue(ret_dict['spw']['13']['flagged'] == 125074.0) self.assertTrue(ret_dict['spw']['14']['flagged'] == 118039.0) def test_mpi4casa_flagdata_list_return_async(self): """Test flagdata summary in async mode""" # First run flagdata sequentially bypassParallelProcessing = ParallelTaskHelper.getBypassParallelProcessing() ParallelTaskHelper.bypassParallelProcessing(2) res = flagdata(vis=self.vis, mode='summary') ParallelTaskHelper.bypassParallelProcessing(bypassParallelProcessing) # Make a copy of the input MMS for each flagdata instance os.system("cp -r %s %s" % (self.vis,self.vis2)) os.system("cp -r %s %s" % (self.vis,self.vis3)) # Set async mode in ParallelTaskHelper ParallelTaskHelper.setAsyncMode(True) # Run applycal in MMS mode with the first set request_id_1 = flagdata(vis=self.vis, mode='summary') # Run applycal in MMS mode with the second set request_id_2 = flagdata(vis=self.vis2, mode='summary') # Run applycal in MMS mode with the third set request_id_3 = flagdata(vis=self.vis3, mode='summary') # Get response in block mode reques_id_list = request_id_1 + request_id_2 + request_id_3 command_response_list = self.client.get_command_response(reques_id_list,True,True) # Get result res1 = ParallelTaskHelper.getResult(request_id_1,'flagdata') res2 = ParallelTaskHelper.getResult(request_id_2,'flagdata') res3 = ParallelTaskHelper.getResult(request_id_3,'flagdata') # Unset async mode in ParallelTaskHelper ParallelTaskHelper.setAsyncMode(False) self.assertEqual(res1,res, "flagdata dictionary does not match for the first flagdata run") self.assertEqual(res2,res, "flagdata dictionary does not match for the second flagdata run") self.assertEqual(res3,res, "flagdata dictionary does not match for the third flagdata run") def test_mpi4casa_flagdata_list_return_multithreading(self): """Test flagdata summary in multithreading mode""" # First run flagdata sequentially bypassParallelProcessing = ParallelTaskHelper.getBypassParallelProcessing() ParallelTaskHelper.bypassParallelProcessing(2) res = flagdata(vis=self.vis, mode='summary') ParallelTaskHelper.bypassParallelProcessing(bypassParallelProcessing) # Make a copy of the input MMS for each flagdata instance os.system("cp -r %s %s" % (self.vis,self.vis2)) os.system("cp -r %s %s" % (self.vis,self.vis3)) ParallelTaskHelper.setMultithreadingMode(True) # Set up workers cmd1 = "flagdata(vis='%s', mode='summary')" % (self.vis) worker1 = ParallelTaskWorker(cmd1) cmd2 = "flagdata(vis='%s', mode='summary')" % (self.vis2) worker2 = ParallelTaskWorker(cmd2) cmd3 = "flagdata(vis='%s', mode='summary')" % (self.vis3) worker3 = ParallelTaskWorker(cmd3) # Spawn worker threads worker1.start() worker2.start() worker3.start() # Get resulting summary ict from each worker res1 = worker1.getResult() res2 = worker2.getResult() res3 = worker3.getResult() ParallelTaskHelper.setMultithreadingMode(False) # Compare return summary dicts with the one generated with a sequential run self.assertEqual(res1,res, "flagdata dictionary does not match for the first flagdata run") self.assertEqual(res2,res, "flagdata dictionary does not match for the second flagdata run") self.assertEqual(res3,res, "flagdata dictionary does not match for the third flagdata run")