def testmix4(self): '''listpartition MMS mix4: check the sizes of the sub-MSs''' output = 'listpartitionmix2.txt' if os.path.exists(output): os.system('rm -rf '+output) listpartition(vis=self.vis, listfile=output) self.assertTrue(os.path.exists(output)) # Compare the sizes of the sub-MSs with the output of du -hs ff = open(output,'r') mslist = ff.readlines() i = 0 for l in mslist: if i == 0: i += 1 continue ll = l.rstrip() rear = ll.rpartition(' ') front = ll.partition(' ') # Now get the du -hs for the same sub-MS # Step into the data directory dusize = ph.getDiskUsage(self.visdata+front[0]) # Compare both self.assertEqual(dusize, rear[2], '%s is not equal to %s for %s'%(dusize,rear[2],front[0])) ff.close()
def test_mms_XXYY_selection(self): '''mstransform: correlation='RR,LL' should select and re-index properly''' self.outputms = '3cRRLL.mms' # spw 0 should not be processed. The selection should happen before the MMS work mstransform(vis=self.vis, outputvis=self.outputms, datacolumn='data', correlation='RR,LL', createmms=True, separationaxis='auto') msmdt = msmdtool() msmdt.open(self.outputms) out_dds = msmdt.datadescids() msmdt.done() ref = [0,1] for i in out_dds: self.assertEqual(out_dds[i], ref[i]) pol_col = th.getVarCol(self.outputms+'/POLARIZATION','NUM_CORR') self.assertEqual(pol_col['r1'][0], 0,'Error in NUM_CORR of POLARIZATION table') self.assertEqual(pol_col['r2'][0], 0,'Error in NUM_CORR of POLARIZATION table') self.assertEqual(pol_col['r3'][0], 2,'Error in NUM_CORR of POLARIZATION table') self.assertEqual(pol_col['r4'][0], 2,'Error in NUM_CORR of POLARIZATION table') # Verify that POLARIZATION table is not re-sized. corr_col = th.getVarCol(self.outputms+'/POLARIZATION', 'NUM_CORR') self.assertEqual(corr_col.keys().__len__(), 4, 'Wrong number of rows in POLARIZATION table') # Check the FEED table # out_feed_spw = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID') # self.assertEqual(len(out_feed_spw.keys()), 52) # listobs, listpartition should not fail listobs(self.outputms, listfile='3c_1.obs') self.assertTrue(os.path.exists('3c_1.obs'), 'Probable error in sub-table re-indexing') listpartition(self.outputms, listfile='3c_2.obs') self.assertTrue(os.path.exists('3c_2.obs'), 'Probable error in sub-table re-indexing')
def testMS2(self): '''listpartition MS2: Save to a file''' output = 'listpartitionms.txt' if os.path.exists(output): os.system('rm -rf '+output) listpartition(vis=self.vis, listfile=output) self.assertTrue(os.path.exists(output), 'Output file does not exist')
def testmix2(self): '''listpartition MMS mix2: Save to a file''' output = 'listpartitionmix1.txt' if os.path.exists(output): os.system('rm -rf '+output) listpartition(vis=self.vis, listfile=output) self.assertTrue(os.path.exists(output), 'Output file %s does not exist'%output) # Check the number of lines in the output file ff = open(output,'r') nlines = len(ff.readlines()) ff.close() self.assertEqual(nlines, 33, 'Wrong number of lines in output')
def test_sepaxis(self): '''Partition: separationaxis=auto''' partition(vis=self.msfile, outputvis=self.mmsfile, spw='0~11',separationaxis='auto', flagbackup=False, disableparallel=True, datacolumn='data') # partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto') self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') # Dictionary with selection to compare with original MS mysel = {'spw':'0~11'} # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Compare nrows of all scans in selection slist = ph.getMMSScans(thisdict) self.assertEqual(slist.__len__(), 2) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s, selection=mysel) # msN = ph.getScanNrows(self.msfile, s) self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %(s, mmsN, msN)) # Compare spw IDs for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel) # ms_spw = ph.getSpwIds(self.msfile, s) self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\ 'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw)) # The separation axis should be written to the output MMS sepaxis = ph.axisType(self.mmsfile) self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
def test_spw_selection(self): '''Partition: create an MMS separated by spws with spw=2,4 selection''' partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='spw', spw='2,4', flagbackup=False, disableparallel=True) self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Dictionary with selection to compare with original MS mysel = {'spw':'2,4'} # Compare nrows of all scans in selection slist = ph.getMMSScans(thisdict) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s, selection=mysel) self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %(s, mmsN, msN)) # spwids are re-indexed. The expected IDs are: # ms_spw = 2 --> mms_spw = 0 # ms_spw = 4 --> mms_spw = 1 # Check that MMS spw IDs have been re-indexed properly indexed_ids = range(2) for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed')
def test_scan_spw(self): '''Partition: separationaxis=scan with spw selection''' partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='scan', spw='1~4,10,11', flagbackup=False, datacolumn='data') self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') self.assertTrue(os.path.exists(self.msfile),'Make sure the input MS is not deleted inside the task') # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Dictionary with selection to compare with original MS mysel = {'spw':'1~4,10,11'} # Compare nrows of all scans in selection slist = ph.getMMSScans(thisdict) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s, selection=mysel) self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %(s, mmsN, msN)) # The comparison should be # ms_spw = 1 --> mms_spw = 0 # ms_spw = 2 --> mms_spw = 1, etc. # Check that MMS spw IDs have been re-indexed properly indexed_ids = range(6) for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed') # The separation axis should be written to the output MMS sepaxis = ph.axisType(self.mmsfile) self.assertEqual(sepaxis, 'scan', 'Partition did not write AxisType correctly in MMS')
def test_scan_selection(self): """Partition: create an MMS using scan selection""" partition( vis=self.msfile, outputvis=self.mmsfile, separationaxis="scan", scan="1,2,3,11", flagbackup=False, disableparallel=True, ) self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test") # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Compare nrows of all scans in selection slist = ph.getMMSScans(thisdict) self.assertEqual(slist.__len__(), 4) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s) self.assertEqual(mmsN, msN, "Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s" % (s, mmsN, msN)) # Compare spw IDs for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) ms_spw = ph.getSpwIds(self.msfile, s) self.assertEqual( mms_spw, ms_spw, "list of spws in scan=%s differs: " "mms_spw=%s <--> ms_spw=%s" % (s, mmsN, msN) )
def test_default_sequential(self): '''Partition: create an MMS with default values in sequential''' partition(vis=self.msfile, outputvis=self.mmsfile, disableparallel=True) self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Get scans of MMS slist = ph.getMMSScans(thisdict) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s) self.assertEqual( mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' % (s, mmsN, msN)) # Compare spw IDs of MS and MMS for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) ms_spw = ph.getSpwIds(self.msfile, s) self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\ 'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN)) # The separation axis should be written to the output MMS sepaxis = ph.axisType(self.mmsfile) self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
def test_all_columns(self): '''Partition: datacolumn=all''' partition(vis=self.msfile, outputvis=self.mmsfile, datacolumn='all', flagbackup=False, disableparallel=True) self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Compare nrows of all scans in selection slist = ph.getMMSScans(thisdict) self.assertEqual(slist.__len__(), 2) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s) self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %(s, mmsN, msN)) # Compare spw IDs for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) ms_spw = ph.getSpwIds(self.msfile, s) self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\ 'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))
def test_default_sequential(self): '''Partition: create an MMS with default values in sequential''' partition(vis=self.msfile, outputvis=self.mmsfile, disableparallel=True) self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Get scans of MMS slist = ph.getMMSScans(thisdict) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s) self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %(s, mmsN, msN)) # Compare spw IDs of MS and MMS for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) ms_spw = ph.getSpwIds(self.msfile, s) self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\ 'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN)) # The separation axis should be written to the output MMS sepaxis = ph.axisType(self.mmsfile) self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
def test_all_columns(self): '''Partition: datacolumn=all''' partition(vis=self.msfile, outputvis=self.mmsfile, datacolumn='all', flagbackup=False, disableparallel=True) self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Compare nrows of all scans in selection slist = ph.getMMSScans(thisdict) self.assertEqual(slist.__len__(), 2) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s) self.assertEqual( mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' % (s, mmsN, msN)) # Compare spw IDs for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) ms_spw = ph.getSpwIds(self.msfile, s) self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\ 'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))
def test_float_data_mms_baseline_auto(self): '''importasdm: Create an MMS with a FLOAT_DATA column separated per baseline ''' myasdmname = 'uid___A002_X6218fb_X264' themsname = myasdmname + ".ms" # The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA importasdm(myasdmname, vis=themsname, ocorr_mode='ao', createmms=True, scans='1', separationaxis='baseline', numsubms=4) self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0) md = msmdtool() md.open(themsname) bsl = md.baselines() md.close() #diagnoals give the auto-corrs ac = bsl.diagonal() self.assertTrue(ac.all(), 'Not all auto-correlations are there') # Take the dictionary and compare with original MS thisdict = listpartition(vis=themsname, createdict=True) self.assertEqual(len(thisdict.keys()), 4, 'There should be 4 subMSs in output MMS')
def test_spw_separation(self): '''Partition: create an MMS separated by spws with observation selection''' # NOTE: ms.getscansummary() used in ph.getScanNrows does not honour several observation # IDs, therefore I need to selection by obs id in partition partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='spw', observation='2', flagbackup=False, disableparallel=True) self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Dictionary with selection to compare with original MS mysel = {'observation':'2'} # Compare nrows of all scans in MMS and MS slist = ph.getMMSScans(thisdict) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s, selection=mysel) self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %(s, mmsN, msN)) # spwids are re-indexed. The expected IDs are: # ms_spw = 2 --> mms_spw = 0 # ms_spw = 3 --> mms_spw = 1, etc. # Check that MMS spw IDs have been re-indexed properly indexed_ids = range(4) for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed')
def test_partition_balanced_multiple_scan(self): partition(self.vis, outputvis=self.outputms, separationaxis='auto', numsubms=2, disableparallel=True) listpartition_dict = listpartition(self.outputms, createdict=True) self.assertEqual(len(listpartition_dict), 2) self.assertEqual(len(listpartition_dict[0]['scanId']), 26) self.assertEqual(len(listpartition_dict[1]['scanId']), 26) self.assertEqual(sum([x['nrows'] for x in listpartition_dict[1]['scanId'].values()]), 4344) self.assertEqual(sum([x['nrows'] for x in listpartition_dict[0]['scanId'].values()]), 4344)
def test_baseline_max(self): '''partition: create an MMS per baseline axis. Use the maximum number of baselines''' self.outputms = 'baseline_max.mms' partition(self.vis, outputvis=self.outputms, separationaxis='baseline', numsubms=50, flagbackup=False) # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.outputms, createdict=True) self.assertEqual(len(thisdict.keys()), 6, 'There should be 6 subMSs in output MMS')
def test_baseline_2subms(self): '''partition: create an MMS per baseline axis. Create 3 SubMS''' self.outputms = 'baseline_3subms.mms' partition(self.vis, outputvis=self.outputms, separationaxis='baseline', numsubms=3, flagbackup=False) # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.outputms, createdict=True) self.assertEqual(len(thisdict.keys()), 3, 'There should be 3 subMSs in output MMS')
def test_baseline_partition(self): '''partition: create an MMS per baseline for an interferometry MS''' partition(vis=self.msfile, outputvis=self.mmsfile, spw='0,1',createmms=True, separationaxis='baseline', flagbackup=False, datacolumn='data') listpartition(self.mmsfile) # This MS doesn't have auto-correlations. It has 4 antennas and 6 baselines (cross) # There will be MSSelectionNullSelection SEVERE messages in the logfile because # it will try to create SubMSs from auto-correlations. This is not an error! md = msmdtool() md.open(self.mmsfile) bsl = md.baselines() md.close() # diagonal give the auto-corrs ac = bsl.diagonal() self.assertFalse(ac.all(), 'There should be no auto-correlations in MMS')
def test_partition_balanced_repeated_nsubms_greater_than_available_pairs(self): '''mstransform: Check that balanced mode properly handles the case when the number of requests subMSs is greater than the number of available ddi,scan pairs''' self.outputms = "test_partition_3c84scan1.mms" partition(self.vis, outputvis=self.outputms,separationaxis='auto',numsubms=64,flagbackup=False) listpartition_dict = listpartition(self.outputms, createdict=True) self.assertEqual(len(listpartition_dict), 4, "Number of subMS should be 3")
def test_default_scan(self): '''Partition: create an MMS with default values and axis=scan''' partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='scan') self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Compare nrows of all scans slist = ph.getMMSScans(thisdict) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s) self.assertEqual( mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' % (s, mmsN, msN)) # Compare spw IDs for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) ms_spw = ph.getSpwIds(self.msfile, s) self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\ 'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN)) # Sort the output MSs so that they can be compared myms = mstool() myms.open(self.msfile) myms.sort('ms_sorted.ms', [ 'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID', 'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME' ]) myms.done() myms.open(self.mmsfile) myms.sort('mms_sorted.ms', [ 'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID', 'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME' ]) myms.done() self.assertTrue( th.compTables('ms_sorted.ms', 'mms_sorted.ms', [ 'FLAG', 'FLAG_CATEGORY', 'TIME_CENTROID', 'WEIGHT_SPECTRUM', 'DATA' ])) # Compare the DATA column self.assertTrue( th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))
def test_partition_balanced_repeated_spws(self): '''mstransform: Check that balanced mode properly distributes the chunks when one SPW is pointed by more than one DDI (with different correlation setups)''' self.outputms = "test_partition_balanced_repeated_spws.mms" partition(self.vis, outputvis=self.outputms,separationaxis='auto',spw='0,2',numsubms=3,flagbackup=False) listpartition_dict = listpartition(self.outputms, createdict=True) self.assertEqual(listpartition_dict[0]['scanId'][1]['spwIds'], [1], "Wrong SPWIds in subMS 0") self.assertEqual(listpartition_dict[1]['scanId'][1]['spwIds'], [0], "Wrong SPWIds in subMS 1") self.assertEqual(listpartition_dict[2]['scanId'][1]['spwIds'], [0], "Wrong SPWIds in subMS 2")
def test_default_scan(self): """Partition: create an MMS with default values and axis=scan""" partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis="scan") self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test") # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Compare nrows of all scans slist = ph.getMMSScans(thisdict) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s) self.assertEqual(mmsN, msN, "Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s" % (s, mmsN, msN)) # Compare spw IDs for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) ms_spw = ph.getSpwIds(self.msfile, s) self.assertEqual( mms_spw, ms_spw, "list of spws in scan=%s differs: " "mms_spw=%s <--> ms_spw=%s" % (s, mmsN, msN) ) # Sort the output MSs so that they can be compared myms = mstool() myms.open(self.msfile) myms.sort( "ms_sorted.ms", ["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"], ) myms.done() myms.open(self.mmsfile) myms.sort( "mms_sorted.ms", ["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"], ) myms.done() self.assertTrue( th.compTables( "ms_sorted.ms", "mms_sorted.ms", ["FLAG", "FLAG_CATEGORY", "TIME_CENTROID", "WEIGHT_SPECTRUM", "DATA"] ) ) # Compare the DATA column self.assertTrue(th.compVarColTables("ms_sorted.ms", "mms_sorted.ms", "DATA"))
def test_baseline_partition(self): '''partition: create an MMS per baseline for an interferometry MS''' partition(vis=self.msfile, outputvis=self.mmsfile, spw='0,1', createmms=True, separationaxis='baseline', flagbackup=False, datacolumn='data') listpartition(self.mmsfile) # This MS doesn't have auto-correlations. It has 4 antennas and 6 baselines (cross) # There will be MSSelectionNullSelection SEVERE messages in the logfile because # it will try to create SubMSs from auto-correlations. This is not an error! md = msmdtool() md.open(self.mmsfile) bsl = md.baselines() md.close() # diagonal give the auto-corrs ac = bsl.diagonal() self.assertFalse(ac.all(), 'There should be no auto-correlations in MMS')
def test_partition_balanced_repeated_nsubms_greater_than_available_pairs( self): '''mstransform: Check that balanced mode properly handles the case when the number of requests subMSs is greater than the number of available ddi,scan pairs''' self.outputms = "test_partition_3c84scan1.mms" partition(self.vis, outputvis=self.outputms, separationaxis='auto', numsubms=64, flagbackup=False) listpartition_dict = listpartition(self.outputms, createdict=True) self.assertEqual(len(listpartition_dict), 4, "Number of subMS should be 3")
def test_partition_balanced_multiple_scan(self): partition(self.vis, outputvis=self.outputms, separationaxis='auto', numsubms=2, disableparallel=True) listpartition_dict = listpartition(self.outputms, createdict=True) self.assertEqual(len(listpartition_dict), 2) self.assertEqual(len(listpartition_dict[0]['scanId']), 26) self.assertEqual(len(listpartition_dict[1]['scanId']), 26) self.assertEqual( sum([x['nrows'] for x in listpartition_dict[1]['scanId'].values()]), 4344) self.assertEqual( sum([x['nrows'] for x in listpartition_dict[0]['scanId'].values()]), 4344)
def testmix3(self): '''listpartition MMS mix3: Create an output dictionary''' resdict = listpartition(vis=self.vis, createdict=True) nkeys = resdict.keys().__len__() self.assertEqual(nkeys, 32) # Check all scans in all sub-MSs for k in resdict.keys(): subms = resdict[k]['MS'] MS = self.visdata+'/'+subms scans = resdict[k]['scanId'].keys() for s in scans: nr = resdict[k]['scanId'][s]['nrows'] refN = ph.getScanNrows(MS, s) self.assertEqual(nr, refN, '%s, scan=%s, nrows=%s do not match reference nrows=%s'\ %(MS, s, nr, refN))
def test_scan_spw(self): '''Partition: separationaxis=scan with spw selection''' partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='scan', spw='1~4,10,11', flagbackup=False, datacolumn='data') self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') self.assertTrue( os.path.exists(self.msfile), 'Make sure the input MS is not deleted inside the task') # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Dictionary with selection to compare with original MS mysel = {'spw': '1~4,10,11'} # Compare nrows of all scans in selection slist = ph.getMMSScans(thisdict) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s, selection=mysel) self.assertEqual( mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' % (s, mmsN, msN)) # The comparison should be # ms_spw = 1 --> mms_spw = 0 # ms_spw = 2 --> mms_spw = 1, etc. # Check that MMS spw IDs have been re-indexed properly indexed_ids = range(6) for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed') # The separation axis should be written to the output MMS sepaxis = ph.axisType(self.mmsfile) self.assertEqual(sepaxis, 'scan', 'Partition did not write AxisType correctly in MMS')
def test_sepaxis(self): '''Partition: separationaxis=auto''' partition(vis=self.msfile, outputvis=self.mmsfile, spw='0~11', separationaxis='auto', flagbackup=False, disableparallel=True, datacolumn='data') # partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto') self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') # Dictionary with selection to compare with original MS mysel = {'spw': '0~11'} # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Compare nrows of all scans in selection slist = ph.getMMSScans(thisdict) self.assertEqual(slist.__len__(), 2) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s, selection=mysel) # msN = ph.getScanNrows(self.msfile, s) self.assertEqual( mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' % (s, mmsN, msN)) # Compare spw IDs for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel) # ms_spw = ph.getSpwIds(self.msfile, s) self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\ 'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw)) # The separation axis should be written to the output MMS sepaxis = ph.axisType(self.mmsfile) self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
def test_numsubms(self): '''Partition: small numsubms value''' # There are 16 spws; we want only 6 sub-MSs. partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='spw', numsubms=6, flagbackup=False, disableparallel=True, datacolumn='corrected') self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Check the number of sub-MSs mmslist = [] klist = thisdict.keys() for kk in klist: mmslist.append(thisdict[kk]['MS']) nsubms = mmslist.__len__() self.assertEqual(nsubms, 6, 'There should be only 6 sub-MSs') # Check that spw list is the same in MS and MMS spwlist = ph.getMMSSpwIds(thisdict) # Reference list of scans in MS slist = ph.getScanList(self.msfile) setlist = set([]) for s in slist: ms_spw = ph.getSpwIds(self.msfile, s) for a in ms_spw: setlist.add(a) self.assertEqual(list(setlist), spwlist) # The separation axis should be written to the output MMS sepaxis = ph.axisType(self.mmsfile) self.assertEqual(sepaxis, 'spw', 'Partition did not write AxisType correctly in MMS')
def test_numsubms(self): """Partition: small numsubms value""" # There are 16 spws; we want only 6 sub-MSs. partition( vis=self.msfile, outputvis=self.mmsfile, separationaxis="spw", numsubms=6, flagbackup=False, disableparallel=True, datacolumn="corrected", ) self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test") # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Check the number of sub-MSs mmslist = [] klist = thisdict.keys() for kk in klist: mmslist.append(thisdict[kk]["MS"]) nsubms = mmslist.__len__() self.assertEqual(nsubms, 6, "There should be only 6 sub-MSs") # Check that spw list is the same in MS and MMS spwlist = ph.getMMSSpwIds(thisdict) # Reference list of scans in MS slist = ph.getScanList(self.msfile) setlist = set([]) for s in slist: ms_spw = ph.getSpwIds(self.msfile, s) for a in ms_spw: setlist.add(a) self.assertEqual(list(setlist), spwlist) # The separation axis should be written to the output MMS sepaxis = ph.axisType(self.mmsfile) self.assertEqual(sepaxis, "spw", "Partition did not write AxisType correctly in MMS")
def test_partition_balanced_repeated_spws(self): '''mstransform: Check that balanced mode properly distributes the chunks when one SPW is pointed by more than one DDI (with different correlation setups)''' self.outputms = "test_partition_balanced_repeated_spws.mms" partition(self.vis, outputvis=self.outputms, separationaxis='auto', spw='0,2', numsubms=3, flagbackup=False) listpartition_dict = listpartition(self.outputms, createdict=True) self.assertEqual(listpartition_dict[0]['scanId'][1]['spwIds'], [1], "Wrong SPWIds in subMS 0") self.assertEqual(listpartition_dict[1]['scanId'][1]['spwIds'], [0], "Wrong SPWIds in subMS 1") self.assertEqual(listpartition_dict[2]['scanId'][1]['spwIds'], [0], "Wrong SPWIds in subMS 2")
def test_sepaxis(self): """Partition: separationaxis=auto""" partition( vis=self.msfile, outputvis=self.mmsfile, spw="0~11", separationaxis="auto", flagbackup=False, disableparallel=True, datacolumn="data", ) # partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto') self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test") # Dictionary with selection to compare with original MS mysel = {"spw": "0~11"} # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Compare nrows of all scans in selection slist = ph.getMMSScans(thisdict) self.assertEqual(slist.__len__(), 2) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s, selection=mysel) # msN = ph.getScanNrows(self.msfile, s) self.assertEqual(mmsN, msN, "Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s" % (s, mmsN, msN)) # Compare spw IDs for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel) # ms_spw = ph.getSpwIds(self.msfile, s) self.assertEqual( mms_spw, ms_spw, "list of spws in scan=%s differs: " "mms_spw=%s <--> ms_spw=%s" % (s, mms_spw, ms_spw) ) # The separation axis should be written to the output MMS sepaxis = ph.axisType(self.mmsfile) self.assertEqual(sepaxis, "scan,spw", "Partition did not write AxisType correctly in MMS")
def test_default_scan(self): '''Partition: create an MMS with default values and axis=scan''' partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='scan') self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Compare nrows of all scans slist = ph.getMMSScans(thisdict) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s) self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %(s, mmsN, msN)) # Compare spw IDs for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) ms_spw = ph.getSpwIds(self.msfile, s) self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\ 'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN)) # Sort the output MSs so that they can be compared myms = mstool() myms.open(self.msfile) myms.sort('ms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME']) myms.done() myms.open(self.mmsfile) myms.sort('mms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME']) myms.done() self.assertTrue(th.compTables('ms_sorted.ms', 'mms_sorted.ms', ['FLAG','FLAG_CATEGORY','TIME_CENTROID', 'WEIGHT_SPECTRUM','DATA'])) # Compare the DATA column self.assertTrue(th.compVarColTables('ms_sorted.ms','mms_sorted.ms','DATA'))
def test_float_data_mms_baseline_auto(self): '''importasdm: Create an MMS with a FLOAT_DATA column separated per baseline ''' myasdmname = 'uid___A002_X6218fb_X264' themsname = myasdmname+".ms" # The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA importasdm(myasdmname, vis=themsname, ocorr_mode='ao', createmms=True, scans='1', separationaxis='baseline', numsubms=4) self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0) md = msmdtool() md.open(themsname) bsl = md.baselines() md.close() #diagnoals give the auto-corrs ac = bsl.diagonal() self.assertTrue(ac.all(), 'Not all auto-correlations are there') # Take the dictionary and compare with original MS thisdict = listpartition(vis=themsname, createdict=True) self.assertEqual(len(thisdict.keys()), 4, 'There should be 4 subMSs in output MMS')
def test_spw_separation(self): '''Partition: create an MMS separated by spws with observation selection''' # NOTE: ms.getscansummary() used in ph.getScanNrows does not honour several observation # IDs, therefore I need to selection by obs id in partition partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='spw', observation='2', flagbackup=False, disableparallel=True) self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Dictionary with selection to compare with original MS mysel = {'observation': '2'} # Compare nrows of all scans in MMS and MS slist = ph.getMMSScans(thisdict) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s, selection=mysel) self.assertEqual( mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' % (s, mmsN, msN)) # spwids are re-indexed. The expected IDs are: # ms_spw = 2 --> mms_spw = 0 # ms_spw = 3 --> mms_spw = 1, etc. # Check that MMS spw IDs have been re-indexed properly indexed_ids = range(4) for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed')
def test_spw_selection(self): '''Partition: create an MMS separated by spws with spw=2,4 selection''' partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='spw', spw='2,4', flagbackup=False, disableparallel=True) self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') # Take the dictionary and compare with original MS thisdict = listpartition(vis=self.mmsfile, createdict=True) # Dictionary with selection to compare with original MS mysel = {'spw': '2,4'} # Compare nrows of all scans in selection slist = ph.getMMSScans(thisdict) for s in slist: mmsN = ph.getMMSScanNrows(thisdict, s) msN = ph.getScanNrows(self.msfile, s, selection=mysel) self.assertEqual( mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' % (s, mmsN, msN)) # spwids are re-indexed. The expected IDs are: # ms_spw = 2 --> mms_spw = 0 # ms_spw = 4 --> mms_spw = 1 # Check that MMS spw IDs have been re-indexed properly indexed_ids = range(2) for s in slist: mms_spw = ph.getSpwIds(self.mmsfile, s) self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed')
def testMS1(self): '''listpartition MS1: Input MS''' res = listpartition(vis=self.vis) self.assertEqual(res, {}, "It should return an empty dictionary")
def testMS3(self): '''listpartition MS3: Create an output dictionary''' resdict = listpartition(vis=self.vis, createdict=True) self.assertEqual(resdict[0]['MS'], self.vis)