def test_default_cols(self): '''hanningsmooth2: Default datacolumn=all and MMS output''' self.createMMS(self.msfile,column='all') self.outputms = 'hannall.ms' hanningsmooth2(vis=self.testmms, outputvis=self.outputms) self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms), 'Output should be an MMS') # Should have all scratch columns in output cd = th.getColDesc(self.outputms, 'DATA') self.assertGreater(len(cd), 0, 'DATA column does not exist') cc = th.getColDesc(self.outputms, 'CORRECTED_DATA') self.assertGreater(len(cc), 0, 'CORRECTED_DATA does not exist') # Now repeat the above steps but create an output MS by setting keepmms=False os.system('rm -rf '+self.outputms) hanningsmooth2(vis=self.testmms, outputvis=self.outputms, keepmms=False) self.assertFalse(ParallelDataHelper.isParallelMS(self.outputms), 'Output should be a normal MS') # Should have all scratch columns in output cd = th.getColDesc(self.outputms, 'DATA') self.assertGreater(len(cd), 0, 'DATA column does not exist') cc = th.getColDesc(self.outputms, 'CORRECTED_DATA') self.assertGreater(len(cc), 0, 'CORRECTED_DATA does not exist')
def hanningsmooth( vis=None, outputvis=None, keepmms=None, field=None, spw=None, scan=None, antenna=None, correlation=None, timerange=None, intent=None, array=None, uvrange=None, observation=None, feed=None, datacolumn=None, ): """Hanning smooth frequency channel data to remove Gibbs ringing """ casalog.origin('hanningsmooth') # Initiate the helper class pdh = ParallelDataHelper("hanningsmooth", locals()) # Validate input and output parameters try: pdh.setupIO() except Exception, instance: casalog.post('%s' % instance, 'ERROR') return False
def hanningsmooth( vis=None, outputvis=None, keepmms=None, field=None, spw=None, scan=None, antenna=None, correlation=None, timerange=None, intent=None, array=None, uvrange=None, observation=None, feed=None, datacolumn=None, ): """Hanning smooth frequency channel data to remove Gibbs ringing """ casalog.origin("hanningsmooth") # Initiate the helper class pdh = ParallelDataHelper("hanningsmooth", locals()) # Validate input and output parameters try: pdh.setupIO() except Exception, instance: casalog.post("%s" % instance, "ERROR") return False
def test_default_cols(self): '''hanningsmooth: Default datacolumn=all and MMS output''' self.createMMS(self.msfile, column='all') self.outputms = 'hannall.ms' hanningsmooth(vis=self.testmms, outputvis=self.outputms) self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms), 'Output should be an MMS') # Should have all scratch columns in output cd = th.getColDesc(self.outputms, 'DATA') self.assertGreater(len(cd), 0, 'DATA column does not exist') cc = th.getColDesc(self.outputms, 'CORRECTED_DATA') self.assertGreater(len(cc), 0, 'CORRECTED_DATA does not exist') # Now repeat the above steps but create an output MS by setting keepmms=False os.system('rm -rf ' + self.outputms) hanningsmooth(vis=self.testmms, outputvis=self.outputms, keepmms=False) self.assertFalse(ParallelDataHelper.isParallelMS(self.outputms), 'Output should be a normal MS') # Should have all scratch columns in output cd = th.getColDesc(self.outputms, 'DATA') self.assertGreater(len(cd), 0, 'DATA column does not exist') cc = th.getColDesc(self.outputms, 'CORRECTED_DATA') self.assertGreater(len(cc), 0, 'CORRECTED_DATA does not exist')
def test_default_cols(self): """hanningsmooth: Default datacolumn=all and MMS output""" self.createMMS(self.msfile, column="all") self.outputms = "hannall.ms" hanningsmooth(vis=self.testmms, outputvis=self.outputms) self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms), "Output should be an MMS") # Should have all scratch columns in output cd = th.getColDesc(self.outputms, "DATA") self.assertGreater(len(cd), 0, "DATA column does not exist") cc = th.getColDesc(self.outputms, "CORRECTED_DATA") self.assertGreater(len(cc), 0, "CORRECTED_DATA does not exist") # Now repeat the above steps but create an output MS by setting keepmms=False os.system("rm -rf " + self.outputms) hanningsmooth(vis=self.testmms, outputvis=self.outputms, keepmms=False) self.assertFalse(ParallelDataHelper.isParallelMS(self.outputms), "Output should be a normal MS") # Should have all scratch columns in output cd = th.getColDesc(self.outputms, "DATA") self.assertGreater(len(cd), 0, "DATA column does not exist") cc = th.getColDesc(self.outputms, "CORRECTED_DATA") self.assertGreater(len(cc), 0, "CORRECTED_DATA does not exist")
def split2(vis, outputvis, keepmms, field, spw, scan, antenna, correlation, timerange, intent, array, uvrange, observation, feed, datacolumn, keepflags, width, timebin, combine ): """Create a visibility subset from an existing visibility set""" casalog.origin('split2') # Initialize the helper class pdh = ParallelDataHelper("split2", locals()) # Validate input and output parameters try: pdh.setupIO() except Exception, instance: casalog.post('%s'%instance,'ERROR') return False
def test_mms5(self): '''test_mms5: Create 2 MMS, 2 flagversions and 2 online flag files''' myasdmname = 'uid___A002_X71e4ae_X317_short' themsname = myasdmname+".ms" wvrmsname = myasdmname+'-wvr-corrected.ms' flagfile1 = myasdmname+'_cmd.txt' flagfile2 = myasdmname+'-wvr-corrected'+'_cmd.txt' importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~4', wvr_corrected_data='both', savecmds=True, createmms=True) self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertTrue(ParallelDataHelper.isParallelMS(wvrmsname), 'Output is not a Multi-MS') self.assertTrue(os.path.exists(flagfile1)) self.assertTrue(os.path.exists(flagfile2))
def test_MMS1(self): '''mstransform: input MMS should be the same as output MMS''' # Create an MMS in the setup self.createMMS(self.vis, axis='scan', spws='0,1') # Create another MS and compare. They should be the same self.outputms = 'thesame.mms' mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data') self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output is not an MMS') # Sort the MSs so that they can be compared myms = mstool() myms.open(self.testmms) myms.sort('input_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME']) myms.done() myms.open(self.outputms) myms.sort('output_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME']) myms.done() # Compare both tables. Ignore the DATA column and compare it in next line self.assertTrue(th.compTables('input_sorted.ms','output_sorted.ms', ['FLAG_CATEGORY','FLAG','WEIGHT_SPECTRUM','SIGMA_SPECTRUM','DATA'])) # Compare the DATA column self.assertTrue(th.compVarColTables('input_sorted.ms','output_sorted.ms','DATA')) # The separation axis should be copied to the output MMS in_sepaxis = ph.axisType(self.testmms) out_sepaxis = ph.axisType(self.outputms) self.assertEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS do not match')
def split(vis, outputvis, keepmms, field, spw, scan, antenna, correlation, timerange, intent, array, uvrange, observation, feed, datacolumn, keepflags, width, timebin, combine): """Create a visibility subset from an existing visibility set""" casalog.origin('split') # Initialize the helper class pdh = ParallelDataHelper("split", locals()) # Validate input and output parameters try: pdh.setupIO() except Exception, instance: casalog.post('%s' % instance, 'ERROR') return False
def test_mms2(self): '''test_mms2: Create an MMS with default name and lazy=True''' myasdmname = 'uid___A002_X71e4ae_X317_short' themsname = myasdmname+".ms" importasdm(myasdmname, createmms=True, lazy=True, scans='2') self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
def test_float_data_mms_baseline_auto(self): '''importasdm: Create an MMS with a FLOAT_DATA column separated per baseline ''' myasdmname = 'uid___A002_X6218fb_X264' themsname = myasdmname + ".ms" # The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA importasdm(myasdmname, vis=themsname, ocorr_mode='ao', createmms=True, scans='1', separationaxis='baseline', numsubms=4) self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0) md = msmdtool() md.open(themsname) bsl = md.baselines() md.close() #diagnoals give the auto-corrs ac = bsl.diagonal() self.assertTrue(ac.all(), 'Not all auto-correlations are there') # Take the dictionary and compare with original MS thisdict = listpartition(vis=themsname, createdict=True) self.assertEqual(len(thisdict.keys()), 4, 'There should be 4 subMSs in output MMS')
def test_output_mms4(self): '''mstransform: timeaverage=True, output axis=scan, timespan=scan''' self.outputms = 'outmms4.mms' # Just give a WARNING mstransform(self.vis, outputvis=self.outputms, datacolumn='corrected', createmms=True, timeaverage=True, spw='12,13', separationaxis='scan',timebin='10s',timespan='scan') self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MMS')
def test_combspws_timespan(self): '''mstransform: combinespws=True, timespan=scan axis=auto''' self.createMMS(self.vis, axis='auto',spws='3') self.outputms = "2transformations.mms" # This should work. mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', combinespws=True, timeaverage=True, timebin='40s',timespan='scan') self.assertFalse(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MS')
def test_mms2(self): '''test_mms2: Create an MMS with default name and lazy=True''' myasdmname = 'uid___A002_X71e4ae_X317_short' themsname = myasdmname + ".ms" importasdm(myasdmname, createmms=True, lazy=True, scans='2') self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
def test_sd_data_mms(self): '''importasdm: Create an MMS from a single-dish MS and DATA column ''' myasdmname = 'uid___A002_X6218fb_X264' themsname = myasdmname+".ms" importasdm(myasdmname, vis=themsname, scans='1,4', createmms=True, separationaxis='scan', numsubms=2, flagbackup=False) self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertTrue(len(th.getColDesc(themsname, 'DATA')) > 0)
def test_mms3(self): '''test_mms3: Create MMS with separationaxis=spw and lazy=True''' myasdmname = 'uid___A002_X71e4ae_X317_short' themsname = myasdmname+".ms" importasdm(myasdmname, createmms=True, lazy=True, scans='1,2', separationaxis='spw', flagbackup=False, process_flags=False) self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertEqual(ph.axisType(themsname), 'spw', 'Separation axis of MMS should be spw')
def test_float_data_mms(self): '''importasdm: Create an MMS from a FLOAT_DATA MS ''' myasdmname = 'uid___A002_X6218fb_X264' themsname = myasdmname+".ms" # The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA importasdm(myasdmname, vis=themsname, ocorr_mode='ao', createmms=True, scans='1') self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0)
def cvel2( vis, outputvis, keepmms, passall, # hidden parameter for backwards compatibiliy field, spw, scan, antenna, correlation, timerange, intent, array, uvrange, observation, feed, datacolumn, mode, nchan, start, width, interpolation, phasecenter, restfreq, outframe, veltype, hanning, ): """ This task used the MSTransform framework. It needs to use the ParallelDataHelper class, implemented in parallel.parallel_data_helper.py. """ # Initialize the helper class pdh = ParallelDataHelper("cvel2", locals()) casalog.origin('cvel2') # Validate input and output parameters try: pdh.setupIO() except Exception, instance: casalog.post('%s'%instance,'ERROR') return False
def cvel2( vis, outputvis, keepmms, passall, # hidden parameter for backwards compatibiliy field, spw, scan, antenna, correlation, timerange, intent, array, uvrange, observation, feed, datacolumn, mode, nchan, start, width, interpolation, phasecenter, restfreq, outframe, veltype, hanning, ): """ This task used the MSTransform framework. It needs to use the ParallelDataHelper class, implemented in parallel.parallel_data_helper.py. """ # Initialize the helper class pdh = ParallelDataHelper("cvel2", locals()) casalog.origin("cvel2") # Validate input and output parameters try: pdh.setupIO() except Exception, instance: casalog.post("%s" % instance, "ERROR") return False
def test_mms5(self): '''test_mms5: Create 2 MMS, 2 flagversions and 2 online flag files''' myasdmname = 'uid___A002_X71e4ae_X317_short' themsname = myasdmname + ".ms" wvrmsname = myasdmname + '-wvr-corrected.ms' flagfile1 = myasdmname + '_cmd.txt' flagfile2 = myasdmname + '-wvr-corrected' + '_cmd.txt' importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~4', wvr_corrected_data='both', savecmds=True, createmms=True) self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertTrue(ParallelDataHelper.isParallelMS(wvrmsname), 'Output is not a Multi-MS') self.assertTrue(os.path.exists(flagfile1)) self.assertTrue(os.path.exists(flagfile2))
def test_combspws_timespan_scan_axis(self): '''mstransform: combinespws=True, timespan=scan axis=scan''' self.createMMS(self.vis, axis='scan',spws='0') self.outputms = "scanaxiserror.mms" # subMSs do not have all scans. Create an MS. try: mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', combinespws=True, timeaverage=True, timebin='20s',timespan='scan') self.assertTrue((ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MMS')) except Exception, instance: print 'Expected error: %s'%instance
def test_combspws_timespan_spw_axis(self): '''mstransform: combinespws=True, timespan=scan axis=spw''' self.createMMS(self.vis, axis='spw',scans='30',spws='10') self.outputms = "spwaxisok.mms" # This should work try: mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', combinespws=True, timeaverage=True, timebin='20s',timespan='scan') self.assertTrue((ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MMS')) except Exception, instance: print 'This error should have not happened %s'%instance
def test_monolithic_combspw1_1(self): '''mstransform: Combine four spws into one using a monolithic-MMS''' self.createMMS(self.vis, axis='spw',spws='0~3') self.outputms = "monocombspw11.ms" mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data',combinespws=True, spw='0~3') self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MMS') ret = th.verifyMS(self.outputms, 1, 256, 0) self.assertTrue(ret[0],ret[1]) listobs(self.outputms, listfile='list2.obs') self.assertTrue(os.path.exists('list2.obs'), 'Probable error in sub-table re-indexing')
def test_combspws_timespan_spw_axis_error(self): '''mstransform: combinespws=True, timespan=scan axis=spw''' self.createMMS(self.vis, axis='spw',scans='30',spws='10,11') self.outputms = "spwaxiserror.mms" # subMSs do not have all spws. Create an MS mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', combinespws=True, timeaverage=True, timebin='20s',timespan='scan') self.assertFalse(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MS') mymsmd = msmdtool() mymsmd.open(self.outputms) nspw = mymsmd.nspw() mymsmd.close() self.assertEqual(nspw,1)
def test_combspws_timespan_error(self): '''mstransform: combinespws=True, timespan=scan axis=auto timebin=40s''' self.createMMS(self.vis, axis='auto',spws='1,3', numms=4) self.outputms = "spanscan_comb.mms" # combinespws is not possible. It should create and MS mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', combinespws=True, timeaverage=True, timebin='40s',timespan='scan') self.assertFalse(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MS') mymsmd = msmdtool() mymsmd.open(self.outputms) nspw = mymsmd.nspw() mymsmd.close() self.assertEqual(nspw,1)
def test_float_data_mms(self): '''importasdm: Create an MMS from a FLOAT_DATA MS ''' myasdmname = 'uid___A002_X6218fb_X264' themsname = myasdmname + ".ms" # The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA importasdm(myasdmname, vis=themsname, ocorr_mode='ao', createmms=True, scans='1') self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0)
def test_combspws_timespan_fail(self): '''mstransform: combinespws=True, timespan=scan axis=auto timebin=200s''' self.createMMS(self.vis, axis='auto',spws='3') self.outputms = "errormms.mms" # Scans are shorter than timebin. Create an MS mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', combinespws=True, timeaverage=True, timebin='200s',timespan='scan') self.assertFalse(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MS') mymsmd = msmdtool() mymsmd.open(self.outputms) nscan = mymsmd.nscans() exposure = mymsmd.exposuretime(30)['value'] mymsmd.close() self.assertEqual(nscan,1) self.assertEqual(exposure, 179)
def test_sd_data_mms(self): '''importasdm: Create an MMS from a single-dish MS and DATA column ''' myasdmname = 'uid___A002_X6218fb_X264' themsname = myasdmname + ".ms" importasdm(myasdmname, vis=themsname, scans='1,4', createmms=True, separationaxis='scan', numsubms=2, flagbackup=False) self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertTrue(len(th.getColDesc(themsname, 'DATA')) > 0)
def test_timespan_spw_axis(self): '''mstransform: timeaverage=True, timespan=scan, separationaxis=spw''' self.createMMS(self.vis, axis='spw',spws='1,3') self.outputms = "spanscan_spw.mms" mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data',timeaverage=True, timebin='100s',timespan='scan') self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MMS') self.assertEqual(ph.axisType(self.outputms),'spw') mymsmd = msmdtool() mymsmd.open(self.outputms) t30 = mymsmd.exposuretime(30)['value'] t31 = mymsmd.exposuretime(31)['value'] mymsmd.close() self.assertEqual(t30, 100) self.assertEqual(t31, 79)
def test_mms3(self): '''test_mms3: Create MMS with separationaxis=spw and lazy=True''' myasdmname = 'uid___A002_X71e4ae_X317_short' themsname = myasdmname + ".ms" importasdm(myasdmname, createmms=True, lazy=True, scans='1,2', separationaxis='spw', flagbackup=False, process_flags=False) self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertEqual(ph.axisType(themsname), 'spw', 'Separation axis of MMS should be spw')
def test_sd_data_mms_baseline_all(self): '''importasdm: Create an MMS separated per baseline, using default numsubms ''' myasdmname = 'uid___A002_X6218fb_X264' themsname = myasdmname+".ms" # Create a single-dish MMS with auto and cross correlations importasdm(myasdmname, vis=themsname, ocorr_mode='ca', createmms=True, scans='1', separationaxis='baseline') self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertTrue(len(th.getColDesc(themsname, 'DATA')) > 0) md = msmdtool() md.open(themsname) bsl = md.baselines() md.close() # Check if all baselines are in there self.assertTrue(bsl.all(), 'Not all baselines are in the MMS')
def test_MMS_as_monolithicMS(self): '''mstransform: MMS should be processed as a monolithic MS''' # Create an MMS in the setup. It creates self.testmms self.createMMS(self.vis, axis='spw', spws='2,4,6') self.outputms = 'monolithicMMS.mms' # Treat MMS as a monolithic MS and create an output MMS with different separation axis. mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', combinespws=True) self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MMS') # The separation axis should be copied to the output MMS in_sepaxis = ph.axisType(self.testmms) out_sepaxis = ph.axisType(self.outputms) self.assertNotEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS should not match') ret = th.verifyMS(self.outputms, 1, 320, 0) self.assertTrue(ret[0],ret[1]) listobs(self.outputms, listfile='list1.obs') self.assertTrue(os.path.exists('list1.obs'), 'Probable error in sub-table re-indexing')
def test_float_data_mms_baseline_auto(self): '''importasdm: Create an MMS with a FLOAT_DATA column separated per baseline ''' myasdmname = 'uid___A002_X6218fb_X264' themsname = myasdmname+".ms" # The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA importasdm(myasdmname, vis=themsname, ocorr_mode='ao', createmms=True, scans='1', separationaxis='baseline', numsubms=4) self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0) md = msmdtool() md.open(themsname) bsl = md.baselines() md.close() #diagnoals give the auto-corrs ac = bsl.diagonal() self.assertTrue(ac.all(), 'Not all auto-correlations are there') # Take the dictionary and compare with original MS thisdict = listpartition(vis=themsname, createdict=True) self.assertEqual(len(thisdict.keys()), 4, 'There should be 4 subMSs in output MMS')
def test_combspw1_4(self): '''mstransform: Combine some channels of two spws using MMS input''' # same test as test_combspw1_2 mmsfile = "inpmms14.mms" # First create an MMS mstransform(vis=self.vis, outputvis=mmsfile, spw='0,1', createmms=True) # Now do the same as in test_combspw1_2. Datacolumn moved to DATA self.outputms = "combspw14.ms" mstransform(vis=mmsfile, outputvis=self.outputms, combinespws=True, spw='0:60~63,1:60~63', datacolumn='data', disableparallel=True) self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms), 'Output should be an MMS') # The spws contain gaps, therefore the number of channels is bigger ret = th.verifyMS(self.outputms, 1, 68, 0) self.assertTrue(ret[0],ret[1]) # Compare with cvel results default(cvel) cvel(vis=self.vis, outputvis='combcvel14.ms', spw='0:60~63,1:60~63') ret = th.verifyMS('combcvel14.ms', 1, 68, 0) self.assertTrue(ret[0],ret[1])
def test_mms_heuristics3(self): '''cvel2 : MMS heuristic tests''' self.setUp_mms_vis_c() self.createMMS(vis_c, axis='auto', scans='8,11', spws='0,1') # It should create an output MS rval = cvel2(vis=self.testmms, outputvis=outfile, field='5,6', spw='0,1', passall=False, mode='frequency', nchan=2, start='4.8101 GHz', width='50 MHz', outframe='', keepmms=False) self.assertFalse(ParallelDataHelper.isParallelMS(outfile), 'Output shold be an MS') self.assertNotEqual(rval, False) ret = verify_ms(outfile, 1, 2, 0) self.assertTrue(ret[0], ret[1])
def test_sd_data_mms_baseline_all(self): '''importasdm: Create an MMS separated per baseline, using default numsubms ''' myasdmname = 'uid___A002_X6218fb_X264' themsname = myasdmname + ".ms" # Create a single-dish MMS with auto and cross correlations importasdm(myasdmname, vis=themsname, ocorr_mode='ca', createmms=True, scans='1', separationaxis='baseline') self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertTrue(len(th.getColDesc(themsname, 'DATA')) > 0) md = msmdtool() md.open(themsname) bsl = md.baselines() md.close() # Check if all baselines are in there self.assertTrue(bsl.all(), 'Not all baselines are in the MMS')
def test_mms_heuristics3(self): '''cvel2 : MMS heuristic tests''' self.setUp_mms_vis_c() self.createMMS(vis_c, axis='auto', scans='8,11', spws='0,1') # It should create an output MS rval = cvel2( vis = self.testmms, outputvis = outfile, field = '5,6', spw = '0,1', passall = False, mode='frequency', nchan = 2, start = '4.8101 GHz', width = '50 MHz', outframe = '', keepmms=False ) self.assertFalse(ParallelDataHelper.isParallelMS(outfile),'Output shold be an MS') self.assertNotEqual(rval,False) ret = verify_ms(outfile, 1, 2, 0) self.assertTrue(ret[0],ret[1])
def cvel2( vis, outputvis, keepmms, passall, # hidden parameter for backwards compatibiliy field, spw, scan, antenna, correlation, timerange, intent, array, uvrange, observation, feed, datacolumn, mode, nchan, start, width, interpolation, phasecenter, restfreq, outframe, veltype, hanning, ): """ This task used the MSTransform framework. It needs to use the ParallelDataHelper class, implemented in parallel.parallel_data_helper.py. """ def check_do_preaveraging(vis, spw, field, mode, nchan, start, width, phasecenter, restfreq, outframe, veltype): """Returns whether this task is going to perform pre-averaging before regridding. Introduced to quick-fix CAS-9798. The aim is to do the same checks done in task_cvel. All parameters as specified in the task. """ # Reproduce as closely as possible the checks done in task_cevel. But note # there is not 'newphasecenter' here as MSTransformManager seems to handle # that issue in a different way. dopreaverage=False ms = casac.ms() ms.open(vis) thespwsel = ms.msseltoindex(vis=vis, spw=spw)['spw'] thefieldsel = ms.msseltoindex(vis=vis, field=field)['field'] outgrid = ms.cvelfreqs(spwids=thespwsel, fieldids=thefieldsel, mode=mode, nchan=nchan, start=start, width=width, phasec=phasecenter, restfreq=restfreq, outframe=outframe, veltype=veltype, verbose=False) if(len(outgrid)>1): tmpavwidth = [] for thespw in thespwsel: outgridw1 = ms.cvelfreqs(spwids=[thespw], fieldids=thefieldsel, mode='channel', nchan=-1, start=0, width=1, # native width phasec=phasecenter, restfreq=restfreq, outframe=outframe, veltype=veltype, verbose=False) if(len(outgridw1)>1): widthratio = abs((outgrid[1]-outgrid[0])/(outgridw1[1]-outgridw1[0])) if(widthratio>=2.0): # do preaverage tmpavwidth.append( int(widthratio+0.001) ) dopreaverage=True else: tmpavwidth.append(1) if dopreaverage: preavwidth = tmpavwidth ms.close() if(dopreaverage and (mode=='channel' or mode=='channel_b')): if(max(preavwidth)==1): dopreaverage = False return dopreaverage # Initialize the helper class pdh = ParallelDataHelper("cvel2", locals()) casalog.origin('cvel2') # Validate input and output parameters try: pdh.setupIO() except Exception, instance: casalog.post('%s'%instance,'ERROR') return False
# When dealing with MMS, process in parallel or sequential # disableparallel is a hidden parameter. Only for debugging purposes! if disableparallel: pdh.bypassParallelProcessing(1) else: pdh.bypassParallelProcessing(0) # Validate input and output parameters try: pdh.setupIO() except Exception, instance: casalog.post('%s'%instance,'ERROR') return False # Process the input Multi-MS if ParallelDataHelper.isParallelMS(vis) == True and monolithic_processing == False: ''' retval{'status': True, 'axis':''} --> can run in parallel retval{'status': False, 'axis':'value'} --> treat MMS as monolithic MS, set new axis for output MMS retval{'status': False, 'axis':''} --> treat MMS as monolithic MS, create an output MS ''' retval = pdh.validateInputParams() # Cannot create an output MMS. if retval['status'] == False and retval['axis'] == '': casalog.post('Cannot process MMS with the requested transformations','WARN') casalog.post('Use task listpartition to see the contents of the MMS') casalog.post('Will create an output MS','WARN') createmms = False # MMS is processed as monolithic MS.
def test4(self): '''hanningsmooth - Test 4: Theoretical and calculated values should be the same for MMS-case''' # Split the input to decrease the running time split(self.msfile, outputvis='splithan.ms', scan='1,2', datacolumn='data') self.msfile = 'splithan.ms' # create a test MMS. It creates self.testmms self.createMMS(self.msfile) self.outputms = 'hann4.mms' # check correct flagging (just for one row as a sample) mslocal = mstool() mslocal.open(self.msfile) mslocal.sort('sorted.ms', [ 'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID', 'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME' ]) mslocal.close() self.msfile = 'sorted.ms' flag_col = th.getVarCol(self.msfile, 'FLAG') self.assertTrue(flag_col['r1'][0][0] == [False]) self.assertTrue(flag_col['r1'][0][1] == [False]) self.assertTrue(flag_col['r1'][0][61] == [False]) self.assertTrue(flag_col['r1'][0][62] == [False]) data_col = th.getVarCol(self.msfile, 'DATA') hanningsmooth(vis=self.testmms, outputvis=self.outputms, datacolumn='data', keepmms=True) self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms), 'Output should be an MMS') # Sort the MMS mslocal.open(self.outputms) mslocal.sort('sorted.mms', [ 'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID', 'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME' ]) mslocal.close() self.outputms = 'sorted.mms' corr_col = th.getVarCol(self.outputms, 'DATA') nrows = len(corr_col) # check correct flagging (just for one row as a sample) flag_col = th.getVarCol(self.outputms, 'FLAG') self.assertTrue(flag_col['r1'][0][0] == [True]) self.assertTrue(flag_col['r1'][0][1] == [False]) self.assertTrue(flag_col['r1'][0][61] == [False]) self.assertTrue(flag_col['r1'][0][62] == [True]) # Loop over every 2nd row,pol and get the data for each channel max = 1e-05 for i in range(1, nrows, 2): row = 'r%s' % i # polarization is 0-1 for pol in range(0, 2): # array's channels is 0-63 for chan in range(1, 62): # channels must start from second and end before the last data = data_col[row][pol][chan] dataB = data_col[row][pol][chan - 1] dataA = data_col[row][pol][chan + 1] Smoothed = th.calculateHanning(dataB, data, dataA) CorData = corr_col[row][pol][chan] # Check the difference self.assertTrue(abs(CorData - Smoothed) < max)
def partition( vis, outputvis, createmms, separationaxis, numsubms, flagbackup, # only for MMS datacolumn, field, spw, scan, antenna, correlation, timerange, intent, array, uvrange, observation, feed, disableparallel, # HIDDEN parameter to create an MMS in sequential ddistart, # HIDDEN parameter to be used when merging sub-tables taql # HIDDEN parameter to be used for balanced partition schema ): """Create a multi visibility set from an existing visibility set: Keyword arguments: vis -- Name of input visibility file (MS) default: none; example: vis='ngc5921.ms' outputvis -- Name of output visibility file (MS) default: none; example: outputvis='ngc5921_src.ms' createmms -- Boolean flag if we're creating Multi MS default: True separationaxis -- what axis do we intend to split on. default = 'auto' Options: 'scan','spw','auto' numsubms -- Number of sub-MSs to create. default: 'auto' flagbackup -- Backup the FLAG column of the output MMS default: True datacolumn -- Which data column to use to create the output default='data'; example: datacolumn='data' Options: 'data', 'corrected', 'model', 'all', 'float_data', 'lag_data', 'float_data,data', and 'lag_data,data'. note: 'all' = whichever of the above that are present. field -- Field name default: field = '' means use all sources field = 1 # will get field_id=1 (if you give it an integer, it will retrieve the source with that index) field = '1328+307' specifies source '1328+307'. Minimum match can be used, egs field = '13*' will retrieve '1328+307' if it is unique or exists. Source names with imbedded blanks cannot be included. spw -- Spectral window index identifier default=-1 (all); example: spw=1 antenna -- antenna names default '' (all), antenna = '3 & 7' gives one baseline with antennaid = 3,7. timerange -- Time range default='' means all times. examples: timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss' timerange='< YYYY/MM/DD/HH:MM:SS.sss' timerange='> YYYY/MM/DD/HH:MM:SS.sss' timerange='< ddd/HH:MM:SS.sss' timerange='> ddd/HH:MM:SS.sss' scan -- Scan numbers to select. default '' (all). intent -- Select based on the scan intent. default '' (all) array -- (Sub)array IDs to select. default '' (all). uvrange -- uv distance range to select. default '' (all). observation -- observation ID(s) to select. default '' (all). """ casalog.origin('partition') # Initiate the helper class pdh = ParallelDataHelper('partition', locals()) # Validate input and output parameters try: pdh.setupIO() except Exception, instance: casalog.post('%s' % instance, 'ERROR') return False
def importasdm( asdm=None, vis=None, createmms=None, separationaxis=None, numsubms=None, corr_mode=None, srt=None, time_sampling=None, ocorr_mode=None, compression=None, lazy=None, asis=None, wvr_corrected_data=None, scans=None, ignore_time=None, process_syspower=None, process_caldevice=None, process_pointing=None, process_flags=None, tbuff=None, applyflags=None, savecmds=None, outfile=None, flagbackup=None, verbose=None, overwrite=None, showversion=None, useversion=None, bdfflags=None, with_pointing_correction=None, remove_ref_undef=None, convert_ephem2geo=None, polyephem_tabtimestep=None ): """Convert an ALMA Science Data Model observation into a CASA visibility file (MS) or single-dish data format (Scantable). The conversion of the ALMA SDM archive format into a measurement set. This version is under development and is geared to handling many spectral windows of different shapes. Keyword arguments: asdm -- Name of input ASDM file (directory) default: none; example: asdm='ExecBlock3' vis -- Root ms or scantable name, note a prefix (.ms or .asap) is NOT appended to this name default: none createmms -- Create a Multi-MS default: False corr_mode -- correlation mode to be considered on input. Could be one or more of the following, ao, co, ac, or all default: all srt -- spectral resolution type. Could be one or more of the following, fr, ca, bw, or all default: all time_sampling -- specifies the time sampling, INTEGRATION and/or SUBINTEGRATION. could be one or more of the following i, si, or all. default: all ocorr_mode -- output data for correlation mode AUTO_ONLY (ao) or CROSS_ONLY (co) or CROSS_AND_AUTO (ca) default: ca compression -- produces comrpressed columns in the resulting measurement set. default: False lazy -- Make the MS DATA column read the ASDM Binary data directly (faster import, smaller MS) default: False asis -- creates verbatim copies of the ASDM tables in the output measurement set. The value given to this option must be a list of table names separated by space characters; the wildcard character '*' is allowed in table names. wvr_corrected_data -- specifies wich values are considered in the ASDM binary data to fill the DATA column in the MAIN table of the MS. Expected values for this option are 'no' for the uncorrected data (this is the default), 'yes' for the corrected data and 'both' for corrected and uncorrected data. In the latter case, two measurement sets are created, one containing the uncorrected data and the other one, whose name is suffixed by '-wvr-corrected', containing the corrected data. scans -- processes only the scans specified in the option's value. This value is a semicolon separated list of scan specifications. A scan specification consists in an exec bock index followed by the character ':' followed by a comma separated list of scan indexes or scan index ranges. A scan index is relative to the exec block it belongs to. Scan indexes are 1-based while exec blocks's are 0-based. "0:1" or "2:2~6" or "0:1,1:2~6,8;2:,3:24~30" "1,2" are valid values for the option. "3:" alone will be interpreted as 'all the scans of the exec block#3'. An scan index or a scan index range not preceded by an exec block index will be interpreted as 'all the scans with such indexes in all the exec blocks'. By default all the scans are considered. ignore_time -- All the rows of the tables Feed, History, Pointing, Source, SysCal, CalDevice, SysPower, and Weather are processed independently of the time range of the selected exec block / scan. process_syspower -- The SysPower table is processed if and only if this parameter is set to True. default: True process_caldevice -- The CalDevice table is processed if and only if this parameter is set to True. default: True process_pointing -- The Pointing table is processed if and only if this parameter is set to True. If the parameter is set to False the resulting MS will have an empty POINTING table. default: True process_flags -- Process the online flags and save them to the FLAG_CMD sub-table. default: True >>> process_flags expandable parameter tbuff -- Time padding buffer (in seconds). default: 0.0 applyflags -- Apply the online flags to the MS. default: False savecmds -- Save the online flags to an ASCII file. default: False outfile -- Filename to save the online flags. default: '' flagbackup -- Backup the FLAG column in the .flagversions. default: True verbose -- produce log output as asdm2MS is being run. overwrite -- Over write an existing MS. showversion -- report the version of the asdm2MS being used. useversion -- Selects the version of asdm2MS to be used (presently only \'v3\' is available). default: v3 bdfflags -- Set the MS FLAG column according to the ASDM _binary_ flags default: false with_pointing_correction -- add (ASDM::Pointing::encoder - ASDM::Pointing::pointingDirection) to the value to be written in MS::Pointing::direction default: false remove_ref_undef -- if set to True then apply fixspwbackport on the resulting MSes. convert_ephem2geo -- if True, convert any attached ephemerides to the GEO reference frame polyephem_tabtimestep -- Timestep (days) for the tabulation of polynomial ephemerides. A value <= 0 disables tabulation. Presently, VLA data can contain polynomial ephemerides. ALMA data uses tabulated values. default: 0. """ # Python script # make agentflagger tool local aflocal = casac.agentflagger() # make table tool local tblocal = casac.table() try: casalog.origin('importasdm') viso = '' visoc = '' # for the wvr corrected version, if needed if len(vis) > 0: viso = vis tmps = vis.rstrip('.ms') if tmps == vis: visoc = vis + '-wvr-corrected' else: visoc = tmps + '-wvr-corrected.ms' else: viso = asdm.rstrip("/") + '.ms' visoc = asdm.rstrip("/") + '-wvr-corrected.ms' vis = asdm.rstrip("/") useversion = 'v3' theexecutable = 'asdm2MS' execute_string = theexecutable + ' --icm "' + corr_mode \ + '" --isrt "' + srt + '" --its "' + time_sampling \ + '" --ocm "' + ocorr_mode + '" --wvr-corrected-data "' \ + wvr_corrected_data + '" --asis "' + asis \ + '" --logfile "' + casalog.logfile() + '"' if len(scans) > 0: execute_string = execute_string + ' --scans ' + scans if ignore_time: execute_string = execute_string + ' --ignore-time' if useversion == 'v3': if not process_syspower: execute_string = execute_string + ' --no-syspower' if not process_caldevice: execute_string = execute_string + ' --no-caldevice' if not process_pointing: execute_string = execute_string + ' --no-pointing' if compression: execute_string = execute_string + ' --compression' elif lazy: execute_string = execute_string + ' --lazy' if verbose: execute_string = execute_string + ' --verbose' # if not overwrite and os.path.exists(viso): # raise Exception, \ # 'You have specified an existing MS and have indicated you do not wish to overwrite it' # Compression if compression: # viso = viso + '.compressed' viso = viso.rstrip('.ms') + '.compressed.ms' visoc = visoc.rstrip('.ms') + '.compressed.ms' vistoproc = [] # the output MSs to post-process if wvr_corrected_data == 'no' or wvr_corrected_data == 'both': vistoproc.append(viso) if (wvr_corrected_data == 'yes' or wvr_corrected_data == 'both') : vistoproc.append(visoc) for ff in vistoproc: if not overwrite and os.path.exists(ff): raise Exception, \ 'You have specified an existing MS and have indicated you do not wish to overwrite it: %s'%ff # If viso+".flagversions" then process differently depending on the value of overwrite.. # if flagbackup: for myviso in vistoproc: dotFlagversion = myviso + '.flagversions' if os.path.exists(dotFlagversion): if overwrite: casalog.post("Found '" + dotFlagversion + "' . It'll be deleted before running the filler." ) os.system('rm -rf %s' % dotFlagversion) else: casalog.post("Found '%s' but can't overwrite it." % dotFlagversion) raise Exception, "Found '%s' but can't overwrite it." \ % dotFlagversion # Make outfile always a list if isinstance(outfile, str): if outfile == '': outfile = [] else: noutfile = [outfile] outfile = noutfile if savecmds: if len(outfile) == 0: # Create default names for the online flags for myviso in vistoproc: outfile.append(myviso.replace('.ms','_cmd.txt')) elif len(outfile) != len(vistoproc): casalog.post('List of outfile names does not match list of MSs','WARN') casalog.post('Will save online flags to temporary filenames', 'WARN') outfile = [] for myviso in vistoproc: online_file = myviso.replace('.ms','_TEMP_cmd.txt') outfile.append(online_file) if not overwrite: for of in outfile: if os.path.exists(of): raise Exception, "Cannot overwrite online flags file '%s'; overwrite is set to False."% of execute_string = execute_string + ' ' + asdm + ' ' + viso if showversion: casalog.post("You set option \'showversion\' to True. Will just output the version information and then terminate." , 'WARN') execute_string = theexecutable + ' --revision' if with_pointing_correction: execute_string = execute_string + ' --with-pointing-correction' if (polyephem_tabtimestep!=None) and (type(polyephem_tabtimestep)==int or type(polyephem_tabtimestep)==float): if polyephem_tabtimestep>0: casalog.post('Will tabulate all attached polynomial ephemerides with a time step of ' +str(polyephem_tabtimestep)+' days.') if polyephem_tabtimestep>1.: casalog.post('A tabulation timestep of <= 1 days is recommended.', 'WARN') execute_string = execute_string + ' --polyephem-tabtimestep '+str(polyephem_tabtimestep) casalog.post('Running ' + theexecutable + ' standalone invoked as:') # print execute_string casalog.post(execute_string) exitcode = os.system(execute_string) if exitcode != 0: if not showversion: casalog.post(theexecutable + ' terminated with exit code ' + str(exitcode), 'SEVERE') raise Exception, \ 'ASDM conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.' if showversion: return # # Possibly remove the name of the measurement set expected to contain the corrected data from the list of of produced measurement # sets if it appears the filler did not find any corrected data. # if not os.path.exists(visoc): vistoproc = [myviso for myviso in vistoproc if myviso != visoc] # CAS-7369. HISTORY should be written after createmms is tested # # Populate the HISTORY table of the MS with information about the context in which it's been created # try: mslocal = mstool() param_names = importasdm.func_code.co_varnames[:importasdm.func_code.co_argcount] param_vals = [eval(p) for p in param_names] for myviso in vistoproc: write_history(mslocal, myviso, 'importasdm', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') return False if mslocal: mslocal = None # # Do we apply fixspwbackport if remove_ref_undef : casalog.post('remove_ref_undef=True: fixspwbackport will be applied ...') for myviso in vistoproc: cmd = 'fixspwbackport ' + myviso casalog.post('Running fixspwbackport standalone invoked as:') casalog.post(cmd) cmdexitcode = os.system(cmd) if cmdexitcode != 0: casalog.post(cmd + ' terminated with exit code ' + str(cmdexitcode), 'SEVERE') raise Exception, 'fixspwbackport error.' # Binary Flag processing if bdfflags: casalog.post('Parameter bdfflags==True: flags from the ASDM binary data will be used to set the MS flags ...') bdffexecutable = 'bdflags2MS ' bdffexecstring_base = bdffexecutable + ' -f ALL' + ' --ocm "' + ocorr_mode \ + '" --logfile "' + casalog.logfile() + '"' if len(scans) > 0: bdffexecstring_base = bdffexecstring_base + ' --scans ' + scans if lazy and not compression: bdffexecstring_base = bdffexecstring_base + ' --lazy=true' for myviso in vistoproc: if myviso.find("wvr-corrected") != -1: options = " --wvr-corrected=True " else: options = " " bdffexecstring = bdffexecstring_base + options + asdm + ' ' + myviso casalog.post('Running '+bdffexecutable+' standalone invoked as:') casalog.post(bdffexecstring) bdffexitcode = os.system(bdffexecstring) if bdffexitcode != 0: casalog.post(bdffexecutable + ' terminated with exit code ' + str(bdffexitcode), 'SEVERE') raise Exception, \ 'ASDM binary flags conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.' theephemfields = ce.findattachedephemfields(myviso,field='*') if len(theephemfields)>0: # until asdm2MS does this internally: recalc the UVW coordinates for ephem fields imt = imtool() imt.open(myviso, usescratch=False) imt.calcuvw(theephemfields, refcode='J2000', reuse=False) imt.close() if convert_ephem2geo: for myviso in vistoproc: ce.convert2geo(myviso, '*') # convert any attached ephemerides to GEO if len(theephemfields)>0: # also set the direction column in the SOURCE table tblocal.open(myviso+'/FIELD', nomodify=False) sourceids = tblocal.getcol('SOURCE_ID') ftimes = tblocal.getcol('TIME') ftimekw = tblocal.getcolkeywords('TIME') tmpa = tblocal.getcol('PHASE_DIR') origphasedir = tmpa affectedsids = [] thesamplefields = [] for fld in theephemfields: # determine all source ids used by the ephem fields if not (sourceids[fld] in affectedsids): # this source id wasn't handled yet affectedsids.append(sourceids[fld]) thesamplefields.append(fld) # need to temporarily change the offset (not all mosaics have an element at (0,0)) tmpa[0][0][fld]=0. tmpa[1][0][fld]=0. #endif #endfor tblocal.putcol('PHASE_DIR', tmpa) tblocal.close() tblocal.open(myviso+'/SOURCE') sourceposref = tblocal.getcolkeywords('DIRECTION')['MEASINFO']['Ref'] tblocal.close() directions = [] msmdlocal = casac.msmetadata() msmdlocal.open(myviso) for fld in thesamplefields: thedirmeas = msmdlocal.phasecenter(fld) if thedirmeas['refer']!=sourceposref: casalog.post('Ephemeris is in '+thedirmeas['refer']+' instead of '+sourceposref +' frame.\nEntry in SOURCE table will be converted to '+sourceposref, 'WARN') melocal = metool() melocal.doframe(thedirmeas) thedirmeas = melocal.measure(thedirmeas, sourceposref) directions.append([thedirmeas['m0']['value'], thedirmeas['m1']['value']]) thetime = me.epoch(v0=str(ftimes[fld])+'s', rf=ftimekw['MEASINFO']['Ref']) casalog.post("Will set SOURCE direction for SOURCE_ID "+str(sourceids[fld]) +" to ephemeris phase center for time "+str(thetime['m0']['value'])+" "+thetime['m0']['unit']+" "+thetime['refer']) #endfor msmdlocal.close() # restore original PHASE_DIR tblocal.open(myviso+'/FIELD', nomodify=False) tblocal.putcol('PHASE_DIR', origphasedir) tblocal.close() # write source directions tblocal.open(myviso+'/SOURCE', nomodify=False) ssourceids = tblocal.getcol('SOURCE_ID') sdirs = tblocal.getcol('DIRECTION') for row in xrange(0,len(ssourceids)): for i in xrange(0,len(affectedsids)): if ssourceids[row]==affectedsids[i]: sdirs[0][row] = directions[i][0] sdirs[1][row] = directions[i][1] break #endfor #endfor tblocal.putcol('DIRECTION', sdirs) # write back corrected directions tblocal.close() #end if ##############################################################################################3 # CAS-7369 - Create an output Multi-MS (MMS) if createmms: # Get the default parameters of partition from tasks import partition fpars = partition.parameters for mypar in fpars.keys(): fpars[mypar] = partition.itsdefault(mypar) # Call the cluster for each MS for myviso in vistoproc: casalog.origin('importasdm') # Move original MS to tempdir tempname = myviso+'.temp.ms' outputmms = myviso shutil.move(myviso, tempname) # Get the proper column datacolumn = 'DATA' dcols = ['DATA', 'FLOAT_DATA'] for dc in dcols: if len(th.getColDesc(tempname, dc)) > 0: datacolumn = dc break fpars['datacolumn'] = datacolumn casalog.post('Will create a Multi-MS for: '+myviso) fpars['vis'] = tempname fpars['flagbackup'] = False fpars['outputvis'] = outputmms fpars['separationaxis'] = separationaxis fpars['numsubms'] = numsubms pdh = ParallelDataHelper('partition', fpars) # Get a cluster pdh.setupCluster(thistask='partition') try: pdh.go() # Remove original MS shutil.rmtree(tempname) except Exception, instance: # Restore MS in case of error in MMS creation shutil.move(tempname, myviso) casalog.post('%s'%instance,'ERROR') return False casalog.origin('importasdm')
def test_mms6(self): '''test_mms6: Create MMS and lazy=True''' retValue = {'success': True, 'msgs': "", 'error_msgs': '' } myasdmname = 'uid___A002_X71e4ae_X317_short' themsname = myasdmname+".ms" flagfile = myasdmname+'_cmd.txt' self.res = importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~4', createmms=True, savecmds=True) # only the first 4 scans to save time self.assertEqual(self.res, None) self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertTrue(os.path.exists(flagfile)) print myname, ": Success! Now checking output ..." mscomponents = set(["ANTENNA/table.dat", "DATA_DESCRIPTION/table.dat", "FEED/table.dat", "FIELD/table.dat", "FLAG_CMD/table.dat", "HISTORY/table.dat", "OBSERVATION/table.dat", "POINTING/table.dat", "POLARIZATION/table.dat", "PROCESSOR/table.dat", "SOURCE/table.dat", "SPECTRAL_WINDOW/table.dat", "STATE/table.dat", "SYSCAL/table.dat", "ANTENNA/table.f0", "DATA_DESCRIPTION/table.f0", "FEED/table.f0", "FIELD/table.f0", "FLAG_CMD/table.f0", "HISTORY/table.f0", "OBSERVATION/table.f0", "POINTING/table.f0", "POLARIZATION/table.f0", "PROCESSOR/table.f0", "SOURCE/table.f0", "SPECTRAL_WINDOW/table.f0", "STATE/table.f0", "SYSCAL/table.f0" ]) for name in mscomponents: if not os.access(themsname+"/"+name, os.F_OK): print myname, ": Error ", themsname+"/"+name, "doesn't exist ..." retValue['success']=False retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist' else: print myname, ": ", name, "present." print myname, ": MS exists. All tables present. Try opening as MS ..." try: mslocal.open(themsname) mslocal.close() print myname, ": MS can be opened. Now testing the changing of the asdmref ..." mslocal.open(themsname) mslocal.asdmref("./moved_"+myasdmname) mslocal.close() os.system("mv "+myasdmname+" moved_"+myasdmname) mslocal.open(themsname) except: print myname, ": Error Cannot open MS table", themsname retValue['success']=False retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname else: mslocal.close() print myname, ": OK. Checking tables in detail ..." importasdm(asdm="moved_"+myasdmname, vis='reference.ms', lazy=False, overwrite=True, scans='0:1~3', createmms=True) if(os.path.exists('reference.ms')): retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from " +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0 if not retValue['success']: print "ERROR: DATA does not agree with reference." else: print "DATA columns agree." retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from " +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.WEIGHT,t2.WEIGHT, 1.e-06)))") == 0 if not retValueTmp: print "ERROR: WEIGHT does not agree with reference." else: print "WEIGHT columns agree." retValueTmp2 = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from " +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") == 0 if not retValueTmp2: print "ERROR: FLAG does not agree with reference." else: print "FLAG columns agree." retValue['success'] = retValue['success'] and retValueTmp and retValueTmp2 for subtname in ["ANTENNA", "DATA_DESCRIPTION", "FEED", "FIELD", "FLAG_CMD", "OBSERVATION", "POLARIZATION", "PROCESSOR", "SOURCE", "SPECTRAL_WINDOW", "STATE", "SYSCAL"]: print "\n*** Subtable ",subtname excllist = [] if subtname=='SOURCE': excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL'] if subtname=='SYSCAL': excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM'] if subtname=='SPECTRAL_WINDOW': excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE'] for colname in excllist: if colname!='ASSOC_NATURE': retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW', themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success'] if subtname=='POLARIZATION': excllist=['CORR_TYPE', 'CORR_PRODUCT'] for colname in excllist: retValue['success'] = th.compVarColTables('reference.ms/POLARIZATION', themsname+'/POLARIZATION', colname, 0.01) and retValue['success'] try: retValue['success'] = th.compTables('reference.ms/'+subtname, themsname+'/'+subtname, excllist, 0.01) and retValue['success'] except: retValue['success'] = False print "ERROR for table ", subtname os.system("mv moved_"+myasdmname+" "+myasdmname) self.assertTrue(retValue['success'],retValue['error_msgs'])
def importasdm( asdm=None, vis=None, createmms=None, separationaxis=None, numsubms=None, singledish=None, antenna=None, corr_mode=None, srt=None, time_sampling=None, ocorr_mode=None, compression=None, lazy=None, asis=None, wvr_corrected_data=None, scans=None, ignore_time=None, process_syspower=None, process_caldevice=None, process_pointing=None, process_flags=None, tbuff=None, applyflags=None, savecmds=None, outfile=None, flagbackup=None, verbose=None, overwrite=None, showversion=None, useversion=None, bdfflags=None, with_pointing_correction=None, remove_ref_undef=None, convert_ephem2geo=None ): """Convert an ALMA Science Data Model observation into a CASA visibility file (MS) or single-dish data format (Scantable). The conversion of the ALMA SDM archive format into a measurement set. This version is under development and is geared to handling many spectral windows of different shapes. Keyword arguments: asdm -- Name of input ASDM file (directory) default: none; example: asdm='ExecBlock3' vis -- Root ms or scantable name, note a prefix (.ms or .asap) is NOT appended to this name default: none createmms -- Create a Multi-MS default: False singledish -- Set True to write data as single-dish format (Scantable) default: False singledish expandable parameter antenna -- antenna name or id. corr_mode -- correlation mode to be considered on input. Could be one or more of the following, ao, co, ac, or all default: all srt -- spectral resolution type. Could be one or more of the following, fr, ca, bw, or all default: all time_sampling -- specifies the time sampling, INTEGRATION and/or SUBINTEGRATION. could be one or more of the following i, si, or all. default: all ocorr_mode -- output data for correlation mode AUTO_ONLY (ao) or CROSS_ONLY (co) or CROSS_AND_AUTO (ca) default: ca compression -- produces comrpressed columns in the resulting measurement set. default: False lazy -- Make the MS DATA column read the ASDM Binary data directly (faster import, smaller MS) default: False asis -- creates verbatim copies of the ASDM tables in the output measurement set. The value given to this option must be a list of table names separated by space characters; the wildcard character '*' is allowed in table names. wvr_corrected_data -- specifies wich values are considered in the ASDM binary data to fill the DATA column in the MAIN table of the MS. Expected values for this option are 'no' for the uncorrected data (this is the default), 'yes' for the corrected data and 'both' for corrected and uncorrected data. In the latter case, two measurement sets are created, one containing the uncorrected data and the other one, whose name is suffixed by '-wvr-corrected', containing the corrected data. scans -- processes only the scans specified in the option's value. This value is a semicolon separated list of scan specifications. A scan specification consists in an exec bock index followed by the character ':' followed by a comma separated list of scan indexes or scan index ranges. A scan index is relative to the exec block it belongs to. Scan indexes are 1-based while exec blocks's are 0-based. "0:1" or "2:2~6" or "0:1,1:2~6,8;2:,3:24~30" "1,2" are valid values for the option. "3:" alone will be interpreted as 'all the scans of the exec block#3'. An scan index or a scan index range not preceded by an exec block index will be interpreted as 'all the scans with such indexes in all the exec blocks'. By default all the scans are considered. ignore_time -- All the rows of the tables Feed, History, Pointing, Source, SysCal, CalDevice, SysPower, and Weather are processed independently of the time range of the selected exec block / scan. process_syspower -- The SysPower table is processed if and only if this parameter is set to True. default: True process_caldevice -- The CalDevice table is processed if and only if this parameter is set to True. default: True process_pointing -- The Pointing table is processed if and only if this parameter is set to True. If the parameter is set to False the resulting MS will have an empty POINTING table. default: True process_flags -- Process the online flags and save them to the FLAG_CMD sub-table. default: True >>> process_flags expandable parameter tbuff -- Time padding buffer (in seconds). default: 0.0 applyflags -- Apply the online flags to the MS. default: False savecmds -- Save the online flags to an ASCII file. default: False outfile -- Filename to save the online flags. default: '' flagbackup -- Backup the FLAG column in the .flagversions. default: True verbose -- produce log output as asdm2MS is being run. overwrite -- Over write an existing MS. showversion -- report the version of the asdm2MS being used. useversion -- Selects the version of asdm2MS to be used (presently only \'v3\' is available). default: v3 bdfflags -- Set the MS FLAG column according to the ASDM _binary_ flags default: false with_pointing_correction -- add (ASDM::Pointing::encoder - ASDM::Pointing::pointingDirection) to the value to be written in MS::Pointing::direction default: false remove_ref_undef -- if set to True then apply fixspwbackport on the resulting MSes. convert_ephem2geo -- if True, convert any attached ephemerides to the GEO reference frame """ # Python script # make agentflagger tool local aflocal = casac.agentflagger() # make table tool local tblocal = casac.table() try: casalog.origin('importasdm') viso = '' visoc = '' # for the wvr corrected version, if needed # ----------------------------------------- # beginning of importasdm_sd implementation # ----------------------------------------- if singledish: theexecutable = 'asdm2ASAP' # if useversion == 'v2': # theexecutable = 'oldasdm2ASAP' if compression: casalog.post('compression=True has no effect for single-dish format.') cmd = 'which %s > /dev/null 2>&1' % theexecutable ret = os.system(cmd) if ret == 0: import commands casalog.post('found %s' % theexecutable) if showversion: execute_string = theexecutable + ' --help' else: execute_string = theexecutable + ' -asdm ' + asdm if len(vis) != 0: execute_string += ' -asap ' + vis.rstrip('/') execute_string += ' -antenna ' + str(antenna) \ + ' -apc ' + wvr_corrected_data \ + ' -time-sampling ' + time_sampling.lower() \ + ' -overwrite ' + str(overwrite) if corr_mode == 'all': execute_string += \ ' -corr-mode ao,ca -ocorr-mode ao' else: execute_string += ' -corr-mode ' \ + corr_mode.replace(' ', ',') \ + ' -ocorr-mode ao' if srt == 'all': execute_string += ' -srt ' + srt else: execute_string += ' -srt ' + srt.replace(' ', ',') execute_string += ' -logfile ' + casalog.logfile() casalog.post('execute_string is') casalog.post(' ' + execute_string) ret = os.system(execute_string) if ret != 0 and not showversion: casalog.post(theexecutable + ' terminated with exit code ' + str(ret), 'SEVERE') # raise Exception, "ASDM conversion error, please check if it is a valid ASDM and/or useversion='%s' is consistent with input ASDM."%(useversion) raise Exception, \ 'ASDM conversion error, please check if it is a valid ASDM.' else: casalog.post('You have to build ASAP to be able to create single-dish data.' , 'SEVERE') # implementation of asis option using tb.fromASDM if asis != '': import commands asdmTables = commands.getoutput('ls %s/*.xml' % asdm).split('\n') asdmTabNames = [] for tab in asdmTables: asdmTabNames.append(tab.split('/')[-1].rstrip('.xml' )) if asis == '*': targetTables = asdmTables targetTabNames = asdmTabNames else: targetTables = [] targetTabNames = [] tmpTabNames = asis.split() for i in xrange(len(tmpTabNames)): tab = tmpTabNames[i] try: targetTables.append(asdmTables[asdmTabNames.index(tab)]) targetTabNames.append(tab) except: pass outTabNames = [] outTables = [] for tab in targetTabNames: out = 'ASDM_' + tab.upper() outTabNames.append(out) outTables.append(vis + '/' + out) tblocal.open(vis, nomodify=False) wtb = casac.table() for i in xrange(len(outTables)): wtb.fromASDM(outTables[i], targetTables[i]) tblocal.putkeyword(outTabNames[i], 'Table: %s' % outTables[i]) tblocal.flush() tblocal.close() return # ----------------------------------- # end of importasdm_sd implementation # ----------------------------------- if len(vis) > 0: viso = vis tmps = vis.rstrip('.ms') if tmps == vis: visoc = vis + '-wvr-corrected' else: visoc = tmps + '-wvr-corrected.ms' if singledish: viso = vis.rstrip('/') + '.importasdm.tmp.ms' else: viso = asdm.rstrip("/") + '.ms' visoc = asdm.rstrip("/") + '-wvr-corrected.ms' vis = asdm.rstrip("/") if singledish: viso = asdm.rstrip('/') + '.importasdm.tmp.ms' vis = asdm.rstrip('/') + '.asap' useversion = 'v3' theexecutable = 'asdm2MS' execute_string = theexecutable + ' --icm "' + corr_mode \ + '" --isrt "' + srt + '" --its "' + time_sampling \ + '" --ocm "' + ocorr_mode + '" --wvr-corrected-data "' \ + wvr_corrected_data + '" --asis "' + asis \ + '" --logfile "' + casalog.logfile() + '"' if len(scans) > 0: execute_string = execute_string + ' --scans ' + scans if ignore_time: execute_string = execute_string + ' --ignore-time' if useversion == 'v3': if not process_syspower: execute_string = execute_string + ' --no-syspower' if not process_caldevice: execute_string = execute_string + ' --no-caldevice' if not process_pointing: execute_string = execute_string + ' --no-pointing' if compression: execute_string = execute_string + ' --compression' elif lazy: execute_string = execute_string + ' --lazy' if verbose: execute_string = execute_string + ' --verbose' # if not overwrite and os.path.exists(viso): # raise Exception, \ # 'You have specified an existing MS and have indicated you do not wish to overwrite it' # Compression if compression: # viso = viso + '.compressed' viso = viso.rstrip('.ms') + '.compressed.ms' visoc = visoc.rstrip('.ms') + '.compressed.ms' vistoproc = [] # the output MSs to post-process if wvr_corrected_data == 'no' or wvr_corrected_data == 'both': vistoproc.append(viso) if (wvr_corrected_data == 'yes' or wvr_corrected_data == 'both') : vistoproc.append(visoc) for ff in vistoproc: if not overwrite and os.path.exists(ff): raise Exception, \ 'You have specified an existing MS and have indicated you do not wish to overwrite it: %s'%ff # If viso+".flagversions" then process differently depending on the value of overwrite.. # if flagbackup: for myviso in vistoproc: dotFlagversion = myviso + '.flagversions' if os.path.exists(dotFlagversion): if overwrite: casalog.post("Found '" + dotFlagversion + "' . It'll be deleted before running the filler." ) os.system('rm -rf %s' % dotFlagversion) else: casalog.post("Found '%s' but can't overwrite it." % dotFlagversion) raise Exception, "Found '%s' but can't overwrite it." \ % dotFlagversion # Make outfile always a list if isinstance(outfile, str): if outfile == '': outfile = [] else: noutfile = [outfile] outfile = noutfile if savecmds: if len(outfile) == 0: # Create default names for the online flags for myviso in vistoproc: outfile.append(myviso.replace('.ms','_cmd.txt')) elif len(outfile) != len(vistoproc): casalog.post('List of outfile names does not match list of MSs','WARN') casalog.post('Will save online flags to temporary filenames', 'WARN') outfile = [] for myviso in vistoproc: online_file = myviso.replace('.ms','_TEMP_cmd.txt') outfile.append(online_file) if not overwrite: for of in outfile: if os.path.exists(of): raise Exception, "Cannot overwrite online flags file '%s'; overwrite is set to False."% of execute_string = execute_string + ' ' + asdm + ' ' + viso if showversion: casalog.post("You set option \'showversion\' to True. Will just output the version information and then terminate." , 'WARN') execute_string = theexecutable + ' --revision' if with_pointing_correction: execute_string = execute_string + ' --with-pointing-correction' casalog.post('Running ' + theexecutable + ' standalone invoked as:') # print execute_string casalog.post(execute_string) exitcode = os.system(execute_string) if exitcode != 0: if not showversion: casalog.post(theexecutable + ' terminated with exit code ' + str(exitcode), 'SEVERE') raise Exception, \ 'ASDM conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.' if showversion: return # # Possibly remove the element the name of the measurement set expected to contain the corrected data from the list of of produced measurement # sets if it appears the filler did not find any corrected data. # if not os.path.exists(visoc): vistoproc = [myviso for myviso in vistoproc if myviso != visoc] # CAS-7369. HISTORY should be written after createmms is tested # # Populate the HISTORY table of the MS with information about the context in which it's been created # try: mslocal = mstool() param_names = importasdm.func_code.co_varnames[:importasdm.func_code.co_argcount] param_vals = [eval(p) for p in param_names] for myviso in vistoproc: write_history(mslocal, myviso, 'importasdm', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') return False if mslocal: mslocal = None # # Do we apply fixspwbackport if remove_ref_undef : casalog.post('remove_ref_undef=True: fixspwbackport will be applied ...') for myviso in vistoproc: cmd = 'fixspwbackport ' + myviso casalog.post('Running fixspwbackport standalone invoked as:') casalog.post(cmd) cmdexitcode = os.system(cmd) if cmdexitcode != 0: casalog.post(cmd + ' terminated with exit code ' + str(cmdexitcode), 'SEVERE') raise Exception, 'fixspwbackport error.' # Binary Flag processing if bdfflags: casalog.post('Parameter bdfflags==True: flags from the ASDM binary data will be used to set the MS flags ...') bdffexecutable = 'bdflags2MS ' bdffexecstring_base = bdffexecutable+' -f ALL' if len(scans) > 0: bdffexecstring_base = bdffexecstring_base + ' --scans ' + scans for myviso in vistoproc: if myviso.find("wvr-corrected") != -1: options = " --wvr-corrected=True " else: options = " " bdffexecstring = bdffexecstring_base + options + asdm + ' ' + myviso casalog.post('Running '+bdffexecutable+' standalone invoked as:') casalog.post(bdffexecstring) bdffexitcode = os.system(bdffexecstring) if bdffexitcode != 0: casalog.post(bdffexecutable + ' terminated with exit code ' + str(bdffexitcode), 'SEVERE') raise Exception, \ 'ASDM binary flags conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.' for myviso in vistoproc: if os.path.exists(myviso) and flagbackup==True: aflocal.open(myviso) aflocal.saveflagversion('Original', comment='Original flags at import into CASA', merge='save') aflocal.done() # Importasdm Flag Parsing if os.access(asdm + '/Flag.xml', os.F_OK): # Find Flag.xml casalog.post('Found Flag.xml in SDM') # Find Antenna.xml if os.access(asdm + '/Antenna.xml', os.F_OK): casalog.post('Found Antenna.xml in SDM') else: raise Exception, 'Failed to find Antenna.xml in SDM' # Find SpectralWindow.xml if os.access(asdm + '/SpectralWindow.xml', os.F_OK): casalog.post('Found SpectralWindow.xml in SDM') else: raise Exception, \ 'Failed to find SpectralWindow.xml in SDM' # # Parse Flag.xml into flag dictionary # if process_flags: flagcmds = fh.parseXML(asdm, float(tbuff)) onlinekeys = flagcmds.keys() nflags = onlinekeys.__len__() # Apply flags to the MS if nflags > 0: idx = 0 for myviso in vistoproc: if applyflags: # Open the MS and attach it to the tool aflocal.open(myviso) # Select the data aflocal.selectdata() # Setup the agent's parameters fh.parseAgents(aflocal, flagcmds, [], True, True, '') # Initialize the agents aflocal.init() # Run the tool aflocal.run(True, True) casalog.post('Applied %s flag commands to %s'%(nflags,myviso)) # Destroy the tool and de-attach the MS aflocal.done() # Save to FLAG_CMD table. APPLIED is set to True. fh.writeFlagCommands(myviso, flagcmds, True, '', '', True) else: casalog.post('Will not apply flags to %s (apply_flags=False), use flagcmd to apply'%myviso) # Write to FLAG_CMD, APPLIED is set to False fh.writeFlagCommands(myviso, flagcmds, False, '', '', True) # Save the flag cmds to an ASCII file if savecmds: # Save to standard filename fh.writeFlagCommands(myviso, flagcmds, False, '', outfile[idx], False) casalog.post('Saved %s flag commands to %s'%(nflags,outfile[idx])) idx += 1 else: casalog.post('There are no flag commands to process') else: casalog.post('There is no Flag.xml in ASDM', 'WARN') import recipes.ephemerides.convertephem as ce theephemfields = ce.findattachedephemfields(myviso,field='*') if len(theephemfields)>0: # temporary fix until asdm2MS does this internally: recalc the UVW coordinates for ephem fields imt = imtool() imt.open(myviso, usescratch=False) imt.calcuvw(theephemfields, refcode='J2000', reuse=False) imt.close() if convert_ephem2geo: for myviso in vistoproc: ce.convert2geo(myviso, '*') # convert any attached ephemerides to GEO # CAS-7369 - Create an output Multi-MS (MMS) if createmms: # Get the default parameters of partition from tasks import partition fpars = partition.parameters for mypar in fpars.keys(): fpars[mypar] = partition.itsdefault(mypar) # Call the cluster for each MS for myviso in vistoproc: casalog.origin('importasdm') outputmms = myviso.replace('.ms','.mms') # Get the proper column datacolumn = 'DATA' dcols = ['DATA', 'FLOAT_DATA'] for dc in dcols: if len(th.getColDesc(myviso, dc)) > 0: datacolumn = dc break fpars['datacolumn'] = datacolumn casalog.post('Will create a Multi-MS for: '+myviso) fpars['vis'] = myviso fpars['flagbackup'] = False fpars['outputvis'] = outputmms fpars['separationaxis'] = separationaxis fpars['numsubms'] = numsubms pdh = ParallelDataHelper('partition', fpars) # Get a cluster pdh.setupCluster(thistask='partition') try: pdh.go() # Rename MMS to MS shutil.rmtree(myviso) shutil.move(outputmms, myviso) except Exception, instance: casalog.post('%s'%instance,'ERROR') return False casalog.origin('importasdm') return
def test_mms4(self): '''test_mms4: Create MMS, lazy=True, with separationaxis=scan and scans selection in ASDM''' retValue = {'success': True, 'msgs': "", 'error_msgs': '' } myasdmname = 'uid___A002_X72bc38_X000' themsname = myasdmname + ".ms" # only the first 3 scans to save time importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~3', createmms=True, separationaxis='scan', process_flags=False) self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertEqual(ph.axisType(themsname), 'scan', 'Separation axis of MMS should be scan') print myname, ": Success! Now checking output ..." mscomponents = set(["ANTENNA/table.dat", "DATA_DESCRIPTION/table.dat", "FEED/table.dat", "FIELD/table.dat", "FLAG_CMD/table.dat", "HISTORY/table.dat", "OBSERVATION/table.dat", "POINTING/table.dat", "POLARIZATION/table.dat", "PROCESSOR/table.dat", "SOURCE/table.dat", "SPECTRAL_WINDOW/table.dat", "STATE/table.dat", "SYSCAL/table.dat", "ANTENNA/table.f0", "DATA_DESCRIPTION/table.f0", "FEED/table.f0", "FIELD/table.f0", "FLAG_CMD/table.f0", "HISTORY/table.f0", "OBSERVATION/table.f0", "POINTING/table.f0", "POLARIZATION/table.f0", "PROCESSOR/table.f0", "SOURCE/table.f0", "SPECTRAL_WINDOW/table.f0", "STATE/table.f0", "SYSCAL/table.f0" ]) for name in mscomponents: if not os.access(themsname+"/"+name, os.F_OK): print myname, ": Error ", themsname+"/"+name, "doesn't exist ..." retValue['success']=False retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist' else: print myname, ": ", name, "present." print myname, ": MS exists. All tables present. Try opening as MS ..." try: mslocal.open(themsname) except: print myname, ": Error Cannot open MS table", themsname retValue['success']=False retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname else: mslocal.close() print myname, ": OK. Checking tables in detail ..." importasdm(asdm=myasdmname, vis='reference.ms', lazy=False, overwrite=True, scans='0:1~3', createmms=True, separationaxis='scan', process_flags=False) if(os.path.exists('reference.ms')): retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from " +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0 if not retValue['success']: print "ERROR: DATA does not agree with reference." else: print "DATA columns agree." retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from " +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") == 0 if not retValueTmp: print "ERROR: FLAG does not agree with reference." else: print "FLAG columns agree." retValue['success'] = retValue['success'] and retValueTmp for subtname in ["ANTENNA", "DATA_DESCRIPTION", "FEED", "FIELD", "FLAG_CMD", "OBSERVATION", "POINTING", "POLARIZATION", "PROCESSOR", "SOURCE", "SPECTRAL_WINDOW", "STATE", "SYSCAL"]: print "\n*** Subtable ",subtname excllist = [] if subtname=='SOURCE': excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL'] if subtname=='SYSCAL': excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM'] if subtname=='SPECTRAL_WINDOW': excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION'] for colname in excllist: retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW', themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success'] try: retValue['success'] = th.compTables('reference.ms/'+subtname, themsname+'/'+subtname, excllist, 0.01) and retValue['success'] except: retValue['success'] = False print "ERROR for table ", subtname print retValue self.assertTrue(retValue['success'],retValue['error_msgs'])
def mstransform( vis, outputvis, # output createmms, # MMS --> partition separationaxis, numsubms, tileshape, # tiling field, spw, scan, antenna, correlation, timerange, intent, array, uvrange, observation, feed, datacolumn, realmodelcol, keepflags, usewtspectrum, combinespws, # spw combination --> cvel chanaverage, # channel averaging --> split chanbin, hanning, # Hanning --> cvel regridms, # regridding to new frame --> cvel mode, nchan, start, width, nspw, # spw separation interpolation, phasecenter, restfreq, outframe, veltype, timeaverage, # time averaging --> split timebin, timespan, maxuvwdistance, docallib, callib, douvcontsub, fitspw, fitorder, want_cont, denoising_lib, nthreads, niter, disableparallel, # HIDDEN parameter to create MMS in sequential ddistart, # HIDDEN internal parameter for the sub-table re-indexing taql, # HIDDEN internal parameter monolithic_processing, # HIDDEN parameter for the MMS monolithic processing reindex # HIDDEN parameter for use in the pipeline context only ): ''' This task can replace split, cvel, partition and hanningsmooth ''' casalog.origin('mstransform') ''' HEURISTICS: input MS --> output MS input MMS --> output MMS user can set createmms=True to create the following: input MS --> output MMS ''' # Initialize the helper class pdh = ParallelDataHelper('mstransform', locals()) # When dealing with MMS, process in parallel or sequential # disableparallel is a hidden parameter. Only for debugging purposes! if disableparallel: pdh.bypassParallelProcessing(1) else: pdh.bypassParallelProcessing(0) # Validate input and output parameters try: pdh.setupIO() except Exception, instance: casalog.post('%s'%instance,'ERROR') return False
def test_mms4(self): '''test_mms4: Create MMS, lazy=True, with separationaxis=scan and scans selection in ASDM''' retValue = {'success': True, 'msgs': "", 'error_msgs': ''} myasdmname = 'uid___A002_X72bc38_X000' themsname = myasdmname + ".ms" # only the first 3 scans to save time importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~3', createmms=True, separationaxis='scan', process_flags=False) self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertEqual(ph.axisType(themsname), 'scan', 'Separation axis of MMS should be scan') print myname, ": Success! Now checking output ..." mscomponents = set([ "ANTENNA/table.dat", "DATA_DESCRIPTION/table.dat", "FEED/table.dat", "FIELD/table.dat", "FLAG_CMD/table.dat", "HISTORY/table.dat", "OBSERVATION/table.dat", "POINTING/table.dat", "POLARIZATION/table.dat", "PROCESSOR/table.dat", "SOURCE/table.dat", "SPECTRAL_WINDOW/table.dat", "STATE/table.dat", "SYSCAL/table.dat", "ANTENNA/table.f0", "DATA_DESCRIPTION/table.f0", "FEED/table.f0", "FIELD/table.f0", "FLAG_CMD/table.f0", "HISTORY/table.f0", "OBSERVATION/table.f0", "POINTING/table.f0", "POLARIZATION/table.f0", "PROCESSOR/table.f0", "SOURCE/table.f0", "SPECTRAL_WINDOW/table.f0", "STATE/table.f0", "SYSCAL/table.f0" ]) for name in mscomponents: if not os.access(themsname + "/" + name, os.F_OK): print myname, ": Error ", themsname + "/" + name, "doesn't exist ..." retValue['success'] = False retValue['error_msgs'] = retValue[ 'error_msgs'] + themsname + '/' + name + ' does not exist' else: print myname, ": ", name, "present." print myname, ": MS exists. All tables present. Try opening as MS ..." try: mslocal.open(themsname) except: print myname, ": Error Cannot open MS table", themsname retValue['success'] = False retValue['error_msgs'] = retValue[ 'error_msgs'] + 'Cannot open MS table ' + themsname else: mslocal.close() print myname, ": OK. Checking tables in detail ..." importasdm(asdm=myasdmname, vis='reference.ms', lazy=False, overwrite=True, scans='0:1~3', createmms=True, separationaxis='scan', process_flags=False) if (os.path.exists('reference.ms')): retValue['success'] = th.checkwithtaql( "select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from " + themsname + " orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))" ) == 0 if not retValue['success']: print "ERROR: DATA does not agree with reference." else: print "DATA columns agree." retValueTmp = th.checkwithtaql( "select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from " + themsname + " orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) " ) == 0 if not retValueTmp: print "ERROR: FLAG does not agree with reference." else: print "FLAG columns agree." retValue['success'] = retValue['success'] and retValueTmp for subtname in [ "ANTENNA", "DATA_DESCRIPTION", "FEED", "FIELD", "FLAG_CMD", "OBSERVATION", "POINTING", "POLARIZATION", "PROCESSOR", "SOURCE", "SPECTRAL_WINDOW", "STATE", "SYSCAL" ]: print "\n*** Subtable ", subtname excllist = [] if subtname == 'SOURCE': excllist = [ 'POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL' ] if subtname == 'SYSCAL': excllist = ['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM'] if subtname == 'SPECTRAL_WINDOW': excllist = [ 'CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION' ] for colname in excllist: retValue['success'] = th.compVarColTables( 'reference.ms/SPECTRAL_WINDOW', themsname + '/SPECTRAL_WINDOW', colname, 0.01) and retValue['success'] try: retValue['success'] = th.compTables( 'reference.ms/' + subtname, themsname + '/' + subtname, excllist, 0.01) and retValue['success'] except: retValue['success'] = False print "ERROR for table ", subtname print retValue self.assertTrue(retValue['success'], retValue['error_msgs'])
def test_mms6(self): '''test_mms6: Create MMS and lazy=True''' retValue = {'success': True, 'msgs': "", 'error_msgs': ''} myasdmname = 'uid___A002_X71e4ae_X317_short' themsname = myasdmname + ".ms" flagfile = myasdmname + '_cmd.txt' self.res = importasdm( myasdmname, vis=themsname, lazy=True, scans='0:1~4', createmms=True, savecmds=True) # only the first 4 scans to save time self.assertEqual(self.res, None) self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS') self.assertTrue(os.path.exists(flagfile)) print myname, ": Success! Now checking output ..." mscomponents = set([ "ANTENNA/table.dat", "DATA_DESCRIPTION/table.dat", "FEED/table.dat", "FIELD/table.dat", "FLAG_CMD/table.dat", "HISTORY/table.dat", "OBSERVATION/table.dat", "POINTING/table.dat", "POLARIZATION/table.dat", "PROCESSOR/table.dat", "SOURCE/table.dat", "SPECTRAL_WINDOW/table.dat", "STATE/table.dat", "SYSCAL/table.dat", "ANTENNA/table.f0", "DATA_DESCRIPTION/table.f0", "FEED/table.f0", "FIELD/table.f0", "FLAG_CMD/table.f0", "HISTORY/table.f0", "OBSERVATION/table.f0", "POINTING/table.f0", "POLARIZATION/table.f0", "PROCESSOR/table.f0", "SOURCE/table.f0", "SPECTRAL_WINDOW/table.f0", "STATE/table.f0", "SYSCAL/table.f0" ]) for name in mscomponents: if not os.access(themsname + "/" + name, os.F_OK): print myname, ": Error ", themsname + "/" + name, "doesn't exist ..." retValue['success'] = False retValue['error_msgs'] = retValue[ 'error_msgs'] + themsname + '/' + name + ' does not exist' else: print myname, ": ", name, "present." print myname, ": MS exists. All tables present. Try opening as MS ..." try: mslocal.open(themsname) mslocal.close() print myname, ": MS can be opened. Now testing the changing of the asdmref ..." mslocal.open(themsname) mslocal.asdmref("./moved_" + myasdmname) mslocal.close() os.system("mv " + myasdmname + " moved_" + myasdmname) mslocal.open(themsname) except: print myname, ": Error Cannot open MS table", themsname retValue['success'] = False retValue['error_msgs'] = retValue[ 'error_msgs'] + 'Cannot open MS table ' + themsname else: mslocal.close() print myname, ": OK. Checking tables in detail ..." importasdm(asdm="moved_" + myasdmname, vis='reference.ms', lazy=False, overwrite=True, scans='0:1~3', createmms=True) if (os.path.exists('reference.ms')): retValue['success'] = th.checkwithtaql( "select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from " + themsname + " orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))" ) == 0 if not retValue['success']: print "ERROR: DATA does not agree with reference." else: print "DATA columns agree." retValueTmp = th.checkwithtaql( "select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from " + themsname + " orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.WEIGHT,t2.WEIGHT, 1.e-06)))" ) == 0 if not retValueTmp: print "ERROR: WEIGHT does not agree with reference." else: print "WEIGHT columns agree." retValueTmp2 = th.checkwithtaql( "select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from " + themsname + " orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) " ) == 0 if not retValueTmp2: print "ERROR: FLAG does not agree with reference." else: print "FLAG columns agree." retValue['success'] = retValue[ 'success'] and retValueTmp and retValueTmp2 for subtname in [ "ANTENNA", "DATA_DESCRIPTION", "FEED", "FIELD", "FLAG_CMD", "OBSERVATION", "POLARIZATION", "PROCESSOR", "SOURCE", "SPECTRAL_WINDOW", "STATE", "SYSCAL" ]: print "\n*** Subtable ", subtname excllist = [] if subtname == 'SOURCE': excllist = [ 'POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL' ] if subtname == 'SYSCAL': excllist = ['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM'] if subtname == 'SPECTRAL_WINDOW': excllist = [ 'CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE' ] for colname in excllist: if colname != 'ASSOC_NATURE': retValue['success'] = th.compVarColTables( 'reference.ms/SPECTRAL_WINDOW', themsname + '/SPECTRAL_WINDOW', colname, 0.01) and retValue['success'] if subtname == 'POLARIZATION': excllist = ['CORR_TYPE', 'CORR_PRODUCT'] for colname in excllist: retValue['success'] = th.compVarColTables( 'reference.ms/POLARIZATION', themsname + '/POLARIZATION', colname, 0.01) and retValue['success'] try: retValue['success'] = th.compTables( 'reference.ms/' + subtname, themsname + '/' + subtname, excllist, 0.01) and retValue['success'] except: retValue['success'] = False print "ERROR for table ", subtname os.system("mv moved_" + myasdmname + " " + myasdmname) self.assertTrue(retValue['success'], retValue['error_msgs'])
def mstransform( vis, outputvis, # output createmms, # MMS --> partition separationaxis, numsubms, tileshape, # tiling field, spw, scan, antenna, correlation, timerange, intent, array, uvrange, observation, feed, datacolumn, realmodelcol, keepflags, usewtspectrum, combinespws, # spw combination --> cvel chanaverage, # channel averaging --> split chanbin, hanning, # Hanning --> cvel regridms, # regridding to new frame --> cvel mode, nchan, start, width, nspw, # spw separation interpolation, phasecenter, restfreq, outframe, veltype, preaverage, timeaverage, # time averaging --> split timebin, timespan, maxuvwdistance, docallib, callib, douvcontsub, fitspw, fitorder, want_cont, denoising_lib, nthreads, niter, disableparallel, # HIDDEN parameter to create MMS in sequential ddistart, # HIDDEN internal parameter for the sub-table re-indexing taql, # HIDDEN internal parameter monolithic_processing, # HIDDEN parameter for the MMS monolithic processing reindex # HIDDEN parameter for use in the pipeline context only ): ''' This task can replace split, cvel, partition and hanningsmooth ''' casalog.origin('mstransform') ''' HEURISTICS: input MS --> output MS input MMS --> output MMS user can set createmms=True to create the following: input MS --> output MMS ''' # Initialize the helper class pdh = ParallelDataHelper('mstransform', locals()) # When dealing with MMS, process in parallel or sequential # disableparallel is a hidden parameter. Only for debugging purposes! if disableparallel: pdh.bypassParallelProcessing(1) else: pdh.bypassParallelProcessing(0) # Validate input and output parameters try: pdh.setupIO() except Exception, instance: casalog.post('%s'%instance,'ERROR') return False
def importasdm(asdm=None, vis=None, createmms=None, separationaxis=None, numsubms=None, corr_mode=None, srt=None, time_sampling=None, ocorr_mode=None, compression=None, lazy=None, asis=None, wvr_corrected_data=None, scans=None, ignore_time=None, process_syspower=None, process_caldevice=None, process_pointing=None, process_flags=None, tbuff=None, applyflags=None, savecmds=None, outfile=None, flagbackup=None, verbose=None, overwrite=None, showversion=None, useversion=None, bdfflags=None, with_pointing_correction=None, remove_ref_undef=None, convert_ephem2geo=None, polyephem_tabtimestep=None): """Convert an ALMA Science Data Model observation into a CASA visibility file (MS) or single-dish data format (Scantable). The conversion of the ALMA SDM archive format into a measurement set. This version is under development and is geared to handling many spectral windows of different shapes. Keyword arguments: asdm -- Name of input ASDM file (directory) default: none; example: asdm='ExecBlock3' vis -- Root ms or scantable name, note a prefix (.ms or .asap) is NOT appended to this name default: none createmms -- Create a Multi-MS default: False corr_mode -- correlation mode to be considered on input. Could be one or more of the following, ao, co, ac, or all default: all srt -- spectral resolution type. Could be one or more of the following, fr, ca, bw, or all default: all time_sampling -- specifies the time sampling, INTEGRATION and/or SUBINTEGRATION. could be one or more of the following i, si, or all. default: all ocorr_mode -- output data for correlation mode AUTO_ONLY (ao) or CROSS_ONLY (co) or CROSS_AND_AUTO (ca) default: ca compression -- produces comrpressed columns in the resulting measurement set. default: False lazy -- Make the MS DATA column read the ASDM Binary data directly (faster import, smaller MS) default: False asis -- creates verbatim copies of the ASDM tables in the output measurement set. The value given to this option must be a list of table names separated by space characters; the wildcard character '*' is allowed in table names. wvr_corrected_data -- specifies wich values are considered in the ASDM binary data to fill the DATA column in the MAIN table of the MS. Expected values for this option are 'no' for the uncorrected data (this is the default), 'yes' for the corrected data and 'both' for corrected and uncorrected data. In the latter case, two measurement sets are created, one containing the uncorrected data and the other one, whose name is suffixed by '-wvr-corrected', containing the corrected data. scans -- processes only the scans specified in the option's value. This value is a semicolon separated list of scan specifications. A scan specification consists in an exec bock index followed by the character ':' followed by a comma separated list of scan indexes or scan index ranges. A scan index is relative to the exec block it belongs to. Scan indexes are 1-based while exec blocks's are 0-based. "0:1" or "2:2~6" or "0:1,1:2~6,8;2:,3:24~30" "1,2" are valid values for the option. "3:" alone will be interpreted as 'all the scans of the exec block#3'. An scan index or a scan index range not preceded by an exec block index will be interpreted as 'all the scans with such indexes in all the exec blocks'. By default all the scans are considered. ignore_time -- All the rows of the tables Feed, History, Pointing, Source, SysCal, CalDevice, SysPower, and Weather are processed independently of the time range of the selected exec block / scan. process_syspower -- The SysPower table is processed if and only if this parameter is set to True. default: True process_caldevice -- The CalDevice table is processed if and only if this parameter is set to True. default: True process_pointing -- The Pointing table is processed if and only if this parameter is set to True. If the parameter is set to False the resulting MS will have an empty POINTING table. default: True process_flags -- Process the online flags and save them to the FLAG_CMD sub-table. default: True >>> process_flags expandable parameter tbuff -- Time padding buffer (in seconds). default: 0.0 applyflags -- Apply the online flags to the MS. default: False savecmds -- Save the online flags to an ASCII file. default: False outfile -- Filename to save the online flags. default: '' flagbackup -- Backup the FLAG column in the .flagversions. default: True verbose -- produce log output as asdm2MS is being run. overwrite -- Over write an existing MS. showversion -- report the version of the asdm2MS being used. useversion -- Selects the version of asdm2MS to be used (presently only \'v3\' is available). default: v3 bdfflags -- Set the MS FLAG column according to the ASDM _binary_ flags default: false with_pointing_correction -- add (ASDM::Pointing::encoder - ASDM::Pointing::pointingDirection) to the value to be written in MS::Pointing::direction default: false remove_ref_undef -- if set to True then apply fixspwbackport on the resulting MSes. convert_ephem2geo -- if True, convert any attached ephemerides to the GEO reference frame polyephem_tabtimestep -- Timestep (days) for the tabulation of polynomial ephemerides. A value <= 0 disables tabulation. Presently, VLA data can contain polynomial ephemerides. ALMA data uses tabulated values. default: 0. """ # Python script # make agentflagger tool local aflocal = casac.agentflagger() # make table tool local tblocal = casac.table() try: casalog.origin('importasdm') viso = '' visoc = '' # for the wvr corrected version, if needed if len(vis) > 0: viso = vis tmps = vis.rstrip('.ms') if tmps == vis: visoc = vis + '-wvr-corrected' else: visoc = tmps + '-wvr-corrected.ms' else: viso = asdm.rstrip("/") + '.ms' visoc = asdm.rstrip("/") + '-wvr-corrected.ms' vis = asdm.rstrip("/") useversion = 'v3' theexecutable = 'asdm2MS' execute_string = theexecutable + ' --icm "' + corr_mode \ + '" --isrt "' + srt + '" --its "' + time_sampling \ + '" --ocm "' + ocorr_mode + '" --wvr-corrected-data "' \ + wvr_corrected_data + '" --asis "' + asis \ + '" --logfile "' + casalog.logfile() + '"' if len(scans) > 0: execute_string = execute_string + ' --scans ' + scans if ignore_time: execute_string = execute_string + ' --ignore-time' if useversion == 'v3': if not process_syspower: execute_string = execute_string + ' --no-syspower' if not process_caldevice: execute_string = execute_string + ' --no-caldevice' if not process_pointing: execute_string = execute_string + ' --no-pointing' if compression: execute_string = execute_string + ' --compression' elif lazy: execute_string = execute_string + ' --lazy' if verbose: execute_string = execute_string + ' --verbose' # if not overwrite and os.path.exists(viso): # raise Exception, \ # 'You have specified an existing MS and have indicated you do not wish to overwrite it' # Compression if compression: # viso = viso + '.compressed' viso = viso.rstrip('.ms') + '.compressed.ms' visoc = visoc.rstrip('.ms') + '.compressed.ms' vistoproc = [] # the output MSs to post-process if wvr_corrected_data == 'no' or wvr_corrected_data == 'both': vistoproc.append(viso) if (wvr_corrected_data == 'yes' or wvr_corrected_data == 'both'): vistoproc.append(visoc) for ff in vistoproc: if not overwrite and os.path.exists(ff): raise Exception, \ 'You have specified an existing MS and have indicated you do not wish to overwrite it: %s'%ff # If viso+".flagversions" then process differently depending on the value of overwrite.. # if flagbackup: for myviso in vistoproc: dotFlagversion = myviso + '.flagversions' if os.path.exists(dotFlagversion): if overwrite: casalog.post( "Found '" + dotFlagversion + "' . It'll be deleted before running the filler.") os.system('rm -rf %s' % dotFlagversion) else: casalog.post("Found '%s' but can't overwrite it." % dotFlagversion) raise Exception, "Found '%s' but can't overwrite it." \ % dotFlagversion # Make outfile always a list if isinstance(outfile, str): if outfile == '': outfile = [] else: noutfile = [outfile] outfile = noutfile if savecmds: if len(outfile) == 0: # Create default names for the online flags for myviso in vistoproc: outfile.append(myviso.replace('.ms', '_cmd.txt')) elif len(outfile) != len(vistoproc): casalog.post( 'List of outfile names does not match list of MSs', 'WARN') casalog.post('Will save online flags to temporary filenames', 'WARN') outfile = [] for myviso in vistoproc: online_file = myviso.replace('.ms', '_TEMP_cmd.txt') outfile.append(online_file) if not overwrite: for of in outfile: if os.path.exists(of): raise Exception, "Cannot overwrite online flags file '%s'; overwrite is set to False." % of execute_string = execute_string + ' ' + asdm + ' ' + viso if showversion: casalog.post( "You set option \'showversion\' to True. Will just output the version information and then terminate.", 'WARN') execute_string = theexecutable + ' --revision' if with_pointing_correction: execute_string = execute_string + ' --with-pointing-correction' if (polyephem_tabtimestep != None) and (type(polyephem_tabtimestep) == int or type(polyephem_tabtimestep) == float): if polyephem_tabtimestep > 0: casalog.post( 'Will tabulate all attached polynomial ephemerides with a time step of ' + str(polyephem_tabtimestep) + ' days.') if polyephem_tabtimestep > 1.: casalog.post( 'A tabulation timestep of <= 1 days is recommended.', 'WARN') execute_string = execute_string + ' --polyephem-tabtimestep ' + str( polyephem_tabtimestep) casalog.post('Running ' + theexecutable + ' standalone invoked as:') # print execute_string casalog.post(execute_string) exitcode = os.system(execute_string) if exitcode != 0: if not showversion: casalog.post( theexecutable + ' terminated with exit code ' + str(exitcode), 'SEVERE') raise Exception, \ 'ASDM conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.' if showversion: return # # Possibly remove the name of the measurement set expected to contain the corrected data from the list of of produced measurement # sets if it appears the filler did not find any corrected data. # if not os.path.exists(visoc): vistoproc = [myviso for myviso in vistoproc if myviso != visoc] # CAS-7369. HISTORY should be written after createmms is tested # # Populate the HISTORY table of the MS with information about the context in which it's been created # try: mslocal = mstool() param_names = importasdm.func_code.co_varnames[:importasdm. func_code. co_argcount] param_vals = [eval(p) for p in param_names] for myviso in vistoproc: write_history(mslocal, myviso, 'importasdm', param_names, param_vals, casalog) except Exception, instance: casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN') return False if mslocal: mslocal = None # # Do we apply fixspwbackport if remove_ref_undef: casalog.post( 'remove_ref_undef=True: fixspwbackport will be applied ...') for myviso in vistoproc: cmd = 'fixspwbackport ' + myviso casalog.post('Running fixspwbackport standalone invoked as:') casalog.post(cmd) cmdexitcode = os.system(cmd) if cmdexitcode != 0: casalog.post( cmd + ' terminated with exit code ' + str(cmdexitcode), 'SEVERE') raise Exception, 'fixspwbackport error.' # Binary Flag processing if bdfflags: casalog.post( 'Parameter bdfflags==True: flags from the ASDM binary data will be used to set the MS flags ...' ) bdffexecutable = 'bdflags2MS ' bdffexecstring_base = bdffexecutable + ' -f ALL' + ' --ocm "' + ocorr_mode \ + '" --logfile "' + casalog.logfile() + '"' if len(scans) > 0: bdffexecstring_base = bdffexecstring_base + ' --scans ' + scans if lazy and not compression: bdffexecstring_base = bdffexecstring_base + ' --lazy=true' for myviso in vistoproc: if myviso.find("wvr-corrected") != -1: options = " --wvr-corrected=True " else: options = " " bdffexecstring = bdffexecstring_base + options + asdm + ' ' + myviso casalog.post('Running ' + bdffexecutable + ' standalone invoked as:') casalog.post(bdffexecstring) bdffexitcode = os.system(bdffexecstring) if bdffexitcode != 0: casalog.post( bdffexecutable + ' terminated with exit code ' + str(bdffexitcode), 'SEVERE') raise Exception, \ 'ASDM binary flags conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.' theephemfields = ce.findattachedephemfields(myviso, field='*') if len(theephemfields) > 0: # until asdm2MS does this internally: recalc the UVW coordinates for ephem fields imt = imtool() imt.open(myviso, usescratch=False) imt.calcuvw(theephemfields, refcode='J2000', reuse=False) imt.close() if convert_ephem2geo: for myviso in vistoproc: ce.convert2geo(myviso, '*') # convert any attached ephemerides to GEO if len(theephemfields) > 0: # also set the direction column in the SOURCE table tblocal.open(myviso + '/FIELD', nomodify=False) sourceids = tblocal.getcol('SOURCE_ID') ftimes = tblocal.getcol('TIME') ftimekw = tblocal.getcolkeywords('TIME') tmpa = tblocal.getcol('PHASE_DIR') origphasedir = tmpa affectedsids = [] thesamplefields = [] for fld in theephemfields: # determine all source ids used by the ephem fields if not (sourceids[fld] in affectedsids): # this source id wasn't handled yet affectedsids.append(sourceids[fld]) thesamplefields.append(fld) # need to temporarily change the offset (not all mosaics have an element at (0,0)) tmpa[0][0][fld] = 0. tmpa[1][0][fld] = 0. #endif #endfor tblocal.putcol('PHASE_DIR', tmpa) tblocal.close() tblocal.open(myviso + '/SOURCE') sourceposref = tblocal.getcolkeywords( 'DIRECTION')['MEASINFO']['Ref'] tblocal.close() directions = [] msmdlocal = casac.msmetadata() msmdlocal.open(myviso) for fld in thesamplefields: thedirmeas = msmdlocal.phasecenter(fld) if thedirmeas['refer'] != sourceposref: casalog.post( 'Ephemeris is in ' + thedirmeas['refer'] + ' instead of ' + sourceposref + ' frame.\nEntry in SOURCE table will be converted to ' + sourceposref, 'WARN') melocal = metool() melocal.doframe(thedirmeas) thedirmeas = melocal.measure(thedirmeas, sourceposref) directions.append( [thedirmeas['m0']['value'], thedirmeas['m1']['value']]) thetime = me.epoch(v0=str(ftimes[fld]) + 's', rf=ftimekw['MEASINFO']['Ref']) casalog.post("Will set SOURCE direction for SOURCE_ID " + str(sourceids[fld]) + " to ephemeris phase center for time " + str(thetime['m0']['value']) + " " + thetime['m0']['unit'] + " " + thetime['refer']) #endfor msmdlocal.close() # restore original PHASE_DIR tblocal.open(myviso + '/FIELD', nomodify=False) tblocal.putcol('PHASE_DIR', origphasedir) tblocal.close() # write source directions tblocal.open(myviso + '/SOURCE', nomodify=False) ssourceids = tblocal.getcol('SOURCE_ID') sdirs = tblocal.getcol('DIRECTION') for row in xrange(0, len(ssourceids)): for i in xrange(0, len(affectedsids)): if ssourceids[row] == affectedsids[i]: sdirs[0][row] = directions[i][0] sdirs[1][row] = directions[i][1] break #endfor #endfor tblocal.putcol('DIRECTION', sdirs) # write back corrected directions tblocal.close() #end if ##############################################################################################3 # CAS-7369 - Create an output Multi-MS (MMS) if createmms: # Get the default parameters of partition from tasks import partition fpars = partition.parameters for mypar in fpars.keys(): fpars[mypar] = partition.itsdefault(mypar) # Call the cluster for each MS for myviso in vistoproc: casalog.origin('importasdm') # Move original MS to tempdir tempname = myviso + '.temp.ms' outputmms = myviso shutil.move(myviso, tempname) # Get the proper column datacolumn = 'DATA' dcols = ['DATA', 'FLOAT_DATA'] for dc in dcols: if len(th.getColDesc(tempname, dc)) > 0: datacolumn = dc break fpars['datacolumn'] = datacolumn casalog.post('Will create a Multi-MS for: ' + myviso) fpars['vis'] = tempname fpars['flagbackup'] = False fpars['outputvis'] = outputmms fpars['separationaxis'] = separationaxis fpars['numsubms'] = numsubms pdh = ParallelDataHelper('partition', fpars) # Get a cluster pdh.setupCluster(thistask='partition') try: pdh.go() # Remove original MS shutil.rmtree(tempname) except Exception, instance: # Restore MS in case of error in MMS creation shutil.move(tempname, myviso) casalog.post('%s' % instance, 'ERROR') return False casalog.origin('importasdm')