def test_flagsrestore(self): '''Partition: check that we can restore the flags''' # Delete any flagversions if os.path.exists(self.msfile + '.flagversions'): shutil.rmtree(self.msfile + '.flagversions') # Unflag the MS flagdata(vis=self.msfile, mode='unflag', flagbackup=False) # Run partition and create the .flagversions partition(vis=self.msfile, outputvis=self.mmsfile, createmms=True, disableparallel=True, datacolumn='data') self.assertTrue(os.path.exists(self.mmsfile + '.flagversions')) # Flag spw=9 and then spw=7 in the MMS flagdata(vis=self.mmsfile, mode='manual', spw='9', flagbackup=True) flagdata(vis=self.mmsfile, mode='manual', spw='7', flagbackup=True) # There should be flags in spw=7 and 9 in MMS res = flagdata(vis=self.mmsfile, mode='summary') self.assertEqual(res['flagged'], 549888) # Restore the original flags (there should be none) flagmanager(vis=self.mmsfile, mode='restore', versionname='partition_1') res = flagdata(vis=self.mmsfile, mode='summary') self.assertEqual(res['flagged'], 0)
def test_CAS2701(self): """flagmanager: Do not allow flagversions with empty versionname''""" try: flagmanager(vis = self.vis,mode = "save",versionname = "") except IOError, e: print 'Expected exception: %s'%e
def test_flagsrestore(self): '''Partition: check that we can restore the flags''' # Delete any flagversions if os.path.exists(self.msfile+'.flagversions'): shutil.rmtree(self.msfile+'.flagversions') # Unflag the MS flagdata(vis=self.msfile, mode='unflag', flagbackup=False) # Run partition and create the .flagversions partition(vis=self.msfile, outputvis=self.mmsfile, createmms=True, disableparallel=True, datacolumn='data') self.assertTrue(os.path.exists(self.mmsfile+'.flagversions')) # Flag spw=9 and then spw=7 in the MMS flagdata(vis=self.mmsfile, mode='manual', spw='9', flagbackup=True) flagdata(vis=self.mmsfile, mode='manual', spw='7', flagbackup=True) # There should be flags in spw=7 and 9 in MMS res = flagdata(vis=self.mmsfile, mode='summary') self.assertEqual(res['flagged'],549888) # Restore the original flags (there should be none) flagmanager(vis=self.mmsfile, mode='restore', versionname='partition_1') res = flagdata(vis=self.mmsfile, mode='summary') self.assertEqual(res['flagged'],0)
def test_caltable_flagbackup(self): '''Flagmanager:: cal table mode=list, flagbackup=True/False''' # Need a fresh start os.system('rm -rf cal.fewscans.bpass*') self.setUp_bpass_case() self.unflag_ms() flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 2) aflocal.done() flagdata(vis=self.vis, mode='unflag', flagbackup=False) flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 2) aflocal.done() flagdata(vis=self.vis, mode='unflag', flagbackup=True) flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 3) aflocal.done() newname = 'Ha! The best version ever!' flagmanager(vis=self.vis, mode='rename', oldname='flagdata_1', versionname=newname, comment='This is a *much* better name') flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 3) aflocal.done() self.assertTrue(os.path.exists(self.vis+'.flagversions/flags.'+newname), 'Flagversion file does not exist: flags.'+newname)
def test_CAS2701(self): """flagmanager: Do not allow flagversions with empty versionname''""" try: flagmanager(vis=self.vis, mode="save", versionname="") except IOError, e: print 'Expected exception: %s' % e
def test_flagsrestore(self): """Partition: check that we can restore the flags""" # Delete any flagversions if os.path.exists(self.msfile + ".flagversions"): shutil.rmtree(self.msfile + ".flagversions") # Unflag the MS flagdata(vis=self.msfile, mode="unflag", flagbackup=False) # Run partition and create the .flagversions partition(vis=self.msfile, outputvis=self.mmsfile, createmms=True, disableparallel=True, datacolumn="data") self.assertTrue(os.path.exists(self.mmsfile + ".flagversions")) # Flag spw=9 and then spw=7 in the MMS flagdata(vis=self.mmsfile, mode="manual", spw="9", flagbackup=True) flagdata(vis=self.mmsfile, mode="manual", spw="7", flagbackup=True) # There should be flags in spw=7 and 9 in MMS res = flagdata(vis=self.mmsfile, mode="summary") self.assertEqual(res["flagged"], 549888) # Restore the original flags (there should be none) flagmanager(vis=self.mmsfile, mode="restore", versionname="partition_1") res = flagdata(vis=self.mmsfile, mode="summary") self.assertEqual(res["flagged"], 0)
def test2m(self): """flagmanager test2m: Create, then restore autoflag""" self.unflag_ms() flagdata(vis=self.vis, mode='summary') flagmanager(vis=self.vis) flagdata(vis=self.vis, mode='manual', antenna="2", flagbackup=True) flagmanager(vis=self.vis) ant2 = flagdata(vis=self.vis, mode='summary')['flagged'] print "After flagging antenna 2 there were", ant2, "flags" # Change flags, then restore flagdata(vis=self.vis, mode='manual', antenna="3", flagbackup=True) flagmanager(vis = self.vis) ant3 = flagdata(vis=self.vis, mode='summary')['flagged'] print "After flagging antenna 2 and 3 there were", ant3, "flags" flagmanager(vis=self.vis, mode='restore', versionname='flagdata_2') restore2 = flagdata(vis=self.vis, mode='summary')['flagged'] print "After restoring pre-antenna 2 flagging, there are", restore2, "flags; should be", ant2 self.assertEqual(restore2, ant2)
def test2m(self): """flagmanager test2m: Create, then restore autoflag""" self.unflag_ms() flagdata(vis=self.vis, mode='summary') flagmanager(vis=self.vis) flagdata(vis=self.vis, mode='manual', antenna="2", flagbackup=True) flagmanager(vis=self.vis) ant2 = flagdata(vis=self.vis, mode='summary')['flagged'] print "After flagging antenna 2 there were", ant2, "flags" # Change flags, then restore flagdata(vis=self.vis, mode='manual', antenna="3", flagbackup=True) flagmanager(vis=self.vis) ant3 = flagdata(vis=self.vis, mode='summary')['flagged'] print "After flagging antenna 2 and 3 there were", ant3, "flags" flagmanager(vis=self.vis, mode='restore', versionname='flagdata_2') restore2 = flagdata(vis=self.vis, mode='summary')['flagged'] print "After restoring pre-antenna 2 flagging, there are", restore2, "flags; should be", ant2 self.assertEqual(restore2, ant2)
def test1m(self): '''flagmanager test1m: mode=list, flagbackup=True/False''' self.unflag_ms() flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 2) aflocal.done() flagdata(vis=self.vis, mode='unflag', flagbackup=False) flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 2) aflocal.done() flagdata(vis=self.vis, mode='unflag', flagbackup=True) flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 3) aflocal.done() newname = 'Ha! The best version ever!' flagmanager(vis=self.vis, mode='rename', oldname='flagdata_1', versionname=newname, comment='This is a *much* better name') flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 3) aflocal.done() self.assertTrue( os.path.exists(self.vis + '.flagversions/flags.' + newname), 'Flagversion file does not exist: flags.' + newname) # Specific for MMS if testmms: areg = self.vis + '/SUBMSS/*flagversions*' import glob print 'Check for .flagversions in the wrong place.' self.assertEqual( glob.glob(areg), [], 'There should not be any .flagversions in the' ' SUBMSS directory')
def test_rename(self): '''flagmanager: do not overwrite an existing versionname''' # Create a flagbackup flagdata(vis=self.vis, mode='manual', antenna="2", flagbackup=True) fname = 'flagdata_1' self.assertTrue(os.path.exists(self.vis+'.flagversions/flags.'+fname), 'Flagversions file does not exist: flags.'+fname) # Rename it newname = 'Rename_Me' flagmanager(vis=self.vis, mode='rename', oldname=fname, versionname=newname, comment='CAS-3080') self.assertTrue(os.path.exists(self.vis+'.flagversions/flags.'+newname), 'Flagversions file does not exist: flags.'+newname) self.assertFalse(os.path.exists(self.vis+'.flagversions/flags.'+fname), 'Flagversions file shuold not exist: flags.'+fname)
def test1m(self): '''flagmanager test1m: mode=list, flagbackup=True/False''' self.unflag_ms() flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 2) aflocal.done() flagdata(vis=self.vis, mode='unflag', flagbackup=False) flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 2) aflocal.done() flagdata(vis=self.vis, mode='unflag', flagbackup=True) flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 3) aflocal.done() newname = 'Ha! The best version ever!' flagmanager(vis=self.vis, mode='rename', oldname='flagdata_1', versionname=newname, comment='This is a *much* better name') flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 3) aflocal.done() self.assertTrue(os.path.exists(self.vis+'.flagversions/flags.'+newname), 'Flagversion file does not exist: flags.'+newname) # Specific for MMS if testmms: areg = self.vis+'/SUBMSS/*flagversions*' import glob print 'Check for .flagversions in the wrong place.' self.assertEqual(glob.glob(areg), [], 'There should not be any .flagversions in the' ' SUBMSS directory')
def test_caltable_flagbackup(self): '''Flagmanager:: cal table mode=list, flagbackup=True/False''' # Need a fresh start os.system('rm -rf cal.fewscans.bpass*') self.setUp_bpass_case() self.unflag_ms() flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 2) aflocal.done() flagdata(vis=self.vis, mode='unflag', flagbackup=False) flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 2) aflocal.done() flagdata(vis=self.vis, mode='unflag', flagbackup=True) flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 3) aflocal.done() newname = 'Ha! The best version ever!' flagmanager(vis=self.vis, mode='rename', oldname='flagdata_1', versionname=newname, comment='This is a *much* better name') flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 3) aflocal.done() self.assertTrue( os.path.exists(self.vis + '.flagversions/flags.' + newname), 'Flagversion file does not exist: flags.' + newname)
def test_rename(self): '''flagmanager: do not overwrite an existing versionname''' # Create a flagbackup flagdata(vis=self.vis, mode='manual', antenna="2", flagbackup=True) fname = 'flagdata_1' self.assertTrue( os.path.exists(self.vis + '.flagversions/flags.' + fname), 'Flagversions file does not exist: flags.' + fname) # Rename it newname = 'Rename_Me' flagmanager(vis=self.vis, mode='rename', oldname=fname, versionname=newname, comment='CAS-3080') self.assertTrue( os.path.exists(self.vis + '.flagversions/flags.' + newname), 'Flagversions file does not exist: flags.' + newname) self.assertFalse( os.path.exists(self.vis + '.flagversions/flags.' + fname), 'Flagversions file shuold not exist: flags.' + fname)
def test_dictionary1(self): '''flagmanager: Test the returned dictionary in list mode''' self.unflag_ms() # MS should start only with the standard info such as: # MS : /diska/scastro/work/cas11462/nosedir/cal.fewscans.bpass # main : working copy in main table aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 2) aflocal.done() flagsdict1 = flagmanager(vis=self.vis, mode='list') self.assertIsInstance( flagsdict1, dict, 'mode=list should return a dictionary of the flag versions') flagdata(vis=self.vis, mode='unflag', flagbackup=False) flagsdict2 = flagmanager(vis=self.vis, mode='list') self.assertDictEqual( flagsdict1, flagsdict2, 'Dictionaries should contain the same flag versions') flagdata(vis=self.vis, mode='unflag', flagbackup=True) flagsdict3 = flagmanager(vis=self.vis, mode='list') self.assertEqual(flagsdict3[0]['name'], 'flagdata_1') # Rename the flagdata_1 version newname = 'A new name' flagmanager(vis=self.vis, mode='rename', oldname='flagdata_1', versionname=newname, comment='A new versions name') flagsdict4 = flagmanager(vis=self.vis, mode='list') self.assertTrue( os.path.exists(self.vis + '.flagversions/flags.' + newname), 'Flagversion file does not exist: flags.' + newname) self.assertEqual(flagsdict4[0]['name'], newname, 'Dictionary should show new name') # Specific for MMS if testmms: areg = self.vis + '/SUBMSS/*flagversions*' import glob print 'Check for .flagversions in the wrong place.' self.assertEqual( glob.glob(areg), [], 'There should not be any .flagversions in the' ' SUBMSS directory')
def test_save(self): '''flagmanager: CAS-3080, do not overwrite an existing versionname''' # Create a flagbackup flagdata(vis=self.vis, mode='manual', antenna="2", flagbackup=True) fname = 'flagdata_1' self.assertTrue(os.path.exists(self.vis+'.flagversions/flags.'+fname), 'Flagversions file does not exist: flags.'+fname) # Rename newname = 'Do_Not_Overwrite_Me' print ('Rename versionname to Do_Not_Overwrite_Me') flagmanager(vis=self.vis, mode='save', versionname=newname) self.assertTrue(os.path.exists(self.vis+'.flagversions/flags.'+newname), 'Flagversions file does not exist: flags.'+newname) print 'Move existing versionname to temporary name' flagmanager(vis=self.vis, mode='save', versionname=newname) flagmanager(vis=self.vis, mode='list') lf = os.listdir(self.vis+'.flagversions') self.assertTrue([s for s in lf if '.old.' in s]) self.assertEqual(len(lf), 4)
def test_save(self): '''flagmanager: CAS-3080, do not overwrite an existing versionname''' # Create a flagbackup flagdata(vis=self.vis, mode='manual', antenna="2", flagbackup=True) fname = 'flagdata_1' self.assertTrue( os.path.exists(self.vis + '.flagversions/flags.' + fname), 'Flagversions file does not exist: flags.' + fname) # Rename newname = 'Do_Not_Overwrite_Me' print('Rename versionname to Do_Not_Overwrite_Me') flagmanager(vis=self.vis, mode='save', versionname=newname) self.assertTrue( os.path.exists(self.vis + '.flagversions/flags.' + newname), 'Flagversions file does not exist: flags.' + newname) print 'Move existing versionname to temporary name' flagmanager(vis=self.vis, mode='save', versionname=newname) flagmanager(vis=self.vis, mode='list') lf = os.listdir(self.vis + '.flagversions') self.assertTrue([s for s in lf if '.old.' in s]) self.assertEqual(len(lf), 4)
# Flag SPW 0, 1 and 4. The pipeline always fails for 0 and usually for 4. # SPW 1 is always returned by the pipeline, but by-eye investigations and # the VLA RFI plots show there are almost no RFI free channels. default("flagdata") flagdata(vis=ms_active, spw="0,1,4", flagbackup=False) # Now remove specific portions of the other SPWs # Channel ranges are for 16B, and are the regions that were found to # persist over multiple SBs. Other strong RFI exists (SPW 6, for instance) # but is intermittent. # This leaves 438 good channels, or 68% of the recoverable 5 SPWs. # This is 43% of the total BW. We assumed 60% recovered in the proposal. # Using the sensitivity calculator, the difference in sensitivity is # 1.88 uJy/bm vs. 1.63 uJy/bm over 48 hours. So not much of a difference. spw2 = "2:0~20;42~54;83~96" spw3 = "3:0~10;30;31;69;70" spw5 = "5:52~67;112;113" # spw6 = "" # There are two narrow, strong, but intermittent RFI sources spw7 = "7:44~127" flag_str = ",".join([spw2, spw3, spw5, spw7]) flagdata(vis=ms_active, spw=flag_str, flagbackup=False) flagmanager(vis=ms_active, mode='save', versionname="known_RFI", comment="Removal of constant L-band RFI in 16B. See " "EVLA_Lband_RFI_flag.py") else: logprint("known_RFI flag version already exists. Skipping flagging.") logprint("Finished EVLA_Lband_RFI_flag.py", logfileout='logs/known_RFI_flag.log')
import clog_classified import tasks casalog.filter(level="DEBUG") anmsname = 'n14c2.ms' fj_table = "sbd_gah.fj" # Set corrected data to original ## Single bands or "manual phase cals tasks.flagmanager(anmsname, versionname='applycal_10', mode='restore') casalog.post("Reset CORRECTED_DATA to original values.") # Set the boolean flags to do single-band delays, multi-band delays or both. if False: tasks.applycal(anmsname, gaintable=['n14c2.gcal', 'n14c2.tsys'], parang=True) ff = clog_classified.FringeFitter(anmsname, fj_table, ref_antenna_name='EF', scans=[5], # antennas=None, # antennas = [0,4,6,7,8,10], snr_threshold=30.0, solint=60.0, solsub=2) ff.run() new_antenna_list = [s for s in ff.antennas2 if s not in ff.bad_antennas] tasks.applycal(anmsname, gaintable=['n14c2.gcal', 'n14c2.tsys', fj_table], parang=True) if True: new_antenna_list = [0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15] ## Compute multiband calibration mb_table = 'mbd_meh.fj'
def test_flagbackup(self): '''Flagcmd: backup cal table flags''' flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 2) aflocal.done() flagcmd(vis=self.vis, inpmode='list', inpfile=["spw='3'"]) flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 3) aflocal.done() flagcmd(vis=self.vis, inpmode='list', inpfile=["spw='4'"], flagbackup=False) flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 3) aflocal.done() newname = 'BackupBeforeSpwFlags' flagmanager(vis=self.vis, mode='rename', oldname='flagcmd_1', versionname=newname, comment='Backup of flags before applying flags on spw') flagmanager(vis=self.vis, mode='list') aflocal.open(self.vis) self.assertEqual(len(aflocal.getflagversionlist()), 3) aflocal.done() # Apply spw=5 flags flagcmd(vis=self.vis, inpmode='list', inpfile=["spw='5'"], flagbackup=True) res = flagdata(vis=self.vis, mode='summary') self.assertEqual(res['spw']['3']['flagged'], 83200) self.assertEqual(res['spw']['4']['flagged'], 83200) self.assertEqual(res['spw']['5']['flagged'], 83200) self.assertEqual(res['flagged'], 83200*3) # Restore backup flagmanager(vis=self.vis, mode='restore', versionname='BackupBeforeSpwFlags', merge='replace') res = flagdata(vis=self.vis, mode='summary') self.assertEqual(res['spw']['5']['flagged'], 0) self.assertEqual(res['flagged'], 0)
widths.append(wid) # these are TOPO freqs: freqs[spw] = msmd.chanfreqs(spw) try: freqs[spw] = ms.cvelfreqs(spwid=[spw], outframe='LSRK') except TypeError: freqs[spw] = ms.cvelfreqs(spwids=[spw], outframe='LSRK') linechannels = contchannels_to_linechannels( cont_channel_selection, freqs) msmd.close() ms.close() flagmanager(vis=visfile, mode='save', versionname='before_cont_flags') # not clear why this is done in other imaging scripts, but it # seems to achieve the wrong effect. #initweights(vis=visfile, wtmode='weight', dowtsp=True) flagdata(vis=visfile, mode='manual', spw=linechannels, flagbackup=False) rmtables(contvis) os.system('rm -rf ' + contvis + '.flagversions') tb.open(invis)
ylabel = '' showmajorgrid = False showminorgrid = False overwrite = False showgui = True async = False plotms() raw_input("Continue?") else: print("No bad phase vs. time scans.") # Get the existing flag version names. flag_folder = "{}.flagversions".format(ms_active) tstamp = datetime.now().strftime("%Y%m%d-%H%M%S") if not os.path.exists(flag_folder): print("No flag versions exist. Using default flag name.") versionname = "badscan_flagging_1_{}".format(tstamp) else: flag_versions = \ glob(os.path.join(flag_folder, "flags.badscan_flagging_*")) if len(flag_versions) == 0: versionname = "badscan_flagging_1_{}".format(tstamp) else: num = len(flag_versions) + 1 versionname = "badscan_flagging_{0}_{1}".format(num, tstamp) # Save this new version of the flags flagmanager(vis=ms_active, mode='save', versionname=versionname)