def run( fetch=False ): """Run the tasks and compare the results.""" #####fetch data if fetch: for f in data( ): copydata( f, os.getcwd( ) ) nsigma = 3.0 advice = None lazy = True equinoxes = ['B1950_VLA', 'B1950', 'J2000'] # 'BMEAN', 'BTRUE', 'APP', 'JMEAN', 'JTRUE'] convert_phasecenter = False origvisses = data() passes = True for origvis in origvisses: peaks = run_tasks(origvis, advice, lazy, equinoxes, convert_phasecenter) passes = passes and compare(peaks, nsigma) if passes: print '' print 'Regression PASSED' print '' return []
def run(fetch=False): #####fetch data if fetch: for f in data(): copydata(f, os.getcwd()) im.open('ic2233_1.ms') npix = 1024 im.selectvis(spw='0', nchan=[6], start=[0], step=[1]) im.defineimage(nx=npix, ny=npix, cellx='3.0arcsec', celly='3.0arcsec', stokes="IV", spw=[0]) im.weight(type='briggs', robust=0.7) im.setoptions(imagetilevol=-1000000) im.setvp(dovp=bool(1), usedefaultvp=bool(1), dosquint=bool(1), parangleinc='5.0deg') im.make('squint_corr') im.clean(algorithm='mfhogbom', niter=1000, model=['squint_corr'], residual=['squint_corr.residual'], image=['squint_corr.restored'], threshold='0Jy') im.done() return ['squint_corr.restored']
def run(fetch=False): """Run the tasks and compare the results.""" #####fetch data if fetch: for f in data(): copydata(f, os.getcwd()) nsigma = 3.0 advice = None lazy = True equinoxes = ['B1950_VLA', 'B1950', 'J2000'] # 'BMEAN', 'BTRUE', 'APP', 'JMEAN', 'JTRUE'] convert_phasecenter = False origvisses = data() passes = True for origvis in origvisses: peaks = run_tasks(origvis, advice, lazy, equinoxes, convert_phasecenter) passes = passes and compare(peaks, nsigma) if passes: print '' print 'Regression PASSED' print '' return []
def run(fetch=False): #####fetch data if fetch: for f in data( ): copydata( f, os.getcwd( ) ) im=gl['casac'].imager() me=gl['me'] time1=time.time() im.open('coma.ms') im.selectvis(spw=0, field=0); mydir=me.direction('J2000', '12h30m48', '12d24m0') im.defineimage(nx=200, ny=200, cellx='30arcsec', celly='30arcsec', phasecenter=mydir); im.make('outlier1re'); im.defineimage(nx=2000, ny=2000, cellx='30arcsec',celly='30arcsec',phasecenter=0, facets=1); im.setoptions(ftmachine='wproject',wprojplanes=400, padding=1.2) im.make('bla1re') im.clean(algorithm='wfclark',model=['bla1re', 'outlier1re'], image=['bla1re.restored', 'outlier1re.restored'], niter=10000); im.done() time2=time.time(); print 'Time taken for wproject= ', (time2-time1)/60,'mins' #resturn the images that will be templated and compared in future runs return ['bla1re.restored', 'outlier1re.restored']
def run(fetch=False): #####fetch data if fetch: for f in data(): copydata(f, os.getcwd()) #####locate the regression script lepath = locatescript('accum_regression.py') gl['regstate'] = True execfile(lepath, gl) print 'regstate =', gl['regstate'] if not gl['regstate']: raise Exception, 'regstate = False' return []
def run(fetch=False): #####fetch data if fetch: for f in data(): copydata(f, os.getcwd()) #####locate the regression script try: lepath = locatescript('alma-m100-analysis-regression.py') print 'Script used is ', lepath execfile(lepath, gl, pass_on) except: print 'execution failed: ', sys.exc_info() raise ###return the images that will be templated and compared in future runs return []
def run(fetch=False): #####fetch data if fetch: for f in data(): copydata(f, os.getcwd()) #####locate the regression script lepath = locatescript('wideband_regression.py') print 'Script used is ', lepath gl['regstate'] = True execfile(lepath, gl) print 'regstate =', gl['regstate'] if not gl['regstate']: raise Exception, 'regstate = False' ### return the images that will be templated and compared in future runs return []
def run(fetch=False): #####fetch data if fetch: for f in data( ): copydata( f, os.getcwd( ) ) #####locate the regression script lepath=locatescript('wideband_regression.py') print 'Script used is ',lepath gl['regstate']=True execfile(lepath, gl) print 'regstate =', gl['regstate'] if not gl['regstate']: raise Exception, 'regstate = False' ### return the images that will be templated and compared in future runs return []; #'reg_3C286.image.tt0','reg_3C286.image.alpha','reg_3C286.image.beta']
def run( fetch=False ): #####fetch data if fetch: for f in data( ): copydata( f, os.getcwd( ) ) #####locate the regression script try: lepath=locatescript('alma-m100-analysis-hpc-regression.py') print 'Script used is ',lepath execfile(lepath, gl, pass_on) except: print 'execution failed: ', sys.exc_info() raise ###return the images that will be templated and compared in future runs return []
def run(fetch=False): #####fetch data if fetch: for f in data( ): copydata( f, os.getcwd( ) ) #####locate the regression script lepath=locatescript('ngc4826_tutorial_regression.py') print 'Script used is ',lepath gl['regstate']=True execfile(lepath, gl) print 'regstate =', gl['regstate'] if not gl['regstate']: raise Exception, 'regstate = False' ###return the images that will be templated and compared in future runs return ['ngc4826.tutorial.16apr98.src.clean.image', 'ngc4826.tutorial.16apr98.moments.mom1']
def run( fetch=False ): #####fetch data if fetch: for f in data( ): copydata( f, os.getcwd( ) ) #####locate the regression script lepath=locatescript('3c129_tutorial_regression.py') print 'Script used is ',lepath gl['regstate']=True execfile(lepath, gl) print 'regstate =', gl['regstate'] if not gl['regstate']: raise Exception, 'regstate = False' ###return the images that will be templated and compared in future runs # return ['3C129BC.clean.image', 'at166B.3c129.image', 'at166C.3c129.image'] return ['3C129BC.core.I']
def run(fetch=False) : #####fetch data if fetch: for f in data( ): copydata( f, os.getcwd( ) ) im.open('ic2233_1.ms') npix=1024 im.selectvis(spw='0', nchan=[6], start=[0], step=[1]) im.defineimage(nx=npix, ny=npix, cellx='3.0arcsec', celly='3.0arcsec', stokes="IV", spw=[0]) im.weight(type='briggs', robust=0.7) im.setoptions(imagetilevol=-1000000); im.setvp(dovp=bool(1), usedefaultvp=bool(1), dosquint=bool(1), parangleinc='5.0deg') im.make('squint_corr') im.clean(algorithm='mfhogbom', niter=1000, model=['squint_corr'], residual=['squint_corr.residual'], image=['squint_corr.restored'], threshold='0Jy') im.done() return ['squint_corr.restored']
def run(fetch=False): #####fetch data if fetch: for f in data( ): copydata( f, os.getcwd( ) ) #####locate the regression script lepath=locatescript('ngc2403_tutorial_regression.py') print 'Script used is ',lepath gl['regstate']=True execfile(lepath, gl) print 'regstate =', gl['regstate'] if not gl['regstate']: raise Exception, 'regstate = False' ###return the images that will be templated and compared in future runs #not in repository: return ['n2403.tutorial.final.clean.image','n2403.tutorial.dirty.image'] return ['n2403.tutorial.final.clean.image']
def run(fetch=False): #####fetch data if fetch: for f in data(): copydata(f, os.getcwd()) #####locate the regression script lepath = locatescript('ngc2403_tutorial_regression.py') print 'Script used is ', lepath gl['regstate'] = True execfile(lepath, gl) print 'regstate =', gl['regstate'] if not gl['regstate']: raise Exception, 'regstate = False' ###return the images that will be templated and compared in future runs #not in repository: return ['n2403.tutorial.final.clean.image','n2403.tutorial.dirty.image'] return ['n2403.tutorial.final.clean.image']
def run(fetch=False): #####fetch data if fetch: for f in data(): copydata(f, os.getcwd()) #####locate the regression script lepath = locatescript('ngc4826_tutorial_regression.py') print 'Script used is ', lepath gl['regstate'] = True execfile(lepath, gl) print 'regstate =', gl['regstate'] if not gl['regstate']: raise Exception, 'regstate = False' ###return the images that will be templated and compared in future runs return [ 'ngc4826.tutorial.16apr98.src.clean.image', 'ngc4826.tutorial.16apr98.moments.mom1' ]
def run(fetch=False): #####fetch data if fetch: for f in data(): copydata(f, os.getcwd()) im = gl['casac'].imager() me = gl['me'] time1 = time.time() im.open('coma.ms') im.selectvis(spw=0, field=0) mydir = me.direction('J2000', '12h30m48', '12d24m0') im.defineimage(nx=200, ny=200, cellx='30arcsec', celly='30arcsec', phasecenter=mydir) im.make('outlier1re') im.defineimage(nx=2000, ny=2000, cellx='30arcsec', celly='30arcsec', phasecenter=0, facets=1) im.setoptions(ftmachine='wproject', wprojplanes=400, padding=1.2) im.make('bla1re') im.clean(algorithm='wfclark', model=['bla1re', 'outlier1re'], image=['bla1re.restored', 'outlier1re.restored'], niter=10000) im.done() time2 = time.time() print 'Time taken for wproject= ', (time2 - time1) / 60, 'mins' #resturn the images that will be templated and compared in future runs return ['bla1re.restored', 'outlier1re.restored']
def run(fetch=False): #####fetch data if fetch: for f in data(): copydata(f, os.getcwd()) input_mses = data() badcells = {} for input_ms in input_mses: tbin = str(expected[input_ms]['tav']) + 's' cbin = expected[input_ms]['cav'] spwsels = expected[input_ms].keys() spwsels.remove('tav') spwsels.remove('cav') spwsels.remove('dep') spwsels.sort() # Prob. not needed. for spwsel in spwsels: tavms = re.sub(r'.ms$', '', input_ms) + '_%s_timebin%s.ms' % (spwsel, tbin) #print "run: tavms =", tavms outputms = re.sub(r'.ms$', '', tavms) + '_width%d.ms' % cbin #print "run: outputms =", outputms for oms in (tavms, outputms): # Get rid of past problems. if os.path.isdir(oms): shutil.rmtree(oms) # The actual run. try: (nrows_aft_tavg, nrows_aft_cavg, shp_aft_cavg) = time_then_chan_avg(input_ms, tbin, cbin, outms=outputms, chanselstr=spwsel, zaptemp=False) except Exception, e: raise Exception, "Error (%s) running time_then_chan_avg()." % e ###### Test # of rows. if nrows_aft_tavg != expected[input_ms][spwsel]['nrows_aft_tavg']: raise Exception, """ The number of rows after time averaging (%d) does not match the expected number (%d) for input MS %s with selection string "%s" """ % (nrows_aft_tavg, expected[input_ms][spwsel]['nrows_aft_tavg'], input_ms, spwsel) if nrows_aft_cavg != expected[input_ms][spwsel]['nrows_aft_cavg']: raise Exception, """ The number of rows after channel averaging (%d) does not match the expected number (%d) for input MS %s with selection string "%s" """ % (nrows_aft_cavg, expected[input_ms][spwsel]['nrows_aft_cavg'], input_ms, spwsel) if shp_aft_cavg != expected[input_ms][spwsel]['datshp']: raise Exception, """ The data shape (%s) after channel averaging does not match the expectation (%s) for input MS %s with selection string "%s" """ % (shp_aft_cavg, expected[input_ms][spwsel]['datshp'], input_ms, spwsel) ###### Compare averaged data with expectation, and rm the averaged ###### data if it passes. outputmses = {'tav': tavms, 'cav': outputms} for avtype, avms in outputmses.iteritems(): expcells = expected[input_ms][spwsel]['cells'][avtype] try: tb.open(avms) except Exception, e: raise Exception, "Error (%s) opening %s." % (e, avms) try: for ((col, row), expent) in expcells.iteritems(): usetol = True if isinstance(expent, tuple): tol = expent[1] expent = expent[0] elif not isquantized(expent): tol = 0.005 * expent else: usetol = False gotval = tb.getcell(col, row) if hasattr(gotval, 'flatten') and gotval.size == 1: gotval = gotval.flatten()[0] if usetol: closeenough = (abs(gotval - expent) < tol) else: closeenough = (gotval == expent) if hasattr(closeenough, 'all'): closeenough = closeenough.all() if not closeenough: if not badcells.has_key(avms): badcells[avms] = {} badcells[avms][(col, row)] = (gotval, expent) except Exception, e: raise Exception, "Error (%s) checking %s's cell %s." % ( e, avms, (col, row)) finally:
def run(fetch=False): #####fetch data if fetch: for f in data( ): copydata( f, os.getcwd( ) ) input_mses = data() badcells = {} for input_ms in input_mses: tbin = str(expected[input_ms]['tav']) + 's' cbin = expected[input_ms]['cav'] spwsels = expected[input_ms].keys() spwsels.remove('tav') spwsels.remove('cav') spwsels.remove('dep') spwsels.sort() # Prob. not needed. for spwsel in spwsels: tavms = re.sub(r'.ms$', '', input_ms) + '_%s_timebin%s.ms' % (spwsel, tbin) #print "run: tavms =", tavms outputms = re.sub(r'.ms$', '', tavms) + '_width%d.ms' % cbin #print "run: outputms =", outputms for oms in (tavms, outputms): # Get rid of past problems. if os.path.isdir(oms): shutil.rmtree(oms) # The actual run. try: (nrows_aft_tavg, nrows_aft_cavg, shp_aft_cavg) = time_then_chan_avg(input_ms, tbin, cbin, outms=outputms, chanselstr=spwsel, zaptemp=False) except Exception, e: raise Exception, "Error (%s) running time_then_chan_avg()." % e ###### Test # of rows. if nrows_aft_tavg != expected[input_ms][spwsel]['nrows_aft_tavg']: raise Exception, """ The number of rows after time averaging (%d) does not match the expected number (%d) for input MS %s with selection string "%s" """ % (nrows_aft_tavg, expected[input_ms][spwsel]['nrows_aft_tavg'], input_ms, spwsel) if nrows_aft_cavg != expected[input_ms][spwsel]['nrows_aft_cavg']: raise Exception, """ The number of rows after channel averaging (%d) does not match the expected number (%d) for input MS %s with selection string "%s" """ % (nrows_aft_cavg, expected[input_ms][spwsel]['nrows_aft_cavg'], input_ms, spwsel) if shp_aft_cavg != expected[input_ms][spwsel]['datshp']: raise Exception, """ The data shape (%s) after channel averaging does not match the expectation (%s) for input MS %s with selection string "%s" """ % (shp_aft_cavg, expected[input_ms][spwsel]['datshp'], input_ms, spwsel) ###### Compare averaged data with expectation, and rm the averaged ###### data if it passes. outputmses = {'tav': tavms, 'cav': outputms} for avtype, avms in outputmses.iteritems(): expcells = expected[input_ms][spwsel]['cells'][avtype] try: tb.open(avms) except Exception, e: raise Exception, "Error (%s) opening %s." % (e, avms) try: for ((col, row), expent) in expcells.iteritems(): usetol = True if isinstance(expent, tuple): tol = expent[1] expent = expent[0] elif not isquantized(expent): tol = 0.005 * expent else: usetol = False gotval = tb.getcell(col, row) if hasattr(gotval, 'flatten') and gotval.size == 1: gotval = gotval.flatten()[0] if usetol: closeenough = (abs(gotval - expent) < tol) else: closeenough = (gotval == expent) if hasattr(closeenough, 'all'): closeenough = closeenough.all() if not closeenough: if not badcells.has_key(avms): badcells[avms] = {} badcells[avms][(col, row)] = (gotval, expent) except Exception, e: raise Exception, "Error (%s) checking %s's cell %s." % (e, avms, (col, row)) finally:
def run(fetch=False): #####fetch data if fetch: for f in data(): copydata(f, os.getcwd()) expected = { 'ggtau.3mm.ph.gcal0': { 'SPLINE_KNOTS_PHASE': { 'rms': 4362063360.0, 'medabsdevmed': 8704.0, 'min': 4362050048.0, 'max': 4362076160.0, 'sum': 872412620800.0, 'quartile': 26112.0, 'median': 4362058752.0, 'sumsq': 3.80551890468e+21, 'stddev': 11866.4301499, 'var': 140812164.503, 'npts': 200, 'mean': 4362063104.0 } }, 'ggtau.1mm.ph.gcal0': { 'SPLINE_KNOTS_PHASE': { 'rms': 4362063360.0, 'medabsdevmed': 8704.0, 'min': 4362050048.0, 'max': 4362076160.0, 'sum': 872412620800.0, 'quartile': 26112.0, 'median': 4362058752.0, 'sumsq': 3.80551890468e+21, 'stddev': 11866.4301499, 'var': 140812164.503, 'npts': 200, 'mean': 4362063104.0 } } } for caltable in caltables: print "Testing with data", caltable, "..." if expected.has_key(caltable): default(calstat) axis = 'spline_knots_phase' s = calstat(caltable=caltable, axis=axis) if s.keys() != expected[caltable].keys(): raise Exception("Wrong dictionary keys. Expected %s, got %s" % \ (expected[caltable], s)) print "Expected =", expected[caltable] print "Got = ", s if not s.has_key('SPLINE_KNOTS_PHASE'): raise Exception( "Dictionary returned from calstat does not have key SPLINE_KNOTS_PHASE" ) for e in expected[caltable]['SPLINE_KNOTS_PHASE'].keys(): print "Checking %s: %s vs %s" % \ (e, expected[caltable]['SPLINE_KNOTS_PHASE'][e], s['SPLINE_KNOTS_PHASE'][e]) failed = False if expected[caltable]['SPLINE_KNOTS_PHASE'][e] == 0: if s['SPLINE_KNOTS_PHASE'][e] != 0: failed = True else: if abs((expected[caltable]['SPLINE_KNOTS_PHASE'][e] - s['SPLINE_KNOTS_PHASE'][e]) / expected[caltable] ['SPLINE_KNOTS_PHASE'][e]) > epsilon: failed = True # Remove these 3 lines of code, once CAS-1671 is solved if failed == True and e in ['var', 'stddev']: print "Ignoring this known problem on 64bit!" failed = False if failed: raise Exception("Numbers differ, expected %s, got %s" % \ (str(expected[caltable]['SPLINE_KNOTS_PHASE'][e]), str(s['SPLINE_KNOTS_PHASE'][e]))) tb.open(caltable) cols = tb.colnames() tb.close() cplx = ['amp', 'amplitude', 'phase', 'imag', 'imaginary', 'real'] for x in cplx: cols.append(x) print cols # remove complex columns cols.remove('GAIN') if 'SCALE_FACTOR' in cols: cols.remove('SCALE_FACTOR') if 'SIDEBAND_REF' in cols: cols.remove('SIDEBAND_REF') # don't try string columns cols.remove('FREQ_GROUP_NAME') cols.remove('FIELD_NAME') cols.remove('FIELD_CODE') cols.remove('SOURCE_NAME') cols.remove('SOURCE_CODE') if 'POLY_TYPE' in cols: cols.remove('POLY_TYPE') if 'POLY_MODE' in cols: cols.remove('POLY_MODE') if 'PHASE_UNITS' in cols: cols.remove('PHASE_UNITS') # empty column: if 'VALID_DOMAIN' in cols: cols.remove('VALID_DOMAIN') cols = [x.lower() for x in cols] print "Trying these column names", cols for col in cols: data_cols = [''] if col in cplx: data_cols = ['gain', 'scale_factor'] for dc in data_cols: print "Call with caltable =", caltable, "; axis =", col, "; datacolumn =", dc if dc != '': s = calstat(caltable=caltable, axis=col, datacolumn=dc) else: s = calstat(caltable=caltable, axis=col) if col.upper() == "FLAG_CATEGORY": # The MSs used have no data in FLAG_CATEGORY, therefore # calstat() should fail if s != None: raise Exception("Error! " + str(s)) elif not type(s) is dict: raise Exception("Error! Return value " + str(s) + " is not a dictionary") print '' print 'Regression PASSED' print '' return []
def run( fetch=False ): #####fetch data if fetch: for f in data( ): copydata( f, os.getcwd( ) ) expected = {'ggtau.3mm.ph.gcal0': {'SPLINE_KNOTS_PHASE':{'rms': 4362063360.0, 'medabsdevmed': 13056.0, 'min': 4362050048.0, 'max': 4362076160.0, 'sum': 872412620800.0, 'quartile': 26112.0, 'median': 4362063104.0, 'sumsq': 3.80551890468e+21, 'stddev': 11866.4301499, 'var': 140812164.503, 'npts': 200, 'mean': 4362063104.0}}, 'ggtau.1mm.ph.gcal0': {'SPLINE_KNOTS_PHASE':{'rms': 4362063360.0, 'medabsdevmed': 13056.0, 'min': 4362050048.0, 'max': 4362076160.0, 'sum': 872412620800.0, 'quartile': 26112.0, 'median': 4362063104.0, 'sumsq': 3.80551890468e+21, 'stddev': 11866.4301499, 'var': 140812164.503, 'npts': 200, 'mean': 4362063104.0}}} for caltable in caltables: print "Testing with data", caltable, "..." if expected.has_key(caltable): default(calstat) axis='spline_knots_phase' s = calstat(caltable=caltable, axis=axis) if s.keys() != expected[caltable].keys(): raise Exception("Wrong dictionary keys. Expected %s, got %s" % \ (expected[caltable], s)) print "Expected =", expected[caltable] print "Got = ", s if not s.has_key('SPLINE_KNOTS_PHASE'): raise Exception("Dictionary returned from calstat does not have key SPLINE_KNOTS_PHASE") for e in expected[caltable]['SPLINE_KNOTS_PHASE'].keys(): print "Checking %s: %s vs %s" % \ (e, expected[caltable]['SPLINE_KNOTS_PHASE'][e], s['SPLINE_KNOTS_PHASE'][e]) failed = False if expected[caltable]['SPLINE_KNOTS_PHASE'][e] == 0: if s['SPLINE_KNOTS_PHASE'][e] != 0: failed = True else: if abs((expected[caltable]['SPLINE_KNOTS_PHASE'][e] - s['SPLINE_KNOTS_PHASE'][e])/expected[caltable]['SPLINE_KNOTS_PHASE'][e]) > epsilon: failed = True # Remove these 3 lines of code, once CAS-1671 is solved if failed == True and e in ['var', 'stddev']: print "Ignoring this known problem on 64bit!" failed = False if failed: raise Exception("Numbers differ, expected %s, got %s" % \ (str(expected[caltable]['SPLINE_KNOTS_PHASE'][e]), str(s['SPLINE_KNOTS_PHASE'][e]))) tb.open(caltable) cols = tb.colnames() tb.close() cplx = ['amp', 'amplitude', 'phase', 'imag', 'imaginary', 'real'] for x in cplx: cols.append(x) print cols # remove complex columns cols.remove('GAIN') if 'SCALE_FACTOR' in cols: cols.remove('SCALE_FACTOR') if 'SIDEBAND_REF' in cols: cols.remove('SIDEBAND_REF') # don't try string columns cols.remove('FREQ_GROUP_NAME') cols.remove('FIELD_NAME') cols.remove('FIELD_CODE') cols.remove('SOURCE_NAME') cols.remove('SOURCE_CODE') if 'POLY_TYPE' in cols: cols.remove('POLY_TYPE') if 'POLY_MODE' in cols: cols.remove('POLY_MODE') if 'PHASE_UNITS' in cols: cols.remove('PHASE_UNITS') # empty column: if 'VALID_DOMAIN' in cols: cols.remove('VALID_DOMAIN') cols = [x.lower() for x in cols] print "Trying these column names", cols for col in cols: data_cols = [''] if col in cplx: data_cols = ['gain', 'scale_factor'] for dc in data_cols: print "Call with caltable =", caltable, "; axis =", col, "; datacolumn =", dc if dc != '': s = calstat(caltable=caltable, axis=col, datacolumn=dc) else: s = calstat(caltable=caltable, axis=col) if col.upper() == "FLAG_CATEGORY": # The MSs used have no data in FLAG_CATEGORY, therefore # calstat() should fail if s != None: raise Exception("Error! " + str(s)) elif not type(s) is dict: raise Exception("Error! Return value " + str(s) + " is not a dictionary") print '' print 'Regression PASSED' print '' return []