def __init__(self, pathsMS, scheduler, check_flags=True, check_sun=False, min_sun_dist=10): """ pathsMS: list of MS paths scheduler: scheduler object check_flag: if true ignore fully flagged ms check_sun: if true check sun distance min_sun_dist: if check_sun and distance from the sun < than this deg, skip """ self.scheduler = scheduler # sort them, useful for some concatenating steps if len(pathsMS) == 0: logger.error('Cannot find MS files.') raise('Cannot find MS files.') self.mssListObj = [] for pathMS in sorted(pathsMS): ms = MS(pathMS) if check_flags and ms.isAllFlagged(): logger.warning('Skip fully flagged ms: %s' % pathMS) elif check_sun and ms.sun_dist.deg < min_sun_dist: logger.warning('Skip too close to sun (%.0f deg) ms: %s' % (ms.sun_dist.deg, pathMS)) else: self.mssListObj.append(MS(pathMS)) if len(self.mssListObj) == 0: raise('ALL MS files flagged.') self.mssListStr = [ms.pathMS for ms in self.mssListObj] self.resolution = self.mssListObj[0].getResolution(check_flags=False)
def check_run(self, log = "", commandType = ""): """ Produce a warning if a command didn't close the log properly i.e. it crashed NOTE: grep, -L inverse match, -l return only filename """ if (not os.path.exists(log)): logger.warning("No log file found to check results: " + log) return 1 if (commandType == "DPPP"): out = subprocess.check_output('grep -L "Finishing processing" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -l "Segmentation fault\|Killed" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) # # out += subprocess.check_output('grep -l "Aborted (core dumped)" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -i -l "Exception" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -l "**** uncaught exception ****" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -l "error" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -l "misspelled" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) elif (commandType == "CASA"): out = subprocess.check_output('grep -l "[a-z]Error" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -l "An error occurred running" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -l "\*\*\* Error \*\*\*" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) elif (commandType == "wsclean"): out = subprocess.check_output('grep -l "exception occur" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -l "Segmentation fault\|Killed" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -l "Aborted" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -L "Cleaning up temporary files..." '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) elif (commandType == "python"): out = subprocess.check_output('grep -l "Traceback (most recent call last):" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -l "Segmentation fault\|Killed" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -i -l \'(?=^((?!error000).)*$).*Error.*\' '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -i -l "Critical" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -l "ERROR" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -l "raise Exception" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) elif (commandType == "singularity"): out = subprocess.check_output('grep -l "Traceback (most recent call last):" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -i -l \'(?=^((?!error000).)*$).*Error.*\' '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) out += subprocess.check_output('grep -i -l "Critical" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) elif (commandType == "general"): out = subprocess.check_output('grep -l -i "error" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) else: logger.warning("Unknown command type for log checking: '" + commandType + "'") return 1 if out != b'': out = out.split(b'\n')[0].decode() logger.error(commandType+' run problem on:\n'+out) raise RuntimeError(commandType+' run problem on:\n'+out) return 0
def todo(self, stepname): """ Return false if stepname has been already done """ with open(self.filename, "r") as f: for stepname_done in f: if stepname == stepname_done.rstrip(): logger.warning('SKIP: %s' % stepname) return False return True
def columnAddSimilar(pathMS, columnNameNew, columnNameSimilar, dataManagerInfoNameNew, overwrite = False, fillWithOnes = True, comment = "", verbose = False): # more to lib_ms """ Add a column to a MS that is similar to a pre-existing column (in shape, but not in values). pathMS: path of the MS columnNameNew: name of the column to be added columnNameSimilar: name of the column from which properties are copied (e.g. "DATA") dataManagerInfoNameNew: string value for the data manager info (DMI) keyword "NAME" (should be unique in the MS) overwrite: whether or not to overwrite column 'columnNameNew' if it already exists fillWithOnes: whether or not to fill the newly-made column with ones verbose: whether or not to produce abundant output """ t = tables.table(pathMS, readonly = False) if (columnExists(t, columnNameNew) and not overwrite): logger.warning("Attempt to add column '" + columnNameNew + "' aborted, as it already exists and 'overwrite = False' in columnAddSimilar(...).") else: # Either the column does not exist yet, or it does but overwriting is allowed. # Remove column if necessary. if (columnExists(t, columnNameNew)): logger.info("Removing column '" + columnNameNew + "'...") t.removecols(columnNameNew) # Add column. columnDescription = t.getcoldesc(columnNameSimilar) dataManagerInfo = t.getdminfo(columnNameSimilar) if (verbose): logger.debug("columnDescription:") logger.debug(columnDescription) logger.debug("dataManagerInfo:") logger.debug(dataManagerInfo) columnDescription["comment"] = "" # What about adding something here like: #columnDescription["dataManagerGroup"] = ...? dataManagerInfo["NAME"] = dataManagerInfoNameNew if (verbose): logger.debug("columnDescription (updated):") logger.debug(columnDescription) logger.debug("dataManagerInfo (updated):") logger.debug(dataManagerInfo) logger.info("Adding column '" + columnNameNew + "'...") t.addcols(tables.makecoldesc(columnNameNew, columnDescription), dataManagerInfo) # Fill with ones if desired. if (fillWithOnes): logger.info("Filling column '" + columnNameNew + "' with ones...") columnDataSimilar = t.getcol(columnNameSimilar) t.putcol(columnNameNew, np.ones_like(columnDataSimilar)) # Close the table to avoid that it is locked for further use. t.close()
def __exit__(self, type, value, traceback): """ Catch "Skip" errors, if not skipped, write to file after exited without exceptions. """ if type is None: with open(self.filename, "a") as f: f.write(self.__step__ + '\n') logger.debug('Done: {}'.format(self.__step__)) return # No exception if issubclass(type, Skip): logger.warning('SKIP: {}'.format(self.__step__)) return True # Suppress special SkipWithBlock exception
def get_cluster(self): """ Find in which computing cluster the pipeline is running """ import socket hostname = socket.gethostname() if (hostname == 'lgc1' or hostname == 'lgc2'): return "Hamburg" elif ('node3' in hostname): return "Hamburg_fat" elif ('leidenuniv' in hostname): return "Leiden" elif (hostname[0:3] == 'lof'): return "CEP3" else: logger.warning('Hostname %s unknown.' % hostname) return "Unknown"
def rescaleModel(self, funct_flux): """ Rescale the model images to a certain total flux estimated using funct_flux(nu) nu is taken from the fits file. funct_flux: is a function of frequency (Hz) which returns the total expected flux (Jy) at that frequency. """ for model_img in sorted(glob.glob(self.root+'*model*.fits')): fits = pyfits.open(model_img) # get frequency assert fits[0].header['CTYPE3'] == 'FREQ' nu = fits[0].header['CRVAL3'] data = fits[0].data # find expected flux flux = funct_flux(nu) current_flux = np.sum(data) # rescale data scaling_factor = flux/current_flux logger.warning('Rescaling model %s by: %f' % (model_img, scaling_factor)) data *= scaling_factor fits[0].data = data fits.writeto(model_img, overwrite=True) fits.close()
def addpol(h5parmFile, soltabname, solsetname='sol000'): """ add pol axes on a soltab """ # open h5parm logger.info('%s: add pol axis to %s.' % (h5parmFile, soltabname)) h5 = h5parm(h5parmFile, readonly=False) ss = h5.getSolset(solsetname) st = ss.getSoltab(soltabname) if 'pol' in st.getAxesNames(): h5.close() logger.warning('%s: polarisation axis already present in %s.' % (h5parmFile, soltabname)) return # create values for new soltab typ = st.getType() axesNames = st.getAxesNames() + ['pol'] axesVals = [st.getAxisValues(axisName) for axisName in st.getAxesNames()] + [np.array(['XX', 'YY'])] vals = st.getValues(retAxesVals=False) vals = np.array([vals, vals]) vals = np.moveaxis(vals, 0, -1) weights = st.getValues(weight=True, retAxesVals=False) weights = np.array([weights, weights]) weights = np.moveaxis(weights, 0, -1) # remove old soltab st.delete() # make new soltab soltabout = ss.makeSoltab(soltype = typ, soltabName = soltabname, axesNames=axesNames, \ axesVals=axesVals, vals=vals, weights=weights) # write h5parm h5.close()
def __init__(self, pathsMS, scheduler, check_flags=True): """ pathsMS: list of MS paths scheduler: scheduler object check_flag: if true ignore fully flagged ms """ self.scheduler = scheduler # sort them, useful for some concatenating steps if len(pathsMS) == 0: raise ('Cannot find MS files.') self.mssListObj = [] for pathMS in sorted(pathsMS): ms = MS(pathMS) if check_flags and ms.isAllFlagged(): logger.warning('Skip fully flagged ms: %s' % pathMS) else: self.mssListObj.append(MS(pathMS)) if len(self.mssListObj) == 0: raise ('ALL MS files flagged.') self.mssListStr = [ms.pathMS for ms in self.mssListObj]
def killms_data(s, logfile, MSs, imagename, outsols, clusterfile=None, colname='DATA', niterkf=6, dicomodel=None, uvrange=None, wtuv=None, robust=None, dt=None, cache_dir='ddfcal/cache', sols_dir='ddfcal/solutions', SolverType="KAFCA", PolMode="Scalar", MergeSmooth=False, NChanSols=1, DISettings=None, EvolutionSolFile=None, CovQ=0.1, InterpToMSListFreqs=None, SkipSmooth=False, PreApplySols=None, SigmaFilterOutliers=None): if not os.path.isdir(cache_dir): raise RuntimeError('Missing cache dir.') if not os.path.isdir(sols_dir): raise RuntimeError('Missing sols dir.') # run killms individually on each MS -- allows restart if it failed in the middle for f in MSs.getListStr(): MSName = os.path.abspath(f).split("/")[-1] solname = os.path.abspath( sols_dir) + "/" + MSName + '/killMS.' + outsols + '.sols.npz' checkname = solname #checkname=f+'/killMS.'+outsols+'.sols.npz' if os.path.isfile(checkname): logger.warning('Solutions file ' + checkname + ' already exists, not running killMS step.') else: runcommand = "kMS.py --MSName %s --SolverType %s --PolMode %s --BaseImageName %s --dt %f --NIterKF %i --CovQ %f --LambdaKF 0.5 --NCPU %i --OutSolsName %s --InCol %s --DoBar 1 --SolsDir %s" \ % (f, SolverType, PolMode, imagename, dt, niterkf, CovQ, s.max_processors, outsols, colname, sols_dir) # set weights if robust is None: runcommand += ' --Weighting Natural' else: runcommand += ' --Weighting Briggs --Robust=%f' % robust # set uv-range if uvrange is not None: if wtuv is not None: runcommand += ' --WTUV=%f --WeightUVMinMax=%f,%f' % ( wtuv, uvrange[0], uvrange[1]) else: runcommand += ' --UVMinMax=%f,%f' % (uvrange[0], uvrange[1]) if PreApplySols: runcommand += ' --PreApplySols=[%s]' % PreApplySols if DISettings is None: runcommand += ' --NChanSols %i' % NChanSols runcommand += ' --BeamMode LOFAR --LOFARBeamMode=A --DDFCacheDir=%s' % cache_dir if clusterfile is not None: runcommand += ' --NodesFile ' + clusterfile if dicomodel is not None: runcommand += ' --DicoModel ' + dicomodel if EvolutionSolFile is not None: runcommand += ' --EvolutionSolFile ' + EvolutionSolFile else: runcommand += " --SolverType %s --PolMode %s --SkyModelCol %s --OutCol %s --ApplyToDir 0" % DISettings _, _, ModelColName, _ = DISettings _, dt, _, n_df = give_dt_dnu(f, DataCol=colname, ModelCol=ModelColName, T=10.) runcommand += " --dt %f --NChanSols %i" % (dt + 1e-4, n_df) s.add(runcommand, log=logfile, commandType='singularity', processors='max') s.run(check=True) # Clip anyway - on IMAGING_WEIGHT by default if DISettings is not None: ClipCol = DISettings[-1] else: ClipCol = colname runcommand = "ClipCal.py --MSName %s --ColName %s" % (f, ClipCol) s.add(runcommand, log=logfile, commandType='singularity', processors='max') s.run(check=True) if MergeSmooth: print('ADD SMOOTHSOL') #outsols=smooth_solutions(mslist,outsols,dryrun=o['dryrun'],InterpToMSListFreqs=InterpToMSListFreqs, # SkipSmooth=SkipSmooth,SigmaFilterOutliers=SigmaFilterOutliers) return outsols
def smooth_solutions(mslist, ddsols, catcher=None, dryrun=False, SkipSmooth=False, SigmaFilterOutliers=None): filenames = [l.strip() for l in open(mslist, 'r').readlines()] full_sollist = [] start_times = [] SolsDir = o["SolsDir"] if SolsDir is None or SolsDir == "": for fname in filenames: solname = fname + '/killMS.' + ddsols + '.sols.npz' t0, t1 = get_solutions_timerange(solname) start_times.append(t0) full_sollist.append(solname) f.write('%s\n' % (solname)) else: for fname in filenames: MSName = os.path.abspath(fname).split("/")[-1] solname = os.path.abspath( SolsDir) + "/" + MSName + '/killMS.' + ddsols + '.sols.npz' t0, t1 = get_solutions_timerange(solname) start_times.append(t0) full_sollist.append(solname) Ustart_times = np.unique(start_times) for start_time in Ustart_times: with open('solslist_%s.txt' % start_time, 'w') as f: for i in range(0, len(full_sollist)): if start_times[i] == start_time: solname = full_sollist[i] f.write('%s\n' % (solname)) checkname = '%s_%s_merged.npz' % (ddsols, start_time) if o['restart'] and os.path.isfile(checkname): warn('Solutions file ' + checkname + ' already exists, not running MergeSols step') else: ss = 'MergeSols.py --SolsFilesIn=solslist_%s.txt --SolFileOut=%s_%s_merged.npz' % ( start_time, ddsols, start_time) if SigmaFilterOutliers: ss += " --SigmaFilterOutliers %f" % SigmaFilterOutliers run(ss, dryrun=dryrun) checkname = '%s_%s_smoothed.npz' % (ddsols, start_time) if o['restart'] and os.path.isfile(checkname): warn('Solutions file ' + checkname + ' already exists, not running SmoothSols step') elif SkipSmooth: warn('Skipping smoothing Solutions file') else: run('SmoothSols.py --SolsFileIn=%s_%s_merged.npz --SolsFileOut=%s_%s_smoothed.npz --InterpMode=%s' % (ddsols, start_time, ddsols, start_time, o['smoothingtype']), dryrun=dryrun) smoothoutname = '%s_%s_smoothed.npz' % (ddsols, start_time) #if InterpToMSListFreqs: # interp_outname="%s_%s_interp.npz"%(smoothoutname,start_time) # checkname=interp_outname # if o['restart'] and os.path.isfile(checkname): # warn('Solutions file '+checkname+' already exists, not running MergeSols step') # else: # command="InterpSols.py --SolsFileIn %s --SolsFileOut %s --MSOutFreq %s"%(smoothoutname,interp_outname,InterpToMSListFreqs) # run(command,dryrun=dryrun) for i in range(0, len(full_sollist)): if start_times[i] == start_time: if not SkipSmooth: symsolname = full_sollist[i].replace( ddsols, ddsols + '_smoothed') else: symsolname = full_sollist[i].replace( ddsols, ddsols + '_merged') # always overwrite the symlink to allow the dataset to move -- costs nothing if os.path.islink(symsolname): logger.warning('Symlink ' + symsolname + ' already exists, recreating') os.unlink(symsolname) if not SkipSmooth: os.symlink( os.path.abspath('%s_%s_smoothed.npz' % (ddsols, start_time)), symsolname) else: os.symlink( os.path.abspath('%s_%s_merged.npz' % (ddsols, start_time)), symsolname) if SkipSmooth: outname = ddsols + '_merged' else: outname = ddsols + '_smoothed' return outname
def check_run(self, log="", commandType=""): """ Produce a warning if a command didn't close the log properly i.e. it crashed NOTE: grep, -L inverse match, -l return only filename """ import subprocess if (not os.path.exists(log)): logger.warning("No log file found to check results: " + log) return 1 if (commandType == "BBS"): out = subprocess.check_output('grep -L success ' + log + ' ; exit 0', shell=False, stderr=subprocess.STDOUT) if out != '': logger.error('BBS run problem on:\n' + out.split("\n")[0]) return 1 elif (commandType == "DPPP"): out = subprocess.check_output('grep -L "Finishing processing" ' + log + ' ; exit 0', shell=True, stderr=subprocess.STDOUT) out += subprocess.check_output('grep -l "Exception" ' + log + ' ; exit 0', shell=True, stderr=subprocess.STDOUT) out += subprocess.check_output( 'grep -l "**** uncaught exception ****" ' + log + ' ; exit 0', shell=True, stderr=subprocess.STDOUT) out += subprocess.check_output('grep -l "error" ' + log + ' ; exit 0', shell=True, stderr=subprocess.STDOUT) #out += subprocess.check_output('grep -l "misspelled" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT) if out != '': logger.error('DPPP run problem on:\n' + out.split("\n")[0]) return 1 elif (commandType == "CASA"): out = subprocess.check_output('grep -l "[a-z]Error" ' + log + ' ; exit 0', shell=True, stderr=subprocess.STDOUT) out += subprocess.check_output( 'grep -l "An error occurred running" ' + log + ' ; exit 0', shell=True, stderr=subprocess.STDOUT) out += subprocess.check_output('grep -l "\*\*\* Error \*\*\*" ' + log + ' ; exit 0', shell=True, stderr=subprocess.STDOUT) if out != '': logger.error('CASA run problem on:\n' + out.split("\n")[0]) return 1 elif (commandType == "wsclean"): out = subprocess.check_output('grep -l "exception occured" ' + log + ' ; exit 0', shell=True, stderr=subprocess.STDOUT) out = subprocess.check_output('grep -l "Segmentation fault" ' + log + ' ; exit 0', shell=True, stderr=subprocess.STDOUT) out += subprocess.check_output( 'grep -L "Cleaning up temporary files..." ' + log + ' ; exit 0', shell=True, stderr=subprocess.STDOUT) if out != '': logger.error('WSClean run problem on:\n' + out.split("\n")[0]) return 1 elif (commandType == "python"): out = subprocess.check_output( 'grep -l "Traceback (most recent call last):" ' + log + ' ; exit 0', shell=True, stderr=subprocess.STDOUT) out += subprocess.check_output( 'grep -i -l \'(?=^((?!error000).)*$).*Error.*\' ' + log + ' ; exit 0', shell=True, stderr=subprocess.STDOUT) out += subprocess.check_output('grep -i -l "Critical" ' + log + ' ; exit 0', shell=True, stderr=subprocess.STDOUT) if out != '': logger.error('Python run problem on:\n' + out.split("\n")[0]) return 1 elif (commandType == "general"): out = subprocess.check_output('grep -l -i "error" ' + log + ' ; exit 0', shell=True, stderr=subprocess.STDOUT) if out != '': logger.error('Run problem on:\n' + out.split("\n")[0]) return 1 else: logger.warning("Unknown command type for log checking: '" + commandType + "'") return 1 return 0