def __init__(self, pathsMS, scheduler, check_flags=True, check_sun=False, min_sun_dist=10):
"""
pathsMS: list of MS paths
scheduler: scheduler object
check_flag: if true ignore fully flagged ms
check_sun: if true check sun distance
min_sun_dist: if check_sun and distance from the sun < than this deg, skip
"""
self.scheduler = scheduler
# sort them, useful for some concatenating steps
if len(pathsMS) == 0:
logger.error('Cannot find MS files.')
raise('Cannot find MS files.')
self.mssListObj = []
for pathMS in sorted(pathsMS):
ms = MS(pathMS)
if check_flags and ms.isAllFlagged():
logger.warning('Skip fully flagged ms: %s' % pathMS)
elif check_sun and ms.sun_dist.deg < min_sun_dist:
logger.warning('Skip too close to sun (%.0f deg) ms: %s' % (ms.sun_dist.deg, pathMS))
else:
self.mssListObj.append(MS(pathMS))
if len(self.mssListObj) == 0:
raise('ALL MS files flagged.')
self.mssListStr = [ms.pathMS for ms in self.mssListObj]
self.resolution = self.mssListObj[0].getResolution(check_flags=False)
def check_run(self, log = "", commandType = ""):
"""
Produce a warning if a command didn't close the log properly i.e. it crashed
NOTE: grep, -L inverse match, -l return only filename
"""
if (not os.path.exists(log)):
logger.warning("No log file found to check results: " + log)
return 1
if (commandType == "DPPP"):
out = subprocess.check_output('grep -L "Finishing processing" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "Segmentation fault\|Killed" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
#
# out += subprocess.check_output('grep -l "Aborted (core dumped)" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -i -l "Exception" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "**** uncaught exception ****" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "error" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "misspelled" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
elif (commandType == "CASA"):
out = subprocess.check_output('grep -l "[a-z]Error" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "An error occurred running" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "\*\*\* Error \*\*\*" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
elif (commandType == "wsclean"):
out = subprocess.check_output('grep -l "exception occur" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "Segmentation fault\|Killed" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "Aborted" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -L "Cleaning up temporary files..." '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
elif (commandType == "python"):
out = subprocess.check_output('grep -l "Traceback (most recent call last):" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "Segmentation fault\|Killed" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -i -l \'(?=^((?!error000).)*$).*Error.*\' '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -i -l "Critical" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "ERROR" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "raise Exception" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
elif (commandType == "singularity"):
out = subprocess.check_output('grep -l "Traceback (most recent call last):" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -i -l \'(?=^((?!error000).)*$).*Error.*\' '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -i -l "Critical" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
elif (commandType == "general"):
out = subprocess.check_output('grep -l -i "error" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
else:
logger.warning("Unknown command type for log checking: '" + commandType + "'")
return 1
if out != b'':
out = out.split(b'\n')[0].decode()
logger.error(commandType+' run problem on:\n'+out)
raise RuntimeError(commandType+' run problem on:\n'+out)
return 0
def todo(self, stepname):
"""
Return false if stepname has been already done
"""
with open(self.filename, "r") as f:
for stepname_done in f:
if stepname == stepname_done.rstrip():
logger.warning('SKIP: %s' % stepname)
return False
return True
def columnAddSimilar(pathMS, columnNameNew, columnNameSimilar, dataManagerInfoNameNew, overwrite = False, fillWithOnes = True, comment = "", verbose = False):
# more to lib_ms
"""
Add a column to a MS that is similar to a pre-existing column (in shape, but not in values).
pathMS: path of the MS
columnNameNew: name of the column to be added
columnNameSimilar: name of the column from which properties are copied (e.g. "DATA")
dataManagerInfoNameNew: string value for the data manager info (DMI) keyword "NAME" (should be unique in the MS)
overwrite: whether or not to overwrite column 'columnNameNew' if it already exists
fillWithOnes: whether or not to fill the newly-made column with ones
verbose: whether or not to produce abundant output
"""
t = tables.table(pathMS, readonly = False)
if (columnExists(t, columnNameNew) and not overwrite):
logger.warning("Attempt to add column '" + columnNameNew + "' aborted, as it already exists and 'overwrite = False' in columnAddSimilar(...).")
else: # Either the column does not exist yet, or it does but overwriting is allowed.
# Remove column if necessary.
if (columnExists(t, columnNameNew)):
logger.info("Removing column '" + columnNameNew + "'...")
t.removecols(columnNameNew)
# Add column.
columnDescription = t.getcoldesc(columnNameSimilar)
dataManagerInfo = t.getdminfo(columnNameSimilar)
if (verbose):
logger.debug("columnDescription:")
logger.debug(columnDescription)
logger.debug("dataManagerInfo:")
logger.debug(dataManagerInfo)
columnDescription["comment"] = ""
# What about adding something here like:
#columnDescription["dataManagerGroup"] = ...?
dataManagerInfo["NAME"] = dataManagerInfoNameNew
if (verbose):
logger.debug("columnDescription (updated):")
logger.debug(columnDescription)
logger.debug("dataManagerInfo (updated):")
logger.debug(dataManagerInfo)
logger.info("Adding column '" + columnNameNew + "'...")
t.addcols(tables.makecoldesc(columnNameNew, columnDescription), dataManagerInfo)
# Fill with ones if desired.
if (fillWithOnes):
logger.info("Filling column '" + columnNameNew + "' with ones...")
columnDataSimilar = t.getcol(columnNameSimilar)
t.putcol(columnNameNew, np.ones_like(columnDataSimilar))
# Close the table to avoid that it is locked for further use.
t.close()
def __exit__(self, type, value, traceback):
"""
Catch "Skip" errors, if not skipped, write to file after exited without exceptions.
"""
if type is None:
with open(self.filename, "a") as f:
f.write(self.__step__ + '\n')
logger.debug('Done: {}'.format(self.__step__))
return # No exception
if issubclass(type, Skip):
logger.warning('SKIP: {}'.format(self.__step__))
return True # Suppress special SkipWithBlock exception
def get_cluster(self):
"""
Find in which computing cluster the pipeline is running
"""
import socket
hostname = socket.gethostname()
if (hostname == 'lgc1' or hostname == 'lgc2'):
return "Hamburg"
elif ('node3' in hostname):
return "Hamburg_fat"
elif ('leidenuniv' in hostname):
return "Leiden"
elif (hostname[0:3] == 'lof'):
return "CEP3"
else:
logger.warning('Hostname %s unknown.' % hostname)
return "Unknown"
def rescaleModel(self, funct_flux):
"""
Rescale the model images to a certain total flux estimated using funct_flux(nu)
nu is taken from the fits file.
funct_flux: is a function of frequency (Hz) which returns the total expected flux (Jy) at that frequency.
"""
for model_img in sorted(glob.glob(self.root+'*model*.fits')):
fits = pyfits.open(model_img)
# get frequency
assert fits[0].header['CTYPE3'] == 'FREQ'
nu = fits[0].header['CRVAL3']
data = fits[0].data
# find expected flux
flux = funct_flux(nu)
current_flux = np.sum(data)
# rescale data
scaling_factor = flux/current_flux
logger.warning('Rescaling model %s by: %f' % (model_img, scaling_factor))
data *= scaling_factor
fits[0].data = data
fits.writeto(model_img, overwrite=True)
fits.close()
def addpol(h5parmFile, soltabname, solsetname='sol000'):
"""
add pol axes on a soltab
"""
# open h5parm
logger.info('%s: add pol axis to %s.' % (h5parmFile, soltabname))
h5 = h5parm(h5parmFile, readonly=False)
ss = h5.getSolset(solsetname)
st = ss.getSoltab(soltabname)
if 'pol' in st.getAxesNames():
h5.close()
logger.warning('%s: polarisation axis already present in %s.' %
(h5parmFile, soltabname))
return
# create values for new soltab
typ = st.getType()
axesNames = st.getAxesNames() + ['pol']
axesVals = [st.getAxisValues(axisName)
for axisName in st.getAxesNames()] + [np.array(['XX', 'YY'])]
vals = st.getValues(retAxesVals=False)
vals = np.array([vals, vals])
vals = np.moveaxis(vals, 0, -1)
weights = st.getValues(weight=True, retAxesVals=False)
weights = np.array([weights, weights])
weights = np.moveaxis(weights, 0, -1)
# remove old soltab
st.delete()
# make new soltab
soltabout = ss.makeSoltab(soltype = typ, soltabName = soltabname, axesNames=axesNames, \
axesVals=axesVals, vals=vals, weights=weights)
# write h5parm
h5.close()
def __init__(self, pathsMS, scheduler, check_flags=True):
"""
pathsMS: list of MS paths
scheduler: scheduler object
check_flag: if true ignore fully flagged ms
"""
self.scheduler = scheduler
# sort them, useful for some concatenating steps
if len(pathsMS) == 0:
raise ('Cannot find MS files.')
self.mssListObj = []
for pathMS in sorted(pathsMS):
ms = MS(pathMS)
if check_flags and ms.isAllFlagged():
logger.warning('Skip fully flagged ms: %s' % pathMS)
else:
self.mssListObj.append(MS(pathMS))
if len(self.mssListObj) == 0:
raise ('ALL MS files flagged.')
self.mssListStr = [ms.pathMS for ms in self.mssListObj]
def killms_data(s,
logfile,
MSs,
imagename,
outsols,
clusterfile=None,
colname='DATA',
niterkf=6,
dicomodel=None,
uvrange=None,
wtuv=None,
robust=None,
dt=None,
cache_dir='ddfcal/cache',
sols_dir='ddfcal/solutions',
SolverType="KAFCA",
PolMode="Scalar",
MergeSmooth=False,
NChanSols=1,
DISettings=None,
EvolutionSolFile=None,
CovQ=0.1,
InterpToMSListFreqs=None,
SkipSmooth=False,
PreApplySols=None,
SigmaFilterOutliers=None):
if not os.path.isdir(cache_dir):
raise RuntimeError('Missing cache dir.')
if not os.path.isdir(sols_dir):
raise RuntimeError('Missing sols dir.')
# run killms individually on each MS -- allows restart if it failed in the middle
for f in MSs.getListStr():
MSName = os.path.abspath(f).split("/")[-1]
solname = os.path.abspath(
sols_dir) + "/" + MSName + '/killMS.' + outsols + '.sols.npz'
checkname = solname
#checkname=f+'/killMS.'+outsols+'.sols.npz'
if os.path.isfile(checkname):
logger.warning('Solutions file ' + checkname +
' already exists, not running killMS step.')
else:
runcommand = "kMS.py --MSName %s --SolverType %s --PolMode %s --BaseImageName %s --dt %f --NIterKF %i --CovQ %f --LambdaKF 0.5 --NCPU %i --OutSolsName %s --InCol %s --DoBar 1 --SolsDir %s" \
% (f, SolverType, PolMode, imagename, dt, niterkf, CovQ, s.max_processors, outsols, colname, sols_dir)
# set weights
if robust is None:
runcommand += ' --Weighting Natural'
else:
runcommand += ' --Weighting Briggs --Robust=%f' % robust
# set uv-range
if uvrange is not None:
if wtuv is not None:
runcommand += ' --WTUV=%f --WeightUVMinMax=%f,%f' % (
wtuv, uvrange[0], uvrange[1])
else:
runcommand += ' --UVMinMax=%f,%f' % (uvrange[0],
uvrange[1])
if PreApplySols:
runcommand += ' --PreApplySols=[%s]' % PreApplySols
if DISettings is None:
runcommand += ' --NChanSols %i' % NChanSols
runcommand += ' --BeamMode LOFAR --LOFARBeamMode=A --DDFCacheDir=%s' % cache_dir
if clusterfile is not None:
runcommand += ' --NodesFile ' + clusterfile
if dicomodel is not None:
runcommand += ' --DicoModel ' + dicomodel
if EvolutionSolFile is not None:
runcommand += ' --EvolutionSolFile ' + EvolutionSolFile
else:
runcommand += " --SolverType %s --PolMode %s --SkyModelCol %s --OutCol %s --ApplyToDir 0" % DISettings
_, _, ModelColName, _ = DISettings
_, dt, _, n_df = give_dt_dnu(f,
DataCol=colname,
ModelCol=ModelColName,
T=10.)
runcommand += " --dt %f --NChanSols %i" % (dt + 1e-4, n_df)
s.add(runcommand,
log=logfile,
commandType='singularity',
processors='max')
s.run(check=True)
# Clip anyway - on IMAGING_WEIGHT by default
if DISettings is not None:
ClipCol = DISettings[-1]
else:
ClipCol = colname
runcommand = "ClipCal.py --MSName %s --ColName %s" % (f, ClipCol)
s.add(runcommand,
log=logfile,
commandType='singularity',
processors='max')
s.run(check=True)
if MergeSmooth:
print('ADD SMOOTHSOL')
#outsols=smooth_solutions(mslist,outsols,dryrun=o['dryrun'],InterpToMSListFreqs=InterpToMSListFreqs,
# SkipSmooth=SkipSmooth,SigmaFilterOutliers=SigmaFilterOutliers)
return outsols
def smooth_solutions(mslist,
ddsols,
catcher=None,
dryrun=False,
SkipSmooth=False,
SigmaFilterOutliers=None):
filenames = [l.strip() for l in open(mslist, 'r').readlines()]
full_sollist = []
start_times = []
SolsDir = o["SolsDir"]
if SolsDir is None or SolsDir == "":
for fname in filenames:
solname = fname + '/killMS.' + ddsols + '.sols.npz'
t0, t1 = get_solutions_timerange(solname)
start_times.append(t0)
full_sollist.append(solname)
f.write('%s\n' % (solname))
else:
for fname in filenames:
MSName = os.path.abspath(fname).split("/")[-1]
solname = os.path.abspath(
SolsDir) + "/" + MSName + '/killMS.' + ddsols + '.sols.npz'
t0, t1 = get_solutions_timerange(solname)
start_times.append(t0)
full_sollist.append(solname)
Ustart_times = np.unique(start_times)
for start_time in Ustart_times:
with open('solslist_%s.txt' % start_time, 'w') as f:
for i in range(0, len(full_sollist)):
if start_times[i] == start_time:
solname = full_sollist[i]
f.write('%s\n' % (solname))
checkname = '%s_%s_merged.npz' % (ddsols, start_time)
if o['restart'] and os.path.isfile(checkname):
warn('Solutions file ' + checkname +
' already exists, not running MergeSols step')
else:
ss = 'MergeSols.py --SolsFilesIn=solslist_%s.txt --SolFileOut=%s_%s_merged.npz' % (
start_time, ddsols, start_time)
if SigmaFilterOutliers:
ss += " --SigmaFilterOutliers %f" % SigmaFilterOutliers
run(ss, dryrun=dryrun)
checkname = '%s_%s_smoothed.npz' % (ddsols, start_time)
if o['restart'] and os.path.isfile(checkname):
warn('Solutions file ' + checkname +
' already exists, not running SmoothSols step')
elif SkipSmooth:
warn('Skipping smoothing Solutions file')
else:
run('SmoothSols.py --SolsFileIn=%s_%s_merged.npz --SolsFileOut=%s_%s_smoothed.npz --InterpMode=%s'
% (ddsols, start_time, ddsols, start_time, o['smoothingtype']),
dryrun=dryrun)
smoothoutname = '%s_%s_smoothed.npz' % (ddsols, start_time)
#if InterpToMSListFreqs:
# interp_outname="%s_%s_interp.npz"%(smoothoutname,start_time)
# checkname=interp_outname
# if o['restart'] and os.path.isfile(checkname):
# warn('Solutions file '+checkname+' already exists, not running MergeSols step')
# else:
# command="InterpSols.py --SolsFileIn %s --SolsFileOut %s --MSOutFreq %s"%(smoothoutname,interp_outname,InterpToMSListFreqs)
# run(command,dryrun=dryrun)
for i in range(0, len(full_sollist)):
if start_times[i] == start_time:
if not SkipSmooth:
symsolname = full_sollist[i].replace(
ddsols, ddsols + '_smoothed')
else:
symsolname = full_sollist[i].replace(
ddsols, ddsols + '_merged')
# always overwrite the symlink to allow the dataset to move -- costs nothing
if os.path.islink(symsolname):
logger.warning('Symlink ' + symsolname +
' already exists, recreating')
os.unlink(symsolname)
if not SkipSmooth:
os.symlink(
os.path.abspath('%s_%s_smoothed.npz' %
(ddsols, start_time)), symsolname)
else:
os.symlink(
os.path.abspath('%s_%s_merged.npz' %
(ddsols, start_time)), symsolname)
if SkipSmooth:
outname = ddsols + '_merged'
else:
outname = ddsols + '_smoothed'
return outname
def check_run(self, log="", commandType=""):
"""
Produce a warning if a command didn't close the log properly i.e. it crashed
NOTE: grep, -L inverse match, -l return only filename
"""
import subprocess
if (not os.path.exists(log)):
logger.warning("No log file found to check results: " + log)
return 1
if (commandType == "BBS"):
out = subprocess.check_output('grep -L success ' + log +
' ; exit 0',
shell=False,
stderr=subprocess.STDOUT)
if out != '':
logger.error('BBS run problem on:\n' + out.split("\n")[0])
return 1
elif (commandType == "DPPP"):
out = subprocess.check_output('grep -L "Finishing processing" ' +
log + ' ; exit 0',
shell=True,
stderr=subprocess.STDOUT)
out += subprocess.check_output('grep -l "Exception" ' + log +
' ; exit 0',
shell=True,
stderr=subprocess.STDOUT)
out += subprocess.check_output(
'grep -l "**** uncaught exception ****" ' + log + ' ; exit 0',
shell=True,
stderr=subprocess.STDOUT)
out += subprocess.check_output('grep -l "error" ' + log +
' ; exit 0',
shell=True,
stderr=subprocess.STDOUT)
#out += subprocess.check_output('grep -l "misspelled" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
if out != '':
logger.error('DPPP run problem on:\n' + out.split("\n")[0])
return 1
elif (commandType == "CASA"):
out = subprocess.check_output('grep -l "[a-z]Error" ' + log +
' ; exit 0',
shell=True,
stderr=subprocess.STDOUT)
out += subprocess.check_output(
'grep -l "An error occurred running" ' + log + ' ; exit 0',
shell=True,
stderr=subprocess.STDOUT)
out += subprocess.check_output('grep -l "\*\*\* Error \*\*\*" ' +
log + ' ; exit 0',
shell=True,
stderr=subprocess.STDOUT)
if out != '':
logger.error('CASA run problem on:\n' + out.split("\n")[0])
return 1
elif (commandType == "wsclean"):
out = subprocess.check_output('grep -l "exception occured" ' +
log + ' ; exit 0',
shell=True,
stderr=subprocess.STDOUT)
out = subprocess.check_output('grep -l "Segmentation fault" ' +
log + ' ; exit 0',
shell=True,
stderr=subprocess.STDOUT)
out += subprocess.check_output(
'grep -L "Cleaning up temporary files..." ' + log +
' ; exit 0',
shell=True,
stderr=subprocess.STDOUT)
if out != '':
logger.error('WSClean run problem on:\n' + out.split("\n")[0])
return 1
elif (commandType == "python"):
out = subprocess.check_output(
'grep -l "Traceback (most recent call last):" ' + log +
' ; exit 0',
shell=True,
stderr=subprocess.STDOUT)
out += subprocess.check_output(
'grep -i -l \'(?=^((?!error000).)*$).*Error.*\' ' + log +
' ; exit 0',
shell=True,
stderr=subprocess.STDOUT)
out += subprocess.check_output('grep -i -l "Critical" ' + log +
' ; exit 0',
shell=True,
stderr=subprocess.STDOUT)
if out != '':
logger.error('Python run problem on:\n' + out.split("\n")[0])
return 1
elif (commandType == "general"):
out = subprocess.check_output('grep -l -i "error" ' + log +
' ; exit 0',
shell=True,
stderr=subprocess.STDOUT)
if out != '':
logger.error('Run problem on:\n' + out.split("\n")[0])
return 1
else:
logger.warning("Unknown command type for log checking: '" +
commandType + "'")
return 1
return 0
