def run(self, executable, nthreads, strategy, indirect, skip_flagged, wd, *infiles):
with log_time(self.logger):
self.logger.info("Processing %s" % " ".join(infiles))
try:
if not os.access(executable, os.X_OK):
raise ExecutableMissing(executable)
working_dir = tempfile.mkdtemp(dir=wd,suffix=".%s" % (os.path.basename(__file__),))
cmd = [executable, "-j", str(nthreads)]
if strategy:
if os.path.exists(strategy):
cmd.extend(["-strategy", strategy])
else:
raise Exception("Strategy definition not available")
if indirect:
cmd.extend(["-indirect-read"])
if skip_flagged:
cmd.extend(["-skip-flagged"])
cmd.extend(infiles)
with CatchLog4CPlus(
working_dir,
self.logger.name,
os.path.basename(executable)
) as logger:
catch_segfaults(cmd, working_dir, None, logger)
except ExecutableMissing, e:
self.logger.error("%s not found" % (e.args[0]))
return 1
except CalledProcessError, e:
self.logger.error(str(e))
return 1
def run(self, executable, file_pattern, input_dir, output_file, clobber):
if not os.path.exists(input_dir):
self.logger.error("Input directory (%s) not found" % input_dir)
return 1
self.logger.info("Processing %s" % input_dir)
if os.path.exists(output_file):
if clobber:
self.logger.warn(
"Deleting previous version of results: %s" % output_file
)
os.unlink(output_file)
else:
self.logger.error(
"Refusing to overwrite existing file %s" % output_file
)
return 1
input_files = glob.glob(os.path.join(input_dir, file_pattern))
command_line = [executable] + input_files + [output_file]
try:
catch_segfaults(command_line, None, None, self.logger)
except Exception, e:
self.logger.error(str(e))
return 1
def _create_mask(self, npix, cell_size, output_image,
concatenated_measurement_set, executable,
working_directory, log4_cplus_name, sourcedb_path,
mask_patch_size, image_path_directory):
"""
(3) create a casa image containing an mask blocking out the
sources in the provided sourcedb.
It expects:
a. the ms for which the mask will be created, it is used to de
termine some image details: (eg. pointing)
b. parameters for running within the catchsegfault framework
c. and the size of the mask_pach.
To create a mask, first a empty measurement set is created using
awimager: ready to be filled with mask data
This function is a wrapper around some functionality written by:
[email protected]
steps:
1. Create a parset with image paramters used by:
2. awimager run. Creating an empty casa image.
3. Fill the casa image with mask data
"""
# ********************************************************************
# 1. Create the parset used to make a mask
mask_file_path = output_image + ".mask"
mask_patch_dictionary = {
"npix": str(npix),
"cellsize": str(cell_size),
"image": str(mask_file_path),
"ms": str(concatenated_measurement_set),
"operation": "empty",
"stokes": "'I'"
}
mask_parset = Parset.fromDict(mask_patch_dictionary)
mask_parset_path = os.path.join(image_path_directory, "mask.par")
mask_parset.writeFile(mask_parset_path)
self.logger.debug(
"Write parset for awimager mask creation: {0}".format(
mask_parset_path))
# *********************************************************************
# 2. Create an empty mask using awimager
cmd = [executable, mask_parset_path]
self.logger.info(" ".join(cmd))
try:
with CatchLog4CPlus(
working_directory,
self.logger.name + "." + os.path.basename(log4_cplus_name),
os.path.basename(executable)) as logger:
catch_segfaults(cmd, working_directory, self.environment,
logger)
# Thrown by catch_segfault
except CalledProcessError, exception:
self.logger.error(str(exception))
return 1
def _create_mask(self, npix, cell_size, output_image,
concatenated_measurement_set, executable,
working_directory, log4_cplus_name, sourcedb_path,
mask_patch_size, image_path_directory):
"""
(3) create a casa image containing an mask blocking out the
sources in the provided sourcedb.
It expects:
a. the ms for which the mask will be created, it is used to de
termine some image details: (eg. pointing)
b. parameters for running within the catchsegfault framework
c. and the size of the mask_pach.
To create a mask, first a empty measurement set is created using
awimager: ready to be filled with mask data
This function is a wrapper around some functionality written by:
[email protected]
steps:
1. Create a parset with image paramters used by:
2. awimager run. Creating an empty casa image.
3. Fill the casa image with mask data
"""
# ********************************************************************
# 1. Create the parset used to make a mask
mask_file_path = output_image + ".mask"
mask_patch_dictionary = {"npix": str(npix),
"cellsize": str(cell_size),
"image": str(mask_file_path),
"ms": str(concatenated_measurement_set),
"operation": "empty",
"stokes": "'I'"
}
mask_parset = Parset.fromDict(mask_patch_dictionary)
mask_parset_path = os.path.join(image_path_directory, "mask.par")
mask_parset.writeFile(mask_parset_path)
self.logger.debug(
"Write parset for awimager mask creation: {0}".format(
mask_parset_path))
# *********************************************************************
# 2. Create an empty mask using awimager
cmd = [executable, mask_parset_path]
self.logger.info(" ".join(cmd))
try:
with CatchLog4CPlus(working_directory,
self.logger.name + "." + os.path.basename(log4_cplus_name),
os.path.basename(executable)
) as logger:
catch_segfaults(cmd, working_directory, self.environment,
logger)
# Thrown by catch_segfault
except CalledProcessError, exception:
self.logger.error(str(exception))
return 1
def run(self, infile, outfile, executable, environment, sigma,
use_parmexportcal):
self.environment.update(environment)
if os.path.exists(infile):
self.logger.info("Processing {0}".format(infile))
else:
self.logger.error(
"Instrument model file %s does not exist" % infile
)
return 1
# Create output directory (if it doesn't already exist)
create_directory(os.path.dirname(outfile))
# Remove the target outfile if there: parexportcall fail otherwise
if os.path.exists(outfile):
shutil.rmtree(outfile)
# ********************************************************************
# 1. Select correction method
if not use_parmexportcal:
# ****************************************************************
# 3. use gainoutliercorrect from Swinbank
self.logger.info(
"Using the gainoutlier correction based on editparmdb")
self._filter_stations_parmdb(infile, outfile, sigma)
return 0
# else:
if not os.access(executable, os.X_OK):
self.logger.error(
"Could not find parmexport call executable at: {0}".format(
executable))
self.logger.error("bailing out!")
return 1
# ********************************************************************
# 2. Call parmexportcal for gain correction
self.logger.info(
"Using the gainoutlier correction based on parmexportcal")
try:
temp_dir = tempfile.mkdtemp()
with CatchLog4CPlus(
temp_dir,
self.logger.name + '.' + os.path.basename(infile),
os.path.basename(executable)
) as logger:
cmd = [executable, '-in', infile, '-out', outfile]
self.logger.debug(
"Parmexportcal call: {0} ".format(" ".join(cmd)))
catch_segfaults(
cmd,
temp_dir,
self.environment,
logger
)
except Exception, excp:
self.logger.error(str(excp))
return 1
def run(self, infile, outfile, executable, environment, sigma,
use_parmexportcal):
self.environment.update(environment)
if os.path.exists(infile):
self.logger.info("Processing {0}".format(infile))
else:
self.logger.error(
"Instrument model file %s does not exist" % infile
)
return 1
# Create output directory (if it doesn't already exist)
create_directory(os.path.dirname(outfile))
# Remove the target outfile if there: parexportcall fail otherwise
if os.path.exists(outfile):
shutil.rmtree(outfile)
# ********************************************************************
# 1. Select correction method
if not use_parmexportcal:
# ****************************************************************
# 3. use gainoutliercorrect from Swinbank
self.logger.info(
"Using the gainoutlier correction based on editparmdb")
self._filter_stations_parmdb(infile, outfile, sigma)
return 0
# else:
if not os.access(executable, os.X_OK):
self.logger.error(
"Could not find parmexport call executable at: {0}".format(
executable))
self.logger.error("bailing out!")
return 1
# ********************************************************************
# 2. Call parmexportcal for gain correction
self.logger.info(
"Using the gainoutlier correction based on parmexportcal")
try:
temp_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
with CatchLog4CPlus(
temp_dir,
self.logger.name + '.' + os.path.basename(infile),
os.path.basename(executable)
) as logger:
cmd = [executable, '-in', infile, '-out', outfile]
self.logger.debug(
"Parmexportcal call: {0} ".format(" ".join(cmd)))
catch_segfaults(
cmd,
temp_dir,
self.environment,
logger
)
except Exception, excp:
self.logger.error(str(excp))
return 1
def _execute(self, cmd):
try:
temp_dir = tempfile.mkdtemp()
with CatchLog4CPlus(temp_dir, self.logger.name,
os.path.basename(cmd[0])) as logger:
catch_segfaults(cmd, temp_dir, self.environment, self.logger)
except Exception, e:
self.logger.error(str(e))
return False
def _dppp_call(self, working_dir, ndppp, cmd, environment):
"""
Muckable function running the dppp executable.
Wraps dppp with catchLog4CPLus and catch_segfaults
"""
with CatchLog4CPlus(working_dir, self.logger.name +
"." + os.path.basename("long_baseline_ndppp"),
os.path.basename(ndppp)) as logger:
catch_segfaults(cmd, working_dir, environment,
logger, cleanup = None)
def _execute(self, cmd):
try:
temp_dir = tempfile.mkdtemp()
with CatchLog4CPlus(temp_dir,
self.logger.name,
os.path.basename(cmd[0])
) as logger:
catch_segfaults(cmd, temp_dir, self.environment, self.logger)
except Exception, e:
self.logger.error(str(e))
return False
def _dppp_call(self, working_dir, ndppp, cmd, environment):
"""
Muckable function running the dppp executable.
Wraps dppp with catchLog4CPLus and catch_segfaults
"""
# TODO: cpu limited is static at this location
environment['OMP_NUM_THREADS'] = str(8)
self.logger.debug("Using %s threads for ndppp" % 8)
with CatchLog4CPlus(working_dir, self.logger.name +
"." + os.path.basename("imager_prepare_ndppp"),
os.path.basename(ndppp)) as logger:
catch_segfaults(cmd, working_dir, environment,
logger, cleanup = None, usageStats=self.resourceMonitor)
def _execute(self, cmd):
try:
temp_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
with CatchLog4CPlus(temp_dir,
self.logger.name,
os.path.basename(cmd[0])
) as logger:
catch_segfaults(cmd, temp_dir, self.environment, self.logger)
except Exception as e:
self.logger.error(str(e))
return False
finally:
shutil.rmtree(temp_dir)
return True
def _dppp_call(self, working_dir, ndppp, cmd, environment):
"""
Muckable function running the dppp executable.
Wraps dppp with catchLog4CPLus and catch_segfaults
"""
with CatchLog4CPlus(
working_dir, self.logger.name + "." +
os.path.basename("imager_prepare_ndppp"),
os.path.basename(ndppp)) as logger:
catch_segfaults(cmd,
working_dir,
environment,
logger,
cleanup=None)
def _create_source_db(self,
source_list,
sourcedb_target_path,
working_directory,
create_sourcdb_exec,
append=False):
"""
Convert a sourcelist to a sourcedb:
1. Remove existing sourcedb if not appending (sourcedb fails else)
2. Call the sourcedb executable with the supplied parameters
"""
# *********************************************************************
# 1. remove existing sourcedb if not appending
if (append == False) and os.path.isdir(sourcedb_target_path):
shutil.rmtree(sourcedb_target_path)
self.logger.debug(
"Removed existing sky model: {0}".format(sourcedb_target_path))
# *********************************************************************
# 2. The command and parameters to be run
cmd = [
create_sourcdb_exec,
"in={0}".format(source_list),
"out={0}".format(sourcedb_target_path),
"format=<", # format according to Ger van Diepen
"append=true"
] # Always set append: no effect on non exist db
self.logger.info(' '.join(cmd))
try:
with CatchLog4CPlus(
working_directory,
self.logger.name + "." + os.path.basename("makesourcedb"),
os.path.basename(create_sourcdb_exec)) as logger:
catch_segfaults(cmd,
working_directory,
self.environment,
logger,
cleanup=None)
except Exception as exception:
self.logger.error("Execution of external failed:")
self.logger.error(" ".join(cmd))
self.logger.error("exception details:")
self.logger.error(str(exception))
return 1
return 0
def _create_source_db(self,
source_list,
sourcedb_target_path,
working_directory,
executable,
append=False):
"""
_create_source_db consumes a sourcelist text file and produces a
source db (pyraptable).
If the append parameter is set to true. It expects an already existing
sourcedb on the supplied path and will then append the sources in
the list. typically used multiple iterations of the imaging pipeline,
with self calibration
"""
#remove existing sourcedb if not appending
if (append == False) and os.path.isdir(sourcedb_target_path):
shutil.rmtree(sourcedb_target_path)
self.logger.debug(
"Removed existing sky model: {0}".format(sourcedb_target_path))
# The command and parameters to be run
cmd = [
executable,
"in={0}".format(source_list),
"out={0}".format(sourcedb_target_path),
"format=<", # format according to Ger van Diepen
"append=true"
] # Always set append flag: no effect on non exist
# db
try:
with CatchLog4CPlus(
working_directory,
self.logger.name + "." + os.path.basename("makesourcedb"),
os.path.basename(executable)) as logger:
catch_segfaults(cmd,
working_directory,
self.environment,
logger,
cleanup=None)
except subprocess.CalledProcessError as called_proc_error:
self.logger.error("Execution of external failed:")
self.logger.error(" ".join(cmd))
self.logger.error("exception details:")
self.logger.error(str(called_proc_error))
return 1
return 0
def execute(self, executable, args):
try:
# ****************************************************************
# Run
cmd = [executable] + args
with CatchLog4CPlus(
self.work_dir,
self.logger.name + "." + os.path.basename(self.infile),
os.path.basename(self.executable),
) as logger:
# Catch segfaults and retry
catch_segfaults(cmd, self.work_dir, self.environment, logger)
except CalledProcessError, err:
# CalledProcessError isn't properly propagated by IPython
self.logger.error(str(err))
return 1
def run(self, files, executable, parset, environment):
"""
Run the bbs-reducer executable.
*Arguments*
- `files`: argument is a tuple of (MS-file, parmdb-file, sourcedb-file)
- `executable`: full path to the bbs-reducer executable
- `parset`: full path to the parset-file
- `environment`: environment variables to use
"""
self.logger.debug("files = %s" % str(files))
self.logger.debug("executable = %s" % executable)
self.logger.debug("parset = %s" % parset)
self.logger.debug("environment = %s" % environment)
self.environment.update(environment)
ms, parmdb, sourcedb = files
# Time execution of this job
with log_time(self.logger):
if os.path.exists(ms):
self.logger.info("Processing %s" % ms)
else:
self.logger.error("Measurement Set %s does not exist" % ms)
return 1
# Run bbs-reducer. Catch log output from bbs-reducer and stdout.
scratch_dir = mkdtemp(suffix=".%s" %
(os.path.basename(__file__), ))
try:
cmd = [
executable,
"--parmdb=%s" % parmdb,
"--sourcedb=%s" % sourcedb, ms, parset
]
with CatchLog4CPlus(
scratch_dir,
self.logger.name + "." + os.path.basename(ms),
os.path.basename(executable),
) as logger:
catch_segfaults(cmd, scratch_dir, self.environment, logger)
except CalledProcessError as err:
self.logger.error(str(err))
return 1
finally:
shutil.rmtree(scratch_dir)
return 0
def run(self, executable, catalogue, skydb, dbtype):
"""
Contains all functionality
"""
with log_time(self.logger):
# ****************************************************************
# 1. Create output directory if it does not yet exist.
skydb_dir = os.path.dirname(skydb)
try:
os.makedirs(skydb_dir)
self.logger.debug("Created output directory %s" % skydb_dir)
except FileExistsError:
pass
# ****************************************************************
# 2 Remove any old sky database
# Create the sourcedb
shutil.rmtree(skydb, ignore_errors=True)
self.logger.info("Creating skymodel: %s" % (skydb))
scratch_dir = tempfile.mkdtemp(suffix=".%s" %
(os.path.basename(__file__), ))
try:
cmd = [
executable,
"in=%s" % catalogue,
"out=%s" % skydb,
"outtype=%s" % dbtype, "format=<", "append=false"
]
with CatchLog4CPlus(
scratch_dir,
self.logger.name + "." + os.path.basename(skydb),
os.path.basename(executable)) as logger:
catch_segfaults(cmd, scratch_dir, None, logger)
# *****************************************************************
# 3. Validate performance and cleanup temp files
except CalledProcessError as err:
# For CalledProcessError isn't properly propagated by IPython
# Temporary workaround...
self.logger.error(str(err))
return 1
finally:
shutil.rmtree(scratch_dir)
return 0
def run(self, executable, nthreads, strategy, indirect, skip_flagged, wd,
*infiles):
with log_time(self.logger):
self.logger.info("Processing %s" % " ".join(infiles))
try:
if not os.access(executable, os.X_OK):
raise ExecutableMissing(executable)
working_dir = tempfile.mkdtemp(dir=wd,
suffix=".%s" %
(os.path.basename(__file__), ))
cmd = [executable, "-j", str(nthreads)]
if strategy:
if os.path.exists(strategy):
cmd.extend(["-strategy", strategy])
else:
raise Exception("Strategy definition not available")
if indirect:
cmd.extend(["-indirect-read"])
if skip_flagged:
cmd.extend(["-skip-flagged"])
cmd.extend(infiles)
with CatchLog4CPlus(working_dir, self.logger.name,
os.path.basename(executable)) as logger:
catch_segfaults(cmd, working_dir, None, logger)
except ExecutableMissing as e:
self.logger.error("%s not found" % (e.args[0]))
return 1
except CalledProcessError as e:
self.logger.error(str(e))
return 1
except Exception as e:
self.logger.exception(e)
return 1
finally:
# Try and clean up the working directory, but don't worry if
# it fails -- it might not have been greated before throwing
# the exception
try:
shutil.rmtree(working_dir)
except:
pass
return 0
def run(self, executable, catalogue, skydb, dbtype):
"""
Contains all functionality
"""
with log_time(self.logger):
# ****************************************************************
# 1. Create output directory if it does not yet exist.
skydb_dir = os.path.dirname(skydb)
try:
os.makedirs(skydb_dir)
self.logger.debug("Created output directory %s" % skydb_dir)
except OSError, err:
# Ignore error if directory already exists, otherwise re-raise
if err[0] != errno.EEXIST:
raise
# ****************************************************************
# 2 Remove any old sky database
# Create the sourcedb
shutil.rmtree(skydb, ignore_errors=True)
self.logger.info("Creating skymodel: %s" % (skydb))
scratch_dir = tempfile.mkdtemp()
try:
cmd = [executable,
"in=%s" % catalogue,
"out=%s" % skydb,
"outtype=%s" % dbtype,
"format=<",
"append=false"
]
with CatchLog4CPlus(
scratch_dir,
self.logger.name + "." + os.path.basename(skydb),
os.path.basename(executable)
) as logger:
catch_segfaults(cmd, scratch_dir, None, logger)
# *****************************************************************
# 3. Validate performance and cleanup temp files
except CalledProcessError, err:
# For CalledProcessError isn't properly propagated by IPython
# Temporary workaround...
self.logger.error(str(err))
return 1
def execute(self, executable, args):
try:
# ****************************************************************
# Run
cmd = [executable] + args
with CatchLog4CPlus(
self.work_dir,
self.logger.name + "." + os.path.basename(self.infile),
os.path.basename(self.executable),
) as logger:
# Catch segfaults and retry
catch_segfaults(
cmd, self.work_dir, self.environment, logger
)
except CalledProcessError, err:
# CalledProcessError isn't properly propagated by IPython
self.logger.error(str(err))
return 1
def run(self, files, executable, parset, environment):
"""
Run the bbs-reducer executable.
*Arguments*
- `files`: argument is a tuple of (MS-file, parmdb-file, sourcedb-file)
- `executable`: full path to the bbs-reducer executable
- `parset`: full path to the parset-file
- `environment`: environment variables to use
"""
self.logger.debug("files = %s" % str(files))
self.logger.debug("executable = %s" % executable)
self.logger.debug("parset = %s" % parset)
self.logger.debug("environment = %s" % environment)
self.environment.update(environment)
ms, parmdb, sourcedb = files
# Time execution of this job
with log_time(self.logger):
if os.path.exists(ms):
self.logger.info("Processing %s" % ms)
else:
self.logger.error("Measurement Set %s does not exist" % ms)
return 1
# Run bbs-reducer. Catch log output from bbs-reducer and stdout.
scratch_dir = mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
try:
cmd = [executable,
"--parmdb=%s" % parmdb,
"--sourcedb=%s" % sourcedb,
ms, parset
]
with CatchLog4CPlus(
scratch_dir,
self.logger.name + "." + os.path.basename(ms),
os.path.basename(executable),
) as logger:
catch_segfaults(cmd, scratch_dir, self.environment, logger)
except CalledProcessError, err:
self.logger.error(str(err))
return 1
finally:
def _create_source_db(self, source_list, sourcedb_target_path,
working_directory, create_sourcdb_exec, append = False):
"""
Convert a sourcelist to a sourcedb:
1. Remove existing sourcedb if not appending (sourcedb fails else)
2. Call the sourcedb executable with the supplied parameters
"""
# *********************************************************************
# 1. remove existing sourcedb if not appending
if (append == False) and os.path.isdir(sourcedb_target_path):
shutil.rmtree(sourcedb_target_path)
self.logger.debug("Removed existing sky model: {0}".format(
sourcedb_target_path))
# *********************************************************************
# 2. The command and parameters to be run
cmd = [create_sourcdb_exec, "in={0}".format(source_list),
"out={0}".format(sourcedb_target_path),
"format=<", # format according to Ger van Diepen
"append=true"] # Always set append: no effect on non exist db
self.logger.info(' '.join(cmd))
try:
with CatchLog4CPlus(working_directory,
self.logger.name + "." + os.path.basename("makesourcedb"),
os.path.basename(create_sourcdb_exec)
) as logger:
catch_segfaults(cmd, working_directory, self.environment,
logger, cleanup = None)
except Exception, exception:
self.logger.error("Execution of external failed:")
self.logger.error(" ".join(cmd))
self.logger.error("exception details:")
self.logger.error(str(exception))
return 1
def run(self, executable, file_pattern, input_dir, output_file, clobber):
if not os.path.exists(input_dir):
self.logger.error("Input directory (%s) not found" % input_dir)
return 1
self.logger.info("Processing %s" % input_dir)
if os.path.exists(output_file):
if clobber:
self.logger.warn("Deleting previous version of results: %s" % output_file)
os.unlink(output_file)
else:
self.logger.error("Refusing to overwrite existing file %s" % output_file)
return 1
input_files = glob.glob(os.path.join(input_dir, file_pattern))
command_line = [executable] + input_files + [output_file]
try:
catch_segfaults(command_line, None, None, self.logger)
except Exception, e:
self.logger.error(str(e))
return 1
def _create_source_db(self, source_list, sourcedb_target_path,
working_directory, executable, append=False):
"""
_create_source_db consumes a sourcelist text file and produces a
source db (pyraptable).
If the append parameter is set to true. It expects an already existing
sourcedb on the supplied path and will then append the sources in
the list. typically used multiple iterations of the imaging pipeline,
with self calibration
"""
#remove existing sourcedb if not appending
if (append == False) and os.path.isdir(sourcedb_target_path):
shutil.rmtree(sourcedb_target_path)
self.logger.debug("Removed existing sky model: {0}".format(
sourcedb_target_path))
# The command and parameters to be run
cmd = [executable, "in={0}".format(source_list),
"out={0}".format(sourcedb_target_path),
"format=<", # format according to Ger van Diepen
"append=true"] # Always set append flag: no effect on non exist
# db
try:
with CatchLog4CPlus(working_directory,
self.logger.name + "." + os.path.basename("makesourcedb"),
os.path.basename(executable)
) as logger:
catch_segfaults(cmd, working_directory, self.environment,
logger, cleanup=None)
except subprocess.CalledProcessError, called_proc_error:
self.logger.error("Execution of external failed:")
self.logger.error(" ".join(cmd))
self.logger.error("exception details:")
self.logger.error(str(called_proc_error))
return 1
def run(self, infile, clusterdesc, outfile, executable):
with log_time(self.logger):
if os.path.exists(infile):
self.logger.info("Processing %s" % (infile))
else:
self.logger.error("Dataset %s does not exist" % (infile))
return 1
try:
if not os.access(executable, os.X_OK):
raise ExecutableMissing(executable)
cmd = [executable, clusterdesc, infile, outfile]
return catch_segfaults(cmd, None, None, self.logger).returncode
except ExecutableMissing, e:
self.logger.error("%s not found" % (e.args[0]))
return 1
except CalledProcessError, e:
# For CalledProcessError isn't properly propagated by IPython
# Temporary workaround...
self.logger.error(str(e))
self.logger.info("A common cause for this failure is the usage"
"of an incorrect cluster.desc file in the pipeline.cfg")
return 1
def run(self, infile, clusterdesc, outfile, executable):
with log_time(self.logger):
if os.path.exists(infile):
self.logger.info("Processing %s" % (infile))
else:
self.logger.error("Dataset %s does not exist" % (infile))
return 1
try:
if not os.access(executable, os.X_OK):
raise ExecutableMissing(executable)
cmd = [executable, clusterdesc, infile, outfile]
return catch_segfaults(cmd, None, None, self.logger).returncode
except ExecutableMissing, e:
self.logger.error("%s not found" % (e.args[0]))
return 1
except CalledProcessError, e:
# For CalledProcessError isn't properly propagated by IPython
# Temporary workaround...
self.logger.error(str(e))
self.logger.info(
"A common cause for this failure is the usage"
"of an incorrect cluster.desc file in the pipeline.cfg")
return 1
def run(self, infile, executable, args, kwargs, work_dir='/tmp', parsetasfile=True, args_format='', environment=''):
"""
This method contains all the needed functionality
"""
# Debugging info
self.logger.debug("infile = %s" % infile)
self.logger.debug("executable = %s" % executable)
self.logger.debug("working directory = %s" % work_dir)
self.logger.debug("arguments = %s" % args)
self.logger.debug("arg dictionary = %s" % kwargs)
self.logger.debug("environment = %s" % environment)
self.environment.update(environment)
self.work_dir = work_dir
self.infile = infile
self.executable = executable
self.msout_original = kwargs['msout'].rstrip('/')
kwargs.pop('msout')
self.msout_destination_dir = os.path.dirname(self.msout_original)
# Set up scratch paths
scratch_dir = kwargs['local_scratch_dir']
kwargs.pop('local_scratch_dir')
try:
os.mkdir(scratch_dir)
except OSError:
pass
self.scratch_dir = tempfile.mkdtemp(dir=scratch_dir)
self.logger.info('Using {} as scratch directory'.format(self.scratch_dir))
self.msout_scratch = os.path.join(self.scratch_dir, os.path.basename(self.msout_original))
args.append('msout=' + self.msout_scratch)
# Time execution of this job
with log_time(self.logger):
#if os.path.exists(infile):
self.logger.info("Processing %s" % infile)
# Check if script is present
if not os.path.isfile(executable):
self.logger.error("Executable %s not found" % executable)
return 1
# hurray! race condition when running with more than one process on one filesystem
if not os.path.isdir(work_dir):
try:
os.mkdir(work_dir, )
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(work_dir):
pass
else:
raise
argsformat = args_format['args_format']
if not parsetasfile:
if argsformat == 'gnu':
for k, v in kwargs.items():
args.append('--' + k + '=' + v)
if argsformat == 'lofar':
for k, v in kwargs.items():
args.append(k + '=' + v)
if argsformat == 'argparse':
for k, v in kwargs.items():
args.append('--' + k + ' ' + v)
if argsformat == 'wsclean':
for k, v in kwargs.items():
multargs = v.split(' ')
if multargs:
multargs.reverse()
for item in multargs:
args.insert(0, item)
else:
args.insert(0, v)
args.insert(0, '-'+ k)
else:
nodeparset = Parset()
parsetname = os.path.join(work_dir, os.path.basename(infile) + '.parset')
for k, v in kwargs.items():
nodeparset.add(k, v)
nodeparset.writeFile(parsetname)
args.insert(0, parsetname)
try:
# ****************************************************************
#Run
cmd = [executable] + args
with CatchLog4CPlus(
work_dir,
self.logger.name + "." + os.path.basename(infile),
os.path.basename(executable),
) as logger:
# Catch segfaults and retry
catch_segfaults(
cmd, work_dir, self.environment, logger
)
except CalledProcessError, err:
# CalledProcessError isn't properly propagated by IPython
self.logger.error(str(err))
self.cleanup()
return 1
except Exception, err:
self.logger.error(str(err))
self.cleanup()
return 1
def run(self,
infile,
executable,
args,
kwargs,
work_dir='/tmp',
parsetasfile=True,
args_format='',
environment=''):
"""
This method contains all the needed functionality
"""
# Debugging info
self.logger.debug("infile = %s" % infile)
self.logger.debug("executable = %s" % executable)
self.logger.debug("working directory = %s" % work_dir)
self.logger.debug("arguments = %s" % args)
self.logger.debug("arg dictionary = %s" % kwargs)
self.logger.debug("environment = %s" % environment)
self.environment.update(environment)
self.work_dir = work_dir
self.infile = infile
self.executable = executable
if 'replace-sourcedb' in kwargs:
self.replace_sourcedb = kwargs['replace-sourcedb']
kwargs.pop('replace-sourcedb')
if 'replace-parmdb' in kwargs:
self.replace_parmdb = kwargs['replace-parmdb']
kwargs.pop('replace-parmdb')
if 'dry-run' in kwargs:
self.dry_run = kwargs['dry-run']
kwargs.pop('dry-run')
if 'sourcedb' in kwargs:
self.sourcedb = kwargs['sourcedb']
kwargs.pop('sourcedb')
if 'parmdb' in kwargs:
self.parmdb = kwargs['parmdb']
kwargs.pop('parmdb')
if 'sourcedb-name' in kwargs:
self.sourcedb_basename = kwargs['sourcedb-name']
self.replace_sourcedb = True
kwargs.pop('sourcedb-name')
if 'parmdb-name' in kwargs:
self.parmdb_basename = kwargs['parmdb-name']
self.replace_parmdb = True
kwargs.pop('parmdb-name')
if 'force' in kwargs:
self.replace_parmdb = True
self.replace_sourcedb = True
kwargs.pop('force')
numthreads = 1
if 'numthreads' in kwargs:
numthreads = kwargs['numthreads']
kwargs.pop('numthreads')
args.append('--numthreads=' + str(numthreads))
if 'observation' in kwargs:
self.observation = kwargs.pop('observation')
if 'catalog' in kwargs:
self.catalog = kwargs.pop('catalog')
self.createsourcedb()
self.createparmdb()
if not 'no-columns' in kwargs:
#if not kwargs['no-columns']:
self.addcolumns()
else:
kwargs.pop('no-columns')
args.append('--sourcedb=' + self.sourcedb_path)
args.append('--parmdb=' + self.parmdb_path)
args.append(self.observation)
#catalog = None
# Time execution of this job
with log_time(self.logger):
#if os.path.exists(infile):
self.logger.info("Processing %s" % infile)
# Check if script is present
if not os.path.isfile(executable):
self.logger.error("Executable %s not found" % executable)
return 1
# hurray! race condition when running with more than one process on one filesystem
if not os.path.isdir(work_dir):
try:
os.mkdir(work_dir, )
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(work_dir):
pass
else:
raise
if parsetasfile:
nodeparset = Parset()
parsetname = os.path.join(work_dir,
os.path.basename(infile) + '.parset')
for k, v in list(kwargs.items()):
nodeparset.add(k, v)
nodeparset.writeFile(parsetname)
#args.insert(0, parsetname)
args.append(parsetname)
#if catalog is not None:
# args.append(catalog)
try:
# ****************************************************************
#Run
cmd = [executable] + args
with CatchLog4CPlus(
work_dir,
self.logger.name + "." + os.path.basename(infile),
os.path.basename(executable),
) as logger:
# Catch segfaults and retry
catch_segfaults(cmd, work_dir, self.environment, logger)
except CalledProcessError as err:
# CalledProcessError isn't properly propagated by IPython
self.logger.error(str(err))
return 1
except Exception as err:
self.logger.error(str(err))
return 1
# We need some signal to the master script that the script ran ok.
self.outputs['ok'] = True
return 0
def run(self,
infile,
executable,
args,
kwargs,
work_dir='/tmp',
parsetasfile=True,
args_format='',
environment=''):
"""
This method contains all the needed functionality
"""
# Debugging info
self.logger.debug("infile = %s" % infile)
self.logger.debug("executable = %s" % executable)
self.logger.debug("working directory = %s" % work_dir)
self.logger.debug("arguments = %s" % args)
self.logger.debug("arg dictionary = %s" % kwargs)
self.logger.debug("environment = %s" % environment)
self.environment.update(environment)
self.work_dir = work_dir
self.infile = infile
self.executable = executable
self.msout_original = kwargs['msout'].rstrip('/')
kwargs.pop('msout')
self.msout_destination_dir = os.path.dirname(self.msout_original)
self.scratch_dir = tempfile.mkdtemp(dir=kwargs['local_scratch_dir'])
kwargs.pop('local_scratch_dir')
self.logger.info('Using {} as scratch directory'.format(
self.scratch_dir))
# Set up scratch paths
self.msout_scratch = os.path.join(
self.scratch_dir, os.path.basename(self.msout_original))
args.append('msout=' + self.msout_scratch)
# Time execution of this job
with log_time(self.logger):
#if os.path.exists(infile):
self.logger.info("Processing %s" % infile)
# Check if script is present
if not os.path.isfile(executable):
self.logger.error("Executable %s not found" % executable)
return 1
# hurray! race condition when running with more than one process on one filesystem
if not os.path.isdir(work_dir):
try:
os.mkdir(work_dir, )
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(work_dir):
pass
else:
raise
argsformat = args_format['args_format']
if not parsetasfile:
if argsformat == 'gnu':
for k, v in kwargs.items():
args.append('--' + k + '=' + v)
if argsformat == 'lofar':
for k, v in kwargs.items():
args.append(k + '=' + v)
if argsformat == 'argparse':
for k, v in kwargs.items():
args.append('--' + k + ' ' + v)
if argsformat == 'wsclean':
for k, v in kwargs.items():
multargs = v.split(' ')
if multargs:
multargs.reverse()
for item in multargs:
args.insert(0, item)
else:
args.insert(0, v)
args.insert(0, '-' + k)
else:
nodeparset = Parset()
parsetname = os.path.join(work_dir,
os.path.basename(infile) + '.parset')
for k, v in kwargs.items():
nodeparset.add(k, v)
nodeparset.writeFile(parsetname)
args.insert(0, parsetname)
try:
# ****************************************************************
#Run
cmd = [executable] + args
with CatchLog4CPlus(
work_dir,
self.logger.name + "." + os.path.basename(infile),
os.path.basename(executable),
) as logger:
# Catch segfaults and retry
catch_segfaults(cmd, work_dir, self.environment, logger)
except CalledProcessError, err:
# CalledProcessError isn't properly propagated by IPython
self.logger.error(str(err))
self.cleanup()
return 1
except Exception, err:
self.logger.error(str(err))
self.cleanup()
return 1
def run(self, infile, executable, args, kwargs, work_dir='/tmp', parsetasfile=False, args_format='', environment=''):
"""
This method contains all the needed functionality
"""
# Debugging info
self.logger.debug("infile = %s" % infile)
self.logger.debug("executable = %s" % executable)
self.logger.debug("working directory = %s" % work_dir)
self.logger.debug("arguments = %s" % args)
self.logger.debug("arg dictionary = %s" % kwargs)
self.logger.debug("environment = %s" % environment)
self.environment.update(environment)
# Time execution of this job
with log_time(self.logger):
#if os.path.exists(infile):
self.logger.info("Processing %s" % infile)
# else:
# self.logger.error("Dataset %s does not exist" % infile)
# return 1
# Check if executable is present
if not os.access(executable, os.X_OK):
self.logger.error("Executable %s not found" % executable)
return 1
# hurray! race condition when running with more than one process on one filesystem
if not os.path.isdir(work_dir):
try:
os.mkdir(work_dir, )
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(work_dir):
pass
else:
raise
argsformat = args_format['args_format']
# deal with multiple input files for wsclean
if argsformat == 'wsclean':
for i in reversed(xrange(len(args))):
if str(args[i]).startswith('[') and str(args[i]).endswith(']'):
tmplist = args.pop(i).lstrip('[').rstrip(']').split(',')
for val in reversed(tmplist):
args.insert(i, val.strip(' \'\"'))
if not parsetasfile:
if argsformat == 'gnu':
for k, v in kwargs.items():
args.append('--' + k + '=' + v)
if argsformat == 'lofar':
for k, v in kwargs.items():
args.append(k + '=' + v)
if argsformat == 'argparse':
for k, v in kwargs.items():
args.append('--' + k + ' ' + v)
if argsformat == 'wsclean':
for k, v in kwargs.items():
if str(v).startswith('[') and str(v).endswith(']'):
v = v.lstrip('[').rstrip(']').replace(' ', '')
multargs = v.split(',')
else:
multargs = v.split(' ')
if multargs:
multargs.reverse()
for item in multargs:
args.insert(0, item)
else:
args.insert(0, v)
args.insert(0, '-'+ k)
else:
nodeparset = Parset()
parsetname = os.path.join(work_dir, os.path.basename(infile) + '.parset')
for k, v in kwargs.items():
nodeparset.add(k, v)
nodeparset.writeFile(parsetname)
if argsformat == 'losoto':
args.append(parsetname)
else:
args.insert(0,parsetname)
try:
# ****************************************************************
# Run
cmd = [executable] + args
with CatchLog4CPlus(
work_dir,
self.logger.name + "." + os.path.basename(infile),
os.path.basename(executable),
) as logger:
# Catch segfaults and retry
catch_segfaults(
cmd, work_dir, self.environment, logger
)
except CalledProcessError, err:
# CalledProcessError isn't properly propagated by IPython
self.logger.error(str(err))
return 1
except Exception, err:
self.logger.error(str(err))
return 1
def run(self, executable, environment, parset, working_directory,
output_image, concatenated_measurement_set, sourcedb_path,
mask_patch_size, autogenerate_parameters, specify_fov, fov,
major_cycle, nr_cycles, perform_self_cal):
"""
:param executable: Path to awimager executable
:param environment: environment for catch_segfaults (executable runner)
:param parset: parameters for the awimager,
:param working_directory: directory the place temporary files
:param output_image: location and filesname to story the output images
the multiple images are appended with type extentions
:param concatenated_measurement_set: Input measurement set
:param sourcedb_path: Path the the sourcedb used to create the image
mask
:param mask_patch_size: Scaling of the patch around the source in the
mask
:param autogenerate_parameters: Turns on the autogeneration of:
cellsize, npix, wprojplanes, wmax, fov
:param fov: if autogenerate_parameters is false calculate
imageparameter (cellsize, npix, wprojplanes, wmax) relative to this
fov
:param major_cycle: number of the self calibration cycle to determine
the imaging parameters: cellsize, npix, wprojplanes, wmax, fov
:param nr_cycles: The requested number of self cal cycles
:param perform_self_cal: Bool used to control the selfcal functionality
or the old semi-automatic functionality
:rtype: self.outputs["image"] The path to the output image
"""
self.logger.info("Start selfcal_awimager node run:")
log4_cplus_name = "selfcal_awimager"
self.environment.update(environment)
with log_time(self.logger):
# Read the parameters as specified in the parset
parset_object = get_parset(parset)
# *************************************************************
# 1. Calculate awimager parameters that depend on measurement set
# and the parset
if perform_self_cal:
# Calculate awimager parameters that depend on measurement set
# and the parset
self.logger.info(
"Calculating selfcalibration parameters ")
cell_size, npix, w_max, w_proj_planes, \
UVmin, UVmax, robust, threshold =\
self._get_selfcal_parameters(
concatenated_measurement_set,
parset, major_cycle, nr_cycles)
self._save_selfcal_info(concatenated_measurement_set,
major_cycle, npix, UVmin, UVmax)
else:
self.logger.info(
"Calculating parameters.. ( NOT selfcalibration)")
cell_size, npix, w_max, w_proj_planes = \
self._get_imaging_parameters(
concatenated_measurement_set,
parset,
autogenerate_parameters,
specify_fov,
fov)
self.logger.info("Using autogenerated parameters; ")
self.logger.info(
"Calculated parameters: cell_size: {0}, npix: {1}".format(
cell_size, npix))
self.logger.info("w_max: {0}, w_proj_planes: {1} ".format(
w_max, w_proj_planes))
# ****************************************************************
# 2. Get the target image location from the mapfile for the parset.
# Create target dir if it not exists
image_path_head = os.path.dirname(output_image)
create_directory(image_path_head)
self.logger.debug("Created directory to place awimager output"
" files: {0}".format(image_path_head))
# ****************************************************************
# 3. Create the mask
#mask_file_path = self._create_mask(npix, cell_size, output_image,
# concatenated_measurement_set, executable,
# working_directory, log4_cplus_name, sourcedb_path,
# mask_patch_size, image_path_head)
# *****************************************************************
# 4. Update the parset with calculated parameters, and output image
patch_dictionary = {'uselogger': 'True', # enables log4cpluscd log
'ms': str(concatenated_measurement_set),
'cellsize': str(cell_size),
'npix': str(npix),
'wmax': str(w_max),
'wprojplanes': str(w_proj_planes),
'image': str(output_image),
'maxsupport': str(npix)
# 'mask':str(mask_file_path), #TODO REINTRODUCE
# MASK, excluded to speed up in this debug stage
}
# Add some aditional keys from the self calibration method
if perform_self_cal:
self_cal_patch_dict = {
'weight': 'briggs',
'padding': str(1.18),
'niter' : str(1000000),
'operation' : 'mfclark',
'timewindow' : '300',
'fits' : '',
'threshold' : str(threshold),
'robust' : str(robust),
'UVmin' : str(UVmin),
'UVmax' : str(UVmax),
'maxbaseline' : str(10000000),
'select' : str("sumsqr(UVW[:2])<1e12"),
}
patch_dictionary.update(self_cal_patch_dict)
# save the parset at the target dir for the image
calculated_parset_path = os.path.join(image_path_head,
"parset.par")
try:
temp_parset_filename = patch_parset(parset, patch_dictionary)
# Copy tmp file to the final location
shutil.copyfile(temp_parset_filename, calculated_parset_path)
self.logger.debug("Wrote parset for awimager run: {0}".format(
calculated_parset_path))
finally:
# remove temp file
os.remove(temp_parset_filename)
# *****************************************************************
# 5. Run the awimager with the parameterset
cmd = [executable, calculated_parset_path]
self.logger.debug("Parset used for awimager run:")
self.logger.debug(cmd)
try:
with CatchLog4CPlus(working_directory,
self.logger.name + "." +
os.path.basename(log4_cplus_name),
os.path.basename(executable)
) as logger:
catch_segfaults(cmd, working_directory, self.environment,
logger, usageStats=self.resourceMonitor)
# Thrown by catch_segfault
except CalledProcessError, exception:
self.logger.error(str(exception))
return 1
except Exception, exception:
self.logger.error(str(exception))
return 1
def run(self, infile, executable, args, kwargs, work_dir='/tmp', parsetasfile=False, args_format='', environment=''):
"""
This function contains all the needed functionality
"""
# Debugging info
self.logger.debug("infile = %s" % infile)
self.logger.debug("executable = %s" % executable)
self.logger.debug("working directory = %s" % work_dir)
self.logger.debug("arguments = %s" % args)
self.logger.debug("arg dictionary = %s" % kwargs)
self.logger.debug("environment = %s" % environment)
self.environment.update(environment)
# Time execution of this job
with log_time(self.logger):
if infile[0] == '[':
infiles = [ms.strip(" []\'\"") for ms in infile.split(',')]
reffile = infiles[0]
else:
reffile = infile
if os.path.exists(reffile):
self.logger.info("Processing %s" % reffile)
else:
self.logger.error("Dataset %s does not exist" % reffile)
return 1
# Check if executable is present
if not os.access(executable, os.X_OK):
self.logger.error("Executable %s not found" % executable)
return 1
# race condition when running with more than one process on one filesystem
if not os.path.isdir(work_dir):
try:
os.mkdir(work_dir, )
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(work_dir):
pass
else:
raise
if not parsetasfile:
self.logger.error("Nodescript \"executable_casa.py\" requires \"parsetasfile\" to be True!")
return 1
else:
nodeparset = Parset()
sublist = []
for k, v in kwargs.items():
nodeparset.add(k, v)
if str(k).find('.'):
if not str(k).split('.')[0] in sublist:
sublist.append(str(k).split('.')[0])
#quick hacks below. for proof of concept.
casastring = ''
for sub in sublist:
subpar = nodeparset.makeSubset(nodeparset.fullModuleName(sub) + '.')
casastring = sub + '('
for k in subpar.keys():
if str(subpar[k]).find('/') == 0:
casastring += str(k) + '=' + "'" + str(subpar[k]) + "'" + ','
elif str(subpar[k]).find('casastr/') == 0:
casastring += str(k) + '=' + "'" + str(subpar[k]).strip('casastr/') + "'" + ','
elif str(subpar[k]).lower() == 'false' or str(subpar[k]).lower() == 'true':
casastring += str(k) + '=' + str(subpar[k]) + ','
else:
# Test if int/float or list of int/float
try:
self.logger.info('value: {}'.format(subpar[k]))
test = float(str(subpar[k]))
is_int_float = True
except:
is_int_float = False
if is_int_float:
casastring += str(k) + '=' + str(subpar[k]) + ','
else:
if '[' in str(subpar[k]) or '(' in str(subpar[k]):
# Check if list of int/float or strings
list_vals = [f.strip() for f in str(subpar[k]).strip('[]()').split(',')]
is_int_float = True
for list_val in list_vals:
try:
test = float(list_val)
except:
is_int_float = False
break
if is_int_float:
casastring += str(k) + '=' + str(subpar[k]) + ','
else:
casastring += str(k) + '=' + '[{}]'.format(','.join(["'"+list_val+"'" for list_val in list_vals])) + ','
else:
# Simple string
casastring += str(k) + '=' + "'" + str(subpar[k]) + "'" + ','
casastring = casastring.rstrip(',')
casastring += ')\n'
# 1) return code of a casapy is not properly recognized by the pipeline
# wrapping in shellscript works for succesful runs.
# failed runs seem to hang the pipeline...
# 2) casapy can not have two instances running from the same directory.
# create tmp dirs
casapydir = tempfile.mkdtemp(dir=work_dir)
if casastring != '':
casafilename = os.path.join(work_dir, os.path.basename(reffile) + '.casacommand.py')
casacommandfile = open(casafilename, 'w')
casacommandfile.write(casastring)
casacommandfile.close()
args.append(casafilename)
somename = os.path.join(work_dir, os.path.basename(reffile) + '.casashell.sh')
commandstring = ''
commandstring += executable
for item in args:
if str(item).find(' ') > -1 or str(item).find('[') > -1:
commandstring += ' "' + item + '"'
else:
commandstring += ' ' + item
crap = open(somename, 'w')
crap.write('#!/bin/bash \n')
crap.write('echo "Trying CASAPY command" \n')
crap.write(commandstring + ' >& casa.log\n')
crap.close()
# file permissions
st = os.stat(somename)
os.chmod(somename, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
try:
# ****************************************************************
# Run
cmd = [somename]
with CatchLog4CPlus(
casapydir,
self.logger.name + "." + os.path.basename(reffile),
os.path.basename(executable),
) as logger:
# Catch segfaults and retry
catch_segfaults(
cmd, casapydir, self.environment, logger
)
except CalledProcessError, err:
# CalledProcessError isn't properly propagated by IPython
self.logger.error(str(err))
return 1
except Exception, err:
self.logger.error(str(err))
return 1
def run(self, executable, environment, parset, working_directory,
output_image, concatenated_measurement_set, sourcedb_path,
mask_patch_size, autogenerate_parameters, specify_fov, fov,
major_cycle, nr_cycles, perform_self_cal):
"""
:param executable: Path to awimager executable
:param environment: environment for catch_segfaults (executable runner)
:param parset: parameters for the awimager,
:param working_directory: directory the place temporary files
:param output_image: location and filesname to story the output images
the multiple images are appended with type extentions
:param concatenated_measurement_set: Input measurement set
:param sourcedb_path: Path the the sourcedb used to create the image
mask
:param mask_patch_size: Scaling of the patch around the source in the
mask
:param autogenerate_parameters: Turns on the autogeneration of:
cellsize, npix, wprojplanes, wmax, fov
:param fov: if autogenerate_parameters is false calculate
imageparameter (cellsize, npix, wprojplanes, wmax) relative to this
fov
:param major_cycle: number of the self calibration cycle to determine
the imaging parameters: cellsize, npix, wprojplanes, wmax, fov
:param nr_cycles: The requested number of self cal cycles
:param perform_self_cal: Bool used to control the selfcal functionality
or the old semi-automatic functionality
:rtype: self.outputs["image"] The path to the output image
"""
self.logger.info("Start selfcal_awimager node run:")
log4_cplus_name = "selfcal_awimager"
self.environment.update(environment)
with log_time(self.logger):
# Read the parameters as specified in the parset
parset_object = get_parset(parset)
# *************************************************************
# 1. Calculate awimager parameters that depend on measurement set
# and the parset
if perform_self_cal:
# Calculate awimager parameters that depend on measurement set
# and the parset
self.logger.info(
"Calculating selfcalibration parameters ")
cell_size, npix, w_max, w_proj_planes, \
UVmin, UVmax, robust, threshold =\
self._get_selfcal_parameters(
concatenated_measurement_set,
parset, major_cycle, nr_cycles)
self._save_selfcal_info(concatenated_measurement_set,
major_cycle, npix, UVmin, UVmax)
else:
self.logger.info(
"Calculating parameters.. ( NOT selfcalibration)")
cell_size, npix, w_max, w_proj_planes = \
self._get_imaging_parameters(
concatenated_measurement_set,
parset,
autogenerate_parameters,
specify_fov,
fov)
self.logger.info("Using autogenerated parameters; ")
self.logger.info(
"Calculated parameters: cell_size: {0}, npix: {1}".format(
cell_size, npix))
self.logger.info("w_max: {0}, w_proj_planes: {1} ".format(
w_max, w_proj_planes))
# ****************************************************************
# 2. Get the target image location from the mapfile for the parset.
# Create target dir if it not exists
image_path_head = os.path.dirname(output_image)
create_directory(image_path_head)
self.logger.debug("Created directory to place awimager output"
" files: {0}".format(image_path_head))
# ****************************************************************
# 3. Create the mask
#mask_file_path = self._create_mask(npix, cell_size, output_image,
# concatenated_measurement_set, executable,
# working_directory, log4_cplus_name, sourcedb_path,
# mask_patch_size, image_path_head)
# *****************************************************************
# 4. Update the parset with calculated parameters, and output image
patch_dictionary = {'uselogger': 'True', # enables log4cpluscd log
'ms': str(concatenated_measurement_set),
'cellsize': str(cell_size),
'npix': str(npix),
'wmax': str(w_max),
'wprojplanes': str(w_proj_planes),
'image': str(output_image),
'maxsupport': str(npix)
# 'mask':str(mask_file_path), #TODO REINTRODUCE
# MASK, excluded to speed up in this debug stage
}
# Add some aditional keys from the self calibration method
if perform_self_cal:
self_cal_patch_dict = {
'weight': 'briggs',
'padding': str(1.18),
'niter' : str(1000000),
'operation' : 'mfclark',
'timewindow' : '300',
'fits' : '',
'threshold' : str(threshold),
'robust' : str(robust),
'UVmin' : str(UVmin),
'UVmax' : str(UVmax),
'maxbaseline' : str(10000000),
'select' : str("sumsqr(UVW[:2])<1e12"),
}
patch_dictionary.update(self_cal_patch_dict)
# save the parset at the target dir for the image
calculated_parset_path = os.path.join(image_path_head,
"parset.par")
try:
temp_parset_filename = patch_parset(parset, patch_dictionary)
# Copy tmp file to the final location
shutil.copyfile(temp_parset_filename, calculated_parset_path)
self.logger.debug("Wrote parset for awimager run: {0}".format(
calculated_parset_path))
finally:
# remove temp file
os.remove(temp_parset_filename)
# *****************************************************************
# 5. Run the awimager with the parameterset
cmd = [executable, calculated_parset_path]
self.logger.debug("Parset used for awimager run:")
self.logger.debug(cmd)
try:
with CatchLog4CPlus(working_directory,
self.logger.name + "." +
os.path.basename(log4_cplus_name),
os.path.basename(executable)
) as logger:
catch_segfaults(cmd, working_directory, self.environment,
logger, usageStats=self.resourceMonitor)
# Thrown by catch_segfault
except CalledProcessError as exception:
self.logger.error(str(exception))
return 1
except Exception as exception:
self.logger.error(str(exception))
return 1
# *********************************************************************
# 6. Return output
# Append static .restored: This might change but prob. not
# The actual output image has this extention always, default of
# awimager
self.outputs["image"] = output_image + ".restored"
return 0
def run(self, executable, environment, parset, working_directory,
output_image, concatenated_measurement_set, sourcedb_path,
mask_patch_size, autogenerate_parameters, specify_fov, fov):
"""
:param executable: Path to awimager executable
:param environment: environment for catch_segfaults (executable runner)
:param parset: parameters for the awimager,
:param working_directory: directory the place temporary files
:param output_image: location and filesname to story the output images
the multiple images are appended with type extentions
:param concatenated_measurement_set: Input measurement set
:param sourcedb_path: Path the the sourcedb used to create the image
mask
:param mask_patch_size: Scaling of the patch around the source in the
mask
:param autogenerate_parameters: Turns on the autogeneration of:
cellsize, npix, wprojplanes, wmax, fov
:param fov: if autogenerate_parameters is false calculate
imageparameter (cellsize, npix, wprojplanes, wmax) relative to this
fov
:rtype: self.outputs["image"] The path to the output image
"""
self.logger.info("Start imager_awimager node run:")
log4_cplus_name = "imager_awimager"
self.environment.update(environment)
with log_time(self.logger):
# Read the parameters as specified in the parset
parset_object = get_parset(parset)
#******************************************************************
# 0. Create the directories used in this recipe
create_directory(working_directory)
# *************************************************************
# 1. Calculate awimager parameters that depend on measurement set
# and the parset
cell_size, npix, w_max, w_proj_planes = \
self._get_imaging_parameters(
concatenated_measurement_set,
parset,
autogenerate_parameters,
specify_fov,
fov)
self.logger.info("Using autogenerated parameters; ")
self.logger.info(
"Calculated parameters: cell_size: {0}, npix: {1}".format(
cell_size, npix))
self.logger.info("w_max: {0}, w_proj_planes: {1} ".format(
w_max, w_proj_planes))
# ****************************************************************
# 2. Get the target image location from the mapfile for the parset.
# Create target dir if it not exists
image_path_head = os.path.dirname(output_image)
create_directory(image_path_head)
self.logger.debug("Created directory to place awimager output"
" files: {0}".format(image_path_head))
# ****************************************************************
# 3. Create the mask
mask_file_path = self._create_mask(npix, cell_size, output_image,
concatenated_measurement_set,
executable, working_directory,
log4_cplus_name, sourcedb_path,
mask_patch_size,
image_path_head)
# *****************************************************************
# 4. Update the parset with calculated parameters, and output image
patch_dictionary = {
'uselogger': 'True', # enables log4cpluscd log
'ms': str(concatenated_measurement_set),
'cellsize': str(cell_size),
'npix': str(npix),
'wmax': str(w_max),
'wprojplanes': str(w_proj_planes),
'image': str(output_image),
'maxsupport': str(npix),
# 'mask':str(mask_file_path), #TODO REINTRODUCE
# MASK, excluded to speed up in this debug stage
}
# save the parset at the target dir for the image
calculated_parset_path = os.path.join(image_path_head,
"parset.par")
try:
temp_parset_filename = patch_parset(parset, patch_dictionary)
# Copy tmp file to the final location
shutil.copyfile(temp_parset_filename, calculated_parset_path)
self.logger.debug("Wrote parset for awimager run: {0}".format(
calculated_parset_path))
finally:
# remove temp file
os.remove(temp_parset_filename)
# *****************************************************************
# 5. Run the awimager with the updated parameterset
cmd = [executable, calculated_parset_path]
try:
with CatchLog4CPlus(
working_directory, self.logger.name + "." +
os.path.basename(log4_cplus_name),
os.path.basename(executable)) as logger:
catch_segfaults(cmd,
working_directory,
self.environment,
logger,
usageStats=self.resourceMonitor)
# Thrown by catch_segfault
except CalledProcessError, exception:
self.logger.error(str(exception))
return 1
except Exception, exception:
self.logger.error(str(exception))
return 1
def run(self, infile, executable, args, kwargs, work_dir='/tmp', parsetasfile=True, args_format='', environment=''):
"""
This method contains all the needed functionality
"""
# Debugging info
self.logger.debug("infile = %s" % infile)
self.logger.debug("executable = %s" % executable)
self.logger.debug("working directory = %s" % work_dir)
self.logger.debug("arguments = %s" % args)
self.logger.debug("arg dictionary = %s" % kwargs)
self.logger.debug("environment = %s" % environment)
self.environment.update(environment)
self.work_dir = work_dir
self.infile = infile
self.executable = executable
if 'replace-sourcedb' in kwargs:
self.replace_sourcedb = kwargs['replace-sourcedb']
kwargs.pop('replace-sourcedb')
if 'replace-parmdb' in kwargs:
self.replace_parmdb = kwargs['replace-parmdb']
kwargs.pop('replace-parmdb')
if 'dry-run' in kwargs:
self.dry_run = kwargs['dry-run']
kwargs.pop('dry-run')
if 'sourcedb' in kwargs:
self.sourcedb = kwargs['sourcedb']
kwargs.pop('sourcedb')
if 'parmdb' in kwargs:
self.parmdb = kwargs['parmdb']
kwargs.pop('parmdb')
if 'sourcedb-name' in kwargs:
self.sourcedb_basename = kwargs['sourcedb-name']
self.replace_sourcedb = True
kwargs.pop('sourcedb-name')
if 'parmdb-name' in kwargs:
self.parmdb_basename = kwargs['parmdb-name']
self.replace_parmdb = True
kwargs.pop('parmdb-name')
if 'force' in kwargs:
self.replace_parmdb = True
self.replace_sourcedb = True
kwargs.pop('force')
numthreads = 1
if 'numthreads' in kwargs:
numthreads = kwargs['numthreads']
kwargs.pop('numthreads')
args.append('--numthreads='+str(numthreads))
if 'observation' in kwargs:
self.observation = kwargs.pop('observation')
if 'catalog' in kwargs:
self.catalog = kwargs.pop('catalog')
self.createsourcedb()
self.createparmdb()
if not 'no-columns' in kwargs:
#if not kwargs['no-columns']:
self.addcolumns()
else:
kwargs.pop('no-columns')
args.append('--sourcedb=' + self.sourcedb_path)
args.append('--parmdb=' + self.parmdb_path)
args.append(self.observation)
#catalog = None
# Time execution of this job
with log_time(self.logger):
#if os.path.exists(infile):
self.logger.info("Processing %s" % infile)
# Check if script is present
if not os.path.isfile(executable):
self.logger.error("Executable %s not found" % executable)
return 1
# hurray! race condition when running with more than one process on one filesystem
if not os.path.isdir(work_dir):
try:
os.mkdir(work_dir, )
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(work_dir):
pass
else:
raise
if parsetasfile:
nodeparset = Parset()
parsetname = os.path.join(work_dir, os.path.basename(infile) + '.parset')
for k, v in kwargs.items():
nodeparset.add(k, v)
nodeparset.writeFile(parsetname)
#args.insert(0, parsetname)
args.append(parsetname)
#if catalog is not None:
# args.append(catalog)
try:
# ****************************************************************
#Run
cmd = [executable] + args
with CatchLog4CPlus(
work_dir,
self.logger.name + "." + os.path.basename(infile),
os.path.basename(executable),
) as logger:
# Catch segfaults and retry
catch_segfaults(
cmd, work_dir, self.environment, logger
)
except CalledProcessError, err:
# CalledProcessError isn't properly propagated by IPython
self.logger.error(str(err))
return 1
except Exception, err:
self.logger.error(str(err))
return 1
class imager_finalize(LOFARnodeTCP):
"""
This script performs the folowing functions:
1. Add the image info to the casa image:
addimg.addImagingInfo (imageName, msNames, sourcedbName, minbl, maxbl)
2. Convert the image to hdf5 and fits image
3. Filling of the HDF5 root group
4. Export fits image to msss image server
5. Export sourcelist to msss server, copy the sourcelist to hdf5 location
6. Return the outputs
"""
def run(self, awimager_output, ms_per_image, sourcelist, target,
output_image, minbaseline, maxbaseline, processed_ms_dir,
fillrootimagegroup_exec, environment, sourcedb):
self.environment.update(environment)
"""
:param awimager_output: Path to the casa image produced by awimager
:param ms_per_image: The X (90) measurements set scheduled to
create the image
:param sourcelist: list of sources found in the image
:param target: <unused>
:param minbaseline: Minimum baseline used for the image
:param maxbaseline: largest/maximum baseline used for the image
:param processed_ms_dir: The X (90) measurements set actually used to
create the image
:param fillrootimagegroup_exec: Executable used to add image data to
the hdf5 image
:rtype: self.outputs['hdf5'] set to "succes" to signal node succes
:rtype: self.outputs['image'] path to the produced hdf5 image
"""
with log_time(self.logger):
ms_per_image_map = DataMap.load(ms_per_image)
# *****************************************************************
# 1. add image info
# Get all the files in the processed measurement dir
file_list = os.listdir(processed_ms_dir)
# TODO: BUG!! the meta data might contain files that were copied
# but failed in imager_bbs
processed_ms_paths = []
for item in ms_per_image_map:
path = item.file
ms_file_name = os.path.split(path)[1]
#if the ms is in the processed dir (additional check)
if (ms_file_name in file_list):
# save the path
processed_ms_paths.append(
os.path.join(processed_ms_dir, ms_file_name))
#add the information the image
try:
addimg.addImagingInfo(awimager_output, processed_ms_paths,
sourcedb, minbaseline, maxbaseline)
except Exception, error:
self.logger.warn("addImagingInfo Threw Exception:")
self.logger.warn(error)
# Catch raising of already done error: allows for rerunning
# of the recipe
if "addImagingInfo already done" in str(error):
pass
else:
raise Exception(error)
#The majority of the tables is updated correctly
# ***************************************************************
# 2. convert to hdf5 image format
output_directory = None
pim_image = pim.image(awimager_output)
try:
self.logger.info(
"Saving image in HDF5 Format to: {0}".format(output_image))
# Create the output directory
output_directory = os.path.dirname(output_image)
create_directory(output_directory)
# save the image
pim_image.saveas(output_image, hdf5=True)
except Exception, error:
self.logger.error(
"Exception raised inside pyrap.images: {0}".format(
str(error)))
raise error
# Convert to fits
# create target location
fits_output = output_image + ".fits"
# To allow reruns a possible earlier version needs to be removed!
# image2fits fails if not done!!
if os.path.exists(fits_output):
os.unlink(fits_output)
try:
temp_dir = tempfile.mkdtemp(suffix=".%s" %
(os.path.basename(__file__), ))
with CatchLog4CPlus(
temp_dir, self.logger.name + '.' +
os.path.basename(awimager_output),
"image2fits") as logger:
catch_segfaults([
"image2fits", '-in', awimager_output, '-out',
fits_output
], temp_dir, self.environment, logger)
except Exception, excp:
self.logger.error(str(excp))
return 1
def run(self,
infile,
executable,
args,
kwargs,
work_dir='/tmp',
parsetasfile=False,
args_format='',
environment=''):
"""
This method contains all the needed functionality
"""
# Debugging info
self.logger.debug("infile = %s" % infile)
self.logger.debug("executable = %s" % executable)
self.logger.debug("working directory = %s" % work_dir)
self.logger.debug("arguments = %s" % args)
self.logger.debug("arg dictionary = %s" % kwargs)
self.logger.debug("environment = %s" % environment)
self.environment.update(environment)
# Time execution of this job
with log_time(self.logger):
#if os.path.exists(infile):
self.logger.info("Processing %s" % infile)
# else:
# self.logger.error("Dataset %s does not exist" % infile)
# return 1
# Check if executable is present
if not os.access(executable, os.X_OK):
self.logger.error("Executable %s not found" % executable)
return 1
# hurray! race condition when running with more than one process on one filesystem
if not os.path.isdir(work_dir):
try:
os.mkdir(work_dir, )
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(work_dir):
pass
else:
raise
argsformat = args_format['args_format']
# deal with multiple input files for wsclean
if argsformat == 'wsclean':
for i in reversed(xrange(len(args))):
if str(args[i]).startswith('[') and str(
args[i]).endswith(']'):
tmplist = args.pop(i).lstrip('[').rstrip(']').split(
',')
for val in reversed(tmplist):
args.insert(i, val.strip(' \'\"'))
if not parsetasfile:
if argsformat == 'gnu':
for k, v in kwargs.items():
args.append('--' + k + '=' + v)
if argsformat == 'lofar':
for k, v in kwargs.items():
args.append(k + '=' + v)
if argsformat == 'argparse':
for k, v in kwargs.items():
args.append('--' + k + ' ' + v)
if argsformat == 'wsclean':
for k, v in kwargs.items():
if str(v).startswith('[') and str(v).endswith(']'):
v = v.lstrip('[').rstrip(']').replace(' ', '')
multargs = v.split(',')
else:
multargs = v.split(' ')
if multargs:
multargs.reverse()
for item in multargs:
args.insert(0, item)
else:
args.insert(0, v)
args.insert(0, '-' + k)
else:
nodeparset = Parset()
parsetname = os.path.join(work_dir,
os.path.basename(infile) + '.parset')
for k, v in kwargs.items():
nodeparset.add(k, v)
nodeparset.writeFile(parsetname)
if argsformat == 'losoto':
args.append(parsetname)
else:
args.insert(0, parsetname)
try:
# ****************************************************************
# Run
cmd = [executable] + args
with CatchLog4CPlus(
work_dir,
self.logger.name + "." + os.path.basename(infile),
os.path.basename(executable),
) as logger:
# Catch segfaults and retry
catch_segfaults(cmd, work_dir, self.environment, logger)
except CalledProcessError, err:
# CalledProcessError isn't properly propagated by IPython
self.logger.error(str(err))
return 1
except Exception, err:
self.logger.error(str(err))
return 1
def run(self, infile, outfile, parmdb, sourcedb,
parsetfile, executable, working_directory, environment, demix_always, demix_if_needed,
start_time, end_time, nthreads, clobber):
"""
This function contains all the needed functionality
"""
# Debugging info
self.logger.debug("infile = %s" % infile)
self.logger.debug("outfile = %s" % outfile)
self.logger.debug("parmdb = %s" % parmdb)
self.logger.debug("sourcedb = %s" % sourcedb)
self.logger.debug("parsetfile = %s" % parsetfile)
self.logger.debug("executable = %s" % executable)
self.logger.debug("working_dir = %s" % working_directory)
self.logger.debug("environment = %s" % environment)
self.logger.debug("demix_always = %s" % demix_always)
self.logger.debug("demix_if_needed = %s" % demix_if_needed)
self.logger.debug("start_time = %s" % start_time)
self.logger.debug("end_time = %s" % end_time)
self.logger.debug("nthreads = %s" % nthreads)
self.logger.debug("clobber = %s" % clobber)
self.environment.update(environment)
# ********************************************************************
# 1. preparations. set nthreads, Validate input, clean workspace
#
if not nthreads:
nthreads = 1
if not outfile:
outfile = infile
tmpfile = outfile + '.tmp'
# Time execution of this job
with log_time(self.logger):
if os.path.exists(infile):
self.logger.info("Processing %s" % (infile))
else:
self.logger.error("Dataset %s does not exist" % (infile))
return 1
# Check if DPPP executable is present
if not os.access(executable, os.X_OK):
self.logger.error("Executable %s not found" % executable)
return 1
# Make sure that we start with a clean slate
shutil.rmtree(tmpfile, ignore_errors=True)
if clobber:
if outfile == infile:
self.logger.warn(
"Input and output are identical, not clobbering %s" %
outfile
)
else:
self.logger.info("Removing previous output %s" % outfile)
shutil.rmtree(outfile, ignore_errors=True)
# *****************************************************************
# 2. Perform house keeping, test if work is already done
# If input and output files are different, and if output file
# already exists, then we're done.
if outfile != infile and os.path.exists(outfile):
self.logger.info(
"Output file %s already exists. We're done." % outfile
)
self.outputs['ok'] = True
return 0
# Create a working copy if input and output are identical, to
# avoid corrupting the original file if things go awry.
if outfile == infile:
self.logger.info(
"Creating working copy: %s --> %s" % (infile, tmpfile)
)
shutil.copytree(infile, tmpfile)
# *****************************************************************
# 3. Update the parset with locally calculate information
# Put arguments we need to pass to some private methods in a dict
kwargs = {
'infile' : infile,
'tmpfile' : tmpfile,
'parmdb' : parmdb,
'sourcedb' : sourcedb,
'parsetfile' : parsetfile,
'demix_always' : demix_always,
'demix_if_needed' : demix_if_needed,
'start_time' : start_time,
'end_time' : end_time
}
# Prepare for the actual DPPP run.
with patched_parset(
# *****************************************************************
# 4. Add ms names to the parset, start/end times if availabe, etc.
# 5. Add demixing parameters to the parset
parsetfile, self._prepare_steps(**kwargs), output_dir=working_directory, unlink=False
) as temp_parset_filename:
self.logger.debug("Created temporary parset file: %s" %
temp_parset_filename
)
try:
working_dir = tempfile.mkdtemp(dir=working_directory,suffix=".%s" % (os.path.basename(__file__),))
# ****************************************************************
# 6. Run ndppp
cmd = [executable, temp_parset_filename, '1']
with CatchLog4CPlus(
working_dir,
self.logger.name + "." + os.path.basename(infile),
os.path.basename(executable),
) as logger:
# Catch NDPPP segfaults (a regular occurance), and retry
catch_segfaults(
cmd, working_dir, self.environment, logger,
cleanup=lambda : shutil.rmtree(tmpfile, ignore_errors=True)
)
# Replace outfile with the updated working copy
shutil.rmtree(outfile, ignore_errors=True)
os.rename(tmpfile, outfile)
except CalledProcessError, err:
# CalledProcessError isn't properly propagated by IPython
self.logger.error(str(err))
return 1
except Exception, err:
self.logger.error(str(err))
return 1
finally:
def run(self, infile, outfile, parmdb, sourcedb, parsetfile, executable,
working_directory, environment, demix_always, demix_if_needed,
start_time, end_time, nthreads, clobber):
"""
This function contains all the needed functionality
"""
# Debugging info
self.logger.debug("infile = %s" % infile)
self.logger.debug("outfile = %s" % outfile)
self.logger.debug("parmdb = %s" % parmdb)
self.logger.debug("sourcedb = %s" % sourcedb)
self.logger.debug("parsetfile = %s" % parsetfile)
self.logger.debug("executable = %s" % executable)
self.logger.debug("working_dir = %s" % working_directory)
self.logger.debug("environment = %s" % environment)
self.logger.debug("demix_always = %s" % demix_always)
self.logger.debug("demix_if_needed = %s" % demix_if_needed)
self.logger.debug("start_time = %s" % start_time)
self.logger.debug("end_time = %s" % end_time)
self.logger.debug("nthreads = %s" % nthreads)
self.logger.debug("clobber = %s" % clobber)
self.environment.update(environment)
# ********************************************************************
# 1. preparations. set nthreads, Validate input, clean workspace
#
if not nthreads:
nthreads = 1
if not outfile:
outfile = infile
tmpfile = outfile + '.tmp'
# Time execution of this job
with log_time(self.logger):
if os.path.exists(infile):
self.logger.info("Processing %s" % (infile))
else:
self.logger.error("Dataset %s does not exist" % (infile))
return 1
# Check if DPPP executable is present
if not os.access(executable, os.X_OK):
self.logger.error("Executable %s not found" % executable)
return 1
# Make sure that we start with a clean slate
shutil.rmtree(tmpfile, ignore_errors=True)
if clobber:
if outfile == infile:
self.logger.warn(
"Input and output are identical, not clobbering %s" %
outfile)
else:
self.logger.info("Removing previous output %s" % outfile)
shutil.rmtree(outfile, ignore_errors=True)
# *****************************************************************
# 2. Perform house keeping, test if work is already done
# If input and output files are different, and if output file
# already exists, then we're done.
if outfile != infile and os.path.exists(outfile):
self.logger.info("Output file %s already exists. We're done." %
outfile)
self.outputs['ok'] = True
return 0
# Create a working copy if input and output are identical, to
# avoid corrupting the original file if things go awry.
if outfile == infile:
self.logger.info("Creating working copy: %s --> %s" %
(infile, tmpfile))
shutil.copytree(infile, tmpfile)
# *****************************************************************
# 3. Update the parset with locally calculate information
# Put arguments we need to pass to some private methods in a dict
kwargs = {
'infile': infile,
'tmpfile': tmpfile,
'parmdb': parmdb,
'sourcedb': sourcedb,
'parsetfile': parsetfile,
'demix_always': demix_always,
'demix_if_needed': demix_if_needed,
'start_time': start_time,
'end_time': end_time
}
# Prepare for the actual DPPP run.
with patched_parset(
# *****************************************************************
# 4. Add ms names to the parset, start/end times if availabe, etc.
# 5. Add demixing parameters to the parset
parsetfile,
self._prepare_steps(**kwargs),
output_dir=working_directory,
unlink=False) as temp_parset_filename:
self.logger.debug("Created temporary parset file: %s" %
temp_parset_filename)
try:
working_dir = tempfile.mkdtemp(
dir=working_directory,
suffix=".%s" % (os.path.basename(__file__), ))
# ****************************************************************
# 6. Run ndppp
cmd = [executable, temp_parset_filename, '1']
with CatchLog4CPlus(
working_dir,
self.logger.name + "." + os.path.basename(infile),
os.path.basename(executable),
) as logger:
# Catch NDPPP segfaults (a regular occurance), and retry
catch_segfaults(cmd,
working_dir,
self.environment,
logger,
cleanup=lambda: shutil.rmtree(
tmpfile, ignore_errors=True))
# Replace outfile with the updated working copy
shutil.rmtree(outfile, ignore_errors=True)
os.rename(tmpfile, outfile)
except CalledProcessError as err:
# CalledProcessError isn't properly propagated by IPython
self.logger.error(str(err))
return 1
except Exception as err:
self.logger.error(str(err))
return 1
finally:
shutil.rmtree(working_dir)
# We need some signal to the master script that the script ran ok.
self.outputs['ok'] = True
return 0
def run(self,
infile,
executable,
args,
kwargs,
work_dir='/tmp',
parsetasfile=False,
args_format='',
environment=''):
"""
This function contains all the needed functionality
"""
# Debugging info
self.logger.debug("infile = %s" % infile)
self.logger.debug("executable = %s" % executable)
self.logger.debug("working directory = %s" % work_dir)
self.logger.debug("arguments = %s" % args)
self.logger.debug("arg dictionary = %s" % kwargs)
self.logger.debug("environment = %s" % environment)
self.environment.update(environment)
# Time execution of this job
with log_time(self.logger):
if infile[0] == '[':
infiles = [ms.strip(" []\'\"") for ms in infile.split(',')]
reffile = infiles[0]
else:
reffile = infile
if os.path.exists(reffile):
self.logger.info("Processing %s" % reffile)
else:
self.logger.error("Dataset %s does not exist" % reffile)
return 1
# Check if executable is present
if not os.access(executable, os.X_OK):
self.logger.error("Executable %s not found" % executable)
return 1
# race condition when running with more than one process on one filesystem
if not os.path.isdir(work_dir):
try:
os.mkdir(work_dir, )
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(work_dir):
pass
else:
raise
if not parsetasfile:
self.logger.error(
"Nodescript \"executable_casa.py\" requires \"parsetasfile\" to be True!"
)
return 1
else:
nodeparset = Parset()
sublist = []
for k, v in kwargs.items():
nodeparset.add(k, v)
if str(k).find('.'):
if not str(k).split('.')[0] in sublist:
sublist.append(str(k).split('.')[0])
#quick hacks below. for proof of concept.
casastring = ''
for sub in sublist:
subpar = nodeparset.makeSubset(
nodeparset.fullModuleName(sub) + '.')
casastring = sub + '('
for k in subpar.keys():
if str(subpar[k]).find('/') == 0:
casastring += str(k) + '=' + "'" + str(
subpar[k]) + "'" + ','
elif str(subpar[k]).find('casastr/') == 0:
casastring += str(k) + '=' + "'" + str(
subpar[k]).strip('casastr/') + "'" + ','
elif str(subpar[k]).lower() == 'false' or str(
subpar[k]).lower() == 'true':
casastring += str(k) + '=' + str(subpar[k]) + ','
else:
# Test if int/float or list of int/float
try:
self.logger.info('value: {}'.format(subpar[k]))
test = float(str(subpar[k]))
is_int_float = True
except:
is_int_float = False
if is_int_float:
casastring += str(k) + '=' + str(
subpar[k]) + ','
else:
if '[' in str(subpar[k]) or '(' in str(
subpar[k]):
# Check if list of int/float or strings
list_vals = [
f.strip() for f in str(
subpar[k]).strip('[]()').split(',')
]
is_int_float = True
for list_val in list_vals:
try:
test = float(list_val)
except:
is_int_float = False
break
if is_int_float:
casastring += str(k) + '=' + str(
subpar[k]) + ','
else:
casastring += str(
k) + '=' + '[{}]'.format(','.join([
"'" + list_val + "'"
for list_val in list_vals
])) + ','
else:
# Simple string
casastring += str(k) + '=' + "'" + str(
subpar[k]) + "'" + ','
casastring = casastring.rstrip(',')
casastring += ')\n'
# 1) return code of a casapy is not properly recognized by the pipeline
# wrapping in shellscript works for succesful runs.
# failed runs seem to hang the pipeline...
# 2) casapy can not have two instances running from the same directory.
# create tmp dirs
casapydir = tempfile.mkdtemp(dir=work_dir)
if casastring != '':
casafilename = os.path.join(
work_dir,
os.path.basename(reffile) + '.casacommand.py')
casacommandfile = open(casafilename, 'w')
casacommandfile.write(casastring)
casacommandfile.close()
args.append(casafilename)
somename = os.path.join(
work_dir,
os.path.basename(reffile) + '.casashell.sh')
commandstring = ''
commandstring += executable
for item in args:
if str(item).find(' ') > -1 or str(item).find('[') > -1:
commandstring += ' "' + item + '"'
else:
commandstring += ' ' + item
crap = open(somename, 'w')
crap.write('#!/bin/bash \n')
crap.write('echo "Trying CASAPY command" \n')
crap.write(commandstring + ' >& casa.log\n')
crap.close()
# file permissions
st = os.stat(somename)
os.chmod(
somename,
st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
try:
# ****************************************************************
# Run
cmd = [somename]
with CatchLog4CPlus(
casapydir,
self.logger.name + "." + os.path.basename(reffile),
os.path.basename(executable),
) as logger:
# Catch segfaults and retry
catch_segfaults(cmd, casapydir, self.environment, logger)
except CalledProcessError, err:
# CalledProcessError isn't properly propagated by IPython
self.logger.error(str(err))
return 1
except Exception, err:
self.logger.error(str(err))
return 1
def run(self,
infile,
executable,
args,
kwargs,
work_dir='/tmp',
parsetasfile=False,
args_format='',
environment=''):
"""
This function contains all the needed functionality
"""
# Debugging info
self.logger.debug("infile = %s" % infile)
self.logger.debug("executable = %s" % executable)
self.logger.debug("working directory = %s" % work_dir)
self.logger.debug("arguments = %s" % args)
self.logger.debug("arg dictionary = %s" % kwargs)
self.logger.debug("environment = %s" % environment)
self.environment.update(environment)
# hack the planet
#executable = 'casa'
# Time execution of this job
with log_time(self.logger):
if os.path.exists(infile):
self.logger.info("Processing %s" % infile)
else:
self.logger.error("Dataset %s does not exist" % infile)
return 1
# Check if executable is present
if not os.access(executable, os.X_OK):
self.logger.error("Executable %s not found" % executable)
return 1
# hurray! race condition when running with than one process on one filesystem
if not os.path.isdir(work_dir):
try:
os.mkdir(work_dir, )
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(work_dir):
pass
else:
raise
#print 'KWARGS: ', kwargs
if not parsetasfile:
for k, v in kwargs.items():
args.append('--' + k + '=' + v)
else:
nodeparset = Parset()
sublist = []
for k, v in kwargs.items():
nodeparset.add(k, v)
if str(k).find('.'):
#print 'DOTPOS: ',str(k).find('.')
#print 'SPLIT: ', str(k).split('.')[0]
#print 'SPLIT: ', str(k).split('.')[1]
if not str(k).split('.')[0] in sublist:
sublist.append(str(k).split('.')[0])
#print 'SUBPARSETLIST: ', sublist
#subpar = Parset()
#quick hacks below. for proof of concept.
subparsetlist = []
casastring = ''
for sub in sublist:
subpar = nodeparset.makeSubset(
nodeparset.fullModuleName(sub) + '.')
#print 'SUBPAR: ',subpar.keys()
casastring = sub + '('
for k in subpar.keys():
#print 'SUBPARSET: ',k ,' ',subpar[k]
#args.append('--' + k + '=' + subpar[k])
if str(subpar[k]).find('/') == 0:
casastring += str(k) + '=' + "'" + str(
subpar[k]) + "'" + ','
elif str(subpar[k]).find('/casastr/') == 0:
casastring += str(k) + '=' + "'" + str(
subpar[k]).strip('/casastr/') + "'" + ','
else:
casastring += str(k) + '=' + str(subpar[k]) + ','
casastring = casastring.rstrip(',')
casastring += ')\n'
#print 'CASASTRING:'
#print casastring
# 1) return code of a casapy is not properly recognized by the pipeline
# wrapping in shellscript works for succesful runs.
# failed runs seem to hang the pipeline...
# 2) casapy can not have two instances running from the same directory.
# create tmp dirs
casapydir = tempfile.mkdtemp(dir=work_dir)
if casastring != '':
casafilename = os.path.join(
work_dir,
os.path.basename(infile) + '.casacommand.py')
casacommandfile = open(casafilename, 'w')
casacommandfile.write('try:\n')
casacommandfile.write(' ' + casastring)
casacommandfile.write('except SystemExit:\n')
casacommandfile.write(' pass\n')
casacommandfile.write('except:\n')
casacommandfile.write(' import os\n')
casacommandfile.write(' os._exit(1)\n')
casacommandfile.close()
args.append(casafilename)
somename = os.path.join(
work_dir,
os.path.basename(infile) + '.casashell.sh')
commandstring = ''
commandstring += executable
for item in args:
commandstring += ' ' + item
#print 'COMMANDSTRING: ',commandstring
crap = open(somename, 'w')
crap.write('#!/bin/bash \n')
crap.write('echo "Trying CASAPY command" \n')
#crap.write('/home/zam/sfroehli/casapy-42.1.29047-001-1-64b/bin/casa' + ' --nologger'+' -c ' + casafilename)
crap.write(commandstring)
# crap.write('\nexit 0')
crap.close()
import stat
st = os.stat(somename)
#os.chmod(casafilename, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
os.chmod(
somename,
st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
try:
# ****************************************************************
# Run
#cmd = [executable] + args
cmd = [somename]
with CatchLog4CPlus(
casapydir,
self.logger.name + "." + os.path.basename(infile),
os.path.basename(executable),
) as logger:
# Catch segfaults and retry
catch_segfaults(cmd, casapydir, self.environment, logger)
except CalledProcessError, err:
# CalledProcessError isn't properly propagated by IPython
self.logger.error(str(err))
return 1
except Exception, err:
self.logger.error(str(err))
return 1
def run(self, executable, environment, parset, working_directory,
output_image, concatenated_measurement_set, sourcedb_path,
mask_patch_size, autogenerate_parameters, specify_fov, fov):
"""
:param executable: Path to awimager executable
:param environment: environment for catch_segfaults (executable runner)
:param parset: parameters for the awimager,
:param working_directory: directory the place temporary files
:param output_image: location and filesname to story the output images
the multiple images are appended with type extentions
:param concatenated_measurement_set: Input measurement set
:param sourcedb_path: Path the the sourcedb used to create the image
mask
:param mask_patch_size: Scaling of the patch around the source in the
mask
:param autogenerate_parameters: Turns on the autogeneration of:
cellsize, npix, wprojplanes, wmax, fov
:param fov: if autogenerate_parameters is false calculate
imageparameter (cellsize, npix, wprojplanes, wmax) relative to this
fov
:rtype: self.outputs["image"] The path to the output image
"""
self.logger.info("Start imager_awimager node run:")
log4_cplus_name = "imager_awimager"
self.environment.update(environment)
with log_time(self.logger):
# Read the parameters as specified in the parset
parset_object = get_parset(parset)
# *************************************************************
# 1. Calculate awimager parameters that depend on measurement set
# and the parset
cell_size, npix, w_max, w_proj_planes = \
self._get_imaging_parameters(
concatenated_measurement_set,
parset,
autogenerate_parameters,
specify_fov,
fov)
self.logger.info("Using autogenerated parameters; ")
self.logger.info(
"Calculated parameters: cell_size: {0}, npix: {1}".format(
cell_size, npix))
self.logger.info("w_max: {0}, w_proj_planes: {1} ".format(
w_max, w_proj_planes))
# ****************************************************************
# 2. Get the target image location from the mapfile for the parset.
# Create target dir if it not exists
image_path_head = os.path.dirname(output_image)
create_directory(image_path_head)
self.logger.debug("Created directory to place awimager output"
" files: {0}".format(image_path_head))
# ****************************************************************
# 3. Create the mask
mask_file_path = self._create_mask(npix, cell_size, output_image,
concatenated_measurement_set, executable,
working_directory, log4_cplus_name, sourcedb_path,
mask_patch_size, image_path_head)
# *****************************************************************
# 4. Update the parset with calculated parameters, and output image
patch_dictionary = {'uselogger': 'True', # enables log4cpluscd log
'ms': str(concatenated_measurement_set),
'cellsize': str(cell_size),
'npix': str(npix),
'wmax': str(w_max),
'wprojplanes': str(w_proj_planes),
'image': str(output_image),
'maxsupport': str(npix),
# 'mask':str(mask_file_path), #TODO REINTRODUCE
# MASK, excluded to speed up in this debug stage
}
# save the parset at the target dir for the image
calculated_parset_path = os.path.join(image_path_head,
"parset.par")
try:
temp_parset_filename = patch_parset(parset, patch_dictionary)
# Copy tmp file to the final location
shutil.copyfile(temp_parset_filename, calculated_parset_path)
self.logger.debug("Wrote parset for awimager run: {0}".format(
calculated_parset_path))
finally:
# remove temp file
os.remove(temp_parset_filename)
# *****************************************************************
# 5. Run the awimager with the updated parameterset
cmd = [executable, calculated_parset_path]
try:
with CatchLog4CPlus(working_directory,
self.logger.name + "." +
os.path.basename(log4_cplus_name),
os.path.basename(executable)
) as logger:
catch_segfaults(cmd, working_directory, self.environment,
logger)
# Thrown by catch_segfault
except CalledProcessError, exception:
self.logger.error(str(exception))
return 1
except Exception, exception:
self.logger.error(str(exception))
return 1
def run(self, awimager_output, ms_per_image, sourcelist, target,
output_image, minbaseline, maxbaseline, processed_ms_dir,
fillrootimagegroup_exec, environment, sourcedb):
self.environment.update(environment)
"""
:param awimager_output: Path to the casa image produced by awimager
:param ms_per_image: The X (90) measurements set scheduled to
create the image
:param sourcelist: list of sources found in the image
:param target: <unused>
:param minbaseline: Minimum baseline used for the image
:param maxbaseline: largest/maximum baseline used for the image
:param processed_ms_dir: The X (90) measurements set actually used to
create the image
:param fillrootimagegroup_exec: Executable used to add image data to
the hdf5 image
:rtype: self.outputs['hdf5'] set to "succes" to signal node succes
:rtype: self.outputs['image'] path to the produced hdf5 image
"""
with log_time(self.logger):
ms_per_image_map = DataMap.load(ms_per_image)
# *****************************************************************
# 1. add image info
# Get all the files in the processed measurement dir
file_list = os.listdir(processed_ms_dir)
# TODO: BUG!! the meta data might contain files that were copied
# but failed in imager_bbs
processed_ms_paths = []
for item in ms_per_image_map:
path = item.file
ms_file_name = os.path.split(path)[1]
#if the ms is in the processed dir (additional check)
if (ms_file_name in file_list):
# save the path
processed_ms_paths.append(
os.path.join(processed_ms_dir, ms_file_name))
#add the information the image
try:
addimg.addImagingInfo(awimager_output, processed_ms_paths,
sourcedb, minbaseline, maxbaseline)
except Exception as error:
self.logger.warn("addImagingInfo Threw Exception:")
self.logger.warn(error)
# Catch raising of already done error: allows for rerunning
# of the recipe
if "addImagingInfo already done" in str(error):
pass
else:
raise Exception(error)
#The majority of the tables is updated correctly
# ***************************************************************
# 2. convert to hdf5 image format
output_directory = None
pim_image = pim.image(awimager_output)
try:
self.logger.info(
"Saving image in HDF5 Format to: {0}".format(output_image))
# Create the output directory
output_directory = os.path.dirname(output_image)
create_directory(output_directory)
# save the image
pim_image.saveas(output_image, hdf5=True)
except Exception as error:
self.logger.error(
"Exception raised inside pyrap.images: {0}".format(
str(error)))
raise error
# Convert to fits
# create target location
fits_output = output_image + ".fits"
# To allow reruns a possible earlier version needs to be removed!
# image2fits fails if not done!!
if os.path.exists(fits_output):
os.unlink(fits_output)
try:
temp_dir = tempfile.mkdtemp(suffix=".%s" %
(os.path.basename(__file__), ))
with CatchLog4CPlus(
temp_dir, self.logger.name + '.' +
os.path.basename(awimager_output),
"image2fits") as logger:
catch_segfaults([
"image2fits", '-in', awimager_output, '-out',
fits_output
], temp_dir, self.environment, logger)
except Exception as excp:
self.logger.error(str(excp))
return 1
finally:
shutil.rmtree(temp_dir)
# ****************************************************************
# 3. Filling of the HDF5 root group
command = [fillrootimagegroup_exec, output_image]
self.logger.info(" ".join(command))
#Spawn a subprocess and connect the pipes
proc = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdoutdata, stderrdata) = communicate_returning_strings(proc)
exit_status = proc.returncode
self.logger.info(stdoutdata)
self.logger.info(stderrdata)
#if copy failed log the missing file
if exit_status != 0:
self.logger.error(
"Error using the fillRootImageGroup command"
"see above lines. Exit status: {0}".format(exit_status))
return 1
# *****************************************************************
# 4 Export the fits image to the msss server
url = "http://tanelorn.astron.nl:8000/upload"
try:
self.logger.info(
"Starting upload of fits image data to server!")
opener = urllib.request.build_opener(mph.MultipartPostHandler)
filedata = {"file": open(fits_output, "rb")}
opener.open(url, filedata, timeout=2)
# HTTPError needs to be caught first.
except urllib.error.HTTPError as httpe:
self.logger.warn("HTTP status is: {0}".format(httpe.code))
self.logger.warn("failed exporting fits image to server")
except urllib.error.URLError as urle:
self.logger.warn(str(urle.reason))
self.logger.warn("failed exporting fits image to server")
except Exception as exc:
self.logger.warn(str(exc))
self.logger.warn("failed exporting fits image to server")
# *****************************************************************
# 5. export the sourcelist to the msss server
url = "http://tanelorn.astron.nl:8000/upload_srcs"
try:
# Copy file to output location
new_sourcelist_path = output_image + ".sourcelist"
if os.path.exists(new_sourcelist_path):
os.unlink(new_sourcelist_path)
shutil.copy(sourcelist, new_sourcelist_path)
self.logger.info(
"Starting upload of sourcelist data to server!")
opener = urllib.request.build_opener(mph.MultipartPostHandler)
filedata = {"file": open(new_sourcelist_path, "rb")}
opener.open(url, filedata, timeout=2)
# HTTPError needs to be caught first.
except urllib.error.HTTPError as httpe:
self.logger.warn("HTTP status is: {0}".format(httpe.code))
self.logger.warn("failed exporting sourcelist to server")
except urllib.error.URLError as urle:
self.logger.warn(str(urle.reason))
self.logger.warn("failed exporting sourcelist image to server")
except Exception as exc:
self.logger.warn(str(exc))
self.logger.warn("failed exporting sourcelist image to serve")
self.outputs["hdf5"] = "succes"
self.outputs["image"] = output_image
return 0
