def run_task(self, configblock, datafiles=[], **kwargs):
"""
A task is a combination of a recipe and a set of parameters.
Tasks can be prefedined in the task file set in the pipeline
configuration (default: tasks.cfg).
Here, we load a task configuration and execute it.
This is a "shorthand" version of
:meth:`lofarpipe.cuisine.WSRTrecipe.WSRTrecipe.cook_recipe`.
"""
self.logger.info("Running task: %s" % (configblock,))
# Does the task definition exist?
try:
recipe = self.task_definitions.get(configblock, "recipe")
except NoSectionError:
raise PipelineException(
"%s not found -- check your task definitions" % configblock
)
# Build inputs dict.
# First, take details from caller.
inputs = LOFARinput(self.inputs)
inputs['args'] = datafiles
# Add parameters from the task file.
# Note that we neither need the recipe name nor any items from the
# DEFAULT config.
parameters = dict(self.task_definitions.items(configblock))
del parameters['recipe']
for key in list(dict(self.config.items("DEFAULT")).keys()):
del parameters[key]
inputs.update(parameters)
# Update inputs with provided kwargs, if any.
inputs.update(kwargs)
# Default outputs dict.
outputs = LOFARoutput()
# Cook the recipe and return the results"
try:
self.cook_recipe(recipe, inputs, outputs)
except CookError:
self.logger.warn(
"%s reports failure (using %s recipe)" % (configblock, recipe)
)
raise PipelineRecipeFailed("%s failed" % configblock)
# Get the (optional) node xml information
if "return_xml" in outputs:
return_node = xml.parseString(
outputs['return_xml']).documentElement
# If no active stack, fail silently.
add_child_to_active_stack_head(self, return_node)
return outputs
def _convert_data_to_ComplexArray(self, data, type_pair):
"""
Performs a conversion of a 2d array to a 1d complex valued array.
with real/imag values or with amplitude phase values
"""
if sorted(type_pair) == sorted(RealImagArray.keys):
# The type_pair is in alphabetical order: Imag on index 0
complex_array = RealImagArray(data[1]["values"], data[0]["values"])
elif sorted(type_pair) == sorted(AmplPhaseArray.keys):
complex_array = AmplPhaseArray(data[0]["values"], data[1]["values"])
else:
self.logger.error("Incorrect data type pair provided: {0}".format(
type_pair))
raise PipelineRecipeFailed(
"Invalid data type retrieved from parmdb: {0}".format(type_pair))
return complex_array
def _filter_stations_parmdb(self, infile, outfile, sigma):
"""
Performs a gain outlier correction of the infile parmdb with
the corrected parmdb written to outfile.
Outliers in the gain with a distance of median of sigma times std
are replaced with the mean. The last value of the complex array
is skipped (John Swinbank: "I found it [the last value] was bad when
I hacked together some code to do this")
"""
sigma = float(sigma)
# Create copy of the input file
# delete target location
if not os.path.exists(infile):
message = "The supplied parmdb path is not available on"
"the filesystem: {0}".format(infile)
self.logger.error(message)
raise PipelineRecipeFailed(message)
delete_directory(outfile)
self.logger.debug(
"cleared target path for filtered parmdb: \n {0}".format(outfile))
# copy
shutil.copytree(infile, outfile)
self.logger.debug(
"Copied raw parmdb to target locations: \n {0}".format(infile))
# Create a local WritableParmDB
parmdb = WritableParmDB(outfile)
#get all stations in the parmdb
stations = list_stations(parmdb)
for station in stations:
self.logger.debug("Processing station {0}".format(station))
# till here implemented
polarization_data, type_pair = \
self._read_polarisation_data_and_type_from_db(parmdb, station)
corrected_data = self._swap_outliers_with_median(
polarization_data, type_pair, sigma)
#print polarization_data
self._write_corrected_data(parmdb, station, polarization_data,
corrected_data)
def _read_polarisation_data_and_type_from_db(self, parmdb, station):
"""
Read the polarisation data and type from the db.
"""
all_matching_names = parmdb.getNames("Gain:*:*:*:{0}".format(station))
# get the polarisation_data eg: 1:1
# This is based on the 1 trough 3th entry in the parmdb name entry
pols = set(":".join(x[1:3])
for x in (x.split(":") for x in all_matching_names))
# Get the im or re name, eg: real. Sort for we need a known order
type_pair = sorted(
set(x[3] for x in (x.split(":") for x in all_matching_names)))
#Check if the retrieved types are valid
sorted_valid_type_pairs = [
sorted(RealImagArray.keys),
sorted(AmplPhaseArray.keys)
]
if not type_pair in sorted_valid_type_pairs:
self.logger.error(
"The parsed parmdb contained an invalid array_type:")
self.logger.error("{0}".format(type_pair))
self.logger.error(
"valid data pairs are: {0}".format(sorted_valid_type_pairs))
raise PipelineRecipeFailed(
"Invalid data type retrieved from parmdb: {0}".format(
type_pair))
polarisation_data = dict()
#for all polarisation_data in the parmdb (2 times 2)
for polarization in pols:
data = []
#for the two types
for key in type_pair:
query = "Gain:{0}:{1}:{2}".format(polarization, key, station)
#append the retrieved data (resulting in dict to arrays
data.append(parmdb.getValuesGrid(query)[query])
polarisation_data[polarization] = data
#return the raw data and the type of the data
return polarisation_data, type_pair
def _write_corrected_data(self, parmdb, station, polarization_data,
corrected_data):
"""
Use pyparmdb to write (now corrected) data to the parmdb
"""
for pol, data in polarization_data.items():
if not pol in corrected_data:
error_message = "Requested polarisation type is unknown:" \
"{0} \n valid polarisations: {1}".format(pol, list(corrected_data.keys()))
self.logger.error(error_message)
raise PipelineRecipeFailed(error_message)
corrected_data_pol = corrected_data[pol]
#get the "complex" converted data from the complex array
for component, value in corrected_data_pol.writeable.items():
#Collect all the data needed to write an array
name = "Gain:{0}:{1}:{2}".format(pol, component, station)
freqscale = data[0]['freqs'][0]
freqstep = data[0]['freqwidths'][0]
timescale = data[0]['times'][0]
timestep = data[0]['timewidths'][0]
#call the write function on the parmdb
parmdb.setValues(name, value, freqscale, freqstep, timescale,
timestep)