def run_task(self, configblock, datafiles=[], **kwargs): """ A task is a combination of a recipe and a set of parameters. Tasks can be prefedined in the task file set in the pipeline configuration (default: tasks.cfg). Here, we load a task configuration and execute it. This is a "shorthand" version of :meth:`lofarpipe.cuisine.WSRTrecipe.WSRTrecipe.cook_recipe`. """ self.logger.info("Running task: %s" % (configblock,)) # Does the task definition exist? try: recipe = self.task_definitions.get(configblock, "recipe") except NoSectionError: raise PipelineException( "%s not found -- check your task definitions" % configblock ) # Build inputs dict. # First, take details from caller. inputs = LOFARinput(self.inputs) inputs['args'] = datafiles # Add parameters from the task file. # Note that we neither need the recipe name nor any items from the # DEFAULT config. parameters = dict(self.task_definitions.items(configblock)) del parameters['recipe'] for key in list(dict(self.config.items("DEFAULT")).keys()): del parameters[key] inputs.update(parameters) # Update inputs with provided kwargs, if any. inputs.update(kwargs) # Default outputs dict. outputs = LOFARoutput() # Cook the recipe and return the results" try: self.cook_recipe(recipe, inputs, outputs) except CookError: self.logger.warn( "%s reports failure (using %s recipe)" % (configblock, recipe) ) raise PipelineRecipeFailed("%s failed" % configblock) # Get the (optional) node xml information if "return_xml" in outputs: return_node = xml.parseString( outputs['return_xml']).documentElement # If no active stack, fail silently. add_child_to_active_stack_head(self, return_node) return outputs
def _convert_data_to_ComplexArray(self, data, type_pair): """ Performs a conversion of a 2d array to a 1d complex valued array. with real/imag values or with amplitude phase values """ if sorted(type_pair) == sorted(RealImagArray.keys): # The type_pair is in alphabetical order: Imag on index 0 complex_array = RealImagArray(data[1]["values"], data[0]["values"]) elif sorted(type_pair) == sorted(AmplPhaseArray.keys): complex_array = AmplPhaseArray(data[0]["values"], data[1]["values"]) else: self.logger.error("Incorrect data type pair provided: {0}".format( type_pair)) raise PipelineRecipeFailed( "Invalid data type retrieved from parmdb: {0}".format(type_pair)) return complex_array
def _filter_stations_parmdb(self, infile, outfile, sigma): """ Performs a gain outlier correction of the infile parmdb with the corrected parmdb written to outfile. Outliers in the gain with a distance of median of sigma times std are replaced with the mean. The last value of the complex array is skipped (John Swinbank: "I found it [the last value] was bad when I hacked together some code to do this") """ sigma = float(sigma) # Create copy of the input file # delete target location if not os.path.exists(infile): message = "The supplied parmdb path is not available on" "the filesystem: {0}".format(infile) self.logger.error(message) raise PipelineRecipeFailed(message) delete_directory(outfile) self.logger.debug( "cleared target path for filtered parmdb: \n {0}".format(outfile)) # copy shutil.copytree(infile, outfile) self.logger.debug( "Copied raw parmdb to target locations: \n {0}".format(infile)) # Create a local WritableParmDB parmdb = WritableParmDB(outfile) #get all stations in the parmdb stations = list_stations(parmdb) for station in stations: self.logger.debug("Processing station {0}".format(station)) # till here implemented polarization_data, type_pair = \ self._read_polarisation_data_and_type_from_db(parmdb, station) corrected_data = self._swap_outliers_with_median( polarization_data, type_pair, sigma) #print polarization_data self._write_corrected_data(parmdb, station, polarization_data, corrected_data)
def _read_polarisation_data_and_type_from_db(self, parmdb, station): """ Read the polarisation data and type from the db. """ all_matching_names = parmdb.getNames("Gain:*:*:*:{0}".format(station)) # get the polarisation_data eg: 1:1 # This is based on the 1 trough 3th entry in the parmdb name entry pols = set(":".join(x[1:3]) for x in (x.split(":") for x in all_matching_names)) # Get the im or re name, eg: real. Sort for we need a known order type_pair = sorted( set(x[3] for x in (x.split(":") for x in all_matching_names))) #Check if the retrieved types are valid sorted_valid_type_pairs = [ sorted(RealImagArray.keys), sorted(AmplPhaseArray.keys) ] if not type_pair in sorted_valid_type_pairs: self.logger.error( "The parsed parmdb contained an invalid array_type:") self.logger.error("{0}".format(type_pair)) self.logger.error( "valid data pairs are: {0}".format(sorted_valid_type_pairs)) raise PipelineRecipeFailed( "Invalid data type retrieved from parmdb: {0}".format( type_pair)) polarisation_data = dict() #for all polarisation_data in the parmdb (2 times 2) for polarization in pols: data = [] #for the two types for key in type_pair: query = "Gain:{0}:{1}:{2}".format(polarization, key, station) #append the retrieved data (resulting in dict to arrays data.append(parmdb.getValuesGrid(query)[query]) polarisation_data[polarization] = data #return the raw data and the type of the data return polarisation_data, type_pair
def _write_corrected_data(self, parmdb, station, polarization_data, corrected_data): """ Use pyparmdb to write (now corrected) data to the parmdb """ for pol, data in polarization_data.items(): if not pol in corrected_data: error_message = "Requested polarisation type is unknown:" \ "{0} \n valid polarisations: {1}".format(pol, list(corrected_data.keys())) self.logger.error(error_message) raise PipelineRecipeFailed(error_message) corrected_data_pol = corrected_data[pol] #get the "complex" converted data from the complex array for component, value in corrected_data_pol.writeable.items(): #Collect all the data needed to write an array name = "Gain:{0}:{1}:{2}".format(pol, component, station) freqscale = data[0]['freqs'][0] freqstep = data[0]['freqwidths'][0] timescale = data[0]['times'][0] timestep = data[0]['timewidths'][0] #call the write function on the parmdb parmdb.setValues(name, value, freqscale, freqstep, timescale, timestep)