Ejemplo n.º 1
0
 def populate_cimager_parset(parset):
     input_parset = Parset(parset)
     patch_dictionary = {
         'Cimager.dataset': dataset,
         'Cimager.restore': restore
     }
     image_names = []
     for image_name in input_parset.getStringVector(
             'Cimager.Images.Names'):
         image_names.append("%s_%s" % (image_name, name))
         subset = input_parset.makeSubset(
             "Cimager.Images.%s" % image_name,
             "Cimager.Images.%s" % image_names[-1])
         patch_dictionary["Cimager.Images.%s.frequency" %
                          image_names[-1]] = frequency
         patch_dictionary["Cimager.Images.%s.direction" %
                          image_names[-1]] = "[ %s,%s,%s ]" % (
                              ms_dir_ra, ms_dir_dec, ms_dir_type)
         for key in subset:
             patch_dictionary[key] = subset[key].get()
     input_parset.subtractSubset('Cimager.Images.image')
     for key in input_parset:
         patch_dictionary[key] = input_parset[key].get()
     patch_dictionary['Cimager.Images.Names'] = "[ %s ]" % ", ".join(
         image_names)
     return patch_parset(None, patch_dictionary,
                         self.config.get("layout", "job_directory"))
Ejemplo n.º 2
0
 def populate_cimager_parset(parset):
     input_parset = Parset(parset)
     patch_dictionary = {
         'Cimager.dataset': dataset,
         'Cimager.restore': restore
     }
     image_names = []
     for image_name in input_parset.getStringVector('Cimager.Images.Names'):
         image_names.append("%s_%s" % (image_name, name))
         subset = input_parset.makeSubset(
             "Cimager.Images.%s" % image_name,
             "Cimager.Images.%s" % image_names[-1]
         )
         patch_dictionary[
             "Cimager.Images.%s.frequency" % image_names[-1]
         ] = frequency
         patch_dictionary[
             "Cimager.Images.%s.direction" % image_names[-1]
         ] = "[ %s,%s,%s ]" % (ms_dir_ra, ms_dir_dec, ms_dir_type)
         for key in subset:
             patch_dictionary[key] = subset[key].get()
     input_parset.subtractSubset('Cimager.Images.image')
     for key in input_parset:
         patch_dictionary[key] = input_parset[key].get()
     patch_dictionary['Cimager.Images.Names'] = "[ %s ]" % ", ".join(image_names)
     return patch_parset(
         None, patch_dictionary,
         self.config.get("layout", "job_directory")
     )
Ejemplo n.º 3
0
    def run(self, imager_exec, vds, parset, resultsdir, start_time, end_time):
        #       imager_exec:                          path to cimager executable
        #               vds:           VDS file describing the data to be imaged
        #            parset:                                imager configuration
        #        resultsdir:                         place resulting images here
        #        start_time:                        )    time range to be imaged
        #          end_time:                        )   in seconds (may be None)
        # ----------------------------------------------------------------------
        with log_time(self.logger):
            self.logger.info("Processing %s" % (vds,))

            #    Bail out if destination exists (can thus resume a partial run).
            #                                            Should be configurable?
            # ------------------------------------------------------------------
            parset_data = Parset(parset)
            image_names = parset_data.getStringVector("Cimager.Images.Names")
            for image_name in image_names:
                outputfile = os.path.join(resultsdir, image_name + ".restored")
                self.logger.info(outputfile)
                if os.path.exists(outputfile):
                    self.logger.info("Image already exists: aborting.")
                    return 0
            try:
                working_dir = mkdtemp(suffix=".%s" % (os.path.basename(__file__),))

                #   If a time range has been specified, copy that section of the
                #                                  input MS and only image that.
                # --------------------------------------------------------------
                query = []
                if start_time:
                    self.logger.debug("Start time is %s" % start_time)
                    start_time = quantity(float(start_time), 's')
                    query.append("TIME > %f" % start_time.get('s').get_value())
                if end_time:
                    self.logger.debug("End time is %s" % end_time)
                    end_time = quantity(float(end_time), 's')
                    query.append("TIME < %f" % end_time.get('s').get_value())
                query = " AND ".join(query)
                if query:
                    #                             Select relevant section of MS.
                    # ----------------------------------------------------------
                    self.logger.debug("Query is %s" % query)
                    output = os.path.join(working_dir, "timeslice.MS")
                    vds_parset = get_parset(vds)
                    t = table(vds_parset.getString("FileName"))
                    t.query(query, name=output)
                    #       Patch updated information into imager configuration.
                    # ----------------------------------------------------------
                    parset = patch_parset(parset,
                        {
                            'Cimager.dataset': output
                        }
                    )
                else:
                    self.logger.debug("No time range selected")

                self.logger.debug("Running cimager")
                with CatchLog4CXX(
                    working_dir,
                    self.logger.name + "." + os.path.basename(vds)
                ):
                    cimager_process = Popen(
                        [imager_exec, "-inputs", parset],
                        stdout=PIPE, stderr=PIPE, cwd=working_dir
                    )
                    sout, serr = cimager_process.communicate()
                log_process_output("cimager", sout, serr, self.logger)
                if cimager_process.returncode != 0:
                    raise CalledProcessError(
                        cimager_process.returncode, imager_exec
                    )

                #        Dump the resulting images in the pipeline results area.
                #    I'm not aware of a foolproof way to predict the image names
                #                that will be produced, so we read them from the
                #                      parset and add standard cimager prefixes.
                # --------------------------------------------------------------
                parset_data = Parset(parset)
                image_names = parset_data.getStringVector("Cimager.Images.Names")
                prefixes = [
                    "image", "psf", "residual", "weights", "sensitivity"
                ]
                self.logger.debug("Copying images to %s" % resultsdir)
                for image_name in image_names:
                    for prefix in prefixes:
                        filename = image_name.replace("image", prefix, 1)
                        shutil.move(
                            os.path.join(working_dir, filename),
                            os.path.join(resultsdir, filename)
                        )
                    if parset_data.getBool('Cimager.restore'):
                        shutil.move(
                            os.path.join(working_dir, image_name + ".restored"),
                            os.path.join(resultsdir, image_name + ".restored")
                        )
            except CalledProcessError, e:
                self.logger.error(str(e))
                return 1
            finally:
Ejemplo n.º 4
0
    def run(self, imager_exec, vds, parset, resultsdir, start_time, end_time):
        #       imager_exec:                          path to cimager executable
        #               vds:           VDS file describing the data to be imaged
        #            parset:                                imager configuration
        #        resultsdir:                         place resulting images here
        #        start_time:                        )    time range to be imaged
        #          end_time:                        )   in seconds (may be None)
        # ----------------------------------------------------------------------
        with log_time(self.logger):
            self.logger.info("Processing %s" % (vds, ))

            #    Bail out if destination exists (can thus resume a partial run).
            #                                            Should be configurable?
            # ------------------------------------------------------------------
            parset_data = Parset(parset)
            image_names = parset_data.getStringVector("Cimager.Images.Names")
            for image_name in image_names:
                outputfile = os.path.join(resultsdir, image_name + ".restored")
                self.logger.info(outputfile)
                if os.path.exists(outputfile):
                    self.logger.info("Image already exists: aborting.")
                    return 0
            try:
                working_dir = mkdtemp(suffix=".%s" %
                                      (os.path.basename(__file__), ))

                #   If a time range has been specified, copy that section of the
                #                                  input MS and only image that.
                # --------------------------------------------------------------
                query = []
                if start_time:
                    self.logger.debug("Start time is %s" % start_time)
                    start_time = quantity(float(start_time), 's')
                    query.append("TIME > %f" % start_time.get('s').get_value())
                if end_time:
                    self.logger.debug("End time is %s" % end_time)
                    end_time = quantity(float(end_time), 's')
                    query.append("TIME < %f" % end_time.get('s').get_value())
                query = " AND ".join(query)
                if query:
                    #                             Select relevant section of MS.
                    # ----------------------------------------------------------
                    self.logger.debug("Query is %s" % query)
                    output = os.path.join(working_dir, "timeslice.MS")
                    vds_parset = get_parset(vds)
                    t = table(vds_parset.getString("FileName"))
                    t.query(query, name=output)
                    #       Patch updated information into imager configuration.
                    # ----------------------------------------------------------
                    parset = patch_parset(parset, {'Cimager.dataset': output})
                else:
                    self.logger.debug("No time range selected")

                self.logger.debug("Running cimager")
                with CatchLog4CXX(
                        working_dir,
                        self.logger.name + "." + os.path.basename(vds)):
                    cimager_process = Popen([imager_exec, "-inputs", parset],
                                            stdout=PIPE,
                                            stderr=PIPE,
                                            cwd=working_dir)
                    sout, serr = cimager_process.communicate()
                log_process_output("cimager", sout, serr, self.logger)
                if cimager_process.returncode != 0:
                    raise CalledProcessError(cimager_process.returncode,
                                             imager_exec)

                #        Dump the resulting images in the pipeline results area.
                #    I'm not aware of a foolproof way to predict the image names
                #                that will be produced, so we read them from the
                #                      parset and add standard cimager prefixes.
                # --------------------------------------------------------------
                parset_data = Parset(parset)
                image_names = parset_data.getStringVector(
                    "Cimager.Images.Names")
                prefixes = [
                    "image", "psf", "residual", "weights", "sensitivity"
                ]
                self.logger.debug("Copying images to %s" % resultsdir)
                for image_name in image_names:
                    for prefix in prefixes:
                        filename = image_name.replace("image", prefix, 1)
                        shutil.move(os.path.join(working_dir, filename),
                                    os.path.join(resultsdir, filename))
                    if parset_data.getBool('Cimager.restore'):
                        shutil.move(
                            os.path.join(working_dir,
                                         image_name + ".restored"),
                            os.path.join(resultsdir, image_name + ".restored"))
            except CalledProcessError, e:
                self.logger.error(str(e))
                return 1
            finally: