Ejemplo n.º 1
0
    def remove_image(self, image_id, delete_observations=False):
        """
        Remove all data from runningcatalog/runningcatalog_fluxes.
        """
        GLOBALS['i'] = image_id
        self.log.info('removing data from image %s' % image_id)
        self.conn.start()
        for sql in [
                'deduct runningcatalog', 'deduct runningcatalog non-zero',
                'deduct runningcatalog extended',
                'deduct runningcatalog extended non-zero',
                'deduct runningcatalog_fluxes',
                'deduct runningcatalog_fluxes non-zero'
        ]:
            run_update(self.conn, sql, image_id)
        self.conn.execute(get_sql('deduct cleanup'))
        self.conn.execute(get_sql('update runningcatalog XYZ'))
        if delete_observations:
            self.conn.execute(get_sql('deduct remove extractedsources'))
            image_status = 99
        else:
            image_status = 2
        self.conn.execute("""
update images
   set status = %s,
       process_date = current_timestamp
 where imageid = %s""" % (image_status, image_id))
        self.conn.commit()
Ejemplo n.º 2
0
    def save_image_info(self, conn):
        """
        Write image info into images table.
        """
        if not self.data.get('frequency').isdigit():
            raise SourceException('Frequency should be digital, %s found' %
                                  self.data.get('frequency'))
        band = conn.exec_return(
            get_sql('get frequency', self.data.get('frequency')),
            'No matching frequency band found for frequency %s' %
            self.data.get('frequency'))

        if not 'pointing_ra' in self.data or \
           not 'pointing_decl' in self.data or \
           not 'beam_size' in self.data:
            data = conn.exec_return("""select min(ldecl), max(ldecl),
                      min(lra), max(lra),
                      avg(ldecl), avg(lra)
                 from detections
                 where run_id = %s;""" % self.run_id,
                                    single_column=False)
            size, avg_decl, avg_ra = get_image_size(*data)
            self.recalculate_pointing = True
        else:
            size = self.data.get('beam_size')
            avg_decl = self.data.get('pointing_decl')
            avg_ra = self.data.get('pointing_ra')

        conn.execute(
            get_sql('insert image', self.parset_id, band, avg_ra, avg_decl,
                    size, get_svn_version(), self.run_id, self.get('bmaj'),
                    self.get('bmin'), self.get('bpa')))
        image_id = conn.exec_return(get_sql('get last image_id'))
        self.log.info('Image %s created' % image_id)
        return image_id
Ejemplo n.º 3
0
    def remove_image(self, image_id, delete_observations=False):
        """
        Remove all data from runningcatalog/runningcatalog_fluxes.
        """
        GLOBALS['i'] = image_id
        self.log.info('removing data from image %s' % image_id)
        self.conn.start()
        for sql in ['deduct runningcatalog',
                    'deduct runningcatalog non-zero',
                    'deduct runningcatalog extended',
                    'deduct runningcatalog extended non-zero',
                    'deduct runningcatalog_fluxes',
                    'deduct runningcatalog_fluxes non-zero']:
            run_update(self.conn, sql, image_id)
        self.conn.execute(get_sql('deduct cleanup'))
        self.conn.execute(get_sql('update runningcatalog XYZ'))
        if delete_observations:
            self.conn.execute(get_sql('deduct remove extractedsources'))
            image_status = 99
        else:
            image_status = 2
        self.conn.execute("""
update images
   set status = %s,
       process_date = current_timestamp
 where imageid = %s""" % (image_status, image_id))
        self.conn.commit()
Ejemplo n.º 4
0
 def match(self, image_id):
     self.conn.execute(
         get_sql('Associate point', self.max_assoc, self.max_dist,
                 self.pix))
     self.conn.execute_set(
         get_sql('Associate extended', self.max_assoc, self.max_dist_ext,
                 self.pix))
Ejemplo n.º 5
0
 def update_image_pointing(self, image_id):
     """
     Update image pointing to average ra/decl of all sources.
     """
     avg_x, avg_y, avg_z, count = self.conn.exec_return(
                         get_sql('Image properties selector', image_id),
                                  single_column=False)
     avg_x, avg_y, avg_z = avg_x / count, avg_y / count, avg_z / count
     decl = math.asin(avg_z)
     ra = math.atan2(avg_x, avg_y)
     self.conn.execute(get_sql('Image properties updater',
                               ra, decl, image_id))
Ejemplo n.º 6
0
 def update_image_pointing(self, image_id):
     """
     Update image pointing to average ra/decl of all sources.
     """
     avg_x, avg_y, avg_z, count = self.conn.exec_return(get_sql(
         'Image properties selector', image_id),
                                                        single_column=False)
     avg_x, avg_y, avg_z = avg_x / count, avg_y / count, avg_z / count
     decl = math.asin(avg_z)
     ra = math.atan2(avg_x, avg_y)
     self.conn.execute(
         get_sql('Image properties updater', ra, decl, image_id))
Ejemplo n.º 7
0
 def run_resolver(self, resolve_class):
     #Running resolver
     resolver = resolve_class(self.conn)
     for group_id in self.conn.get_cursor(get_sql("GroupCycle")):
         if not resolver.run_resolve(group_id[0]):
             #Failed to resolve
             self.log.debug("Group id %s not resolved by %s." %
                            (group_id[0], resolver.__class__.__name__))
             self.conn.log.debug("Group id %s not resolved." % group_id[0])
             self.conn.execute_set(
                 get_sql("GroupUpdate runcat", group_id[0]))
         else:
             self.log.debug("Group id %s resolved by %s." %
                            (group_id[0], resolver.__class__.__name__))
             self.conn.log.debug("Group id %s resolved." % group_id[0])
Ejemplo n.º 8
0
 def run_resolver(self, resolve_class):
     #Running resolver
     resolver = resolve_class(self.conn)
     for group_id in self.conn.get_cursor(get_sql("GroupCycle")):
         if not resolver.run_resolve(group_id[0]):
             #Failed to resolve
             self.log.debug("Group id %s not resolved by %s." % 
                                (group_id[0], resolver.__class__.__name__))
             self.conn.log.debug("Group id %s not resolved." % group_id[0])
             self.conn.execute_set(get_sql("GroupUpdate runcat",
                                   group_id[0]))
         else:
             self.log.debug("Group id %s resolved by %s."  % 
                                (group_id[0], resolver.__class__.__name__))
             self.conn.log.debug("Group id %s resolved." % group_id[0])
Ejemplo n.º 9
0
 def run_grouper(self):
     """
     Detect/update and store groups of sources for later processing.
     """
     #Update groups by merging overlapping patches.
     cursor = self.conn.get_cursor(get_sql("GroupFinder"))
     grouper = Grouper(cursor.fetchall())
     while grouper.is_completed():
         grouper.one_cycle()
         self.conn.execute_set(get_sql("GroupUpdate",
                                   grouper.group,
                                   ",".join(map(str, grouper.runcatset))))
         grouper.cleanup()
     for resolver in [SimpleResolver]:
         self.run_resolver(resolver)
     self.conn.execute(get_sql("GroupFill"))
Ejemplo n.º 10
0
 def run_grouper(self):
     """
     Detect/update and store groups of sources for later processing.
     """
     #Update groups by merging overlapping patches.
     cursor = self.conn.get_cursor(get_sql("GroupFinder"))
     grouper = Grouper(cursor.fetchall())
     while grouper.is_completed():
         grouper.one_cycle()
         self.conn.execute_set(
             get_sql("GroupUpdate", grouper.group,
                     ",".join(map(str, grouper.runcatset))))
         grouper.cleanup()
     for resolver in [SimpleResolver]:
         self.run_resolver(resolver)
     self.conn.execute(get_sql("GroupFill"))
Ejemplo n.º 11
0
 def get_image_properties(self, image_id):
     """
     List basic image properties.
     """
     return {'header': ['Name', 'Sources', 'MinFlux', 'NewSources'],
             'data': self.connection.get_cursor(
                                      get_sql("APIimage", image_id)
                                      ).fetchall()}
Ejemplo n.º 12
0
def run_update(conn, sql_name, *params):
    """
    Run update on a given connection. Refactor it for MonetDB if needed.
    """
    sql = get_sql(sql_name, *params)
    if conn.is_monet():
        conn.execute_set(_refactor_update(sql))
    else:
        conn.execute(sql)
Ejemplo n.º 13
0
def run_update(conn, sql_name, *params):
    """
    Run update on a given connection. Refactor it for MonetDB if needed.
    """
    sql = get_sql(sql_name, *params)
    if conn.is_monet():
        conn.execute_set(_refactor_update(sql))
    else:
        conn.execute(sql)
Ejemplo n.º 14
0
    def save_image_info(self, conn):
        """
        Write image info into images table.
        """
        if not self.data.get('frequency').isdigit():
            raise SourceException('Frequency should be digital, %s found'
                                    % self.data.get('frequency'))
        band = conn.exec_return(get_sql('get frequency',
                                        self.data.get('frequency')),
                        'No matching frequency band found for frequency %s' %
                            self.data.get('frequency'))

        if not 'pointing_ra' in self.data or \
           not 'pointing_decl' in self.data or \
           not 'beam_size' in self.data:
            data = conn.exec_return(
            """select min(ldecl), max(ldecl),
                      min(lra), max(lra),
                      avg(ldecl), avg(lra)
                 from detections
                 where run_id = %s;""" % self.run_id, single_column=False)
            size, avg_decl, avg_ra = get_image_size(*data)
            self.recalculate_pointing = True
        else:
            size = self.data.get('beam_size')
            avg_decl = self.data.get('pointing_decl')
            avg_ra = self.data.get('pointing_ra')

        conn.execute(get_sql('insert image', self.parset_id, band,
                             avg_ra, avg_decl, size,
                             get_svn_version(), self.run_id,
                             self.get('bmaj'),
                             self.get('bmin'),
                             self.get('bpa')))
        image_id = conn.exec_return(get_sql('get last image_id'))
        self.log.info('Image %s created' % image_id)
        return image_id
Ejemplo n.º 15
0
 def match(self, image_id):
     self.conn.execute(get_sql('Associate point', self.max_assoc, 
                          self.max_dist, self.pix))
     self.conn.execute_set(get_sql('Associate extended', self.max_assoc, 
                           self.max_dist_ext, self.pix))
Ejemplo n.º 16
0
    def process_image(self, image_id, run_id=None, sources_loaded=False):
        """
        Process single image.
        @sources_loaded: True if there are records in the extractedsources
        already.
        """
        self.conn.start()
        status, band, stokes, fov_radius, \
        centr_ra, centr_decl, run_loaded, bmaj = \
        self.conn.exec_return("""
        select status, band, stokes, fov_radius, 
               centr_ra, centr_decl, run_id, bmaj
          from images
         where imageid = %s;""" % image_id, single_column=False)
        if not run_id:
            run_id = run_loaded
        if status == 1:
            raise ImageStateError('Image %s in state 1 (Ok). Cannot process' %
                                  image_id)
        GLOBALS.update({'i': image_id, 'r': run_id,
                        'b': band, 's': stokes})
        if not sources_loaded:
            self.conn.execute(get_sql('insert_extractedsources'))
            self.conn.execute(get_sql('insert dummysources'))
        if bmaj:
            max_assoc = float(bmaj)
        else:
            max_assoc = float(self.options.get('maximum_association_distance'))
        self.log.debug('Using options: %s' % self.options)
        self.log.debug('Final max_assoc_dist %s' % max_assoc)
        
        #Now do the matching!
        if self.options.get('matcher') == 'F90':
            matcher_class = MatcherF90
        else:
            matcher_class = MatcherSQL
        matcher = matcher_class(self.conn, max_assoc, 
                  self.options.get('match_distance'),
                  self.options.get('match_distance_extended'),
                  get_pixels(centr_ra, centr_decl, fov_radius + 0.5))
        matcher.match(image_id)

        self.conn.call_procedure("fill_temp_assoc_kind(%s);" % image_id)
        #Process many-to-many;
        self.run_grouper()

        # Process one-to-one associations;
        self.conn.execute(get_sql('add 1 to 1'))
        #process one-to-many associations;
        self.conn.execute(get_sql('add 1 to N'))
        self.conn.execute_set(get_sql('update flux_fraction'))
        #process many-to-one associations;
        self.conn.execute_set(get_sql('add N to 1'))
        #updating runningcatalog
        run_update(self.conn, 'update runningcatalog')
        run_update(self.conn, 'update runningcatalog extended')
        self.conn.execute(get_sql('update runningcatalog XYZ'))
        #First update, then insert new (!!!)
        run_update(self.conn, 'update runningcatalog_fluxes')
        self.conn.execute(get_sql('insert new bands for point sources'))
        #inserting new sources
        self.conn.execute_set(get_sql('Insert new sources'))
        self.conn.execute_set(get_sql('Join extended'))
        #update image status and save current svn verion.
        self.conn.execute_set(get_sql('Cleanup', get_svn_version()))
        if self.parset.recalculate_pointing:
            self.update_image_pointing(image_id)
        self.conn.commit()
Ejemplo n.º 17
0
    def process_image(self, image_id, run_id=None, sources_loaded=False):
        """
        Process single image.
        @sources_loaded: True if there are records in the extractedsources
        already.
        """
        self.conn.start()
        status, band, stokes, fov_radius, \
        centr_ra, centr_decl, run_loaded, bmaj = \
        self.conn.exec_return("""
        select status, band, stokes, fov_radius, 
               centr_ra, centr_decl, run_id, bmaj
          from images
         where imageid = %s;""" % image_id, single_column=False)
        if not run_id:
            run_id = run_loaded
        if status == 1:
            raise ImageStateError('Image %s in state 1 (Ok). Cannot process' %
                                  image_id)
        GLOBALS.update({'i': image_id, 'r': run_id, 'b': band, 's': stokes})
        if not sources_loaded:
            self.conn.execute(get_sql('insert_extractedsources'))
            self.conn.execute(get_sql('insert dummysources'))
        if bmaj:
            max_assoc = float(bmaj)
        else:
            max_assoc = float(self.options.get('maximum_association_distance'))
        self.log.debug('Using options: %s' % self.options)
        self.log.debug('Final max_assoc_dist %s' % max_assoc)

        #Now do the matching!
        if self.options.get('matcher') == 'F90':
            matcher_class = MatcherF90
        else:
            matcher_class = MatcherSQL
        matcher = matcher_class(
            self.conn, max_assoc, self.options.get('match_distance'),
            self.options.get('match_distance_extended'),
            get_pixels(centr_ra, centr_decl, fov_radius + 0.5))
        matcher.match(image_id)

        self.conn.call_procedure("fill_temp_assoc_kind(%s);" % image_id)
        #Process many-to-many;
        self.run_grouper()

        # Process one-to-one associations;
        self.conn.execute(get_sql('add 1 to 1'))
        #process one-to-many associations;
        self.conn.execute(get_sql('add 1 to N'))
        self.conn.execute_set(get_sql('update flux_fraction'))
        #process many-to-one associations;
        self.conn.execute_set(get_sql('add N to 1'))
        #updating runningcatalog
        run_update(self.conn, 'update runningcatalog')
        run_update(self.conn, 'update runningcatalog extended')
        self.conn.execute(get_sql('update runningcatalog XYZ'))
        #First update, then insert new (!!!)
        run_update(self.conn, 'update runningcatalog_fluxes')
        self.conn.execute(get_sql('insert new bands for point sources'))
        #inserting new sources
        self.conn.execute_set(get_sql('Insert new sources'))
        self.conn.execute_set(get_sql('Join extended'))
        #update image status and save current svn verion.
        self.conn.execute_set(get_sql('Cleanup', get_svn_version()))
        if self.parset.recalculate_pointing:
            self.update_image_pointing(image_id)
        self.conn.commit()