def remove_image(self, image_id, delete_observations=False): """ Remove all data from runningcatalog/runningcatalog_fluxes. """ GLOBALS['i'] = image_id self.log.info('removing data from image %s' % image_id) self.conn.start() for sql in [ 'deduct runningcatalog', 'deduct runningcatalog non-zero', 'deduct runningcatalog extended', 'deduct runningcatalog extended non-zero', 'deduct runningcatalog_fluxes', 'deduct runningcatalog_fluxes non-zero' ]: run_update(self.conn, sql, image_id) self.conn.execute(get_sql('deduct cleanup')) self.conn.execute(get_sql('update runningcatalog XYZ')) if delete_observations: self.conn.execute(get_sql('deduct remove extractedsources')) image_status = 99 else: image_status = 2 self.conn.execute(""" update images set status = %s, process_date = current_timestamp where imageid = %s""" % (image_status, image_id)) self.conn.commit()
def save_image_info(self, conn): """ Write image info into images table. """ if not self.data.get('frequency').isdigit(): raise SourceException('Frequency should be digital, %s found' % self.data.get('frequency')) band = conn.exec_return( get_sql('get frequency', self.data.get('frequency')), 'No matching frequency band found for frequency %s' % self.data.get('frequency')) if not 'pointing_ra' in self.data or \ not 'pointing_decl' in self.data or \ not 'beam_size' in self.data: data = conn.exec_return("""select min(ldecl), max(ldecl), min(lra), max(lra), avg(ldecl), avg(lra) from detections where run_id = %s;""" % self.run_id, single_column=False) size, avg_decl, avg_ra = get_image_size(*data) self.recalculate_pointing = True else: size = self.data.get('beam_size') avg_decl = self.data.get('pointing_decl') avg_ra = self.data.get('pointing_ra') conn.execute( get_sql('insert image', self.parset_id, band, avg_ra, avg_decl, size, get_svn_version(), self.run_id, self.get('bmaj'), self.get('bmin'), self.get('bpa'))) image_id = conn.exec_return(get_sql('get last image_id')) self.log.info('Image %s created' % image_id) return image_id
def remove_image(self, image_id, delete_observations=False): """ Remove all data from runningcatalog/runningcatalog_fluxes. """ GLOBALS['i'] = image_id self.log.info('removing data from image %s' % image_id) self.conn.start() for sql in ['deduct runningcatalog', 'deduct runningcatalog non-zero', 'deduct runningcatalog extended', 'deduct runningcatalog extended non-zero', 'deduct runningcatalog_fluxes', 'deduct runningcatalog_fluxes non-zero']: run_update(self.conn, sql, image_id) self.conn.execute(get_sql('deduct cleanup')) self.conn.execute(get_sql('update runningcatalog XYZ')) if delete_observations: self.conn.execute(get_sql('deduct remove extractedsources')) image_status = 99 else: image_status = 2 self.conn.execute(""" update images set status = %s, process_date = current_timestamp where imageid = %s""" % (image_status, image_id)) self.conn.commit()
def match(self, image_id): self.conn.execute( get_sql('Associate point', self.max_assoc, self.max_dist, self.pix)) self.conn.execute_set( get_sql('Associate extended', self.max_assoc, self.max_dist_ext, self.pix))
def update_image_pointing(self, image_id): """ Update image pointing to average ra/decl of all sources. """ avg_x, avg_y, avg_z, count = self.conn.exec_return( get_sql('Image properties selector', image_id), single_column=False) avg_x, avg_y, avg_z = avg_x / count, avg_y / count, avg_z / count decl = math.asin(avg_z) ra = math.atan2(avg_x, avg_y) self.conn.execute(get_sql('Image properties updater', ra, decl, image_id))
def update_image_pointing(self, image_id): """ Update image pointing to average ra/decl of all sources. """ avg_x, avg_y, avg_z, count = self.conn.exec_return(get_sql( 'Image properties selector', image_id), single_column=False) avg_x, avg_y, avg_z = avg_x / count, avg_y / count, avg_z / count decl = math.asin(avg_z) ra = math.atan2(avg_x, avg_y) self.conn.execute( get_sql('Image properties updater', ra, decl, image_id))
def run_resolver(self, resolve_class): #Running resolver resolver = resolve_class(self.conn) for group_id in self.conn.get_cursor(get_sql("GroupCycle")): if not resolver.run_resolve(group_id[0]): #Failed to resolve self.log.debug("Group id %s not resolved by %s." % (group_id[0], resolver.__class__.__name__)) self.conn.log.debug("Group id %s not resolved." % group_id[0]) self.conn.execute_set( get_sql("GroupUpdate runcat", group_id[0])) else: self.log.debug("Group id %s resolved by %s." % (group_id[0], resolver.__class__.__name__)) self.conn.log.debug("Group id %s resolved." % group_id[0])
def run_resolver(self, resolve_class): #Running resolver resolver = resolve_class(self.conn) for group_id in self.conn.get_cursor(get_sql("GroupCycle")): if not resolver.run_resolve(group_id[0]): #Failed to resolve self.log.debug("Group id %s not resolved by %s." % (group_id[0], resolver.__class__.__name__)) self.conn.log.debug("Group id %s not resolved." % group_id[0]) self.conn.execute_set(get_sql("GroupUpdate runcat", group_id[0])) else: self.log.debug("Group id %s resolved by %s." % (group_id[0], resolver.__class__.__name__)) self.conn.log.debug("Group id %s resolved." % group_id[0])
def run_grouper(self): """ Detect/update and store groups of sources for later processing. """ #Update groups by merging overlapping patches. cursor = self.conn.get_cursor(get_sql("GroupFinder")) grouper = Grouper(cursor.fetchall()) while grouper.is_completed(): grouper.one_cycle() self.conn.execute_set(get_sql("GroupUpdate", grouper.group, ",".join(map(str, grouper.runcatset)))) grouper.cleanup() for resolver in [SimpleResolver]: self.run_resolver(resolver) self.conn.execute(get_sql("GroupFill"))
def run_grouper(self): """ Detect/update and store groups of sources for later processing. """ #Update groups by merging overlapping patches. cursor = self.conn.get_cursor(get_sql("GroupFinder")) grouper = Grouper(cursor.fetchall()) while grouper.is_completed(): grouper.one_cycle() self.conn.execute_set( get_sql("GroupUpdate", grouper.group, ",".join(map(str, grouper.runcatset)))) grouper.cleanup() for resolver in [SimpleResolver]: self.run_resolver(resolver) self.conn.execute(get_sql("GroupFill"))
def get_image_properties(self, image_id): """ List basic image properties. """ return {'header': ['Name', 'Sources', 'MinFlux', 'NewSources'], 'data': self.connection.get_cursor( get_sql("APIimage", image_id) ).fetchall()}
def run_update(conn, sql_name, *params): """ Run update on a given connection. Refactor it for MonetDB if needed. """ sql = get_sql(sql_name, *params) if conn.is_monet(): conn.execute_set(_refactor_update(sql)) else: conn.execute(sql)
def save_image_info(self, conn): """ Write image info into images table. """ if not self.data.get('frequency').isdigit(): raise SourceException('Frequency should be digital, %s found' % self.data.get('frequency')) band = conn.exec_return(get_sql('get frequency', self.data.get('frequency')), 'No matching frequency band found for frequency %s' % self.data.get('frequency')) if not 'pointing_ra' in self.data or \ not 'pointing_decl' in self.data or \ not 'beam_size' in self.data: data = conn.exec_return( """select min(ldecl), max(ldecl), min(lra), max(lra), avg(ldecl), avg(lra) from detections where run_id = %s;""" % self.run_id, single_column=False) size, avg_decl, avg_ra = get_image_size(*data) self.recalculate_pointing = True else: size = self.data.get('beam_size') avg_decl = self.data.get('pointing_decl') avg_ra = self.data.get('pointing_ra') conn.execute(get_sql('insert image', self.parset_id, band, avg_ra, avg_decl, size, get_svn_version(), self.run_id, self.get('bmaj'), self.get('bmin'), self.get('bpa'))) image_id = conn.exec_return(get_sql('get last image_id')) self.log.info('Image %s created' % image_id) return image_id
def match(self, image_id): self.conn.execute(get_sql('Associate point', self.max_assoc, self.max_dist, self.pix)) self.conn.execute_set(get_sql('Associate extended', self.max_assoc, self.max_dist_ext, self.pix))
def process_image(self, image_id, run_id=None, sources_loaded=False): """ Process single image. @sources_loaded: True if there are records in the extractedsources already. """ self.conn.start() status, band, stokes, fov_radius, \ centr_ra, centr_decl, run_loaded, bmaj = \ self.conn.exec_return(""" select status, band, stokes, fov_radius, centr_ra, centr_decl, run_id, bmaj from images where imageid = %s;""" % image_id, single_column=False) if not run_id: run_id = run_loaded if status == 1: raise ImageStateError('Image %s in state 1 (Ok). Cannot process' % image_id) GLOBALS.update({'i': image_id, 'r': run_id, 'b': band, 's': stokes}) if not sources_loaded: self.conn.execute(get_sql('insert_extractedsources')) self.conn.execute(get_sql('insert dummysources')) if bmaj: max_assoc = float(bmaj) else: max_assoc = float(self.options.get('maximum_association_distance')) self.log.debug('Using options: %s' % self.options) self.log.debug('Final max_assoc_dist %s' % max_assoc) #Now do the matching! if self.options.get('matcher') == 'F90': matcher_class = MatcherF90 else: matcher_class = MatcherSQL matcher = matcher_class(self.conn, max_assoc, self.options.get('match_distance'), self.options.get('match_distance_extended'), get_pixels(centr_ra, centr_decl, fov_radius + 0.5)) matcher.match(image_id) self.conn.call_procedure("fill_temp_assoc_kind(%s);" % image_id) #Process many-to-many; self.run_grouper() # Process one-to-one associations; self.conn.execute(get_sql('add 1 to 1')) #process one-to-many associations; self.conn.execute(get_sql('add 1 to N')) self.conn.execute_set(get_sql('update flux_fraction')) #process many-to-one associations; self.conn.execute_set(get_sql('add N to 1')) #updating runningcatalog run_update(self.conn, 'update runningcatalog') run_update(self.conn, 'update runningcatalog extended') self.conn.execute(get_sql('update runningcatalog XYZ')) #First update, then insert new (!!!) run_update(self.conn, 'update runningcatalog_fluxes') self.conn.execute(get_sql('insert new bands for point sources')) #inserting new sources self.conn.execute_set(get_sql('Insert new sources')) self.conn.execute_set(get_sql('Join extended')) #update image status and save current svn verion. self.conn.execute_set(get_sql('Cleanup', get_svn_version())) if self.parset.recalculate_pointing: self.update_image_pointing(image_id) self.conn.commit()
def process_image(self, image_id, run_id=None, sources_loaded=False): """ Process single image. @sources_loaded: True if there are records in the extractedsources already. """ self.conn.start() status, band, stokes, fov_radius, \ centr_ra, centr_decl, run_loaded, bmaj = \ self.conn.exec_return(""" select status, band, stokes, fov_radius, centr_ra, centr_decl, run_id, bmaj from images where imageid = %s;""" % image_id, single_column=False) if not run_id: run_id = run_loaded if status == 1: raise ImageStateError('Image %s in state 1 (Ok). Cannot process' % image_id) GLOBALS.update({'i': image_id, 'r': run_id, 'b': band, 's': stokes}) if not sources_loaded: self.conn.execute(get_sql('insert_extractedsources')) self.conn.execute(get_sql('insert dummysources')) if bmaj: max_assoc = float(bmaj) else: max_assoc = float(self.options.get('maximum_association_distance')) self.log.debug('Using options: %s' % self.options) self.log.debug('Final max_assoc_dist %s' % max_assoc) #Now do the matching! if self.options.get('matcher') == 'F90': matcher_class = MatcherF90 else: matcher_class = MatcherSQL matcher = matcher_class( self.conn, max_assoc, self.options.get('match_distance'), self.options.get('match_distance_extended'), get_pixels(centr_ra, centr_decl, fov_radius + 0.5)) matcher.match(image_id) self.conn.call_procedure("fill_temp_assoc_kind(%s);" % image_id) #Process many-to-many; self.run_grouper() # Process one-to-one associations; self.conn.execute(get_sql('add 1 to 1')) #process one-to-many associations; self.conn.execute(get_sql('add 1 to N')) self.conn.execute_set(get_sql('update flux_fraction')) #process many-to-one associations; self.conn.execute_set(get_sql('add N to 1')) #updating runningcatalog run_update(self.conn, 'update runningcatalog') run_update(self.conn, 'update runningcatalog extended') self.conn.execute(get_sql('update runningcatalog XYZ')) #First update, then insert new (!!!) run_update(self.conn, 'update runningcatalog_fluxes') self.conn.execute(get_sql('insert new bands for point sources')) #inserting new sources self.conn.execute_set(get_sql('Insert new sources')) self.conn.execute_set(get_sql('Join extended')) #update image status and save current svn verion. self.conn.execute_set(get_sql('Cleanup', get_svn_version())) if self.parset.recalculate_pointing: self.update_image_pointing(image_id) self.conn.commit()