def insert_and_associate_forced_fits(image_id,successful_fits,successful_ids): assert len(successful_ids) == len(successful_fits) nd_extractions=[] nd_runcats=[] ms_extractions=[] ms_ids = [] for idx, id in enumerate(successful_ids): if id[0] == 'ff_nd': nd_extractions.append(successful_fits[idx]) nd_runcats.append(id[1]) elif id[0] == 'ff_ms': ms_extractions.append(successful_fits[idx]) ms_ids.append(id[1]) else: raise ValueError("Forced fit type id not recognised:" + id[0]) if nd_extractions: logger.debug("adding null detections") dbgen.insert_extracted_sources(image_id, nd_extractions, extract_type='ff_nd', ff_runcat_ids=nd_runcats) dbnd.associate_nd(image_id) else: logger.debug("No successful nulldetection fits") if ms_extractions: dbgen.insert_extracted_sources(image_id, ms_extractions, extract_type='ff_ms', ff_monitor_ids=ms_ids) logger.debug("adding monitoring sources") dbmon.associate_ms(image_id) else: logger.debug("No successful monitor fits")
def insert_and_associate_forced_fits(image_id, successful_fits, successful_ids): assert len(successful_ids) == len(successful_fits) nd_extractions = [] nd_runcats = [] ms_extractions = [] ms_ids = [] for idx, id in enumerate(successful_ids): if id[0] == 'ff_nd': nd_extractions.append(successful_fits[idx]) nd_runcats.append(id[1]) elif id[0] == 'ff_ms': ms_extractions.append(successful_fits[idx]) ms_ids.append(id[1]) else: raise ValueError("Forced fit type id not recognised:" + id[0]) if nd_extractions: logger.info("adding null detections") dbgen.insert_extracted_sources(image_id, nd_extractions, extract_type='ff_nd', ff_runcat_ids=nd_runcats) dbnd.associate_nd(image_id) else: logger.info("No successful nulldetection fits") if ms_extractions: dbgen.insert_extracted_sources(image_id, ms_extractions, extract_type='ff_ms', ff_monitor_ids=ms_ids) logger.info("adding monitoring sources") dbmon.associate_ms(image_id) else: logger.info("No successful monitor fits")
def test_basic_case(self): im_params = self.im_params blind_src = db_subs.example_extractedsource_tuple( ra=im_params[0]['centre_ra'], dec=im_params[0]['centre_decl'], ) superimposed_mon_src = blind_src mon_src_in_field = blind_src._replace(ra=blind_src.ra + 0.001) # Simulate a source that does not get fit, for good measure: mon_src_out_of_field = blind_src._replace(ra=blind_src.ra + 90.) #Sorted by increasing RA: mon_srcs = [ superimposed_mon_src, mon_src_in_field, mon_src_out_of_field ] mon_posns = [(m.ra, m.dec) for m in mon_srcs] dbgen.insert_monitor_positions(self.dataset.id, mon_posns) images = [] for img_pars in self.im_params: img = tkp.db.Image(dataset=self.dataset, data=img_pars) dbgen.insert_extracted_sources(img.id, [blind_src], 'blind') associate_extracted_sources(img.id, deRuiter_r=5.68) nd_requests = get_nulldetections(img.id) self.assertEqual(len(nd_requests), 0) mon_requests = dbmon.get_monitor_entries(self.dataset.id) self.assertEqual(len(mon_requests), len(mon_srcs)) # mon requests is a list of tuples [(id,ra,decl)] # Ensure sorted by RA for cross-checking: mon_requests = sorted(mon_requests, key=lambda s: s[1]) for idx in range(len(mon_srcs)): self.assertAlmostEqual(mon_requests[idx][1], mon_srcs[idx].ra) self.assertAlmostEqual(mon_requests[idx][2], mon_srcs[idx].dec) #Insert fits for the in-field sources and then associate dbgen.insert_extracted_sources( img.id, [superimposed_mon_src, mon_src_in_field], 'ff_ms', ff_monitor_ids=[mon_requests[0][0], mon_requests[1][0]]) dbmon.associate_ms(img.id) query = """\ SELECT r.id ,r.mon_src ,rf.f_datapoints FROM runningcatalog r ,runningcatalog_flux rf WHERE r.dataset = %(dataset_id)s AND rf.runcat = r.id ORDER BY r.wm_ra ,r.mon_src """ cursor = tkp.db.execute(query, {'dataset_id': self.dataset.id}) runcat_flux = get_db_rows_as_dicts(cursor) self.assertEqual(len(runcat_flux), 3) # First entry (lowest RA, mon_src = False) is the regular one; self.assertEqual(runcat_flux[0]['mon_src'], False) # The higher RA source is the monitoring one self.assertEqual(runcat_flux[1]['mon_src'], True) self.assertEqual(runcat_flux[2]['mon_src'], True) for entry in runcat_flux: self.assertEqual(entry['f_datapoints'], len(self.im_params)) #Let's verify the association types blind_src_assocs = get_assoc_entries(self.dataset.database, runcat_flux[0]['id']) superimposed_mon_src_assocs = get_assoc_entries( self.dataset.database, runcat_flux[1]['id']) offset_mon_src_assocs = get_assoc_entries(self.dataset.database, runcat_flux[2]['id']) assoc_lists = [ blind_src_assocs, superimposed_mon_src_assocs, offset_mon_src_assocs ] for al in assoc_lists: self.assertEqual(len(al), 3) # The individual light-curve datapoints for the "normal" source # It was new at first timestep self.assertEqual(blind_src_assocs[0]['type'], 4) self.assertEqual(superimposed_mon_src_assocs[0]['type'], 8) self.assertEqual(offset_mon_src_assocs[0]['type'], 8) for idx, img_pars in enumerate(self.im_params): if idx != 0: self.assertEqual(blind_src_assocs[idx]['type'], 3) self.assertEqual(superimposed_mon_src_assocs[idx]['type'], 9) self.assertEqual(offset_mon_src_assocs[idx]['type'], 9) #And the extraction types: self.assertEqual(blind_src_assocs[idx]['extract_type'], 0) self.assertEqual(superimposed_mon_src_assocs[idx]['extract_type'], 2) self.assertEqual(offset_mon_src_assocs[idx]['extract_type'], 2) #Sanity check the timestamps while we're at it for al in assoc_lists: self.assertEqual(al[idx]['taustart_ts'], img_pars['taustart_ts'])
def test_basic_case(self): im_params = self.im_params blind_src = db_subs.example_extractedsource_tuple( ra=im_params[0]['centre_ra'], dec=im_params[0]['centre_decl'], ) superimposed_mon_src = blind_src mon_src_in_field = blind_src._replace(ra = blind_src.ra+0.001) # Simulate a source that does not get fit, for good measure: mon_src_out_of_field = blind_src._replace(ra = blind_src.ra+90.) #Sorted by increasing RA: mon_srcs = [superimposed_mon_src, mon_src_in_field, mon_src_out_of_field] mon_posns = [(m.ra, m.dec) for m in mon_srcs] dbgen.insert_monitor_positions(self.dataset.id,mon_posns) images = [] for img_pars in self.im_params: img = tkp.db.Image(dataset=self.dataset, data=img_pars) dbgen.insert_extracted_sources(img.id, [blind_src], 'blind') associate_extracted_sources(img.id, deRuiter_r=5.68) nd_requests = get_nulldetections(img.id) self.assertEqual(len(nd_requests),0) mon_requests = dbmon.get_monitor_entries(self.dataset.id) self.assertEqual(len(mon_requests),len(mon_srcs)) # mon requests is a list of tuples [(id,ra,decl)] # Ensure sorted by RA for cross-checking: mon_requests = sorted(mon_requests, key = lambda s: s[1]) for idx in range(len(mon_srcs)): self.assertAlmostEqual(mon_requests[idx][1],mon_srcs[idx].ra) self.assertAlmostEqual(mon_requests[idx][2],mon_srcs[idx].dec) #Insert fits for the in-field sources and then associate dbgen.insert_extracted_sources(img.id, [superimposed_mon_src, mon_src_in_field], 'ff_ms', ff_monitor_ids=[mon_requests[0][0], mon_requests[1][0]]) dbmon.associate_ms(img.id) query = """\ SELECT r.id ,r.mon_src ,rf.f_datapoints FROM runningcatalog r ,runningcatalog_flux rf WHERE r.dataset = %(dataset_id)s AND rf.runcat = r.id ORDER BY r.wm_ra ,r.mon_src """ cursor = tkp.db.execute(query, {'dataset_id': self.dataset.id}) runcat_flux = get_db_rows_as_dicts(cursor) self.assertEqual(len(runcat_flux), 3) # First entry (lowest RA, mon_src = False) is the regular one; self.assertEqual(runcat_flux[0]['mon_src'], False) # The higher RA source is the monitoring one self.assertEqual(runcat_flux[1]['mon_src'], True) self.assertEqual(runcat_flux[2]['mon_src'], True) for entry in runcat_flux: self.assertEqual(entry['f_datapoints'], len(self.im_params)) #Let's verify the association types blind_src_assocs = get_assoc_entries(self.dataset.database, runcat_flux[0]['id']) superimposed_mon_src_assocs = get_assoc_entries(self.dataset.database, runcat_flux[1]['id']) offset_mon_src_assocs = get_assoc_entries(self.dataset.database, runcat_flux[2]['id']) assoc_lists = [blind_src_assocs, superimposed_mon_src_assocs, offset_mon_src_assocs] for al in assoc_lists: self.assertEqual(len(al), 3) # The individual light-curve datapoints for the "normal" source # It was new at first timestep self.assertEqual(blind_src_assocs[0]['type'], 4) self.assertEqual(superimposed_mon_src_assocs[0]['type'], 8) self.assertEqual(offset_mon_src_assocs[0]['type'], 8) for idx, img_pars in enumerate(self.im_params): if idx != 0: self.assertEqual(blind_src_assocs[idx]['type'], 3) self.assertEqual(superimposed_mon_src_assocs[idx]['type'], 9) self.assertEqual(offset_mon_src_assocs[idx]['type'], 9) #And the extraction types: self.assertEqual(blind_src_assocs[idx]['extract_type'],0) self.assertEqual(superimposed_mon_src_assocs[idx]['extract_type'],2) self.assertEqual(offset_mon_src_assocs[idx]['extract_type'],2) #Sanity check the timestamps while we're at it for al in assoc_lists: self.assertEqual(al[idx]['taustart_ts'], img_pars['taustart_ts'])
def run(job_name, mon_coords, local=False): setup_event_listening(celery_app) pipe_config = initialize_pipeline_config( os.path.join(os.getcwd(), "pipeline.cfg"), job_name) debug = pipe_config.logging.debug #Setup logfile before we do anything else log_dir = pipe_config.logging.log_dir setup_log_file(log_dir, debug) job_dir = pipe_config.DEFAULT.job_directory if not os.access(job_dir, os.X_OK): msg = "can't access job folder %s" % job_dir logger.error(msg) raise IOError(msg) logger.info("Job dir: %s", job_dir) db_config = get_database_config(pipe_config.database, apply=True) dump_database_backup(db_config, job_dir) job_config = load_job_config(pipe_config) se_parset = job_config.source_extraction deruiter_radius = job_config.association.deruiter_radius all_images = imp.load_source('images_to_process', os.path.join(job_dir, 'images_to_process.py')).images logger.info("dataset %s contains %s images" % (job_name, len(all_images))) logger.info("performing database consistency check") if not dbconsistency.check(): logger.error("Inconsistent database found; aborting") return 1 dataset_id = create_dataset(job_config.persistence.dataset_id, job_config.persistence.description) if job_config.persistence.dataset_id == -1: store_config(job_config, dataset_id) # new data set else: job_config_from_db = fetch_config(dataset_id) # existing data set if check_job_configs_match(job_config, job_config_from_db): logger.debug("Job configs from file / database match OK.") else: logger.warn("Job config file has changed since dataset was " "first loaded into database. ") logger.warn("Using job config settings loaded from database, see " "log dir for details") job_config = job_config_from_db dump_configs_to_logdir(log_dir, job_config, pipe_config) logger.info("performing persistence step") image_cache_params = pipe_config.image_cache imgs = [[img] for img in all_images] metadatas = runner(tasks.persistence_node_step, imgs, [image_cache_params], local) metadatas = [m[0] for m in metadatas] logger.info("Storing images") image_ids = store_images(metadatas, job_config.source_extraction.extraction_radius_pix, dataset_id) db_images = [Image(id=image_id) for image_id in image_ids] logger.info("performing quality check") urls = [img.url for img in db_images] arguments = [job_config] rejecteds = runner(tasks.quality_reject_check, urls, arguments, local) good_images = [] for image, rejected in zip(db_images, rejecteds): if rejected: reason, comment = rejected steps.quality.reject_image(image.id, reason, comment) else: good_images.append(image) if not good_images: logger.warn("No good images under these quality checking criteria") return grouped_images = group_per_timestep(good_images) timestep_num = len(grouped_images) for n, (timestep, images) in enumerate(grouped_images): msg = "processing %s images in timestep %s (%s/%s)" logger.info(msg % (len(images), timestep, n+1, timestep_num)) logger.info("performing source extraction") urls = [img.url for img in images] arguments = [se_parset] extract_sources = runner(tasks.extract_sources, urls, arguments, local) logger.info("storing extracted to database") for image, sources in zip(images, extract_sources): dbgen.insert_extracted_sources(image.id, sources, 'blind') logger.info("performing database operations") for image in images: logger.info("performing DB operations for image %s" % image.id) logger.info("performing source association") dbass.associate_extracted_sources(image.id, deRuiter_r=deruiter_radius) logger.info("performing null detections") null_detections = dbnd.get_nulldetections(image.id) logger.info("Found %s null detections" % len(null_detections)) # Only if we found null_detections the next steps are necessary if len(null_detections) > 0: logger.info("performing forced fits") ff_nd = forced_fits(image.url, null_detections, se_parset) dbgen.insert_extracted_sources(image.id, ff_nd, 'ff_nd') logger.info("adding null detections") dbnd.associate_nd(image.id) if len(mon_coords) > 0: logger.info("performing monitoringlist") ff_ms = forced_fits(image.url, mon_coords, se_parset) dbgen.insert_extracted_sources(image.id, ff_ms, 'ff_ms') logger.info("adding monitoring sources") dbmon.associate_ms(image.id) transients = search_transients(image.id, job_config['transient_search']) dbgen.update_dataset_process_end_ts(dataset_id)
def test_monitoringSource(self): data = {'description': "monitoringlist:" + self._testMethodName} dataset = DataSet(data=data) # Three timesteps, 1 band -> 3 images. taustart_tss = [datetime.datetime(2013, 8, 1), datetime.datetime(2013, 9, 1), datetime.datetime(2013, 10, 1)] #freq_effs = [124, 149, 156, 185] freq_effs = [124] freq_effs = [f * 1e6 for f in freq_effs] im_params = db_subs.example_dbimage_datasets(len(freq_effs) * len(taustart_tss)) timestamps = itertools.repeat(taustart_tss, len(freq_effs)) for im, freq, ts in zip(im_params, itertools.cycle(freq_effs), itertools.chain.from_iterable(zip(*timestamps))): im['freq_eff'] = freq im['taustart_ts'] = ts images = [] for im in im_params: image = tkp.db.Image(dataset=dataset, data=im) images.append(image) # Arbitrary parameters, except that they fall inside our image. # We have one to be monitored source and one "normal" source src0 = db_subs.example_extractedsource_tuple(ra=122.5, dec=9.5) src1_mon = db_subs.example_extractedsource_tuple(ra=123.5, dec=10.5) # Group images in blocks of 1, corresponding to all frequency bands at # a given timestep. for images in zip(*(iter(images),) * len(freq_effs)): for image in images: # The "normal" source is seen at all timesteps dbgen.insert_extracted_sources(image.id, [src0], 'blind') for image in images: dbass.associate_extracted_sources(image.id, deRuiter_r=5.68) # The monitoring sources are the positional inputs for the forced # fits, which on their turn return additional parameters, # e.g. from src0_mon # src1_mon is the monitoring source at all timesteps dbgen.insert_extracted_sources(image.id, [src1_mon], 'ff_ms') # And here we have to associate the monitoring sources with the # runcat sources... dbmon.associate_ms(image.id) query = """\ SELECT id ,mon_src FROM runningcatalog r WHERE dataset = %(dataset_id)s AND datapoints = 3 ORDER BY id """ cursor = tkp.db.execute(query, {'dataset_id': dataset.id}) result = cursor.fetchall() # We should have two runningcatalog sources, one for the "normal" # source and one for the monitoring source. # Both should have three datapoints. print "dp_result:",result self.assertEqual(len(result), 2) # The first source is the "normal" one self.assertEqual(result[0][1], False) # The second source is the monitoring one self.assertEqual(result[1][1], True) query = """\ SELECT r.id ,r.mon_src ,rf.f_datapoints FROM runningcatalog r ,runningcatalog_flux rf WHERE r.dataset = %(dataset_id)s AND rf.runcat = r.id ORDER BY r.id """ cursor = tkp.db.execute(query, {'dataset_id': dataset.id}) result = cursor.fetchall() # We should have two runningcatalog_flux entries, # one for every source, where every source has # three f_datapoints self.assertEqual(len(result), 2) # "Normal" source: three flux datapoints self.assertEqual(result[0][1], False) self.assertEqual(result[0][2], 3) # Monitoring source: three flux datapoints self.assertEqual(result[1][1], True) self.assertEqual(result[1][2], 3) # We should also have two lightcurves for both sources, # where both sources have three datapoints. # The association types of the "normal" source are # 3 (first) or 4 (later ones), while the monitoring source # association types are 8 (first) or 9 (later ones). query = """\ SELECT a.runcat ,a.xtrsrc ,a.type ,i.taustart_ts ,r.mon_src ,x.extract_type FROM assocxtrsource a ,extractedsource x ,image i ,runningcatalog r WHERE a.xtrsrc = x.id AND x.image = i.id AND i.dataset = %(dataset_id)s AND a.runcat = r.id ORDER BY a.runcat ,i.taustart_ts """ cursor = tkp.db.execute(query, {'dataset_id': dataset.id}) result = cursor.fetchall() # 3 + 3 entries for source 1 and 2 resp. self.assertEqual(len(result), 6) # The individual light-curve datapoints for the "normal" source # It was new at first timestep self.assertEqual(result[0][2], 4) self.assertEqual(result[0][3], taustart_tss[0]) self.assertEqual(result[0][4], False) self.assertEqual(result[0][5], 0) # It was known at second timestep self.assertEqual(result[1][2], 3) self.assertEqual(result[1][3], taustart_tss[1]) self.assertEqual(result[1][4], result[0][4]) self.assertEqual(result[1][5], result[0][5]) # It was known at third timestep self.assertEqual(result[2][2], result[1][2]) self.assertEqual(result[2][3], taustart_tss[2]) self.assertEqual(result[2][4], result[1][4]) self.assertEqual(result[2][5], result[1][5]) # The individual light-curve datapoints for the monitoring source # It was new at first timestep self.assertEqual(result[3][2], 8) self.assertEqual(result[3][3], taustart_tss[0]) self.assertEqual(result[3][4], True) self.assertEqual(result[3][5], 2) # It was known at second timestep self.assertEqual(result[4][2], 9) self.assertEqual(result[4][3], taustart_tss[1]) self.assertEqual(result[4][4], result[3][4]) self.assertEqual(result[4][5], result[3][5]) # It was known at third timestep self.assertEqual(result[5][2], result[4][2]) self.assertEqual(result[5][3], taustart_tss[2]) self.assertEqual(result[5][4], result[4][4]) self.assertEqual(result[5][5], result[4][5])