def test_intermittentToMonitorlist(self): dataset = tkp.db.DataSet(database=self.database, data={'description': "Monlist:" + self._testMethodName}) n_images = 3 im_params = db_subs.example_dbimage_datasets(n_images) steady_srcs = [] # We will work with 2 sources per image # one being detected in all images and not in the monlist # the second having a null-detection in the second image # and stored in the monlist n_steady_srcs = 2 for i in range(n_steady_srcs): src = db_subs.example_extractedsource_tuple() src = src._replace(ra=src.ra + 2 * i) steady_srcs.append(src) for idx, im in enumerate(im_params): image = tkp.db.Image(database=self.database, dataset=dataset, data=im) if idx == 1: # The second image has a null detection, so only the first source is detected image.insert_extracted_sources(steady_srcs[0:1]) else: image.insert_extracted_sources(steady_srcs) # First, we check for null detections nd = monitoringlist.get_nulldetections(image.id, deRuiter_r=3.717) if idx == 0: self.assertEqual(len(nd), 0) elif idx == 1: self.assertEqual(len(nd), 1) # The null detection is found, # We simulate the forced fit result back into extractedsource # Check that the null-detection ra is the ra of source two self.assertEqual(nd[0][0], steady_srcs[1].ra) #print "nd=",nd tuple_ff_nd = steady_srcs[1:2] monitoringlist.insert_forcedfits_into_extractedsource(image.id, tuple_ff_nd, 'ff_nd') elif idx == 2: self.assertEqual(len(nd), 0) # Secondly, we do the source association dbass.associate_extracted_sources(image.id, deRuiter_r=3.717) monitoringlist.add_nulldetections(image.id) # We also need to run the transient search in order to pick up the variable # eta_lim, V_lim, prob_threshold, minpoints, resp. transients = tr_search.multi_epoch_transient_search(image.id, 0.0, 0.0, 0.5, 1) # Adjust (insert/update/remove) transients in monlist as well monitoringlist.adjust_transients_in_monitoringlist(image.id, transients) # So after the three images have been processed, # We should have the null-detection source in the monlist # Get the null detection in extractedsource # These are of extract_type = 1 query = """\ select x.id from extractedsource x ,image i where x.image = i.id and i.dataset = %s and x.extract_type = 1 """ self.database.cursor.execute(query, (dataset.id,)) result = zip(*self.database.cursor.fetchall()) null_det = result[0] self.assertEqual(len(null_det), 1) query = """\ select a.runcat ,a.xtrsrc ,r.wm_ra ,r.wm_decl from assocxtrsource a ,extractedsource x ,image i ,runningcatalog r where a.xtrsrc = x.id and x.id = %s and x.image = i.id and i.dataset = %s and a.runcat = r.id and r.dataset = i.dataset """ self.database.cursor.execute(query, (null_det[0], dataset.id,)) result = zip(*self.database.cursor.fetchall()) assocruncat = result[0] xtrsrc = result[1] wm_ra = result[2] wm_decl = result[3] self.assertEqual(len(assocruncat), 1) query = """\ SELECT runcat ,ra ,decl FROM monitoringlist WHERE dataset = %s """ self.database.cursor.execute(query, (dataset.id,)) result = zip(*self.database.cursor.fetchall()) # print "len(result)=",len(result) self.assertEqual(len(result), 3) #self.assertEqual(0, 1) monruncat = result[0] ra = result[1] decl = result[2] self.assertEqual(len(monruncat), 1) self.assertEqual(monruncat[0], assocruncat[0]) self.assertEqual(ra[0], wm_ra[0]) self.assertAlmostEqual(decl[0], wm_decl[0])
def run(job_name, local=False): pipe_config = initialize_pipeline_config( os.path.join(os.getcwd(), "pipeline.cfg"), job_name) database_config(pipe_config) job_dir = pipe_config.get('layout', 'job_directory') if not os.access(job_dir, os.X_OK): msg = "can't access job folder %s" % job_dir logger.error(msg) raise IOError(msg) logger.info("Job dir: %s", job_dir) images = imp.load_source('images_to_process', os.path.join(job_dir, 'images_to_process.py')).images logger.info("dataset %s contains %s images" % (job_name, len(images))) job_config = load_job_config(pipe_config) dump_job_config_to_logdir(pipe_config, job_config) p_parset = parset.load_section(job_config, 'persistence') se_parset = parset.load_section(job_config, 'source_extraction') nd_parset = parset.load_section(job_config, 'null_detections') tr_parset = parset.load_section(job_config, 'transient_search') # persistence imgs = [[img] for img in images] arguments = [p_parset] metadatas = runner(tasks.persistence_node_step, imgs, arguments, local) metadatas = [m[0] for m in metadatas] dataset_id, image_ids = steps.persistence.master_steps(metadatas, se_parset['radius'], p_parset) # manual monitoringlist entries if not add_manual_monitoringlist_entries(dataset_id, []): return 1 images = [Image(id=image_id) for image_id in image_ids] # quality_check urls = [img.url for img in images] arguments = [job_config] rejecteds = runner(tasks.quality_reject_check, urls, arguments, local) good_images = [] for image, rejected in zip(images, rejecteds): if rejected: reason, comment = rejected steps.quality.reject_image(image.id, reason, comment) else: good_images.append(image) if not good_images: logger.warn("No good images under these quality checking criteria") return # Sourcefinding urls = [img.url for img in good_images] arguments = [se_parset] extract_sources = runner(tasks.extract_sources, urls, arguments, local) for image, sources in zip(good_images, extract_sources): dbgen.insert_extracted_sources(image.id, sources, 'blind') # null_detections deRuiter_radius = nd_parset['deruiter_radius'] null_detectionss = [dbmon.get_nulldetections(image.id, deRuiter_radius) for image in good_images] iters = zip([i.url for i in good_images], null_detectionss) arguments = [nd_parset] ff_nds = runner(tasks.forced_fits, iters, arguments, local) for image, ff_nd in zip(good_images, ff_nds): dbgen.insert_extracted_sources(image.id, ff_nd, 'ff_nd') for image in good_images: logger.info("performing DB operations for image %s" % image.id) dbass.associate_extracted_sources(image.id, deRuiter_r=deRuiter_radius) dbmon.add_nulldetections(image.id) transients = steps.transient_search.search_transients(image.id, tr_parset) dbmon.adjust_transients_in_monitoringlist(image.id, transients) for transient in transients: steps.feature_extraction.extract_features(transient) # ingred.classification.classify(transient, cl_parset) now = datetime.datetime.utcnow() dbgen.update_dataset_process_ts(dataset_id, now)