def testFSGenerator(self):
object_ids = np.array([1, 2, 3, 4, 5, 6, 7, 8])
object_ra_decs = [(1.1, 1.1), (1.1, 1.2), (1.1, 1.3), (2.0, 2.1),
(3.0, 3.1), (3.1, 3.0), (3.05, 3.05), (3.1, 3.1)]
visit_ids = np.array([101, 102, 103, 104])
visit_ra_decs = [(1.0, 1.0), (2.0, 2.0), (3.0, 3.0), (3.02, 3.02)]
visit_df = pd.DataFrame({
'ccdVisitId': visit_ids,
'ra': [x[0] for x in visit_ra_decs],
'decl': [x[1] for x in visit_ra_decs],
'filterName': ['g'] * len(visit_ids)
})
object_df = pd.DataFrame({
'objectId': object_ids,
'ra': [x[0] for x in object_ra_decs],
'decl': [x[1] for x in object_ra_decs],
'mag_g': [20] * len(object_ids)
})
prereq_tables = {'CcdVisit': visit_df, 'Object': object_df}
cell_id = 1
length = 0
seed = 1
fs_generator = columns.ForcedSourceGenerator(visit_radius=1.00)
# Expected result in the form of a list for each object,
# containing (objectId, visitId) tuples.
expected = [[(1, 101), (1, 102)], [(2, 101), (2, 102)],
[(3, 101), (3, 102)], [(4, 101), (4, 102)]]
box = columns.SimpleBox(0.5, 0.5, 2.5, 2.5)
chunk_center = SkyCoord(1.5, 1.5, frame="icrs", unit="deg")
for object_row_id, expected_res in enumerate(expected):
fs_output = fs_generator(box,
length,
seed,
prereq_tables=prereq_tables,
box_center=chunk_center)
output_obj_ids, output_ccdvisits, _, _ = fs_output
self.assertEqual(len(output_obj_ids), len(expected_res))
for res_row_obj, res_row_visit in expected_res:
self.assertIn(res_row_obj, output_obj_ids)
self.assertIn(res_row_visit, output_ccdvisits)
def testForcedSource(self):
with tempfile.TemporaryDirectory() as data_dir:
with open(os.path.join(data_dir, "visit_table.csv"), "w") as f:
print(chunk3525_visits, file=f)
generator_spec = {
"Object": {
"columns": {
"objectId":
columns.ObjIdGenerator(),
"psRa,psDecl":
columns.RaDecGenerator(),
"uPsFlux,gPsFlux,rPsFlux,iPsFlux,zPsFlux,yPsFlux":
columns.MagnitudeGenerator(n_mags=6)
},
"density": UniformSpatialModel(100),
},
"CcdVisit": {
"from_file": os.path.join(data_dir, "visit_table.csv"),
"columns": "ccdVisitId,filterName,ra,decl"
},
"ForcedSource": {
"prereq_tables": ["CcdVisit", "Object"],
"columns": {
"objectId,ccdVisitId,psFlux,psFlux_Sigma":
columns.ForcedSourceGenerator(visit_radius=3.0,
filters="ugr"),
},
}
}
chunk_id = 3525
seed = 1
generator = DataGenerator(generator_spec, chunker, seed=seed)
edge_width = 0.017 # degrees
chunk_tables = generator.make_chunk(chunk_id,
edge_width=edge_width,
edge_only=False,
return_pregenerated=True)
self.assertIn("ForcedSource", chunk_tables.keys())
self.assertIn("Object", chunk_tables.keys())
print(len(chunk_tables['ForcedSource']))
print(len(chunk_tables['Object']))
self.assertEqual(set(chunk_tables['ForcedSource']['ccdVisitId']),
set(chunk_tables['CcdVisit']['ccdVisitId']))
def testFSGenerator(self):
object_ids = np.array([1, 2, 3, 4, 5, 6, 7, 8])
object_ra_decs = [(1.1, 1.1), (1.1, 1.2), (1.1, 1.3), (2.0, 2.1),
(3.0, 3.1), (3.1, 3.0), (3.05, 3.05), (3.1, 3.1)]
visit_ids = np.array([101, 102, 103, 104])
visit_ra_decs = [(1.0, 1.0), (2.0, 2.0), (3.0, 3.0), (3.02, 3.02)]
visit_df = pd.DataFrame({
'ccdVisitId': visit_ids,
'ra': [x[0] for x in visit_ra_decs],
'decl': [x[1] for x in visit_ra_decs],
'filterName': ['g'] * len(visit_ids)
})
prereq_tables = {'CcdVisit': visit_df}
object_df = pd.DataFrame({
'objectId': object_ids,
'ra': [x[0] for x in object_ra_decs],
'decl': [x[1] for x in object_ra_decs],
'mag_g': [20] * len(object_ids)
})
cell_id = 1
length = 0
fs_generator = columns.ForcedSourceGenerator(visit_radius=0.30)
# Expected result in the form of a list for each object,
# containing (objectId, visitId) tuples.
expected = [[(1, 101)], [(2, 101)], [], [(4, 102)], [(5, 103),
(5, 104)],
[(6, 103), (6, 104)], [(7, 103), (7, 104)],
[(8, 103), (8, 104)]]
for object_row_id, expected_res in enumerate(expected):
fs_output = fs_generator(cell_id,
length,
prereq_row=object_df.iloc[object_row_id],
prereq_tables=prereq_tables)
output_obj_ids, output_ccdvisits, _, _ = fs_output
self.assertEqual(len(output_obj_ids), len(expected_res))
for res_row_obj, res_row_visit in expected_res:
self.assertIn(res_row_obj, output_obj_ids)
self.assertIn(res_row_visit, output_ccdvisits)
def testForcedSource(self):
generator_spec = {
"Object": {
"columns": {"objectId": columns.ObjIdGenerator(),
"ra,decl": columns.RaDecGenerator(chunker),
"mag_u,mag_g,mag_r": columns.MagnitudeGenerator(n_mags=3)
}
},
"CcdVisit": {
"columns": {"ccdVisitId": columns.VisitIdGenerator(),
"filterName": columns.FilterGenerator(filters="ugr"),
"ra,decl": columns.RaDecGenerator(chunker)
}
},
"ForcedSource": {
"prereq_row": "Object",
"prereq_tables": ["CcdVisit"],
"columns": {
"objectId,ccdVisitId,psFlux,psFlux_Sigma":
columns.ForcedSourceGenerator(visit_radius=3.0, filters="ugr"),
},
}
}
chunk_id = 3525
generator = DataGenerator(generator_spec)
row_counts = {"CcdVisit": 80,
"Object": 100,
"ForcedSource": 0}
chunk_tables = generator.make_chunk(chunk_id, row_counts)
self.assertIn("ForcedSource", chunk_tables.keys())
self.assertIn("Object", chunk_tables.keys())
self.assertIn("CcdVisit", chunk_tables.keys())
print(len(chunk_tables['ForcedSource']))
print(len(chunk_tables['CcdVisit']))
print(len(chunk_tables['Object']))
self.assertEqual(set(chunk_tables['ForcedSource']['ccdVisitId']),
set(chunk_tables['CcdVisit']['ccdVisitId']))
"uApN,gApN,rApN,iApN,zApN,yApN":
columns.PoissonGenerator(mean_val=10, n_columns=6, column_seed=27),
"extendedness":
columns.UniformGenerator(max_val=10, n_columns=1, column_seed=28),
"FLAGS1,FLAGS2":
columns.PoissonGenerator(mean_val=1000,
n_columns=2,
column_seed=29),
},
"density": UniformSpatialModel(327248),
},
"ForcedSource": {
"prereq_tables": ["CcdVisit", "Object"],
"columns": {
"objectId,ccdVisitId,psFlux,psFlux_Err,flags":
columns.ForcedSourceGenerator(visit_radius=1.7, filters="ugriz"),
},
},
"Source": {
"prereq_tables": ["CcdVisit", "Object"],
"columns": {
"sourceId:SID,ccdVisitId:VID,filterName:CHAR(1),objectId:OID,ssObjectId:BIGINT,"
"parentSourceId:BIGINT,htmId20:BIGINT,sky:FLOAT,skyErr:FLOATE,"
"psFlux:FLOAT,psFluxErr:FLOATE,psX:FLOAT,psXErr:FLOATE,psY:FLOAT,psYErr:FLOATE,"
"psFlux_psX_Cov:FLOAT,psFlux_psY_Cov:FLOAT,psX_Y_Cov:FLOAT,"
"psLnL:FLOAT,psChi2:FLOAT,psN:INT,psRa:RA,psRaErr:FLOATE,psDecl:DEC,psDeclErr:FLOATE,"
"psFlux_psRa_Cov:FLOAT,psFlux_psDecl_Cov:FLOAT,x:FLOAT,xErr:FLOATE,y:FLOAT,yErr:FLOATE,"
"x_y_Cov:FLOAT,ra:RA,raErr:FLOATE,decl:DEC,declErr:FLOATE,ra_decl_Cov:FLOAT,"
"E1:FLOAT,E1Err:FLOATE,E2:FLOAT,E2Err:FLOATE,E1_E2_Cov:FLOAT,mSum:FLOAT,mSumErr:FLOATE,"
"m4:FLOAT,apN:TINYINT,flags:BIGINT":
columns.SourceGenerator(visit_radius=1.7,