def testObjectTable(self):
generator_spec = {
"Object": {
"columns": {
"objectId":
columns.ObjIdGenerator(),
"ra,decl":
columns.RaDecGenerator(),
"uPsFlux,gPsFlux,rPsFlux,iPsFlux,zPsFlux,yPsFlux":
columns.MagnitudeGenerator(n_mags=6)
},
"density": UniformSpatialModel(500),
}
}
chunk_id = 3525
seed = 1
generator = DataGenerator(generator_spec, chunker, seed=seed)
edge_width = 0.017 # degrees
data = generator.make_chunk(chunk_id,
edge_width=edge_width,
edge_only=False)
chunk_area = chunker.getChunkBounds(
chunk_id).getArea() / RAD_PER_DEG**2
self.assertIn('Object', data.keys())
self.assertAlmostEqual(len(data["Object"]),
int(500 * chunk_area),
delta=3)
def testObjIdGenerator(self):
obs_generator = columns.ObjIdGenerator()
cell_id = 5000
output_ids = obs_generator(cell_id, 20)
self.assertEqual(len(output_ids), 20)
# Test for uniqueness
self.assertEqual(len(set(output_ids)), 20)
output_ids2 = obs_generator(cell_id + 1, 20)
# Test for uniqueness
self.assertEqual(len(set(output_ids) | set(output_ids2)), 40)
def testObjIdGenerator(self):
obs_generator = columns.ObjIdGenerator()
box = columns.SimpleBox(0.5, 0.5, 2.5, 2.5)
cell_id = 5000
seed = 1
output_ids = obs_generator(box, 20, seed, unique_box_id=0)
self.assertEqual(len(output_ids), 20)
# Test for uniqueness
self.assertEqual(len(set(output_ids)), 20)
output_ids2 = obs_generator(box, 20, seed, unique_box_id=1)
# Test for uniqueness
self.assertEqual(len(set(output_ids) | set(output_ids2)), 40)
def testForcedSource(self):
with tempfile.TemporaryDirectory() as data_dir:
with open(os.path.join(data_dir, "visit_table.csv"), "w") as f:
print(chunk3525_visits, file=f)
generator_spec = {
"Object": {
"columns": {
"objectId":
columns.ObjIdGenerator(),
"psRa,psDecl":
columns.RaDecGenerator(),
"uPsFlux,gPsFlux,rPsFlux,iPsFlux,zPsFlux,yPsFlux":
columns.MagnitudeGenerator(n_mags=6)
},
"density": UniformSpatialModel(100),
},
"CcdVisit": {
"from_file": os.path.join(data_dir, "visit_table.csv"),
"columns": "ccdVisitId,filterName,ra,decl"
},
"ForcedSource": {
"prereq_tables": ["CcdVisit", "Object"],
"columns": {
"objectId,ccdVisitId,psFlux,psFlux_Sigma":
columns.ForcedSourceGenerator(visit_radius=3.0,
filters="ugr"),
},
}
}
chunk_id = 3525
seed = 1
generator = DataGenerator(generator_spec, chunker, seed=seed)
edge_width = 0.017 # degrees
chunk_tables = generator.make_chunk(chunk_id,
edge_width=edge_width,
edge_only=False,
return_pregenerated=True)
self.assertIn("ForcedSource", chunk_tables.keys())
self.assertIn("Object", chunk_tables.keys())
print(len(chunk_tables['ForcedSource']))
print(len(chunk_tables['Object']))
self.assertEqual(set(chunk_tables['ForcedSource']['ccdVisitId']),
set(chunk_tables['CcdVisit']['ccdVisitId']))
def testObjectTable(self):
generator_spec = {
"Object": {
"columns": {"objectId": columns.ObjIdGenerator(),
"ra,decl": columns.RaDecGenerator(chunker),
"mag_u,mag_g,mag_r": columns.MagnitudeGenerator(n_mags=3)
}
}
}
cell_id = 5000
generator = DataGenerator(generator_spec)
data = generator.make_chunk(cell_id, num_rows=50)
self.assertIn('Object', data.keys())
self.assertEqual(len(data['Object']), 50)
def testForcedSource(self):
generator_spec = {
"Object": {
"columns": {"objectId": columns.ObjIdGenerator(),
"ra,decl": columns.RaDecGenerator(chunker),
"mag_u,mag_g,mag_r": columns.MagnitudeGenerator(n_mags=3)
}
},
"CcdVisit": {
"columns": {"ccdVisitId": columns.VisitIdGenerator(),
"filterName": columns.FilterGenerator(filters="ugr"),
"ra,decl": columns.RaDecGenerator(chunker)
}
},
"ForcedSource": {
"prereq_row": "Object",
"prereq_tables": ["CcdVisit"],
"columns": {
"objectId,ccdVisitId,psFlux,psFlux_Sigma":
columns.ForcedSourceGenerator(visit_radius=3.0, filters="ugr"),
},
}
}
chunk_id = 3525
generator = DataGenerator(generator_spec)
row_counts = {"CcdVisit": 80,
"Object": 100,
"ForcedSource": 0}
chunk_tables = generator.make_chunk(chunk_id, row_counts)
self.assertIn("ForcedSource", chunk_tables.keys())
self.assertIn("Object", chunk_tables.keys())
self.assertIn("CcdVisit", chunk_tables.keys())
print(len(chunk_tables['ForcedSource']))
print(len(chunk_tables['CcdVisit']))
print(len(chunk_tables['Object']))
self.assertEqual(set(chunk_tables['ForcedSource']['ccdVisitId']),
set(chunk_tables['CcdVisit']['ccdVisitId']))
import lsst.dax.data_generator.columns as columns
from lsst.dax.data_generator import Chunker, UniformSpatialModel
# Verify files in ingestCfgs all have correct database name, stripes, and substripes.
num_stripes = 340
num_substripes = 3
chunker = Chunker(0, num_stripes, num_substripes)
edge_width = 0.0167 # degrees, must be >= overlap
spec = {
"Object": {
"columns": {
"objectId":
columns.ObjIdGenerator(),
"parentObjectId,prv_inputId":
columns.PoissonGenerator(mean_val=10, n_columns=2, column_seed=1),
"psRa,psRaErr,psDecl,psDeclErr":
columns.RaDecGenerator(include_err=True),
"psMuRa,psMuRaErr,psMuDecl,psMuDeclErr,psParallax,psParallaxErr":
columns.UniformGenerator(n_columns=6,
min_val=-5,
max_val=5,
column_seed=2),
"uPsFlux":
columns.MagnitudeGenerator(n_mags=1, column_seed=102),
"uPsFluxErr":
columns.UniformGenerator(min_val=0.05,
max_val=0.5,
n_columns=1,
column_seed=103),
"gPsFlux":
import lsst.dax.data_generator.columns as columns
from lsst.dax.data_generator import Chunker
num_stripes = 200
num_substripes = 5
chunker = Chunker(0, num_stripes, num_substripes)
spec = {
"Object": {
"columns": {
"objectId": columns.ObjIdGenerator(),
"ra,decl": columns.RaDecGenerator(chunker),
"mag_u,mag_g,mag_r,mag_i,mag_z":
columns.MagnitudeGenerator(n_mags=5)
}
},
"CcdVisit": {
"columns": {
"ccdVisitId": columns.VisitIdGenerator(),
"filterName": columns.FilterGenerator(filters="ugriz"),
"ra,decl": columns.RaDecGenerator(chunker)
}
},
"ForcedSource": {
"prereq_row": "Object",
"prereq_tables": ["CcdVisit"],
"columns": {
"objectId,ccdVisitId,psFlux,psFlux_Sigma":
columns.ForcedSourceGenerator(visit_radius=1.4, filters="ugriz"),
},
}