def testObjectTable(self):
generator_spec = {
"Object": {
"columns": {
"objectId":
columns.ObjIdGenerator(),
"ra,decl":
columns.RaDecGenerator(),
"uPsFlux,gPsFlux,rPsFlux,iPsFlux,zPsFlux,yPsFlux":
columns.MagnitudeGenerator(n_mags=6)
},
"density": UniformSpatialModel(500),
}
}
chunk_id = 3525
seed = 1
generator = DataGenerator(generator_spec, chunker, seed=seed)
edge_width = 0.017 # degrees
data = generator.make_chunk(chunk_id,
edge_width=edge_width,
edge_only=False)
chunk_area = chunker.getChunkBounds(
chunk_id).getArea() / RAD_PER_DEG**2
self.assertIn('Object', data.keys())
self.assertAlmostEqual(len(data["Object"]),
int(500 * chunk_area),
delta=3)
def testForcedSource(self):
generator_spec = {
"Object": {
"columns": {"objectId": columns.ObjIdGenerator(),
"ra,decl": columns.RaDecGenerator(chunker),
"mag_u,mag_g,mag_r": columns.MagnitudeGenerator(n_mags=3)
}
},
"CcdVisit": {
"columns": {"ccdVisitId": columns.VisitIdGenerator(),
"filterName": columns.FilterGenerator(filters="ugr"),
"ra,decl": columns.RaDecGenerator(chunker)
}
},
"ForcedSource": {
"prereq_row": "Object",
"prereq_tables": ["CcdVisit"],
"columns": {
"objectId,ccdVisitId,psFlux,psFlux_Sigma":
columns.ForcedSourceGenerator(visit_radius=3.0, filters="ugr"),
},
}
}
chunk_id = 3525
generator = DataGenerator(generator_spec)
row_counts = {"CcdVisit": 80,
"Object": 100,
"ForcedSource": 0}
chunk_tables = generator.make_chunk(chunk_id, row_counts)
self.assertIn("ForcedSource", chunk_tables.keys())
self.assertIn("Object", chunk_tables.keys())
self.assertIn("CcdVisit", chunk_tables.keys())
print(len(chunk_tables['ForcedSource']))
print(len(chunk_tables['CcdVisit']))
print(len(chunk_tables['Object']))
self.assertEqual(set(chunk_tables['ForcedSource']['ccdVisitId']),
set(chunk_tables['CcdVisit']['ccdVisitId']))
def testCcdVisit(self):
generator_spec = {
"CcdVisit": {
"columns": {"ccdVisitId": columns.VisitIdGenerator(),
"filterName": columns.FilterGenerator(filters="ugr"),
"ra,decl": columns.RaDecGenerator(chunker)
}
},
}
chunk_id = 5000
generator = DataGenerator(generator_spec)
chunk_table = generator.make_chunk(chunk_id, 50)
self.assertIn("CcdVisit", chunk_table.keys())
self.assertEqual(len(chunk_table["CcdVisit"]), 50)
def testForcedSource(self):
with tempfile.TemporaryDirectory() as data_dir:
with open(os.path.join(data_dir, "visit_table.csv"), "w") as f:
print(chunk3525_visits, file=f)
generator_spec = {
"Object": {
"columns": {
"objectId":
columns.ObjIdGenerator(),
"psRa,psDecl":
columns.RaDecGenerator(),
"uPsFlux,gPsFlux,rPsFlux,iPsFlux,zPsFlux,yPsFlux":
columns.MagnitudeGenerator(n_mags=6)
},
"density": UniformSpatialModel(100),
},
"CcdVisit": {
"from_file": os.path.join(data_dir, "visit_table.csv"),
"columns": "ccdVisitId,filterName,ra,decl"
},
"ForcedSource": {
"prereq_tables": ["CcdVisit", "Object"],
"columns": {
"objectId,ccdVisitId,psFlux,psFlux_Sigma":
columns.ForcedSourceGenerator(visit_radius=3.0,
filters="ugr"),
},
}
}
chunk_id = 3525
seed = 1
generator = DataGenerator(generator_spec, chunker, seed=seed)
edge_width = 0.017 # degrees
chunk_tables = generator.make_chunk(chunk_id,
edge_width=edge_width,
edge_only=False,
return_pregenerated=True)
self.assertIn("ForcedSource", chunk_tables.keys())
self.assertIn("Object", chunk_tables.keys())
print(len(chunk_tables['ForcedSource']))
print(len(chunk_tables['Object']))
self.assertEqual(set(chunk_tables['ForcedSource']['ccdVisitId']),
set(chunk_tables['CcdVisit']['ccdVisitId']))
def testObjectTable(self):
generator_spec = {
"Object": {
"columns": {"objectId": columns.ObjIdGenerator(),
"ra,decl": columns.RaDecGenerator(chunker),
"mag_u,mag_g,mag_r": columns.MagnitudeGenerator(n_mags=3)
}
}
}
cell_id = 5000
generator = DataGenerator(generator_spec)
data = generator.make_chunk(cell_id, num_rows=50)
self.assertIn('Object', data.keys())
self.assertEqual(len(data['Object']), 50)
# Verify files in ingestCfgs all have correct database name, stripes, and substripes.
num_stripes = 340
num_substripes = 3
chunker = Chunker(0, num_stripes, num_substripes)
edge_width = 0.0167 # degrees, must be >= overlap
spec = {
"Object": {
"columns": {
"objectId":
columns.ObjIdGenerator(),
"parentObjectId,prv_inputId":
columns.PoissonGenerator(mean_val=10, n_columns=2, column_seed=1),
"psRa,psRaErr,psDecl,psDeclErr":
columns.RaDecGenerator(include_err=True),
"psMuRa,psMuRaErr,psMuDecl,psMuDeclErr,psParallax,psParallaxErr":
columns.UniformGenerator(n_columns=6,
min_val=-5,
max_val=5,
column_seed=2),
"uPsFlux":
columns.MagnitudeGenerator(n_mags=1, column_seed=102),
"uPsFluxErr":
columns.UniformGenerator(min_val=0.05,
max_val=0.5,
n_columns=1,
column_seed=103),
"gPsFlux":
columns.MagnitudeGenerator(n_mags=1, column_seed=104),
"gPsFluxErr":
import lsst.dax.data_generator.columns as columns
from lsst.dax.data_generator import Chunker
num_stripes = 200
num_substripes = 5
chunker = Chunker(0, num_stripes, num_substripes)
spec = {
"Object": {
"columns": {
"objectId": columns.ObjIdGenerator(),
"ra,decl": columns.RaDecGenerator(chunker),
"mag_u,mag_g,mag_r,mag_i,mag_z":
columns.MagnitudeGenerator(n_mags=5)
}
},
"CcdVisit": {
"columns": {
"ccdVisitId": columns.VisitIdGenerator(),
"filterName": columns.FilterGenerator(filters="ugriz"),
"ra,decl": columns.RaDecGenerator(chunker)
}
},
"ForcedSource": {
"prereq_row": "Object",
"prereq_tables": ["CcdVisit"],
"columns": {
"objectId,ccdVisitId,psFlux,psFlux_Sigma":
columns.ForcedSourceGenerator(visit_radius=1.4, filters="ugriz"),
},
}
import lsst.dax.data_generator.columns as columns
from lsst.dax.data_generator import Chunker, UniformSpatialModel
num_stripes = 200
num_substripes = 5
chunker = Chunker(0, num_stripes, num_substripes)
edge_width = 0.017 # degrees, must be >= overlap
spec = {
"Object": {
"columns": {
"objectId": columns.ObjIdGenerator(),
"ra,decl": columns.RaDecGenerator(),
"uPsFlux,gPsFlux,rPsFlux,iPsFlux,zPsFlux,yPsFlux":
columns.MagnitudeGenerator(n_mags=6),
"uPsFluxErr,gPsFluxErr,rPsFluxErr,iPsFluxErr,zPsFluxErr,yPsFluxErr":
columns.UniformGenerator(min_val=0.05, max_val=0.5, n_columns=6, column_seed=6),
"psLnL,psChi2,psN": columns.UniformGenerator(n_columns=6, column_seed=7),
"uBbdRa,uBdRaErr,uBdDecl,uBdDeclErr,uBdE1,uBdE1Err,uBdE2,uBdE2Err,uBdFluxB,uBdFluxBErr,uBdFluxD,uBdFluxDErr,uBdReB,uBdReBErr,uBdReD,uBdReDErr,uBdLnL,uBdChi2,uBdN,gBbdRa,gBdRaErr,gBdDecl,gBdDeclErr,gBdE1,gBdE1Err,gBdE2,gBdE2Err,gBdFluxB,gBdFluxBErr,gBdFluxD,gBdFluxDErr,gBdReB,gBdReBErr,gBdReD,gBdReDErr,gBdLnL,gBdChi2,gBdN,rBbdRa,rBdRaErr,rBdDecl,rBdDeclErr,rBdE1,rBdE1Err,rBdE2,rBdE2Err,rBdFluxB,rBdFluxBErr,rBdFluxD,rBdFluxDErr,rBdReB,rBdReBErr,rBdReD,rBdReDErr,rBdLnL,rBdChi2,rBdN,iBbdRa,iBdRaErr,iBdDecl,iBdDeclErr,iBdE1,iBdE1Err,iBdE2,iBdE2Err,iBdFluxB,iBdFluxBErr,iBdFluxD,iBdFluxDErr,iBdReB,iBdReBErr,iBdReD,iBdReDErr,iBdLnL,iBdChi2,iBdN,zBbdRa,zBdRaErr,zBdDecl,zBdDeclErr,zBdE1,zBdE1Err,zBdE2,zBdE2Err,zBdFluxB,zBdFluxBErr,zBdFluxD,zBdFluxDErr,zBdReB,zBdReBErr,zBdReD,zBdReDErr,zBdLnL,zBdChi2,zBdN,yBbdRa,yBdRaErr,yBdDecl,yBdDeclErr,yBdE1,yBdE1Err,yBdE2,yBdE2Err,yBdFluxB,yBdFluxBErr,yBdFluxD,yBdFluxDErr,yBdReB,yBdReBErr,yBdReD,yBdReDErr,yBdLnL,yBdChi2,yBdN":
columns.UniformGenerator(n_columns=114, column_seed=8),
"ugStd,ugStdErr,grStd,grStdErr,riStd,riStdErr,izStd,izStdErr,zyStd,zyStdErr":
columns.UniformGenerator(min_val=-2, max_val=2, n_columns=10, column_seed=9),
"uRa,uRaErr,uDecl,uDeclErr,gRa,gRaErr,gDecl,gDeclErr,rRa,rRaErr,rDecl,rDeclErr,iRa,iRaErr,iDecl,iDeclErr,zRa,zRaErr,zDecl,zDeclErr,yRa,yRaErr,yDecl,yDeclErr":
columns.UniformGenerator(n_columns=24, column_seed=10),
"uE1,uE1Err,uE2,uE2Err,uE1_E2_Cov,gE1,gE1Err,gE2,gE2Err,gE1_E2_Cov,rE1,rE1Err,rE2,rE2Err,rE1_E2_Cov,iE1,iE1Err,iE2,iE2Err,iE1_E2_Cov,zE1,zE1Err,zE2,zE2Err,zE1_E2_Cov,yE1,yE1Err,yE2,yE2Err,yE1_E2_Cov":
columns.UniformGenerator(min_val=-1, max_val=1, n_columns=30, column_seed=11),
"uMSum,uMSumErr,gMSum,gMSumErr,rMSum,rMSumErr,iMSum,iMSumErr,zMSum,zMSumErr,yMSum,yMSumErr,uM4,gM4,rM4,iM4,zM4,yM4":
columns.UniformGenerator(min_val=-1, max_val=1, n_columns=18, column_seed=12),
"uPetroRad,uPetroRadErr,gPetroRad,gPetroRadErr,rPetroRad,rPetroRadErr,iPetroRad,iPetroRadErr,zPetroRad,zPetroRadErr,yPetroRad,yPetroRadErr,petroFilter,uPetroFlux,uPetroFluxErr,gPetroFlux,gPetroFluxErr,rPetroFlux,rPetroFluxErr,iPetroFlux,iPetroFluxErr,zPetroFlux,zPetroFluxErr,yPetroFlux,yPetroFluxErr,uPetroRad50,uPetroRad50Err,gPetroRad50,gPetroRad50Err,rPetroRad50,rPetroRad50Err,iPetroRad50,iPetroRad50Err,zPetroRad50,zPetroRad50Err,yPetroRad50,yPetroRad50Err,uPetroRad90,uPetroRad90Err,gPetroRad90,gPetroRad90Err,rPetroRad90,rPetroRad90Err,iPetroRad90,iPetroRad90Err,zPetroRad90,zPetroRad90Err,yPetroRad90,yPetroRad90Err":
columns.UniformGenerator(max_val=10, n_columns=49, column_seed=13),
import lsst.dax.data_generator.columns as columns
from lsst.dax.data_generator import Chunker, UniformSpatialModel
num_stripes = 200
num_substripes = 5
chunker = Chunker(0, num_stripes, num_substripes)
edge_width = 0.017 # degrees, must be >= overlap
spec = {
"Object": {
"columns": {"objectId": columns.ObjIdGenerator(),
"psRa,psDecl": columns.RaDecGenerator(),
"uPsFlux,gPsFlux,rPsFlux,iPsFlux,zPsFlux,yPsFlux": columns.MagnitudeGenerator(n_mags=6)
},
"density": UniformSpatialModel(1000),
},
"ForcedSource": {
"prereq_tables": ["CcdVisit", "Object"],
"columns": {
"objectId,ccdVisitId,psFlux,psFlux_Sigma,flags":
columns.ForcedSourceGenerator(visit_radius=1.4, filters="ugriz"),
},
}
}
spec["CcdVisit"] = {
"from_file": "visit_table_chunk3525.csv",
"columns": "ccdVisitId,filterName,ra,decl"
}