def get_ops():
"""
Fetch and ingest ZTF ops data
"""
# connect to MongoDB:
print(f'{time_stamp()}: Connecting to DB.')
mongo = Mongo(host=config['database']['host'],
port=config['database']['port'],
username=config['database']['username'],
password=config['database']['password'],
db=config['database']['db'],
verbose=0)
print(f'{time_stamp()}: Successfully connected.')
collection = 'ZTF_ops'
print(f'{time_stamp()}: Checking indexes.')
mongo.db[collection].create_index(
[('coordinates.radec_geojson', '2dsphere')], background=True)
mongo.db[collection].create_index([('utc_start', pymongo.ASCENDING),
('utc_end', pymongo.ASCENDING),
('fileroot', pymongo.ASCENDING)],
background=True)
mongo.db[collection].create_index([('jd_start', pymongo.ASCENDING),
('jd_end', pymongo.ASCENDING),
('fileroot', pymongo.ASCENDING)],
background=True)
mongo.db[collection].create_index([('jd_start', pymongo.DESCENDING),
('pid', pymongo.ASCENDING),
('field', pymongo.ASCENDING)],
background=True)
# fetch full table
print(f'{time_stamp()}: Fetching data.')
url = config['ztf_ops']['url']
r = requests.get(url,
auth=(config['ztf_ops']['username'],
config['ztf_ops']['password']),
verify=False)
if r.status_code == requests.codes.ok:
with open(os.path.join(config['path']['tmp'], 'allexp.tbl'),
'wb') as f:
f.write(r.content)
else:
raise Exception(f'{time_stamp()}: Failed to fetch allexp.tbl')
latest = list(mongo.db[collection].find({},
sort=[["$natural", -1]],
limit=1))
print(f'{time_stamp()}: Loading data.')
df = pd.read_fwf(os.path.join(config['path']['tmp'], 'allexp.tbl'),
comment='|',
header=None,
names=[
'utc_start', 'sun_elevation', 'exp', 'filter', 'type',
'field', 'pid', 'ra', 'dec', 'slew', 'wait',
'fileroot', 'programpi', 'qcomment'
])
# drop comments:
comments = df['utc_start'] == 'UT_START'
df = df.loc[~comments]
for col in ['sun_elevation', 'exp', 'filter', 'field', 'pid']:
df[col] = df[col].apply(lambda x: int(x))
for col in ['ra', 'dec', 'slew', 'wait']:
df[col] = df[col].apply(lambda x: float(x))
df['utc_start'] = df['utc_start'].apply(
lambda x: datetime.datetime.strptime(x, '%Y-%m-%dT%H:%M:%S.%f'))
df['utc_end'] = df['utc_start'].add(
df['exp'].apply(lambda x: datetime.timedelta(seconds=x)))
df['jd_start'] = df['utc_start'].apply(lambda x: datetime_to_jd(x))
df['jd_end'] = df['utc_end'].apply(lambda x: datetime_to_jd(x))
# drop rows with utc_start <= c['utc_start]
if len(latest) > 0:
new = df['jd_start'] > latest[0].get('jd_start', 0)
if sum(new):
print(f'{time_stamp()}: Found {sum(new)} new records.')
df = df.loc[new]
else:
# no new data? take a nap...
print(f'{time_stamp()}: No new data found.')
# close connection to db
mongo.client.close()
print(f'{time_stamp()}: Disconnected from db.')
return
documents = df.to_dict('records')
documents = [mongify(doc) for doc in documents]
print(f'{time_stamp()}: Inserting {len(documents)} documents.')
mongo.insert_many(collection=collection, documents=documents)
# close connection to db
mongo.client.close()
print(f'{time_stamp()}: Disconnected from db.')
def process_file(file, collection, batch_size):
# connect to MongoDB:
log("Connecting to DB")
mongo = Mongo(
host=config["database"]["host"],
port=config["database"]["port"],
replica_set=config["database"]["replica_set"],
username=config["database"]["username"],
password=config["database"]["password"],
db=config["database"]["db"],
verbose=0,
)
log("Successfully connected")
collection = "VLASS_DR1"
log(f"Processing {file}")
names = [
"Component_name",
"RA",
"DEC",
"E_RA",
"E_DEC",
"Total_flux",
"E_Total_flux",
"Peak_flux",
"E_Peak_flux",
"Maj",
"E_Maj",
"Min",
"E_Min",
"Duplicate_flag",
"Quality_flag",
]
for chunk_index, dataframe_chunk in enumerate(
pd.read_csv(file, chunksize=batch_size)
):
log(f"{file}: processing batch # {chunk_index + 1}")
dataframe_chunk = dataframe_chunk[names]
dataframe_chunk = dataframe_chunk[dataframe_chunk["Duplicate_flag"] < 2]
dataframe_chunk = dataframe_chunk[dataframe_chunk["Quality_flag"] == 0]
batch = dataframe_chunk.to_dict(orient="records")
bad_document_indexes = []
for document_index, document in enumerate(batch):
try:
# GeoJSON for 2D indexing
document["coordinates"] = dict()
# string format: H:M:S, D:M:S
document["coordinates"]["radec_str"] = [
deg2hms(document["RA"]),
deg2dms(document["DEC"]),
]
# for GeoJSON, must be lon:[-180, 180], lat:[-90, 90] (i.e. in deg)
_radec_geojson = [document["RA"] - 180.0, document["DEC"]]
document["coordinates"]["radec_geojson"] = {
"type": "Point",
"coordinates": _radec_geojson,
}
except Exception as e:
log(str(e))
bad_document_indexes.append(document_index)
if len(bad_document_indexes) > 0:
log("Removing bad docs")
for index in sorted(bad_document_indexes, reverse=True):
del batch[index]
# ingest batch
mongo.insert_many(collection=collection, documents=batch)
# disconnect from db:
try:
mongo.client.close()
finally:
log("Successfully disconnected from db")
def process_file(file, collection, batch_size):
# connect to MongoDB:
log("Connecting to DB")
mongo = Mongo(
host=config["database"]["host"],
port=config["database"]["port"],
replica_set=config["database"]["replica_set"],
username=config["database"]["username"],
password=config["database"]["password"],
db=config["database"]["db"],
verbose=0,
)
log("Successfully connected")
collection = "IGAPS_DR2"
log(f"Processing {file}")
names = [
"name",
"RA",
"DEC",
"gal_long",
"gal_lat",
"sourceID",
"posErr",
"mergedClass",
"pStar",
"pGalaxy",
"pNoise",
"i",
"iErr",
"iAB",
"iEll",
"iClass",
"iDeblend",
"iSaturated",
"iVignetted",
"iTrail",
"iTruncated",
"iBadPix",
"iMJD",
"iSeeing",
"iDetectionID",
"iDeltaRA",
"iDeltaDEC",
"ha",
"haErr",
"haAB",
"haEll",
"haClass",
"haDeblend",
"haSaturated",
"haVignetted",
"haTrail",
"haTruncated",
"haBadPix",
"haMJD",
"haSeeing",
"haDetectionID",
"haDeltaRA",
"haDeltaDEC",
"r_I",
"rErr_I",
"rAB_I",
"rEll_I",
"rClass_I",
"rDeblend_I",
"rSaturated_I",
"rVignetted_I",
"rTrail_I",
"rTruncated_I",
"rBadPix_I",
"rMJD_I",
"rSeeing_I",
"rDetectionID_I",
"r_U",
"rErr_U",
"rAB_U",
"rEll_U",
"rClass_U",
"rDeblend_U",
"rSaturated_U",
"rVignetted_U",
"rTrail_U",
"rTruncated_U",
"rBadPix_U",
"rMJD_U",
"rSeeing_U",
"rDetectionID_U",
"rDeltaRA_U",
"rDeltaDEC_U",
"g",
"gErr",
"gAB",
"gEll",
"gClass",
"gDeblend",
"gSaturated",
"gVignetted",
"gTrail",
"gTruncated",
"gBadPix",
"gmask",
"gMJD",
"gSeeing",
"gDetectionID",
"gDeltaRA",
"gDeltaDEC",
"U_RGO",
"UErr",
"UEll",
"UClass",
"UDeblend",
"USaturated",
"UVignetted",
"UTrail",
"UTruncated",
"UBadPix",
"UMJD",
"USeeing",
"UDetectionID",
"UDeltaRA",
"UDeltaDEC",
"brightNeighb",
"deblend",
"saturated",
"nBands",
"errBits",
"nObs_I",
"nObs_U",
"fieldID_I",
"fieldID_U",
"fieldGrade_I",
"fieldGrade_U",
"emitter",
"variable",
"SourceID2",
"i2",
"i2Err",
"i2Class",
"i2Seeing",
"i2MJD",
"i2DeltaRA",
"i2DeltaDEC",
"i2DetectionID",
"i2ErrBits",
"ha2",
"ha2Err",
"ha2Class",
"ha2Seeing",
"ha2MJD",
"ha2DeltaRA",
"ha2DeltaDEC",
"ha2DetectionID",
"ha2ErrBits",
"r2_I",
"r2Err_I",
"r2Class_I",
"r2Seeing_I",
"r2MJD_I",
"r2DeltaRA_I",
"r2DeltaDEC_I",
"r2DetectionID_I",
"r2ErrBits_I",
"r2_U",
"r2Err_U",
"r2Class_U",
"r2Seeing_U",
"r2MJD_U",
"r2DeltaRA_U",
"r2DeltaDEC_U",
"r2DetectionID_U",
"r2ErrBits_U",
"g2",
"g2Err",
"g2Class",
"g2Seeing",
"g2MJD",
"g2DeltaRA",
"g2DeltaDEC",
"g2DetectionID",
"g2ErrBits",
"U_RGO2",
"U2Err",
"U2Class",
"U2Seeing",
"U2MJD",
"U2DeltaRA",
"U2DeltaDEC",
"U2DetectionID",
"U2ErrBits",
"errBits2",
]
with fits.open(file) as hdulist:
nhdu = 1
dataframe = pd.DataFrame(np.asarray(hdulist[nhdu].data), columns=names)
for chunk_index, dataframe_chunk in dataframe.groupby(
np.arange(len(dataframe)) // batch_size):
log(f"{file}: processing batch # {chunk_index + 1}")
for col, dtype in dataframe_chunk.dtypes.items():
if dtype == np.object:
dataframe_chunk[col] = dataframe_chunk[col].apply(
lambda x: x.decode("utf-8"))
batch = dataframe_chunk.to_dict(orient="records")
batch = dataframe_chunk.fillna("DROPMEPLEASE").to_dict(
orient="records")
# pop nulls - save space
batch = [{
key: value
for key, value in document.items() if value != "DROPMEPLEASE"
} for document in batch]
bad_document_indexes = []
for document_index, document in enumerate(batch):
try:
# GeoJSON for 2D indexing
document["coordinates"] = dict()
# string format: H:M:S, D:M:S
document["coordinates"]["radec_str"] = [
deg2hms(document["RA"]),
deg2dms(document["DEC"]),
]
# for GeoJSON, must be lon:[-180, 180], lat:[-90, 90] (i.e. in deg)
_radec_geojson = [document["RA"] - 180.0, document["DEC"]]
document["coordinates"]["radec_geojson"] = {
"type": "Point",
"coordinates": _radec_geojson,
}
except Exception as e:
log(str(e))
bad_document_indexes.append(document_index)
if len(bad_document_indexes) > 0:
log("Removing bad docs")
for index in sorted(bad_document_indexes, reverse=True):
del batch[index]
# ingest batch
mongo.insert_many(collection=collection, documents=batch)
# disconnect from db:
try:
mongo.client.close()
finally:
log("Successfully disconnected from db")
def process_file(args):
file, collection, batch_size, rm, verbose = args
# connect to MongoDB:
if verbose:
log("Connecting to DB")
mongo = Mongo(
host=config["database"]["host"],
port=config["database"]["port"],
replica_set=config["database"]["replica_set"],
username=config["database"]["username"],
password=config["database"]["password"],
db=config["database"]["db"],
verbose=0,
)
if verbose:
log("Successfully connected")
collection = "Gaia_EDR3"
if verbose:
log(f"Processing {file}")
for chunk_index, dataframe_chunk in enumerate(
pd.read_csv(file, chunksize=batch_size)):
if verbose:
log(f"{file}: processing batch # {chunk_index + 1}")
dataframe_chunk["_id"] = dataframe_chunk["source_id"].apply(
lambda x: str(x))
batch = dataframe_chunk.fillna("DROPMEPLEASE").to_dict(
orient="records")
# pop nulls - save space
batch = [{
key: value
for key, value in document.items()
if value not in ("DROPMEPLEASE", "NOT_AVAILABLE")
} for document in batch]
bad_document_indexes = []
for document_index, document in enumerate(batch):
try:
# GeoJSON for 2D indexing
document["coordinates"] = dict()
# string format: H:M:S, D:M:S
document["coordinates"]["radec_str"] = [
deg2hms(document["ra"]),
deg2dms(document["dec"]),
]
# for GeoJSON, must be lon:[-180, 180], lat:[-90, 90] (i.e. in deg)
_radec_geojson = [document["ra"] - 180.0, document["dec"]]
document["coordinates"]["radec_geojson"] = {
"type": "Point",
"coordinates": _radec_geojson,
}
except Exception as e:
if verbose:
log(str(e))
bad_document_indexes.append(document_index)
if len(bad_document_indexes) > 0:
if verbose:
log("Removing bad docs")
for index in sorted(bad_document_indexes, reverse=True):
del batch[index]
# ingest batch
mongo.insert_many(collection=collection, documents=batch)
# disconnect from db:
try:
mongo.client.close()
finally:
if verbose:
log("Successfully disconnected from db")
# clean up:
if rm:
os.remove(file)
if verbose:
log(f"Successfully removed {file}")
def get_ops():
"""
Fetch and ingest ZTF ops data
"""
# connect to MongoDB:
print(f"{time_stamp()}: Connecting to DB.")
mongo = Mongo(
host=config["database"]["host"],
port=config["database"]["port"],
replica_set=config["database"]["replica_set"],
username=config["database"]["username"],
password=config["database"]["password"],
db=config["database"]["db"],
verbose=0,
)
print(f"{time_stamp()}: Successfully connected.")
collection = "ZTF_ops"
print(f"{time_stamp()}: Checking indexes.")
mongo.db[collection].create_index(
[("coordinates.radec_geojson", "2dsphere")], background=True)
mongo.db[collection].create_index(
[
("utc_start", pymongo.ASCENDING),
("utc_end", pymongo.ASCENDING),
("fileroot", pymongo.ASCENDING),
],
background=True,
)
mongo.db[collection].create_index(
[
("jd_start", pymongo.ASCENDING),
("jd_end", pymongo.ASCENDING),
("fileroot", pymongo.ASCENDING),
],
background=True,
)
mongo.db[collection].create_index(
[
("jd_start", pymongo.DESCENDING),
("pid", pymongo.ASCENDING),
("field", pymongo.ASCENDING),
],
background=True,
)
# fetch full table
print(f"{time_stamp()}: Fetching data.")
url = config["ztf_ops"]["url"]
r = requests.get(
url,
auth=(config["ztf_ops"]["username"], config["ztf_ops"]["password"]),
verify=False,
)
if r.status_code == requests.codes.ok:
with open(os.path.join(config["path"]["tmp"], "allexp.tbl"),
"wb") as f:
f.write(r.content)
else:
raise Exception(f"{time_stamp()}: Failed to fetch allexp.tbl")
latest = list(mongo.db[collection].find({},
sort=[["$natural", -1]],
limit=1))
print(f"{time_stamp()}: Loading data.")
df = pd.read_fwf(
os.path.join(config["path"]["tmp"], "allexp.tbl"),
comment="|",
widths=[22, 4, 6, 4, 5, 8, 4, 9, 9, 7, 8, 29, 11, 25],
header=None,
names=[
"utc_start",
"sun_elevation",
"exp",
"filter",
"type",
"field",
"pid",
"ra",
"dec",
"slew",
"wait",
"fileroot",
"programpi",
"qcomment",
],
)
# drop comments:
comments = df["utc_start"] == "UT_START"
df = df.loc[~comments]
for col in ["sun_elevation", "exp", "filter", "field", "pid"]:
df[col] = df[col].apply(lambda x: int(x))
for col in ["ra", "dec", "slew", "wait"]:
df[col] = df[col].apply(lambda x: float(x))
df["utc_start"] = df["utc_start"].apply(
lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%f"))
df["utc_end"] = df["utc_start"].add(
df["exp"].apply(lambda x: datetime.timedelta(seconds=x)))
df["jd_start"] = df["utc_start"].apply(lambda x: datetime_to_jd(x))
df["jd_end"] = df["utc_end"].apply(lambda x: datetime_to_jd(x))
# drop rows with utc_start <= c['utc_start]
if len(latest) > 0:
new = df["jd_start"] > latest[0].get("jd_start", 0)
if sum(new):
print(f"{time_stamp()}: Found {sum(new)} new records.")
df = df.loc[new]
else:
# no new data? take a nap...
print(f"{time_stamp()}: No new data found.")
# close connection to db
mongo.client.close()
print(f"{time_stamp()}: Disconnected from db.")
return
documents = df.to_dict("records")
documents = [mongify(doc) for doc in documents]
print(f"{time_stamp()}: Inserting {len(documents)} documents.")
mongo.insert_many(collection=collection, documents=documents)
# close connection to db
mongo.client.close()
print(f"{time_stamp()}: Disconnected from db.")
def process_file(argument_list: Sequence):
file_name, collections, batch_size, rm_file, dry_run = argument_list
try:
# connect to MongoDB:
mongo = Mongo(
host=config["database"]["host"],
port=config["database"]["port"],
replica_set=config["database"]["replica_set"],
username=config["database"]["username"],
password=config["database"]["password"],
db=config["database"]["db"],
verbose=0,
)
with tables.open_file(file_name, "r+") as f:
for group in f.walk_groups():
pass
ff_basename = pathlib.Path(file_name).name
# base id:
_, field, filt, ccd, _, _ = ff_basename.split("_")
field = int(field[1:])
filt = int(filt[1:])
readout_channel = int(ccd[1:])
baseid = int(1e13 + field * 1e9 + readout_channel * 1e7 +
filt * 1e6)
exposure_baseid = int(1e16 + field * 1e12 +
readout_channel * 1e10 + filt * 1e9)
def clean_up_document(group):
""" Format passed in dicts for Mongo insertion """
document = {}
for k, v in group.items():
if k == "matchedSourceID":
document[k] = group[k]
continue
if k in sources_int_fields:
document[k] = [
int(group[k][key2]) for key2 in group[k].keys()
]
else:
document[k] = [
float(group[k][key2]) for key2 in group[k].keys()
]
# document["ra"] = document["ra"][0]
# document["dec"] = document["dec"][0]
# generate unique _id:
document["_id"] = baseid + document["matchedSourceID"]
document["filter"] = filt
document["field"] = field
document["ccd"] = ccd
# GeoJSON for 2D indexing
document["coordinates"] = dict()
_ra = np.median(document["ra"])
_dec = np.median(document["dec"])
_radec_str = [deg2hms(_ra), deg2dms(_dec)]
document["coordinates"]["radec_str"] = _radec_str
# for GeoJSON, must be lon:[-180, 180], lat:[-90, 90] (i.e. in deg)
_radec_geojson = [_ra - 180.0, _dec]
document["coordinates"]["radec_geojson"] = {
"type": "Point",
"coordinates": _radec_geojson,
}
document["data"] = []
for t, m, e, f, _ra, _dec in zip(
document["mjd"],
document["mag"],
document["magErr"],
document["ipacFlags"],
document["ra"],
document["dec"],
):
data_point = {
"mjd": t,
"mag": m,
"magerr": e,
"ipacflags": f,
"ra": _ra,
"dec": _dec,
}
# convert types for pymongo:
for k, v in data_point.items():
if k in sourcedata_int_fields:
data_point[k] = int(data_point[k])
else:
data_point[k] = float(data_point[k])
if k == "mjd":
data_point[k] = round(data_point[k], 5)
elif k not in ("ra", "dec"):
data_point[k] = round(data_point[k], 3)
document["data"].append(data_point)
del (
document["mjd"],
document["mag"],
document["magErr"],
document["ipacFlags"],
document["ra"],
document["dec"],
)
document["data"].sort(key=lambda x: x["mjd"])
return document
exposures = pd.DataFrame.from_records(group.exposures[:])
# prepare docs to ingest into db:
docs_exposures = []
for index, row in exposures.iterrows():
try:
doc = row.to_dict()
# unique exposure id:
doc["_id"] = exposure_baseid + doc["expid"]
doc["matchfile"] = ff_basename
doc["filter"] = filt
doc["field"] = field
doc["ccd"] = ccd
docs_exposures.append(doc)
except Exception as exception:
log(str(exception))
# ingest exposures in one go:
if not dry_run:
mongo.insert_many(collection=collections["exposures"],
documents=docs_exposures)
sources = pd.DataFrame.from_records(
group["sources"].read(),
index="matchedSourceID",
exclude=sources_fields_to_exclude,
)
sourcedatas = pd.DataFrame.from_records(
group["sourcedata"].read(),
index="matchedSourceID",
exclude=sourcedata_to_exclude,
)
merged = sources.merge(sourcedatas,
left_index=True,
right_index=True)
groups = merged.groupby("matchedSourceID")
# light curves
docs_sources = []
batch_num = 1
# fixme? skip transients
for row, group in groups:
try:
groupdict = group.to_dict()
groupdict["matchedSourceID"] = row
current_doc = clean_up_document(groupdict)
docs_sources.append(current_doc)
except Exception as exception:
log(str(exception))
# ingest in batches
try:
if len(docs_sources) % batch_size == 0 and len(
docs_sources) != 0:
if not dry_run:
mongo.insert_many(
collection=collections["sources"],
documents=docs_sources,
)
# flush:
docs_sources = []
batch_num += 1
except Exception as exception:
log(str(exception))
# ingest remaining
while len(docs_sources) > 0:
try:
# In case mongo crashed and disconnected, docs will accumulate in documents
# keep on trying to insert them until successful
if not dry_run:
mongo.insert_many(collection=collections["sources"],
documents=docs_sources)
# flush:
docs_sources = []
except Exception as e:
traceback.print_exc()
log(e)
log("Failed, waiting 5 seconds to retry")
time.sleep(5)
mongo.client.close()
except Exception as e:
traceback.print_exc()
log(e)
# if there was an error, return without potentially deleting the file
return
try:
if rm_file:
os.remove(file_name)
finally:
pass
def process_file(argument_list: Sequence):
file_name, collections, batch_size, rm_file, dry_run = argument_list
try:
# connect to MongoDB:
mongo = Mongo(
host=config["database"]["host"],
port=config["database"]["port"],
replica_set=config["database"]["replica_set"],
username=config["database"]["username"],
password=config["database"]["password"],
db=config["database"]["db"],
verbose=0,
)
with tables.open_file(file_name, "r+") as f:
group = f.root.matches
ff_basename = pathlib.Path(file_name).name
# base id:
_, field, filt, ccd, quadrant, _ = ff_basename.split("_")
field = int(field)
filt = filters[filt]
ccd = int(ccd[1:])
quadrant = int(quadrant[1:])
readout_channel = ccd_quad_to_rc(ccd=ccd, quad=quadrant)
baseid = int(1e13 + field * 1e9 + readout_channel * 1e7 +
filt * 1e6)
exposure_baseid = int(1e16 + field * 1e12 +
readout_channel * 1e10 + filt * 1e9)
def clean_up_document(document):
""" Format passed in dicts for Mongo insertion """
# convert types for pymongo:
for k, v in document.items():
if k != "data":
if k in sources_int_fields:
document[k] = int(document[k])
else:
document[k] = float(document[k])
if k not in ("ra", "dec"):
# this will save a lot of space:
document[k] = round(document[k], 3)
# generate unique _id:
document["_id"] = baseid + document["matchid"]
document["filter"] = filt
document["field"] = field
document["ccd"] = ccd
document["quad"] = quadrant
document["rc"] = readout_channel
# GeoJSON for 2D indexing
document["coordinates"] = dict()
_ra = document["ra"]
_dec = document["dec"]
_radec_str = [deg2hms(_ra), deg2dms(_dec)]
document["coordinates"]["radec_str"] = _radec_str
# for GeoJSON, must be lon:[-180, 180], lat:[-90, 90] (i.e. in deg)
_radec_geojson = [_ra - 180.0, _dec]
document["coordinates"]["radec_geojson"] = {
"type": "Point",
"coordinates": _radec_geojson,
}
document["data"].sort(key=lambda x: x["hjd"])
for data_point in document["data"]:
# convert types for pymongo:
for k, v in data_point.items():
if k in sourcedata_int_fields:
data_point[k] = int(data_point[k])
else:
data_point[k] = float(data_point[k])
if k not in ("ra", "dec", "hjd"):
data_point[k] = round(data_point[k], 3)
elif k == "hjd":
data_point[k] = round(data_point[k], 5)
# generate unique exposure id's that match _id's in exposures collection
data_point[
"uexpid"] = exposure_baseid + data_point["expid"]
return document
exposures = pd.DataFrame.from_records(group.exposures[:])
# prepare docs to ingest into db:
docs_exposures = []
for index, row in exposures.iterrows():
try:
doc = row.to_dict()
# unique exposure id:
doc["_id"] = exposure_baseid + doc["expid"]
doc["matchfile"] = ff_basename
doc["filter"] = filt
doc["field"] = field
doc["ccd"] = ccd
doc["quad"] = quadrant
doc["rc"] = readout_channel
docs_exposures.append(doc)
except Exception as exception:
log(str(exception))
# ingest exposures in one go:
if not dry_run:
mongo.insert_many(collection=collections["exposures"],
documents=docs_exposures)
# light curves
docs_sources = []
batch_num = 1
# fixme? skip transients
# for source_type in ('source', 'transient'):
for source_type in ("source", ):
sources = pd.DataFrame.from_records(
group[f"{source_type}s"].read(),
index="matchid",
exclude=sources_fields_to_exclude,
)
# Load in percentiles separately to compute the IQR column
# because Pandas DF from_records() only wants 2-D tables
percentiles = group[f"{source_type}s"].col("percentiles")
# Ignore float errors due to infinity values
old_settings = np.seterr(all="ignore")
iqr = np.round(percentiles[:, 8] - percentiles[:, 3], 3)
np.seterr(**old_settings)
sources["iqr"] = iqr
sourcedatas = pd.DataFrame.from_records(
group[f"{source_type}data"][:],
index="matchid",
exclude=[
"ypos",
"xpos",
"mjd",
"psfflux",
"psffluxerr",
"magerrmodel",
],
)
sourcedatas.rename(columns={
"ra": "ra_data",
"dec": "dec_data"
},
inplace=True)
sourcedata_colnames = sourcedatas.columns.values
# Join sources and their data
merged = sources.merge(sourcedatas,
left_index=True,
right_index=True)
prev_matchid = None
current_doc = None
for row in merged.itertuples():
matchid = row[0]
try:
# At a new source
if matchid != prev_matchid:
# Done with last source; save
if current_doc is not None:
current_doc = clean_up_document(current_doc)
docs_sources.append(current_doc)
# Set up new doc
doc = dict(row._asdict())
doc["matchid"] = doc["Index"]
doc.pop("Index")
# Coerce the source data info into its own nested array
first_data_row = {}
for col in sourcedata_colnames:
if col not in ["dec_data", "ra_data"]:
first_data_row[col] = doc[col]
else:
real_col = col.split("_data")[0]
first_data_row[real_col] = doc[col]
doc.pop(col)
doc["data"] = [first_data_row]
current_doc = doc
# For continued source, just append new data row
else:
data_row = {}
data = dict(row._asdict())
for col in sourcedata_colnames:
if col not in ["dec_data", "ra_data"]:
data_row[col] = data[col]
else:
real_col = col.split("_data")[0]
data_row[real_col] = data[col]
current_doc["data"].append(data_row)
prev_matchid = matchid
except Exception as exception:
log(str(exception))
# ingest in batches
try:
if (len(docs_sources) % batch_size == 0
and len(docs_sources) != 0):
if not dry_run:
mongo.insert_many(
collection=collections["sources"],
documents=docs_sources,
)
# flush:
docs_sources = []
batch_num += 1
except Exception as exception:
log(str(exception))
# Clean up and append the last doc
if current_doc is not None:
current_doc = clean_up_document(current_doc)
docs_sources.append(current_doc)
# ingest remaining
while len(docs_sources) > 0:
try:
# In case mongo crashed and disconnected, docs will accumulate in documents
# keep on trying to insert them until successful
if not dry_run:
mongo.insert_many(collection=collections["sources"],
documents=docs_sources)
# flush:
docs_sources = []
except Exception as e:
traceback.print_exc()
log(e)
log("Failed, waiting 5 seconds to retry")
time.sleep(5)
mongo.client.close()
except Exception as e:
traceback.print_exc()
log(e)
# if there was an error, return without potentially deleting the file
return
try:
if rm_file:
os.remove(file_name)
finally:
pass