def store_smf_list(db, exp_tabname, new_exps): """ Store a human-readable list of loaded SMF files and their exp_ids, for fast lookup by ps1-load and lsd-make-object-catalog. Note: For forward-compatibility, readers of this file should assume it has an unspecified number of columns. I.e., don't do (a, b) = line.split(), but do (a, b) = line.split()[:2] """ uri = 'lsd:%s:cache:all_exposures.txt' % (exp_tabname) # Try to load from cache, query otherwise try: with db.open_uri(uri) as f: old_exps = np.loadtxt(f, dtype=[('_EXP', 'u8'), ('smf_fn', 'a40')], ndmin=1) except IOError: old_exps = db.query("select _EXP, smf_fn from %s" % exp_tabname).fetch( progress_callback=pool2.progress_pass) exps = colgroup.fromiter([old_exps, new_exps], blocks=True) if len(old_exps) else new_exps # Store to cache with db.open_uri(uri, mode='w') as f: for exp_id, exp_fn in exps: f.write("%d %s\n" % (exp_id, exp_fn))
def _sanity_check_object_table_reducer(kv, db, det_tabname, explist): cell_id, rowlist = kv rows = colgroup.fromiter(rowlist, blocks=True) rows.sort(["det_id"]) # Verify that each detection appears only once (cond #1) if not np.all(np.diff(rows.det_id) != 0): #a1 = np.arange(len(rows))[np.diff(rows.det_id) == 0] with open("cell.%s.txt" % cell_id, 'w') as fp: for row in rows: fp.write("%s\t%s\n" % (row['det_id'], row['obj_id'])) print "ERROR -- same detection assigned to multiple objects" x = rows.det_id a1 = np.arange(len(x))[np.diff(x) == 0] print rows[a1] print rows[a1+1] #assert np.all(np.diff(rows.det_id) != 0) # Verify that all detections in this cell are linked (cond #2) # This can be restricted to only detections from exposures existing in explist det_rows = db.query("_ID as det_id, _EXP as exp_id FROM '%s'" % det_tabname).fetch_cell(cell_id) if explist is not None: det_rows = det_rows[np.in1d(det_rows.exp_id, explist)] det_rows.sort(["det_id"]) ok = np.all(np.unique(rows.det_id) == det_rows.det_id) if not ok: print "ERROR -- Not all detections were linked to objects (need to rerun make-object-catalog?): nlinked=%d ntotal=%d cell_id=%s" % (len(rows.det_id), len(det_rows), cell_id) yield ok
def _sanity_check_object_table_reducer(kv, db, det_tabname, explist): cell_id, rowlist = kv rows = colgroup.fromiter(rowlist, blocks=True) rows.sort(["det_id"]) # Verify that each detection appears only once (cond #1) if not np.all(np.diff(rows.det_id) != 0): #a1 = np.arange(len(rows))[np.diff(rows.det_id) == 0] with open("cell.%s.txt" % cell_id, 'w') as fp: for row in rows: fp.write("%s\t%s\n" % (row['det_id'], row['obj_id'])) print "ERROR -- same detection assigned to multiple objects" x = rows.det_id a1 = np.arange(len(x))[np.diff(x) == 0] print rows[a1] print rows[a1 + 1] #assert np.all(np.diff(rows.det_id) != 0) # Verify that all detections in this cell are linked (cond #2) # This can be restricted to only detections from exposures existing in explist det_rows = db.query("_ID as det_id, _EXP as exp_id FROM '%s'" % det_tabname).fetch_cell(cell_id) if explist is not None: det_rows = det_rows[np.in1d(det_rows.exp_id, explist)] det_rows.sort(["det_id"]) ok = np.all(np.unique(rows.det_id) == det_rows.det_id) if not ok: print "ERROR -- Not all detections were linked to objects (need to rerun make-object-catalog?): nlinked=%d ntotal=%d cell_id=%s" % ( len(rows.det_id), len(det_rows), cell_id) yield ok
def store_smf_list(db, exp_tabname, new_exps): """ Store a human-readable list of loaded SMF files and their exp_ids, for fast lookup by ps1-load and lsd-make-object-catalog. Note: For forward-compatibility, readers of this file should assume it has an unspecified number of columns. I.e., don't do (a, b) = line.split(), but do (a, b) = line.split()[:2] """ uri = 'lsd:%s:cache:all_exposures.txt' % (exp_tabname) # Try to load from cache, query otherwise try: with db.open_uri(uri) as f: old_exps = np.loadtxt(f, dtype=[('_EXP', 'u8'),('smf_fn', 'a40')], ndmin=1) except IOError: old_exps = db.query("select _EXP, smf_fn from %s" % exp_tabname).fetch(progress_callback=pool2.progress_pass) exps = colgroup.fromiter([old_exps, new_exps], blocks=True) if len(old_exps) else new_exps # Store to cache with db.open_uri(uri, mode='w') as f: for exp_id, exp_fn in exps: f.write("%d %s\n" % (exp_id, exp_fn))