def process_loop(fname):
"""
Thread to process satobs FourFrame FITS files in a multi-thread compatible manner
"""
# Generate star catalog
if not os.path.exists(fname + ".cat"):
pix_catalog = generate_star_catalog(fname)
else:
pix_catalog = pixel_catalog(fname + ".cat")
# Calibrate from reference
calibrate_from_reference(fname, "test.fits", pix_catalog)
# Store calibration
store_calibration(pix_catalog, fname + ".cal")
# Generate satellite predictions
generate_satellite_predictions(fname)
# Detect lines with 3D Hough transform
ids = find_hough3d_lines(fname, nhoughmin, houghrmin)
# Get properties
ff = fourframe(fname)
# Extract tracks
if is_calibrated(ff):
screenoutput_idents = extract_tracks(fname, trkrmin, drdtmin, drdtmax, trksig, ntrkmin, root_dir, results_dir, tle_dir)
else:
screenoutput_idents = None
# Stars available and used
nused = np.sum(pix_catalog.flag == 1)
nstars = pix_catalog.nstars
# Write output
screenoutput = "%s %10.6f %10.6f %4d/%4d %5.1f %5.1f %6.2f +- %6.2f" % (
ff.fname, ff.crval[0], ff.crval[1], nused, nstars,
3600.0 * ff.crres[0], 3600.0 * ff.crres[1], np.mean(
ff.zavg), np.std(ff.zavg))
if is_calibrated(ff):
screenoutput = colored(screenoutput, "green")
else:
screenoutput = colored(screenoutput, "red")
imgstat_output = ("%s,%.8lf,%.6f,%.6f,%.3f,%.3f,%.3f," + "%.3f,%d,%d\n") % (
(ff.fname, ff.mjd, ff.crval[0], ff.crval[1],
3600 * ff.crres[0], 3600 * ff.crres[1], np.mean(
ff.zavg), np.std(ff.zavg), nstars, nused))
# Move processed files
shutil.move(fname, os.path.join(processed_dir, fname))
shutil.move(fname + ".png", os.path.join(processed_dir, fname + ".png"))
shutil.move(fname + ".id", os.path.join(processed_dir, fname + ".id"))
shutil.move(fname + ".cat", os.path.join(processed_dir, fname + ".cat"))
shutil.move(fname + ".cal", os.path.join(processed_dir, fname + ".cal"))
return (screenoutput, imgstat_output, screenoutput_idents)
def generate_keogram(path):
# Get files
fnames = sorted(glob.glob(os.path.join(path, "processed/2*.fits")))
# Allocate arrays
nx = len(fnames)
ny = cfg.getint('Camera', 'camera_y')
ixmid = cfg.getint('Camera', 'camera_x') // 2
keogram = np.zeros(nx * ny).reshape(ny, nx)
mjds = np.zeros(nx)
# Get data
for i, fname in enumerate(fnames):
if i % 10 == 0:
print(i, fname)
# Read file
ff = fourframe(fname)
# Extract data
keogram[:, i] = ff.zavg[:, ixmid]
mjds[i] = ff.mjd
# Save npy arrays
np.save(os.path.join(path, "mjds"), mjds)
np.save(os.path.join(path, "keogram"), keogram)
fnames = sorted(glob.glob("2*.fits"))
# Loop over files
for fname in fnames:
# Generate star catalog
pix_catalog = generate_star_catalog(fname)
# Create reference calibration file
if not os.path.exists("test.fits"):
solved = generate_reference_with_anet(fname, "")
# Calibrate astrometry
calibrate_from_reference(fname, "test.fits", pix_catalog)
# Redo refence if astrometry failed
if not is_calibrated(fourframe(fname)) and pix_catalog.nstars > 10:
print(
colored(
"Recomputing astrometric calibration for %s" % fname,
"yellow"))
solved = generate_reference_with_anet(fname, "")
# Calibrate astrometry
calibrate_from_reference(fname, "test.fits", pix_catalog)
# Generate satellite predictions
generate_satellite_predictions(fname)
# Detect lines with 3D Hough transform
ids = find_hough3d_lines(fname)
def find_hough3d_lines(fname, ntrkmin, trkrmin):
# Read four frame
ff = fourframe(fname)
# Mask frame
ff.mask(10, 10, 5, 5)
# Compute selection mask
x, y, z, t, sig = ff.selection_mask(5.0, 40.0)
# Skip if not enough points
if len(t) < 2:
return []
# Save points to a unique temporary file
(fd, tmpfile_path) = tempfile.mkstemp(prefix="hough_tmp", suffix=".dat")
try:
with os.fdopen(fd, "w") as f:
for i in range(len(t)):
f.write("%f,%f,%f\n" % (x[i], y[i], z[i]))
# Run 3D Hough line-finding algorithm
command = "hough3dlines -dx %d -minvotes %d %s" % (trkrmin, ntrkmin,
tmpfile_path)
try:
output = subprocess.check_output(command,
shell=True,
stderr=subprocess.STDOUT)
except Exception:
return []
finally:
os.remove(tmpfile_path)
# FIXME: Is this still needed? hough3dlines output does not appear to have these items -- interplanetarychris
# Clean output (a bit cluncky)
cleaned_output = output.decode("utf-8").replace("npoints=", "")
cleaned_output = cleaned_output.replace(", a=(", " ")
cleaned_output = cleaned_output.replace("), b=(", " ")
cleaned_output = cleaned_output.replace(")", "")
cleaned_output = cleaned_output.replace(",", " ")
# Generate identifications
lines = []
s = cleaned_output.split()
for i in range(len(s) // 7):
# Extract points
x0, y0, z0 = float(s[1 + 7 * i]), float(s[2 + 7 * i]), float(s[3 +
7 * i])
dx, dy, dz = float(s[4 + 7 * i]), float(s[5 + 7 * i]), float(s[6 +
7 * i])
# Reconstruct start and end points
xmin = x0 - z0 * dx / (dz + 1e-9)
xmax = x0 + (ff.nz - z0) * dx / (dz + 1e-9)
ymin = y0 - z0 * dy / (dz + 1e-9)
ymax = y0 + (ff.nz - z0) * dy / (dz + 1e-9)
# Output line
line = "%.23s %8.3f %8.3f %8.3f %8.3f %8.5f %s unidentified sunlit\n" %\
(ff.nfd, xmin, ymin, xmax, ymax, ff.texp, 99999-i)
lines.append(line)
# Store identifications
with open(fname + ".id", "a") as fp:
for line in lines:
fp.write(line)
return [satid(line) for line in lines]
def extract_tracks(fname, trkrmin, drdtmin, trksig, ntrkmin, path,
results_path):
# Read four frame
ff = fourframe(fname)
# Skip saturated frames
if np.sum(ff.zavg > 240.0) / float(ff.nx * ff.ny) > 0.95:
return
# Read satelite IDs
try:
f = open(fname + ".id")
except OSError:
print("Cannot open", fname + ".id")
else:
lines = f.readlines()
f.close()
tr = np.array([-0.5, 1.0, 0.0, -0.5, 0.0, 1.0])
# Parse identifications
idents = [satid(line) for line in lines]
# Identify unknowns
for ident0 in idents:
if ident0.catalog == "unidentified":
for ident1 in idents:
if ident1.catalog == "unidentified":
continue
# Find matches
p1 = inside_selection(ident1, ident0.t0, ident0.x0, ident0.y0)
p2 = inside_selection(ident1, ident0.t1, ident0.x1, ident0.y1)
# Match found
if p1 and p2:
# Copy info
ident0.norad = ident1.norad
ident0.catalog = ident1.catalog
ident0.state = ident1.state
ident1.state = "remove"
break
# Loop over identifications
for ident in idents:
# Skip superseded unknowns
if ident.state == "remove":
continue
# Skip slow moving objects
drdt = np.sqrt(ident.dxdt**2 + ident.dydt**2)
if drdt < drdtmin:
continue
# Extract significant pixels along a track
x, y, t, sig = ff.significant_pixels_along_track(
trksig, ident.x0, ident.y0, ident.dxdt, ident.dydt, trkrmin)
# Fit tracks
if len(t) > ntrkmin:
# Get times
tmin = np.min(t)
tmax = np.max(t)
tmid = 0.5 * (tmax + tmin)
mjd = ff.mjd + tmid / 86400.0
# Skip if no variance in time
if np.std(t - tmid) == 0.0:
continue
# Very simple polynomial fit; no weighting, no cleaning
px = np.polyfit(t - tmid, x, 1)
py = np.polyfit(t - tmid, y, 1)
# Extract results
x0, y0 = px[1], py[1]
dxdt, dydt = px[0], py[0]
xmin = x0 + dxdt * (tmin - tmid)
ymin = y0 + dydt * (tmin - tmid)
xmax = x0 + dxdt * (tmax - tmid)
ymax = y0 + dydt * (tmax - tmid)
cospar = get_cospar(ident.norad, ff.nfd)
obs = observation(ff, mjd, x0, y0)
iod_line = "%s" % format_iod_line(ident.norad, cospar, ff.site_id,
obs.nfd, obs.ra, obs.de)
# Create diagnostic plot
plot_header(fname.replace(".fits", "_%05d.png/png" % ident.norad),
ff, iod_line)
ppg.pgimag(ff.zmax, ff.nx, ff.ny, 0, ff.nx - 1, 0, ff.ny - 1,
ff.zmaxmax, ff.zmaxmin, tr)
ppg.pgbox("BCTSNI", 0., 0, "BCTSNI", 0., 0)
ppg.pgstbg(1)
ppg.pgsci(0)
if ident.catalog.find("classfd.tle") > 0:
ppg.pgsci(4)
elif ident.catalog.find("inttles.tle") > 0:
ppg.pgsci(3)
ppg.pgpt(np.array([x0]), np.array([y0]), 4)
ppg.pgmove(xmin, ymin)
ppg.pgdraw(xmax, ymax)
ppg.pgsch(0.65)
ppg.pgtext(np.array([x0]), np.array([y0]), " %05d" % ident.norad)
ppg.pgsch(1.0)
ppg.pgsci(1)
ppg.pgend()
# Store results
store_results(ident, fname, results_path, iod_line)
elif ident.catalog.find("classfd.tle") > 0:
# Track and stack
t = np.linspace(0.0, ff.texp)
x, y = ident.x0 + ident.dxdt * t, ident.y0 + ident.dydt * t
c = (x > 0) & (x < ff.nx) & (y > 0) & (y < ff.ny)
# Skip if no points selected
if np.sum(c) == 0:
store_not_seen(ident, fname, results_path)
continue
# Compute track
tmid = np.mean(t[c])
mjd = ff.mjd + tmid / 86400.0
xmid = ident.x0 + ident.dxdt * tmid
ymid = ident.y0 + ident.dydt * tmid
ztrk = ndimage.gaussian_filter(
ff.track(ident.dxdt, ident.dydt, tmid), 1.0)
vmin = np.mean(ztrk) - 2.0 * np.std(ztrk)
vmax = np.mean(ztrk) + 6.0 * np.std(ztrk)
# Select region
xmin = int(xmid - 100)
xmax = int(xmid + 100)
ymin = int(ymid - 100)
ymax = int(ymid + 100)
if xmin < 0:
xmin = 0
if ymin < 0:
ymin = 0
if xmax > ff.nx:
xmax = ff.nx - 1
if ymax > ff.ny:
ymax = ff.ny - 1
# Find peak
x0, y0, w, sigma = peakfind(ztrk[ymin:ymax, xmin:xmax])
x0 += xmin
y0 += ymin
# Skip if peak is not significant
if sigma < trksig:
store_not_seen(ident, fname, results_path)
continue
# Skip if point is outside selection area
if inside_selection(ident, tmid, x0, y0) is False:
store_not_seen(ident, fname, results_path)
continue
# Format IOD line
cospar = get_cospar(ident.norad, ff.nfd)
obs = observation(ff, mjd, x0, y0)
iod_line = "%s" % format_iod_line(ident.norad, cospar, ff.site_id,
obs.nfd, obs.ra, obs.de)
# Create diagnostic plot
pngfile = fname.replace(".fits", "_%05d.png" % ident.norad)
plot_header(pngfile + "/png", ff, iod_line)
ppg.pgimag(ztrk, ff.nx, ff.ny, 0, ff.nx - 1, 0, ff.ny - 1, vmax,
vmin, tr)
ppg.pgbox("BCTSNI", 0., 0, "BCTSNI", 0., 0)
ppg.pgstbg(1)
plot_selection(ident, xmid, ymid)
ppg.pgsci(0)
if ident.catalog.find("classfd.tle") > 0:
ppg.pgsci(4)
elif ident.catalog.find("inttles.tle") > 0:
ppg.pgsci(3)
ppg.pgpt(np.array([ident.x0]), np.array([ident.y0]), 17)
ppg.pgmove(ident.x0, ident.y0)
ppg.pgdraw(ident.x1, ident.y1)
ppg.pgpt(np.array([x0]), np.array([y0]), 4)
ppg.pgsch(0.65)
ppg.pgtext(np.array([ident.x0]), np.array([ident.y0]),
" %05d" % ident.norad)
ppg.pgsch(1.0)
ppg.pgsci(1)
ppg.pgend()
# Store results
store_results(ident, fname, results_path, iod_line)
def find_hough3d_lines(fname, ntrkmin=20, dr=8):
# Read four frame
ff = fourframe(fname)
# Mask frame
ff.mask(10, 10, 5, 5)
# Compute selection mask
x, y, z, t, sig = ff.selection_mask(5.0, 40.0)
# Skip if not enough points
if len(t) < 2:
return []
# Save points to temporary file
with open("/tmp/hough.dat", "w") as f:
for i in range(len(t)):
f.write("%f,%f,%f\n" % (x[i], y[i], z[i]))
f.close()
# Run 3D Hough line-finding algorithm
command = "hough3dlines -dx %d -minvotes %d %s" % (dr, ntrkmin,
"/tmp/hough.dat")
try:
output = subprocess.check_output(command,
shell=True,
stderr=subprocess.STDOUT)
except Exception:
return []
# Clean output (a bit cluncky)
cleaned_output = output.decode("utf-8").replace("npoints=", "")
cleaned_output = cleaned_output.replace(", a=(", " ")
cleaned_output = cleaned_output.replace("), b=(", " ")
cleaned_output = cleaned_output.replace(")", "")
cleaned_output = cleaned_output.replace(",", " ")
# Generate identifications
lines = []
s = cleaned_output.split()
for i in range(len(s) // 7):
# Extract points
x0, y0, z0 = float(s[1 + 7 * i]), float(s[2 + 7 * i]), float(s[3 +
7 * i])
dx, dy, dz = float(s[4 + 7 * i]), float(s[5 + 7 * i]), float(s[6 +
7 * i])
# Reconstruct start and end points
xmin = x0 - z0 * dx / (dz + 1e-9)
xmax = x0 + (ff.nz - z0) * dx / (dz + 1e-9)
ymin = y0 - z0 * dy / (dz + 1e-9)
ymax = y0 + (ff.nz - z0) * dy / (dz + 1e-9)
# Output line
line = "%.23s %8.3f %8.3f %8.3f %8.3f %8.5f %s unidentified sunlit\n" %\
(ff.nfd, xmin, ymin, xmax, ymax, ff.texp, 99999-i)
lines.append(line)
# Store identifications
fp = open(fname + ".id", "a")
for line in lines:
fp.write(line)
fp.close()
return [satid(line) for line in lines]
for fname in fnames:
# Generate star catalog
if not os.path.exists(fname + ".cat"):
pix_catalog = generate_star_catalog(fname)
# Calibrate from reference
calibrate_from_reference(fname, "test.fits", pix_catalog)
# Generate satellite predictions
generate_satellite_predictions(fname)
# Detect lines with 3D Hough transform
ids = find_hough3d_lines(fname)
# Get properties
ff = fourframe(fname)
# Extract tracks
if is_calibrated(ff):
extract_tracks(fname, trkrmin, drdtmin, trksig, ntrkmin,
root_dir, results_dir)
# Stars available and used
nused = np.sum(pix_catalog.flag == 1)
nstars = pix_catalog.nstars
# Write output
output = "%s %10.6f %10.6f %4d/%4d %5.1f %5.1f %6.2f +- %6.2f" % (
ff.fname, ff.crval[0], ff.crval[1], nused, nstars,
3600.0 * ff.crres[0], 3600.0 * ff.crres[1], np.mean(
ff.zavg), np.std(ff.zavg))