def on_filter(self, max_alt: int, max_time: datetime) -> None:
assert self._traffic is not None
west, east, south, north = self.map_plot.ax.get_extent(
crs=PlateCarree())
self._tview = self._traffic.before(max_time).sort_values("timestamp")
if self._tview is None:
return
filtered = Traffic.from_flights(
Flight(f.data.ffill().bfill()) for f in self._tview)
if "altitude" in filtered.data.columns:
filtered = filtered.query(
f"altitude != altitude or altitude <= {max_alt}")
if "latitude" in self._tview.data.columns:
filtered = filtered.query("latitude != latitude or "
f"({west} <= longitude <= {east} and "
f"{south} <= latitude <= {north})")
self.identifier_select.clear()
text = self.identifier_input.text()
# cast is necessary because of the @lru_cache on callsigns which hides
# the type annotation
for callsign in sorted(cast(Set[str], filtered.callsigns)):
if re.match(text, callsign, flags=re.IGNORECASE):
self.identifier_select.addItem(callsign)
callsigns = cast(Set[str], filtered.callsigns)
self.map_plot.default_plot(self._tview[callsigns])
self.set_float_columns()
def main():
indir = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/DULLES_GOAROUND/IN_LRG/'
outdir_kml = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/DULLES_GOAROUND/OUT_KML/'
main_types = OSFD.make_typlist_all()
types_list = main_types[0][1]
files = glob.glob(indir + '*2018*.pkl')
files.sort()
fli_len = len(files)
# Number of files to open in one go
n_files_proc = 24
pool_proc = 50
f_data = []
pool = mp.Pool(processes=pool_proc)
for i in range(0, fli_len, n_files_proc):
print("Processing batch starting with " + str(i + 1).zfill(5) +
" of " + str(fli_len).zfill(5))
p_list = []
# First we load several files at once
for j in range(0, n_files_proc):
if (i + j < fli_len):
p_list.append(
pool.apply_async(OSFD.get_flight, args=(files[i + j], )))
for p in p_list:
t_res = p.get()
for fli in t_res:
f_data.append(fli)
traf_arr = Traffic.from_flights(f_data)
p_list = []
results = []
f_data = []
end_time = traf_arr.end_time
# Now we process the results
for flight in traf_arr:
if (flight.stop + timedelta(minutes=5) < end_time):
p_list.append(
pool.apply_async(p_f,
args=(
flight,
runway_list,
outdir_kml,
types_list,
)))
else:
f_data.append(flight)
for p in p_list:
t_res = p.get()
results.append(t_res)
def __call__(
self,
fun: Callable[..., "LazyTraffic"],
*args,
**kwargs,
) -> Optional["Flight"]:
from traffic.core import Flight, Traffic # noqa: F811
in_ = Traffic.from_flights(
segment.assign(index_=i) for i, segment in enumerate(self))
if in_ is None:
return None
out_ = fun(in_, *args, **kwargs).eval()
if out_ is None:
return None
return Flight(out_.data)
print("Processing batch starting with " + str(i + 1).zfill(5) + " of " +
str(fli_len).zfill(5))
p_list = []
# First we load several files at once
for j in range(0, n_files_proc):
if (i + j < fli_len):
p_list.append(
pool.apply_async(DF.get_flight_basic, args=(files[i + j], )))
for p in p_list:
t_res = p.get()
for fli in t_res:
f_data.append(fli)
try:
traf_arr = Traffic.from_flights(f_data)
except:
counter = counter + n_files_proc
continue
p_list = []
results = []
f_data = []
end_time = traf_arr.end_time
# Now we process the results
for flight in traf_arr:
p_list.append(pool.apply_async(DF.get_acft, args=(flight, )))
for p in p_list:
def main():
indir = '/gf2/eodg/SRP001_PROUD_TURBREP/DULLES_GOAROUND/IN/'
outdir = '/gf2/eodg/SRP001_PROUD_TURBREP/DULLES_GOAROUND/OUT/'
files = glob.glob(indir + '*201802*.pkl')
files.sort()
do_plots = False
do_kml = False
first = True
counters = 0
main_types = OSF.make_typlist()
alt_hist_main = np.zeros((110, 540))
gs_hist_main = np.zeros((110, 50))
lalo_hist_main = np.zeros((220, 300))
vsp_hist_main = np.zeros((110, 120))
gal_hist_main = np.zeros((110, 500))
hdg_hist_main = np.zeros((110, 80))
fli_len = len(files)
# Number of files to open in one go
n_files_proc = 50
plot_lim = 50
f_data = []
pool = mp.Pool(processes=55)
for typr in main_types:
kml = simplekml.Kml(open=1)
alt_hist_main[:, :] = 0.
gs_hist_main[:, :] = 0.
lalo_hist_main[:, :] = 0.
vsp_hist_main[:, :] = 0.
gal_hist_main[:, :] = 0.
hdg_hist_main[:, :] = 0.
plt.clf()
print("Now processing:", typr[0])
types_list = typr[1]
outf_format = typr[2]
figs, nums = OSF.setup_figs()
for i in range(0, fli_len, n_files_proc):
print("Processing batch starting with " + str(i + 1).zfill(5) +
" of " + str(fli_len).zfill(5))
p_list = []
# First we load several files at once
for j in range(0, n_files_proc):
if (i + j < fli_len):
p_list.append(
pool.apply_async(OSF.get_flight,
args=(files[i + j], )))
for p in p_list:
t_res = p.get()
for fli in t_res:
f_data.append(fli)
traf_arr = Traffic.from_flights(f_data)
p_list = []
results = []
f_data = []
end_time = traf_arr.end_time
# Now we process the results
for flight in traf_arr:
if (flight.stop + timedelta(minutes=5) < end_time):
p_list.append(
pool.apply_async(proc_flight,
args=(
flight,
rwy,
gate,
types_list,
do_plots,
do_kml,
)))
else:
f_data.append(flight)
for p in p_list:
t_res = p.get()
results.append(t_res)
# Below is for either processing method
for ddi in results:
if (ddi == None):
continue
if (do_plots):
OSF.add_data_to_fig(ddi, nums)
alt_hist_main = alt_hist_main + ddi['alt_hi']
gs_hist_main = gs_hist_main + ddi['gspd_hi']
lalo_hist_main = lalo_hist_main + ddi['pos_hi']
vsp_hist_main = vsp_hist_main + ddi['vspd_hi']
gal_hist_main = gal_hist_main + ddi['galt_hi']
hdg_hist_main = hdg_hist_main + ddi['hdgs_hi']
if (do_kml):
ls = kml.newlinestring(name=ddi['callsign'])
ls.altitudemode = simplekml.AltitudeMode.relativetoground
ls.extrude = 0
for irng in range(0, len(ddi['lats'])):
ls.coords.addcoordinates([
(ddi['lons'][irng], ddi['lats'][irng],
ddi['galt'][irng] / 3.28084)
])
counters = counters + n_files_proc
if (counters > plot_lim):
if (do_plots):
OSF.plot_figs(nums, outdir, outf_format)
try:
outi = {}
outi['alt_hist'] = alt_hist_main
outi['gs_hist'] = gs_hist_main
outi['lalo_hist'] = lalo_hist_main
outi['vsp_hist'] = vsp_hist_main
outi['gal_hist'] = gal_hist_main
outi['hdg_hist'] = hdg_hist_main
np.save(outdir + outf_format + 'AC_Data.npy', outi)
if (do_kml):
kml.save(outdir + outf_format + 'TRAJ.kml')
except:
print("ERROR")
counters = 0
outi = {}
outi['alt_hist'] = alt_hist_main
outi['gs_hist'] = gs_hist_main
outi['lalo_hist'] = lalo_hist_main
outi['vsp_hist'] = vsp_hist_main
outi['gal_hist'] = gal_hist_main
outi['hdg_hist'] = hdg_hist_main
if (do_plots):
OSF.plot_figs(nums, outdir, outf_format)
np.save(outdir + outf_format + 'AC_Data.npy', outi)
if (do_kml):
kml.save(outdir + outf_format + 'TRAJ.kml')
def main():
indir = '/gf2/eodg/SRP001_PROUD_TURBREP/DULLES_GOAROUND/IN/'
outdir = '/gf2/eodg/SRP001_PROUD_TURBREP/DULLES_GOAROUND/OUT/'
files = glob.glob(indir + '*2018*.pkl')
files.sort()
do_plots = False
do_kml = False
first = True
counters = 0
main_types = OSF.make_typlist()
fli_len = len(files)
# Number of files to open in one go
n_files_proc = 24
plot_lim = 200
pool_proc = 50
f_data = []
pool = mp.Pool(processes=pool_proc)
for typr in main_types:
print("Now processing:", typr[0])
kml = simplekml.Kml(open=1)
rwy01l = kml.newdocument(name='RWY01L')
rwy01c = kml.newdocument(name='RWY01C')
rwy01r = kml.newdocument(name='RWY01R')
rwy19l = kml.newdocument(name='RWY19L')
rwy19c = kml.newdocument(name='RWY19C')
rwy19r = kml.newdocument(name='RWY19R')
types_list = typr[1]
outf_format = typr[2]
for i in range(0, fli_len, n_files_proc):
print("Processing batch starting with " + str(i + 1).zfill(5) +
" of " + str(fli_len).zfill(5))
p_list = []
# First we load several files at once
for j in range(0, n_files_proc):
if (i + j < fli_len):
p_list.append(
pool.apply_async(OSF.get_flight,
args=(files[i + j], )))
for p in p_list:
t_res = p.get()
for fli in t_res:
f_data.append(fli)
traf_arr = Traffic.from_flights(f_data)
p_list = []
results = []
f_data = []
end_time = traf_arr.end_time
# Now we process the results
for flight in traf_arr:
if (flight.stop + timedelta(minutes=5) < end_time):
p_list.append(
pool.apply_async(p_f,
args=(
flight,
runway_list,
types_list,
do_plots,
do_kml,
)))
else:
f_data.append(flight)
for p in p_list:
t_res = p.get()
results.append(t_res)
# Below is for either processing method
for ddi in results:
if (ddi == None):
continue
if (ddi['ga'] != True):
continue
if (ddi['rwy'] == "01L"):
ls = rwy01l.newlinestring()
elif (ddi['rwy'] == "01C"):
ls = rwy01c.newlinestring()
elif (ddi['rwy'] == "01R"):
ls = rwy01r.newlinestring()
elif (ddi['rwy'] == "19L"):
ls = rwy19l.newlinestring()
elif (ddi['rwy'] == "19C"):
ls = rwy19c.newlinestring()
elif (ddi['rwy'] == "19R"):
ls = rwy19r.newlinestring()
else:
print("Unknown runway!", ddi['rwy'])
ls = kml.newlinestring()
ls.name = ddi['callsign']
ls.description = ddi['time'].strftime("%Y%m%d%H%M%S") + "\n" +\
ddi['icao24'] + '\n' +\
str(ddi['bdist'])
ls.style.linestyle.color = simplekml.Color.red
ls.altitudemode = simplekml.AltitudeMode.relativetoground
ls.extrude = 0
for irng in range(0, len(ddi['lats'])):
ls.coords.addcoordinates([
(ddi['lons'][irng], ddi['lats'][irng],
ddi['galt'][irng] / 3.28084)
])
counters = counters + n_files_proc
if (counters > plot_lim):
print("SAVING KML")
try:
kml.save(outdir + outf_format + 'TRAJ.kml')
except:
print("ERROR SAVING KML:",
outdir + outf_format + 'TRAJ.kml')
counters = 0
kml.save(outdir + outf_format + 'TRAJ.kml')
def main():
indir = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/DULLES/IN/'
outdir = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/DULLES/OUT/'
dater = datetime(2018,1,1,0,0)
main_types = OSF.make_typlist()
# Number of files to open in one go
n_files_proc = 24
plot_lim = 200
pool_proc = 50
n_01l = 0
n_01c = 0
n_01r = 0
n_19l = 0
n_19c = 0
n_19r = 0
f_data = []
pool = mp.Pool(processes=pool_proc)
while True:
files = glob.glob(indir+'*'+dater.strftime("%Y%m%d")+'*.pkl')
files.sort()
fli_len = len(files)
print("Now processing:",dater)
for typr in main_types:
figs, nums = OSF.setup_figs()
types_list = typr[1]
outf_format = typr[2]
for i in range(0, fli_len, n_files_proc):
p_list = []
# First we load several files at once
for j in range(0, n_files_proc):
if (i+j < fli_len):
p_list.append(pool.apply_async(OSF.get_flight, args=(files[i+j],)))
for p in p_list:
t_res = p.get()
for fli in t_res:
f_data.append(fli)
traf_arr = Traffic.from_flights(f_data)
p_list = []
results = []
f_data = []
end_time = traf_arr.end_time
# Now we process the results
for flight in traf_arr:
if (flight.stop + timedelta(minutes=5) < end_time):
p_list.append(pool.apply_async(p_f, args=(flight,
runway_list,
types_list,)))
else:
f_data.append(flight)
for p in p_list:
t_res = p.get()
results.append(t_res)
# Below is for either processing method
for ddi in results:
if (ddi == None):
continue
if (ddi['rwy'] == '01L'):
n_01l = n_01l + 1
if (ddi['rwy'] == '01C'):
n_01c = n_01c + 1
if (ddi['rwy'] == '01R'):
n_01r = n_01r + 1
if (ddi['rwy'] == '19L'):
n_19l = n_19l + 1
if (ddi['rwy'] == '19C'):
n_19c = n_19c + 1
if (ddi['rwy'] == '19R'):
n_19r = n_19r + 1
OSF.add_data_to_fig(ddi, nums)
OSF.plot_figs(nums, outdir, dater, outf_format)
dater = dater + timedelta(days = 1)
print(' - ',n_01l,n_01c,n_01r,n_19l,n_19c,n_19r)
# quit()
if (dater.year > 2018):
break
def main(start_n, fidder, do_write):
"""The main code for detecting go-arounds.
Arguments:
start_n -- The index of the first file to read
fidder -- the id of an open file to write log information into
do_write -- boolean flag specifying whether to output data to textfile
"""
# Total number of aircraft seen
tot_n_ac = 0
# Of which go-arounds
tot_n_ga = 0
top_dir = '/gf2/eodg/SRP002_PROUD_ADSBREP/GO_AROUNDS/VABB/'
# indir stores the opensky data
indir = top_dir + 'INDATA/'
# odir_nm stores plots for normal landings
odir_pl_nm = top_dir + 'OUT_PLOT/NORM/'
# odir_ga stores plots for detected go-arounds
odir_pl_ga = top_dir + 'OUT_PLOT/PSGA/'
# odir_nm stores plots for normal landings
odir_da_nm = top_dir + 'OUT_DATA/NORM/'
# odir_ga stores plots for detected go-arounds
odir_da_ga = top_dir + 'OUT_DATA/PSGA/'
odirs = [odir_pl_nm, odir_pl_ga, odir_da_nm, odir_da_ga]
# Output filenames for saving data about go-arounds
out_file_ga = 'GA_MET_NEW.csv'
# Output filenames for saving data about non-go-arounds
out_file_noga = 'GA_NOGA_NEW.csv'
t_frmt = "%Y/%m/%d %H:%M:%S"
# File to save met info for g/a flights
if (do_write):
metfid = open(out_file_ga, 'w')
nogfid = open(out_file_noga, 'w')
metfid.write('ICAO24, Callsign, GA_Time, L_Time, Runway, Heading, Alt, Lat, \
Lon, gapt, rocvar, hdgvar, latvar, lonvar, gspvar, \
Temp, Dewp, Wind_Spd, Wind_Gust, Wind_Dir,Cld_Base,\
CB, Vis, Pressure\n')
nogfid.write('ICAO24, Callsign, GA_Time, L_Time, Runway, gapt, rocvar, \
hdgvar, latvar, lonvar, gspvar, \
Temp, Dewp, Wind_Spd, Wind_Gust, Wind_Dir,Cld_Base,\
CB, Vis, Pressure\n')
files = []
files = glob.glob(indir+'**.pkl')
files.sort()
fli_len = len(files)
colormap = {'GND': 'black', 'CL': 'green', 'CR': 'blue',
'DE': 'orange', 'LVL': 'purple', 'NA': 'red'}
# Number of files to open in one go
n_files_proc = 55
pool_proc = 100
f_data = []
pool = mp.Pool(processes=pool_proc)
for main_count in range(start_n, fli_len, n_files_proc):
print("Processing batch starting with "
+ str(main_count + 1).zfill(5) + " of "
+ str(fli_len).zfill(5))
fidder.write("Processing batch starting with "
+ str(main_count + 1).zfill(5) + " of "
+ str(fli_len).zfill(5) + '\n')
p_list = []
# First we load several files at once
for j in range(0, n_files_proc):
if (main_count+j < fli_len):
p_list.append(pool.apply_async(OSF.get_flight,
args=(files[main_count+j],)))
for p in p_list:
t_res = p.get()
for fli in t_res:
f_data.append(fli)
if(len(f_data) < 1):
continue
traf_arr = Traffic.from_flights(f_data)
p_list = []
f_data = []
# Extend array end time if there's only one flight, else processing
# will fail as we check to ensure latest flight is > 5 mins from end
if (len(traf_arr) == 1):
end_time = traf_arr.end_time + timedelta(minutes=10)
print("Extending timespan due to single aircraft")
else:
end_time = traf_arr.end_time
# Now we process the results
for flight in traf_arr:
if (flight.stop + timedelta(minutes=5) < end_time):
p_list.append(pool.apply_async(OSF.proc_fl,
args=(flight,
VABB.rwy_list,
odirs,
colormap,
True,
False,)))
else:
f_data.append(flight)
for p in p_list:
t_res = p.get()
if (t_res != -1):
tot_n_ac += 1
# If there's a go-around, this will be True
if (t_res[0]):
if (do_write):
metfid.write(t_res[1] + ',' + t_res[2] + ',')
metfid.write(t_res[3].strftime(t_frmt) + ',')
metfid.write(t_res[4].strftime(t_frmt) + ',')
metfid.write(t_res[5] + ',')
if (t_res[6] < 0):
t_res[6] = 360 + t_res[6]
metfid.write(str(t_res[6]) + ',')
metfid.write(str(t_res[7]) + ',')
metfid.write(str(t_res[8]) + ',')
metfid.write(str(t_res[9]) + ',')
metfid.write(str(t_res[10]) + ',')
metfid.write(str(t_res[11]) + ',')
metfid.write(str(t_res[12]) + ',')
metfid.write(str(t_res[13]) + ',')
metfid.write(str(t_res[14]) + ',')
metfid.write(str(t_res[15]) + ',')
metfid.write(str(t_res[16].temp) + ',')
metfid.write(str(t_res[16].dewp) + ',')
metfid.write(str(t_res[16].w_s) + ',')
metfid.write(str(t_res[16].w_g) + ',')
metfid.write(str(t_res[16].w_d) + ',')
metfid.write(str(t_res[16].cld) + ',')
if t_res[16].cb:
metfid.write('1,')
else:
metfid.write('0,')
metfid.write(str(t_res[16].vis) + ',')
metfid.write(str(t_res[16].pres) + '\n')
tot_n_ga += 1
# Otherwise, do the following (doesn't save g/a location etc).
else:
if (do_write):
nogfid.write(t_res[1] + ',' + t_res[2] + ',')
nogfid.write(t_res[3].strftime(t_frmt) + ',')
nogfid.write(t_res[4].strftime(t_frmt) + ',')
nogfid.write(t_res[5] + ',')
nogfid.write(str(t_res[10]) + ',')
nogfid.write(str(t_res[11]) + ',')
nogfid.write(str(t_res[12]) + ',')
nogfid.write(str(t_res[13]) + ',')
nogfid.write(str(t_res[14]) + ',')
nogfid.write(str(t_res[15]) + ',')
try:
outstr = str(t_res[16].temp) + ','
outstr = outstr + str(t_res[16].dewp) + ','
outstr = outstr + str(t_res[16].w_s) + ','
outstr = outstr + str(t_res[16].w_g) + ','
outstr = outstr + str(t_res[16].w_d) + ','
outstr = outstr + str(t_res[16].cld) + ','
if t_res[16].cb:
outstr = outstr + '1,'
else:
outstr = outstr + '0,'
outstr = outstr + str(t_res[16].vis) + ','
outstr = outstr + str(t_res[16].pres)
except Exception as e:
print("No METAR data for this flight")
outstr = ''
nogfid.write(outstr + '\n')
print("\t-\tHave processed " + str(tot_n_ac) +
" aircraft. Have seen " + str(tot_n_ga) + " go-arounds.")
if (do_write):
metfid.close()
nogfid.close()
def main():
figs, nums = OSP.setup_figs_rwy_vabb()
indir = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/VABB/INDATA/'
outdir_pl = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/VABB/PLOT/'
outdir_kml = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/VABB/KML/'
outdir_np = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/VABB/ODATA/'
files = glob.glob(indir + '*.pkl')
files.sort()
counters = 0
good_calls = DD.good_calls_vabb()
fli_len = len(files)
# Number of files to open in one go
n_files_proc = 24
plot_lim = 200
pool_proc = 100
f_data = []
pool = mp.Pool(processes=pool_proc)
for main_count in range(0, fli_len, n_files_proc):
print("Processing batch starting with " +
str(main_count + 1).zfill(5) + " of " + str(fli_len).zfill(5))
p_list = []
# First we load several files at once
for j in range(0, n_files_proc):
if (main_count + j < fli_len):
p_list.append(
pool.apply_async(DF.get_flight,
args=(files[main_count + j], good_calls,
True, False)))
for p in p_list:
t_res = p.get()
for fli in t_res:
f_data.append(fli)
traf_arr = Traffic.from_flights(f_data)
p_list = []
results = []
f_data = []
end_time = traf_arr.end_time
# Now we process the results
for flight in traf_arr:
if (flight.stop + timedelta(minutes=5) < end_time):
p_list.append(
pool.apply_async(p_f_appr,
args=(
flight,
VABB.rwy_list,
outdir_np,
outdir_kml,
)))
else:
f_data.append(flight)
for p in p_list:
t_res = p.get()
results.append(t_res)
# Below is for either processing method
for ddi in results:
if (ddi is None):
continue
rwy = ddi['rwy']
for i in range(0, len(nums)):
if (rwy == VABB.rwy_list[i].name):
OSP.add_data_single_rwy(nums[i], ddi)
counters = counters + n_files_proc
if (counters > plot_lim):
for i in range(0, len(nums)):
OSP.save_plot_single_rwy(nums[i], VABB.rwy_list[i], main_count,
outdir_pl)
figs, nums = OSP.setup_figs_rwy_vabb()
counters = 0
for i in range(0, len(nums)):
OSP.save_plot_single_rwy(nums[i], VABB.rwy_list[i], main_count,
outdir_pl)
