def openFile(self, *args, **kwargs) -> None:
options = {
"caption":
"Open file",
"filter": ("Pandas DataFrame (*.pkl);;"
"CSV files (*.csv);;"
"Sqlite3 files (*.db)"),
# "filter": "Data files (*.csv *.pkl)",
"directory":
os.path.expanduser("~"),
}
self.filename = QFileDialog.getOpenFileName(self, **options)[0]
if self.filename == "":
return
self.filename = Path(self.filename)
self._traffic = Traffic.from_file(self.filename)
assert self._traffic is not None
self._tview = self._traffic.sort_values("timestamp")
assert self._tview is not None
self.open_dropdown.setItemText(0, self.filename.name)
self.map_plot.default_plot(self._tview)
self.identifier_select.clear()
for callsign in sorted(cast(Set[str], self._tview.callsigns)):
self.identifier_select.addItem(callsign)
self.set_time_range()
self.set_float_columns()
def get_flight(inf, calls=['*'], landing=False, takeoff=False):
flist = []
try:
fdata = Traffic.from_file(inf).clean_invalid()
fdata = fdata.filter().eval()
except:
return flist
for flight in fdata:
if (flight.callsign[0:7] == 'WILDLIF'):
continue
if (flight.callsign[0:7] == 'FOLOWME'):
continue
if (flight.callsign[0:5] == 'RADAR'):
continue
if (flight.callsign[0:7] == 'FIRETEN'):
continue
if (flight.callsign[0:8] == 'DUTYOFIR'):
continue
if (np.nanmax(flight.data['altitude'].values) < 300):
continue
if (flight.callsign[0:3] not in calls):
continue
if (landing):
if (np.nanmean(flight.data['vertical_rate'].values) > 200):
continue
if (takeoff):
if (np.nanmean(flight.data['vertical_rate'].values) < -200):
continue
if (flight.typecode is None):
flist.append(flight)
return flist
def on_filter(self, max_alt: int, max_time: datetime) -> None:
assert self._traffic is not None
west, east, south, north = self.map_plot.ax.get_extent(
crs=PlateCarree())
self._tview = self._traffic.before(max_time).sort_values("timestamp")
if self._tview is None:
return
filtered = Traffic.from_flights(
Flight(f.data.ffill().bfill()) for f in self._tview)
if "altitude" in filtered.data.columns:
filtered = filtered.query(
f"altitude != altitude or altitude <= {max_alt}")
if "latitude" in self._tview.data.columns:
filtered = filtered.query("latitude != latitude or "
f"({west} <= longitude <= {east} and "
f"{south} <= latitude <= {north})")
self.identifier_select.clear()
text = self.identifier_input.text()
# cast is necessary because of the @lru_cache on callsigns which hides
# the type annotation
for callsign in sorted(cast(Set[str], filtered.callsigns)):
if re.match(text, callsign, flags=re.IGNORECASE):
self.identifier_select.addItem(callsign)
callsigns = cast(Set[str], filtered.callsigns)
self.map_plot.default_plot(self._tview[callsigns])
self.set_float_columns()
def main(args):
if not os.path.exists(args.savepath):
os.makedirs(args.savepath)
traffic = Traffic.from_file(args.data)
list_features = ["track_unwrapped", "longitude", "latitude", "altitude"]
nb_samples = len(traffic[0])
nb_features = len(list_features)
algo_clustering = AutoencoderTSNE(
gpu=args.gpu,
model=Autoencoder((nb_samples * nb_features, 32, 8, 2)),
learning_rate=args.learning_rate,
weight_decay=args.learning_rate,
lambda_kl=args.lambda_kl,
nb_iterations=args.nb_iterations,
batch_size=args.batch_size,
algo_clustering=DBSCAN(eps=0.06, min_samples=20),
distance_trajectory="euclidean", # delta_max
savepath=f"{args.savepath}/model.pth",
)
t_tsne = traffic.clustering(
nb_samples=None, # nb_samples,
features=list_features,
clustering=algo_clustering,
transform=MinMaxScaler(feature_range=(-1, 1)),
).fit_predict()
t_tsne.to_parquet(f"{args.savepath}/t_tsne.parquet")
def get_flight(inf):
"""Load a series of flights from a file using Xavier's 'traffic' library.
Input:
- inf, the input filename
Returns:
- a list of flights
"""
flist = []
# try:
fdata = Traffic.from_file(inf).query("latitude == latitude")
fdata = fdata.clean_invalid().filter().eval()
# except:
# return flist
for flight in fdata:
pos = flight.callsign.find(CNS.search_call)
if (pos < 0):
continue
f_data = flight.data
f_data = f_data.drop_duplicates('timestamp')
f_data = f_data.drop_duplicates('longitude')
f_data = f_data.drop_duplicates('latitude')
f_data = f_data.query('altitude<10000')
f_data = f_data.dropna()
flight.data = f_data
flist.append(flight)
return flist
def loadFlights(start_day, end_day):
filename = getFlightsFilename(start_day, end_day)
flights = Traffic.from_file(filename)
if logLvl >= 1:
print("Loaded " + str(len(flights)) + " flights")
return flights
def loadFlights(start_day, end_day, callsign):
filename = recordings_folder + getFlightsFilename(start_day, end_day,
callsign)
flights = Traffic.from_file(filename)
print("Loaded " + str(len(flights)) + " flights")
return flights
def main():
indir = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/DULLES_GOAROUND/IN_LRG/'
outdir_kml = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/DULLES_GOAROUND/OUT_KML/'
main_types = OSFD.make_typlist_all()
types_list = main_types[0][1]
files = glob.glob(indir + '*2018*.pkl')
files.sort()
fli_len = len(files)
# Number of files to open in one go
n_files_proc = 24
pool_proc = 50
f_data = []
pool = mp.Pool(processes=pool_proc)
for i in range(0, fli_len, n_files_proc):
print("Processing batch starting with " + str(i + 1).zfill(5) +
" of " + str(fli_len).zfill(5))
p_list = []
# First we load several files at once
for j in range(0, n_files_proc):
if (i + j < fli_len):
p_list.append(
pool.apply_async(OSFD.get_flight, args=(files[i + j], )))
for p in p_list:
t_res = p.get()
for fli in t_res:
f_data.append(fli)
traf_arr = Traffic.from_flights(f_data)
p_list = []
results = []
f_data = []
end_time = traf_arr.end_time
# Now we process the results
for flight in traf_arr:
if (flight.stop + timedelta(minutes=5) < end_time):
p_list.append(
pool.apply_async(p_f,
args=(
flight,
runway_list,
outdir_kml,
types_list,
)))
else:
f_data.append(flight)
for p in p_list:
t_res = p.get()
results.append(t_res)
def load_flights():
flights=list()
for file in files:
flights.extend(Traffic.from_file(file))
callsigns=dict()
for flight in flights:
if flight.callsign not in callsigns:
callsigns[flight.callsign]=0
callsigns[flight.callsign]+=1
print(callsigns)
filtered=set()
for c in callsigns:
if callsigns[c]>monthly_flights:
filtered.add(c)
print(filtered)
def __call__(
self,
fun: Callable[..., "LazyTraffic"],
*args,
**kwargs,
) -> Optional["Flight"]:
from traffic.core import Flight, Traffic # noqa: F811
in_ = Traffic.from_flights(
segment.assign(index_=i) for i, segment in enumerate(self))
if in_ is None:
return None
out_ = fun(in_, *args, **kwargs).eval()
if out_ is None:
return None
return Flight(out_.data)
def pretrained_clust(
traffic_file,
list_features,
algo_clustering,
model,
pretrained_path,
to_pickle,
gpu=0,
):
t = Traffic.from_file(traffic_file)
ae_tsne = AutoencoderTSNE(
gpu=gpu,
model=model,
pretrained_path=pretrained_path,
algo_clustering=algo_clustering,
)
t_c = t.clustering(
nb_samples=None,
features=list_features,
clustering=ae_tsne,
transform=MinMaxScaler(feature_range=(-1, 1)),
).fit_predict()
re, scores = ae_tsne.score_samples()
t_c_re = pd.DataFrame.from_records(
[dict(flight_id=f.flight_id, re=re) for f, re in zip(t_c, re)])
t_c = t_c.merge(t_c_re, on="flight_id")
d = {"flight_id": "nunique", "re": ["mean", "min", "max"]}
if scores is not None:
t_c_scores = pd.DataFrame.from_records([
dict(flight_id=f.flight_id, score=score)
for f, score in zip(t_c, scores)
])
t_c = t_c.merge(t_c_scores, on="flight_id")
d["score"] = ["mean", "min", "max"]
print(t_c.groupby(["cluster"]).agg(d))
t_c.to_pickle(to_pickle)
return ae_tsne, t_c
def callsign_monthly_frequency():
all_callsigns=set()
for file in files:
flights=Traffic.from_file(file)
callsigns=dict()
for flight in flights:
if flight.callsign not in callsigns:
callsigns[flight.callsign]=0
callsigns[flight.callsign]+=1
#print(callsigns)
filtered=set()
for c in callsigns:
if callsigns[c]>monthly_flights:
filtered.add((c, callsigns[c]))
all_callsigns.add(c)
print(file[len(recording_folder)+5:len(recording_folder)+12], filtered)
return all_callsigns
def icaos_monthly_frequency():
all_icaos=set()
for file in files:
flights=Traffic.from_file(file)
icaos=dict()
for flight in flights:
if flight.icao24 not in icaos:
icaos[flight.icao24]=0
icaos[flight.icao24]+=1
#print(icaos)
filtered=set()
for c in icaos:
if icaos[c]>monthly_flights:
filtered.add((c, icaos[c]))
all_icaos.add(c)
print(file[len(recording_folder)+5:len(recording_folder)+12], filtered)
return all_icaos
def get_flight_basic(inf):
flist = []
try:
fdata = Traffic.from_file(inf)
except:
return flist
for flight in fdata:
if (flight.callsign[0:7] == 'WILDLIF'):
continue
if (flight.callsign[0:7] == 'FOLOWME'):
continue
if (flight.callsign[0:5] == 'RADAR'):
continue
if (flight.callsign[0:7] == 'FIRETEN'):
continue
if (flight.callsign[0:8] == 'DUTYOFIR'):
continue
if (flight.typecode is None):
flist.append(flight)
return flist
def plot_data(flights: Traffic, airport: Airport, output: Path,
arrival: int=1):
fig = plt.figure(figsize=(10, 10))
ax = plt.axes(projection=UTM((airport.lon + 180) // 6,
southern_hemisphere=(airport.lat > 0)))
ax.add_feature(countries(scale='50m'))
ax.add_feature(rivers(scale='50m'))
ax.add_feature(lakes(scale='50m'))
ax.gridlines()
ax.set_extent(airport.extent)
try:
from cartotools.osm import request, tags
request(f'{airport.icao} airport', **tags.airport).plot(ax)
except Exception:
pass
fd = flights.data
fd = fd[fd.longitude > airport.lon - 1]
fd = fd[fd.longitude < airport.lon + 1]
fd = fd[fd.latitude > airport.lat - 1]
fd = fd[fd.latitude < airport.lat + 1]
fd = fd.groupby('callsign').filter(
lambda f: arrival * f.vertical_rate[:-100].mean() < -10)
for flight in Traffic(fd):
flight.plot(ax)
fig.suptitle(f"{airport.name}")
fig.set_tight_layout(True)
logging.info(f"Image saved as {output}")
fig.savefig(output.as_posix())
for p in processes:
p.join()
# Move on to the next block of times
start_dt = start_dt + timedelta(hours=nummer)
# If we have reached the end of the block then exit
if (start_dt >= end_dt):
break
files = glob(odir + '*.pkl')
files.sort()
first = True
for inf in files:
if first:
fdata = Traffic.from_file(inf).query("latitude == latitude")
fdata = fdata.clean_invalid().filter().eval()
first = False
else:
tdata = Traffic.from_file(inf).query("latitude == latitude")
tdata = tdata.clean_invalid().filter().eval()
fdata = fdata + tdata
print(len(fdata))
figsize = (20, 20)
projparm = ccrs.PlateCarree(central_longitude=(bounds[2] - bounds[0]) / 2)
alt_cut = 500
fig, ax = plt.subplots(1,
1,
def to_bluesky(
traffic: Traffic,
filename: Union[str, Path],
minimum_time: Optional[timelike] = None,
) -> None:
"""Generates a Bluesky scenario file."""
if minimum_time is not None:
traffic = traffic.after(minimum_time)
if isinstance(filename, str):
filename = Path(filename)
if not filename.parent.exists():
filename.parent.mkdir(parents=True)
altitude = ("baro_altitude"
if "baro_altitude" in traffic.data.columns else "altitude")
if "typecode" not in traffic.data.columns:
traffic = Traffic(
traffic.data.merge(
aircraft.data[["icao24",
"typecode"]].drop_duplicates("icao24"),
on="icao24",
how="inner",
))
if "cas" not in traffic.data.columns:
traffic = Traffic(
traffic.data.assign(cas=vtas2cas(traffic.data.ground_speed,
traffic.data[altitude])))
with filename.open("w") as fh:
t_delta = traffic.data.timestamp - traffic.start_time
data = (traffic.assign_id().data.groupby("flight_id").filter(
lambda x: x.shape[0] > 3).assign(
timedelta=t_delta.apply(fmt_timedelta)).sort_values(
by="timestamp"))
for column in data.columns:
data[column] = data[column].astype(str)
is_created: List[str] = []
is_deleted: List[str] = []
start_time = cast(pd.Timestamp, traffic.start_time).time()
fh.write(f"00:00:00> TIME {start_time}\n")
# Add some bluesky command for the visualisation
# fh.write("00:00:00>trail on\n")
# fh.write("00:00:00>ssd conflicts\n")
# We remove an object when it's its last data point
buff = data.groupby("flight_id").timestamp.max()
dd = pd.DataFrame(columns=["timestamp"],
data=buff.values,
index=buff.index.values)
map_icao24_last_point = {}
for i, v in dd.iterrows():
map_icao24_last_point[i] = v[0]
# Main loop to write lines in the scenario file
for _, v in data.iterrows():
if v.flight_id not in is_created:
# If the object is not created then create it
is_created.append(v.flight_id)
fh.write(f"{v.timedelta}> CRE {v.callsign} {v.typecode} "
f"{v.latitude} {v.longitude} {v.track} "
f"{v[altitude]} {v.cas}\n")
elif v.timestamp == map_icao24_last_point[v.flight_id]:
# Remove an aircraft when no data are available
if v.flight_id not in is_deleted:
is_deleted.append(v.flight_id)
fh.write(f"{v.timedelta}> DEL {v.callsign}\n")
elif v.flight_id not in is_deleted:
# Otherwise update the object position
fh.write(f"{v.timedelta}> MOVE {v.callsign} "
f"{v.latitude} {v.longitude} {v[altitude]} "
f"{v.track} {v.cas} {v.vertical_rate}\n")
logging.info(f"Scenario file {filename} written")
print("Processing batch starting with " + str(i + 1).zfill(5) + " of " +
str(fli_len).zfill(5))
p_list = []
# First we load several files at once
for j in range(0, n_files_proc):
if (i + j < fli_len):
p_list.append(
pool.apply_async(DF.get_flight_basic, args=(files[i + j], )))
for p in p_list:
t_res = p.get()
for fli in t_res:
f_data.append(fli)
try:
traf_arr = Traffic.from_flights(f_data)
except:
counter = counter + n_files_proc
continue
p_list = []
results = []
f_data = []
end_time = traf_arr.end_time
# Now we process the results
for flight in traf_arr:
p_list.append(pool.apply_async(DF.get_acft, args=(flight, )))
for p in p_list:
def main():
indir = '/gf2/eodg/SRP001_PROUD_TURBREP/DULLES_GOAROUND/IN/'
outdir = '/gf2/eodg/SRP001_PROUD_TURBREP/DULLES_GOAROUND/OUT/'
files = glob.glob(indir + '*201802*.pkl')
files.sort()
do_plots = False
do_kml = False
first = True
counters = 0
main_types = OSF.make_typlist()
alt_hist_main = np.zeros((110, 540))
gs_hist_main = np.zeros((110, 50))
lalo_hist_main = np.zeros((220, 300))
vsp_hist_main = np.zeros((110, 120))
gal_hist_main = np.zeros((110, 500))
hdg_hist_main = np.zeros((110, 80))
fli_len = len(files)
# Number of files to open in one go
n_files_proc = 50
plot_lim = 50
f_data = []
pool = mp.Pool(processes=55)
for typr in main_types:
kml = simplekml.Kml(open=1)
alt_hist_main[:, :] = 0.
gs_hist_main[:, :] = 0.
lalo_hist_main[:, :] = 0.
vsp_hist_main[:, :] = 0.
gal_hist_main[:, :] = 0.
hdg_hist_main[:, :] = 0.
plt.clf()
print("Now processing:", typr[0])
types_list = typr[1]
outf_format = typr[2]
figs, nums = OSF.setup_figs()
for i in range(0, fli_len, n_files_proc):
print("Processing batch starting with " + str(i + 1).zfill(5) +
" of " + str(fli_len).zfill(5))
p_list = []
# First we load several files at once
for j in range(0, n_files_proc):
if (i + j < fli_len):
p_list.append(
pool.apply_async(OSF.get_flight,
args=(files[i + j], )))
for p in p_list:
t_res = p.get()
for fli in t_res:
f_data.append(fli)
traf_arr = Traffic.from_flights(f_data)
p_list = []
results = []
f_data = []
end_time = traf_arr.end_time
# Now we process the results
for flight in traf_arr:
if (flight.stop + timedelta(minutes=5) < end_time):
p_list.append(
pool.apply_async(proc_flight,
args=(
flight,
rwy,
gate,
types_list,
do_plots,
do_kml,
)))
else:
f_data.append(flight)
for p in p_list:
t_res = p.get()
results.append(t_res)
# Below is for either processing method
for ddi in results:
if (ddi == None):
continue
if (do_plots):
OSF.add_data_to_fig(ddi, nums)
alt_hist_main = alt_hist_main + ddi['alt_hi']
gs_hist_main = gs_hist_main + ddi['gspd_hi']
lalo_hist_main = lalo_hist_main + ddi['pos_hi']
vsp_hist_main = vsp_hist_main + ddi['vspd_hi']
gal_hist_main = gal_hist_main + ddi['galt_hi']
hdg_hist_main = hdg_hist_main + ddi['hdgs_hi']
if (do_kml):
ls = kml.newlinestring(name=ddi['callsign'])
ls.altitudemode = simplekml.AltitudeMode.relativetoground
ls.extrude = 0
for irng in range(0, len(ddi['lats'])):
ls.coords.addcoordinates([
(ddi['lons'][irng], ddi['lats'][irng],
ddi['galt'][irng] / 3.28084)
])
counters = counters + n_files_proc
if (counters > plot_lim):
if (do_plots):
OSF.plot_figs(nums, outdir, outf_format)
try:
outi = {}
outi['alt_hist'] = alt_hist_main
outi['gs_hist'] = gs_hist_main
outi['lalo_hist'] = lalo_hist_main
outi['vsp_hist'] = vsp_hist_main
outi['gal_hist'] = gal_hist_main
outi['hdg_hist'] = hdg_hist_main
np.save(outdir + outf_format + 'AC_Data.npy', outi)
if (do_kml):
kml.save(outdir + outf_format + 'TRAJ.kml')
except:
print("ERROR")
counters = 0
outi = {}
outi['alt_hist'] = alt_hist_main
outi['gs_hist'] = gs_hist_main
outi['lalo_hist'] = lalo_hist_main
outi['vsp_hist'] = vsp_hist_main
outi['gal_hist'] = gal_hist_main
outi['hdg_hist'] = hdg_hist_main
if (do_plots):
OSF.plot_figs(nums, outdir, outf_format)
np.save(outdir + outf_format + 'AC_Data.npy', outi)
if (do_kml):
kml.save(outdir + outf_format + 'TRAJ.kml')
def main():
figs, nums = OSP.setup_figs_rwy_vabb()
indir = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/VABB/INDATA/'
outdir_pl = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/VABB/PLOT/'
outdir_kml = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/VABB/KML/'
outdir_np = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/VABB/ODATA/'
files = glob.glob(indir + '*.pkl')
files.sort()
counters = 0
good_calls = DD.good_calls_vabb()
fli_len = len(files)
# Number of files to open in one go
n_files_proc = 24
plot_lim = 200
pool_proc = 100
f_data = []
pool = mp.Pool(processes=pool_proc)
for main_count in range(0, fli_len, n_files_proc):
print("Processing batch starting with " +
str(main_count + 1).zfill(5) + " of " + str(fli_len).zfill(5))
p_list = []
# First we load several files at once
for j in range(0, n_files_proc):
if (main_count + j < fli_len):
p_list.append(
pool.apply_async(DF.get_flight,
args=(files[main_count + j], good_calls,
True, False)))
for p in p_list:
t_res = p.get()
for fli in t_res:
f_data.append(fli)
traf_arr = Traffic.from_flights(f_data)
p_list = []
results = []
f_data = []
end_time = traf_arr.end_time
# Now we process the results
for flight in traf_arr:
if (flight.stop + timedelta(minutes=5) < end_time):
p_list.append(
pool.apply_async(p_f_appr,
args=(
flight,
VABB.rwy_list,
outdir_np,
outdir_kml,
)))
else:
f_data.append(flight)
for p in p_list:
t_res = p.get()
results.append(t_res)
# Below is for either processing method
for ddi in results:
if (ddi is None):
continue
rwy = ddi['rwy']
for i in range(0, len(nums)):
if (rwy == VABB.rwy_list[i].name):
OSP.add_data_single_rwy(nums[i], ddi)
counters = counters + n_files_proc
if (counters > plot_lim):
for i in range(0, len(nums)):
OSP.save_plot_single_rwy(nums[i], VABB.rwy_list[i], main_count,
outdir_pl)
figs, nums = OSP.setup_figs_rwy_vabb()
counters = 0
for i in range(0, len(nums)):
OSP.save_plot_single_rwy(nums[i], VABB.rwy_list[i], main_count,
outdir_pl)
from traffic.data import opensky
from traffic.core import Traffic
import pandas as pd
"""
t_dcm = opensky.history(
"2020-02-26 00:00",
"2020-02-26 23:59",
callsign="DCM%",
)
"""
t_dcm = Traffic.from_file("test.parquet")
#t_dcm = pd.read_parquet("test.parquet")
print(len(t_dcm))
unique_callsigns = set()
for f in t_dcm:
unique_callsigns.add(f.callsign)
print(len(unique_callsigns))
by_callsign = {}
for c in t_dcm:
if c.callsign in by_callsign:
by_callsign[c.callsign].append(c)
else:
by_callsign[c.callsign] = [c]
for c in by_callsign:
if len(by_callsign[c]) == 1:
def main(start_n, fidder, do_write):
"""The main code for detecting go-arounds.
Arguments:
start_n -- The index of the first file to read
fidder -- the id of an open file to write log information into
do_write -- boolean flag specifying whether to output data to textfile
"""
# Total number of aircraft seen
tot_n_ac = 0
# Of which go-arounds
tot_n_ga = 0
top_dir = '/gf2/eodg/SRP002_PROUD_ADSBREP/GO_AROUNDS/VABB/'
# indir stores the opensky data
indir = top_dir + 'INDATA/'
# odir_nm stores plots for normal landings
odir_pl_nm = top_dir + 'OUT_PLOT/NORM/'
# odir_ga stores plots for detected go-arounds
odir_pl_ga = top_dir + 'OUT_PLOT/PSGA/'
# odir_nm stores plots for normal landings
odir_da_nm = top_dir + 'OUT_DATA/NORM/'
# odir_ga stores plots for detected go-arounds
odir_da_ga = top_dir + 'OUT_DATA/PSGA/'
odirs = [odir_pl_nm, odir_pl_ga, odir_da_nm, odir_da_ga]
# Output filenames for saving data about go-arounds
out_file_ga = 'GA_MET_NEW.csv'
# Output filenames for saving data about non-go-arounds
out_file_noga = 'GA_NOGA_NEW.csv'
t_frmt = "%Y/%m/%d %H:%M:%S"
# File to save met info for g/a flights
if (do_write):
metfid = open(out_file_ga, 'w')
nogfid = open(out_file_noga, 'w')
metfid.write('ICAO24, Callsign, GA_Time, L_Time, Runway, Heading, Alt, Lat, \
Lon, gapt, rocvar, hdgvar, latvar, lonvar, gspvar, \
Temp, Dewp, Wind_Spd, Wind_Gust, Wind_Dir,Cld_Base,\
CB, Vis, Pressure\n')
nogfid.write('ICAO24, Callsign, GA_Time, L_Time, Runway, gapt, rocvar, \
hdgvar, latvar, lonvar, gspvar, \
Temp, Dewp, Wind_Spd, Wind_Gust, Wind_Dir,Cld_Base,\
CB, Vis, Pressure\n')
files = []
files = glob.glob(indir+'**.pkl')
files.sort()
fli_len = len(files)
colormap = {'GND': 'black', 'CL': 'green', 'CR': 'blue',
'DE': 'orange', 'LVL': 'purple', 'NA': 'red'}
# Number of files to open in one go
n_files_proc = 55
pool_proc = 100
f_data = []
pool = mp.Pool(processes=pool_proc)
for main_count in range(start_n, fli_len, n_files_proc):
print("Processing batch starting with "
+ str(main_count + 1).zfill(5) + " of "
+ str(fli_len).zfill(5))
fidder.write("Processing batch starting with "
+ str(main_count + 1).zfill(5) + " of "
+ str(fli_len).zfill(5) + '\n')
p_list = []
# First we load several files at once
for j in range(0, n_files_proc):
if (main_count+j < fli_len):
p_list.append(pool.apply_async(OSF.get_flight,
args=(files[main_count+j],)))
for p in p_list:
t_res = p.get()
for fli in t_res:
f_data.append(fli)
if(len(f_data) < 1):
continue
traf_arr = Traffic.from_flights(f_data)
p_list = []
f_data = []
# Extend array end time if there's only one flight, else processing
# will fail as we check to ensure latest flight is > 5 mins from end
if (len(traf_arr) == 1):
end_time = traf_arr.end_time + timedelta(minutes=10)
print("Extending timespan due to single aircraft")
else:
end_time = traf_arr.end_time
# Now we process the results
for flight in traf_arr:
if (flight.stop + timedelta(minutes=5) < end_time):
p_list.append(pool.apply_async(OSF.proc_fl,
args=(flight,
VABB.rwy_list,
odirs,
colormap,
True,
False,)))
else:
f_data.append(flight)
for p in p_list:
t_res = p.get()
if (t_res != -1):
tot_n_ac += 1
# If there's a go-around, this will be True
if (t_res[0]):
if (do_write):
metfid.write(t_res[1] + ',' + t_res[2] + ',')
metfid.write(t_res[3].strftime(t_frmt) + ',')
metfid.write(t_res[4].strftime(t_frmt) + ',')
metfid.write(t_res[5] + ',')
if (t_res[6] < 0):
t_res[6] = 360 + t_res[6]
metfid.write(str(t_res[6]) + ',')
metfid.write(str(t_res[7]) + ',')
metfid.write(str(t_res[8]) + ',')
metfid.write(str(t_res[9]) + ',')
metfid.write(str(t_res[10]) + ',')
metfid.write(str(t_res[11]) + ',')
metfid.write(str(t_res[12]) + ',')
metfid.write(str(t_res[13]) + ',')
metfid.write(str(t_res[14]) + ',')
metfid.write(str(t_res[15]) + ',')
metfid.write(str(t_res[16].temp) + ',')
metfid.write(str(t_res[16].dewp) + ',')
metfid.write(str(t_res[16].w_s) + ',')
metfid.write(str(t_res[16].w_g) + ',')
metfid.write(str(t_res[16].w_d) + ',')
metfid.write(str(t_res[16].cld) + ',')
if t_res[16].cb:
metfid.write('1,')
else:
metfid.write('0,')
metfid.write(str(t_res[16].vis) + ',')
metfid.write(str(t_res[16].pres) + '\n')
tot_n_ga += 1
# Otherwise, do the following (doesn't save g/a location etc).
else:
if (do_write):
nogfid.write(t_res[1] + ',' + t_res[2] + ',')
nogfid.write(t_res[3].strftime(t_frmt) + ',')
nogfid.write(t_res[4].strftime(t_frmt) + ',')
nogfid.write(t_res[5] + ',')
nogfid.write(str(t_res[10]) + ',')
nogfid.write(str(t_res[11]) + ',')
nogfid.write(str(t_res[12]) + ',')
nogfid.write(str(t_res[13]) + ',')
nogfid.write(str(t_res[14]) + ',')
nogfid.write(str(t_res[15]) + ',')
try:
outstr = str(t_res[16].temp) + ','
outstr = outstr + str(t_res[16].dewp) + ','
outstr = outstr + str(t_res[16].w_s) + ','
outstr = outstr + str(t_res[16].w_g) + ','
outstr = outstr + str(t_res[16].w_d) + ','
outstr = outstr + str(t_res[16].cld) + ','
if t_res[16].cb:
outstr = outstr + '1,'
else:
outstr = outstr + '0,'
outstr = outstr + str(t_res[16].vis) + ','
outstr = outstr + str(t_res[16].pres)
except Exception as e:
print("No METAR data for this flight")
outstr = ''
nogfid.write(outstr + '\n')
print("\t-\tHave processed " + str(tot_n_ac) +
" aircraft. Have seen " + str(tot_n_ga) + " go-arounds.")
if (do_write):
metfid.close()
nogfid.close()
def main():
indir = '/gf2/eodg/SRP001_PROUD_TURBREP/DULLES_GOAROUND/IN/'
outdir = '/gf2/eodg/SRP001_PROUD_TURBREP/DULLES_GOAROUND/OUT/'
files = glob.glob(indir + '*2018*.pkl')
files.sort()
do_plots = False
do_kml = False
first = True
counters = 0
main_types = OSF.make_typlist()
fli_len = len(files)
# Number of files to open in one go
n_files_proc = 24
plot_lim = 200
pool_proc = 50
f_data = []
pool = mp.Pool(processes=pool_proc)
for typr in main_types:
print("Now processing:", typr[0])
kml = simplekml.Kml(open=1)
rwy01l = kml.newdocument(name='RWY01L')
rwy01c = kml.newdocument(name='RWY01C')
rwy01r = kml.newdocument(name='RWY01R')
rwy19l = kml.newdocument(name='RWY19L')
rwy19c = kml.newdocument(name='RWY19C')
rwy19r = kml.newdocument(name='RWY19R')
types_list = typr[1]
outf_format = typr[2]
for i in range(0, fli_len, n_files_proc):
print("Processing batch starting with " + str(i + 1).zfill(5) +
" of " + str(fli_len).zfill(5))
p_list = []
# First we load several files at once
for j in range(0, n_files_proc):
if (i + j < fli_len):
p_list.append(
pool.apply_async(OSF.get_flight,
args=(files[i + j], )))
for p in p_list:
t_res = p.get()
for fli in t_res:
f_data.append(fli)
traf_arr = Traffic.from_flights(f_data)
p_list = []
results = []
f_data = []
end_time = traf_arr.end_time
# Now we process the results
for flight in traf_arr:
if (flight.stop + timedelta(minutes=5) < end_time):
p_list.append(
pool.apply_async(p_f,
args=(
flight,
runway_list,
types_list,
do_plots,
do_kml,
)))
else:
f_data.append(flight)
for p in p_list:
t_res = p.get()
results.append(t_res)
# Below is for either processing method
for ddi in results:
if (ddi == None):
continue
if (ddi['ga'] != True):
continue
if (ddi['rwy'] == "01L"):
ls = rwy01l.newlinestring()
elif (ddi['rwy'] == "01C"):
ls = rwy01c.newlinestring()
elif (ddi['rwy'] == "01R"):
ls = rwy01r.newlinestring()
elif (ddi['rwy'] == "19L"):
ls = rwy19l.newlinestring()
elif (ddi['rwy'] == "19C"):
ls = rwy19c.newlinestring()
elif (ddi['rwy'] == "19R"):
ls = rwy19r.newlinestring()
else:
print("Unknown runway!", ddi['rwy'])
ls = kml.newlinestring()
ls.name = ddi['callsign']
ls.description = ddi['time'].strftime("%Y%m%d%H%M%S") + "\n" +\
ddi['icao24'] + '\n' +\
str(ddi['bdist'])
ls.style.linestyle.color = simplekml.Color.red
ls.altitudemode = simplekml.AltitudeMode.relativetoground
ls.extrude = 0
for irng in range(0, len(ddi['lats'])):
ls.coords.addcoordinates([
(ddi['lons'][irng], ddi['lats'][irng],
ddi['galt'][irng] / 3.28084)
])
counters = counters + n_files_proc
if (counters > plot_lim):
print("SAVING KML")
try:
kml.save(outdir + outf_format + 'TRAJ.kml')
except:
print("ERROR SAVING KML:",
outdir + outf_format + 'TRAJ.kml')
counters = 0
kml.save(outdir + outf_format + 'TRAJ.kml')
from traffic.data import opensky
from traffic.core import Traffic, Flight
import pandas as pd
start = "2020-02-26"
end = "2020-02-26"
filename = start + "_" + end + "_DCM_recordings.parquet"
if __name__ == "__main__":
t_dcm = Traffic.from_file(filename)
print(len(t_dcm.callsigns))
flight = t_dcm[8]
typer = 'loiter'
col = 'cyan'
plt.plot(lons, lats, color=col, linewidth=0.3)
return
outf1 = '/home/proud/Desktop/test.svg'
outf2 = '/home/proud/Desktop/test.png'
count = 0
aclist = {}
for inf in files:
print(inf)
# try:
indata = Traffic.from_file(inf)
for flight in indata:
try:
acft = aircraft[flight.icao24]
typ = acft.typecode.values[0]
if (typ in aclist):
aclist[typ][0] = aclist[typ][0] + 1
else:
aclist[typ] = [1, acft.model.values[0]]
except KeyboardInterrupt:
continue
except Exception as e:
print("OH", e)
continue
continue
proc_flight(flight)
def main():
indir = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/DULLES/IN/'
outdir = '/gf2/eodg/SRP001_PROUD_TURBREP/GO_AROUNDS/DULLES/OUT/'
dater = datetime(2018,1,1,0,0)
main_types = OSF.make_typlist()
# Number of files to open in one go
n_files_proc = 24
plot_lim = 200
pool_proc = 50
n_01l = 0
n_01c = 0
n_01r = 0
n_19l = 0
n_19c = 0
n_19r = 0
f_data = []
pool = mp.Pool(processes=pool_proc)
while True:
files = glob.glob(indir+'*'+dater.strftime("%Y%m%d")+'*.pkl')
files.sort()
fli_len = len(files)
print("Now processing:",dater)
for typr in main_types:
figs, nums = OSF.setup_figs()
types_list = typr[1]
outf_format = typr[2]
for i in range(0, fli_len, n_files_proc):
p_list = []
# First we load several files at once
for j in range(0, n_files_proc):
if (i+j < fli_len):
p_list.append(pool.apply_async(OSF.get_flight, args=(files[i+j],)))
for p in p_list:
t_res = p.get()
for fli in t_res:
f_data.append(fli)
traf_arr = Traffic.from_flights(f_data)
p_list = []
results = []
f_data = []
end_time = traf_arr.end_time
# Now we process the results
for flight in traf_arr:
if (flight.stop + timedelta(minutes=5) < end_time):
p_list.append(pool.apply_async(p_f, args=(flight,
runway_list,
types_list,)))
else:
f_data.append(flight)
for p in p_list:
t_res = p.get()
results.append(t_res)
# Below is for either processing method
for ddi in results:
if (ddi == None):
continue
if (ddi['rwy'] == '01L'):
n_01l = n_01l + 1
if (ddi['rwy'] == '01C'):
n_01c = n_01c + 1
if (ddi['rwy'] == '01R'):
n_01r = n_01r + 1
if (ddi['rwy'] == '19L'):
n_19l = n_19l + 1
if (ddi['rwy'] == '19C'):
n_19c = n_19c + 1
if (ddi['rwy'] == '19R'):
n_19r = n_19r + 1
OSF.add_data_to_fig(ddi, nums)
OSF.plot_figs(nums, outdir, dater, outf_format)
dater = dater + timedelta(days = 1)
print(' - ',n_01l,n_01c,n_01r,n_19l,n_19c,n_19r)
# quit()
if (dater.year > 2018):
break
def plot_latent_and_trajs_outliers(
t,
lat,
outliers,
savefig,
nb_top_outliers=10,
airport="LSZH",
runway=None,
plot_callsigns=True,
re_or_score="re",
):
outliers = outliers.query("initial_flow != 'N/A'")
if outliers is None:
return
if runway is not None:
subset = t.query(f"runway == '{runway}' and initial_flow != 'N/A'")
else:
subset = t.query("initial_flow != 'N/A'")
df = pd.DataFrame.from_records([{
"flight_id": id_,
"x": x,
"y": y
} for (id_, x, y) in zip(
list(f.flight_id for f in t),
lat[:, 0],
lat[:, 1],
)])
top_outliers = (
outliers.query("initial_flow != 'N/A'").groupby("flight_id").agg({
re_or_score:
np.mean
}).sort_values(
re_or_score,
ascending=(re_or_score == "score")).head(nb_top_outliers))
print("\n\ntop outliers\n", top_outliers)
cols = ["flight_id", "simple", "initial_flow", "cluster"]
subset = df.merge(subset.data[cols].drop_duplicates())
subset_o = df.merge(outliers.data, on="flight_id")
subset_o = top_outliers.merge(subset_o, on="flight_id")
with plt.style.context("traffic"):
text_style = dict(
verticalalignment="top",
horizontalalignment="right",
fontname="Ubuntu",
fontsize=18,
bbox=dict(facecolor="white", alpha=0.6, boxstyle="round"),
)
colors = [
"#1f77b4",
"#ff7f0e",
"#2ca02c",
"#d62728",
"#9467bd",
"#8c564b",
"#e377c2",
"#7f7f7f",
"#bcbd22",
"#17becf",
]
fig = plt.figure(figsize=(30, 15))
ax = fig.add_subplot(121)
m = fig.add_subplot(122, projection=EuroPP())
m.add_feature(
countries(edgecolor="white",
facecolor="#d9dadb",
alpha=1,
linewidth=2,
zorder=-2))
m.outline_patch.set_visible(False)
m.background_patch.set_visible(False)
airports[airport].point.plot(
m,
shift=dict(units="dots", x=-15, y=-15),
marker=atc_tower,
s=300,
zorder=5,
text_kw={**text_style},
)
subset.plot.scatter(
x="x",
y="y",
ax=ax,
color="#aaaaaa",
alpha=0.4,
)
for (flow, d), color in zip(subset_o.groupby("cluster"), colors):
d.plot.scatter(x="x",
y="y",
ax=ax,
color=color,
label=flow,
alpha=0.4,
s=100)
for f in Traffic(d):
f.plot(
m,
linewidth=3,
label=f"{f.callsign}, {f.stop:%b %d}",
color=color,
)
if plot_callsigns:
point = subset_o.query(
f'flight_id == "{f.flight_id}"').iloc[0]
ax.text(
point.x + 0.5,
point.y + 0.5,
f"{f.callsign}, {f.stop:%b %d}",
**{
**text_style,
**dict(horizontalalignment="left", fontsize=15),
},
)
# ax.legend(prop=dict(family="Ubuntu", size=18))
ax.grid(linestyle="solid", alpha=0.5, zorder=-2)
ax.set_xlabel("1st component on latent space", labelpad=10)
ax.set_ylabel("2nd component on latent space", labelpad=10)
fig.savefig(savefig)
def loadFlights(start_day, end_day):
filename = getFlightsFilename(start_day, end_day)
return Traffic.from_file(filename)