def getter(init_time, bounds, timer, anam):
'''
This function downloads the data, which is done in
one hour segments. Each hour is downloaded separately
using multiprocessing for efficiency.
'''
try:
times = init_time + timedelta(hours=timer)
dtst = times.strftime("%Y%m%d%H%M")
outf = outdir + 'OS_' + dtst + '_' + anam + '.pkl'
# Check if the file has already been retrieved
if (os.path.exists(outf)):
print("Already retrieved", outf)
return
# Otherwise use 'traffic' to download
flights = opensky.history(start=times,
stop=times + timedelta(hours=1),
bounds=bounds,
other_params=" and time-lastcontact<=15 ")
flights.to_pickle(outf)
except Exception as e:
print("There is a problem with this date/time combination:", e, times)
return
def getter(init_time, bounds, timer, anam, outdir, subdir):
"""Get data from the opensky server.
This is done in one hour segments. Each hour is downloaded
separately using multiprocessing for efficiency.
"""
try:
times = init_time + timedelta(hours=timer)
dtst = times.strftime("%Y%m%d%H%M")
# Check if we're saving into YYYMMDD subdirectories,
# create directories if necessary
if subdir is True:
odir = outdir + times.strftime("%Y%m%d") + '/'
else:
odir = outdir
pathlib.Path(odir).mkdir(parents=True, exist_ok=True)
outf = odir + 'OS_' + dtst + '_' + anam + '.pkl'
# Check if the file has already been retrieved
if (os.path.exists(outf)):
print("Already retrieved", outf)
return
# Otherwise use 'traffic' to download
flights = opensky.history(start=times,
stop=times + timedelta(hours=1),
bounds=bounds,
other_params=" and time-lastcontact<=15 ")
flights.to_pickle(outf)
except Exception as e:
print("There is a problem with this date/time combination:", e, times)
return
def query(start_day, end_day):
t_dcm = opensky.history(
start_day+" 00:00",
end_day+" 23:59",
callsign=callsign+'%',
#airport=airport,
)
return t_dcm
def queryFlights(start_day, end_day):
t_dcm = opensky.history(
start_day + " 00:00",
end_day + " 23:59",
callsign="DCM%",
)
filename = getFlightsFilename(start_day, end_day)
t_dcm.to_parquet(filename, allow_truncated_timestamps=True)
return t_dcm
def test_history():
t_aib: Optional[Traffic] = cast(
Optional[Traffic],
opensky.history(
"2019-11-01 09:00",
"2019-11-01 12:00",
departure_airport="LFBO",
arrival_airport="LFBO",
callsign="AIB%",
),
)
assert t_aib is not None
flight = t_aib["AIB04FI"]
assert flight is not None
assert flight.icao24 == "388dfb"
t_tma: Optional[Traffic] = cast(
Optional[Traffic],
opensky.history(
"2019-11-11 10:00",
"2019-11-11 10:10",
bounds=lfbo_tma,
serials=1433801924,
),
)
assert t_tma is not None
assert len(t_tma) == 33
df = opensky.extended(
"2019-11-11 10:00", "2019-11-11 10:10", bounds=lfbo_tma
)
t_decoded = t_tma.filter().query_ehs(df).eval(desc="", max_workers=4)
assert len(t_decoded) == len(t_tma)
def get_data(before_time: timelike, after_time: timelike,
airport: Airport) -> Optional[Traffic]:
logging.info(f"Getting callsigns arriving at {airport.name}")
callsigns = opensky.at_airport(before_time, after_time, airport)
if callsigns is None or callsigns.shape[0] == 0:
return None
callsigns_list = callsigns.loc[callsigns.callsign.notnull(), 'callsign']
logging.info(f"{len(callsigns_list)} callsigns found")
logging.info(f"Fetching data from OpenSky database")
flights = opensky.history(before_time, after_time,
callsign=callsigns_list,
progressbar=tqdm)
return flights
def test_timebuffer() -> None:
f = cast(
Flight,
opensky.history(
"2021-09-06 14:00",
"2021-09-07 16:00",
callsign="SAS44A",
airport="ESSA",
time_buffer="1H",
return_flight=True,
),
)
assert f is not None
g = f.resample("1s").cumulative_distance().query("compute_gs > 5")
assert g is not None
h = g.on_taxiway("ESSA").max(key="stop")
assert h is not None
assert h.taxiway_max == "Z"
def get_history_data(start_datetime,
end_datetime,
interval_datetime=datetime.timedelta(hours=1),
callsign=None,
icao24=None,
departure_airport=None,
arrival_airport=None,
onground=False,
min_ft=33000,
time_interval=datetime.timedelta(minutes=1)):
start_datetime_temp = deepcopy(start_datetime)
end_datetime_temp = start_datetime_temp + interval_datetime
list_out = []
pbar = tqdm(total=(end_datetime - start_datetime) // interval_datetime)
while end_datetime_temp <= end_datetime:
pbar.update(1)
start_str = start_datetime_temp.strftime('%Y-%m-%d %H:%M')
end_str = end_datetime_temp.strftime('%Y-%m-%d %H:%M')
flight = opensky.history(start=start_str,
stop=end_str,
callsign=callsign,
icao24=icao24,
departure_airport=departure_airport,
arrival_airport=arrival_airport)
try:
df_temp = flight.data
df_temp = remove_row_flight_df(df_temp,
onground=onground,
min_ft=min_ft,
time_interval=time_interval,
start_str=start_str,
end_str=end_str)
list_out.append(df_temp)
except AttributeError:
print('nodata {0}-{1}'.format(start_str, end_str))
start_datetime_temp = start_datetime_temp + interval_datetime
end_datetime_temp = start_datetime_temp + interval_datetime
pbar.close()
start_str = start_datetime_temp.strftime('%Y-%m-%d %H:%M')
end_str = end_datetime.strftime('%Y-%m-%d %H:%M')
flight = opensky.history(start=start_str,
stop=end_str,
callsign=callsign,
icao24=icao24,
departure_airport=departure_airport,
arrival_airport=arrival_airport)
try:
df_temp = flight.data
df_temp = remove_row_flight_df(
df_temp,
onground=onground,
min_ft=min_ft,
time_interval=datetime.timedelta(minutes=1),
start_str=start_str,
end_str=end_str)
list_out.append(df_temp)
except AttributeError:
print('nodata {0}-{1}'.format(start_str, end_str))
return pd.concat(list_out)
def get_df_one_unit(self,
target_date,
callsign=None,
icao24=None,
departure_airport=None,
arrival_airport=None,
calc_interval_datetime=datetime.timedelta(hours=1),
save_local=False,
dir_save=None,
pickle=True,
tqdm_count=True):
if self.file_batch_unit == 'daily':
filename_head = target_date.strftime('%Y%m%d') + '_' + 'D'
start_datetime = datetime.datetime.combine(
target_date, datetime.datetime.min.time())
stop_datetime = start_datetime + datetime.timedelta(days=1)
# start_str = start_datetime.strftime('%Y-%m-%d %H:%M')
# stop_str = stop_datetime.strftime('%Y-%m-%d %H:%M')
# start_str = target_date.strftime('%Y-%m-%d') + ' 00:00'
# stop_str = (target_date + datetime.timedelta(days=1)).strftime('%Y-%m-%d') + ' 00:00'
elif self.file_batch_unit == 'monthly':
filename_head = target_date.strftime('%Y%m') + '_' + 'M'
target_date = datetime.date(year=target_date.year,
month=target_date.month,
day=1)
start_datetime = datetime.datetime.combine(
target_date, datetime.datetime.min.time())
stop_datetime = datetime.datetime.combine(
(target_date + relativedelta(months=+1)),
datetime.datetime.min.time())
# start_str = target_date.strftime('%Y-%m') + '-01 00:00'
# stop_str = (target_date + relativedelta(months=+1)).strftime('%Y-%m') + '-01 00:00'
else:
print('check arg file_batch_unit')
return
if callsign is not None:
filename_head = filename_head + '_' + 'L'
else:
filename_head = filename_head + '_' + 'A'
if icao24 is not None:
filename_head = filename_head + '_' + 'L'
else:
filename_head = filename_head + '_' + 'A'
if departure_airport is not None:
filename_head = filename_head + '_' + 'dpt-' + departure_airport
else:
filename_head = filename_head + '_' + 'dpt-all'
if arrival_airport is not None:
filename_head = filename_head + '_' + 'arr-' + arrival_airport
else:
filename_head = filename_head + '_' + 'arr-all'
if calc_interval_datetime is None:
start_str = start_datetime.strftime('%Y-%m-%d %H:%M')
stop_str = stop_datetime.strftime('%Y-%m-%d %H:%M')
flight = opensky.history(start=start_str,
stop=stop_str,
callsign=callsign,
icao24=icao24,
departure_airport=departure_airport,
arrival_airport=arrival_airport,
cached=False)
try:
df_out = flight.data
df_out = self._remove_row_flight_df(df_out,
start_str=start_str,
end_str=stop_str)
except AttributeError:
print('nodata {0}-{1}'.format(start_str, stop_str))
return None
else:
start_datetime_temp = deepcopy(start_datetime)
stop_datetime_temp = start_datetime_temp + calc_interval_datetime
list_df_out = []
if tqdm_count:
pbar = tqdm(total=(stop_datetime - start_datetime) //
calc_interval_datetime)
while stop_datetime_temp <= stop_datetime:
start_str = start_datetime_temp.strftime('%Y-%m-%d %H:%M')
stop_str = stop_datetime_temp.strftime('%Y-%m-%d %H:%M')
if tqdm_count:
pbar.update(1)
pbar.set_description(start_str)
flight = opensky.history(start=start_str,
stop=stop_str,
callsign=callsign,
icao24=icao24,
departure_airport=departure_airport,
arrival_airport=arrival_airport,
cached=False)
try:
df_temp = flight.data
df_temp = self._remove_row_flight_df(df_temp,
start_str=start_str,
end_str=stop_str)
list_df_out.append(df_temp)
except AttributeError:
print('nodata {0}-{1}'.format(start_str, stop_str))
start_datetime_temp = start_datetime_temp + calc_interval_datetime
stop_datetime_temp = start_datetime_temp + calc_interval_datetime
if tqdm_count:
pbar.close()
start_str = start_datetime_temp.strftime('%Y-%m-%d %H:%M')
stop_str = stop_datetime.strftime('%Y-%m-%d %H:%M')
flight = opensky.history(start=start_str,
stop=stop_str,
callsign=callsign,
icao24=icao24,
departure_airport=departure_airport,
arrival_airport=arrival_airport)
try:
df_temp = flight.data
df_temp = self._remove_row_flight_df(df_temp,
start_str=start_str,
end_str=stop_str)
list_df_out.append(df_temp)
except AttributeError:
print('nodata {0}-{1}'.format(start_str, stop_str))
df_out = pd.concat(list_df_out)
if save_local:
if not os.path.exists(dir_save):
os.makedirs(dir_save)
path_dest = os.path.join(dir_save, filename_head)
if pickle:
path_dest = path_dest + '.pkl'
df_out.to_pickle(path=path_dest)
else:
path_dest = path_dest + '.csv'
df_out.to_csv(path_dest, index=False)
return df_out, path_dest
return df_out, None
#!/usr/bin/env python
# coding: utf-8
from traffic.data import opensky
#pooling air traffic data
opensky.history(
"2020-01-01",
"2020-05-31",
airport="LOWS",
bounds=(12.2, 47.3, 13.7, 48.2)).to_csv(
r'C:\Users\Dell\Documents\opensky\airtraff_jan_may2020.csv')
#pooling flights info
opensky.flightlist("2020-01-01", "2020-05-31", airport='LOWS').to_csv(
r'C:\Users\Dell\Documents\opensky\flightlist_jan_may2020.csv')
def test_complex_queries() -> None:
error_msg = "airport may not be set if arrival_airport is set"
with pytest.raises(RuntimeError, match=error_msg):
_ = opensky.history(
start="2021-08-24 00:00",
stop="2021-08-24 01:00",
airport="ESSA",
arrival_airport="EGLL",
)
# test that `limit` generate correct query
t2 = opensky.history(
start="2021-08-24 00:00",
stop="2021-08-24 01:00",
airport="ESSA",
limit=3,
)
assert t2 is not None
assert len(t2.data) == 3
t2_1 = opensky.history(
start="2021-08-24 09:00",
stop="2021-08-24 09:10",
airport="ESSA",
)
assert t2_1 is not None
assert len(t2_1) == 23
t3 = opensky.history(
start="2021-08-24 09:00",
stop="2021-08-24 09:10",
arrival_airport="ESSA",
)
assert t3 is not None
assert len(t3) == 13
t4 = opensky.history(
start="2021-08-24 11:32",
stop="2021-08-24 11:42",
departure_airport="ESSA",
arrival_airport="EGLL",
)
assert t4 is not None
assert len(t4) == 1
flight = cast(Traffic, t4)["BAW777C"]
assert flight is not None
assert flight.icao24 == "400936"
with pytest.raises(RuntimeError, match=error_msg):
_ = opensky.history(
start="2021-08-24 00:00",
stop="2021-08-24 01:00",
airport="ESSA",
arrival_airport="EGLL",
limit=3,
)
t6 = opensky.history(
start="2021-08-24 00:00",
stop="2021-08-24 00:10",
arrival_airport="ESSA",
serials=-1408232560,
)
assert t6 is not None
assert len(t6) == 1
flight = cast(Traffic, t6)[0]
assert flight is not None
assert flight.callsign == "SAS6906"
assert flight.icao24 == "4ca863"
t7 = opensky.history(
start="2021-08-24 00:00",
stop="2021-08-24 00:10",
serials=(-1408232560, -1408232534),
)
assert t7 is not None
assert len(t7) == 12
t8 = opensky.history(
start="2021-08-24 09:00",
stop="2021-08-24 09:10",
departure_airport="ESSA",
serials=(-1408232560, -1408232534),
)
assert t8 is not None
assert len(t8) == 1
flight = cast(Traffic, t8)[0]
assert flight is not None
assert flight.callsign == "LOT454"
assert flight.icao24 == "489789"
t9 = opensky.history(
start="2021-08-24 09:00",
stop="2021-08-24 09:10",
bounds=[17.8936, 59.6118, 17.9894, 59.6716],
serials=(-1408232560, -1408232534),
)
assert t9 is not None
assert len(t9) == 9
flight = cast(Traffic, t9)["SAS1136"]
assert flight is not None
assert flight.icao24 == "51110b"
tA = opensky.history(
start="2021-08-24 09:30",
stop="2021-08-24 09:40",
departure_airport="ESSA",
bounds=[17.8936, 59.6118, 17.9894, 59.6716],
serials=(-1408232560, -1408232534),
)
assert tA is not None
assert len(tA) == 1
flight = cast(Traffic, tA)[0]
assert flight is not None
assert flight.callsign == "THY5HT"
assert flight.icao24 == "4bb1c5"
tB = opensky.history(
start="2021-08-24 09:45",
stop="2021-08-24 09:50",
departure_airport="ESSA",
count=True,
bounds=[17.8936, 59.6118, 17.9894, 59.6716],
serials=(-1408232560, -1408232534),
)
assert tB is not None
assert len(tB) == 1
flight = cast(Traffic, tB)[0]
assert flight is not None
assert flight.callsign == "SAS69E"
assert flight.icao24 == "4ac9e5"
tC = opensky.history(
start="2021-08-24 09:45",
stop="2021-08-24 09:50",
departure_airport="ESSA",
count=True,
bounds=[17.8936, 59.6118, 17.9894, 59.6716],
)
assert tC is not None
assert len(tC) == 1
flight = cast(Traffic, tC)[0]
assert flight is not None
assert flight.callsign == "SAS69E"
assert flight.icao24 == "4ac9e5"
tD = opensky.history(
start="2021-08-24 09:45",
stop="2021-08-24 09:50",
bounds=[17.8936, 59.6118, 17.9894, 59.6716],
)
assert tD is not None
assert len(tD) == 9
from traffic.data import opensky
airport = 'KSDM'
start_day = '2019-10-01'
end_day = '2019-10-31'
t_dcm = opensky.history(
start_day+" 00:00",
end_day+" 23:59",
airport=airport,
)
filename = '../data/recordings/KSDM_oct2019_recordings.parquet'
t_dcm.to_parquet(filename,allow_truncated_timestamps=True)
"""
print(len(t_dcm))
icaos = set()
for f in t_dcm:
icaos.add(f.icao24)
icaos = list(icaos)
print(len(icaos))
string = ""
for i in icaos:
string += i + '\n'
f = open('../data/recordings/KSDM_flights.txt', 'w')
f.write(string)
f.close()
"""
from traffic.data import opensky
t_dcm = opensky.history(
"2020-02-24 00:00",
"2020-02-24 23:59",
#departure_airport="KLAX",
#arrival_airport="KLAX",
callsign="DCM%",
)
cs = set()
for f in t_dcm:
cs.add(f.callsign)
print(cs)
print(len(cs))
