def initial_flight(config):
flight = {}
try:
# TODO Use file pattern to filter only active files(having _ file) by joe
list_date_from_flight_info = os.listdir(config['runtime_flight_info'])
print("list_date_from_flight_info:", list_date_from_flight_info)
if list_date_from_flight_info:
last_update_date = sorted(list_date_from_flight_info,
reverse=True)[0]
if not last_update_date.endswith('_'):
print("last_update_date:", last_update_date)
date = datetime.datetime.strptime(str(last_update_date),
'%Y%m%d')
print("date:", date)
filename = config[
'runtime_flight_info'] + "\\" + last_update_date
buffer_size = 2**16
with open(filename) as f:
flight_info_by_date = {
'date': last_update_date,
'flight_info': {}
}
while True:
lines_buffer = f.readlines(buffer_size)
if not lines_buffer:
break
for line in lines_buffer:
data = Util.json_load(line)
flight_info_by_date['flight_info']['ac_id'] = data
flight = flight_info_by_date
else:
print("No Old Data")
except Exception as error:
print("Cannot load old data " + repr(error))
return flight
def process_file(config, aircraft, filename):
lines_seen = Util.CircularBuffer(size=int(config['lines_seen_size']))
print("process_file: " + filename)
# ? need to be change according to config['input_filename_pattern']
temp = re.findall(r'data[\s.]csv[\s.](\d{8})(\d{2})', filename)
date_from_filename = temp[0][0]
hour = temp[0][1]
date_of_flight = datetime.datetime.strptime(date_from_filename, "%Y%m%d")
buffer_size = 2 ** 16
with open(filename) as f:
while True:
lines_buffer = f.readlines(buffer_size)
if not lines_buffer:
break
for line in lines_buffer:
data = process_line(config, aircraft, lines_seen, line, date_of_flight)
if data != {}:
after_process(config, data)
shutil.move(filename, config['runtime_archived_raw_data'])
if hour == config['time_to_make_unchanged_folder']:
target_date_of_flight = date_of_flight - datetime.timedelta(days=1)
path = config["runtime_flight_data_by_date"] + "\\" + target_date_of_flight.strftime("%Y%m%d")
output_path = config["output_flight_data_by_date"] + "\\" + target_date_of_flight.strftime("%Y%m%d")
if os.path.exists(path):
shutil.copytree(path, output_path, symlinks=False, ignore=None)
os.renames(path, path + "_")
def get_all_coordinates(input_filename):
result = []
with open(input_filename, "r") as f:
for line in f:
data = Util.json_load(line)
lat_long = data['position']
result.append(lat_long)
return result
def load_flight_movement(config, date):
# ? should use pattern to get file
f_input_file = config[
'input_flight_movement_by_date'] + "\\FLIGHT_" + date + ".txt"
flight_movement_at_date = {}
with open(f_input_file, "r") as f:
first_line = f.readline()
for line in f:
data = Util.extract_flight_movement(line)
add_flight_movement(flight_movement_at_date, data)
return flight_movement_at_date
def process_file(config, flight_at_date, flight_movement_at_date, filename):
date = flight_at_date['date']
runtime_input_file = config[
'runtime_flight_data_by_date'] + "\\" + date + "_\\" + config[
'cat'] + "\\" + filename
flight_info = process_flight(config, runtime_input_file,
flight_movement_at_date)
ProcessFlightMap.process_plot(config, runtime_input_file, filename, date,
flight_info)
if flight_info:
if filename in flight_at_date['flight_info']:
if config['replace'] == 'true':
flight_at_date['flight_info'][filename] = flight_info
Util.dump_flight_info(config['runtime_flight_info'], date,
flight_at_date['flight_info'])
else:
print('duplicate data ' + filename + ":\n" +
Util.json_dump(flight_info))
else:
flight_at_date['flight_info'][filename] = flight_info
Util.append_filename(config['runtime_flight_info'], date,
Util.json_dump(flight_info))
else:
print('no summary for ' + filename)
archive_flight_data(config, runtime_input_file, date)
def initial_config(current_path, filename):
config_data = configparser.ConfigParser()
config_data.read_file(open(filename, 'r'))
config = {}
config['input_raw_data'] = Util.check_current_path(
config_data.get('Path', 'input_raw_data'), current_path)
config['input_flight_movement_by_date'] = Util.check_current_path(
config_data.get('Path', 'input_flight_movement_by_date'), current_path)
config['runtime_archived_raw_data'] = Util.check_current_path(
config_data.get('Path', 'runtime_archived_raw_data'), current_path)
config['runtime_archived_flight_data'] = Util.check_current_path(
config_data.get('Path', 'runtime_archived_flight_data'), current_path)
config['runtime_flight_data_by_date'] = Util.check_current_path(
config_data.get('Path', 'runtime_flight_data_by_date'), current_path)
config['runtime_flight_info'] = Util.check_current_path(
config_data.get('Path', 'runtime_flight_info'), current_path)
config['output_flight_data_by_date'] = Util.check_current_path(
config_data.get('Path', 'output_flight_data_by_date'), current_path)
config['output_flight_map'] = Util.check_current_path(
config_data.get('Path', 'output_flight_map'), current_path)
config['output_flight_info'] = Util.check_current_path(
config_data.get('Path', 'output_flight_info'), current_path)
config['cat'] = config_data.get('FlightSeparator', 'cat')
config['server_time_type'] = config_data.get('FlightSeparator',
'server_time_type')
config['input_filename_pattern'] = config_data.get(
'FlightSeparator', 'input_filename_pattern')
config['separated_flight_time_gap'] = config_data.get(
'FlightSeparator', 'separated_flight_time_gap')
config['time_to_make_unchanged_folder'] = config_data.get(
'FlightSeparator', 'time_to_make_unchanged_folder')
config['lines_seen_size'] = config_data.get('FlightSeparator',
'lines_seen_size')
config['input_folder_pattern'] = config_data.get('FlightDataProcessor',
'input_folder_pattern')
config['replace'] = config_data.get('FlightDataProcessor', 'replace')
return config
def process_flight(config, input_file, flight_movement_at_date):
flight_info = {}
a = open(input_file, 'r')
line1 = a.readlines()
first_line = line1[0]
for line in line1:
last_line = line
a.close()
# last_line = first_line[-1]
with open(input_file, "r") as f:
for line in f:
# last_line = line[-1]
data = Util.json_load(line)
process_by_line(config, flight_info, data)
ProcessFlightTime.process_final(config, flight_info, first_line,
last_line)
process_final(config, flight_info, flight_movement_at_date)
return flight_info
def initial_aircraft(config):
aircraft = {}
try:
list_date_from_flight_data_by_date = os.listdir(config['runtime_flight_data_by_date'])
print("list_date_from_flight_data_by_date:", list_date_from_flight_data_by_date)
if list_date_from_flight_data_by_date:
last_update_date = sorted(list_date_from_flight_data_by_date, reverse=True)[0]
print("last_update_date:", last_update_date)
date = datetime.datetime.strptime(str(last_update_date), '%Y%m%d')
print("date:", date)
last_line_dict = read_last_line_for_each_flight(config['cat'], config['runtime_flight_data_by_date'],
last_update_date)
# print(last_line_dict)
for filename in last_line_dict:
line = last_line_dict[filename]
data = Util.json_load(line)
data_time = data['data_time']
assign_filename(aircraft, filename, data_time)
else:
print("No Old Data")
except Exception as error:
print(traceback.format_exc())
print("Cannot load old data " + repr(error))
return aircraft
import os
import pprint
from Lib import Config
from Lib import Util
Util.test('Hello')
current_path = os.path.dirname(os.path.abspath(__file__))
pp = pprint.PrettyPrinter(indent=4)
# initial config
# all_config = Config.initial_config(current_path, '.\\..\\config.txt')
# print("config:")
# pp.pprint(all_config)
filename = "C:\\Users\\Pipat_P\\Documents\\GitHub\\RunwayOccupancyTime\\Input\\FlightMovementByDate\\FLIGHT_20180321.txt"
with open(filename, "r") as f:
for data in f:
# lastline = f.readlines()[0]
# print(lastline)
flight_movement = {}
elements = data.split(";")
flight_movement["STATUS"] = elements[0]
flight_movement["DOF"] = elements[1]
flight_movement["FlightType"] = elements[2]
flight_movement["REG"] = elements[3]
flight_movement["MessageType"] = elements[4]
flight_movement["CS"] = elements[5]
flight_movement["AcType"] = elements[6]
flight_movement["DEP"] = elements[7]
flight_movement["ETD"] = elements[8]
def after_process(config, data):
date = str(data["assign_date"].strftime("%Y%m%d"))
output_path = config["runtime_flight_data_by_date"] + "\\" + date + "\\" + config['cat']
Util.append_filename(output_path, data['filename'], Util.json_dump(data))
def extract_data(aircraft, date_of_flight, line, separated_flight_time_gap):
elements = line.split('|')
if (elements[1] == 'CAT20') and (elements[2] == '1.1'):
return Util.extract_cat20_v1_1(aircraft, date_of_flight, line, separated_flight_time_gap)
else:
raise ValueError('Cat not support')
def process_final(config, flight_info, first_line, last_line):
first_record = Util.json_load(first_line)
last_record = Util.json_load(last_line)
flight_info["flight_time"] = {"first_time": first_record["data_time"], "last_time": last_record["data_time"]}
from Lib import Util
from Lib import Config
import os
import pprint
# import FlightSeparator
print('hello')
Util.test('world')
# current_path = os.path.dirname(os.path.abspath(__file__))
# pp = pprint.PrettyPrinter(indent=4)
#
# # initial config
# all_config = Config.initial_config(current_path, 'config.txt')
# print("config:")
# pp.pprint(all_config)
#
# cb = Util.CircularBuffer(size=10)
# for i in range(20):
# cb.append(i)
# print("@%s, Average: %s", cb, cb.average)
#
# x = Util.CircularBuffer(size=10)
# for i in range(20):
# x.append(str(i))
# print("@%s ", x)
# if '1' not in x:
# print('1 not in x')
# if '6' not in x:
# print('6 not in x')
#