def process_fr24_flights(date, airports_filename): """ Convert the IATA airport ids in an FR24 flight file to ICAO ids. Also extracts aircraft fleet data from an FR24 flight file. Parameters ---------- date: string The date in ISO8601 format, e.g. 2017-08-16 airports_filename: string The name of a file containing airport codes for IATA to ICAO conversion. """ get_airports(airports_filename, '.') iata_filename, _ = create_convert_fr24_filenames(date) error_code = convert_airport_ids(iata_filename, airports_filename) if error_code: sys.exit(error_code) put_processed(REFINED_FR24, [create_flights_filename(FR24, date)]) error_code = extract_fleet_data(iata_filename) if error_code: sys.exit(error_code) put_processed(PRODUCTS_FLEET, [create_fleet_data_filename(date)])
def match_overnight_flights_on_day( date, max_time_difference=DEFAULT_MAXIMUM_TIME_DELTA): """ Match flights for the given day with flights for the previous day. Parameters ---------- date: string The date in ISO8601 format, e.g. 2017-08-16 """ if is_valid_iso8601_date(date): # get the CPR data from the Google bucket log.info(f'Getting data for date: {date}') match_flights_files = create_match_overnight_flights_input_filenames( date) if not get_processed(REFINED_MERGED_DAILY_CPR_FR24, match_flights_files): log.error('Flights file not found in daily_cpr_fr24 bucket') return errno.ENOENT error_code = match_overnight_flights(match_flights_files, max_time_difference) if error_code: return error_code gc.collect() prev_ids_filename = create_matching_ids_filename(PREV_DAY, date) if not put_processed(REFINED_MERGED_OVERNIGHT_CPR_FR24_IDS, [prev_ids_filename]): log.error('Could not write ids to overnight_cpr_fr24/ids bucket') return errno.EACCES extract_data_input_files = create_extract_overnight_data_input_filenames( date) if not get_processed(REFINED_MERGED_DAILY_CPR_FR24, extract_data_input_files[2:]): log.error( 'Positions or events file not found in daily_cpr_fr24 bucket') return errno.ENOENT error_code = extract_overnight_data(extract_data_input_files) if error_code: return error_code extract_data_output_files = create_extract_overnight_data_output_filenames( date) if not put_processed(REFINED_MERGED_OVERNIGHT_CPR_FR24, extract_data_output_files): log.error('Could not write to overnight_cpr_fr24 bucket') return errno.EACCES else: log.error(f'invalid date: {date}') return errno.EINVAL return 0
def find_trajectory_user_airspace_intersections( trajectory_filename, source=CPR_FR24, logging_msg_count=DEFAULT_LOGGING_COUNT): """ """ if not path_exists(trajectory_filename): source_path = '/'.join([PRODUCTS, TRAJECTORIES, source]) log.debug("Getting trajectory file: %s", trajectory_filename) if not get_processed(source_path, [trajectory_filename]): log.error('Trajectory file not found in %s bucket', source_path) return False log.info("find_user_airspace_intersections for: %s", trajectory_filename) if find_user_airspace_intersections(trajectory_filename, logging_msg_count): return False output_filename = trajectory_filename.replace(TRAJECTORIES, USER_INTERSECTIONS) output_filename = output_filename.replace(JSON_FILE_EXTENSION, CSV_FILE_EXTENSION) intersections_path = '/'.join([PRODUCTS_INTERSECTIONS_USER, source]) return put_processed(intersections_path, [output_filename])
def convert_cpr_file(date): """ Converts a CPR file for the given date into PRU format. Parameters ---------- date: string The date in ISO8601 format, e.g. 2017-08-16 Returns ------- True if succesful, False otherwise. """ get_unprocessed(CPR, date) cpr_filename = '/'.join( [UPLOAD_DIR, CPR, create_original_cpr_filename(date)]) log.debug("converting cpr data from file: %s", cpr_filename) if convert_cpr_data(cpr_filename): return False gc.collect() os.remove(cpr_filename) # delete the uploaded CPR file return put_processed(REFINED_CPR, create_convert_cpr_filenames(date))
def extract_fleet_date(date): """ Extract aircraft fleet data from an FR24 flights file for the fleet database. Parameters ---------- date: string The date in ISO8601 format, e.g. 2017-08-16 Returns ------- True if succesful, False otherwise. """ if extract_fleet_data(create_flights_filename(FR24, date)): # if the normal flights file does not exist yet, try the IATA file iata_filename, _ = create_convert_fr24_filenames(date) if extract_fleet_data(iata_filename): return False gc.collect() fleet_filename = create_fleet_data_filename(date) copied_ok = put_processed(PRODUCTS_FLEET, [fleet_filename]) if copied_ok: new_filepath = '/'.join([FLEET_DATA_DIR, fleet_filename]) os.rename(fleet_filename, new_filepath) # os.remove(fleet_filename) # delete the fleet data file return copied_ok
def convert_airport_codes(date, airports_filename=DEFAULT_AIRPORTS_FILENAME): """ Convert FR24 IATA departure and destination airport codes to ICAO codes. Parameters ---------- date: string The date in ISO8601 format, e.g. 2017-08-16 airports_filename: string The name of a file containing airport codes for IATA to ICAO conversion, default: airports.csv Returns ------- True if succesful, False otherwise. """ iata_filename, _ = create_convert_fr24_filenames(date) if (airports_filename == DEFAULT_AIRPORTS_FILENAME): log.debug("Using the default airports file, %s", "airports.csv") get_airports(airports_filename) airports_file_path = '/'.join([DATA_HOME, AIRPORTS, airports_filename]) if convert_airport_ids(iata_filename, airports_file_path): return False gc.collect() os.remove(iata_filename) # delete the iata flights file return put_processed(REFINED_FR24, [create_flights_filename(FR24, date)])
def convert_fr24_date(date): """ Converts a FR24 ADS-B data for the given date into PRU format. Parameters ---------- date: string The date in ISO8601 format, e.g. 2017-08-16 Returns ------- True if succesful, False otherwise. """ get_unprocessed(FR24, date) filenames = [ '/'.join([UPLOAD_DIR, FR24, file]) for file in create_original_fr24_data_filenames(date) ] if convert_fr24_data(filenames): return False gc.collect() # delete the uploaded FR24 files [os.remove(file_path) for file_path in filenames] return put_processed(REFINED_FR24, create_convert_fr24_filenames(date))
def import_apds_file(apds_filename, stands_filename=DEFAULT_STANDS_FILENAME): """ Import an APDS file into PRU format flights, positions and events files. Parameters ---------- apds_filename: string The name of the APDS file, e.g.: FAC_APDS_FLIGHT_IR691_2017-08-01_2017-09-01.csv.bz2 stands_filename: string The name of the airport stands file, default DEFAULT_STANDS_FILENAME. Note: it won't create a positions file if run with a empty stands_filename. Returns ------- True if succesful, False otherwise. """ # Get the files from the bucket get_apds(apds_filename, '.') get_stands(stands_filename, '.') log.info("Converting APDS data with stands file: %s", stands_filename) if convert_apds_data(apds_filename, stands_filename): return False from_date, to_date = split_dual_date(apds_filename) return put_processed(REFINED_APDS, create_convert_apds_filenames(from_date, to_date))
def clean_overnight_cpr_fr24_positions( date, max_speed=DEFAULT_MAX_SPEED, distance_accuracy=DEFAULT_DISTANCE_ACCURACY): """ Clean merged CPR and FR24 ADS-B data after merging with the 'next day'. Parameters ---------- date: string The date (of the previous day!) in ISO8601 format, e.g. 2017-08-16 max_speed: string The maximum ground speed permitted between adjacent positions [Knots], default: 750 Knots. distance_accuracy: string The maximum distance between positions at the same time [Nautical Miles], default: 0.25 NM. Returns- ------- True if succesful, False otherwise. """ date = iso8601_previous_day(date) raw_filename = create_raw_positions_filename(CPR_FR24, date) # TODO this call is failing, investigate. if clean_position_data(raw_filename, max_speed, distance_accuracy): return False os.remove(raw_filename) # delete the raw positions file # Put the error metrics file first filenames = create_clean_position_data_filenames(CPR_FR24, date) if not put_processed(REFINED_MERGED_OVERNIGHT_CPR_FR24, filenames[:1]): return False if not put_processed(PRODUCTS_ERROR_METRICS_CPR_FR24_OVERNIGHT, filenames[1:]): return False # delete the cleaned file and metrics file [os.remove(file_path) for file_path in filenames] return True
def clean_raw_positions_data(source, date, max_speed=DEFAULT_MAX_SPEED, distance_accuracy=DEFAULT_DISTANCE_ACCURACY): """ Clean the refined CPR positions file for the given date. Parameters ---------- source: string The source of the positions data: CPR, FR24 or CPR_FR24 date: string The date in ISO8601 format, e.g. 2017-08-16 max_speed: string The maximum ground speed permitted between adjacent positions [Knots], default: 750 Knots. distance_accuracy: string The maximum distance between positions at the same time [Nautical Miles], default: 0.25 NM. Returns ------- True if succesful, False otherwise. """ raw_filename = create_raw_positions_filename(source, date) if clean_position_data(raw_filename, max_speed, distance_accuracy): return False os.remove(raw_filename) # delete the raw positions file filenames = create_clean_position_data_filenames(source, date) # Put the error metrics file first so that it can be deleted errors_path = '/'.join([PRODUCTS, ERROR_METRICS, source]) if not put_processed(errors_path, filenames[1:]): return False os.remove(filenames[1]) # delete the error metrics file source_path = '/'.join([REFINED, MERGED, DAILY_CPR_FR24]) \ if (source == CPR_FR24) else '/'.join([REFINED, source]) return put_processed(source_path, filenames[:1])
def merge_apds_trajectories_on_day(from_date, to_date, date): """ Merge refined APDS data with CPR and FR24 ADS-B data for the given date. Parameters ---------- from_date: string The APDS start date in ISO8601 format, e.g. 2017-08-01 to_date: string The APDS finish date in ISO8601 format, e.g. 2017-08-31 date: string The date in ISO8601 format, e.g. 2017-08-16 Returns ------- True if succesful, False otherwise. """ filenames = create_merge_apds_input_filenames(from_date, to_date, date) log.info("merging apds trajectories with files %s", str(filenames)) if merge_apds_trajectories(filenames): return False gc.collect() # delete the source daily input files day_filenames = create_daily_filenames(date) [os.remove(file_path) for file_path in day_filenames[1:]] # rename the flights file to add APDS to the start apds_flight_filename = '_'.join([APDS, day_filenames[0]]) os.rename(day_filenames[0], apds_flight_filename) ok = put_processed(REFINED_MERGED_APDS_CPR_FR24, [apds_flight_filename]) if ok: os.remove(apds_flight_filename) else: return False output_filenames = create_merge_apds_output_filenames(date) ok = put_processed(REFINED_MERGED_APDS_CPR_FR24, output_filenames) if ok: # delete the output files [os.remove(file_path) for file_path in output_filenames] return ok
def write_clean_positions_data(source, date): """ Write the cleaned position data and error metrics to the Google bucket. Parameters ---------- source: string The source of the positions data: CPR, FR24 or CPR_FR24 date: string The date in ISO8601 format, e.g. 2017-08-16 """ filenames = create_clean_position_data_filenames(source, date) source_path = REFINED_MERGED_DAILY_CPR_FR24 \ if (source == CPR_FR24) else '/'.join([REFINED, source]) put_processed(source_path, filenames[:1]) errors_path = '/'.join([PRODUCTS, ERROR_METRICS, source]) put_processed(errors_path, filenames[1:])
def match_previous_days_flights(date): """ Match merged CPR and FR24 ADS-B data for the given date, with merged CPR and FR24 ADS-B data for the previous day. Parameters ---------- date: string The date in ISO8601 format, e.g. 2017-08-16 Returns ------- True if succesful, False otherwise. """ # Get the previous days files from the overnight bucket, if not available locally merge_files = create_merge_consecutive_day_input_filenames(date) if not path_exists(merge_files[1]) or \ not path_exists(merge_files[3]) or \ not path_exists(merge_files[5]): log.debug("Getting files %s, %s, %s", merge_files[1], merge_files[3], merge_files[5]) if not get_processed(REFINED_MERGED_OVERNIGHT_CPR_FR24, merge_files[1::2]): log.error( 'Previous days files not found in overnight_cpr_fr24 bucket') return False # Get the current days files from the daily bucket, if not available locally if not path_exists(merge_files[2]) or \ not path_exists(merge_files[4]) or \ not path_exists(merge_files[6]): log.debug("Getting files %s, %s, %s", merge_files[2], merge_files[4], merge_files[6]) if not get_processed(REFINED_MERGED_DAILY_CPR_FR24, merge_files[2::2]): log.error('Current days files not found in daily_cpr_fr24 bucket') return False filenames = create_match_consecutive_day_input_filenames(date) if match_consecutive_day_trajectories(filenames): return False gc.collect() prev_ids_filename = create_matching_ids_filename(PREV_DAY, date) return put_processed(REFINED_MERGED_OVERNIGHT_CPR_FR24_IDS, [prev_ids_filename])
def interpolate_trajectory_file(trajectory_filename, source=CPR_FR24, straight_interval=DEFAULT_STRAIGHT_INTERVAL, turn_interval=DEFAULT_TURN_INTERVAL, logging_msg_count=DEFAULT_LOGGING_COUNT): """ Interpolate a trajectories file for the given date. Parameters ---------- trajectory_filename: string The trajectory filename. source: string The data to interpolate, default cpr_fr24 straight_interval: float The time interval between points on straight legs. turn_interval: float The time interval between points around turns. Returns ------- True if succesful, False otherwise. """ source_path = '/'.join([PRODUCTS, TRAJECTORIES, source]) get_processed(source_path, [trajectory_filename]) log.debug("Interpolating trajectories for file %s", trajectory_filename) if interpolate_trajectories(trajectory_filename, straight_interval, turn_interval, logging_msg_count): return False gc.collect() os.remove(trajectory_filename) # delete the trajectory file output_filename = trajectory_filename.replace(TRAJECTORIES, SYNTH_POSITIONS) output_filename = output_filename.replace(JSON_FILE_EXTENSION, CSV_FILE_EXTENSION) output_path = '/'.join([PRODUCTS, SYNTH_POSITIONS, source]) return put_processed(output_path, [output_filename])
def match_apds_trajectories_on_day(from_date, to_date, date): """ Match refined APDS data with CPR and FR24 ADS-B data for the given date. Parameters ---------- from_date: string The APDS start date in ISO8601 format, e.g. 2017-08-01 to_date: string The APDS finish date in ISO8601 format, e.g. 2017-08-31 date: string The date in ISO8601 format, e.g. 2017-08-16 Returns ------- True if succesful, False otherwise. """ apds_filenames = create_convert_apds_filenames(from_date, to_date) if not path_exists(apds_filenames[0]) or \ not path_exists(apds_filenames[1]) or \ not path_exists(apds_filenames[2]): log.debug("Getting apds files: %s", str(apds_filenames)) if not get_processed(REFINED_APDS, apds_filenames): log.error('APDS files not found in sources/apds bucket') return False day_filenames = create_daily_filenames(date) log.debug("Getting days files: %s", str(day_filenames)) if not get_processed(REFINED_MERGED_OVERNIGHT_CPR_FR24, day_filenames): log.error('Data files not found in overnight_cpr_fr24 bucket') return False filenames = create_match_apds_input_filenames(from_date, to_date, date) log.info("matching apds trajectories with files %s", str(filenames)) if match_apds_trajectories(filenames): return False gc.collect() return put_processed(REFINED_MERGED_APDS_CPR_FR24_IDS, [create_matching_ids_filename(APDS, date)])
def match_cpr_adsb_trajectories_on(date): """ Match refined CPR and FR24 ADS-B data for the given date. Parameters ---------- date: string The date in ISO8601 format, e.g. 2017-08-16 Returns ------- True if succesful, False otherwise. """ filenames = create_match_cpr_adsb_input_filenames(date) if match_cpr_adsb_trajectories(filenames): return False gc.collect() return put_processed(REFINED_MERGED_DAILY_CPR_FR24_IDS, create_match_cpr_adsb_output_filenames(date))
def find_trajectory_airport_intersections( trajectory_filename, source=CPR_FR24, radius=DEFAULT_RADIUS, airports_filename=DEFAULT_MOVEMENTS_AIRPORTS_FILENAME, distance_tolerance=DEFAULT_DISTANCE_TOLERANCE): """ """ if not path_exists(trajectory_filename): source_path = '/'.join([PRODUCTS, TRAJECTORIES, source]) log.debug("Getting trajectory file: %s", trajectory_filename) if not get_processed(source_path, [trajectory_filename]): log.error('Trajectory file not found in %s bucket', source_path) return False date = read_iso8601_date_string(trajectory_filename, is_json=True) flights_filename = create_flights_filename(source, date) if not path_exists(flights_filename): source_path = REFINED_MERGED_OVERNIGHT_CPR_FR24 \ if (source == CPR_FR24) else '/'.join([REFINED, source]) if not get_processed(source_path, [flights_filename]): log.error('Flights file not found in %s bucket', source_path) return False get_airports(airports_filename, '.') log.info("find_sector_intersections for: %s and %s", flights_filename, trajectory_filename) if find_airport_intersections(flights_filename, trajectory_filename, radius, airports_filename, distance_tolerance): return False output_filename = trajectory_filename.replace(TRAJECTORIES, AIRPORT_INTERSECTIONS) output_filename = output_filename.replace(JSON_FILE_EXTENSION, CSV_FILE_EXTENSION) intersections_path = '/'.join([PRODUCTS_INTERSECTIONS_AIRPORT, source]) return put_processed(intersections_path, [output_filename])
def refine_apds_file(apds_filename, stands_filename=DEFAULT_STANDS_FILENAME): """ Converts an APDS file into PRU format flights, positions and events files. Parameters ---------- apds_filename: string The name of the APDS file, e.g.: FAC_APDS_FLIGHT_IR691_2017-08-01_2017-09-01.csv.bz2 stands_filename: string The name of the airport stands file, default DEFAULT_STANDS_FILENAME. Note: it won't create a positions file if run with a empty stands_filename. Returns ------- True if succesful, False otherwise. """ # Get the files from the bucket get_apds(apds_filename) get_stands(stands_filename) apds_path_name = '/'.join([UPLOAD_DIR, APDS, apds_filename]) stands_path_name = '/'.join([DATA_HOME, AIRPORTS_STANDS, stands_filename]) log.info("Converting APDS data with stands file: %s", stands_filename) if convert_apds_data(apds_path_name, stands_path_name): return False gc.collect() os.remove(apds_path_name) # delete the uploaded APDS file from_date, to_date = split_dual_date(apds_filename) return put_processed(REFINED_APDS, create_convert_apds_filenames(from_date, to_date))
def merge_previous_days_data(date): """ Merge merged CPR and FR24 ADS-B data for the given date, with merged CPR and FR24 ADS-B data for the previous day. Parameters ---------- date: string The date in ISO8601 format, e.g. 2017-08-16 Returns ------- True if succesful, False otherwise. """ filenames = create_merge_consecutive_day_input_filenames(date) if merge_consecutive_day_trajectories(filenames): return False gc.collect() # rename output filenames to input files new_files = create_merge_consecutive_day_output_filenames(date) log.debug("Moving files: %s", str(new_files)) [subprocess.run(['mv', file, file[len(NEW) + 1:]]) for file in new_files] # rename the previous days positions filename to raw_... raw_positions_filename = filenames[3].replace(CPR_FR24, RAW_CPR_FR24) os.rename(filenames[3], raw_positions_filename) filenames[3] = raw_positions_filename copied_ok = put_processed(REFINED_MERGED_OVERNIGHT_CPR_FR24, filenames[1:]) if copied_ok: [os.remove(file_path) for file_path in filenames[:3]] # Don't remove the raw positions file [os.remove(file_path) for file_path in filenames[4:]] return copied_ok
def merge_cpr_adsb_trajectories_on(date): """ Merge refined CPR and FR24 ADS-B data for the given date. Parameters ---------- date: string The date in ISO8601 format, e.g. 2017-08-16 Returns ------- True if succesful, False otherwise. """ filenames = create_merge_cpr_adsb_input_filenames(date) if merge_cpr_adsb_trajectories(filenames): return False gc.collect() # delete the source input files [os.remove(file_path) for file_path in filenames] return put_processed(REFINED_MERGED_DAILY_CPR_FR24, create_merge_cpr_adsb_output_filenames(date))
def merge_overnight_data_on_day(date, max_speed=DEFAULT_MAX_SPEED, distance_accuracy=DEFAULT_DISTANCE_ACCURACY): """ Match flights for the given day with flights for the previous day. Parameters ---------- date: string The date in ISO8601 format, e.g. 2017-08-16 max_speed: string The maximum ground speed permitted between adjacent positions [Knots], default: 750 Knots. distance_accuracy: string The maximum distance between positions at the same time [Nautical Miles], default: 0.25 NM. """ if is_valid_iso8601_date(date): # get the CPR data from the Google bucket log.info(f'Getting data for date: {date}') merge_files = create_merge_overnight_flight_data_input_filenames(date) if not get_processed(REFINED_MERGED_OVERNIGHT_CPR_FR24, merge_files): log.error('Flights file not found in overnight_cpr_fr24 bucket') return errno.ENOENT error_code = merge_overnight_flight_data(merge_files) if error_code: return error_code output_files = create_merge_overnight_flight_data_output_filenames( date) if not put_processed(REFINED_MERGED_OVERNIGHT_CPR_FR24, output_files): log.error('Could not merged files to overnight_cpr_fr24 bucket') return errno.EACCES raw_filename = output_files[1] error_code = clean_position_data(raw_filename, max_speed, distance_accuracy) if error_code: log.error('clean_position_data error file: {raw_filename}') return error_code filenames = create_clean_position_data_filenames(CPR_FR24, date) source_path = REFINED_MERGED_OVERNIGHT_CPR_FR24 if not put_processed(source_path, filenames[:1]): log.error('Could not write file: {filenames[:1]} to bucket') return errno.EACCES errors_path = PRODUCTS_ERROR_METRICS_CPR_FR24_OVERNIGHT if not put_processed(errors_path, filenames[1:]): log.error('Could not write file: {filenames[1:]} to bucket') return errno.EACCES else: log.error(f'invalid date: {date}') return errno.EINVAL return 0
def import_data_on_day(date, max_speed=DEFAULT_MAX_SPEED, distance_accuracy=DEFAULT_DISTANCE_ACCURACY): """ Import, refine and merge the CPR and FR24 data for the given date. Parameters ---------- date: string The date in ISO8601 format, e.g. 2017-08-16 max_speed: string The maximum ground speed permitted between adjacent positions [Knots], default: 750 Knots. distance_accuracy: string The maximum distance between positions at the same time [Nautical Miles], default: 0.25 NM. """ if is_valid_iso8601_date(date): log.info(f'Getting data for date: {date}') # convert the FR24 data # Note: needs over 13GB memory if run with convert_cpr_data. get_unprocessed(FR24, date, '.') convert_fr24_data_on_day(date) gc.collect() # Clean the FR24 data in parallel procs = [] procs.append( clean_raw_positions_data(FR24, date, max_speed, distance_accuracy)) # convert the CPR data in parallel get_unprocessed(CPR, date, '.') cpr_proc = convert_cpr_data_on_day(date) # write the converted FR24 data to the Google bucket put_processed(REFINED_FR24, create_convert_fr24_filenames(date)) process_fr24_flights(date, DEFAULT_AIRPORTS_FILENAME) gc.collect() # Wait for CPR conversion to finish out, err = cpr_proc.communicate() print(out) gc.collect() # Clean the CPR and FR24 data in parallel procs.append( clean_raw_positions_data(CPR, date, max_speed, distance_accuracy)) # write the converted CPR data to the Google bucket put_processed(REFINED_CPR, create_convert_cpr_filenames(date)) # Wait for the CPR and FR24 cleaning tasks for proc in procs: out, err = proc.communicate() print(out) # write the CPR and FR24 positions to the Google bucket write_clean_positions_data(CPR, date) write_clean_positions_data(FR24, date) gc.collect() merge_cpr_and_fr24_data(date) gc.collect() # Clean the merged positions proc = clean_raw_positions_data(CPR_FR24, date, max_speed, distance_accuracy) # put the merged CPR and FR24 data to the Google bucket put_processed(REFINED_MERGED_DAILY_CPR_FR24_IDS, create_match_cpr_adsb_output_filenames(date)) put_processed(REFINED_MERGED_DAILY_CPR_FR24, create_merge_cpr_adsb_output_filenames(date)) # Wait for cleaning to finish out, err = proc.communicate() print(out) # put the merged CPR and FR24 positions to the Google bucket write_clean_positions_data(CPR_FR24, date) else: log.error(f'invalid date: {date}') return errno.EINVAL return 0
def analyse_positions_on_date( date, source=CPR_FR24, *, distance_tolerance=DEFAULT_ACROSS_TRACK_TOLERANCE, time_method=MOVING_AVERAGE_SPEED, N=DEFAULT_MOVING_MEDIAN_SAMPLES, M=DEFAULT_MOVING_AVERAGE_SAMPLES, max_duration=DEFAULT_SPEED_MAX_DURATION, logging_msg_count=DEFAULT_LOGGING_COUNT): """ Analyse refined CPR and FR24 ADS-B data for the given date. Parameters ---------- date: string The date in ISO8601 format, e.g. 2017-08-16 source: string The data to analyse, default cpr_fr24 distance_tolerance: float The maximum across track distance[Nautical Miles], default: 0.25 NM. time_method: string The time analysis method, default 'lm'. N : integer The number of samples to consider for the speed moving median filter, default 5. M : integer The number of samples to consider for the speed moving average filter, default 5. max_duration: float The maximum time between points to smooth when calculating speed, default 120 [Seconds]. logging_msg_count : integer The number of analysed flights between each log message. Returns ------- True if succesful, False otherwise. """ positions_filename = create_positions_filename(source, date) source_path = REFINED_MERGED_OVERNIGHT_CPR_FR24 if (source == CPR_FR24) else \ '/'.join([REFINED, source]) get_processed(source_path, [positions_filename]) log.info("Analysing position data for file %s", positions_filename) if analyse_position_data(positions_filename, distance_tolerance, time_method): return False gc.collect() filenames = create_analyse_position_data_filenames(source, date, distance_tolerance, time_method) # Put the traj metrics file first so that it can be deleted metrics_path = '/'.join([PRODUCTS, TRAJ_METRICS, source]) if not put_processed(metrics_path, filenames[1:]): return False os.remove(filenames[1]) # delete the traj metrics file output_path = '/'.join([PRODUCTS, TRAJECTORIES, source]) return put_processed(output_path, filenames[:1])