def validate_auto_graph_changes_applied(coverage_name, default_coverage_name, default_cov_prev_sop_date, docker_client, navitia_docker_compose_file_path, navitia_docker_compose_file_name, navitia_docker_compose_default_file_name): """ Validate that the new default coverage returns results for heat map query for current_start_service_date (as in dates file or gtfs date) and that secondary-cov has results for the previous production date of the default. :param default_coverage_name: The coverage that gets a new (usually more recent) start of production date :param secondary_custom_coverage_name: The coverage that gets a the original default_coverage start of production date :param default_cov_sop_date: start of production date of original default coverage (before changes applied) :return: whether the graph changes were applied """ current_start_service_date = dt.strptime(process_date.get_date_now(), "%Y%m%d") if cfg.ttm_server_on == "aws_ec2": time_map_server_url = cfg.time_map_server_aws_url else: time_map_server_url = cfg.time_map_server_local_url # Check that the current default coverage is up-to-date by comparing sop dates stop_all_containers(docker_client) start_navitia_with_single_coverage(navitia_docker_compose_file_path, navitia_docker_compose_default_file_name, default_coverage_name, False) if not check_prod_date_is_valid_using_heat_map(time_map_server_url, default_coverage_name, current_start_service_date.strftime("%Y%m%d")): _log.error("The %s coverage seems not to be up-to-date following update attempts.", default_coverage_name) return False else: _log.info("%s coverage is up-to-date with production date %s", default_coverage_name, current_start_service_date.strftime("%Y%m%d")) # Check that the coverage_name (the previous one) is up-to-date by comparing sop dates stop_all_containers(docker_client) is_up = start_navitia_with_single_coverage(navitia_docker_compose_file_path, navitia_docker_compose_file_name, coverage_name, False) if not is_up: _log.error("The %s coverage seems not to be up", coverage_name) cov_sop_date = get_coverage_start_production_date(coverage_name) if cov_sop_date == "": _log.info("If this is the first time you're running Transit Analyst Israel data processing, you need to " "copy the generated default.nav.lz4 graph to secondary-cov.nav.lz4 - See docs.") return True if not check_prod_date_is_valid_using_heat_map(time_map_server_url, coverage_name, current_start_service_date.strftime("%Y%m%d")): _log.error("The %s coverage seems not to be up-to-date following update attempts.\nA call for heat map data with" " %s date returned no results", coverage_name, current_start_service_date.strftime("%Y%m%d")) return False _log.info("%s coverage is now updated with new start-of-production date %s. " "Can be accessed via %s%s", coverage_name, current_start_service_date.strftime("%Y%m%d"), time_map_server_url, coverage_name) return True
def gtfs_unzip(): """ Unzip GTFS and remove BOM charcters that interrupt with reasing the file :param _log: :return: """ try: processdate = process_date.get_date_now() gtfs_zip_file_name = cfg.gtfsdirbase + processdate + ".zip" unzip_gtfs(gtfs_zip_file_name, cfg.gtfspath, _log) remove_bom_characters_from_unzipped_files( os.path.join(cfg.gtfspath, cfg.gtfsdirbase + processdate)) except Exception as e: raise e
def validate_graph_changes_applied(coverage_name): """ Validate that the coverage has a different start of production date different from before """ current_start_service_date = process_date.get_date_now() if cfg.ttm_server_on == "aws_ec2": time_map_server_url = cfg.time_map_server_aws_url else: time_map_server_url = cfg.time_map_server_local_url cov_sop_date = get_coverage_start_production_date(coverage_name) if cov_sop_date == "" or not check_prod_date_is_valid_using_heat_map(time_map_server_url, coverage_name, current_start_service_date): _log.error("The %s coverage seems not to be up-to-date following update attempts." "\n A call for heat map data with %s date returned no results", coverage_name, current_start_service_date) return False _log.info("%s coverage is now updated with new start-of-production date %s\n." "Can be accessed via %s%s", coverage_name, current_start_service_date, time_map_server_url, coverage_name) return True
def get_config_params(): """ Reads monthly_update_config_params.conf file and returns the configuration parameters :return: configuration parameters """ # Get parameters default_coverage_name = cfg.default_coverage_name secondary_custom_coverage_name = cfg.secondary_custom_coverage_name # navitia_docker_compose_file_name = "docker-israel-custom-instances.yml"-needed only when 2 coverages should be up # Currently we only bring up one coverage and it's always the previous month to be aligned with past and current # websites. The regular coverage (most current) file is: "docker-compose.yml" coverage_name = secondary_custom_coverage_name navitia_docker_compose_file_name = "docker-compose-secondary-cov.yml" navitia_docker_compose_default_file_name = "docker-compose.yml" navitia_docker_compose_file_path = Path(os.getcwd()).parent.parent / "navitia-docker-compose" / "compose_files" if cfg.get_service_date == "on_demand": navitia_docker_compose_file_name = "navitia-docker-ondemand-" + cfg.gtfsdate + ".yml" coverage_name = "ondemand-" + cfg.gtfsdate gtfs_file_path = Path(os.getcwd()).parent / cfg.gtfspath processdate = process_date.get_date_now() gtfs_zip_file_name = cfg.gtfsdirbase + processdate + ".zip" return default_coverage_name, coverage_name, navitia_docker_compose_file_path, navitia_docker_compose_file_name, \ navitia_docker_compose_default_file_name, gtfs_file_path, gtfs_zip_file_name
#!/usr/bin/env python # -*- coding: utf-8 -*- # # convert stopsinmuni_pre_edit.txt to stopsinmuni_post_edit.txt without any changes # can be replaced with smart algorithem to deal with edge cases... # import transitanalystisrael_config as cfg import process_date import os from pathlib import Path import shutil processdate = process_date.get_date_now() cwd = Path.cwd() prefile = 'stopsinmuni_pre_edit_' + processdate + '.txt' postfile = 'stopsinmuni_post_edit_' + processdate + '.txt' os.chdir(cwd.parent / cfg.processedpath) shutil.copy(prefile, postfile)
def get_gtfs_file_from_url_ftp(url, file_name_on_server, _log): """ Downloads a GTFS file from an FTP server to the working directory :param url: the FTP server URL that points to file's containing folder :param file_name_on_server: The file name on the FTP server :return: file name of the downloaded content in the working directory """ _log.info("Going to download the latest GTFS from %s ", url) download_complete = False download_attempts = 1 max_download_attemtps = 24 while not download_complete: if not download_complete and 24 > download_attempts > 1: _log.error( "%s is unreachable. Sleeping for 60 minutes and trying again. This is attempt %s out of " "%s attempts", url, download_attempts, max_download_attemtps) time.sleep(60 * 60) if not download_complete and download_attempts > 24: _log.error( "%s is unreachable for more than 24 hours. Aborting update", url) raise Exception download_attempts += 1 try: # Connect to FTP ftp = ftplib.FTP(url) ftp.login() # Get the GTFS time stamp and generate local file name, "israel20190225" file_lines = [] size = 0 local_file_name = cfg.gtfsdirbase processdate = process_date.get_date_now() ftp.dir("", file_lines.append) for line in file_lines: tokens = line.split(maxsplit=4) name = tokens[3] if name == file_name_on_server: time_str = tokens[0] actual_time = parser.parse(time_str) local_file_name = local_file_name + processdate + ".zip" size = float(tokens[2]) pardir = Path(os.getcwd()).parent local_file_path_and_name = pardir / cfg.gtfspath / local_file_name # Generate a progress bar and download local_file = open(local_file_path_and_name, 'wb') pbar = createProgressBar(size) # Download global size_iterator size_iterator = 0 ftp.retrbinary( "RETR " + file_name_on_server, lambda data, : file_write_update_progress_bar( data, local_file, pbar)) # Finish local_file.close() ftp.quit() pbar.finish() sys.stdout.flush() download_complete = True _log.info("Finished loading latest GTFS to: %s", local_file_path_and_name) return local_file_name except ftplib.all_errors as err: error_code = err.args[0] # file not found on server if error_code == 2: _log.error(file_name_on_server, "is not found on %s", url) raise err # Unvalid URL if error_code == 11001: _log.error("URL %s is not valid", url) continue