def validate_auto_graph_changes_applied(coverage_name, default_coverage_name, default_cov_prev_sop_date, docker_client,
navitia_docker_compose_file_path, navitia_docker_compose_file_name,
navitia_docker_compose_default_file_name):
"""
Validate that the new default coverage returns results for heat map query for current_start_service_date (as in dates file
or gtfs date) and that secondary-cov has results for the previous production date of the default.
:param default_coverage_name: The coverage that gets a new (usually more recent) start of production date
:param secondary_custom_coverage_name: The coverage that gets a the original default_coverage start of production date
:param default_cov_sop_date: start of production date of original default coverage (before changes applied)
:return: whether the graph changes were applied
"""
current_start_service_date = dt.strptime(process_date.get_date_now(), "%Y%m%d")
if cfg.ttm_server_on == "aws_ec2":
time_map_server_url = cfg.time_map_server_aws_url
else:
time_map_server_url = cfg.time_map_server_local_url
# Check that the current default coverage is up-to-date by comparing sop dates
stop_all_containers(docker_client)
start_navitia_with_single_coverage(navitia_docker_compose_file_path, navitia_docker_compose_default_file_name,
default_coverage_name, False)
if not check_prod_date_is_valid_using_heat_map(time_map_server_url, default_coverage_name,
current_start_service_date.strftime("%Y%m%d")):
_log.error("The %s coverage seems not to be up-to-date following update attempts.", default_coverage_name)
return False
else:
_log.info("%s coverage is up-to-date with production date %s", default_coverage_name,
current_start_service_date.strftime("%Y%m%d"))
# Check that the coverage_name (the previous one) is up-to-date by comparing sop dates
stop_all_containers(docker_client)
is_up = start_navitia_with_single_coverage(navitia_docker_compose_file_path, navitia_docker_compose_file_name,
coverage_name, False)
if not is_up:
_log.error("The %s coverage seems not to be up", coverage_name)
cov_sop_date = get_coverage_start_production_date(coverage_name)
if cov_sop_date == "":
_log.info("If this is the first time you're running Transit Analyst Israel data processing, you need to "
"copy the generated default.nav.lz4 graph to secondary-cov.nav.lz4 - See docs.")
return True
if not check_prod_date_is_valid_using_heat_map(time_map_server_url, coverage_name,
current_start_service_date.strftime("%Y%m%d")):
_log.error("The %s coverage seems not to be up-to-date following update attempts.\nA call for heat map data with"
" %s date returned no results", coverage_name, current_start_service_date.strftime("%Y%m%d"))
return False
_log.info("%s coverage is now updated with new start-of-production date %s. "
"Can be accessed via %s%s", coverage_name, current_start_service_date.strftime("%Y%m%d"), time_map_server_url,
coverage_name)
return True
def gtfs_unzip():
"""
Unzip GTFS and remove BOM charcters that interrupt with reasing the file
:param _log:
:return:
"""
try:
processdate = process_date.get_date_now()
gtfs_zip_file_name = cfg.gtfsdirbase + processdate + ".zip"
unzip_gtfs(gtfs_zip_file_name, cfg.gtfspath, _log)
remove_bom_characters_from_unzipped_files(
os.path.join(cfg.gtfspath, cfg.gtfsdirbase + processdate))
except Exception as e:
raise e
def validate_graph_changes_applied(coverage_name):
"""
Validate that the coverage has a different start of production date different from before
"""
current_start_service_date = process_date.get_date_now()
if cfg.ttm_server_on == "aws_ec2":
time_map_server_url = cfg.time_map_server_aws_url
else:
time_map_server_url = cfg.time_map_server_local_url
cov_sop_date = get_coverage_start_production_date(coverage_name)
if cov_sop_date == "" or not check_prod_date_is_valid_using_heat_map(time_map_server_url, coverage_name,
current_start_service_date):
_log.error("The %s coverage seems not to be up-to-date following update attempts."
"\n A call for heat map data with %s date returned no results",
coverage_name, current_start_service_date)
return False
_log.info("%s coverage is now updated with new start-of-production date %s\n."
"Can be accessed via %s%s", coverage_name, current_start_service_date, time_map_server_url,
coverage_name)
return True
def get_config_params():
"""
Reads monthly_update_config_params.conf file and returns the configuration parameters
:return: configuration parameters
"""
# Get parameters
default_coverage_name = cfg.default_coverage_name
secondary_custom_coverage_name = cfg.secondary_custom_coverage_name
# navitia_docker_compose_file_name = "docker-israel-custom-instances.yml"-needed only when 2 coverages should be up
# Currently we only bring up one coverage and it's always the previous month to be aligned with past and current
# websites. The regular coverage (most current) file is: "docker-compose.yml"
coverage_name = secondary_custom_coverage_name
navitia_docker_compose_file_name = "docker-compose-secondary-cov.yml"
navitia_docker_compose_default_file_name = "docker-compose.yml"
navitia_docker_compose_file_path = Path(os.getcwd()).parent.parent / "navitia-docker-compose" / "compose_files"
if cfg.get_service_date == "on_demand":
navitia_docker_compose_file_name = "navitia-docker-ondemand-" + cfg.gtfsdate + ".yml"
coverage_name = "ondemand-" + cfg.gtfsdate
gtfs_file_path = Path(os.getcwd()).parent / cfg.gtfspath
processdate = process_date.get_date_now()
gtfs_zip_file_name = cfg.gtfsdirbase + processdate + ".zip"
return default_coverage_name, coverage_name, navitia_docker_compose_file_path, navitia_docker_compose_file_name, \
navitia_docker_compose_default_file_name, gtfs_file_path, gtfs_zip_file_name
#!/usr/bin/env python # -*- coding: utf-8 -*- # # convert stopsinmuni_pre_edit.txt to stopsinmuni_post_edit.txt without any changes # can be replaced with smart algorithem to deal with edge cases... # import transitanalystisrael_config as cfg import process_date import os from pathlib import Path import shutil processdate = process_date.get_date_now() cwd = Path.cwd() prefile = 'stopsinmuni_pre_edit_' + processdate + '.txt' postfile = 'stopsinmuni_post_edit_' + processdate + '.txt' os.chdir(cwd.parent / cfg.processedpath) shutil.copy(prefile, postfile)
def get_gtfs_file_from_url_ftp(url, file_name_on_server, _log):
"""
Downloads a GTFS file from an FTP server to the working directory
:param url: the FTP server URL that points to file's containing folder
:param file_name_on_server: The file name on the FTP server
:return: file name of the downloaded content in the working directory
"""
_log.info("Going to download the latest GTFS from %s ", url)
download_complete = False
download_attempts = 1
max_download_attemtps = 24
while not download_complete:
if not download_complete and 24 > download_attempts > 1:
_log.error(
"%s is unreachable. Sleeping for 60 minutes and trying again. This is attempt %s out of "
"%s attempts", url, download_attempts, max_download_attemtps)
time.sleep(60 * 60)
if not download_complete and download_attempts > 24:
_log.error(
"%s is unreachable for more than 24 hours. Aborting update",
url)
raise Exception
download_attempts += 1
try:
# Connect to FTP
ftp = ftplib.FTP(url)
ftp.login()
# Get the GTFS time stamp and generate local file name, "israel20190225"
file_lines = []
size = 0
local_file_name = cfg.gtfsdirbase
processdate = process_date.get_date_now()
ftp.dir("", file_lines.append)
for line in file_lines:
tokens = line.split(maxsplit=4)
name = tokens[3]
if name == file_name_on_server:
time_str = tokens[0]
actual_time = parser.parse(time_str)
local_file_name = local_file_name + processdate + ".zip"
size = float(tokens[2])
pardir = Path(os.getcwd()).parent
local_file_path_and_name = pardir / cfg.gtfspath / local_file_name
# Generate a progress bar and download
local_file = open(local_file_path_and_name, 'wb')
pbar = createProgressBar(size)
# Download
global size_iterator
size_iterator = 0
ftp.retrbinary(
"RETR " + file_name_on_server,
lambda data, : file_write_update_progress_bar(
data, local_file, pbar))
# Finish
local_file.close()
ftp.quit()
pbar.finish()
sys.stdout.flush()
download_complete = True
_log.info("Finished loading latest GTFS to: %s",
local_file_path_and_name)
return local_file_name
except ftplib.all_errors as err:
error_code = err.args[0]
# file not found on server
if error_code == 2:
_log.error(file_name_on_server, "is not found on %s", url)
raise err
# Unvalid URL
if error_code == 11001:
_log.error("URL %s is not valid", url)
continue
