def check_feed(self, url, file_name, force_update=False):
''' download feed from url, and check it against the cache
if newer, then replace cached feed .zip file with new version
'''
# step 1: file name
file_name = file_name
file_path = os.path.join(self.cache_dir, file_name)
# step 2: download new gtfs file
url = url
tmp_path = os.path.join(self.tmp_dir, file_name)
# step 2b: don't keep downloading a file ... make sure the tmp file is at least 2 hours
if os.path.exists(tmp_path) is False or \
file_utils.file_age_seconds(tmp_path) > 7200:
web_utils.wget(url, tmp_path)
# step 3: check the cache whether we should update or not
update = force_update
if not force_update:
if self.is_fresh_in_cache(file_path):
log.info("diff {} against cached {}".format(
tmp_path, file_path))
diff = Diff(file_path, tmp_path)
if diff.is_different():
update = True
else:
update = True
# step 4: mv old file to backup then mv new file in tmp dir to cache
if update:
log.info("cp {} to cache {}".format(tmp_path, file_path))
file_utils.bkup(file_path)
file_utils.cp(tmp_path, file_path)
def download_pbf(self, pbf_url, meta_url=None):
pbf_path = self.path_from_url(pbf_url, self.cache_dir)
log.info("wget {} to {}".format(pbf_url, pbf_path))
file_utils.bkup(pbf_path)
web_utils.wget(pbf_url, pbf_path)
try:
if meta_url:
meta_path = self.path_from_url(meta_url, self.cache_dir)
web_utils.wget(meta_url, meta_path)
except Exception as e:
log.info(e)
def check_otp_jar(graph_dir, jar=OTP_NAME, expected_size=50000000, download_url=OTP_DOWNLOAD_URL):
""" utility to make sure otp.jar exists in the particular graph dir...
if not, download it
:return full-path to otp.jar
"""
jar_path = os.path.join(graph_dir, jar)
exists = os.path.exists(jar_path)
if not exists or file_utils.file_size(jar_path) < expected_size:
log.info("we don't see OTP {} in {}, so will download {} now".format(jar, graph_dir, download_url))
web_utils.wget(download_url, jar_path)
return jar_path
def check_feed(self, url, file_name, force_update=False):
"""
download feed from url, and check it against the cache
if newer, then replace cached feed .zip file with new version
"""
# step 1: file name
file_name = file_name
file_path = os.path.join(self.cache_dir, file_name)
# step 2: download new gtfs file
url = url
tmp_path = os.path.join(self.tmp_dir, file_name)
# step 2b: don't keep downloading a file ... make sure the tmp file is at least 2 hours
if os.path.exists(tmp_path) is False or file_utils.file_age_seconds(tmp_path) > 7200:
web_utils.wget(url, tmp_path)
# step 3: check the cache whether we should update or not
update = force_update
if not force_update:
if self.is_fresh_in_cache(file_path):
log.info("diff {} against cached {}".format(tmp_path, file_path))
diff = Diff(file_path, tmp_path)
if diff.is_different():
update = True
else:
update = True
# step 4: test new .zip for validity and also
if update:
# step 4a: make sure this new .zip feed has a trips.txt, routes.txt and stops.txt file ... if not no update
if GtfsInfo.feed_looks_valid(tmp_path):
# step 4b: mv old file to backup then mv new file in tmp dir to cache
log.info("cp {} to cache {}".format(tmp_path, file_path))
file_utils.bkup(file_path)
file_utils.cp(tmp_path, file_path)
else:
log.warning("something *WRONG* with file: {}".format(tmp_path))
update = False
return update
def diff_vlog_files(svr, graph_dir, vlog_name=VLOG_NAME):
""" return True if the files are different and need to be redeployed ...
- grab vlog from remote server that builds new OTP graphs
- compare it to our local vlog
- send email if we can't find remote vlog...
"""
ret_val = False
# step 1: grab otp.v from build server
url = "{}/{}".format(svr, vlog_name)
vlog_path = get_vlog_file_path(graph_dir, vlog_name)
tmp_vlog_path = vlog_path + ".tmp"
ok = web_utils.wget(url, tmp_vlog_path, 10)
if not ok:
# step 2: remote server doesn't have otp.v exposed ... send an error email...
msg = "No vlog available at {0}".format(url)
web_utils.email(msg, msg)
ret_val = False
else:
# step 3: make sure the otp.v we just downloaded has content ... if not, send an error email
if not file_utils.is_min_sized(tmp_vlog_path, 20):
msg = "vlog file {0} (grabbed from {1}) isn't right ".format(tmp_vlog_path, url)
email(msg, msg)
ret_val = False
else:
# step 4a: we currently don't have a vlog, so assume we don't have an existing OTP ... let's deploy new download...
if not file_utils.is_min_sized(vlog_path, 20):
ret_val = True
logging.info("{0} doesn't exist ... try to grab new OTP from {1} and deploy".format(vlog_path, svr))
else:
# step 4b: check if the vlog files are different...if so, we'll assume the remote is newer and start a new deploy...
if filecmp.cmp(tmp_vlog_path, vlog_path):
logging.info("{0} == {1} ... we're done, going to keep the current graph running".format(tmp_vlog_path, vlog_path))
else:
ret_val = True
logging.info("{0} != {1} ... will try to grab new OTP from {2} and deploy".format(tmp_vlog_path, vlog_path, svr))
return ret_val
def update_old_otp(self):
gok = web_utils.wget(URL_GRAPH, TMP_GRAPH, 1000000)
aok = web_utils.wget(URL_API, TMP_API, 1000000)
jok = web_utils.wget(URL_JAR, TMP_JAR, 1000000)
return gok and aok
def update_new_otp(self):
gok = web_utils.wget(URL_GRAPH, TMP_GRAPH, 1000000)
aok = web_utils.wget(URL_API, TMP_API, 1000000)
return gok and aok
def download_pbf(self):
log.info("wget {} to {}".format(self.pbf_url, self.pbf_path))
file_utils.bkup(self.pbf_path)
web_utils.wget(self.pbf_url, self.pbf_path)
if self.meta_url:
web_utils.wget(self.meta_url, self.meta_path)