def download_gtfs_file():
""" download gtfs zip file from mot, and put it in DATA_DIR in its own subfolder """
local_dir = os.path.join(GTFS_DATA_DIR,ot_utils.get_utc_time_underscored())
ot_utils.mkdir_p(local_dir)
local_path = os.path.join(local_dir,FILE_NAME)
ot_utils.ftp_get_file(MOT_FTP,FILE_NAME,local_path)
ot_utils.unzip_file(local_path,local_dir)
def download_gtfs_file(force=False):
""" download gtfs zip file from mot, and put it in DATA_DIR in its own subfolder """
import shutil
time_suffix = ot_utils.get_utc_time_underscored()
if not os.path.exists(GTFS_DATA_DIR):
ot_utils.mkdir_p(GTFS_DATA_DIR)
tmp_file = '/tmp/{0}_tmp.zip'.format(time_suffix)
LOGGER.info('downloading GTFS to tmp file')
ot_utils.ftp_get_file(MOT_FTP, FILE_NAME, tmp_file)
if not force:
tmp_md5 = ot_utils.md5_for_file(tmp_file)
last_dir = ot_utils.find_lastest_in_dir(GTFS_DATA_DIR)
if last_dir:
was_success = os.path.exists(os.path.join(last_dir, 'success'))
if not was_success:
LOGGER.info('Last time was not success')
else:
last_file = os.path.join(last_dir, FILE_NAME)
try:
last_md5 = ot_utils.md5_for_file(last_file)
except Exception as e:
LOGGER.exception('failed in md5 for last file - ignoring')
last_md5 = 'error_in_md5'
if last_md5 == tmp_md5:
LOGGER.info('Checksum is identical - removing tmp file')
os.remove(tmp_file)
return None
LOGGER.info('Checksum is different or force -- copying')
local_dir = os.path.join(GTFS_DATA_DIR, time_suffix)
ot_utils.mkdir_p(local_dir)
try:
os.remove(os.path.join(GTFS_DATA_DIR,'latest'))
except (IOError, OSError):
pass
os.symlink(local_dir, os.path.join(GTFS_DATA_DIR,'latest'))
local_file = os.path.join(local_dir, FILE_NAME)
shutil.move(tmp_file, local_file)
ot_utils.unzip_file(local_file, local_dir)
LOGGER.info('All gtfs files are in %s' % local_dir)
return local_dir
last_file = os.path.join(last_dir,FILE_NAME)
try:
last_md5 = ot_utils.md5_for_file(last_file)
except Exception,e:
print e
last_md5 = 'error_in_md5'
if last_md5 != tmp_md5:
print 'Checksum is different- copying'
ot_utils.mkdir_p(local_dir)
local_file = os.path.join(local_dir,FILE_NAME)
shutil.move(tmp_file,local_file)
else:
print 'Checksum is identical - removing tmp file'
os.remove(tmp_file)
return
if not download_only:
ot_utils.unzip_file(local_file,local_dir)
def find_gtfs_data_dir():
""" returns the lastest subfolder in DATA_DIR """
dirnames = glob.glob("%s/*" % (GTFS_DATA_DIR))
if not dirnames:
raise Exception("No data dir found in %s" % (GTFS_DATA_DIR))
# return the latest
return sorted(dirnames)[-1]