def gtfs_download(self, url, dt, zone):
"""Do downloading of one file."""
print("Downloading", self.slug, url, zone, dt)
# Use only standard library functions to avoid dependencies.
#furl = urllib.urlopen(url)
opener = FancyURLopener()
# We have to set up an authentication method on the opener if
# we will need to authenticate. This does HTTP BASIC only so
# far.
if 'authentication' in self.data:
auth_name = self.data['authentication']
auth = auth_data['sites'][auth_name]
# A callback method which performs the authentication.
# Return (user, pass) tuple.
opener.prompt_user_passwd = \
lambda host, realm: (auth['username'], auth['password'])
# URL parameters auth method
if 'url_suffix' in auth:
url = url + auth['url_suffix']
if "{API_KEY}" in url:
try:
auth_name = self.data['authentication']
except KeyError:
auth_name = self.name
auth = auth_data['sites'][auth_name]
url = url.format(API_KEY=auth['API_KEY'])
# Make GTFS path.
gtfs_path = self.path_gtfszip(dt, zone)
util.makedirs(os.path.dirname(gtfs_path))
# Open the URL.
print("**** Connecting to %s" % url)
# Open GTFS and relay data from web to file.
with util.create_file(gtfs_path) as tmp_gtfs_path:
opener.retrieve(url, tmp_gtfs_path)
self.test_corrupted_zip(gtfs_path)
def main():
import argparse
parser = argparse.ArgumentParser(
description="Create network extracts from already imported GTFS files."
)
subparsers = parser.add_subparsers(dest='cmd')
# parsing import
parser_routingnets = subparsers.add_parser(
'extract_temporal', help="Direct import GTFS->sqlite")
parser_routingnets.add_argument(
'gtfs', help='Input GTFS .sqlite (must end in .sqlite)')
parser_routingnets.add_argument(
'basename', help='Basename for the output files') # Parsing copy
args = parser.parse_args()
# if the first argument is import, import a GTFS directory to a .sqlite database.
# Both directory and
if args.cmd == 'extract_temporal':
gtfs_fname = args.gtfs
output_basename = args.basename
from gtfspy.gtfs import GTFS
gtfs = GTFS(gtfs_fname)
nodes_filename = output_basename + ".nodes.csv"
with util.create_file(nodes_filename, tmpdir=True,
keepext=True) as tmpfile:
write_nodes(gtfs, tmpfile)
transfers_filename = output_basename + ".transfers.csv"
with util.create_file(transfers_filename, tmpdir=True,
keepext=True) as tmpfile:
write_walk_transfer_edges(gtfs, tmpfile)
temporal_network_filename = output_basename + ".temporal_network.csv"
with util.create_file(temporal_network_filename,
tmpdir=True,
keepext=True) as tmpfile:
write_temporal_network(gtfs, tmpfile)
else:
print("Unrecognized command: %s" % args.cmd)
exit(1)
def write_walk_transfer_edges(gtfs, output_file_name):
"""
Parameters
----------
gtfs: gtfspy.GTFS
output_file_name: str
"""
transfers = gtfs.get_table("stop_distances")
transfers.drop([u"min_transfer_time", u"timed_transfer"], 1, inplace=True)
with util.create_file(output_file_name, tmpdir=True, keepext=True) as tmpfile:
transfers.to_csv(tmpfile, encoding='utf-8', index=False)
def write_nodes(gtfs, output, fields=None):
"""
Parameters
----------
gtfs: gtfspy.GTFS
output: str
Path to the output file
fields: list, optional
which pieces of information to provide
"""
nodes = gtfs.get_table("stops")
if fields is not None:
nodes = nodes[fields]
with util.create_file(output, tmpdir=True, keepext=True) as tmpfile:
nodes.to_csv(tmpfile, encoding='utf-8', index=False, sep=";")
def write_stops_geojson(gtfs, out_file, fields=None):
"""
Parameters
----------
gtfs: gtfspy.GTFS
out_file: file-like or path to file
fields: dict
simultaneously map each original_name to the new_name
Returns
-------
"""
geojson = create_stops_geojson_dict(gtfs, fields)
if hasattr(out_file, "write"):
out_file.write(json.dumps(geojson))
else:
with util.create_file(out_file, tmpdir=True, keepext=True) as tmpfile_path:
tmpfile = open(tmpfile_path, 'w')
tmpfile.write(json.dumps(geojson))
def create_filtered_copy(self):
# this with statement
# is used to ensure that no corrupted/uncompleted files get created in case of problems
with util.create_file(self.copy_db_path) as tempfile:
logging.info("copying database")
shutil.copy(self.this_db_path, tempfile)
self.copy_db_conn = sqlite3.connect(tempfile)
assert isinstance(self.copy_db_conn, sqlite3.Connection)
self._delete_rows_by_start_and_end_date()
if self.copy_db_conn.execute(
'SELECT count(*) FROM days').fetchone() == (0, ):
raise ValueError('No data left after filtering')
self._filter_by_calendar()
self._filter_by_agency()
self._filter_by_area()
if self.update_metadata:
self._update_metadata()
return
def main():
import argparse
parser = argparse.ArgumentParser(description="""
Import GTFS files. Imports gtfs. There are two subcommands. The
'import' subcommand converts from a GTFS directory or zipfile to a
sqlite database. Both must be specified on the command line. The
import-auto subcommand is older, and uses the logic in code/db.py to
automatically find databases and output files (in scratch/gtfs and
scratch/db) based on the shortname given on the command line. This
should probably not be used much anymore, instead automate that before
calling this program.""")
subparsers = parser.add_subparsers(dest='cmd')
# parsing import
parser_import = subparsers.add_parser('import',
help="Direct import GTFS->sqlite")
parser_import.add_argument('gtfs',
help='Input GTFS filename (dir or .zip)')
parser_import.add_argument(
'output', help='Output .sqlite filename (must end in .sqlite)')
parser.add_argument('--fast',
action='store_true',
help='Skip stop_times and shapes tables.')
# parsing import-auto
parser_importauto = subparsers.add_parser(
'import-auto', help="Automatic GTFS import from files")
parser_importauto.add_argument('gtfsname', help='Input GTFS filename')
# parsing import-multiple
parser_import_multiple = subparsers.add_parser(
'import-multiple', help="GTFS import from multiple zip-files")
parser_import_multiple.add_argument('zipfiles',
metavar='zipfiles',
type=str,
nargs=argparse.ONE_OR_MORE,
help='zipfiles for the import')
parser_import_multiple.add_argument(
'output', help='Output .sqlite filename (must end in .sqlite)')
# parsing import-list
# Parsing copy
parser_copy = subparsers.add_parser('copy', help="Copy database")
parser_copy.add_argument('source', help='Input GTFS .sqlite')
parser_copy.add_argument('dest', help='Output GTFS .sqlite')
parser_copy.add_argument('--start', help='Start copy time (inclusive)')
parser_copy.add_argument('--end', help='Start copy time (exclusive)')
# Parsing copy
parser_copy = subparsers.add_parser('make-views', help="Re-create views")
parser_copy.add_argument('gtfs', help='Input GTFS .sqlite')
# make-weekly-download
parser_copy = subparsers.add_parser('make-weekly')
parser_copy.add_argument('source', help='Input GTFS .sqlite')
parser_copy.add_argument('dest', help='Output GTFS .sqlite')
parser_copy = subparsers.add_parser('make-daily')
parser_copy.add_argument('source', help='Input GTFS .sqlite')
parser_copy.add_argument('dest', help='Output GTFS .sqlite')
# Export stop distances
parser_copy = subparsers.add_parser('export-stop-distances')
parser_copy.add_argument('gtfs', help='Input GTFS .sqlite')
parser_copy.add_argument('output', help='Output for .txt file')
# Custom stuff
parser_copy = subparsers.add_parser('custom')
parser_copy.add_argument('gtfs', help='Input GTFS .sqlite')
args = parser.parse_args()
if args.fast:
ignore_tables.update(('stop_times', 'shapes'))
# if the first argument is import, import a GTFS directory to a .sqlite database.
# Both directory and
if args.cmd == 'import':
gtfs = args.gtfs
output = args.output
# This context manager makes a tmpfile for import. If there
# is corruption during import, it won't leave a incomplete or
# corrupt file where it will be noticed.
with util.create_file(output, tmpdir=True, keepext=True) as tmpfile:
import_gtfs(gtfs, output=tmpfile)
elif args.cmd == "import-multiple":
zipfiles = args.zipfiles
output = args.output
print("loaders")
with util.create_file(output, tmpdir=True, keepext=True) as tmpfile:
import_gtfs(zipfiles, output=tmpfile)
elif args.cmd == 'make-views':
main_make_views(args.gtfs)
# This is now implemented in gtfs.py, please remove the commented code
# if no one has touched this in a while.:
#
# elif args.cmd == 'make-weekly':
# G = GTFS(args.source)
# download_date = G.meta['download_date']
# d = datetime.strptime(download_date, '%Y-%m-%d').date()
# date_start = d + timedelta(7-d.isoweekday()+1) # inclusive
# date_end = d + timedelta(7-d.isoweekday()+1 + 7) # exclusive
# G.copy_and_filter(args.dest, start_date=date_start, end_date=date_end)
# elif args.cmd == 'make-daily':
# G = GTFS(args.source)
# download_date = G.meta['download_date']
# d = datetime.strptime(download_date, '%Y-%m-%d').date()
# date_start = d + timedelta(7-d.isoweekday()+1) # inclusive
# date_end = d + timedelta(7-d.isoweekday()+1 + 1) # exclusive
# G.copy_and_filter(args.dest, start_date=date_start, end_date=date_end)
elif args.cmd == 'export-stop-distances':
conn = sqlite3.connect(args.gtfs)
L = StopDistancesLoader(conn)
L.export_stop_distances(conn, open(args.output, 'w'))
elif args.cmd == 'custom':
pass
# This is designed for just testing things. This code should
# always be commented out in the VCS.
# conn = sqlite3.connect(args.gtfs)
# L = StopDistancesLoader(conn)
# L.post_import(None)
# L.export_stop_distances(conn, open('sd.txt', 'w'))
else:
print("Unrecognized command: %s" % args.cmd)
exit(1)