def test_all_gtfs(self):
if not ENABLE:
print("This test is disabled as it is very time-consuming.")
print("If you want to enable it, please see in the code.")
return
# Create temporary directory if not there
if not os.path.isdir(DIR):
os.mkdir(DIR)
# Create a DAO. Re-use any existing present.
logging.basicConfig(level=logging.INFO)
dao = Dao("%s/all_gtfs.sqlite" % (DIR))
deids = IDS_TO_LOAD
if deids is None:
print("Downloading meta-info for all agencies...")
resource_url = "http://www.gtfs-data-exchange.com/api/agencies?format=json"
response = requests.get(resource_url).json()
if response.get('status_code') != 200:
raise IOError()
deids = []
for entry in response.get('data'):
deid = entry.get('dataexchange_id')
deids.append(deid)
# Randomize the list, otherwise we will always load ABCBus, then ...
random.shuffle(deids)
for deid in deids:
try:
local_filename = "%s/%s.gtfs.zip" % (DIR, deid)
if os.path.exists(local_filename) and SKIP_EXISTING:
print("Skipping [%s], GTFS already present." % (deid))
continue
print("Downloading meta-info for ID [%s]" % (deid))
resource_url = "http://www.gtfs-data-exchange.com/api/agency?agency=%s&format=json" % deid
response = requests.get(resource_url).json()
status_code = response.get('status_code')
if status_code != 200:
raise IOError("Error %d (%s)" %
(status_code, response.get('status_txt')))
data = response.get('data')
agency_data = data.get('agency')
agency_name = agency_data.get('name')
agency_area = agency_data.get('area')
agency_country = agency_data.get('country')
print("Processing [%s] %s (%s / %s)" %
(deid, agency_name, agency_country, agency_area))
date_max = 0.0
file_url = None
file_size = 0
file_md5 = None
for datafile in data.get('datafiles'):
date_added = datafile.get('date_added')
if date_added > date_max:
date_max = date_added
file_url = datafile.get('file_url')
file_size = datafile.get('size')
file_md5 = datafile.get('md5sum')
if file_url is None:
print("No datafile available, skipping.")
continue
if file_size > MAX_GTFS_SIZE:
print("GTFS too large (%d bytes > max %d), skipping." %
(file_size, MAX_GTFS_SIZE))
continue
# Check if the file is present and do not download it.
try:
existing_md5 = hashlib.md5(
open(local_filename, 'rb').read()).hexdigest()
except:
existing_md5 = None
if existing_md5 == file_md5:
print("Using existing file '%s': MD5 checksum matches." %
(local_filename))
else:
print("Downloading file '%s' to '%s' (%d bytes)" %
(file_url, local_filename, file_size))
with open(local_filename, 'wb') as local_file:
cnx = requests.get(file_url, stream=True)
for block in cnx.iter_content(1024):
local_file.write(block)
cnx.close()
feed = dao.feed(deid)
if feed is not None:
print("Removing existing data for feed [%s]" % (deid))
dao.delete_feed(deid)
print("Importing into DAO as ID [%s]" % (deid))
try:
dao.load_gtfs("%s/%s.gtfs.zip" % (DIR, deid), feed_id=deid)
except:
error_filename = "%s/%s.error" % (DIR, deid)
print("Import of [%s]: FAILED. Logging error to '%s'" %
(deid, error_filename))
with open(error_filename, 'wb') as errfile:
errfile.write(traceback.format_exc())
raise
print("Import of [%s]: OK." % (deid))
except Exception as error:
logging.exception(error)
continue
class TestProviders(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
clear_mappers()
self.dao_object = Dao()
@mock.patch("exporter.api.radcom.Request")
def test_radcom_provider(self, mocked_request):
mocked_request.side_effect = _request_mock
radcomProvider = RadcomApiDataProvider("", feed_id="1")
assert radcomProvider is not None
# load data from provider
load_result = radcomProvider.load_data_source(self.dao_object)
assert load_result is True
# check calendars
assert len(self.dao_object.calendars()) == 2
# check routes
routes_len = 0
for route in self.dao_object.routes():
routes_len = routes_len + 1
assert route.route_id == '43'
assert route.route_short_name == '1'
assert route.route_long_name == 'Bulevardul Tomis - Sere (C.L.)'
assert routes_len == 1
# check stops
stops_len = 0
for stop in self.dao_object.stops():
stops_len = stops_len + 1
assert len(stop.stop_times) > 0
assert stop.stop_lat != 0.0
assert stop.stop_lon != 0.0
assert stops_len == 35
# check trips
trips_len = 0
for trip in self.dao_object.trips():
trips_len = trips_len + 1
assert trip.shape != None
expected_shape_id = "shp1_{0}_{1}".format(trip.route_id,
trip.direction_id)
assert trip.shape_id == expected_shape_id
assert trips_len == 14
# cleanup
self.dao_object.delete_feed('1')
self.dao_object.flush()
def test_brasov_export(self):
provider = BrasovApiDataProvider("2")
provider.dao = self.dao_object
provider._load_agencies()
assert provider.agency_id is not None
assert len(self.dao_object.agencies()) == 1
provider._load_service()
assert len(self.dao_object.calendars()) == 2
assert len(provider.routes) == 0
provider._load_routes()
assert len(provider.routes) > 0
assert len(self.dao_object.routes()) > 0
assert len(provider.stop_map) == 0
provider._load_schedule()
assert len(provider.stop_map) > 0
assert len(list(self.dao_object.stops())) > 0
assert len(list(self.dao_object.trips())) > 0
# stops_visited_by_route_trips = len(list(dao_fixture.stops()))
# provider._load_stops()
# assert len(list(dao_fixture.stops())) == stops_visited_by_route_trips
# cleanup
self.dao_object.delete_feed('2')
self.dao_object.flush()
class Exporter:
def __init__(self, arguments):
self.logger = logging.getLogger('gtfsexporter')
self._arguments = arguments
if arguments['--id'] is None:
arguments['--id'] = "default"
database_path = os.path.join(__cwd_path__,
arguments['--id'] + ".sqlite")
self._dao = Dao(database_path,
sql_logging=arguments['--logsql'],
schema=arguments['--schema'])
if arguments['--list']:
for feed in self._dao.feeds():
print(feed.feed_id if feed.feed_id != "" else "(default)")
if arguments['--delete']:
feed_id = arguments['--id']
existing_feed = self._dao.feed(feed_id)
if existing_feed:
self.logger.warning("Deleting existing feed ID '%s'" % feed_id)
self._dao.delete_feed(feed_id)
self._dao.commit()
@property
def dao(self):
return self._dao
def load(self, provider: DataProvider):
self.logger.info("Importing data from provided source")
provider.load_data_source(self._dao)
def process(self, processor: Processor = None):
self.logger.info("Processing data from provided source")
# Here we should use a rule processor to have more flexibility when processing data
# processor.process(ruleset)
for route in self._dao.routes():
print("updating route [%s] setting correct color" %
route.route_long_name)
route.route_text_color = "FFFFFF"
if route.route_type == Route.TYPE_BUS:
route.route_color = "195BAD"
elif route.route_type == Route.TYPE_TRAM:
route.route_color = "FFAD33"
elif route.route_type == Route.TYPE_RAIL:
route.route_color = "FF5B33"
elif route.route_type == Route.TYPE_CABLECAR:
route.route_color = "FF8433"
elif route.route_type == Route.TYPE_SUBWAY:
route.route_color = "D13333"
elif route.route_type == Route.TYPE_FERRY:
route.route_color = "62A9DD"
self._dao.commit()
def export(self, bundle=False, out_path: str = __output_path__) -> str:
self.logger.info(
f"Generating archive with name gtfs-{self._arguments['--id']}.zip")
class __Args:
filter = None
context = Context(self._dao, __Args(), out_path)
if bundle:
w = Writer(context, bundle=f"gtfs-{self._arguments['--id']}.zip")
else:
w = Writer(context)
w.run()
return out_path