Beispiel #1
0
    def test_broken(self):
        exception = False
        try:
            clear_mappers()
            dao = Dao("")
            dao.load_gtfs(BROKEN_GTFS, lenient=False)
        except KeyError:
            exception = True
        self.assertTrue(exception)

        clear_mappers()
        dao = Dao("")
        dao.load_gtfs(BROKEN_GTFS, lenient=True)

        # The following are based on BROKEN GTFS content,
        # that is the entities count minus broken ones.
        self.assertTrue(len(dao.routes()) == 4)
        self.assertTrue(len(list(dao.stops())) == 12)
        self.assertTrue(len(dao.calendars()) == 2)
        self.assertTrue(len(list(dao.trips())) == 104)
        self.assertTrue(len(dao.stoptimes()) == 500)
        self.assertTrue(len(dao.fare_attributes()) == 2)
        self.assertTrue(len(dao.fare_rules()) == 4)
        # This stop has missing coordinates in the broken file
        stop00 = dao.stop('FUR_CREEK_RES3')
        self.assertAlmostEqual(stop00.stop_lat, 0.0, 5)
        self.assertAlmostEqual(stop00.stop_lon, 0.0, 5)
Beispiel #2
0
    def test_gtfs_data(self):
        dao = Dao(DAO_URL, sql_logging=SQL_LOG)
        dao.load_gtfs(MINI_GTFS)

        # Check feed
        feed = dao.feed()
        self.assertTrue(feed.feed_id == "")
        self.assertTrue(feed.feed_publisher_name is None)
        self.assertTrue(feed.feed_publisher_url is None)
        self.assertTrue(feed.feed_contact_email is None)
        self.assertTrue(feed.feed_contact_url is None)
        self.assertTrue(feed.feed_start_date is None)
        self.assertTrue(feed.feed_end_date is None)
        self.assertTrue(len(dao.agencies()) == 1)
        self.assertTrue(len(dao.routes()) == 1)
        self.assertTrue(len(feed.agencies) == 1)
        self.assertTrue(len(feed.routes) == 1)

        # Check if optional route agency is set
        a = dao.agency("A")
        self.assertTrue(a is not None)
        self.assertTrue(len(a.routes) == 1)

        # Check for frequency-generated trips
        # They should all have the same delta
        trips = dao.trips(fltr=(Trip.frequency_generated == True),
                          prefetch_stop_times=True)
        n_trips = 0
        deltas = {}
        for trip in trips:
            original_trip_id = trip.trip_id.rsplit('@', 1)[0]
            delta1 = []
            for st1, st2 in trip.hops():
                delta1.append(st2.arrival_time - st1.departure_time)
            delta2 = deltas.get(original_trip_id)
            if delta2 is not None:
                self.assertTrue(delta1 == delta2)
            else:
                deltas[original_trip_id] = delta1
            n_trips += 1
        self.assertTrue(n_trips == 8)
Beispiel #3
0
    def test_non_overlapping_feeds(self):
        dao = Dao(DAO_URL, sql_logging=SQL_LOG)
        # Load twice the same data under two distinct namespaces
        dao.load_gtfs(DUMMY_GTFS, feed_id='A')
        dao.load_gtfs(DUMMY_GTFS, feed_id='B')

        # Check that each feed only return it's own data
        feed_a = dao.feed('A')
        self.assertTrue(feed_a.feed_id == 'A')
        feed_b = dao.feed('B')
        self.assertTrue(feed_b.feed_id == 'B')
        self.assertTrue(len(dao.agencies()) == 4)
        self.assertTrue(len(feed_a.agencies) == 2)
        self.assertTrue(len(feed_b.agencies) == 2)
        self.assertTrue(len(feed_a.routes) * 2 == len(dao.routes()))
        self.assertTrue(len(feed_b.routes) * 2 == len(dao.routes()))
        self.assertTrue(len(feed_a.stops) * 2 == len(list(dao.stops())))
        self.assertTrue(len(feed_b.stops) * 2 == len(list(dao.stops())))
        self.assertTrue(len(feed_a.calendars) * 2 == len(dao.calendars()))
        self.assertTrue(len(feed_b.calendars) * 2 == len(dao.calendars()))
        self.assertTrue(len(feed_a.trips) * 2 == len(list(dao.trips())))
        self.assertTrue(len(feed_b.trips) * 2 == len(list(dao.trips())))
Beispiel #4
0
class TestProviders(unittest.TestCase):
    def setUp(self):
        unittest.TestCase.setUp(self)
        clear_mappers()
        self.dao_object = Dao()

    @mock.patch("exporter.api.radcom.Request")
    def test_radcom_provider(self, mocked_request):
        mocked_request.side_effect = _request_mock
        radcomProvider = RadcomApiDataProvider("", feed_id="1")
        assert radcomProvider is not None

        # load data from provider
        load_result = radcomProvider.load_data_source(self.dao_object)
        assert load_result is True

        # check calendars
        assert len(self.dao_object.calendars()) == 2

        # check routes
        routes_len = 0
        for route in self.dao_object.routes():
            routes_len = routes_len + 1
            assert route.route_id == '43'
            assert route.route_short_name == '1'
            assert route.route_long_name == 'Bulevardul Tomis - Sere (C.L.)'
        assert routes_len == 1

        # check stops
        stops_len = 0
        for stop in self.dao_object.stops():
            stops_len = stops_len + 1
            assert len(stop.stop_times) > 0
            assert stop.stop_lat != 0.0
            assert stop.stop_lon != 0.0
        assert stops_len == 35

        # check trips
        trips_len = 0
        for trip in self.dao_object.trips():
            trips_len = trips_len + 1
            assert trip.shape != None
            expected_shape_id = "shp1_{0}_{1}".format(trip.route_id,
                                                      trip.direction_id)
            assert trip.shape_id == expected_shape_id
        assert trips_len == 14

        # cleanup
        self.dao_object.delete_feed('1')
        self.dao_object.flush()

    def test_brasov_export(self):
        provider = BrasovApiDataProvider("2")
        provider.dao = self.dao_object

        provider._load_agencies()
        assert provider.agency_id is not None
        assert len(self.dao_object.agencies()) == 1

        provider._load_service()
        assert len(self.dao_object.calendars()) == 2

        assert len(provider.routes) == 0
        provider._load_routes()
        assert len(provider.routes) > 0
        assert len(self.dao_object.routes()) > 0

        assert len(provider.stop_map) == 0
        provider._load_schedule()
        assert len(provider.stop_map) > 0
        assert len(list(self.dao_object.stops())) > 0
        assert len(list(self.dao_object.trips())) > 0

        # stops_visited_by_route_trips = len(list(dao_fixture.stops()))
        # provider._load_stops()
        # assert len(list(dao_fixture.stops())) == stops_visited_by_route_trips

        # cleanup
        self.dao_object.delete_feed('2')
        self.dao_object.flush()
Beispiel #5
0
    def test_gtfs_data(self):
        dao = Dao(DAO_URL, sql_logging=False)
        dao.load_gtfs(DUMMY_GTFS)

        # Check feed
        feed = dao.feed()
        self.assertTrue(feed.feed_id == "")
        self.assertTrue(feed.feed_publisher_name == "Mecatran")
        self.assertTrue(feed.feed_publisher_url == "http://www.mecatran.com/")
        self.assertTrue(feed.feed_contact_email == "*****@*****.**")
        self.assertTrue(feed.feed_lang == "fr")
        self.assertTrue(len(dao.agencies()) == 2)
        self.assertTrue(len(dao.routes()) == 3)
        self.assertTrue(len(feed.agencies) == 2)
        self.assertTrue(len(feed.routes) == 3)

        # Check agencies
        at = dao.agency("AT")
        self.assertTrue(at.agency_name == "Agency Train")
        self.assertTrue(len(at.routes) == 1)
        ab = dao.agency("AB")
        self.assertTrue(ab.agency_name == "Agency Bus")
        self.assertTrue(len(ab.routes) == 2)

        # Check calendars
        week = dao.calendar("WEEK")
        self.assertTrue(len(week.dates) == 253)
        summer = dao.calendar("SUMMER")
        self.assertTrue(len(summer.dates) == 42)
        mon = dao.calendar("MONDAY")
        self.assertTrue(len(mon.dates) == 49)
        sat = dao.calendar("SAT")
        self.assertTrue(len(sat.dates) == 53)
        for date in mon.dates:
            self.assertTrue(date.dow() == 0)
        for date in sat.dates:
            self.assertTrue(date.dow() == 5)
        for date in week.dates:
            self.assertTrue(date.dow() >= 0 and date.dow() <= 4)
        for date in summer.dates:
            self.assertTrue(date >= CalendarDate.ymd(2016, 7, 1)
                            and date <= CalendarDate.ymd(2016, 8, 31))
        empty = dao.calendars(
            func.date(CalendarDate.date) == datetime.date(2016, 5, 1))
        # OR USE: empty = dao.calendars(CalendarDate.date == "2016-05-01")
        self.assertTrue(len(empty) == 0)
        july4 = CalendarDate.ymd(2016, 7, 4)
        summer_mon = dao.calendars(func.date(CalendarDate.date) == july4.date)
        n = 0
        for cal in summer_mon:
            self.assertTrue(july4 in cal.dates)
            n += 1
        self.assertTrue(n == 3)

        # Check stops
        sbq = dao.stop("BQ")
        self.assertAlmostEqual(sbq.stop_lat, 44.844, places=2)
        self.assertAlmostEqual(sbq.stop_lon, -0.573, places=2)
        self.assertTrue(sbq.stop_name == "Bordeaux Quinconces")
        n = 0
        for stop in dao.stops(Stop.stop_name.like("Gare%")):
            self.assertTrue(stop.stop_name.startswith("Gare"))
            n += 1
        self.assertTrue(n == 7)
        n = 0
        for stop in dao.stops(
                fltr=dao.in_area(RectangularArea(44.7, -0.6, 44.9, -0.4))):
            self.assertTrue(stop.stop_lat >= 44.7 and stop.stop_lat <= 44.9
                            and stop.stop_lon >= -0.6
                            and stop.stop_lon <= -0.4)
            n += 1
        self.assertTrue(n == 16)
        for station in dao.stops(Stop.location_type == Stop.TYPE_STATION):
            self.assertTrue(station.location_type == Stop.TYPE_STATION)
            self.assertTrue(len(station.sub_stops) >= 2)
            for stop in station.sub_stops:
                self.assertTrue(stop.parent_station == station)

        # Check zones
        z_inexistant = dao.zone("ZX")
        self.assertTrue(z_inexistant is None)
        z1 = dao.zone("Z1")
        self.assertEqual(16, len(z1.stops))
        z2 = dao.zone("Z2")
        self.assertEqual(4, len(z2.stops))

        # Check transfers
        transfers = dao.transfers()
        self.assertTrue(len(transfers) == 3)
        transfers = dao.transfers(
            fltr=(dao.transfer_from_stop().stop_id == 'GBSJB'))
        self.assertTrue(len(transfers) == 1)
        self.assertTrue(transfers[0].from_stop.stop_id == 'GBSJB')

        # Check routes
        tgv = dao.route("TGVBP")
        self.assertTrue(tgv.agency == at)
        self.assertTrue(tgv.route_type == 2)
        r1 = dao.route("BR")
        self.assertTrue(r1.route_short_name == "R1")
        self.assertTrue(r1.route_long_name == "Bus Red")
        n = 0
        for route in dao.routes(Route.route_type == 3):
            self.assertTrue(route.route_type == 3)
            n += 1
        self.assertTrue(n == 2)

        # Check trip for route
        n = 0
        trips = dao.trips(fltr=Route.route_type == Route.TYPE_BUS)
        for trip in trips:
            self.assertTrue(trip.route.route_type == Route.TYPE_BUS)
            n += 1
        self.assertTrue(n > 20)

        # Check trips on date
        trips = dao.trips(fltr=func.date(CalendarDate.date) == july4.date,
                          prefetch_calendars=True)
        n = 0
        for trip in trips:
            self.assertTrue(july4 in trip.calendar.dates)
            n += 1
        self.assertTrue(n > 30)
Beispiel #6
0
    def test_complex_queries(self):
        dao = Dao(DAO_URL, sql_logging=SQL_LOG)
        dao.load_gtfs(DUMMY_GTFS)

        # Get the list of departures:
        # 1) from "Porte de Bourgogne"
        # 2) on 4th July
        # 3) between 10:00 and 14:00
        # 4) on route type BUS
        # 5) not the last of trip (only departing)
        porte_bourgogne = dao.stop("BBG")
        july4 = CalendarDate.ymd(2016, 7, 4)
        from_time = gtfstime(10, 00)
        to_time = gtfstime(14, 00)
        departures = dao.stoptimes(
            fltr=(StopTime.stop == porte_bourgogne) &
            (StopTime.departure_time >= from_time) &
            (StopTime.departure_time <= to_time)
            & (Route.route_type == Route.TYPE_BUS) &
            (func.date(CalendarDate.date) == july4.date),
            prefetch_trips=True)

        n = 0
        for dep in departures:
            self.assertTrue(dep.stop == porte_bourgogne)
            self.assertTrue(july4 in dep.trip.calendar.dates)
            self.assertTrue(dep.trip.route.route_type == Route.TYPE_BUS)
            self.assertTrue(dep.departure_time >= from_time
                            and dep.departure_time <= to_time)
            n += 1
        self.assertTrue(n > 10)

        # Plage is a stop that is used only in summer (hence the name!)
        plage = dao.stop("BPG")
        # Get the list of stops used by some route:
        # 1) All-year round
        route_red = dao.route("BR")
        stoplist_all = list(dao.stops(fltr=Trip.route == route_red))
        # 2) Only in january
        from_date = CalendarDate.ymd(2016, 1, 1)
        to_date = CalendarDate.ymd(2016, 1, 31)
        stoplist_jan = list(
            dao.stops(fltr=(Trip.route == route_red)
                      & (func.date(CalendarDate.date) >= from_date.date)
                      & (func.date(CalendarDate.date) <= to_date.date)))
        # Now, make some tests
        self.assertTrue(len(stoplist_all) > 5)
        self.assertTrue(plage in stoplist_all)
        self.assertFalse(plage in stoplist_jan)
        stoplist = list(stoplist_all)
        stoplist.remove(plage)
        self.assertTrue(set(stoplist) == set(stoplist_jan))

        # Get all routes passing by the set of stops
        routes = dao.routes(fltr=or_(StopTime.stop == stop
                                     for stop in stoplist_jan))
        stopset = set()
        for route in routes:
            for trip in route.trips:
                for stoptime in trip.stop_times:
                    stopset.add(stoptime.stop)
        self.assertTrue(set(stoplist_jan).issubset(stopset))
Beispiel #7
0
    def test_demo(self):
        dao = Dao(DAO_URL, sql_logging=False)
        dao.load_gtfs(DUMMY_GTFS)

        print("List of stops named '...Bordeaux...':")
        stops_bordeaux = list(
            dao.stops(fltr=(Stop.stop_name.ilike('%Bordeaux%'))
                      & (Stop.location_type == Stop.TYPE_STOP)))
        for stop in stops_bordeaux:
            print(stop.stop_name)

        print("List of routes passing by those stops:")
        routes_bordeaux = dao.routes(fltr=or_(StopTime.stop == stop
                                              for stop in stops_bordeaux))
        for route in routes_bordeaux:
            print("%s - %s" % (route.route_short_name, route.route_long_name))

        july4 = CalendarDate.ymd(2016, 7, 4)
        print("All departures from those stops on %s:" % (july4.as_date()))
        departures = list(
            dao.stoptimes(fltr=(or_(StopTime.stop == stop
                                    for stop in stops_bordeaux))
                          & (StopTime.departure_time != None)
                          & (func.date(CalendarDate.date) == july4.date)))
        print("There is %d departures" % (len(departures)))
        for departure in departures:
            print("%30.30s %10.10s %-20.20s > %s" %
                  (departure.stop.stop_name, fmttime(departure.departure_time),
                   departure.trip.route.route_long_name,
                   departure.trip.trip_headsign))

        print("Number of departures and time range per stop on %s:" %
              (july4.as_date()))
        departure_by_stop = defaultdict(list)
        for departure in departures:
            departure_by_stop[departure.stop].append(departure)
        for stop, deps in departure_by_stop.items():
            min_dep = min(d.departure_time for d in deps)
            max_dep = max(d.departure_time for d in deps)
            print("%30.30s %3d departures (from %s to %s)" %
                  (stop.stop_name, len(deps), fmttime(min_dep),
                   fmttime(max_dep)))

        # Compute the average distance and time to next stop by route type
        ntd = [[0, 0, 0.0] for type in range(0, Route.TYPE_FUNICULAR + 1)]
        for departure in departures:
            # The following is guaranteed to succeed as we have departure_time == Null for last stop time in trip
            next_arrival = departure.trip.stop_times[departure.stop_sequence +
                                                     1]
            hop_dist = next_arrival.shape_dist_traveled - departure.shape_dist_traveled
            hop_time = next_arrival.arrival_time - departure.departure_time
            route_type = departure.trip.route.route_type
            ntd[route_type][0] += 1
            ntd[route_type][1] += hop_time
            ntd[route_type][2] += hop_dist
        for route_type in range(0, len(ntd)):
            n, t, d = ntd[route_type]
            if n > 0:
                print(
                    "The average distance to the next stop on those departures for route type %d is %.2f meters"
                    % (route_type, d / n))
                print(
                    "The average time in sec to the next stop on those departures for route type %d is %s"
                    % (route_type, fmttime(t / n)))
Beispiel #8
0
class Exporter:
    def __init__(self, arguments):
        self.logger = logging.getLogger('gtfsexporter')
        self._arguments = arguments

        if arguments['--id'] is None:
            arguments['--id'] = "default"

        database_path = os.path.join(__cwd_path__,
                                     arguments['--id'] + ".sqlite")

        self._dao = Dao(database_path,
                        sql_logging=arguments['--logsql'],
                        schema=arguments['--schema'])

        if arguments['--list']:
            for feed in self._dao.feeds():
                print(feed.feed_id if feed.feed_id != "" else "(default)")

        if arguments['--delete']:
            feed_id = arguments['--id']
            existing_feed = self._dao.feed(feed_id)
            if existing_feed:
                self.logger.warning("Deleting existing feed ID '%s'" % feed_id)
                self._dao.delete_feed(feed_id)
                self._dao.commit()

    @property
    def dao(self):
        return self._dao

    def load(self, provider: DataProvider):
        self.logger.info("Importing data from provided source")
        provider.load_data_source(self._dao)

    def process(self, processor: Processor = None):
        self.logger.info("Processing data from provided source")

        # Here we should use a rule processor to have more flexibility when processing data
        # processor.process(ruleset)

        for route in self._dao.routes():
            print("updating route [%s] setting correct color" %
                  route.route_long_name)

            route.route_text_color = "FFFFFF"

            if route.route_type == Route.TYPE_BUS:
                route.route_color = "195BAD"
            elif route.route_type == Route.TYPE_TRAM:
                route.route_color = "FFAD33"
            elif route.route_type == Route.TYPE_RAIL:
                route.route_color = "FF5B33"
            elif route.route_type == Route.TYPE_CABLECAR:
                route.route_color = "FF8433"
            elif route.route_type == Route.TYPE_SUBWAY:
                route.route_color = "D13333"
            elif route.route_type == Route.TYPE_FERRY:
                route.route_color = "62A9DD"

        self._dao.commit()

    def export(self, bundle=False, out_path: str = __output_path__) -> str:
        self.logger.info(
            f"Generating archive with name gtfs-{self._arguments['--id']}.zip")

        class __Args:
            filter = None

        context = Context(self._dao, __Args(), out_path)

        if bundle:
            w = Writer(context, bundle=f"gtfs-{self._arguments['--id']}.zip")
        else:
            w = Writer(context)

        w.run()

        return out_path