def handle_archive(self, archive_name):
self.route = None
self.trip = None
self.routes = {}
self.calendars = {}
self.stop_times = []
self.notes = []
if 'ulb' in archive_name.lower():
source_name = 'ULB'
else:
source_name = 'MET'
self.source, source_created = DataSource.objects.get_or_create(name=source_name)
self.source.datetime = timezone.localtime()
with zipfile.ZipFile(archive_name) as archive:
for filename in archive.namelist():
if filename.endswith('.cif'):
with archive.open(filename) as open_file:
self.handle_file(open_file)
assert self.stop_times == []
for route in self.routes.values():
groupings = get_stop_usages(route.trip_set.all())
route.service.stops.clear()
stop_usages = [
StopUsage(service=route.service, stop_id=stop_time.stop_id, timing_status=stop_time.timing_status,
direction='outbound', order=i)
for i, stop_time in enumerate(groupings[0])
] + [
StopUsage(service=route.service, stop_id=stop_time.stop_id, timing_status=stop_time.timing_status,
direction='inbound', order=i)
for i, stop_time in enumerate(groupings[1])
]
StopUsage.objects.bulk_create(stop_usages)
# self.stops doesn't contain all stops, and has latlongs in the Irish Grid projection
stops = StopPoint.objects.in_bulk(stop_usage.stop_id for stop_usage in stop_usages)
line_strings = []
for pattern in get_journey_patterns(route.trip_set.all()):
points = (stops[stop_code].latlong for stop_code in pattern if stop_code in stops)
line_strings.append(LineString(*points))
route.service.geometry = MultiLineString(*line_strings)
services = {route.service.id: route.service for route in self.routes.values()}.values()
Service.objects.bulk_update(services,
fields=['geometry', 'description', 'outbound_description', 'inbound_description'])
for service in services:
service.update_search_vector()
self.source.route_set.exclude(code__in=self.routes.keys()).delete()
self.source.service_set.filter(current=True).exclude(service_code__in=self.routes.keys()).update(current=False)
self.source.save(update_fields=['datetime'])
def handle_stop(cls, line):
record_identity = line[:2]
atco_code = line[2:14]
if atco_code not in cls.services[cls.service_code][cls.direction]:
if not StopPoint.objects.filter(atco_code=atco_code).exists():
if not atco_code.startswith('7'):
print(atco_code)
return
cls.deferred_stop_codes.append(atco_code)
if record_identity == 'QI':
timing_status = line[26:28]
order = 1
else:
timing_status = line[21:23]
if record_identity == 'QO':
order = 0
else:
order = 2
stop_usage = StopUsage(
service_id=cls.service_code,
stop_id=atco_code,
direction=('Outbound' if cls.direction == 'O' else 'Inbound'),
timing_status=('PTP' if timing_status == 'T1' else 'OTH'),
order=order)
cls.services[cls.service_code][
cls.direction][atco_code] = stop_usage
cls.stop_usages.append(stop_usage)
def handle_service(self, filename, transxchange, txc_service, today,
stops):
if txc_service.operating_period.end:
if txc_service.operating_period.end < today:
print(filename, txc_service.operating_period.end)
return
elif txc_service.operating_period.end < txc_service.operating_period.start:
return
operators = self.get_operators(transxchange, txc_service)
if not operators:
basename = os.path.basename(filename) # e.g. 'KCTB_'
if basename[4] == '_':
maybe_operator_code = basename[:4]
if maybe_operator_code.isupper(
) and maybe_operator_code.isalpha():
try:
operators = [
Operator.objects.get(id=maybe_operator_code)
]
except Operator.DoesNotExist:
pass
if self.is_tnds() and self.source.name != 'L':
if operators and all(operator.id in self.open_data_operators
for operator in operators):
return
linked_services = []
description = self.get_description(txc_service)
if description == 'Origin - Destination':
description = ''
if re.match(r'^P[BCDFGHKM]\d+:\d+.*.$', txc_service.service_code):
unique_service_code = txc_service.service_code
else:
unique_service_code = None
for line in txc_service.lines:
existing = None
service_code = None
if unique_service_code:
# first try getting by BODS profile compliant service code
existing = Service.objects.filter(
service_code=unique_service_code,
line_name__iexact=line.line_name).order_by(
'-current', 'id').first()
if not existing and operators and line.line_name:
if self.source.name in {'Go South West', 'Oxford Bus Company'}:
assert operators[0].parent
existing = Service.objects.filter(
operator__parent=operators[0].parent)
if self.source.name == 'Oxford Bus Company':
if txc_service.service_code.startswith('T'):
operators = Operator.objects.filter(id='THTR')
elif txc_service.service_code.startswith('C'):
operators = Operator.objects.filter(id='CSLB')
elif self.source.name == 'Go South West':
if txc_service.service_code.startswith('GC'):
operators = Operator.objects.filter(id='TFCN')
elif all(operator.parent == 'Go South Coast'
for operator in operators):
existing = Service.objects.filter(
operator__parent='Go South Coast')
elif self.source.name.startswith('Stagecoach'):
existing = Service.objects.filter(
Q(source=self.source) | Q(operator__in=operators))
if description:
existing = existing.filter(description=description)
else:
existing = Service.objects.filter(operator__in=operators)
if len(transxchange.services) == 1:
has_stop_time = Exists(
StopTime.objects.filter(
stop__in=stops,
trip__route__service=OuterRef('id')))
has_stop_usage = Exists(
StopUsage.objects.filter(stop__in=stops,
service=OuterRef('id')))
has_no_route = ~Exists(
Route.objects.filter(service=OuterRef('id')))
existing = existing.filter(has_stop_time
| (has_stop_usage
& has_no_route))
elif len(txc_service.lines) == 1:
existing = existing.filter(
Exists(
Route.objects.filter(
service_code=txc_service.service_code,
service=OuterRef('id'))))
elif description:
existing = existing.filter(description=description)
existing = existing.filter(
line_name__iexact=line.line_name).order_by(
'-current', 'id').first()
if self.is_tnds():
if self.should_defer_to_other_source(operators,
line.line_name):
continue
service_code = get_service_code(filename)
if service_code is None:
service_code = txc_service.service_code
if not existing:
# assume service code is at least unique within a TNDS region
existing = self.source.service_set.filter(
service_code=service_code).first()
elif unique_service_code:
service_code = unique_service_code
if existing:
service = existing
else:
service = Service()
service.line_name = line.line_name
service.date = today
service.current = True
service.source = self.source
service.show_timetable = True
if service_code:
service.service_code = service_code
if description:
service.description = description
line_brand = line.line_brand
if txc_service.marketing_name and txc_service.marketing_name != 'CornwallbyKernow':
line_brand = txc_service.marketing_name
if line.line_name in line_brand:
line_brand_parts = line_brand.split()
if line.line_name in line_brand_parts:
line_brand_parts.remove(line.line_name)
line_brand = ' '.join(line_brand_parts)
print(line_brand)
if line_brand:
service.line_brand = line_brand
if txc_service.mode:
service.mode = txc_service.mode
if self.region_id:
service.region_id = self.region_id
if self.service_descriptions: # NCSD
service.outbound_description, service.inbound_description = self.get_service_descriptions(
filename)
service.description = service.outbound_description or service.inbound_description
if service.id:
service_created = False
else:
service_created = True
service.save()
if not service_created:
if '_' in service.slug or '-' not in service.slug or existing and not existing.current:
service.slug = ''
service.save(update_fields=['slug'])
if operators:
if service_created:
service.operator.set(operators)
else:
if self.source.name in {
'Oxford Bus Company', 'Go South West'
}:
pass
elif service.id in self.service_ids or all(
o.parent == 'Go South Coast' for o in operators):
service.operator.add(*operators)
else:
service.operator.set(operators)
self.service_ids.add(service.id)
linked_services.append(service.id)
journeys = transxchange.get_journeys(txc_service.service_code,
line.id)
if journeys:
journey = journeys[0]
ticket_machine_service_code = journey.ticket_machine_service_code
if ticket_machine_service_code and ticket_machine_service_code != line.line_name:
try:
ServiceCode.objects.create(
scheme='SIRI',
code=ticket_machine_service_code,
service=service)
except IntegrityError:
pass
# a code used in Traveline Cymru URLs:
if self.source.name == 'W':
private_code = journey.private_code
if private_code and ':' in private_code:
ServiceCode.objects.update_or_create(
{'code': private_code.split(':', 1)[0]},
service=service,
scheme='Traveline Cymru')
# timetable data:
route_defaults = {
'line_name': line.line_name,
'line_brand': line_brand,
'start_date': txc_service.operating_period.start,
'end_date': txc_service.operating_period.end,
'dates': txc_service.operating_period.dates(),
'service': service,
'revision_number': transxchange.attributes['RevisionNumber'],
'service_code': txc_service.service_code
}
if description:
route_defaults['description'] = description
geometry = []
if transxchange.route_sections:
patterns = {
journey.journey_pattern.id: journey.journey_pattern
for journey in journeys
}
routes = [
pattern.route_ref for pattern in patterns.values()
if pattern.route_ref
]
if routes:
routes = [
transxchange.routes[route_id]
for route_id in transxchange.routes
if route_id in routes
]
for route in routes:
for section_ref in route.route_section_refs:
section = transxchange.route_sections[section_ref]
for link in section.links:
if link.track:
geometry.append(link.track)
else:
route_links = {}
for section in transxchange.route_sections.values():
for link in section.links:
route_links[link.id] = link
for journey in journeys:
if journey.journey_pattern:
for section in journey.journey_pattern.sections:
for link in section.timinglinks:
link = route_links[link.route_link_ref]
if link.track:
geometry.append(link.track)
if geometry:
geometry = MultiLineString(geometry).simplify()
if not isinstance(geometry, MultiLineString):
geometry = MultiLineString(geometry)
route_defaults['geometry'] = geometry
route_code = filename
if len(transxchange.services) > 1:
route_code += f'#{txc_service.service_code}'
if len(txc_service.lines) > 1:
route_code += f'#{line.id}'
route, route_created = Route.objects.update_or_create(
route_defaults, source=self.source, code=route_code)
self.route_ids.add(route.id)
if not route_created:
# if 'opendata.ticketer' in self.source.url and route.service_id == service_id:
# continue
route.trip_set.all().delete()
self.handle_journeys(route, stops, journeys, txc_service, line.id)
service.stops.clear()
outbound, inbound = get_stop_usages(
Trip.objects.filter(route__service=service))
changed_fields = []
if self.source.name.startswith(
'Arriva ') or self.source.name == 'Yorkshire Tiger':
if outbound:
changed = 0
origin_stop = outbound[0].stop
destination_stop = outbound[-1].stop
if txc_service.origin in origin_stop.common_name:
if origin_stop.locality.name not in txc_service.origin:
txc_service.origin = f'{origin_stop.locality.name} {txc_service.origin}'
changed += 1
if txc_service.destination in destination_stop.common_name:
if destination_stop.locality.name not in txc_service.destination:
txc_service.destination = f'{destination_stop.locality.name} {txc_service.destination}'
changed += 1
if changed == 2:
service.description = f'{txc_service.origin} - {txc_service.destination}'
changed_fields.append('description')
stop_usages = [
StopUsage(service=service,
stop_id=stop_time.stop_id,
timing_status=stop_time.timing_status,
direction='outbound',
order=i) for i, stop_time in enumerate(outbound)
] + [
StopUsage(service=service,
stop_id=stop_time.stop_id,
timing_status=stop_time.timing_status,
direction='inbound',
order=i) for i, stop_time in enumerate(inbound)
]
StopUsage.objects.bulk_create(stop_usages)
if outbound:
outbound = Grouping(txc_service, outbound[0].stop,
outbound[-1].stop)
outbound_description = str(outbound)
if outbound_description != service.outbound_description:
service.outbound_description = outbound_description
changed_fields.append('outbound_description')
if inbound:
inbound = Grouping(txc_service, inbound[0].stop,
inbound[-1].stop)
inbound_description = str(inbound)
if inbound_description != service.inbound_description:
service.inbound_description = inbound_description
changed_fields.append('inbound_description')
if changed_fields:
service.save(update_fields=changed_fields)
service_code = service.service_code
if service_code in self.corrections:
corrections = {}
for field in self.corrections[service_code]:
if field == 'operator':
service.operator.set(
self.corrections[service_code][field])
else:
corrections[field] = self.corrections[service_code][
field]
Service.objects.filter(service_code=service_code).update(
**corrections)
service.update_search_vector()
if len(linked_services) > 1:
for i, from_service in enumerate(linked_services):
for i, to_service in enumerate(linked_services[i + 1:]):
kwargs = {
'from_service_id': from_service,
'to_service_id': to_service,
}
if not ServiceLink.objects.filter(**kwargs).exists():
ServiceLink.objects.create(**kwargs, how='also')
def handle_zipfile(path, collection, url):
source = DataSource.objects.update_or_create(
{
'url': url,
'datetime': timezone.now()
}, name=f'{collection} GTFS')[0]
shapes = {}
service_shapes = {}
operators = {}
routes = {}
services = set()
headsigns = {}
with zipfile.ZipFile(path) as archive:
for line in read_file(archive, 'shapes.txt'):
shape_id = line['shape_id']
if shape_id not in shapes:
shapes[shape_id] = []
shapes[shape_id].append(
Point(float(line['shape_pt_lon']),
float(line['shape_pt_lat'])))
for line in read_file(archive, 'agency.txt'):
operator, created = Operator.objects.get_or_create(
{
'name': line['agency_name'],
'region_id': 'LE'
},
id=line['agency_id'],
region__in=['CO', 'UL', 'MU', 'LE', 'NI'])
if not created and operator.name != line['agency_name']:
print(operator, line)
operators[line['agency_id']] = operator
for line in read_file(archive, 'routes.txt'):
if line['route_short_name'] and len(line['route_short_name']) <= 8:
route_id = line['route_short_name']
elif line['route_long_name'] and len(line['route_long_name']) <= 4:
route_id = line['route_long_name']
else:
route_id = line['route_id'].split()[0]
service_code = collection + '-' + route_id
assert len(service_code) <= 24
defaults = {
'region_id': 'LE',
'line_name': line['route_short_name'],
'description': line['route_long_name'],
'date': time.strftime('%Y-%m-%d'),
'mode': MODES.get(int(line['route_type']), ''),
'current': True,
'show_timetable': True
}
service, created = Service.objects.update_or_create(
defaults, service_code=service_code, source=source)
try:
operator = operators[line['agency_id']]
if service in services:
service.operator.add(operator)
else:
service.operator.set([operator])
except KeyError:
pass
services.add(service)
route, created = Route.objects.update_or_create(
{
'line_name': line['route_short_name'],
'description': line['route_long_name'],
'service': service,
},
source=source,
code=line['route_id'],
)
if not created:
route.trip_set.all().delete()
routes[line['route_id']] = route
stops, stops_not_created = do_stops(archive)
calendars = {}
for line in read_file(archive, 'calendar.txt'):
calendar = Calendar(
mon='1' == line['monday'],
tue='1' == line['tuesday'],
wed='1' == line['wednesday'],
thu='1' == line['thursday'],
fri='1' == line['friday'],
sat='1' == line['saturday'],
sun='1' == line['sunday'],
start_date=parse_date(line['start_date']),
end_date=parse_date(line['end_date']),
)
calendar.save()
calendars[line['service_id']] = calendar
for line in read_file(archive, 'calendar_dates.txt'):
CalendarDate.objects.create(
calendar=calendars[line['service_id']],
start_date=parse_date(line['date']),
end_date=parse_date(line['date']),
operation=line['exception_type'] == '1')
trips = {}
for line in read_file(archive, 'trips.txt'):
route = routes[line['route_id']]
trips[line['trip_id']] = Trip(
route=route,
calendar=calendars[line['service_id']],
inbound=line['direction_id'] == '1')
if route.service_id not in service_shapes:
service_shapes[route.service_id] = set()
service_shapes[route.service_id].add(line['shape_id'])
if line['trip_headsign']:
if line['route_id'] not in headsigns:
headsigns[line['route_id']] = {
'0': set(),
'1': set(),
}
headsigns[line['route_id']][line['direction_id']].add(
line['trip_headsign'])
for route_id in headsigns:
route = routes[route_id]
if not route.service.description:
origins = headsigns[route_id]['1']
destinations = headsigns[route_id]['0']
origin = None
destination = None
if len(origins) <= 1 and len(destinations) <= 1:
if len(origins) == 1:
origin = list(origins)[0]
if len(destinations) == 1:
destination = list(destinations)[0]
if origin and ' - ' in origin:
route.service.inbound_description = origin
route.service.description = origin
if destination and ' - ' in destination:
route.service.outbound_description = destination
route.service.description = destination
if origin and destination and ' - ' not in origin:
route.service.description = route.service.outbound_description = f'{origin} - {destination}'
route.service.inbound_description = f'{destination} - {origin}'
route.service.save(update_fields=[
'description', 'inbound_description',
'outbound_description'
])
stop_times = []
trip_id = None
trip = None
for line in read_file(archive, 'stop_times.txt'):
if trip_id != line['trip_id']:
if trip:
trip.start = stop_times[0].departure
trip.end = stop_times[-1].arrival
trip.save()
for stop_time in stop_times:
stop_time.trip = trip
StopTime.objects.bulk_create(stop_times)
stop_times = []
trip = Trip()
trip_id = line['trip_id']
trip = trips[trip_id]
stop = stops.get(line['stop_id'])
stop_time = StopTime(
stop=stop,
arrival=line['arrival_time'],
departure=line['departure_time'],
sequence=line['stop_sequence'],
)
if stop:
trip.destination = stop
elif line['stop_id'] in stops_not_created:
stop_time.stop_code = stops_not_created[line['stop_id']]
else:
stop_time.stop_code = line['stop_id']
print(line)
stop_times.append(stop_time)
trip.start = stop_times[0].departure
trip.end = stop_times[-1].arrival
trip.save()
for stop_time in stop_times:
stop_time.trip = trip
StopTime.objects.bulk_create(stop_times)
for service in services:
if service.id in service_shapes:
linestrings = [
LineString(*shapes[shape])
for shape in service_shapes[service.id] if shape in shapes
]
service.geometry = MultiLineString(*linestrings)
service.save(update_fields=['geometry'])
groupings = get_stop_usages(
Trip.objects.filter(route__service=service))
service.stops.clear()
stop_usages = [
StopUsage(service=service,
stop_id=stop_time.stop_id,
timing_status=stop_time.timing_status,
direction='outbound',
order=i) for i, stop_time in enumerate(groupings[0])
] + [
StopUsage(service=service,
stop_id=stop_time.stop_id,
timing_status=stop_time.timing_status,
direction='inbound',
order=i) for i, stop_time in enumerate(groupings[1])
]
StopUsage.objects.bulk_create(stop_usages)
service.region = Region.objects.filter(
adminarea__stoppoint__service=service).annotate(
Count('adminarea__stoppoint__service')).order_by(
'-adminarea__stoppoint__service__count').first()
if service.region:
service.save(update_fields=['region'])
service.update_search_vector()
for operator in operators.values():
operator.region = Region.objects.filter(
adminarea__stoppoint__service__operator=operator).annotate(
Count('adminarea__stoppoint__service__operator')).order_by(
'-adminarea__stoppoint__service__operator__count').first()
if operator.region_id:
operator.save(update_fields=['region'])
print(
source.service_set.filter(current=True).exclude(
route__in=routes.values()).update(current=False))
print(
source.service_set.filter(current=True).exclude(
route__trip__isnull=False).update(current=False))
print(
source.route_set.exclude(
id__in=(route.id for route in routes.values())).delete())
StopPoint.objects.filter(active=False,
service__current=True).update(active=True)
StopPoint.objects.filter(active=True,
service__isnull=True).update(active=False)
def handle_service(self, filename, parts, transxchange, txc_service, today,
stops):
if txc_service.operating_period.end:
if txc_service.operating_period.end < today:
return
if txc_service.operating_period.end < txc_service.operating_period.start:
return
operators = self.get_operators(transxchange, txc_service)
if self.is_tnds() and operators and all(
operator.id in self.open_data_operators
for operator in operators):
return
lines = get_lines(txc_service.element)
linked_services = []
description = self.get_description(txc_service)
for line_id, line_name, line_brand in lines:
existing = None
if operators and description and line_name:
if all(operator.parent == 'Go South Coast'
for operator in operators):
existing = Service.objects.filter(
operator__parent='Go South Coast')
else:
existing = Service.objects.filter(operator__in=operators)
existing = existing.filter(description=description,
line_name=line_name)
existing = existing.order_by('-current',
'service_code').first()
if self.is_tnds():
if operators and all(operator.id in self.incomplete_operators
for operator in operators):
if Service.objects.filter(
operator__in=operators,
line_name=line_name,
current=True).exclude(source=self.source).exists():
continue
service_code = get_service_code(filename)
if service_code is None:
service_code = txc_service.service_code
else:
if self.source.name in {'Go East Anglia', 'Go South West'
} and not existing:
try:
existing = Service.objects.get(
operator__parent=self.source.name,
current=True,
line_name__iexact=line_name)
except (Service.DoesNotExist,
Service.MultipleObjectsReturned):
pass
operator_code = '-'.join(operator.id for operator in operators)
if operator_code == 'TDTR' and 'Swindon-Rural' in filename:
operator_code = 'SBCR'
if parts:
service_code = f'{self.source.id}-{parts}-{line_name}'
if not existing:
existing = Service.objects.filter(
service_code=service_code,
line_name=line_name).first()
if not existing:
existing = self.source.service_set.filter(
line_name=line_name,
route__code__contains=f'/{parts}_').order_by(
'-current', 'service_code').first()
else:
service_code = f'{self.source.id}-{operator_code}-{txc_service.service_code}'
if len(lines) > 1:
service_code += '-' + line_name
if not existing and operator_code != 'SBCR':
existing = self.get_existing_service(
line_name, operators)
if existing and existing.id in self.service_ids and description and existing.description != description:
existing = None
if not existing:
existing = Service.objects.filter(
service_code=service_code).first()
if existing and existing.current and existing.line_name != line_name:
service_code = f'{service_code}-{line_name}'
existing = None
if existing:
services = Service.objects.filter(id=existing.id)
else:
services = Service.objects.filter(service_code=service_code)
defaults = {
'line_name': line_name,
'date': today,
'current': True,
'source': self.source,
'show_timetable': True
}
if not existing:
defaults['service_code'] = service_code
if description:
defaults['description'] = description
if txc_service.mode:
defaults['mode'] = txc_service.mode
if line_brand:
defaults['line_brand'] = line_brand
if self.region_id:
defaults['region_id'] = self.region_id
if self.service_descriptions: # NCSD
defaults['outbound_description'], defaults[
'inbound_description'] = self.get_service_descriptions(
filename)
defaults['description'] = defaults[
'outbound_description'] or defaults['inbound_description']
try:
service, service_created = services.update_or_create(defaults)
except IntegrityError as e:
print(e, service_code)
continue
if service_created:
service.operator.set(operators)
else:
if '_' in service.slug or '-' not in service.slug or existing and not existing.current:
service.slug = ''
service.save(update_fields=['slug'])
if self.source.name in {'Go East Anglia', 'Go South West'}:
pass
elif service.id in self.service_ids or all(
o.parent == 'Go South Coast' for o in operators):
service.operator.add(*operators)
else:
service.operator.set(operators)
self.service_ids.add(service.id)
linked_services.append(service.id)
journeys = transxchange.get_journeys(txc_service.service_code,
line_id)
# a code used in Traveline Cymru URLs:
if self.source.name == 'W':
if transxchange.journeys and journeys[0].private_code:
private_code = journeys[0].private_code
if ':' in private_code:
ServiceCode.objects.update_or_create(
{'code': private_code.split(':', 1)[0]},
service=service,
scheme='Traveline Cymru')
# timetable data:
route_defaults = {
'line_name': line_name,
'line_brand': line_brand,
'start_date': txc_service.operating_period.start,
'end_date': txc_service.operating_period.end,
'dates': txc_service.operating_period.dates(),
'service': service,
}
if 'description' in defaults:
route_defaults['description'] = defaults['description']
route_code = filename
if len(transxchange.services) > 1:
route_code += f'#{txc_service.service_code}'
if len(lines) > 1:
route_code += f'#{line_id}'
route, route_created = Route.objects.update_or_create(
route_defaults, source=self.source, code=route_code)
if not route_created:
route.trip_set.all().delete()
self.route_ids.add(route.id)
self.handle_journeys(route, stops, journeys, txc_service, line_id)
service.stops.clear()
outbound, inbound = get_stop_usages(
Trip.objects.filter(route__service=service))
changed_fields = []
if self.source.name.startswith(
'Arriva ') or self.source.name == 'Yorkshire Tiger':
if outbound:
changed = 0
origin_stop = outbound[0].stop
destination_stop = outbound[-1].stop
if origin_stop.common_name == txc_service.origin:
if origin_stop.locality.name not in txc_service.origin:
txc_service.origin = f'{origin_stop.locality.name} {txc_service.origin}'
changed += 1
if destination_stop.common_name == txc_service.destination:
if destination_stop.locality.name not in txc_service.destination:
txc_service.destination = f'{destination_stop.locality.name} {txc_service.destination}'
changed += 1
if changed == 2:
service.description = f'{txc_service.origin} - {txc_service.destination}'
changed_fields.append('description')
stop_usages = [
StopUsage(service=service,
stop_id=stop_time.stop_id,
timing_status=stop_time.timing_status,
direction='outbound',
order=i) for i, stop_time in enumerate(outbound)
] + [
StopUsage(service=service,
stop_id=stop_time.stop_id,
timing_status=stop_time.timing_status,
direction='inbound',
order=i) for i, stop_time in enumerate(inbound)
]
StopUsage.objects.bulk_create(stop_usages)
if outbound:
outbound = Grouping(txc_service, outbound[0].stop,
outbound[-1].stop)
outbound_description = str(outbound)
if outbound_description != service.outbound_description:
service.outbound_description = outbound_description
changed_fields.append('outbound_description')
if inbound:
inbound = Grouping(txc_service, inbound[0].stop,
inbound[-1].stop)
inbound_description = str(inbound)
if inbound_description != service.inbound_description:
service.inbound_description = inbound_description
changed_fields.append('inbound_description')
if changed_fields:
service.save(update_fields=changed_fields)
service_code = service.service_code
if service_code in self.corrections:
corrections = {}
for field in self.corrections[service_code]:
if field == 'operator':
service.operator.set(
self.corrections[service_code][field])
else:
corrections[field] = self.corrections[service_code][
field]
Service.objects.filter(service_code=service_code).update(
**corrections)
if service_code == 'twm_5-501-A-y11': # Lakeside Coaches
Trip.objects.filter(route__service=service,
start='15:05').delete()
Trip.objects.filter(route__service=service,
start='15:30').delete()
service.update_search_vector()
if len(linked_services) > 1:
for i, from_service in enumerate(linked_services):
for i, to_service in enumerate(linked_services[i + 1:]):
kwargs = {
'from_service_id': from_service,
'to_service_id': to_service,
}
if not ServiceLink.objects.filter(**kwargs).exists():
ServiceLink.objects.create(**kwargs, how='also')
def handle_file(self, open_file, filename):
transxchange = TransXChange(open_file)
try:
service_element = transxchange.element.find('txc:Services/txc:Service', NS)
except AttributeError:
return
if transxchange.mode == 'underground':
return
today = self.source.datetime.date()
if transxchange.operating_period.end and transxchange.operating_period.end < today:
return
line_name, line_brand = get_line_name_and_brand(service_element)
net, service_code, line_ver = infer_from_filename(filename)
if service_code is None:
service_code = transxchange.service_code
if service_code not in self.service_codes:
self.service_codes.add(service_code)
self.source.route_set.filter(service=service_code).delete()
defaults = {
'line_name': line_name,
'line_brand': line_brand,
'mode': transxchange.mode,
'net': net,
'line_ver': line_ver,
'region_id': self.region_id,
'date': today,
'current': True,
'source': self.source,
'show_timetable': True
}
description = transxchange.description
if description:
if self.region_id == 'NE':
description = sanitize_description(description)
defaults['description'] = description
# stops:
stops = StopPoint.objects.in_bulk(transxchange.stops.keys())
groupings = {
'outbound': Grouping('outbound', transxchange),
'inbound': Grouping('inbound', transxchange)
}
for journey_pattern in transxchange.journey_patterns.values():
if journey_pattern.direction == 'inbound':
grouping = groupings['inbound']
else:
grouping = groupings['outbound']
grouping.add_journey_pattern(journey_pattern)
try:
stop_usages = []
for grouping in groupings.values():
if grouping.rows:
stop_usages += [
StopUsage(
service_id=service_code, stop_id=row.part.stop.atco_code,
direction=grouping.direction, order=i, timing_status=row.part.timingstatus
)
for i, row in enumerate(grouping.rows) if row.part.stop.atco_code in stops
]
if grouping.direction == 'outbound' or grouping.direction == 'inbound':
# grouping.description_parts = transxchange.description_parts
defaults[grouping.direction + '_description'] = str(grouping)
line_strings = []
for pattern in transxchange.journey_patterns.values():
line_string = line_string_from_journeypattern(pattern, stops)
if line_string not in line_strings:
line_strings.append(line_string)
multi_line_string = MultiLineString(*(ls for ls in line_strings if ls))
except (AttributeError, IndexError) as error:
logger.error(error, exc_info=True)
defaults['show_timetable'] = False
stop_usages = [StopUsage(service_id=service_code, stop_id=stop, order=0) for stop in stops]
multi_line_string = None
defaults['geometry'] = multi_line_string
if self.service_descriptions:
defaults['outbound_description'], defaults['inbound_description'] = self.get_service_descriptions(filename)
defaults['description'] = defaults['outbound_description'] or defaults['inbound_description']
service, service_created = Service.objects.update_or_create(service_code=service_code, defaults=defaults)
operators = self.get_operators(transxchange)
if service_created:
service.operator.add(*operators)
else:
if service.slug == service_code.lower():
service.slug = ''
service.save()
service.operator.set(operators)
if service_code not in self.service_codes:
service.stops.clear()
StopUsage.objects.bulk_create(stop_usages)
# a code used in Traveline Cymru URLs:
if transxchange.journeys and transxchange.journeys[0].private_code:
private_code = transxchange.journeys[0].private_code
if ':' in private_code:
ServiceCode.objects.update_or_create({
'code': private_code.split(':', 1)[0]
}, service=service, scheme='Traveline Cymru')
# timetable data:
route_defaults = {
'line_name': line_name,
'line_brand': line_brand,
'start_date': transxchange.operating_period.start,
'end_date': transxchange.operating_period.end,
'service': service,
}
if 'description' in defaults:
route_defaults['description'] = defaults['description']
route, route_created = Route.objects.get_or_create(route_defaults, source=self.source, code=filename)
default_calendar = None
for journey in transxchange.journeys:
calendar = None
if journey.operating_profile:
calendar = self.get_calendar(journey.operating_profile, transxchange.operating_period)
if not calendar:
continue
else:
if not default_calendar:
default_calendar = self.get_calendar(transxchange.operating_profile, transxchange.operating_period)
calendar = default_calendar
trip = Trip(
inbound=journey.journey_pattern.direction == 'inbound',
calendar=calendar,
route=route,
journey_pattern=journey.journey_pattern.id,
)
stop_times = [
StopTime(
stop_code=cell.stopusage.stop.atco_code,
trip=trip,
arrival=cell.arrival_time,
departure=cell.departure_time,
sequence=i,
timing_status=cell.stopusage.timingstatus,
activity=cell.stopusage.activity or ''
) for i, cell in enumerate(journey.get_times())
]
trip.start = stop_times[0].arrival or stop_times[0].departure
trip.end = stop_times[-1].departure or stop_times[-1].arrival
if stop_times[-1].stop_code in stops:
trip.destination_id = stop_times[-1].stop_code
trip.save()
else:
print(stop_times[-1].stop_code, service)
if stop_times[-2].stop_code in stops:
trip.destination_id = stop_times[-2].stop_code
trip.save()
else:
print(stop_times[-2].stop_code, service)
return
for note in journey.notes:
note, _ = Note.objects.get_or_create(code=note, text=journey.notes[note])
trip.notes.add(note)
for stop_time in stop_times:
stop_time.trip = stop_time.trip # set trip_id
StopTime.objects.bulk_create(stop_times)
if service_code in self.corrections:
corrections = {}
for field in self.corrections[service_code]:
if field == 'operator':
service.operator.set(self.corrections[service_code][field])
else:
corrections[field] = self.corrections[service_code][field]
Service.objects.filter(service_code=service_code).update(**corrections)