def handle(self, *args, **options):
if (len(args) < 1):
return
csv.register_dialect('quotescolon', quotechar='"', delimiter=';', doublequote=False, lineterminator='\n', quoting=csv.QUOTE_NONE)
f = codecs.open(args[0], mode='rU')
stops = file.UnicodeDictReader(f, 'utf-8', dialect=csv.get_dialect('quotescolon'))
with reversion.create_revision():
source, created = Source.objects.get_or_create(source_id=u'govi', defaults={u'name': "GOVI"})
for stop in stops:
split = unicode(stop['TimingPointName']).split(',')
if len(split) > 1:
city = split[0]
name = split[1].lstrip()
else:
city = stop['TimingPointTown'].capitalize()
name = stop['TimingPointName']
point = geo.transform_rd(Point(x=int(stop['LocationX_EW']), y=int(stop['LocationY_NS']), srid=28992))
s, created = UserStop.objects.get_or_create(tpc=stop[u"TimingPointCode"],
defaults={u'common_name' : name, u'common_city' : city, 'point' : point.wkt})
# Get or create our source
for attr in stop.keys():
self.get_create_update(SourceAttribute, {'stop' : s, 'source' : source, 'key' : attr.capitalize()}, {'value' : stop[attr]} )
reversion.set_comment(u"GOVI Import")
f.close()
def handle(self, *args, **options):
if (len(args) < 2):
return self.do_help()
# Resolve all filenames
usrstop_filename = None
usrstar_filename = None
point_filename = None
for filename in listdir(args[0]):
if usrstop_filename is None and filename.lower().startswith('usrstop'):
usrstop_filename = filename
if usrstar_filename is None and filename.lower().startswith('usrstar'):
usrstar_filename = filename
if point_filename is None and filename.lower().startswith('point'):
point_filename = filename
mapping = "usrstop - %s, usrstar - %s and point - %s" % (usrstop_filename, usrstar_filename, point_filename)
if usrstop_filename is None or usrstar_filename is None or point_filename is None:
return "Couldn't find all 3 required files (%s)" % mapping
else:
print "Using: %s" % mapping
# Retrieve source - used for encoding
source = Source.objects.get(source_id=str(args[1]).lower())
if source is None:
return "Couldn't find the specified source - specify a data source"
# 2. Read files
# Create mapping
csv.register_dialect('kv1', quotechar='"',
delimiter=(options['delimiter'] if options['delimiter'] is not None else '|'),
doublequote=False, lineterminator='\n', quoting=csv.QUOTE_NONE)
# Read files
print "Using %s as encoding" % source.encoding
f1 = codecs.open(join(args[0], usrstop_filename), mode='rU')
if 'dataownercode' not in f1.readline().lower():
return "Huston, we have no headers!\n"
f1.close()
stops = file.UnicodeDictReader(codecs.open(join(args[0], usrstop_filename), mode='rU'), source.encoding, dialect=csv.get_dialect('kv1'))
point_rows = file.UnicodeDictReader(codecs.open(join(args[0], point_filename), mode='rU'), source.encoding, dialect=csv.get_dialect('kv1'))
stoparea_rows = file.UnicodeDictReader(codecs.open(join(args[0], usrstar_filename), mode='rU'), source.encoding, dialect=csv.get_dialect('kv1'))
# Do some translation
points = { point['PointCode'] : point for point in point_rows }
areas = { area['UserStopAreaCode'] : area for area in stoparea_rows }
for stop in stops:
with db.transaction.commit_on_success():
with reversion.create_revision():
if stop['TimingPointCode'] is '':
print "Huston, TPC was none, falling back to USC"
stop['TimingPointCode'] = stop['UserStopCode']
if stop['TimingPointCode'] is None:
return "We had no TPC or USC - import halted"
# Figure out our location
stop_location = points[stop['UserStopCode']]
pnt = geo.transform_rd(Point(int(stop_location['LocationX_EW']), int(stop_location['LocationY_NS']), srid=28992))
s, created = UserStop.objects.get_or_create(tpc=stop['TimingPointCode'],
defaults={u'common_name' : stop['Name'].replace(stop['Town']+', ', ''),
u'common_city' : stop['Town'],
'point' : pnt.wkt })
# Check for stop areas
if stop['UserStopAreaCode'] is not None and stop['UserStopAreaCode'] in areas:
s.parent = self.get_create_star(areas[stop['UserStopAreaCode']], source)
s.save()
# Save as much as the original data as possible
self.create_source_attr(s, source, 'id', stop['TimingPointCode'])
self.create_source_attr(s, source, 'GetIn', stop['GetIn'])
self.create_source_attr(s, source, 'GetOut', stop['GetOut'])
self.create_source_attr(s, source, 'Name', stop['Name'])
self.create_source_attr(s, source, 'Town', stop['Town'])
self.create_source_attr(s, source, 'UserStopAreaCode', stop['UserStopAreaCode'])
self.create_source_attr(s, source, 'MinimalStopTime', stop['MinimalStopTime'])
self.create_source_attr(s, source, 'UserStopType', stop['UserStopType'])
self.create_source_attr(s, source, 'latitude', int(stop_location['LocationX_EW']))
self.create_source_attr(s, source, 'longitude', int(stop_location['LocationY_NS']))
reversion.set_comment("KV1 Import")
