def handle(self, *args, **options): if (len(args) < 1): return csv.register_dialect('quotescolon', quotechar='"', delimiter=';', doublequote=False, lineterminator='\n', quoting=csv.QUOTE_NONE) f = codecs.open(args[0], mode='rU') stops = file.UnicodeDictReader(f, 'utf-8', dialect=csv.get_dialect('quotescolon')) with reversion.create_revision(): source, created = Source.objects.get_or_create(source_id=u'govi', defaults={u'name': "GOVI"}) for stop in stops: split = unicode(stop['TimingPointName']).split(',') if len(split) > 1: city = split[0] name = split[1].lstrip() else: city = stop['TimingPointTown'].capitalize() name = stop['TimingPointName'] point = geo.transform_rd(Point(x=int(stop['LocationX_EW']), y=int(stop['LocationY_NS']), srid=28992)) s, created = UserStop.objects.get_or_create(tpc=stop[u"TimingPointCode"], defaults={u'common_name' : name, u'common_city' : city, 'point' : point.wkt}) # Get or create our source for attr in stop.keys(): self.get_create_update(SourceAttribute, {'stop' : s, 'source' : source, 'key' : attr.capitalize()}, {'value' : stop[attr]} ) reversion.set_comment(u"GOVI Import") f.close()
def handle(self, *args, **options): if (len(args) < 2): return self.do_help() # Resolve all filenames usrstop_filename = None usrstar_filename = None point_filename = None for filename in listdir(args[0]): if usrstop_filename is None and filename.lower().startswith('usrstop'): usrstop_filename = filename if usrstar_filename is None and filename.lower().startswith('usrstar'): usrstar_filename = filename if point_filename is None and filename.lower().startswith('point'): point_filename = filename mapping = "usrstop - %s, usrstar - %s and point - %s" % (usrstop_filename, usrstar_filename, point_filename) if usrstop_filename is None or usrstar_filename is None or point_filename is None: return "Couldn't find all 3 required files (%s)" % mapping else: print "Using: %s" % mapping # Retrieve source - used for encoding source = Source.objects.get(source_id=str(args[1]).lower()) if source is None: return "Couldn't find the specified source - specify a data source" # 2. Read files # Create mapping csv.register_dialect('kv1', quotechar='"', delimiter=(options['delimiter'] if options['delimiter'] is not None else '|'), doublequote=False, lineterminator='\n', quoting=csv.QUOTE_NONE) # Read files print "Using %s as encoding" % source.encoding f1 = codecs.open(join(args[0], usrstop_filename), mode='rU') if 'dataownercode' not in f1.readline().lower(): return "Huston, we have no headers!\n" f1.close() stops = file.UnicodeDictReader(codecs.open(join(args[0], usrstop_filename), mode='rU'), source.encoding, dialect=csv.get_dialect('kv1')) point_rows = file.UnicodeDictReader(codecs.open(join(args[0], point_filename), mode='rU'), source.encoding, dialect=csv.get_dialect('kv1')) stoparea_rows = file.UnicodeDictReader(codecs.open(join(args[0], usrstar_filename), mode='rU'), source.encoding, dialect=csv.get_dialect('kv1')) # Do some translation points = { point['PointCode'] : point for point in point_rows } areas = { area['UserStopAreaCode'] : area for area in stoparea_rows } for stop in stops: with db.transaction.commit_on_success(): with reversion.create_revision(): if stop['TimingPointCode'] is '': print "Huston, TPC was none, falling back to USC" stop['TimingPointCode'] = stop['UserStopCode'] if stop['TimingPointCode'] is None: return "We had no TPC or USC - import halted" # Figure out our location stop_location = points[stop['UserStopCode']] pnt = geo.transform_rd(Point(int(stop_location['LocationX_EW']), int(stop_location['LocationY_NS']), srid=28992)) s, created = UserStop.objects.get_or_create(tpc=stop['TimingPointCode'], defaults={u'common_name' : stop['Name'].replace(stop['Town']+', ', ''), u'common_city' : stop['Town'], 'point' : pnt.wkt }) # Check for stop areas if stop['UserStopAreaCode'] is not None and stop['UserStopAreaCode'] in areas: s.parent = self.get_create_star(areas[stop['UserStopAreaCode']], source) s.save() # Save as much as the original data as possible self.create_source_attr(s, source, 'id', stop['TimingPointCode']) self.create_source_attr(s, source, 'GetIn', stop['GetIn']) self.create_source_attr(s, source, 'GetOut', stop['GetOut']) self.create_source_attr(s, source, 'Name', stop['Name']) self.create_source_attr(s, source, 'Town', stop['Town']) self.create_source_attr(s, source, 'UserStopAreaCode', stop['UserStopAreaCode']) self.create_source_attr(s, source, 'MinimalStopTime', stop['MinimalStopTime']) self.create_source_attr(s, source, 'UserStopType', stop['UserStopType']) self.create_source_attr(s, source, 'latitude', int(stop_location['LocationX_EW'])) self.create_source_attr(s, source, 'longitude', int(stop_location['LocationY_NS'])) reversion.set_comment("KV1 Import")