def save(self, force_insert=False, force_update=False, commit=True): ''' Save each of the journeys in the model. This is a disaster, we return the XML TODO: Figure out a better solution fo this! ''' xml_output = [] for journey in self.data['journeys'].split(',')[0:-1]: qry = Kv1Journey.objects.filter(id=journey, dates__date=get_operator_date()) if qry.count() == 1: self.instance.pk = None self.instance.journey = qry[0] self.instance.line = qry[0].line self.instance.operatingday = get_operator_date() self.instance.is_cancel = True # Unfortunately, we can't place this any earlier, because we don't have the dataownercode there if self.instance.journey.dataownercode == self.instance.dataownercode: self.instance.save() # Add details if self.data['reasontype'] != '0' or self.data['advicetype'] != '0': Kv17JourneyChange(change=self.instance, reasontype=self.data['reasontype'], subreasontype=self.data['subreasontype'], reasoncontent=self.data['reasoncontent'], advicetype=self.data['advicetype'], subadvicetype=self.data['subadvicetype'], advicecontent=self.data['advicecontent']).save() xml_output.append(self.instance.to_xml()) else: log.error("Oops! mismatch between dataownercode of line (%s) and of user (%s) when saving journey cancel" % (self.instance.journey.dataownercode, self.instance.dataownercode)) else: log.error("Failed to find journey %s" % journey) return xml_output
def get_context_data(self, **kwargs): context = super(ChangeListView, self).get_context_data(**kwargs) # Get the currently active changes context['active_list'] = self.model.objects.filter(operatingday=get_operator_date(), is_recovered=False, dataownercode=self.request.user.userprofile.company) context['active_list'] = context['active_list'].order_by('line__publiclinenumber', 'line__lineplanningnumber', 'journey__departuretime', 'created') # Add the no longer active changes context['archive_list'] = self.model.objects.filter(Q(operatingday__lt=get_operator_date()) | Q(is_recovered=True), dataownercode=self.request.user.userprofile.company, created__gt=get_operator_date()-timedelta(days=3)) context['archive_list'] = context['archive_list'].order_by('-created') return context
def get_object(self, queryset=None): line = FerryLine.objects.get(pk=self.request.POST.get('ferry', None)) obj, created = self.model.objects.get_or_create( ferry=line, operatingday=get_operator_date(), journeynumber=self.request.POST.get('journeynumber', None)) return obj
def find_from_realtime(dataowner, realtime_id, date=None): if date is None: date = get_operator_date() j = realtime_id.split(':') log.info("Looking for realtime trip id %s on date %s" % (j, date)) if len(j) == 3: line = Kv1Line.objects.filter(dataownercode=dataowner, lineplanningnumber=j[1]) if line.count() == 1: journey_pk = Kv1Journey.objects.filter(dataownercode=dataowner, line=line[0].pk, journeynumber=int(j[2]), dates__date=date) if journey_pk.count() == 1: log.info("Found journey with id '%s' for realtime_id %s" % (journey_pk[0].id, realtime_id)) return journey_pk[0] else: log.warn( "Realtime trip id '%s' has an journey count of %s (not exactly 1)" % (realtime_id, journey_pk)) else: log.warn("Realtime trip id '%s' has an invalid line" % (realtime_id, )) else: log.warn("Realtime trip id '%s' has incorrect # of arguments" % (realtime_id, )) log.error("Couldn't find trip %s on date %s :(" % (j, date)) return None
def handle_ferry(self, ferry): date = get_operator_date() journeys = ferry.line.journeys.filter(dates__date=date).order_by('departuretime') print("Checking ferry: %s" % ferry.line) ## READY: Init + Arrive init_target = self.get_target_time(5) for journey in journeys.filter(departuretime__lte=init_target): msg, created = FerryKv6Messages.objects.get_or_create(operatingday=date, ferry=ferry, journeynumber=journey.journeynumber) if created or ( msg.status == FerryKv6Messages.Status.INITIALIZED and not msg.cancelled and (msg.delay is None or journey.departuretime + msg.delay <= init_target)): msg.status = FerryKv6Messages.Status.READY msg.save() if self.pusher.push_message(msg.to_kv6_ready(journey.direction)): self.log('info', "Sent KV6 init & arrival message for %s" % msg) else: self.log('error', "Failed to send KV6 init & arrival message for %s" % msg) else: self.log('debug', "Already sent for %s" % journey.departuretime_as_time()) ## DEPARTED: Depart depart_target = self.get_target_time(0) for journey in journeys.filter(departuretime__lte=depart_target): try: msg = FerryKv6Messages.objects.get(operatingday=date, ferry=ferry, journeynumber=journey.journeynumber, status=FerryKv6Messages.Status.READY, cancelled=False) if msg.delay > 0 and journey.departuretime + msg.delay > depart_target: continue msg.status = FerryKv6Messages.Status.DEPARTED msg.save() if self.pusher.push_message(msg.to_kv6_departed(journey.direction)): self.log('info', "Sent KV6 depart message for %s" % msg) else: self.log('error', "Failed to send KV6 depart message for %s" % msg) except FerryKv6Messages.DoesNotExist: self.log('debug', "Skip for departed") ## ARRIVED: Arrive arrival_target = self.get_target_time(-20) for journey in journeys.filter(departuretime__lte=arrival_target): try: msg = FerryKv6Messages.objects.get(operatingday=date, ferry=ferry, journeynumber=journey.journeynumber, status=FerryKv6Messages.Status.DEPARTED, cancelled=False) if msg.delay > 0 and journey.departuretime + msg.delay > arrival_target: continue msg.status = FerryKv6Messages.Status.ARRIVED msg.save() if self.pusher.push_message(msg.to_kv6_arrived(journey.direction)): self.log('info', "Sent KV6 arrival message for %s" % msg) else: self.log('error', "Failed to send KV6 arrival message for %s" % msg) except FerryKv6Messages.DoesNotExist: self.log('debug', "Skip for arrived")
def get_object(self): operating_day = parse_date( self.request.GET['operatingday'] ) if 'operatingday' in self.request.GET else get_operator_date() # Note, can't set this on the view, because it triggers the queryset cache queryset = self.model.objects.filter( Q(is_alljourneysofline=True) | Q(is_alllines=True), operatingday=operating_day, is_recovered=False, dataownercode=self.request.user.userprofile.company).distinct() # TODO: is it possible to apply a function on a value of a queryset? start_of_day = datetime.combine(operating_day, datetime.min.time()).timestamp() return list({ 'id': x['line'], 'begintime': int(x['begintime'].timestamp() - start_of_day) if x['begintime'] is not None else None, 'endtime': int(x['endtime'].timestamp() - start_of_day) if x['endtime'] is not None else None, 'dataownercode': x['dataownercode'], 'alljourneysofline': x['is_alljourneysofline'], 'all_lines': x['is_alllines'] } for x in queryset.values('line', 'begintime', 'endtime', 'dataownercode', 'is_alljourneysofline', 'is_alllines'))
def add_journeys_from_request(self, data): operating_day = parse_date( self.request.GET['operatingday'] ) if 'operatingday' in self.request.GET else get_operator_date() data['operator_date'] = operating_day journey_errors = 0 journeys = [] for journey in self.request.GET['journey'].split(','): if journey == "": continue log.info("Finding journey %s for '%s'" % (journey, self.request.user)) j = Kv1Journey.find_from_realtime( self.request.user.userprofile.company, journey) if j: journeys.append(j) else: journey_errors += 1 log.error("User '%s' (%s) failed to find journey '%s' " % (self.request.user, self.request.user.userprofile.company, journey)) data['journeys'] = journeys if journey_errors > 0: data['journey_errors'] = journey_errors
def post(self, request, *args, **kwargs): resp = super(FerryRecoveredView, self).get(request, *args, **kwargs) messages = FerryKv6Messages.objects.filter( ferry=self.request.POST.get('ferry', None), operatingday=get_operator_date(), journeynumber=self.request.POST.get('journeynumber', None)) xml = [] for message in messages: output = message.to_recover() if output: xml.append(output) else: log.info("Couldn't delete message %s" % self.message) if len(xml) > 0 and self.push_message(''.join(xml)): log.info( "Sent KV17 ferry journey recovered message to subscribers: %s" % self.request.POST.get('journeynumber', None)) else: log.error( "Failed to communicate KV17 ferry journey cancelled to subscribers: %s" % xml) # Try to delete any KV15 message about less ferries if messages[0].ferry and messages[0].ferry.scenario_cancelled: kv15_xml = messages[0].ferry.scenario_cancelled.delete_all() if len(kv15_xml) > 0 and self.kv15_push.push_message(kv15_xml): log.info("Sent KV15 delete for cancellation message: %s" % self.request.POST.get('journeynumber', None)) else: log.info( "Failed to send KV15 delete for cancellation message: %s" % self.request.POST.get('journeynumber', None)) return resp
def handle_ferry(self, ferry): date = get_operator_date() journeys = ferry.line.journeys.filter(dates__date=date).order_by('departuretime') print "Checking ferry: %s" % ferry.line ## READY: Init + Arrive init_target = self.get_target_time(5) for journey in journeys.filter(departuretime__lte=init_target): msg, created = FerryKv6Messages.objects.get_or_create(operatingday=date, ferry=ferry, journeynumber=journey.journeynumber) if created or (msg.status == FerryKv6Messages.Status.INITIALIZED and not msg.cancelled and (msg.delay is None or journey.departuretime+msg.delay <= init_target)): msg.status = FerryKv6Messages.Status.READY msg.save() if self.pusher.push_message(msg.to_kv6_ready(journey.direction)): self.log('info', "Sent KV6 init & arrival message for %s" % msg) else: self.log('error', "Failed to send KV6 init & arrival message for %s" % msg) else: self.log('debug', "Already sent for %s" % journey.departuretime_as_time()) ## DEPARTED: Depart depart_target = self.get_target_time(0) for journey in journeys.filter(departuretime__lte=depart_target): try: msg = FerryKv6Messages.objects.get(operatingday=date, ferry=ferry, journeynumber=journey.journeynumber, status=FerryKv6Messages.Status.READY, cancelled=False) if msg.delay > 0 and journey.departuretime+msg.delay > depart_target: continue msg.status = FerryKv6Messages.Status.DEPARTED msg.save() if self.pusher.push_message(msg.to_kv6_departed(journey.direction)): self.log('info', "Sent KV6 depart message for %s" % msg) else: self.log('error', "Failed to send KV6 depart message for %s" % msg) except FerryKv6Messages.DoesNotExist: self.log('debug', "Skip for departed") ## ARRIVED: Arrive arrival_target = self.get_target_time(-20) for journey in journeys.filter(departuretime__lte=arrival_target): try: msg = FerryKv6Messages.objects.get(operatingday=date, ferry=ferry, journeynumber=journey.journeynumber, status=FerryKv6Messages.Status.DEPARTED, cancelled=False) if msg.delay > 0 and journey.departuretime+msg.delay > arrival_target: continue msg.status = FerryKv6Messages.Status.ARRIVED msg.save() if self.pusher.push_message(msg.to_kv6_arrived(journey.direction)): self.log('info', "Sent KV6 arrival message for %s" % msg) else: self.log('error', "Failed to send KV6 arrival message for %s" % msg) except FerryKv6Messages.DoesNotExist: self.log('debug', "Skip for arrived")
def get_object(self): # Note, can't set this on the view, because it triggers the queryset cache queryset = self.model.objects.filter(changes__operatingday=get_operator_date(), # changes__is_recovered=False, # TODO Fix this - see bug #61 # These two are double, but just in case changes__dataownercode=self.request.user.userprofile.company, dataownercode=self.request.user.userprofile.company).distinct() return list(queryset.values('id', 'dataownercode'))
def get_object(self, queryset=None): """ Forces our output as json and do some queries """ obj = get_object_or_404(self.model, pk=self.kwargs.get('pk', None)) if obj: # Note, the list() is required to serialize correctly # We're filtering on todays trips # journeys = obj.journeys.filter(dates__date=get_operator_date()).order_by('departuretime') \ .values('id', 'journeynumber', 'direction', 'departuretime') return {'trips_1': list(journeys.filter(direction=1)), 'trips_2': list(journeys.filter(direction=2))} return obj
def find_from_realtime(dataowner, realtime_id, date=get_operator_date()): j = realtime_id.split(':') if len(j) == 3: line = Kv1Line.objects.filter(dataownercode=dataowner, lineplanningnumber=j[1]) if line.count() == 1: journey_pk = Kv1Journey.objects.filter(dataownercode=dataowner, line=line[0].pk, journeynumber=int(j[2]), dates__date=date) if journey_pk.count() == 1: return journey_pk[0] return None
class Command(BaseCommand): pusher = Kv15PushMixin() BATCH_SIZE = 100 date = get_operator_date() current = datetime.now() if not is_aware(current): current = make_aware(current) def handle(self, *args, **options): to_send = [] to_send_messages = [] message_changes = Kv15Stopmessage.objects.filter(messageendtime__gte=self.current) \ .order_by('dataownercode', 'messagecodenumber', 'messagestarttime', 'messageendtime') if len(message_changes) != 0: for message in message_changes: res = message.to_xml() if res is not None: self.log("Resending: %s on %s" % (message.realtime_id(), self.date)) to_send.append(res) to_send_messages.append(message) else: self.log("ERROR!: %s on %s" % (message.realtime_id(), self.date)) else: self.log("Not sent: %s on %s" % (message.realtime_id(), self.date)) if len(to_send) > 0 and len(to_send) % self.BATCH_SIZE == 0: to_send, to_send_messages = self.send(to_send, to_send_messages) if len(to_send) > 0: self.send(to_send, to_send_messages) def send(self, to_send, to_send_messages): start = datetime.now() success = self.pusher.push_message(to_send) if success: self.log("Sending batch of %s, took %s seconds" % (len(to_send), (datetime.now() - start).seconds)) else: self.log("Failed to send batch! %s" % to_send_messages) to_send = [] to_send_messages = [] return to_send, to_send_messages def log(self, msg): self.stdout.write("%s - %s" % (datetime.now().isoformat(), msg))
def get_object(self): """ Forces our output as json and do some queries """ obj = self.get_queryset().get(pk=self.kwargs.get('pk', None)) if obj: # TODO : This is a slight hacky way to join three tables changes = Kv17Change.objects.filter( line=obj.line, operatingday=get_operator_date()) changed_trips = {t.journey.journeynumber: t for t in changes} ferry = FerryKv6Messages.objects.filter( ferry=obj, operatingday=get_operator_date()) ferry_trips = {t.journeynumber: t for t in ferry} journeys = obj.line.journeys.filter(dates__date=get_operator_date()).order_by('departuretime') \ .values('id', 'journeynumber', 'direction', 'departuretime') return { 'outward': self.merge_filter_direction(journeys, 1, ferry_trips, changed_trips), 'return': self.merge_filter_direction(journeys, 2, ferry_trips, changed_trips) }
class Command(BaseCommand): # TODO: This defines a default timeout, we may want to/need to change this for batch operations pusher = Kv17PushMixin() last_row_date = "" date = get_operator_date() BATCH_SIZE = 100 def add_arguments(self, parser): parser.add_argument('filename', nargs='+', type=str) def handle(self, *args, **options): with open(options['filename'][0], 'rb') as csvfile: reader = csv.reader(csvfile, delimiter=',') first = True to_send = [] to_send_trips = [] for row in reader: if first: first = False else: dataowner, lineplanningnumber, journeynumber = row[0].split(':') # TODO: Fix date here cancelled = Kv17Change.objects.filter(dataownercode=dataowner, operatingday=get_operator_date(), line__lineplanningnumber=lineplanningnumber, journey__journeynumber=journeynumber) if cancelled.count() == 1: cancelled[0].delete() to_send.append(cancelled[0].to_xml()) to_send_trips.append(row[0]) print ("Restored: %s:%s:%s on %s" % (cancelled[0].dataownercode, cancelled[0].line.lineplanningnumber, cancelled[0].journey.journeynumber, cancelled[0].operatingday)) else: print ("Not found: %s on %s" % (row[0], row[1])) if len(to_send) > 0 and len(to_send) % self.BATCH_SIZE == 0: self.stdout.write("Sending batch of %s" % self.BATCH_SIZE) start = datetime.now() success = self.pusher.push_message(to_send) self.stdout.write("Took %s seconds" % (datetime.now()-start).seconds) if not success: self.stdout.write("Failed to send batch! %s" % to_send_trips) to_send = [] to_send_trips = [] # TODO: last
def clean(self): cleaned_data = super(Kv17ChangeForm, self).clean() if 'journeys' not in self.data: raise ValidationError(_("Een of meer geselecteerde ritten zijn ongeldig")) valid_journeys = 0 for journey in self.data['journeys'].split(',')[0:-1]: journey_qry = Kv1Journey.objects.filter(pk=journey, dates__date=get_operator_date()) if journey_qry.count() == 0: raise ValidationError(_("Een of meer geselecteerde ritten zijn ongeldig")) if Kv17Change.objects.filter(journey__pk=journey, line=journey_qry[0].line, operatingday=get_operator_date()).count() != 0: raise ValidationError(_("Een of meer geselecteerde ritten zijn al aangepast")) valid_journeys += 1 if valid_journeys == 0: raise ValidationError(_("Er zijn geen ritten geselecteerd om op te heffen")) return cleaned_data
class Command(BaseCommand): # TODO: This defines a default timeout, we may want to/need to change this for batch operations pusher = Kv17PushMixin() last_row_date = "" date = get_operator_date() def add_arguments(self, parser): parser.add_argument('filename', nargs='+', type=str) def handle(self, *args, **options): with open(options['filename'][0], 'rb') as csvfile: reader = csv.reader(csvfile, delimiter=',') first = True to_send = [] to_send_trips = [] for row in reader: if first: first = False else: dataowner, lineplanningnumber, journeynumber = row[ 0].split(':') if self.last_row_date != row[1]: split = row[1].split('-') self.date = date(int(split[0]), int(split[1]), int(split[2])) cancelled = Kv17Change.objects.filter( dataownercode=dataowner, operatingday=self.date, line__lineplanningnumber=lineplanningnumber, journey__journeynumber=journeynumber) if cancelled.count() == 1: print("Deleted: %s:%s:%s on %s" % (cancelled[0].dataownercode, cancelled[0].line.lineplanningnumber, cancelled[0].journey.journeynumber, cancelled[0].operatingday)) cancelled[0].force_delete() else: print("Not found: %s on %s" % (row[0], self.date)) self.last_row_date = row[1]
def get_object(self, queryset=None): """ Forces our output as json and do some queries """ operating_day = get_operator_date() if 'operatingday' in self.request.GET: operating_day = parse_date(self.request.GET['operatingday']) obj = get_object_or_404(self.model, pk=self.kwargs.get('pk', None)) if obj: # Note, the list() is required to serialize correctly # We're filtering on todays trips # journeys = obj.journeys.filter(dates__date=operating_day).order_by('departuretime') \ .values('id', 'journeynumber', 'direction', 'departuretime') return { 'trips_1': list(journeys.filter(direction=1)), 'trips_2': list(journeys.filter(direction=2)) } return obj
def cancel_all(line_pk): date = get_operator_date() try: ferry = FerryLine.objects.get(pk=line_pk) except FerryLine.DoesNotExist: ferry = None if ferry: xml_out = [] journeys = ferry.line.journeys.filter(dates__date=date) for j in journeys: m, created = FerryKv6Messages.objects.get_or_create( ferry=ferry, journeynumber=j.journeynumber, operatingday=date) if m.status < FerryKv6Messages.Status.DEPARTED and not m.cancelled: msg = m.to_cancel() if msg: xml_out.append(msg) return xml_out return []
class Command(BaseCommand): pusher = Kv17PushMixin() BATCH_SIZE = 100 date = get_operator_date() def handle(self, *args, **options): to_send = [] to_send_trips = [] changes = Kv17Change.objects.filter(operatingday=self.date).order_by('dataownercode', 'line__lineplanningnumber', 'journey__journeynumber') for trip in changes: if trip.is_cancel: res = trip.to_xml() if res is not None: self.log("Resending: %s on %s" % (trip.realtime_id(), self.date)) to_send.append(res) to_send_trips.append(trip) else: self.log("ERROR!: %s on %s" % (trip.realtime_id(), self.date)) else: self.log("Not cancelled: %s on %s" % (trip.realtime_id(), self.date)) if len(to_send) > 0 and len(to_send) % self.BATCH_SIZE == 0: to_send, to_send_trips = self.send(to_send, to_send_trips) if len(to_send) > 0: self.send(to_send, to_send_trips) def send(self, to_send, to_send_trips): start = datetime.now() success = self.pusher.push_message(to_send) if success: self.log("Sending batch of %s, took %s seconds" % (len(to_send), (datetime.now() - start).seconds)) else: self.log("Failed to send batch! %s" % to_send_trips) to_send = [] to_send_trips = [] return to_send, to_send_trips def log(self, msg): self.stdout.write("%s - %s" % (datetime.now().isoformat(), msg))
def get_context_data(self, **kwargs): data = super(ChangeCreateView, self).get_context_data(**kwargs) data['operator_date'] = get_operator_date() if 'journey' in self.request.GET: self.add_journeys_from_request(data) return data
def get_object(self): operating_day = parse_date(self.request.GET['operatingday']) if 'operatingday' in \ self.request.GET else get_operator_date() # Note, can't set this on the view, because it triggers the queryset cache queryset = self.model.objects.filter( operatingday=operating_day, is_recovered=False, dataownercode=self.request.user.userprofile.company).distinct() return list( queryset.values('journey_id', 'dataownercode', 'is_recovered'))
class Command(BaseCommand): pusher = Kv17PushMixin() BATCH_SIZE = 100 last_row_date = "" date = get_operator_date() def add_arguments(self, parser): parser.add_argument('filename', nargs='+', type=str) def handle(self, *args, **options): with open(options['filename'][0], 'rb') as csvfile: reader = csv.reader(csvfile, delimiter=',') first = True to_send = [] to_send_trips = [] for row in reader: if first: first = False else: dataowner, lineplanningnumber, journeynumber = row[ 0].split(':') if self.last_row_date != row[1]: split = row[1].split('-') self.date = date(int(split[0]), int(split[1]), int(split[2])) cancelled = Kv17Change.objects.filter( dataownercode=dataowner, operatingday=self.date, line__lineplanningnumber=lineplanningnumber, journey__journeynumber=journeynumber) if cancelled.count() == 1: if cancelled[0].is_cancel: res = cancelled[0].to_xml() if res is not None: print("Resending: %s on %s" % (row[0], self.date)) to_send.append(res) to_send_trips.append(row[0]) else: print("ERROR!: %s on %s" % (row[0], self.date)) else: print("Not cancelled: %s on %s" % (row[0], self.date)) else: print("Not found: %s on %s" % (row[0], self.date)) if len(to_send ) > 0 and len(to_send) % self.BATCH_SIZE == 0: to_send, to_send_trips = self.send( to_send, to_send_trips) self.last_row_date = row[1] if len(to_send) > 0: self.send(to_send, to_send_trips) def send(self, to_send, to_send_trips): self.stdout.write("Sending batch of %s" % len(to_send)) start = datetime.now() success = self.pusher.push_message(to_send) self.stdout.write("Took %s seconds" % (datetime.now() - start).seconds) if not success: self.stdout.write("Failed to send batch! %s" % to_send_trips) to_send = [] to_send_trips = [] return to_send, to_send_trips
class Command(BaseCommand): # TODO: This defines a default timeout, we may want to/need to change this for batch operations pusher = Kv17PushMixin() last_row_date = "" date = get_operator_date() BATCH_SIZE = 100 def add_arguments(self, parser): parser.add_argument('filename', nargs='+', type=str) def handle(self, *args, **options): with open(options['filename'][0], 'rb') as csvfile: reader = csv.reader(csvfile, delimiter=',') first = True to_send = [] to_send_trips = [] for row in reader: if first: first = False else: dataowner, lineplanningnumber, journeynumber = row[ 0].split(':') if self.last_row_date != row[1]: split = row[1].split('-') self.date = date(int(split[0]), int(split[1]), int(split[2])) trip = Kv1Journey.find_from_realtime(dataowner, row[0], date=self.date) if trip is None: self.stdout.write("Not found: %s on %s " % (row[0], row[1])) else: cancelled = Kv17Change( dataownercode=trip.dataownercode, operatingday=self.date, line=trip.line, journey=trip) cancelled.save() cancelled.delete() to_send.append(cancelled.to_xml()) to_send_trips.append(row[0]) print("Restored: %s:%s:%s on %s" % (cancelled.dataownercode, cancelled.line.lineplanningnumber, cancelled.journey.journeynumber, cancelled.operatingday)) if len(to_send ) > 0 and len(to_send) % self.BATCH_SIZE == 0: to_send, to_send_trips = self.send( to_send, to_send_trips) if len(to_send) > 0: to_send, to_send_trips = self.send(to_send, to_send_trips) def send(self, to_send, to_send_trips): self.stdout.write("Sending batch of %s" % self.BATCH_SIZE) start = datetime.now() success = self.pusher.push_message(to_send) self.stdout.write("Took %s seconds" % (datetime.now() - start).seconds) if not success: self.stdout.write("Failed to send batch! %s" % to_send_trips) to_send = [] to_send_trips = [] return to_send, to_send_trips
class Command(BaseCommand): # TODO: This defines a default timeout, we may want to/need to change this for batch operations pusher = Kv17PushMixin() BATCH_SIZE = 100 last_row_date = "" date = get_operator_date() def add_arguments(self, parser): parser.add_argument('filename', nargs='+', type=str) def handle(self, *args, **options): with open(options['filename'][0], 'rb') as csvfile: reader = csv.reader(csvfile, delimiter=',') first = True to_send = [] to_send_trips = [] for row in reader: if first: first = False else: dataowner, lineplanningnumber, journeynumber = row[ 0].split(':') if self.last_row_date != row[1]: split = row[1].split('-') self.date = date(int(split[0]), int(split[1]), int(split[2])) trips = Kv1Journey.find_from_realtime(dataowner, row[0], date=self.date) if trips is None: self.stdout.write("Not found: %s on %s " % (row[0], row[1])) else: res = self.cancel_trip(trips, self.date) if res is not None: to_send.append(res) to_send_trips.append(row[0]) if len(to_send ) > 0 and len(to_send) % self.BATCH_SIZE == 0: to_send, to_send_trips = self.send( to_send, to_send_trips) self.last_row_date = row[1] if len(to_send) > 0: self.send(to_send, to_send_trips) def send(self, to_send, to_send_trips): self.stdout.write("Sending batch of %s" % len(to_send)) start = datetime.now() success = self.pusher.push_message(to_send) self.stdout.write("Took %s seconds" % (datetime.now() - start).seconds) if not success: self.stdout.write("Failed to send batch! %s" % to_send_trips) to_send = [] to_send_trips = [] return to_send, to_send_trips def cancel_trip(self, journey, date): if Kv17Change.objects.filter(dataownercode=journey.dataownercode, operatingday=date, line=journey.line, journey=journey).count() == 0: self.stdout.write( "Cancelling: %s:%s:%s on %s " % (journey.dataownercode, journey.line.lineplanningnumber, journey.journeynumber, date)) modification = Kv17Change(dataownercode=journey.dataownercode, operatingday=date, line=journey.line, journey=journey) modification.save() return modification.to_xml() else: self.stdout.write( "Already cancelled: %s:%s:%s on %s " % (journey.dataownercode, journey.line.lineplanningnumber, journey.journeynumber, date)) return None
def handle(self, *args, **options): with open(options['filename'][0], 'rb') as csvfile: reader = csv.reader(csvfile, delimiter=',') first = True to_send = [] to_send_trips = [] for row in reader: if first: first = False else: dataowner, lineplanningnumber, journeynumber = row[0].split(':') # TODO: Fix date here cancelled = Kv17Change.objects.filter(dataownercode=dataowner, operatingday=get_operator_date(), line__lineplanningnumber=lineplanningnumber, journey__journeynumber=journeynumber) if cancelled.count() == 1: cancelled[0].delete() to_send.append(cancelled[0].to_xml()) to_send_trips.append(row[0]) print ("Restored: %s:%s:%s on %s" % (cancelled[0].dataownercode, cancelled[0].line.lineplanningnumber, cancelled[0].journey.journeynumber, cancelled[0].operatingday)) else: print ("Not found: %s on %s" % (row[0], row[1])) if len(to_send) > 0 and len(to_send) % self.BATCH_SIZE == 0: self.stdout.write("Sending batch of %s" % self.BATCH_SIZE) start = datetime.now() success = self.pusher.push_message(to_send) self.stdout.write("Took %s seconds" % (datetime.now()-start).seconds) if not success: self.stdout.write("Failed to send batch! %s" % to_send_trips) to_send = [] to_send_trips = [] # TODO: last