Exemplo n.º 1
0
 def get_row(self, item):
     live_time = self._get_time(item.get('expected_departure_time'))
     time = self._get_time(item['aimed_departure_time'])
     if not time:
         time = live_time
     if not time:
         return
     if item.get('date') is not None:
         time = timezone.make_aware(ciso8601.parse_datetime(item['date'] + ' ' + time))
         if live_time:
             live_time = timezone.make_aware(ciso8601.parse_datetime(item['date'] + ' ' + live_time))
         if (item['source'].startswith('Traveline timetable') and time.date() > self.today):
             return
     else:
         time = timezone.make_aware(datetime.datetime.combine(
             self.today, dateutil.parser.parse(time).time()
         ))
         if live_time:
             live_time = timezone.make_aware(datetime.datetime.combine(
                 self.today, dateutil.parser.parse(live_time).time()
             ))
     return {
         'time': time,
         'live': live_time,
         'service': self.get_service(item.get('line').split('--', 1)[0].split('|', 1)[0]),
         'destination': self._get_destination(item),
     }
 def run(self):
     self.remove_output_on_overwrite()
     with self.input().open("r") as input_file:
         course_structure = json.load(input_file)
         with self.output().open("w") as output_file:
             courses_list = course_structure.get("results")
             if not courses_list:  # If there are no courses, or 'results' is not a key in the json, output nothing.
                 return
             for course in courses_list:
                 # To maintain robustness, ignore any non-dictionary data that finds its way into the API response.
                 try:
                     start_string = course.get("start")
                     end_string = course.get("end")
                     if start_string is None:
                         cleaned_start_string = "\N"
                     else:
                         cleaned_start_string = ciso8601.parse_datetime(start_string)
                     if end_string is None:
                         cleaned_end_string = "\N"
                     else:
                         cleaned_end_string = ciso8601.parse_datetime(end_string)
                     line = [
                         course.get("id", "\N"),
                         course.get("org", "\N"),
                         course.get("course", "\N"),
                         course.get("run", "\N"),
                         coerce_timestamp_for_hive(cleaned_start_string),
                         coerce_timestamp_for_hive(cleaned_end_string),
                         course.get("name", "\N"),
                     ]
                     output_file.write("\t".join([v.encode("utf-8") for v in line]))
                     output_file.write("\n")
                 except AttributeError:  # If the course is not a dictionary, move on to the next one.
                     continue
 def run(self):
     self.remove_output_on_overwrite()
     with self.input().open('r') as input_file:
         course_structure = json.load(input_file)
         with self.output().open('w') as output_file:
             courses_list = course_structure.get('results')
             if not courses_list:  # If there are no courses, or 'results' is not a key in the json, output nothing.
                 return
             for course in courses_list:
                 # To maintain robustness, ignore any non-dictionary data that finds its way into the API response.
                 try:
                     start_string = course.get('start')
                     end_string = course.get('end')
                     if start_string is None:
                         cleaned_start_string = '\N'
                     else:
                         cleaned_start_string = ciso8601.parse_datetime(start_string)
                     if end_string is None:
                         cleaned_end_string = '\N'
                     else:
                         cleaned_end_string = ciso8601.parse_datetime(end_string)
                     line = [
                         course.get('id', '\N'),
                         course.get('org', '\N'),
                         course.get('course', '\N'),
                         course.get('run', '\N'),
                         coerce_timestamp_for_hive(cleaned_start_string),
                         coerce_timestamp_for_hive(cleaned_end_string),
                         course.get('name', '\N')
                     ]
                     output_file.write('\t'.join([v.encode('utf-8') for v in line]))
                     output_file.write('\n')
                 except AttributeError:  # If the course is not a dictionary, move on to the next one.
                     continue
Exemplo n.º 4
0
 def departures_from_response(self, res):
     return sorted([{
         'time': ciso8601.parse_datetime(item['ScheduledArrival']),
         'live': ciso8601.parse_datetime(item['ExpectedArrival']),
         'service': self.get_service(item['LineName']),
         'destination': item['DestinationName'],
     } for item in res.json()['Predictions']['Prediction'] if item['ExpectedArrival']], key=lambda d: d['live'])
 def test_play_video(self):
     template = self.event_templates['play_video']
     event = self.create_event_log_line(template=template)
     expected_key = (self.DEFAULT_DATE, self.task.PROJECT_NAME)
     expected_dict = {
         'input_file': '',
         'source': self.task.PROJECT_NAME,
         'event_type': 'play_video',
         'emitter_type': 'browser',
         'timestamp': ciso8601.parse_datetime('2013-12-17T15:38:32.805444+00:00'),
         'received_at': ciso8601.parse_datetime('2013-12-17T15:38:32.805444+00:00'),
         'date': datetime.date(*[int(x) for x in self.DEFAULT_DATE.split('-')]),
         'username': '******',
         'course_id': self.encoded_course_id,
         'org_id': self.encoded_org_id,
         'user_id': '10',
         'referrer': 'long meaningful url',
         'agent_type': 'desktop',
         'agent_device_name': 'Other',
         'agent_os': 'Mac OS X',
         'agent_browser': 'Safari',
         'agent_touch_capable': False,
         'raw_event': self.get_raw_event(event),
     }
     expected_value = JsonEventRecord(**expected_dict).to_separated_values()
     self.assert_single_map_output(event, expected_key, expected_value)
Exemplo n.º 6
0
 def graph_one(self, start_timestamp, end_timestamp, payload_list):
     series = []
     payload = {"from": parse_datetime(start_timestamp).isoformat(),
         "to":parse_datetime(end_timestamp).isoformat(),
         "temporary":False,
         "averages":True,
         }
     for sys, subsys, col, idx in payload_list:
         for s in series:
             if s['system'] == sys and s['subsystem'] == subsys:
                 if col:
                     if col not in s['columns']:
                         s['columns'].append(col)
                 else:
                     raise ValueError('Need a column')
                 if idx:
                     if idx not in s['indexes']:
                         s['indexes'].append(idx)
                 break
         else:
             if idx:
                 idx = [idx]
             else:
                 idx = []
             series.append({'system':sys, 'subsystem':subsys, 'columns':[col], 'indexes':idx})
     payload["series"] = series
     print payload
     self.graph(payload)
Exemplo n.º 7
0
def munge(dct: Dict) -> AttrDict:
    dct = AttrDict(**dct)
    if 'start' in dct:
        dct.start = parse_datetime(dct.start)
    if 'end' in dct:
        dct.end = parse_datetime(dct.end)
    return dct
Exemplo n.º 8
0
 def departures_from_response(self, res):
     res = res.json()
     if '_embedded' in res:
         return [{
             'time': ciso8601.parse_datetime(item['aimedDepartureTime']),
             'live': ciso8601.parse_datetime(item['expectedDepartureTime']),
             'service': self.get_service(item['_links']['transmodel:line']['name']),
             'destination': item['destinationName'],
         } for item in res['_embedded']['timetable:visit'] if 'expectedDepartureTime' in item]
Exemplo n.º 9
0
 def test_aware_offset(self):
     self.assertEqual(
         ciso8601.parse_datetime('2014-12-05T12:30:45.123456+05:30'),
         datetime.datetime(2014, 12, 5, 12, 30, 45, 123456, pytz.FixedOffset(330))
     )
     self.assertEqual(
         ciso8601.parse_datetime('2014-12-05T12:30:45.123456-05:30'),
         datetime.datetime(2014, 12, 5, 12, 30, 45, 123456, pytz.FixedOffset(-330))
     )
     self.assertEqual(
         ciso8601.parse_datetime('2014-12-05T12:30:45.123456-06:00'),
         datetime.datetime(2014, 12, 5, 12, 30, 45, 123456, pytz.FixedOffset(-360))
     )
Exemplo n.º 10
0
 def test_aware_utc(self):
     expected = datetime.datetime(2014, 12, 5, 12, 30, 45, 123456, pytz.UTC)
     self.assertEqual(
         ciso8601.parse_datetime('2014-12-05T12:30:45.123456Z'),
         expected
     )
     self.assertEqual(
         ciso8601.parse_datetime('2014-12-05T12:30:45.123456+00:00'),
         expected,
     )
     self.assertEqual(
         ciso8601.parse_datetime('2014-12-05T12:30:45.123456-00:00'),
         expected,
     )
Exemplo n.º 11
0
def parse_datetime_user(string):
    """Parse datetime string from user.

    We accept the normal ISO-8601 formats, but kick through to the formats
    supported by the system's date command if parsing fails.

    Args:
        string (str): Datetime string to parse

    Returns:
        datetime.datetime: Parsed datetime object

    """
    try:
        datetime_ = parse_datetime(string)
    except ValueError:
        try:
            output = check_output(['date', '--utc', '--iso-8601=seconds', '-d',
                                   string])
            datetime_ = ciso8601.parse_datetime(output.strip()[:19])
        except subprocess.CalledProcessError:
            datetime_ = None
    if not datetime_:
        raise ValueError('Unable to parse timestamp %r'
                         % (safer_repr(string), ))
    return datetime_
Exemplo n.º 12
0
def service_vehicles_history(request, slug):
    service = get_object_or_404(Service, slug=slug)
    date = request.GET.get('date')
    today = timezone.now().date()
    if date:
        try:
            date = ciso8601.parse_datetime(date).date()
        except ValueError:
            date = None
    journeys = service.vehiclejourney_set
    if not date:
        try:
            date = journeys.values_list('datetime', flat=True).latest('datetime').date()
        except VehicleJourney.DoesNotExist:
            date = today
    locations = VehicleLocation.objects.filter(journey=OuterRef('pk'))
    journeys = journeys.filter(datetime__date=date).select_related('vehicle').annotate(locations=Exists(locations))
    operator = service.operator.select_related('region').first()
    return render(request, 'vehicles/vehicle_detail.html', {
        'breadcrumb': [operator.region, operator, service],
        'date': date,
        'today': today,
        'object': service,
        'journeys': journeys,
    })
Exemplo n.º 13
0
def str_to_datetime(str_repr):
    """Convert a string into a datetime."""
    # Allow the caller to be stupid.
    if type(str_repr) == datetime.datetime:
        return str_repr
    if not str_repr:
        return None
    return ciso8601.parse_datetime(str_repr)
def decode_datetime_objects(nested_value):
    if isinstance(nested_value, list):
        return [decode_datetime_objects(item) for item in nested_value]
    elif isinstance(nested_value, dict):
        for key, value in nested_value.items():
            if isinstance(value, dict) and 'type' in value.keys():
                if value['type'] == 'encoded_datetime':
                    nested_value[key] = ciso8601.parse_datetime(value['value'])
                if value['type'] == 'encoded_date':
                    nested_value[key] = ciso8601.parse_datetime(value['value']).date()
                if value['type'] == 'encoded_time':
                    nested_value[key] = ciso8601.parse_datetime(value['value']).time()
                if value['type'] == 'encoded_decimal':
                    nested_value[key] = Decimal(value['value'])
            elif isinstance(value, dict):
                nested_value[key] = decode_datetime_objects(value)
            elif isinstance(value, list):
                nested_value[key] = decode_datetime_objects(value)
        return nested_value
    return nested_value
 def test_problem_check(self):
     template = self.event_templates['problem_check']
     event = self.create_event_log_line(template=template)
     expected_key = (self.DEFAULT_DATE, self.task.PROJECT_NAME)
     expected_dict = {
         'input_file': '',
         'source': self.task.PROJECT_NAME,
         'event_type': 'problem_check',
         'emitter_type': 'server',
         'timestamp': ciso8601.parse_datetime('2013-12-17T15:38:32.805444+00:00'),
         'received_at': ciso8601.parse_datetime('2013-12-17T15:38:32.805444+00:00'),
         'date': datetime.date(*[int(x) for x in self.DEFAULT_DATE.split('-')]),
         'username': '******',
         'course_id': self.encoded_course_id,
         'org_id': self.encoded_org_id,
         'user_id': '10',
         'raw_event': self.get_raw_event(event),
     }
     expected_value = JsonEventRecord(**expected_dict).to_separated_values()
     self.assert_single_map_output(event, expected_key, expected_value)
Exemplo n.º 16
0
def parse_date(date_str):
    """Parse elastic datetime string."""
    if not date_str:
        return None

    try:
        date = ciso8601.parse_datetime(date_str)
        if not date:
            date = arrow.get(date_str).datetime
    except TypeError:
        date = arrow.get(date_str[0]).datetime
    return date
Exemplo n.º 17
0
    def from_json(self, serialized_value):
        if self.is_nullable and serialized_value is None:
            return None

        try:
            # ciso8601 is significantly faster than dateutil.parser for parsing iso8601 strings, so we try it first
            parsed_value = ciso8601.parse_datetime(serialized_value)
            assert parsed_value is not None  # Caveat: asserts won't run if python is run with -O.
        except Exception as e:
            parsed_value = dateutil.parser.parse(serialized_value)

        return make_naive(parsed_value)
Exemplo n.º 18
0
 def test_invalid(self):
     self.assertEqual(
         ciso8601.parse_datetime_unaware('asdf'),
         None,
     )
     self.assertEqual(
         ciso8601.parse_datetime_unaware('Z'),
         None,
     )
     self.assertEqual(
         ciso8601.parse_datetime('2014-12-05asdfasdf'),
         datetime.datetime(2014, 12, 5)
     )
Exemplo n.º 19
0
def poll_stats(container_name, delay_sec, out):
	with docker.Client(base_url='unix://var/run/docker.sock') as cli:
		delta = timedelta(seconds=4)
		try:
			last_timestamp = datetime.today() + timedelta(seconds=-4)
			for stat in cli.stats(container_name, decode=True):
				timestamp = ciso8601.parse_datetime(stat['read'])
				if timestamp - last_timestamp > delta:
					out.write(json.dumps(stat) + '\n')
					last_timestamp = timestamp
				# cpu_percent = (data['cpu_stats']['cpu_usage']['total_usage'] - data['precpu_stats']['cpu_usage']['total_usage']) / (data['cpu_stats']['system_cpu_usage'] - data['precpu_stats']['system_cpu_usage']) * len(data['cpu_stats']['cpu_usage']['percpu_usage'])
		except KeyboardInterrupt:
			return
Exemplo n.º 20
0
 def departures_from_response(self, response):
     departures = []
     for item in response.json():
         time = ciso8601.parse_datetime(item['ETA'])
         row = {
             'time': time,
             'destination': item['Destination'],
             'service': self.get_service(item['ServiceNumber'])
         }
         if item['IsTracked']:
             row['live'] = time
         departures.append(row)
     return departures
Exemplo n.º 21
0
def on_message(data):
    print(data)
    vehicle, created = Vehicle.objects.get_or_create(operator_id='MSOT', code=data['ass'])
    datetime = parse_datetime(data['date'])
    latlong = Point(data['lng'], data['lat'])

    journey = None
    service = None

    if not created:
        latest_location = vehicle.latest_location
        current = latest_location and (datetime - latest_location.datetime).total_seconds() < 180
        if current and latest_location.journey.service and latest_location.journey.service.geometry.overlaps(latlong):
            journey = latest_location.journey

        if not journey:
            try:
                service = Service.objects.filter(
                    operator='MSOT',
                    journey__datetime__lte=datetime,
                    journey__stopusageusage__datetime__gte=datetime,
                    geometry__bboverlaps=latlong
                ).distinct().get()
            except (Service.DoesNotExist, Service.MultipleObjectsReturned):
                pass

        if current and not service and not latest_location.journey.service:
            journey = latest_location.journey

    if not journey:
        journey = VehicleJourney.objects.create(
            vehicle=vehicle,
            datetime=datetime,
            source=globalism['source'],
            service=service,
            route_name=data['extra'].get('custR', '')
        )

    with atomic():
        if not created and latest_location:
            latest_location.journey.vehiclelocation_set.update(current=False)
        vehicle.latest_location = VehicleLocation.objects.create(
            journey=journey,
            datetime=datetime,
            latlong=latlong,
            heading=data['dir'],
            current=True
        )
        vehicle.save()
 def test_android_screen(self):
     template = self.event_templates['android_screen']
     event = self.create_event_log_line(template=template)
     expected_key = (self.DEFAULT_DATE, self.DEFAULT_PROJECT)
     expected_dict = {
         'input_file': '',
         'source': self.DEFAULT_PROJECT,
         'event_type': 'screen',
         'emitter_type': 'server',
         'timestamp': ciso8601.parse_datetime('2013-12-17T15:38:32.700000+00:00'),
         'received_at': ciso8601.parse_datetime('2013-12-17T15:38:32.796000+00:00'),
         'date': datetime.date(*[int(x) for x in self.DEFAULT_DATE.split('-')]),
         'agent_type': 'tablet',
         'agent_device_name': 'Samsung SM-N920A',
         'agent_os': 'Android',
         'agent_browser': 'Android',
         'agent_touch_capable': True,
         'anonymous_id': self.DEFAULT_ANONYMOUS_ID,
         'category': 'screen',
         'label': 'Launch\\0',
         'raw_event': self.get_raw_event(event),
     }
     expected_value = JsonEventRecord(**expected_dict).to_separated_values()
     self.assert_single_map_output(event, expected_key, expected_value)
Exemplo n.º 23
0
 def graph(self, payload):
     req = post("http://commonscontrol.harleyschool.org:8000/data/api/series/", data=dumps(payload)).json()
     for series in req:
         data = series['data']
         dates = [self.zone.normalize(parse_datetime(point["Time"])) for point in data]
         cols = [key for key in data[0] if key != "Time"]
         for col in cols:
             if series["index"]:
                 label = "%s %s %s" % (series['index'], series['subsystem'], col)
             else:
                 label = "%s %s" % (series['subsystem'], col)
             plt.plot(dates, [point[col] for point in data], label=label)
         plt.gcf().autofmt_xdate()
         plt.legend(loc=0, fontsize=12, fancybox=True)
         plt.show()
Exemplo n.º 24
0
def parse_datetime(string):
    """Parse ISO-8601 datetime string.

    Args:
        string (str): Datetime string to parse
    Returns:
        datetime.datetime: Parsed datetime object
    """
    if not string:
        datetime_ = datetime.datetime.utcnow().replace(tzinfo=utc)
    else:
        datetime_ = ciso8601.parse_datetime(string)
        if not datetime_:
            raise ValueError('Unable to parse timestamp %r' % string)
    return datetime_
Exemplo n.º 25
0
def parse_datetime(__string: str) -> datetime.datetime:
    """Parse ISO-8601 datetime string.

    Args:
        __string: Datetime string to parse
    Returns:
        Parsed datetime object
    """
    if not __string:
        datetime_ = datetime.datetime.now(datetime.timezone.utc)
    else:
        # pylint: disable=no-member
        datetime_ = ciso8601.parse_datetime(__string)
    if datetime_.tzinfo is None:
        datetime_ = datetime_.replace(tzinfo=datetime.timezone.utc)
    return datetime_
 def get_items(self):
     fifteen_minutes_ago = timezone.now() - timedelta(minutes=15)
     for params in self.get_extents():
         try:
             response = self.session.get(self.url, params=params, timeout=5)
             items = response.json()['items']
             any_items = False
             if items:
                 for item in items:
                     if parse_datetime(item['reported']) > fifteen_minutes_ago:
                         any_items = True
                         yield item
             if not any_items:
                 print(response.url)
         except (RequestException, KeyError):
             continue
         sleep(1)
Exemplo n.º 27
0
def parse_datetime(string):
    """Parse datetime string.

    Args:
        string (str): Datetime string to parse

    Returns:
        datetime.datetime: Parsed datetime object

    """
    if not string:
        datetime_ = datetime.datetime.utcnow()
    else:
        datetime_ = ciso8601.parse_datetime(string[:19])
        if not datetime_:
            raise ValueError('Unable to parse timestamp %r'
                             % (safer_repr(string), ))
    return datetime_
Exemplo n.º 28
0
    def set_date(self, date):
        if date and not isinstance(date, datetime.date):
            date = ciso8601.parse_datetime(date).date()

        if hasattr(self, 'date'):
            if date == self.date:
                return
            for grouping in self.groupings:
                for row in grouping.rows:
                    row.times.clear()
                grouping.column_feet.clear()

        self.date = date

        for grouping in self.groupings:
            for journey in grouping.journeys:
                if journey.should_show(self.date, self):
                    journey.add_times()
            grouping.do_heads_and_feet()
Exemplo n.º 29
0
    def test_unaware(self):
        expected = datetime.datetime(2014, 12, 5, 12, 30, 45, 123456)
        self.assertEqual(
            ciso8601.parse_datetime('2014-12-05T12:30:45.123456'),
            expected
        )

        # parse_datetime_unaware ignores tz offset
        self.assertEqual(
            ciso8601.parse_datetime_unaware('2014-12-05T12:30:45.123456Z'),
            expected
        )
        self.assertEqual(
            ciso8601.parse_datetime_unaware('2014-12-05T12:30:45.123456+00:00'),
            expected,
        )
        self.assertEqual(
            ciso8601.parse_datetime_unaware('2014-12-05T12:30:45.123456-05:00'),
            expected,
        )
Exemplo n.º 30
0
 def get_context_data(self, **kwargs):
     context = super().get_context_data(**kwargs)
     if self.object.operator:
         context['breadcrumb'] = [self.object.operator.region, self.object.operator]
     date = self.request.GET.get('date')
     context['today'] = timezone.now().date()
     if date:
         try:
             date = ciso8601.parse_datetime(date).date()
         except ValueError:
             date = None
     journeys = self.object.vehiclejourney_set
     if not date:
         try:
             date = journeys.values_list('datetime', flat=True).latest('datetime').date()
         except VehicleJourney.DoesNotExist:
             date = context['today']
     context['date'] = date
     journeys = journeys.filter(datetime__date=date)
     locations = VehicleLocation.objects.filter(journey=OuterRef('pk'))
     context['journeys'] = journeys.select_related('service').annotate(locations=Exists(locations))
     return context