def get_row(self, item):
live_time = self._get_time(item.get('expected_departure_time'))
time = self._get_time(item['aimed_departure_time'])
if not time:
time = live_time
if not time:
return
if item.get('date') is not None:
time = timezone.make_aware(ciso8601.parse_datetime(item['date'] + ' ' + time))
if live_time:
live_time = timezone.make_aware(ciso8601.parse_datetime(item['date'] + ' ' + live_time))
if (item['source'].startswith('Traveline timetable') and time.date() > self.today):
return
else:
time = timezone.make_aware(datetime.datetime.combine(
self.today, dateutil.parser.parse(time).time()
))
if live_time:
live_time = timezone.make_aware(datetime.datetime.combine(
self.today, dateutil.parser.parse(live_time).time()
))
return {
'time': time,
'live': live_time,
'service': self.get_service(item.get('line').split('--', 1)[0].split('|', 1)[0]),
'destination': self._get_destination(item),
}
def run(self):
self.remove_output_on_overwrite()
with self.input().open("r") as input_file:
course_structure = json.load(input_file)
with self.output().open("w") as output_file:
courses_list = course_structure.get("results")
if not courses_list: # If there are no courses, or 'results' is not a key in the json, output nothing.
return
for course in courses_list:
# To maintain robustness, ignore any non-dictionary data that finds its way into the API response.
try:
start_string = course.get("start")
end_string = course.get("end")
if start_string is None:
cleaned_start_string = "\N"
else:
cleaned_start_string = ciso8601.parse_datetime(start_string)
if end_string is None:
cleaned_end_string = "\N"
else:
cleaned_end_string = ciso8601.parse_datetime(end_string)
line = [
course.get("id", "\N"),
course.get("org", "\N"),
course.get("course", "\N"),
course.get("run", "\N"),
coerce_timestamp_for_hive(cleaned_start_string),
coerce_timestamp_for_hive(cleaned_end_string),
course.get("name", "\N"),
]
output_file.write("\t".join([v.encode("utf-8") for v in line]))
output_file.write("\n")
except AttributeError: # If the course is not a dictionary, move on to the next one.
continue
def run(self):
self.remove_output_on_overwrite()
with self.input().open('r') as input_file:
course_structure = json.load(input_file)
with self.output().open('w') as output_file:
courses_list = course_structure.get('results')
if not courses_list: # If there are no courses, or 'results' is not a key in the json, output nothing.
return
for course in courses_list:
# To maintain robustness, ignore any non-dictionary data that finds its way into the API response.
try:
start_string = course.get('start')
end_string = course.get('end')
if start_string is None:
cleaned_start_string = '\N'
else:
cleaned_start_string = ciso8601.parse_datetime(start_string)
if end_string is None:
cleaned_end_string = '\N'
else:
cleaned_end_string = ciso8601.parse_datetime(end_string)
line = [
course.get('id', '\N'),
course.get('org', '\N'),
course.get('course', '\N'),
course.get('run', '\N'),
coerce_timestamp_for_hive(cleaned_start_string),
coerce_timestamp_for_hive(cleaned_end_string),
course.get('name', '\N')
]
output_file.write('\t'.join([v.encode('utf-8') for v in line]))
output_file.write('\n')
except AttributeError: # If the course is not a dictionary, move on to the next one.
continue
def departures_from_response(self, res):
return sorted([{
'time': ciso8601.parse_datetime(item['ScheduledArrival']),
'live': ciso8601.parse_datetime(item['ExpectedArrival']),
'service': self.get_service(item['LineName']),
'destination': item['DestinationName'],
} for item in res.json()['Predictions']['Prediction'] if item['ExpectedArrival']], key=lambda d: d['live'])
def test_play_video(self):
template = self.event_templates['play_video']
event = self.create_event_log_line(template=template)
expected_key = (self.DEFAULT_DATE, self.task.PROJECT_NAME)
expected_dict = {
'input_file': '',
'source': self.task.PROJECT_NAME,
'event_type': 'play_video',
'emitter_type': 'browser',
'timestamp': ciso8601.parse_datetime('2013-12-17T15:38:32.805444+00:00'),
'received_at': ciso8601.parse_datetime('2013-12-17T15:38:32.805444+00:00'),
'date': datetime.date(*[int(x) for x in self.DEFAULT_DATE.split('-')]),
'username': '******',
'course_id': self.encoded_course_id,
'org_id': self.encoded_org_id,
'user_id': '10',
'referrer': 'long meaningful url',
'agent_type': 'desktop',
'agent_device_name': 'Other',
'agent_os': 'Mac OS X',
'agent_browser': 'Safari',
'agent_touch_capable': False,
'raw_event': self.get_raw_event(event),
}
expected_value = JsonEventRecord(**expected_dict).to_separated_values()
self.assert_single_map_output(event, expected_key, expected_value)
def graph_one(self, start_timestamp, end_timestamp, payload_list):
series = []
payload = {"from": parse_datetime(start_timestamp).isoformat(),
"to":parse_datetime(end_timestamp).isoformat(),
"temporary":False,
"averages":True,
}
for sys, subsys, col, idx in payload_list:
for s in series:
if s['system'] == sys and s['subsystem'] == subsys:
if col:
if col not in s['columns']:
s['columns'].append(col)
else:
raise ValueError('Need a column')
if idx:
if idx not in s['indexes']:
s['indexes'].append(idx)
break
else:
if idx:
idx = [idx]
else:
idx = []
series.append({'system':sys, 'subsystem':subsys, 'columns':[col], 'indexes':idx})
payload["series"] = series
print payload
self.graph(payload)
def munge(dct: Dict) -> AttrDict:
dct = AttrDict(**dct)
if 'start' in dct:
dct.start = parse_datetime(dct.start)
if 'end' in dct:
dct.end = parse_datetime(dct.end)
return dct
def departures_from_response(self, res):
res = res.json()
if '_embedded' in res:
return [{
'time': ciso8601.parse_datetime(item['aimedDepartureTime']),
'live': ciso8601.parse_datetime(item['expectedDepartureTime']),
'service': self.get_service(item['_links']['transmodel:line']['name']),
'destination': item['destinationName'],
} for item in res['_embedded']['timetable:visit'] if 'expectedDepartureTime' in item]
def test_aware_offset(self):
self.assertEqual(
ciso8601.parse_datetime('2014-12-05T12:30:45.123456+05:30'),
datetime.datetime(2014, 12, 5, 12, 30, 45, 123456, pytz.FixedOffset(330))
)
self.assertEqual(
ciso8601.parse_datetime('2014-12-05T12:30:45.123456-05:30'),
datetime.datetime(2014, 12, 5, 12, 30, 45, 123456, pytz.FixedOffset(-330))
)
self.assertEqual(
ciso8601.parse_datetime('2014-12-05T12:30:45.123456-06:00'),
datetime.datetime(2014, 12, 5, 12, 30, 45, 123456, pytz.FixedOffset(-360))
)
def test_aware_utc(self):
expected = datetime.datetime(2014, 12, 5, 12, 30, 45, 123456, pytz.UTC)
self.assertEqual(
ciso8601.parse_datetime('2014-12-05T12:30:45.123456Z'),
expected
)
self.assertEqual(
ciso8601.parse_datetime('2014-12-05T12:30:45.123456+00:00'),
expected,
)
self.assertEqual(
ciso8601.parse_datetime('2014-12-05T12:30:45.123456-00:00'),
expected,
)
def parse_datetime_user(string):
"""Parse datetime string from user.
We accept the normal ISO-8601 formats, but kick through to the formats
supported by the system's date command if parsing fails.
Args:
string (str): Datetime string to parse
Returns:
datetime.datetime: Parsed datetime object
"""
try:
datetime_ = parse_datetime(string)
except ValueError:
try:
output = check_output(['date', '--utc', '--iso-8601=seconds', '-d',
string])
datetime_ = ciso8601.parse_datetime(output.strip()[:19])
except subprocess.CalledProcessError:
datetime_ = None
if not datetime_:
raise ValueError('Unable to parse timestamp %r'
% (safer_repr(string), ))
return datetime_
def service_vehicles_history(request, slug):
service = get_object_or_404(Service, slug=slug)
date = request.GET.get('date')
today = timezone.now().date()
if date:
try:
date = ciso8601.parse_datetime(date).date()
except ValueError:
date = None
journeys = service.vehiclejourney_set
if not date:
try:
date = journeys.values_list('datetime', flat=True).latest('datetime').date()
except VehicleJourney.DoesNotExist:
date = today
locations = VehicleLocation.objects.filter(journey=OuterRef('pk'))
journeys = journeys.filter(datetime__date=date).select_related('vehicle').annotate(locations=Exists(locations))
operator = service.operator.select_related('region').first()
return render(request, 'vehicles/vehicle_detail.html', {
'breadcrumb': [operator.region, operator, service],
'date': date,
'today': today,
'object': service,
'journeys': journeys,
})
def str_to_datetime(str_repr):
"""Convert a string into a datetime."""
# Allow the caller to be stupid.
if type(str_repr) == datetime.datetime:
return str_repr
if not str_repr:
return None
return ciso8601.parse_datetime(str_repr)
def decode_datetime_objects(nested_value):
if isinstance(nested_value, list):
return [decode_datetime_objects(item) for item in nested_value]
elif isinstance(nested_value, dict):
for key, value in nested_value.items():
if isinstance(value, dict) and 'type' in value.keys():
if value['type'] == 'encoded_datetime':
nested_value[key] = ciso8601.parse_datetime(value['value'])
if value['type'] == 'encoded_date':
nested_value[key] = ciso8601.parse_datetime(value['value']).date()
if value['type'] == 'encoded_time':
nested_value[key] = ciso8601.parse_datetime(value['value']).time()
if value['type'] == 'encoded_decimal':
nested_value[key] = Decimal(value['value'])
elif isinstance(value, dict):
nested_value[key] = decode_datetime_objects(value)
elif isinstance(value, list):
nested_value[key] = decode_datetime_objects(value)
return nested_value
return nested_value
def test_problem_check(self):
template = self.event_templates['problem_check']
event = self.create_event_log_line(template=template)
expected_key = (self.DEFAULT_DATE, self.task.PROJECT_NAME)
expected_dict = {
'input_file': '',
'source': self.task.PROJECT_NAME,
'event_type': 'problem_check',
'emitter_type': 'server',
'timestamp': ciso8601.parse_datetime('2013-12-17T15:38:32.805444+00:00'),
'received_at': ciso8601.parse_datetime('2013-12-17T15:38:32.805444+00:00'),
'date': datetime.date(*[int(x) for x in self.DEFAULT_DATE.split('-')]),
'username': '******',
'course_id': self.encoded_course_id,
'org_id': self.encoded_org_id,
'user_id': '10',
'raw_event': self.get_raw_event(event),
}
expected_value = JsonEventRecord(**expected_dict).to_separated_values()
self.assert_single_map_output(event, expected_key, expected_value)
def parse_date(date_str):
"""Parse elastic datetime string."""
if not date_str:
return None
try:
date = ciso8601.parse_datetime(date_str)
if not date:
date = arrow.get(date_str).datetime
except TypeError:
date = arrow.get(date_str[0]).datetime
return date
def from_json(self, serialized_value):
if self.is_nullable and serialized_value is None:
return None
try:
# ciso8601 is significantly faster than dateutil.parser for parsing iso8601 strings, so we try it first
parsed_value = ciso8601.parse_datetime(serialized_value)
assert parsed_value is not None # Caveat: asserts won't run if python is run with -O.
except Exception as e:
parsed_value = dateutil.parser.parse(serialized_value)
return make_naive(parsed_value)
def test_invalid(self):
self.assertEqual(
ciso8601.parse_datetime_unaware('asdf'),
None,
)
self.assertEqual(
ciso8601.parse_datetime_unaware('Z'),
None,
)
self.assertEqual(
ciso8601.parse_datetime('2014-12-05asdfasdf'),
datetime.datetime(2014, 12, 5)
)
def poll_stats(container_name, delay_sec, out): with docker.Client(base_url='unix://var/run/docker.sock') as cli: delta = timedelta(seconds=4) try: last_timestamp = datetime.today() + timedelta(seconds=-4) for stat in cli.stats(container_name, decode=True): timestamp = ciso8601.parse_datetime(stat['read']) if timestamp - last_timestamp > delta: out.write(json.dumps(stat) + '\n') last_timestamp = timestamp # cpu_percent = (data['cpu_stats']['cpu_usage']['total_usage'] - data['precpu_stats']['cpu_usage']['total_usage']) / (data['cpu_stats']['system_cpu_usage'] - data['precpu_stats']['system_cpu_usage']) * len(data['cpu_stats']['cpu_usage']['percpu_usage']) except KeyboardInterrupt: return
def departures_from_response(self, response):
departures = []
for item in response.json():
time = ciso8601.parse_datetime(item['ETA'])
row = {
'time': time,
'destination': item['Destination'],
'service': self.get_service(item['ServiceNumber'])
}
if item['IsTracked']:
row['live'] = time
departures.append(row)
return departures
def on_message(data):
print(data)
vehicle, created = Vehicle.objects.get_or_create(operator_id='MSOT', code=data['ass'])
datetime = parse_datetime(data['date'])
latlong = Point(data['lng'], data['lat'])
journey = None
service = None
if not created:
latest_location = vehicle.latest_location
current = latest_location and (datetime - latest_location.datetime).total_seconds() < 180
if current and latest_location.journey.service and latest_location.journey.service.geometry.overlaps(latlong):
journey = latest_location.journey
if not journey:
try:
service = Service.objects.filter(
operator='MSOT',
journey__datetime__lte=datetime,
journey__stopusageusage__datetime__gte=datetime,
geometry__bboverlaps=latlong
).distinct().get()
except (Service.DoesNotExist, Service.MultipleObjectsReturned):
pass
if current and not service and not latest_location.journey.service:
journey = latest_location.journey
if not journey:
journey = VehicleJourney.objects.create(
vehicle=vehicle,
datetime=datetime,
source=globalism['source'],
service=service,
route_name=data['extra'].get('custR', '')
)
with atomic():
if not created and latest_location:
latest_location.journey.vehiclelocation_set.update(current=False)
vehicle.latest_location = VehicleLocation.objects.create(
journey=journey,
datetime=datetime,
latlong=latlong,
heading=data['dir'],
current=True
)
vehicle.save()
def test_android_screen(self):
template = self.event_templates['android_screen']
event = self.create_event_log_line(template=template)
expected_key = (self.DEFAULT_DATE, self.DEFAULT_PROJECT)
expected_dict = {
'input_file': '',
'source': self.DEFAULT_PROJECT,
'event_type': 'screen',
'emitter_type': 'server',
'timestamp': ciso8601.parse_datetime('2013-12-17T15:38:32.700000+00:00'),
'received_at': ciso8601.parse_datetime('2013-12-17T15:38:32.796000+00:00'),
'date': datetime.date(*[int(x) for x in self.DEFAULT_DATE.split('-')]),
'agent_type': 'tablet',
'agent_device_name': 'Samsung SM-N920A',
'agent_os': 'Android',
'agent_browser': 'Android',
'agent_touch_capable': True,
'anonymous_id': self.DEFAULT_ANONYMOUS_ID,
'category': 'screen',
'label': 'Launch\\0',
'raw_event': self.get_raw_event(event),
}
expected_value = JsonEventRecord(**expected_dict).to_separated_values()
self.assert_single_map_output(event, expected_key, expected_value)
def graph(self, payload):
req = post("http://commonscontrol.harleyschool.org:8000/data/api/series/", data=dumps(payload)).json()
for series in req:
data = series['data']
dates = [self.zone.normalize(parse_datetime(point["Time"])) for point in data]
cols = [key for key in data[0] if key != "Time"]
for col in cols:
if series["index"]:
label = "%s %s %s" % (series['index'], series['subsystem'], col)
else:
label = "%s %s" % (series['subsystem'], col)
plt.plot(dates, [point[col] for point in data], label=label)
plt.gcf().autofmt_xdate()
plt.legend(loc=0, fontsize=12, fancybox=True)
plt.show()
def parse_datetime(string):
"""Parse ISO-8601 datetime string.
Args:
string (str): Datetime string to parse
Returns:
datetime.datetime: Parsed datetime object
"""
if not string:
datetime_ = datetime.datetime.utcnow().replace(tzinfo=utc)
else:
datetime_ = ciso8601.parse_datetime(string)
if not datetime_:
raise ValueError('Unable to parse timestamp %r' % string)
return datetime_
def parse_datetime(__string: str) -> datetime.datetime:
"""Parse ISO-8601 datetime string.
Args:
__string: Datetime string to parse
Returns:
Parsed datetime object
"""
if not __string:
datetime_ = datetime.datetime.now(datetime.timezone.utc)
else:
# pylint: disable=no-member
datetime_ = ciso8601.parse_datetime(__string)
if datetime_.tzinfo is None:
datetime_ = datetime_.replace(tzinfo=datetime.timezone.utc)
return datetime_
def get_items(self):
fifteen_minutes_ago = timezone.now() - timedelta(minutes=15)
for params in self.get_extents():
try:
response = self.session.get(self.url, params=params, timeout=5)
items = response.json()['items']
any_items = False
if items:
for item in items:
if parse_datetime(item['reported']) > fifteen_minutes_ago:
any_items = True
yield item
if not any_items:
print(response.url)
except (RequestException, KeyError):
continue
sleep(1)
def parse_datetime(string):
"""Parse datetime string.
Args:
string (str): Datetime string to parse
Returns:
datetime.datetime: Parsed datetime object
"""
if not string:
datetime_ = datetime.datetime.utcnow()
else:
datetime_ = ciso8601.parse_datetime(string[:19])
if not datetime_:
raise ValueError('Unable to parse timestamp %r'
% (safer_repr(string), ))
return datetime_
def set_date(self, date):
if date and not isinstance(date, datetime.date):
date = ciso8601.parse_datetime(date).date()
if hasattr(self, 'date'):
if date == self.date:
return
for grouping in self.groupings:
for row in grouping.rows:
row.times.clear()
grouping.column_feet.clear()
self.date = date
for grouping in self.groupings:
for journey in grouping.journeys:
if journey.should_show(self.date, self):
journey.add_times()
grouping.do_heads_and_feet()
def test_unaware(self):
expected = datetime.datetime(2014, 12, 5, 12, 30, 45, 123456)
self.assertEqual(
ciso8601.parse_datetime('2014-12-05T12:30:45.123456'),
expected
)
# parse_datetime_unaware ignores tz offset
self.assertEqual(
ciso8601.parse_datetime_unaware('2014-12-05T12:30:45.123456Z'),
expected
)
self.assertEqual(
ciso8601.parse_datetime_unaware('2014-12-05T12:30:45.123456+00:00'),
expected,
)
self.assertEqual(
ciso8601.parse_datetime_unaware('2014-12-05T12:30:45.123456-05:00'),
expected,
)
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
if self.object.operator:
context['breadcrumb'] = [self.object.operator.region, self.object.operator]
date = self.request.GET.get('date')
context['today'] = timezone.now().date()
if date:
try:
date = ciso8601.parse_datetime(date).date()
except ValueError:
date = None
journeys = self.object.vehiclejourney_set
if not date:
try:
date = journeys.values_list('datetime', flat=True).latest('datetime').date()
except VehicleJourney.DoesNotExist:
date = context['today']
context['date'] = date
journeys = journeys.filter(datetime__date=date)
locations = VehicleLocation.objects.filter(journey=OuterRef('pk'))
context['journeys'] = journeys.select_related('service').annotate(locations=Exists(locations))
return context
