def remove_expired_notifications():
print('Cleaning expired notifications...')
# удаляем старые шаблоны уведомлений
notifications = Notification.objects\
.filter(expired_at__lt=utcnow())\
.select_related('job', 'job__user')
session_cache = {}
i = 0
for notification in notifications.iterator():
user = notification.job.user
if user not in session_cache:
sess_id = get_wialon_session_key(user)
session_cache[user] = sess_id
else:
sess_id = session_cache[user]
try:
remove_notification(notification, user, sess_id)
except WialonException as e:
print(e)
else:
notification.delete()
i += 1
for user, sess_id in session_cache.items():
logout_session(user, sess_id)
print('%s expired notifications removed' % i)
def post(self, request, *args, **kwargs):
doc = request.data.xpath('/routesRequest')
if len(doc) < 1:
return error_response('Не найден объект routesRequest',
code='routesRequest_not_found')
doc = doc[0]
doc_id = doc.get('idDoc', '')
if not doc_id:
return error_response('Не указан параметр idDoc',
code='idDoc_not_found')
try:
org_id = int(doc.get('idOrg', ''))
except ValueError:
org_id = 0
sess_id = get_wialon_session_key(request.user)
try:
routes = get_routes(sess_id)
except APIProcessError as e:
return error_response(str(e), code=e.code)
finally:
logout_session(request.user, sess_id)
context = self.get_context_data(**kwargs)
context.update({
'doc_id': doc_id,
'create_date': utcnow(),
'routes': routes,
'org_id': org_id
})
return XMLResponse('ura/routes.xml', context)
def email_reports():
print('Mailing monthly driving style total report...')
reports = models.DrivingStyleTotalReportDelivery.objects.published()
now = utcnow()
for report in reports:
print('Monthly VCHM Driving style total report %s' % report)
for user in report.users.all():
local_now = utc_to_local_time(now, user.timezone)
if not user.email:
print('Skipping user %s (no email)' % user)
continue
print('User %s' % user)
sess_id = None
try:
# получаем отчеты через HTTP
sess_id = get_wialon_session_key(user)
# если время выполнения - 1е число месяца, то отчет сформируется за прошлый месяц,
# иначе отчет подготовится за текущий месяц вчерашний день включительно.
date_to = (local_now - datetime.timedelta(days=1)).date()
date_from = date_to.replace(day=1)
res = make_report(report,
user,
sess_id,
date_from=date_from,
date_to=date_to,
attempts=0)
filename = 'total_vchm_driving_report_%s.xls' % user.pk
log = models.ReportEmailDeliveryLog(
user=user,
email=user.email,
report_type=EmailDeliveryReportTypeEnum.DRIVING_STYLE,
subject='Сводный отчет о качестве вождения (ВЧМ)',
body='Здравствуйте, %s. Отчет по вложении.' %
user.full_name)
content = ContentFile(res.content)
log.report.save(filename, content, save=False)
log.save()
log.send(reraise=True)
except Exception as e:
print('Error: %s' % e)
send_trigger_email('Ошибка в работе системы рассылки отчетов',
extra_data={
'Exception': str(e),
'Traceback': traceback.format_exc(),
'report': report,
'user': user
})
finally:
if sess_id:
logout_session(user, sess_id)
def forwards_func(apps, schema_editor):
job_model = apps.get_model('ura', 'UraJob')
user_model = apps.get_model('users', 'User')
orgs = user_model.objects.filter(is_active=True, supervisor__isnull=False)
units_cache = {}
routes_cache = {}
for org in orgs.iterator():
print('Organization %s' % org.username)
sess_id = get_wialon_session_key(org)
units = get_units(sess_id)
units_cache[org.pk] = {
u['id']: '%s (%s) [%s]' % (u['name'], u['number'], u['vin'])
for u in units
}
routes = get_routes(sess_id)
logout_session(org, sess_id)
routes_cache[org.pk] = {r['id']: r['name'] for r in routes}
sess_id = None
print('Jobs count: %s' % job_model.objects.count())
i = 0
for job in job_model.objects.iterator():
i += 1
if not job.user_id:
print('%s: Job %s missed due to lack of user' % (i, job.pk))
continue
unit_title = route_title = None
try:
unit_id = int(job.unit_id)
unit_title = units_cache.get(job.user_id, {}).get(unit_id)
if unit_title:
job.unit_title = unit_title
except (ValueError, TypeError, AttributeError):
pass
try:
route_id = int(job.route_id)
route_title = routes_cache.get(job.user_id, {}).get(route_id)
if route_title:
job.route_title = route_title
except (ValueError, TypeError, AttributeError):
pass
if unit_title or route_title:
job.save()
print('%s: unit=%s, route=%s (%s)' %
(i, unit_title, route_title, job.created))
def get(self, request, **kwargs):
now = utc_to_local_time(utcnow(), request.user.timezone)
local_dt_from = now.replace(hour=0, minute=0, second=0)
local_dt_to = now.replace(hour=23, minute=59, second=59)
user = User.objects.get(pk=1)
sess_id = get_wialon_session_key(user)
moving_service = self.service_class(
user=user,
local_dt_from=local_dt_from,
local_dt_to=local_dt_to,
sess_id=sess_id
)
moving_service.exec_report()
moving_service.analyze()
logout_session(user, sess_id)
return success_response()
def email_reports(period=None):
print('Mailing %s driving style report...' % period)
if not period or period not in ('daily', 'weekly', 'monthly'):
print('Period not specified')
return
reports = models.DrivingStyleReportDelivery.objects.published()
period_verbose = ''
if period == 'daily':
period_verbose = 'Ежедневный'
reports = reports.filter(is_daily=True)
elif period == 'weekly':
period_verbose = 'Еженедельный'
reports = reports.filter(is_weekly=True)
elif period == 'monthly':
period_verbose = 'Ежемесячный'
reports = reports.filter(is_monthly=True)
now = utcnow()
for report in reports:
print('%s VCHM Driving style report %s' % (period, report))
for user in report.users.all():
local_now = utc_to_local_time(now, user.timezone)
if not user.email:
print('Skipping user %s (no email)' % user)
continue
if local_now.hour != SEND_HOUR:
print('Skipping user %s (%s != %s)' %
(user, local_now.hour, SEND_HOUR))
continue
print('User %s' % user)
sess_id = None
try:
# получаем отчеты через HTTP
sess_id = get_wialon_session_key(user)
date_from = date_to = (local_now -
datetime.timedelta(days=1)).date()
if period == 'daily':
date_from = date_to
elif period == 'weekly':
date_from = (local_now - datetime.timedelta(days=7)).date()
elif period == 'monthly':
date_from = date_to.replace(day=1)
res = make_report(report,
user,
sess_id,
date_from=date_from,
date_to=date_to,
attempts=0)
filename = '%s_vchm_driving_report_%s.xls' % (period, user.pk)
log = models.ReportEmailDeliveryLog(
user=user,
email=user.email,
report_type=EmailDeliveryReportTypeEnum.DRIVING_STYLE,
subject='%s отчет о качестве вождения (ВЧМ)' %
period_verbose,
body='Здравствуйте, %s. Отчет по вложении.' %
user.full_name)
content = ContentFile(res.content)
log.report.save(filename, content)
log.save()
log.send(reraise=True)
except Exception as e:
print('Error: %s' % e)
send_trigger_email('Ошибка в работе системы рассылки отчетов',
extra_data={
'Exception': str(e),
'Traceback': traceback.format_exc(),
'report': report,
'user': user
})
finally:
if sess_id:
logout_session(user, sess_id)
def pre_view_trigger(self, request, **kwargs):
super(BaseUraRidesView, self).pre_view_trigger(request, **kwargs)
self.sess_id = get_wialon_session_key(request.user)
def cache_geozones():
print('Caching geozones...')
with transaction.atomic():
i = 1
users = User.objects.filter(supervisor__isnull=False,
wialon_username__isnull=False,
wialon_password__isnull=False).exclude(
wialon_username='', wialon_password='')
print('%s users found' % len(users))
for user in users:
sess_id = get_wialon_session_key(user)
print('%s) User %s processing' % (i, user))
routes = get_routes(sess_id, with_points=True)
logout_session(user, sess_id)
del sess_id
print('%s routes found' % len(routes))
for route in routes:
print('Route %s' % route['name'])
name = route['name'].strip()
job_template, created = StandardJobTemplate.objects.get_or_create(
wialon_id=str(route['id']),
defaults={
'title': name,
'user': user
})
existing_points = set()
if not created:
if job_template.title != name:
job_template.title = name
job_template.user = user
job_template.save()
existing_points = {p for p in job_template.points.all()}
# убираем дубли (когда одна и та же геозона дублируется в маршруте)
route['points'] = {r['id']: r
for r in route['points']}.values()
print('%s points found, already exist: %s' %
(len(route['points']), len(existing_points)))
for point in route['points']:
name = point['name'].strip()
standard_point, created = StandardPoint.objects.get_or_create(
job_template=job_template,
wialon_id=str(point['id']),
defaults={'title': name})
if not created:
existing_points.discard(standard_point)
if standard_point.title != name:
standard_point.title = name
standard_point.save()
if existing_points:
print('Points to remove: %s' %
', '.join([str(x) for x in existing_points]))
for existing_point in existing_points:
existing_point.delete()
sleep(.3)
i += 1
def exec_report(user,
template_id,
sess_id,
dt_from,
dt_to,
report_resource_id=None,
object_id=None,
attempts=3):
if report_resource_id is None:
error = 'не выявлено'
try:
report_resource_id = get_wialon_report_resource_id(user, sess_id)
except WialonException as e:
error = e
if not report_resource_id:
raise ReportException(
'Не удалось получить ID ресурса для пользователя %s '
'(наименование ресурса: %s). Ошибка: %s' %
(str(user), user.wialon_resource_name, error))
if object_id is None:
error = 'не выявлено'
try:
object_id = get_wialon_report_object_id(user, sess_id)
except WialonException as e:
error = e
if not object_id:
raise ReportException(
'Не удалось получить ID группового объекта для пользователя %s '
'(наименование группового объекта: %s). Ошибка: %s' %
(str(user), user.wialon_group_object_name, error))
# замедляем в случае чего, для прохождения лимита
throttle_report(user)
r = requests.post(
settings.WIALON_BASE_URL + '?svc=report/exec_report&sid=' + sess_id, {
'params':
json.dumps({
'reportResourceId': report_resource_id,
'reportTemplateId': template_id,
'reportTemplate': None,
'reportObjectId': object_id,
'reportObjectSecId': 0,
'interval': {
'flags': 0,
'from': dt_from,
'to': dt_to
}
}),
'sid':
sess_id
})
result = load_requests_json(r)
if 'error' in result:
# сессия неожиданно устарела (такое очень редко и необъяснимо бывает) - отправляем еще раз
if result['error'] == 1:
if attempts > 1:
# генерируем новую сессию
sess_id = get_wialon_session_key(user, invalidate=True)
result = exec_report(user,
template_id,
sess_id,
dt_from,
dt_to,
report_resource_id=report_resource_id,
object_id=object_id,
attempts=attempts - 1)
logout_session(user, sess_id)
return result
raise ReportException(WIALON_SESSION_EXPIRED)
raise ReportException(WIALON_INTERNAL_EXCEPTION % result)
return result
def post(self, request, *args, **kwargs):
jobs = []
jobs_els = request.data.xpath('/setJobs/job')
sess_id = get_wialon_session_key(request.user)
if jobs_els:
for j in jobs_els:
data = parse_xml_input_data(request, self.model_mapping, j)
name = data.get('name')
if not name:
logout_session(request.user, sess_id)
return error_response('Не указан параметр jobName',
code='jobName_not_found')
routes = get_routes(sess_id, with_points=True)
units = get_units(sess_id)
routes_ids = [x['id'] for x in routes]
if data['route_id'] not in routes_ids:
return error_response(
'Шаблон задания idRoute неверный или не принадлежит текущей организации',
code='route_permission')
units_cache = {
u['id']:
'%s (%s) [%s]' % (u['name'], u['number'], u['vin'])
for u in units
}
try:
data['unit_title'] = units_cache.get(int(data['unit_id']))
except (ValueError, TypeError, AttributeError):
pass
if not data['unit_title']:
return error_response(
'Объект ID=%s не найден в текущем ресурсе организации'
% data['unit_id'],
code='unit_not_found_permission')
routes_cache = {r['id']: r for r in routes}
try:
data['route_title'] = routes_cache.get(
int(data['route_id']), {}).get('name')
except (ValueError, TypeError, AttributeError):
pass
data['user'] = request.user
self.job = self.model.objects.create(**data)
register_job_notifications(self.job,
sess_id,
routes_cache=routes_cache)
logout_session(request.user, sess_id)
jobs.append(self.job)
context = self.get_context_data(**kwargs)
context.update({'now': utcnow(), 'acceptedJobs': jobs})
return XMLResponse('ura/ackjobs.xml', context)
def get_context_data(self, **kwargs):
kwargs = super(VchmDrivingStyleView, self).get_context_data(**kwargs)
self.is_total = bool(self.request.POST.get('total_report'))
total_report_data = []
form = kwargs['form']
sess_id = self.request.session.get('sid')
if not sess_id:
raise ReportException(WIALON_NOT_LOGINED)
try:
units_list = get_units(sess_id, extra_fields=True)
except WialonException as e:
raise ReportException(str(e))
kwargs['units'] = units_list
if not self.request.POST:
return kwargs
units_dict = {u['name']: u for u in units_list}
if form.is_valid():
if self.is_total:
user = User.objects.filter(
is_active=True,
username=self.request.session.get('user')
).first()
if not user:
raise ReportException(WIALON_USER_NOT_FOUND)
report_users = UserTotalReportUser.objects\
.published()\
.order_by('ordering')\
.filter(executor_user=user)\
.select_related('report_user', 'report_user__ura_user')
users = set(user.report_user for user in report_users)
else:
user = User.objects.filter(
is_active=True,
wialon_username=self.request.session.get('user')
).first()
if not user:
raise ReportException(WIALON_USER_NOT_FOUND)
users = {user}
dt_from = local_to_utc_time(datetime.datetime.combine(
form.cleaned_data['dt_from'],
datetime.time(0, 0, 0)
), user.timezone)
dt_to = local_to_utc_time(datetime.datetime.combine(
form.cleaned_data['dt_to'],
datetime.time(23, 59, 59)
), user.timezone)
for user in users:
report_data = []
print('Evaluating report for user %s' % user)
ura_user = user.ura_user if user.ura_user_id else user
print('URA user is %s' % ura_user)
if self.request.POST.get('total_report'):
sess_id = get_wialon_session_key(user)
if not sess_id:
raise ReportException(WIALON_NOT_LOGINED)
try:
units_list = get_units(sess_id, extra_fields=True)
except WialonException as e:
raise ReportException(str(e))
kwargs['units'] = units_list
units_dict = {u['name']: u for u in units_list}
jobs = Job.objects\
.filter(user=ura_user, date_begin__lt=dt_to, date_end__gt=dt_from)
if form.cleaned_data.get('unit'):
jobs = jobs.filter(unit_id=str(form.cleaned_data['unit']))
self.driver_cache = {
j.driver_id: j.driver_fio
for j in jobs
if j.driver_fio.lower() != 'нет в.а.'
}
self.driver_id_cache = {
int(j.unit_id): j.driver_id
for j in jobs
if j.driver_fio.lower() != 'нет в.а.'
}
template_id = get_wialon_report_template_id('driving_style', user, sess_id)
mobile_vehicle_types = set()
if user.wialon_mobile_vehicle_types:
mobile_vehicle_types = set(
x.strip() for x in user.wialon_mobile_vehicle_types.lower().split(',')
)
cleanup_and_request_report(user, template_id, sess_id)
report_kwargs = {}
if form.cleaned_data.get('unit'):
report_kwargs['object_id'] = form.cleaned_data['unit']
print('Executing report...')
r = exec_report(
user,
template_id,
sess_id,
int(time.mktime(dt_from.timetuple())),
int(time.mktime(dt_to.timetuple())),
**report_kwargs
)
wialon_report_rows = {}
for table_index, table_info in enumerate(r['reportResult']['tables']):
wialon_report_rows[table_info['name']] = get_report_rows(
sess_id,
table_index,
table_info['rows'],
level=2 if table_info['name'] == 'unit_group_ecodriving' else 1
)
self.mileage_cache = {
row['c'][0]: parse_float(row['c'][1], default=.0)
for row in wialon_report_rows.get('unit_group_trips', [])
}
self.duration_cache = {
row['c'][0]: parse_timedelta(row['c'][2]).total_seconds()
for row in wialon_report_rows.get('unit_group_trips', [])
}
i = 0
for row in wialon_report_rows.get('unit_group_ecodriving', []):
i += 1
violations = [
self.parse_report_row(x['c'], user, total=False)
for x in row['r']
]
row = self.parse_report_row(row['c'], user, total=True)
unit = units_dict.get(row.unit_name)
if not unit:
print('%s) Unit not found: %s' % (i, row.unit_name))
continue
vehicle_type = unit['vehicle_type'].lower()
if mobile_vehicle_types and vehicle_type \
and vehicle_type not in mobile_vehicle_types:
print('%s) Skip vehicle type "%s" of item %s' % (
i, vehicle_type, row.unit_name
))
continue
print('%s) Processing %s' % (i, row.unit_name))
ecodriving = get_drive_rank_settings(unit['id'], sess_id)
ecodriving = {k.lower(): v for k, v in ecodriving.items()}
report_row = self.new_grouping(row, unit)
# собственно расчеты метрик
for violation in violations:
verbose = violation.violation_name
violation_name = ''
violation_scope = 'violations_measures'
if 'превышение скорости' in verbose:
if 'cреднее' in verbose or 'среднее' in verbose:
violation_name = 'avg_overspeed'
elif 'опасное' in verbose:
violation_name = 'critical_overspeed'
elif 'ремень' in verbose or 'ремня' in verbose:
violation_name = 'belt'
elif 'фар' in verbose:
violation_name = 'lights'
elif 'кму' in verbose or 'стрел' in verbose:
violation_name = 'jib'
elif 'разгон' in verbose or 'ускорение' in verbose:
violation_scope = 'per_100km_count'
violation_name = 'accelerations'
elif 'торможение' in verbose:
violation_scope = 'per_100km_count'
violation_name = 'brakings'
elif 'поворот' in verbose:
violation_scope = 'per_100km_count'
violation_name = 'turns'
if not violation_name:
print(
'%s) %s: unknown violaton name %s' % (
i, row.unit_name, verbose
)
)
continue
scope = report_row[violation_scope][violation_name]
if violation_scope == 'per_100km_count':
scope['count'] += violation.violation_count / row.mileage * 100
scope['total_count'] += violation.violation_count
else:
scope['count'] += violation.violation_count
scope['total_time_percentage'] += (
violation.duration / row.duration * 100
)
scope['time_sec'] += violation.duration
# суммируем штрафы
rating_violation_name = violation_name
if 'overspeed' in violation_name:
rating_violation_name = 'overspeed'
# извлечем настройки объектов и узнаем, нужно ли рассчитывать
# относительно пробега
settings = ecodriving.get(verbose)
devider = 1
if settings and settings.get('flags', 0) in (2, 3, 7, 10):
devider = max(1.0, row.mileage)
fine = violation.fine / devider
report_row['rating'][rating_violation_name]['fine'] += fine
report_row['rating_total']['avg']['fine'] += fine
if rating_violation_name in ('belt', 'lights', 'jib', 'brakings'):
report_row['rating_total']['critical_avg']['fine'] += fine
# расчет статистики (рейтинга)
for key in report_row['rating']:
scope = report_row['rating'][key]
scope['rating'] = self.calculate_rating(scope['fine'])
report_row['rating_total']['avg']['rating'] = self.calculate_rating(
report_row['rating_total']['avg']['fine']
)
report_row['rating_total']['critical_avg']['rating'] = self.calculate_rating(
report_row['rating_total']['critical_avg']['fine']
)
report_data.append(report_row)
# группируем строки по нарушителям и сортируем, самых нарушающих наверх
groups = defaultdict(lambda: {
'rows': [],
'driver_id': '',
'driver_fio': '',
'company_name': ''
})
# сначала отсортируем без группировки,
# чтобы внутри групп была правильная сортировка
report_data = sorted(
report_data,
key=lambda x: x['rating_total']['critical_avg']['rating']
)
for row in report_data:
group = groups[row['driver_id']]
group['driver_id'] = row['driver_id']
group['driver_fio'] = self.driver_cache.get(row['driver_id'], DRIVER_NO_NAME)
group['company_name'] = user.company_name or user.wialon_resource_name or ''
group['stats'] = self.new_grouping()
group['rows'].append(row)
# собираем суммарную статистику по каждому водителю
report_data = list(groups.values())
for group in report_data:
for row in group['rows']:
group['stats']['total_mileage'] += row['total_mileage']
group['stats']['total_duration'] += row['total_duration']
for field in ('avg_overspeed', 'critical_overspeed', 'belt', 'lights', 'jib'):
for row in group['rows']:
group['stats']['violations_measures'][field]['count'] += \
row['violations_measures'][field]['count']
group['stats']['violations_measures'][field]['time_sec'] += \
row['violations_measures'][field]['time_sec']
group['stats']['violations_measures'][field]['total_time_percentage'] = \
group['stats']['violations_measures'][field]['time_sec'] \
/ group['stats']['total_duration'] * 100.0
for field in ('brakings', 'accelerations', 'turns'):
for row in group['rows']:
group['stats']['per_100km_count'][field]['total_count'] += \
row['per_100km_count'][field]['total_count']
group['stats']['per_100km_count'][field]['count'] = \
group['stats']['per_100km_count'][field]['total_count'] \
/ group['stats']['total_mileage'] * 100
for field in (
'overspeed', 'belt', 'lights', 'brakings', 'accelerations', 'turns', 'jib'
):
for row in group['rows']:
fine = row['rating'][field]['fine']
group['stats']['rating'][field]['fine'] += fine
group['stats']['rating_total']['avg']['fine'] += fine
if field in ('belt', 'lights', 'jib', 'brakings'):
group['stats']['rating_total']['critical_avg']['fine'] += fine
# расчет статистики (рейтинга)
for key in group['stats']['rating']:
scope = group['stats']['rating'][key]
scope['rating'] = self.calculate_rating(scope['fine'])
group['stats']['rating_total']['avg']['rating'] = self.calculate_rating(
group['stats']['rating_total']['avg']['fine']
)
group['stats']['rating_total']['critical_avg']['rating'] = \
self.calculate_rating(
group['stats']['rating_total']['critical_avg']['fine']
)
total_report_data.extend(report_data)
# финально отсортируем всю группу
total_report_data = sorted(
total_report_data,
key=lambda x: x['stats']['rating_total']['critical_avg']['rating']
)
kwargs['report_data'] = total_report_data
return kwargs
