def get_source_data(logger):
source = HtmlSource(logger)
for station_name in source.get_stations_list():
try:
html = source.get_station_html(station_name)
yield station_name, parse_html(html)
logger.info('action=[get_source_data] result=[success] '
'station=[%s]', station_name)
except RequestError as error:
logger.error('action=[get_source_data] result=[fail] station=[%s] '
'%s', station_name, make_one_line(error))
except Exception as error:
logger.error('action=[get_source_data] result=[error] station=[%s] '
'reason=[%s]', station_name, make_one_line(error))
def handle_exception(request_id, logger, error):
logger.error('class=[%s] reason=[%s]', type(error).__name__,
make_one_line(error))
if settings.DEBUG:
raise error
return Response(dict(id=request_id, status=error, message='internal error'),
status=HTTP_500_INTERNAL_SERVER_ERROR)
def monitor_excesses(logger, data):
try:
current = get_current_performed(data)
if MonitorExcessLog.objects.all().exists():
latest = (MonitorExcessLog.objects.all().aggregate(
Max('performed'))['performed__max'])
if latest >= current:
logger.info(
'action=[monitor_excesses] result=[nothing to do] '
'later_than=[%s]', current)
return
else:
latest = current
excess_warnings = find_excesses(logger, data, latest)
MonitorExcessLog(performed=current,
data=json.dumps(list(x._asdict()
for x in excess_warnings),
default=date_handler,
ensure_ascii=False)).save()
logger.info(
'action=[monitor_excesses] result=[success] '
'later_than=[%s]', latest)
except Exception as error:
logger.error('action=[monitor_excesses] result=[error] reason=[%s]',
make_one_line(error))
def get_source_data(logger):
source = HtmlSource(logger)
for station_name in source.get_stations_list():
try:
html = source.get_station_html(station_name)
yield station_name, parse_html(html)
logger.info(
'action=[get_source_data] result=[success] '
'station=[%s]', station_name)
except RequestError as error:
logger.error(
'action=[get_source_data] result=[fail] station=[%s] '
'%s', station_name, make_one_line(error))
except Exception as error:
logger.error(
'action=[get_source_data] result=[error] station=[%s] '
'reason=[%s]', station_name, make_one_line(error))
def handle_invalid_form(request_id, logger, error):
logger.warning('class=[%s] reason=[%s]',
type(error).__name__, make_one_line(error))
return Response(dict(id=request_id,
status='error',
message=str(error),
errors=error.errors),
status=HTTP_400_BAD_REQUEST)
def handle_exception(request_id, logger, error):
logger.error('class=[%s] reason=[%s]',
type(error).__name__, make_one_line(error))
if settings.DEBUG:
raise error
return Response(dict(id=request_id, status=error,
message='internal error'),
status=HTTP_500_INTERNAL_SERVER_ERROR)
def add_source_data(logger, data):
for station_name, station_data in data.iteritems():
try:
add_data(station_name, station_data)
logger.info('action=[add_source_data] result=[success] '
'station=[%s]', station_name)
except Exception as error:
logger.error('action=[add_source_data] result=[error] station=[%s] '
'reason=[%s]', station_name, make_one_line(error))
def add_source_data(logger, data):
for station_name, station_data in data.iteritems():
try:
add_data(station_name, station_data)
logger.info(
'action=[add_source_data] result=[success] '
'station=[%s]', station_name)
except Exception as error:
logger.error(
'action=[add_source_data] result=[error] station=[%s] '
'reason=[%s]', station_name, make_one_line(error))
def monitor_excesses(logger, data):
try:
current = get_current_performed(data)
if MonitorExcessLog.objects.all().exists():
latest = (MonitorExcessLog.objects.all()
.aggregate(Max('performed'))['performed__max'])
if latest >= current:
logger.info('action=[monitor_excesses] result=[nothing to do] '
'later_than=[%s]', current)
return
else:
latest = current
excess_warnings = find_excesses(logger, data, latest)
MonitorExcessLog(
performed=current,
data=json.dumps(list(x._asdict() for x in excess_warnings),
default=date_handler, ensure_ascii=False)
).save()
logger.info('action=[monitor_excesses] result=[success] '
'later_than=[%s]', latest)
except Exception as error:
logger.error('action=[monitor_excesses] result=[error] reason=[%s]',
make_one_line(error))
def handle_invalid_form(request_id, logger, error):
logger.warning('class=[%s] reason=[%s]', type(error).__name__,
make_one_line(error))
return Response(dict(id=request_id, status='error', message=str(error),
errors=error.errors),
status=HTTP_400_BAD_REQUEST)
def handle_object_does_not_exists(request_id, logger, error):
logger.warning('class=[%s] reason=[%s]', type(error).__name__,
make_one_line(error))
return Response(dict(id=request_id, status='error', message=str(error)),
status=HTTP_404_NOT_FOUND)
def handle_object_does_not_exists(request_id, logger, error):
logger.warning('class=[%s] reason=[%s]',
type(error).__name__, make_one_line(error))
return Response(dict(id=request_id, status='error', message=str(error)),
status=HTTP_404_NOT_FOUND)