def handle_exception(self, request, e):
"""Default exception handling that kicks in when an exception
occours that is not caught. In debug mode the exception will
be re-raised immediately, otherwise it is logged and the handler
for a 500 internal server error is used. If no such handler
exists, a default 500 internal server error message is displayed.
"""
exc_type, exc_value, tb = sys.exc_info()
handler = self.error_handlers.get(500)
if self.propagate_exceptions:
# if we want to repropagate the exception, we can attempt to
# raise it with the whole traceback in case we can do that
# (the function was actually called from the except part)
# otherwise, we just raise the error again
if exc_value is e:
raise exc_type, exc_value, tb
else:
raise e
logbook.exception()
if handler is None:
return werkzeug.exceptions.InternalServerError()
return self.call_error_handler(handler, request, exception = e)
def run(self):
try:
self.server = StoppableServer(**configs.fake_openstack)
bottle.run(server=self.server)
except:
logbook.exception()
raise
def tokens(self):
"""Handle auth requests, patch service catalog endpoint urls"""
response = self.make_requests_request(bottle.request, urljoin(self.auth_url, 'tokens'))
if response.status_code != 200 or not self.patch_service_catalog:
logbook.info('Proxing tokens request to openstack without patching ({})', response.status_code)
return self.make_bottle_response(response)
try:
parsed = response.json()
for service_dict in parsed.get('access', {}).get('serviceCatalog', []):
service_name = service_dict['name']
endpoint = service_dict['endpoints'][0]
for item in endpoint:
if item.endswith('URL'):
name = service_name+'_'+item[:-3]
self.service_mapping[name] = endpoint[item] # e.g. nova_public, keystone_admin
endpoint[item] = self.urljoin(self.fake_auth_url, 'mock', name) + '/'
dump = json.dumps(parsed)
except Exception:
logbook.exception('Error while patching service catalog')
logbook.warning('Tokens content: {}', response.content)
raise
logbook.debug('service mapping is: {}', self.service_mapping)
headers = self.filter_headers(response.headers)
headers['Content-Length'] = len(dump)
return bottle.HTTPResponse(dump, response.status_code, headers)
def response_process(self, response, work_time):
if response is None:
logbook.error("Response can't be None")
response = body_to_log = "{}"
status = 200
elif isinstance(response, bottle.Response):
if response.content_type.startswith("text/") or response.content_type == "application/json":
body_to_log = str(response.body) or getattr(response, "message", "")
else:
body_to_log = response.content_type
status = response.status
elif isinstance(response, dict):
status = 200
try:
body_to_log = json.dumps(response, cls=DateTimeJSONEncoder)
if self.debug:
response = json.dumps(response, cls=DateTimeJSONEncoder, indent=4)
else:
response = body_to_log
except TypeError:
logbook.exception("Can't encode reply: {}", response)
raise bottle.HTTPError(body="Internal Server Error")
bottle.response.content_type = 'application/json'
else:
logbook.error("Incorrect response ({}): {}", type(response), response)
body_to_log = str(response)
status = 200
self.log_response(body_to_log, status, work_time)
return response
def wrapper(*args, **kwargs):
start_time = time.time()
request_id = self.generate_id()
def inject_request_id(record):
record.extra['request_id'] = request_id
with logbook.Processor(inject_request_id):
logbook.notice(self.request_str(), extra={"api": True})
try:
response = callback(*args, **kwargs)
except OperationalError as e:
logbook.warning("Database is down {}: {}", conf.database.uri, e, exc_info=True)
logbook.error("Database is down {}: {}", conf.database.uri, e)
response = errors.DatabaseIsDown()
except errors.BadRequest as e:
e.format_response()
response = e
except bottle.HTTPResponse as e:
response = e
except Exception as e:
if self.under_test:
import traceback
traceback.print_exc()
logbook.exception("Exception during processing request: %s %s" %
(bottle.request.method, bottle.request.path))
self.log_response(str(e), 500, time.time() - start_time)
raise
finally:
from model import db
db.session.remove()
response = self.response_process(response, time.time() - start_time)
return response
def handle_exception(self, request, e):
"""Default exception handling that kicks in when an exception
occours that is not caught. In debug mode the exception will
be re-raised immediately, otherwise it is logged and the handler
for a 500 internal server error is used. If no such handler
exists, a default 500 internal server error message is displayed.
"""
exc_type, exc_value, tb = sys.exc_info()
handler = self.error_handlers.get(500)
if self.propagate_exceptions:
# if we want to repropagate the exception, we can attempt to
# raise it with the whole traceback in case we can do that
# (the function was actually called from the except part)
# otherwise, we just raise the error again
if exc_value is e:
raise exc_type, exc_value, tb
else:
raise e
logbook.exception()
if handler is None:
return werkzeug.exceptions.InternalServerError()
return self.call_error_handler(handler, request, exception=e)
def parse_bank_metadata(
bank: Path,
queue: mp.Queue) -> Tuple[List[BankMetaData], List[BankMetaData]]:
setup_logging(queue)
streamed_flag = False
memory_flag = False
ret_memory = []
ret_streamed = []
try:
logbook.info("parsing bank metadata '{bank}'", bank=bank.absolute())
with bank.open("r", encoding="utf-8") as f:
for line in f:
if line.lower().startswith("in memory audio"):
memory_flag = True
continue
elif line.lower().startswith("streamed audio"):
streamed_flag = True
continue
elif line == "\r\n" or line == "\n":
streamed_flag = False
memory_flag = False
continue
if memory_flag:
ret_memory.append(parse_memory_audio_meta(line))
elif streamed_flag:
ret_streamed.append(parse_streamed_audio_meta(line))
except Exception as e:
logbook.exception("error parsing {bank}: {e}", bank=bank, e=repr(e))
return ret_memory, ret_streamed
def wrapped_f(s, source, *args, **kwargs):
if self.perm.name in source.permissions:
try:
return (yield from f(s, source, *args, **kwargs))
except (IndexError, KeyError, ValueError) as e:
exception("Unhandled exception in permission wrapper. "
"Silently dropping.")
else:
raise PermissionError(self.perm)
def log_exception(record, **kwargs):
msg = json.dumps(record) if isinstance(record, dict) else record
msg = '`'.join(map(str, record))
with err_handler:
exception(msg)
try:
sentry_exception_handler(extra={'msg': str(msg)}, **kwargs)
except:
pass
def revorb(ogg_file: Path, queue: mp.Queue):
setup_logging(queue)
logbook.info("running revorb on {f}", f=ogg_file.absolute())
try:
subprocess.check_call([REVORB, str(ogg_file.absolute())])
except subprocess.CalledProcessError as cpe:
logbook.exception("error running revorb on {f}: {e}",
f=ogg_file.absolute(),
e=cpe)
def log_exception(record, **kwargs):
msg = json.dumps(record) if isinstance(record, dict) else record
msg = '`'.join(map(str, record))
with err_handler:
exception(msg)
try:
import traceback
traceback.print_exc()
sentry_exception_handler(extra={'msg': str(msg)}, **kwargs)
except:
pass
def on_celery_task_failure(sender=None,
task_id=None,
exception=None,
args=None,
kwargs=None,
traceback=None,
einfo=None,
**other_kwargs):
import logbook
logbook.exception('%s' % exception,
exc_info=(einfo.type, einfo.exception, einfo.tb),
extra=dict(task_id=task_id, args=args, kwargs=kwargs))
def thread_wrapped_task():
with inner_thread_nested_setup(outer_handlers):
try:
# Reload from db
from kettle.tasks import Task
task = Task._from_id(task_id)
getattr(task, method_name)()
except Exception:
# TODO: Fix logging
print traceback.format_exc()
logbook.exception()
abort.set()
def wrap(*args, **kwargs):
try:
res = fn(*args, **kwargs)
db.session.commit()
return res
except errors.BadRequest:
db.session.rollback()
raise
except Exception:
logbook.exception("Exception in function {}:", fn)
db.session.rollback()
raise
def on_celery_task_failure(
sender=None,
task_id=None,
exception=None,
args=None,
kwargs=None,
traceback=None,
einfo=None,
**other_kwargs):
import logbook
logbook.exception(
'%s' % exception,
exc_info=(einfo.type, einfo.exception, einfo.tb),
extra=dict(task_id=task_id, args=args, kwargs=kwargs))
def thread_wait(thread, abort):
try:
while True:
if abort.is_set():
pass # TODO: Set some kinda timeout
if thread.is_alive():
thread.join(1)
else:
break
except Exception:
# TODO: Fix logging
print traceback.format_exc()
logbook.exception()
abort.set()
def get(cls, token_id):
key = cls.prefixed_key(token_id)
data = cls.redis().hgetall(key)
if not data:
logbook.debug("Token {} not found in memdb {}", cls.__name__, key)
raise cls.invalid_token()
try:
data = {
k.decode("ascii"): v.decode("ascii")
for k, v in data.items()
}
return cls.Token(**data)
except TypeError:
logbook.exception()
raise cls.invalid_token()
def exception_to_sentry(extra=None):
import conf
import traceback
from raven import Client
if conf.test or conf.region.lower() == "local" or not conf.sentry.backend:
traceback.print_exc()
logbook.exception("")
return
data = {
"version": getattr(conf, "version", None)
}
data.update(extra or {})
client = Client(conf.sentry.backend)
client.captureException(extra=data, tags=conf.logging.handlers.Sentry.tags)
def _render(cls, text, context):
try:
env = Environment(undefined=StrictUndefined)
ast = env.parse(text)
unexpected = meta.find_undeclared_variables(ast) - context.keys()
if unexpected:
logbook.warning("Unexpected variables in template: {}. Context: {}, Template: {}",
", ".join(unexpected), context, text)
raise errors.MessageTemplateError(_("Unexpected variables in template: {}"), ", ".join(unexpected))
template = Template(text, undefined=StrictUndefined)
rendered = template.render(context)
return rendered
except TemplateSyntaxError as e:
logbook.exception("Render template error: {}. Context: {}, template: {}", e.message, context, text)
raise errors.MessageTemplateError(_("Template syntax error: {} at line {}.\nText: {}\nContext: {}"),
e.message, e.lineno, text, context)
except TemplateError as e:
logbook.exception("Render template error: {}. Context: {}, template: {}, ", e, context, text)
raise errors.MessageTemplateError(_("Error while rendering template: %s") % e.message)
def collect_usage(self, tenant, mutex, end=None):
# Collects usage for a given tenant from when they were last collected,
# up to the given end, and breaks the range into one hour windows.
end = end or datetime.utcnow()
time_label = TimeLabel(tenant.last_collected + timedelta(minutes=1))
end_time_label = TimeLabel(end)
usage = {}
logbook.info(
'collect_usage for {}, from {} till {} (last_collected: {})',
tenant, time_label, end_time_label, tenant.last_collected)
customer = Customer.get_by_tenant_id(tenant.tenant_id)
if not customer:
logbook.error("Customer for tenant {} not found", tenant)
return usage
while time_label < end_time_label:
try:
usages = self._collect_usage(tenant, time_label, customer)
tenant.last_collected = time_label.datetime_range()[1]
if usages:
db.session.add(customer)
total_cost = customer.calculate_usage_cost(usages)
customer.withdraw(total_cost)
if not conf.test:
db.session.commit()
usage[time_label] = [usage.to_dict()
for usage in usages], total_cost
except Exception:
self.errors += 1
import traceback
traceback.print_exc()
logbook.exception("Usage process failed for {} and {}", tenant,
time_label)
db.session.rollback()
return usage
time_label = time_label.next()
mutex.update_ttl()
return usage
async def vote_click(chat: Chat, cq: CallbackQuery, match):
logbook.info("{}", cq)
vote_id = match.group(1)
point = match.group(2)
result = "Answer {} accepted".format(point)
game = await storage.get_game(chat.id, vote_id)
if not game:
return await cq.answer(text="No such game")
if game.revealed:
return await cq.answer(text="Can't change vote after cards are opened")
game.add_vote(cq.src["from"], point)
await storage.save_game(game)
try:
await bot.edit_message_text(chat.id, game.reply_message_id,
**game.get_send_kwargs())
except BotApiError:
logbook.exception("Error when updating markup")
await cq.answer(text=result)
def call_error_handler(self, handler_class, request=None, **kwargs):
"""calls an error handler with the given arguments
error handlers are instances of :class:`~starflyer.Handler` and are
called the same way. If a handler raises an exception themselves
the default internal server error is used.
:param handler_class: the Handler class to use
:param request: The request object which was used when the exception occurred.
:param kwargs: additional keyword arguments being passed to the ``get()`` method of
the handler.
"""
handler = handler_class(self, request)
# process the request via the handler
try:
return handler(**kwargs)
except Exception, e:
logbook.exception()
return werkzeug.exceptions.InternalServerError("we got an exception in the error handler")
def wrapper(*args, **kwargs):
start_time = time.time()
request_id = self.generate_id()
def inject_request_id(record):
record.extra['request_id'] = request_id
with logbook.Processor(inject_request_id):
logbook.notice(self.request_str(), extra={"api": True})
try:
response = callback(*args, **kwargs)
except OperationalError as e:
logbook.warning("Database is down {}: {}",
conf.database.uri,
e,
exc_info=True)
logbook.error("Database is down {}: {}", conf.database.uri,
e)
response = errors.DatabaseIsDown()
except errors.BadRequest as e:
e.format_response()
response = e
except bottle.HTTPResponse as e:
response = e
except Exception as e:
if self.under_test:
import traceback
traceback.print_exc()
logbook.exception(
"Exception during processing request: %s %s" %
(bottle.request.method, bottle.request.path))
self.log_response(str(e), 500, time.time() - start_time)
raise
finally:
from model import db
db.session.remove()
response = self.response_process(response,
time.time() - start_time)
return response
def call_error_handler(self, handler_class, request=None, **kwargs):
"""calls an error handler with the given arguments
error handlers are instances of :class:`~starflyer.Handler` and are
called the same way. If a handler raises an exception themselves
the default internal server error is used.
:param handler_class: the Handler class to use
:param request: The request object which was used when the exception occurred.
:param kwargs: additional keyword arguments being passed to the ``get()`` method of
the handler.
"""
handler = handler_class(self, request)
# process the request via the handler
try:
return handler(**kwargs)
except Exception, e:
logbook.exception()
return werkzeug.exceptions.InternalServerError(
"we got an exception in the error handler")
def collect_usage(self, tenant, mutex, end=None):
# Collects usage for a given tenant from when they were last collected,
# up to the given end, and breaks the range into one hour windows.
end = end or datetime.utcnow()
time_label = TimeLabel(tenant.last_collected + timedelta(minutes=1))
end_time_label = TimeLabel(end)
usage = {}
logbook.info('collect_usage for {}, from {} till {} (last_collected: {})',
tenant, time_label, end_time_label, tenant.last_collected)
customer = Customer.get_by_tenant_id(tenant.tenant_id)
if not customer:
logbook.error("Customer for tenant {} not found", tenant)
return usage
while time_label < end_time_label:
try:
usages = self._collect_usage(tenant, time_label, customer)
tenant.last_collected = time_label.datetime_range()[1]
if usages:
db.session.add(customer)
total_cost = customer.calculate_usage_cost(usages)
customer.withdraw(total_cost)
if not conf.test:
db.session.commit()
usage[time_label] = [usage.to_dict() for usage in usages], total_cost
except Exception:
self.errors += 1
import traceback
traceback.print_exc()
logbook.exception("Usage process failed for {} and {}", tenant, time_label)
db.session.rollback()
return usage
time_label = time_label.next()
mutex.update_ttl()
return usage
def ww2ogg(src: Path, queue: mp.Queue):
setup_logging(queue)
out = b""
try:
logbook.info("converting '{src}' to OGG", src=src.resolve().absolute())
out = subprocess.check_output([
WW2OGG,
str(src.resolve().absolute()), "--pcb",
str(PCB.resolve().absolute())
],
stderr=subprocess.STDOUT)
src.unlink()
logbook.info("removed {src}", src=src.resolve().absolute())
revorb(src.with_suffix(".ogg"), queue)
except subprocess.CalledProcessError as cpe:
logbook.exception(
"ww2ogg error for '{src}': code={code}, out={o}",
src=src.resolve().absolute(),
code=cpe.returncode,
o=out.decode("utf-8"),
)
def response_process(self, response, work_time):
if response is None:
logbook.error("Response can't be None")
response = body_to_log = "{}"
status = 200
elif isinstance(response, bottle.Response):
if response.content_type.startswith(
"text/") or response.content_type == "application/json":
body_to_log = str(response.body) or getattr(
response, "message", "")
else:
body_to_log = response.content_type
status = response.status
elif isinstance(response, dict):
status = 200
try:
body_to_log = json.dumps(response, cls=DateTimeJSONEncoder)
if self.debug:
response = json.dumps(response,
cls=DateTimeJSONEncoder,
indent=4)
else:
response = body_to_log
except TypeError:
logbook.exception("Can't encode reply: {}", response)
raise bottle.HTTPError(body="Internal Server Error")
bottle.response.content_type = 'application/json'
else:
logbook.error("Incorrect response ({}): {}", type(response),
response)
body_to_log = str(response)
status = 200
self.log_response(body_to_log, status, work_time)
return response
async def reveal_click(chat: Chat, cq: CallbackQuery, match):
operation = match.group(1)
vote_id = match.group(2)
game = await storage.get_game(chat.id, vote_id)
if not game:
return await cq.answer(text="No such game")
if cq.src["from"]["id"] != game.initiator["id"]:
return await cq.answer(
text="{} is available only for initiator".format(operation))
current_text = game.get_text()
if operation in (Game.OP_RESTART, Game.OP_RESTART_NEW):
game.restart()
else:
game.revealed = True
current_text = game.get_text()
if operation in (Game.OP_RESTART, Game.OP_REVEAL):
try:
await bot.edit_message_text(chat.id, game.reply_message_id,
**game.get_send_kwargs())
except BotApiError:
logbook.exception("Error when updating markup")
else:
try:
await bot.edit_message_text(chat.id,
game.reply_message_id,
text=current_text)
except BotApiError:
logbook.exception("Error when updating markup")
resp = await chat.send_text(**game.get_send_kwargs())
game.reply_message_id = resp["result"]["message_id"]
await storage.save_game(game)
await cq.answer()
def force_delete(cls, table, prefix=None, field=None) -> int:
"""
Run force delete.
:param table: table to force delete on.
:param prefix: entity name prefix (current test prefix if None).
:param field: table field (get from _table_field attribute by default).
:return int: count of deleted objects.
"""
if prefix is None:
prefix = cls.get_test_prefix()
if field is None:
field = cls._table_field[table]
count = None
class BadHTTPError(HTTPError): pass
try:
for r in cls.retries(60, 1, exception=BadHTTPError):
with r:
try:
count = cls.default_admin_client.utility.force_delete(table, prefix, field)['deleted'][table]
except HTTPError as e:
if e.response.status_code > 500:
raise BadHTTPError(response=e.response, request=e.request)
raise
except:
logbook.exception('Force delete error of {}, prefix={}, field={}'.format(table, prefix, field))
raise
else:
if count == 0:
method = logbook.warning
else:
method = logbook.info
method('Force delete {} objects of {}, prefix={}, field={}'.format(count, table, prefix, field))
return count
if 'AACGM_v2_DAT_PREFIX' in _os.environ.keys():
stderr.write("resetting environment variable AACGM_v2_DAT_PREFIX in " +
"python script\n")
reset_warn = True
_os.environ['AACGM_v2_DAT_PREFIX'] = AACGM_v2_DAT_PREFIX
if reset_warn:
stderr.write("non-default coefficient files may be specified by running " +
"aacgmv2.wrapper.set_coeff_path before any other functions\n")
# Imports
#---------------------------------------------------------------------
try:
from aacgmv2.wrapper import (convert_latlon, convert_mlt, get_aacgm_coord)
from aacgmv2.wrapper import (convert_latlon_arr, get_aacgm_coord_arr)
from aacgmv2.wrapper import (convert_bool_to_bit, convert_str_to_bit)
except Exception as err:
logging.exception(__file__ + ' -> aacgmv2: ' + str(err))
try:
from aacgmv2 import (deprecated)
from aacgmv2.deprecated import (convert)
except Exception as err:
logging.exception(__file__ + ' -> aacgmv2: ' + str(err))
try:
from aacgmv2 import (_aacgmv2)
except Exception as err:
logging.exception(__file__ + ' -> aacgmv2: ' + str(err))
def wrapper(*args, **kwargs):
try:
return fn(*args, **kwargs)
except Exception:
logbook.exception("{} raise exception", fn.__name__)
Classes
---------------------------------------------------------------------------
OCBoundary OCB data for different times
VectorData Vector data point
---------------------------------------------------------------------------
Modules
---------------------------------------------------------------------------
instruments Instrument-specific OCB gridding functions
ocb_time Time manipulation routines
---------------------------------------------------------------------------
"""
import logbook as logging
__version__ = str('0.2b2')
__default_file__ = "boundaries/si13_north_circle"
# Imports
#---------------------------------------------------------------------
try:
from ocbpy import (ocboundary, ocb_scaling, ocb_time)
from ocbpy.ocboundary import (OCBoundary, match_data_ocb)
except ImportError as err:
logging.exception('problem importing ocboundary: ' + str(err))
try:
from ocbpy import (instruments)
except ImportError as err:
logging.exception('problem importing instruments: ' + str(err))
