def fetch_package_infos(self):
self.load()
get_url = urllib2.urlopen
pool = Pool(self.concurrency)
pkg_idx = self.package_index
if not pkg_idx:
pkg_idx = self._fetch_package_index()
pkg_info_map = self.pkg_info_map
if not pkg_info_map:
pkg_info_map = PackageInfoMap(pkg_idx=pkg_idx)
pkg_info_map.path = self.package_info_path
pb = ProgressBar(widgets=[Percentage(),
' ', Bar(),
' ', SimpleProgress()],
maxval=len(pkg_idx) + 1)
pb.start()
pb.update(len(pkg_info_map))
to_fetch = sorted(set(pkg_idx.package_rel_urls) -
set(pkg_info_map.pkg_infos.viewkeys()))
def _get_package_info(package_rel_url):
pkg_url = self.default_pypi_url + 'pypi/%s/json' % package_rel_url
try:
resp = get_url(pkg_url)
except Exception as e:
ret = {'error': repr(e)}
else:
ret = json.loads(resp.read())
ret['rel_url'] = package_rel_url
return ret
pkg_info_iter = pool.imap(_get_package_info, to_fetch)
err_count = 0
for pkg_info in pkg_info_iter:
try:
pkg_info_map.add_dict(pkg_info)
except KeyError:
err_count += 1
pkg_info_map.add_dict(pkg_info)
pb.update(len(pkg_info_map))
if len(pkg_info_map) % self.concurrency == 0:
pkg_info_map.save()
pool.join(timeout=0.3, raise_error=True)
print 'Done fetching. Saving', len(pkg_info_map), 'package infos.'
try:
pkg_info_map.save()
except Exception:
print ExceptionInfo.from_current().get_formatted()
import pdb;pdb.post_mortem()
import pdb;pdb.set_trace()
return
def endpoint(self, next, response_dict, request, _route):
# TODO: autoswitch resp status code
try:
ret = next()
except Exception:
if self.raise_errors:
raise
ret = None
exc_info = ExceptionInfo.from_current()
err = '%s: %s' % (exc_info.exc_type, exc_info.exc_msg)
response_dict['errors'].append(err)
response_dict['status'] = 'exception'
else:
if response_dict.get('errors'):
response_dict['status'] = 'failure'
if isinstance(ret, BaseResponse):
# preserialized responses (and 404s, etc.) TODO: log that
# we're skipping over response_dict if the response status
# code == 2xx
# TODO: autoserialize body if no body is set
return ret
elif isinstance(ret, dict) and (ret.get('use_ashes')
or self.use_ashes):
return ret
elif isinstance(ret, dict):
response_dict.update(ret)
else:
response_dict.update({'data': ret})
return render_basic(context=response_dict,
request=request,
_route=_route)
def _exception(self, exc_type, exc_val, exc_tb, message, fargs, data):
exc_type = exc_type or DefaultException
# have to capture the time now in case the on_exception sinks
# take their sweet time
etime = time.time()
exc_info = ExceptionInfo.from_exc_info(exc_type, exc_val, exc_tb)
if not message:
cp = exc_info.tb_info.frames[-1]
t = "%s raised exception: "
exc_repr = "%s(%r)"
errno = getattr(exc_val, 'errno', None)
if errno and str(errno) not in exc_repr:
t += exc_repr + ' (errno %s)' % exc_val.errno
else:
t += exc_repr
t += " from %s on line %s of file '%s'"
if self.data_map:
t += ' - ({data_map_repr})'
message = t % (self.name, exc_info.exc_type, exc_info.exc_msg,
cp.func_name, cp.lineno, cp.module_path)
exc_event = ExceptionEvent(self, etime, message, fargs, exc_info)
self.exc_events.append(exc_event)
self.logger.on_exception(exc_event, exc_type, exc_val, exc_tb)
return self._end('exception', message, fargs, data, etime, exc_info)
def publish_error(self, exception, action, region="eu-central-1"):
subject = self.formatter.format(
'ERROR : while performing %s in environment %s: %s',
[repr(action), self.env, repr(exception)])
result = self.formatter.to_str(ExceptionInfo.from_current().to_dict())
message = self.formatter.to_str({'default': result})
self.publish(subject, message, region)
def _exception(self, exc_type, exc_val, exc_tb, message, fargs, data):
exc_type = exc_type or DefaultException
# have to capture the time now in case the on_exception sinks
# take their sweet time
etime = time.time()
exc_info = ExceptionInfo.from_exc_info(exc_type, exc_val, exc_tb)
if not message:
cp = exc_info.tb_info.frames[-1]
t = "%s raised exception: "
exc_repr = "%s(%r)"
errno = getattr(exc_val, 'errno', None)
if errno and str(errno) not in exc_repr:
t += exc_repr + ' (errno %s)' % exc_val.errno
else:
t += exc_repr
t += " from %s on line %s of file '%s'"
if self.data_map:
t += ' - ({data_map_repr})'
message = t % (self.name, exc_info.exc_type, exc_info.exc_msg,
cp.func_name, cp.lineno, cp.module_path)
exc_event = ExceptionEvent(self, etime, message, fargs, exc_info)
self.exc_events.append(exc_event)
self.logger.on_exception(exc_event, exc_type, exc_val, exc_tb)
return self._end('exception', message, fargs, data,
etime, exc_info)
def test_exception_info():
# test ExceptionInfo and TracebackInfo and hooks, via StringIOs
builtin_exc_hook = sys.excepthook
fix_print_exception()
tbi_str = ''
def test():
raise ValueError('yay fun')
fake_stderr1 = StringIO()
fake_stderr2 = StringIO()
sys.stderr = fake_stderr1
try:
test()
except:
_, _, exc_traceback = sys.exc_info()
tbi = TracebackInfo.from_traceback(exc_traceback)
exc_info = ExceptionInfo.from_exc_info(*sys.exc_info())
exc_info2 = ExceptionInfo.from_current()
tbi_str = str(tbi)
print_exception(*sys.exc_info(), file=fake_stderr2)
new_exc_hook_res = fake_stderr2.getvalue()
builtin_exc_hook(*sys.exc_info())
builtin_exc_hook_res = fake_stderr1.getvalue()
finally:
sys.stderr = sys.__stderr__
# Single frame
single_frame_str = tbi.frames[-1].tb_frame_str()
assert 'in test' in single_frame_str
assert 'yay fun' in single_frame_str
# Traceback info
assert len(tbi_str.splitlines()) == 5
assert 'yay fun' in tbi_str
# Full except hook output
assert 'ValueError: yay fun' in new_exc_hook_res
assert "ValueError('yay fun')" in new_exc_hook_res
assert len(new_exc_hook_res) > len(tbi_str)
assert new_exc_hook_res == builtin_exc_hook_res
def test_exception_info():
# test ExceptionInfo and TracebackInfo and hooks, via StringIOs
builtin_exc_hook = sys.excepthook
fix_print_exception()
tbi_str = ''
def test():
raise ValueError('yay fun')
fake_stderr1 = StringIO()
fake_stderr2 = StringIO()
sys.stderr = fake_stderr1
try:
test()
except:
_, _, exc_traceback = sys.exc_info()
tbi = TracebackInfo.from_traceback(exc_traceback)
exc_info = ExceptionInfo.from_exc_info(*sys.exc_info())
exc_info2 = ExceptionInfo.from_current()
tbi_str = str(tbi)
print_exception(*sys.exc_info(), file=fake_stderr2)
new_exc_hook_res = fake_stderr2.getvalue()
builtin_exc_hook(*sys.exc_info())
builtin_exc_hook_res = fake_stderr1.getvalue()
finally:
sys.stderr = sys.__stderr__
# Single frame
single_frame_str = tbi.frames[-1].tb_frame_str()
assert 'in test' in single_frame_str
assert 'yay fun' in single_frame_str
# Traceback info
assert len(tbi_str.splitlines()) == 5
assert 'yay fun' in tbi_str
# Full except hook output
assert 'ValueError: yay fun' in new_exc_hook_res
assert "ValueError('yay fun')" in new_exc_hook_res
assert len(new_exc_hook_res) > len(tbi_str)
assert new_exc_hook_res == builtin_exc_hook_res
def invoke(self, *args):
""" Executes Click's CLI, print output and return results. """
result = self.runner.invoke(cli, args)
self.print_cli_output(['mpm'] + list(args), result.output)
# Print some more debug info.
print(result)
if result.exception:
print(
ExceptionInfo.from_exc_info(*result.exc_info).get_formatted())
return result
def invoke(self, *args):
""" Executes CLI, print output and return results. """
result = self.runner.invoke(cli, args)
# Simulate CLI output.
print('-' * 70)
print("$ mdedupe {}".format(' '.join(args)))
print(result.output)
print('-' * 70)
# Print some more debug info.
print(result)
if result.exception:
print(ExceptionInfo.from_exc_info(
*result.exc_info).get_formatted())
return result
def _run(*args, color=False):
# We allow for nested iterables and None values as args for
# convenience. We just need to flatten and filters them out.
args = list(filter(None.__ne__, flatten(args)))
if args:
assert same(map(type, args), str)
result = runner.invoke(mdedup, args, color=color)
print_cli_output([CLI_NAME] + args, result.output)
# Print some more debug info.
print(result)
if result.exception:
print(
ExceptionInfo.from_exc_info(*result.exc_info).get_formatted())
return result
def request(self, next, request, _route):
try:
act_name = '%s%s' % (request.method, _route.pattern)
with self.api_log.critical(act_name) as api_act:
# basic redacted url
api_act['path'] = request.path
api_act.data_map.update(request.args.items())
try:
ret = next(api_act=api_act, api_log=self.api_log)
except clastic.errors.BadRequest as br:
api_act.data_map.update(br.to_dict())
api_act.failure()
ret = br
api_act['code'] = ret.status_code
except Exception:
exc_info = ExceptionInfo.from_current()
text = u'\n\n' + exc_info.get_formatted() + '\n\n'
self.exc_log_file_obj.write(text.encode('utf8'))
self.exc_log_file_obj.flush()
raise
return ret
def _run(*args, **kwargs):
color = kwargs.get('color', False)
# We allow for nested iterables and None values as args for
# convenience. We just need to flatten and filters them out.
args = list(filter(None.__ne__, flatten(args)))
if args:
assert set(map(type, args)) == {str}
result = runner.invoke(cli, args, color=color)
# Strip colors out of results.
result.stdout_bytes = strip_ansi(result.stdout_bytes)
result.stderr_bytes = strip_ansi(result.stderr_bytes)
print_cli_output(['mpm'] + args, result.output)
# Print some more debug info.
print(result)
if result.exception:
print(ExceptionInfo.from_exc_info(
*result.exc_info).get_formatted())
return result