def test_base(self):
backtrace.hook()
try:
test_message = 'Test message'
with pytest.raises(RuntimeError) as ex:
raise RuntimeError(test_message)
assert ex._excinfo[0].__name__ == 'RuntimeError'
assert test_message in str(ex)
finally:
backtrace.unhook()
def print_with_backtrace(tb_arg,
file,
is_tb_extracted=False,
post_mortem=False):
if not pkgutil.find_loader('backtrace'):
raise
import backtrace
exc_type, exc, tb = tb_arg
if post_mortem:
if pkgutil.find_loader('ipdb'):
import ipdb
ipdb.post_mortem(tb)
else:
import pdb
pdb.post_mortem(tb)
if not is_tb_extracted:
tb = traceback.extract_tb(tb)
try:
orig_tb = simplify_traceback(tb)
tb = replace_genfile_in_tb(tb, file)
tb = [
new_elem for idx, tbelem in enumerate(tb)
for new_elem in add_generator_file_to_tb(idx, tbelem)
]
except:
tb = orig_tb
tb = simplify_traceback(tb)
backtrace.hook(tpe=exc_type,
value=exc,
tb=tb,
reverse=True,
align=True,
strip_path=False,
enable_on_envvar_only=False,
on_tty=False,
conservative=False,
styles={})
def get_backtrace_lines(tb):
"""As backtrace.hook prints to stderr, its output has to be captured to get it as a text."""
import backtrace
import io
import contextlib
with io.StringIO() as buf:
with contextlib.redirect_stderr(buf):
backtrace.hook(tpe='ignored-tpe',
value='ignored-value',
tb=tb,
reverse=True,
align=True,
strip_path=False,
enable_on_envvar_only=False,
on_tty=False,
conservative=False)
return reformat_lines(buf.getvalue())
def parse_item(self, response):
try:
key = BlogSpider.pattern.search(response.url).group(1)
episodes = {}
for html in response.css('.dv-episode-container'):
self.parse_episode(episodes, html)
value = {
'title':
app.sanitize(
response.css('section > h1::text').extract_first()),
'story':
app.sanitize(
response.css('div.av-synopsis.avu-full-width > p::text').
extract_first()),
'year':
app.sanitize(
response.css(
'[data-automation-id="release-year-badge"]::text').
extract_first()),
'episodes':
episodes
}
# Masterファイルを読み込み（シングルトン）
master = master_file.MasterFile()
master.append('amazon', key, value)
self.check_format(key, value)
except TypeError as error:
logger.error(str(error) + ' ' + response.url + ' ')
tpe, v, tb = sys.exc_info()
backtrace.hook(reverse=True,
strip_path=True,
tb=tb,
tpe=tpe,
value=v)
open_in_browser(response)
pass
"""
import backtrace
import sys
from .base import *
# LOCAL ENVS
# ------------------------------------------------------------------------------
env = environ.Env()
# BACKTRACE
# ------------------------------------------------------------------------------
backtrace.hook(
reverse=True,
align=False,
strip_path=True,
enable_on_envvar_only=False,
on_tty=False,
conservative=False,
styles={}
)
# GENERAL
# ------------------------------------------------------------------------------
DEBUG = True
SECRET_KEY = env.str("DJANGO_SECRET_KEY")
ALLOWED_HOSTS = ["localhost", "0.0.0.0", "127.0.0.1"]
# DEBUG TOOLBAR
DEBUG_TOOLBAR_CONFIG = {
'SHOW_TOOLBAR_CALLBACK': lambda _request: DEBUG
# Copyright (c) 2019 Manuel Barkhau ([email protected]) - MIT License # SPDX-License-Identifier: MIT import os import sys import json import typing as typ import subprocess as sp import markdown_katex # To enable pretty tracebacks: # echo "export ENABLE_BACKTRACE=1;" >> ~/.bashrc if os.environ.get('ENABLE_BACKTRACE') == '1': import backtrace backtrace.hook(align=True, strip_path=True, enable_on_envvar_only=True) HTML_TEMPLATE = """ <!DOCTYPE html> <html> <head> <title>Test Katex</title> <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/katex.css" integrity="sha256-SSjvSe9BDSZMUczwnbB1ywCyIk2XaNly9nn6yRm6WJo=" crossorigin="anonymous" /> <style type="text/css"> body{background: white; } </style> </head> <body>
#!/usr/bin/env python3
from pathlib import Path
from typing import Optional
from subprocess import run, PIPE
import argh
import backtrace
import steno_summary.parse_dict as pd
from steno_summary.brief_info import Brief, brief_grid
from steno_summary.parse_dict import read_dict, _validate_path
""" Manager for the steno summary dictonary. """
backtrace.hook(align=True, strip_path=True)
@argh.aliases("cont")
def contains(string: Optional[str] = None, block: bool = False):
""" Print the names that contains the string. """
string = _query_user_if_none(string, "Search for words containing: ")
briefs = read_dict()
filtered_briefs = [b for b in briefs if string.lower() in b.name.lower()]
print(brief_grid(filtered_briefs))
_wait_if(block)
@argh.aliases("start")
def starting_with(string: Optional[str] = None, block: bool = False):
""" Print the names that starts with the string. """
string = _query_user_if_none(string, "Words starting with: ")
briefs = read_dict()
import os
import sys
import coloredlogs, logging
import colored_traceback
from termcolor import colored
from colorama import Fore, Back, Style
import backtrace
backtrace.hook(reverse=True,
align=True,
strip_path=True,
enable_on_envvar_only=True,
on_tty=True,
conservative=True,
styles={
'backtrace': Fore.YELLOW + '{0}',
'error': Fore.RED + Style.BRIGHT + '{0}',
'line': Fore.RED + Style.BRIGHT + '{0}',
'module': '{0}',
'context': Style.BRIGHT + Fore.GREEN + '{0}',
'call': Fore.YELLOW + '--> ' + Style.BRIGHT + '{0}'
})
colored_traceback.add_hook(always=True)
# Create a logger object.
logger = logging.getLogger(__name__)
# By default the install() function installs a handler on the root logger,
# this means that log messages from your code and log messages from the
# libraries that you use will all show up on the terminal.
orig_tb = simplify_traceback(tb)
tb = replace_genfile_in_tb(tb, file)
tb = [
new_elem for idx, tbelem in enumerate(tb)
for new_elem in add_generator_file_to_tb(idx, tbelem)
]
except:
tb = orig_tb
tb = simplify_traceback(tb)
backtrace.hook(tpe=exc_type,
value=exc,
tb=tb,
reverse=True,
align=True,
strip_path=False,
enable_on_envvar_only=False,
on_tty=False,
conservative=False,
styles={})
def errmsg_srcfile(genline, genlineno):
split = genline.split(" ")
if len(split) <= 2 or split[-2] != "##":
raise T4P4SHandledException()
split2 = split[-1].split(":")
file = split2[0]
lineno = int(split2[1])
def setup(
trace=False,
pdb_on_error=True,
default_config=None,
autolog=False,
autolog_dir="pinkblack_autolog",
):
"""
:param trace:
:param pdb_on_error:
:param default_config: dict or str(yaml file)
:param autolog:
:param autolog_dir:
gpu -> CUDA_VISIBLE_DEVICES
:return: argparsed config
Example >>
```bash
CUDA_VISIBLE_DEVICES=1,3 python myscript.py --batch_size 32 --ckpt ckpt.pth --epochs 100 --lr 0.001
```
```python3
setup(default_config=dict(gpu="1,3",
batch_size=32,
lr=1e-3,
epochs=100,
ckpt="ckpt.pth"))
```
```bash
python myscript.py --gpu 1,3 --batch_size 32 --ckpt ckpt.pth --epochs 100 --lr 0.001
```
"""
if trace:
import backtrace
backtrace.hook(align=True)
if pdb_on_error:
old_hook = sys.excepthook
def new_hook(type_, value, tb):
old_hook(type_, value, tb)
if type_ != KeyboardInterrupt:
import pdb
pdb.post_mortem(tb)
sys.excepthook = new_hook
args = None
if default_config is not None:
if isinstance(default_config, str):
default_config = OmegaConf.load(default_config)
args = get_args(default_config)
import time, datetime
dt = datetime.datetime.fromtimestamp(time.time())
dt = datetime.datetime.strftime(dt, f"{os.path.basename(sys.argv[0])}.%Y%m%d_%H%M%S.log")
if args is not None and hasattr(args, "ckpt"):
logpath = args.ckpt + "_" + dt
else:
logpath = os.path.join(autolog_dir, dt)
os.makedirs(os.path.dirname(logpath), exist_ok=True)
if args is not None:
conf = OmegaConf.create(args.__dict__)
conf.save(logpath[:-4] + ".yaml")
conf.save(args.ckpt + ".yaml")
if autolog:
fp = open(logpath, "w")
sys.stdout = PinkBlackLogger(fp, sys.stdout)
sys.stderr = PinkBlackLogger(fp, sys.stderr)
print("PinkBlack :: args :", args.__dict__)
return args
# Licensed under GNU General Public License v3 or later, see COPYING.
# Copyright (c) 2019 Red Hat, Inc., see CONTRIBUTORS.
try:
import backtrace
styles = backtrace.STYLES.copy()
styles['call'] = styles['call'].replace('--> ', '')
backtrace.hook(strip_path=True, on_tty=True, styles=styles)
except ImportError:
pass
def setup(
trace=False,
pdb_on_error=True,
default_config=None,
autolog=False,
autolog_dir="pinkblack_autolog",
):
"""
:param trace:
:param pdb_on_error:
:param default_config: dict or str(yaml file)
:param autolog:
:param autolog_dir:
:return: config
"""
if trace:
import backtrace
backtrace.hook(align=True)
if pdb_on_error:
old_hook = sys.excepthook
def new_hook(type_, value, tb):
old_hook(type_, value, tb)
if type_ != KeyboardInterrupt:
import pdb
pdb.post_mortem(tb)
sys.excepthook = new_hook
args = None
if default_config is not None:
if isinstance(default_config, str):
default_config = OmegaConf.load(default_config)
elif isinstance(default_config, dict):
default_config = OmegaConf.create(default_config)
args = from_cli(default_config)
import time, datetime
dt = datetime.datetime.fromtimestamp(time.time())
dt = datetime.datetime.strftime(
dt, f"{os.path.basename(sys.argv[0])}.%Y%m%d_%H%M%S.log")
if args is not None and hasattr(args, "ckpt"):
logpath = args.ckpt + "_" + dt
else:
logpath = os.path.join(autolog_dir, dt)
os.makedirs(os.path.dirname(logpath), exist_ok=True)
if args is not None:
conf = OmegaConf.create(args.__dict__)
conf.save(logpath[:-4] + ".yaml")
conf.save(args.ckpt + ".yaml")
if autolog:
fp = open(logpath, "w")
sys.stdout = PinkBlackLogger(fp, sys.stdout)
sys.stderr = PinkBlackLogger(fp, sys.stderr)
return args
import re
import sys
import scrapy
import backtrace
import app
from app import master_file
from app.lib import logger
from scrapy.utils.response import open_in_browser
logger = logger.get_module_logger(__name__)
backtrace.hook(
reverse=True, # 逆順
strip_path=True # ファイル名のみ
)
# ジャンル一覧（取り方が２つあるが同じもの）
# https://www.amazon.co.jp/gp/search/other/ref=sr_sa_p_n_theme_browse-bin?rh=n%3A2351649051%2Cp_n_ways_to_watch%3A3746328051&bbn=2351649051&sort=date-desc-rank&pickerToList=theme_browse-bin&ie=UTF8&qid=1544453307
# https://www.amazon.co.jp/gp/search/other/ref=sr_sa_p_n_theme_browse-bin?rh=n:2351649051,p_n_ways_to_watch:3746328051&bbn=2351649051&sort=date-desc-rank&pickerToList=theme_browse-bin&ie=UTF8&qid=1544453307
# 16 * 400 = 6400
# ジャンル：アニメ
# https://www.amazon.co.jp/s/ref=sr_in_-2_p_n_theme_browse-bin_2?fst=as%3Aoff&rh=n%3A2351649051%2Cp_n_ways_to_watch%3A3746328051%2Cp_n_theme_browse-bin%3A4435524051&bbn=2351649051&sort=date-desc-rank&ie=UTF8&qid=1549894109&rnid=4435522051
# https://www.amazon.co.jp/dp/B07MWDNS4R
class BlogSpider(scrapy.Spider):
name = 'blogspider'
