예제 #1
0
파일: metaproduct.py 프로젝트: cxz/ploomber
    def __init__(self, products):
        container = ProductsContainer(products)

        self.products = container
        self.metadata = MetadataCollection(container)
        self.clients = ClientContainer(container)

        self._repr = Repr()
예제 #2
0
    def __init__(self, source, path_to_here=None, defaults=None):

        # if initialized from another EnvDict, copy the attributes to
        # initialize
        # this happens in the  CLI parser, which instanttiates the env
        # because it needs to create one and then replace cli args, then
        # passes this modified object to DAGSpec
        if isinstance(source, EnvDict):
            for attr in (
                    '_path_to_env',
                    '_preprocessed',
                    '_expander',
                    '_data',
                    '_repr',
                    '_default_keys',
            ):
                original = getattr(source, attr)
                setattr(self, attr, deepcopy(original))
        else:
            (
                # load data
                raw_data,
                # this will be None if source is a dict
                self._path_to_env) = load_from_source(source)

            if defaults:
                raw_data = {**defaults, **raw_data}

            # add default placeholders but override them if they are defined
            # in the raw data
            default = self._default_dict(include_here=path_to_here is not None)
            self._default_keys = set(default) - set(raw_data)
            raw_data = {**default, **raw_data}

            # check raw data is ok
            validate.raw_data_keys(raw_data)

            # expand _module special key, return its expanded value
            self._preprocessed = raw_preprocess(raw_data, self._path_to_env)

            # initialize expander, which converts placeholders to their values
            # we need to pass path_to_env since the {{here}} placeholder
            # resolves to its parent
            if path_to_here is None:
                # if no pat_to_here, use path_to_end
                path_to_here = (None if self._path_to_env is None else Path(
                    self._path_to_env).parent)
            else:
                path_to_here = Path(path_to_here).resolve()

            self._expander = EnvironmentExpander(self._preprocessed,
                                                 path_to_here=path_to_here)
            # now expand all values
            self._data = self._expander.expand_raw_dictionary(raw_data)

            self._repr = Repr()
예제 #3
0
    def __init__(self):
        Repr.__init__(self)

        # Note: These levels can get adjusted dynamically!  My goal is to get more info when printing important debug stuff like exceptions and stack traces and less info when logging normal events.  --Zooko 2000-10-14
        self.maxlevel = 6
        self.maxdict = 6
        self.maxlist = 6
        self.maxtuple = 6
        self.maxstring = 300
        self.maxother = 300
예제 #4
0
    def test_tuple(self):
        eq = self.assertEqual
        eq(r((1,)), "(1,)")

        t3 = (1, 2, 3)
        eq(r(t3), "(1, 2, 3)")

        r2 = Repr()
        r2.maxtuple = 2
        expected = repr(t3)[:-2] + "...)"
        eq(r2.repr(t3), expected)
예제 #5
0
    def test_tuple(self):
        eq = self.assertEqual
        eq(r((1,)), "(1,)")

        t3 = (1, 2, 3)
        eq(r(t3), "(1, 2, 3)")

        r2 = Repr()
        r2.maxtuple = 2
        expected = repr(t3)[:-2] + "...)"
        eq(r2.repr(t3), expected)
예제 #6
0
def get_prices_from_api(request_uri: str) -> dict:
    """using the provided URI, request data from the Octopus API and return a JSON object.
    Try to handle errors gracefully with retries when appropriate."""

    # Try to handle issues with the API - rare but do happen, using an
    # exponential sleep time up to 2**14 (16384) seconds, approx 4.5 hours.
    # We will keep trying for over 9 hours and then give up.

    print('Requesting Agile prices from Octopus API...')
    retry_count = 0
    my_repr = Repr()
    my_repr.maxstring = 80  # let's avoid truncating our error messages too much

    while retry_count <= MAX_RETRIES:

        if retry_count == MAX_RETRIES:
            raise SystemExit('API retry limit exceeded.')

        try:
            success = False
            response = requests.get(request_uri, timeout=5)
            response.raise_for_status()
            if response.status_code // 100 == 2:
                success = True
                return response.json()

        except requests.exceptions.HTTPError as error:
            print(('API HTTP error ' + str(response.status_code) +
                   ',retrying in ' + str(2**retry_count) + 's'))
            time.sleep(2**retry_count)
            retry_count += 1

        except requests.exceptions.ConnectionError as error:
            print(('API connection error: ' + my_repr.repr(str(error)) +
                   ', retrying in ' + str(2**retry_count) + 's'))
            time.sleep(2**retry_count)
            retry_count += 1

        except requests.exceptions.Timeout:
            print('API request timeout, retrying in ' + str(2**retry_count) +
                  's')
            time.sleep(2**retry_count)
            retry_count += 1

        except requests.exceptions.RequestException as error:
            raise SystemExit('API Request error: ' + str(error)) from error

        if success:
            print('API request successful, status ' +
                  str(response.status_code) + '.')
            break
예제 #7
0
    def __init__(self, subsequentIndent=""):
        Repr.__init__(self)
        self.maxtuple = 20
        self.maxset = 160
        self.maxlist = 20
        self.maxdict = 20
        self.maxstring = 1600
        self.maxother = 160

        self.maxLineLen = 160

        self.subsequentIndent = subsequentIndent
        # Pretty-print?
        self._pretty = True
예제 #8
0
    def repr_instance(self, obj, level):
        """
        If it is an instance of Exception, format it nicely (trying to emulate
        the format that you see when an exception is actually raised, plus
        bracketing '<''s).  If it is an instance of dict call self.repr_dict()
        on it.  If it is an instance of list call self.repr_list() on it. Else
        call Repr.repr_instance().
        """
        if isinstance(obj, Exception):
            # Don't cut down exception strings so much.
            tms = self.maxstring
            self.maxstring = max(512, tms * 4)
            tml = self.maxlist
            self.maxlist = max(12, tml * 4)
            try:
                if hasattr(obj, 'args'):
                    if len(obj.args) == 1:
                        return '<' + obj.__class__.__name__ + ': ' + self.repr1(
                            obj.args[0], level - 1) + '>'
                    else:
                        return '<' + obj.__class__.__name__ + ': ' + self.repr1(
                            obj.args, level - 1) + '>'
                else:
                    return '<' + obj.__class__.__name__ + '>'
            finally:
                self.maxstring = tms
                self.maxlist = tml

        if isinstance(obj, dict):
            return self.repr_dict(obj, level)

        if isinstance(obj, list):
            return self.repr_list(obj, level)

        return Repr.repr_instance(self, obj, level)
 def repr(self, obj):
     if isinstance(obj.__class__, DeclarativeMeta):
         return self.repr_Base(obj, self.maxlevel)
     if sys.version_info < (3,):
         return _Repr.repr(self, obj)
     else:
         return super(Repr, self).repr(obj)
예제 #10
0
 def test_init_kwargs(self):
     example_kwargs = {
         "maxlevel": 101,
         "maxtuple": 102,
         "maxlist": 103,
         "maxarray": 104,
         "maxdict": 105,
         "maxset": 106,
         "maxfrozenset": 107,
         "maxdeque": 108,
         "maxstring": 109,
         "maxlong": 110,
         "maxother": 111,
         "fillvalue": "x" * 112,
     }
     r1 = Repr()
     for attr, val in example_kwargs.items():
         setattr(r1, attr, val)
     r2 = Repr(**example_kwargs)
     for attr in example_kwargs:
         self.assertEqual(getattr(r1, attr), getattr(r2, attr), msg=attr)
예제 #11
0
    def test_tuple(self):
        eq = self.assertEqual
        eq(r((1,)), "(1,)")

        t3 = (1, 2, 3)
        eq(r(t3), "(1, 2, 3)")

        r2 = Repr()
        r2.maxtuple = 2
        expected = repr(t3)[:-2] + "...)"
        eq(r2.repr(t3), expected)

        # modified fillvalue:
        r3 = Repr()
        r3.fillvalue = '+++'
        r3.maxtuple = 2
        expected = repr(t3)[:-2] + "+++)"
        eq(r3.repr(t3), expected)
예제 #12
0
    def __init__(self, core_obj, opts=None):
        Mprocessor.Processor.__init__(self, core_obj)

        self.response = {"errs": [], "msg": []}
        self.continue_running = False  # True if we should leave command loop

        self.cmd_instances = self._populate_commands()

        # command name before alias or macro resolution
        self.cmd_name = ""

        # Current command getting run
        self.current_command = ""

        self.debug_nest = 1
        self.display_mgr = Mdisplay.DisplayMgr()
        self.intf = core_obj.debugger.intf
        self.last_command = None  # Initially a no-op
        self.precmd_hooks = []

        # If not:
        # self.location         = lambda : print_location(self)

        self.preloop_hooks = []
        self.postcmd_hooks = []

        self._populate_cmd_lists()

        # Stop only if line/file is different from last time
        self.different_line = None

        # These values updated on entry. Set initial values.
        self.curframe = None
        self.event = None
        self.event_arg = None
        self.frame = None
        self.list_lineno = 0

        # Create a custom safe Repr instance and increase its maxstring.
        # The default of 30 truncates error messages too easily.
        self._repr = Repr()
        self._repr.maxstring = 100
        self._repr.maxother = 60
        self._repr.maxset = 10
        self._repr.maxfrozen = 10
        self._repr.array = 10
        self._saferepr = self._repr.repr
        self.stack = []
        self.thread_name = None
        self.frame_thread_name = None
        return
예제 #13
0
def repr_values(condition: Callable[..., bool],
                lambda_inspection: Optional[ConditionLambdaInspection],
                condition_kwargs: Mapping[str, Any],
                a_repr: reprlib.Repr) -> List[str]:
    # pylint: disable=too-many-locals
    """
    Represent function arguments and frame values in the error message on contract breach.

    :param condition: condition function of the contract
    :param lambda_inspection:
        inspected lambda AST node corresponding to the condition function (None if the condition was not given as a
        lambda function)
    :param condition_kwargs: condition arguments
    :param a_repr: representation instance that defines how the values are represented.
    :return: list of value representations
    """
    if is_lambda(a_function=condition):
        assert lambda_inspection is not None, "Expected a lambda inspection when given a condition as a lambda function"
    else:
        assert lambda_inspection is None, "Expected no lambda inspection in a condition given as a non-lambda function"

    reprs = dict()  # type: MutableMapping[str, Any]

    if lambda_inspection is not None:
        variable_lookup = collect_variable_lookup(
            condition=condition, condition_kwargs=condition_kwargs)

        # pylint: disable=protected-access
        recompute_visitor = icontract._recompute.Visitor(
            variable_lookup=variable_lookup)

        recompute_visitor.visit(node=lambda_inspection.node.body)
        recomputed_values = recompute_visitor.recomputed_values

        repr_visitor = Visitor(recomputed_values=recomputed_values,
                               variable_lookup=variable_lookup,
                               atok=lambda_inspection.atok)
        repr_visitor.visit(node=lambda_inspection.node.body)

        reprs = repr_visitor.reprs
    else:
        for key, val in condition_kwargs.items():
            if _representable(value=val):
                reprs[key] = val

    parts = []  # type: List[str]
    for key in sorted(reprs.keys()):
        parts.append('{} was {}'.format(key, a_repr.repr(reprs[key])))

    return parts
예제 #14
0
    def print_kwargs(self, kwargs, show_size=False):
        repr_instance = Repr()
        print()

        for arg_name, arg_val in kwargs.items():
            print(f"\x1b[1m\x1b[34m{arg_name}\x1b[0m", end="")

            if isinstance(arg_val, (list, tuple, set, dict)):
                print(f"({len(arg_val)})", end="")

            if show_size:
                size = round(get_size(arg_val) / 1048576, 4)
                print(f" {size} MB", end="")

            print(f": {repr_instance.repr(arg_val)}")
        print()
예제 #15
0
 def __init__(self, identifier, client=None):
     super().__init__(identifier)
     self._client = client
     self._repr = Repr()
     self._repr.maxstring = 40
     self._remote_ = _RemoteFile(self)
예제 #16
0
import linecache
import bdb
import re

try:
    from reprlib import Repr
except ImportError:
    from reprlib import Repr

from pycopia import IO
from pycopia import UI
from pycopia import CLI

# Create a custom safe Repr instance and increase its maxstring.
# The default of 30 truncates error messages too easily.
_repr = Repr()
_repr.maxstring = 200
_repr.maxother = 50
_saferepr = _repr.repr

DebuggerQuit = bdb.BdbQuit

def find_function(funcname, filename):
    cre = re.compile(r'def\s+%s\s*[(]' % funcname)
    try:
        fp = open(filename)
    except IOError:
        return None
    # consumer of this info expects the first line to be 1
    lineno = 1
    answer = None
예제 #17
0
from functools import wraps
from reprlib import Repr
import inspect
from flask import request, g
from flask_logger_decorator.config import config
from flask_logger_decorator.logger import debug

__r = Repr()
__r.maxarray = __r.maxarray * 10
__r.maxdict = __r.maxdict * 10
__r.maxstring = __r.maxstring * 10


def request_tracing(fn):
    """
    A decorator to tracing request.
    """
    @wraps(fn)
    def wrapper(*args, **kwargs):
        tracing_request(fn, *args, **kwargs)
        return fn(*args, **kwargs)

    return wrapper


def tracing_request(fn, *args, **kwargs):
    function_args = ' '.join(_get_fn_args(fn, *args, **kwargs))
    trace_info = ' '.join(_get_fn_extra_info(fn))
    func_msg = 'func_name:{} func_args:{} trace_info:{}'.format(
        fn.__name__, function_args, trace_info)
    request_msg = get_request_trace_info()
예제 #18
0
from collections import deque
from itertools import chain
from reprlib import Repr
from sys import getsizeof, stderr

_repr = Repr()
_repr.maxdict = 1000
repr = _repr.repr


def total_size(o, handlers={}, verbose=False):
    """Returns the approximate memory footprint an object and all of its contents.

    Automatically finds the contents of the following builtin containers and
    their subclasses:  tuple, list, deque, dict, set and frozenset.
    To search other containers, add handlers to iterate over their contents:

        handlers = {SomeContainerClass: iter,
                    OtherContainerClass: OtherContainerClass.get_elements}

    """
    dict_handler = lambda d: chain.from_iterable((k, v) for k, v in d.items()
                                                 if k not in {
                                                     "sys",
                                                     "__builtins__",
                                                 })
    all_handlers = {
        tuple: iter,
        list: iter,
        deque: iter,
        dict: dict_handler,
예제 #19
0
import cmd
import bdb
from reprlib import Repr
import os
import re
import pprint
import traceback


class Restart(Exception):
    """Causes a debugger to be restarted for the debugged python program."""
    pass

# Create a custom safe Repr instance and increase its maxstring.
# The default of 30 truncates error messages too easily.
_repr = Repr()
_repr.maxstring = 200
_saferepr = _repr.repr

__all__ = ["run", "pm", "Pdb", "runeval", "runctx", "runcall", "set_trace",
           "post_mortem", "help"]

def find_function(funcname, filename):
    cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname))
    try:
        fp = open(filename)
    except IOError:
        return None
    # consumer of this info expects the first line to be 1
    lineno = 1
    answer = None
예제 #20
0
    def __init__(self, core_obj, opts=None):
        get_option = lambda key: \
            Mmisc.option_set(opts, key, DEFAULT_PROC_OPTS)
        Mprocessor.Processor.__init__(self, core_obj)

        self.continue_running = False  # True if we should leave command loop
        self.event2short = dict(EVENT2SHORT)
        self.event2short['signal'] = '?!'
        self.event2short['brkpt'] = 'xx'

        self.optional_modules = ('ipython', 'bpy')
        self.cmd_instances = self._populate_commands()

        # command argument string. Is like current_command, but the part
        # after cmd_name has been removed.
        self.cmd_argstr = ''

        # command name before alias or macro resolution
        self.cmd_name = ''
        self.cmd_queue = []  # Queued debugger commands
        self.completer        = lambda text, state: \
          Mcomplete.completer(self, text, state)
        self.current_command = ''  # Current command getting run
        self.debug_nest = 1
        self.display_mgr = Mdisplay.DisplayMgr()
        self.intf = core_obj.debugger.intf
        self.last_command = None  # Initially a no-op
        self.precmd_hooks = []

        self.location = lambda: print_location(self)

        self.preloop_hooks = []
        self.postcmd_hooks = []

        self._populate_cmd_lists()
        self.prompt_str = '(trepan3k) '

        # Stop only if line/file is different from last time
        self.different_line = None

        # These values updated on entry. Set initial values.
        self.curframe = None
        self.event = None
        self.event_arg = None
        self.frame = None
        self.list_lineno = 0  # last list number used in "list"
        self.list_filename = None  # last filename used in list

        self.macros = {}  # Debugger Macros

        # Create a custom safe Repr instance and increase its maxstring.
        # The default of 30 truncates error messages too easily.
        self._repr = Repr()
        self._repr.maxstring = 100
        self._repr.maxother = 60
        self._repr.maxset = 10
        self._repr.maxfrozen = 10
        self._repr.array = 10
        self.stack = []
        self.thread_name = None
        self.frame_thread_name = None
        initfile_list = get_option('initfile_list')
        for init_cmdfile in initfile_list:
            self.queue_startfile(init_cmdfile)
        return
예제 #21
0
파일: metaproduct.py 프로젝트: cxz/ploomber
class MetaProduct(Mapping):
    """
    Exposes a Product-like API to allow Tasks to create more than one Product,
    it is automatically instantiated when a Task is initialized with a
    sequence or a mapping object in the product parameter. While it is
    recommended for Tasks to only have one Product (to keep them simple),
    in some cases it makes sense. For example, a Jupyter notebook
    (executed via NotebookRunner), for fitting a model might as well serialize
    the things such as the model and any data preprocessors
    """
    def __init__(self, products):
        container = ProductsContainer(products)

        self.products = container
        self.metadata = MetadataCollection(container)
        self.clients = ClientContainer(container)

        self._repr = Repr()

    @property
    def task(self):
        # TODO: validate same task
        return self.products[0].task

    @property
    def client(self):
        return self.clients

    @task.setter
    def task(self, value):
        for p in self.products:
            try:
                p.task = value
            except AttributeError as e:
                raise AttributeError(
                    "Expected MetaProduct to initialize with Product "
                    "instancess (which have a 'task' attribute), but "
                    f"got {p!r}, an object of type {type(p)}. Replace it "
                    "with a valid Product object. If this is a file, use "
                    f"File({p!r})") from e

    def exists(self):
        return all([p.exists() for p in self.products])

    def delete(self, force=False):
        for product in self.products:
            product.delete(force)

    def download(self):
        for product in self.products:
            product.download()

    def upload(self):
        for product in self.products:
            product.upload()

    def _is_outdated(self, outdated_by_code=True):
        is_outdated = [
            p._is_outdated(outdated_by_code=outdated_by_code)
            for p in self.products
        ]

        if set(is_outdated) == {False}:
            return False

        if set(is_outdated) <= {TaskStatus.WaitingDownload, False}:
            return TaskStatus.WaitingDownload

        return any(is_outdated)

    def _outdated_data_dependencies(self):
        return any([p._outdated_data_dependencies() for p in self.products])

    def _outdated_code_dependency(self):
        return any([p._outdated_code_dependency() for p in self.products])

    def to_json_serializable(self):
        """Returns a JSON serializable version of this product
        """
        # NOTE: this is used in tasks where only JSON serializable parameters
        # are supported such as NotebookRunner that depends on papermill
        return self.products.to_json_serializable()

    def render(self, params, **kwargs):
        for p in self.products:
            p.render(params, **kwargs)

    def __repr__(self):
        content = self._repr.repr1(self.products.products, level=2)
        return f'{type(self).__name__}({content})'

    def __str__(self):
        return str(self.products)

    def __iter__(self):
        for product in self.products:
            yield product

    def __getitem__(self, key):
        return self.products[key]

    def __len__(self):
        return len(self.products)

    @property
    def _remote(self):
        return self.products.first._remote

    def _is_remote_outdated(self, outdated_by_code):
        return self.products.first._is_remote_outdated(outdated_by_code)
예제 #22
0
class Placeholder(abc.AbstractPlaceholder):
    """
    Placeholder powers all the objects that use placeholder variables (
    between curly brackets). It uses a jinja2.Template object under the hood
    but adds a few important things:

    * Keeps the raw (undendered) value: template.raw
    * Keeps path to raw value: template.location
    * Strict: will not render if missing or extra parameters
    * Upon calling .render, saves the rendered value for later access

    End users should not manipulate Placeholder objects, they should be
    automatically created from strings, pathlib.Path or jinja2.Template
    objects.

    Placeholder is mostly used by sources whose source code are parametrized
    strings (e.g. SQL scripts)

    Parameters
    ----------
    hot_reload : bool, optional
        Makes the placeholder always read the template from the file before
        rendering

    required
        Set of keys required for rendering

    Attributes
    ----------
    variables : set
        Returns the set of variables in the template (values sourrounded by
        {{ and }})

    path : pathlib.Path
        The location of the raw object. None if initialized with a str or with
        a jinja2.Template created from a str

    Notes
    -----
    You can use "raise" in a placeholder to raise exceptions, useful for
    validating input parameters: "{% raise 'some error message' %}"
    """
    def __init__(self, primitive, hot_reload=False, required=None):
        self._logger = logging.getLogger('{}.{}'.format(
            __name__,
            type(self).__name__))
        self._hot_reload = hot_reload

        self._variables = None
        self.__template = None

        # we have to take care of 4 possible cases and make sure we have
        # all we need to initialize the template, this includes having
        # access to the raw template (str) and a way to re-initialize
        # the jinja.environment.loader object (to make sure copies and
        # pickles work)

        if isinstance(primitive, Path):
            self._path = primitive
            self.__raw = primitive.read_text()
            self._loader_init = None
        elif isinstance(primitive, str):
            self._path = None
            self.__raw = primitive
            self._loader_init = None

        elif isinstance(primitive, Template):
            # NOTE: primitive.filename will be '<template>' if Template was
            # loaded from a string
            path = Path(primitive.filename)

            if primitive.environment.undefined != StrictUndefined:
                raise ValueError('Placeholder can only be initialized '
                                 'from jinja2.Templates whose undefined '
                                 'parameter is set to '
                                 'jinja2.StrictUndefined, set it explicitely '
                                 'either in the Template or Environment '
                                 'constructors')

            # we cannot get the raw template on this case, raise error
            # check '<template>' first, because Path('<template>').exists()
            # breaks on windows
            if primitive.filename == '<template>' or not path.exists():
                raise ValueError(
                    'Could not load raw source from '
                    'jinja2.Template. This usually happens '
                    'when the placeholder is initialised with a '
                    'jinja.Template which was initialized with '
                    'a string. Only jinja2.Templates loaded from '
                    'the filesystem are supported. Use '
                    'ploomber.SourceLoader or jinja\'s '
                    'FileSystemLoader/PackageLoader to fix '
                    'this issue, if you want to create a template from '
                    'a string pass it directly '
                    'Placeholder("some {{placeholder}}")')

            self._path = path
            self.__raw = path.read_text()
            self._loader_init = _make_loader_init(primitive.environment.loader)
        # SourceLoader returns Placeholder objects, which could inadvertedly
        # be passed to another Placeholder constructor when instantiating
        # a source object, since they sometimes use placeholders
        #  make sure this case is covered
        elif isinstance(primitive, Placeholder):
            self._path = primitive.path
            self.__raw = primitive._raw
            self._loader_init = _make_loader_init(
                primitive._template.environment.loader)
        else:
            raise TypeError('{} must be initialized with a Template, '
                            'Placeholder, pathlib.Path or str, '
                            'got {} instead'.format(
                                type(self).__name__,
                                type(primitive).__name__))

        if self._path is None and hot_reload:
            raise ValueError('hot_reload only works when Placeholder is '
                             'initialized from a file')

        # TODO: remove
        self.needs_render = self._needs_render()

        self._str = None if self.needs_render else self._raw
        self._repr = Repr()
        self._repr.maxstring = 40

        if required:
            self._validate_required(required)

    def _validate_required(self, required):
        missing_required = set(required) - self.variables

        if missing_required:
            msg = ('The following tags are required. ' +
                   display_error(missing_required, required))
            raise SourceInitializationError(msg)

    @property
    def _template(self):
        if self.__template is None or self._hot_reload:
            self.__template = _init_template(self._raw, self._loader_init)

        return self.__template

    @property
    def _raw(self):
        """A string with the raw jinja2.Template contents
        """
        if self._hot_reload:
            self.__raw = self._path.read_text()

        return self.__raw

    @_raw.setter
    def _raw(self, value):
        self.__raw = value

    @property
    def path(self):
        return self._path

    def _needs_render(self):
        """
        Returns true if the template is a literal and does not need any
        parameters to render
        """
        env = self._template.environment

        # check if the template has the variable or block start string
        # is there any better way of checking this?
        needs_variables = (env.variable_start_string in self._raw
                           and env.variable_end_string in self._raw)
        needs_blocks = (env.block_start_string in self._raw
                        and env.block_end_string in self._raw)

        return needs_variables or needs_blocks

    def __str__(self):
        if self._str is None:
            raise RuntimeError('Tried to read {} {} without '
                               'rendering first'.format(
                                   type(self).__name__, repr(self)))

        return self._str

    def render(self, params, optional=None, required=None):
        """
        """
        optional = optional or set()
        optional = set(optional)

        passed = set(params.keys())
        available = passed | set(self._template.environment.globals)

        missing = self.variables - available
        extra = passed - self.variables - optional

        # FIXME: self.variables should also be updated on hot_reload
        if missing:
            raise RenderError('in {}, missing required '
                              'parameters: {}, params passed: {}'.format(
                                  repr(self), missing, params))

        if extra:
            raise RenderError('in {}, unused parameters: {}, params '
                              'declared: {}'.format(repr(self), extra,
                                                    self.variables))

        try:
            self._str = self._template.render(**params)
        except UndefinedError as e:
            # TODO: we can use e.message to see which param caused the
            # error
            raise RenderError('in {}, jinja2 raised an UndefinedError, this '
                              'means the template is using an attribute '
                              'or item that does not exist, the original '
                              'traceback is shown above. For jinja2 '
                              'implementation details see: '
                              'http://jinja.pocoo.org/docs/latest'
                              '/templates/#variables'.format(
                                  repr(self))) from e

        return str(self)

    def best_repr(self, shorten):
        """
        Returns the rendered version (if available), otherwise the raw version
        """
        best = self._raw if self._str is None else self._str

        if shorten:
            best = self._repr.repr(best)

        return best

    @property
    def variables(self):
        """Returns declared variables in the template
        """
        # this requires parsing the raw template, do lazy load, but override
        # it if hot_reload is True
        if self._variables is None or self._hot_reload:
            self._variables = util.get_tags_in_str(self._raw)

        return self._variables

    def __repr__(self):
        content = self.best_repr(shorten=True)
        return f'{type(self).__name__}({content})'

    def __getstate__(self):
        state = self.__dict__.copy()
        del state['_logger']
        del state['_Placeholder__template']
        return state

    def __setstate__(self, state):
        self.__dict__.update(state)
        self._logger = logging.getLogger('{}.{}'.format(
            __name__,
            type(self).__name__))

        self.__template = None
예제 #23
0
 def __init__(self):
     Repr.__init__(self)
     self.maxlist = self.maxtuple = 20
     self.maxdict = 10
     self.maxstring = self.maxother = 100
예제 #24
0
# XXX TO DO:
# - popup menu
# - support partial or total redisplay
# - more doc strings
# - tooltips

# object browser

# XXX TO DO:
# - for classes/modules, add "open source" to object browser

from .TreeWidget import TreeItem, TreeNode, ScrolledCanvas

from reprlib import Repr

myrepr = Repr()
myrepr.maxstring = 100
myrepr.maxother = 100


class ObjectTreeItem(TreeItem):
    def __init__(self, labeltext, object, setfunction=None):
        self.labeltext = labeltext
        self.object = object
        self.setfunction = setfunction

    def GetLabelText(self):
        return self.labeltext

    def GetText(self):
        return myrepr.repr(self.object)
예제 #25
0
    def __init__(self, primitive, hot_reload=False, required=None):
        self._logger = logging.getLogger('{}.{}'.format(
            __name__,
            type(self).__name__))
        self._hot_reload = hot_reload

        self._variables = None
        self.__template = None

        # we have to take care of 4 possible cases and make sure we have
        # all we need to initialize the template, this includes having
        # access to the raw template (str) and a way to re-initialize
        # the jinja.environment.loader object (to make sure copies and
        # pickles work)

        if isinstance(primitive, Path):
            self._path = primitive
            self.__raw = primitive.read_text()
            self._loader_init = None
        elif isinstance(primitive, str):
            self._path = None
            self.__raw = primitive
            self._loader_init = None

        elif isinstance(primitive, Template):
            # NOTE: primitive.filename will be '<template>' if Template was
            # loaded from a string
            path = Path(primitive.filename)

            if primitive.environment.undefined != StrictUndefined:
                raise ValueError('Placeholder can only be initialized '
                                 'from jinja2.Templates whose undefined '
                                 'parameter is set to '
                                 'jinja2.StrictUndefined, set it explicitely '
                                 'either in the Template or Environment '
                                 'constructors')

            # we cannot get the raw template on this case, raise error
            # check '<template>' first, because Path('<template>').exists()
            # breaks on windows
            if primitive.filename == '<template>' or not path.exists():
                raise ValueError(
                    'Could not load raw source from '
                    'jinja2.Template. This usually happens '
                    'when the placeholder is initialised with a '
                    'jinja.Template which was initialized with '
                    'a string. Only jinja2.Templates loaded from '
                    'the filesystem are supported. Use '
                    'ploomber.SourceLoader or jinja\'s '
                    'FileSystemLoader/PackageLoader to fix '
                    'this issue, if you want to create a template from '
                    'a string pass it directly '
                    'Placeholder("some {{placeholder}}")')

            self._path = path
            self.__raw = path.read_text()
            self._loader_init = _make_loader_init(primitive.environment.loader)
        # SourceLoader returns Placeholder objects, which could inadvertedly
        # be passed to another Placeholder constructor when instantiating
        # a source object, since they sometimes use placeholders
        #  make sure this case is covered
        elif isinstance(primitive, Placeholder):
            self._path = primitive.path
            self.__raw = primitive._raw
            self._loader_init = _make_loader_init(
                primitive._template.environment.loader)
        else:
            raise TypeError('{} must be initialized with a Template, '
                            'Placeholder, pathlib.Path or str, '
                            'got {} instead'.format(
                                type(self).__name__,
                                type(primitive).__name__))

        if self._path is None and hot_reload:
            raise ValueError('hot_reload only works when Placeholder is '
                             'initialized from a file')

        # TODO: remove
        self.needs_render = self._needs_render()

        self._str = None if self.needs_render else self._raw
        self._repr = Repr()
        self._repr.maxstring = 40

        if required:
            self._validate_required(required)
예제 #26
0
import logging
from pprintpp import pformat as pf, pprint as pp
from reprlib import Repr
import shutil
import sys
from textwrap import indent, wrap

import sqlparse as sp

from tlbx.object_utils import get_class_vars, get_caller


logging.basicConfig(format="%(message)s")


_repr = Repr()
_repr.maxstring = 60
_repr.maxother = 20
_repr.maxlist = 5
_repr.maxtuple = 5
_repr.maxset = 5
repr = _repr.repr

DEFAULT_TERM_WIDTH = 80


class FontSpecialChars:
    ENDC = "\033[0m"


class FontColors:
예제 #27
0
# XXX TO DO:
# - popup menu
# - support partial or total redisplay
# - more doc strings
# - tooltips

# object browser

# XXX TO DO:
# - for classes/modules, add "open source" to object browser
from reprlib import Repr

from idlelib.tree import TreeItem, TreeNode, ScrolledCanvas

myrepr = Repr()
myrepr.maxstring = 100
myrepr.maxother = 100


class ObjectTreeItem(TreeItem):
    def __init__(self, labeltext, object, setfunction=None):
        self.labeltext = labeltext
        self.object = object
        self.setfunction = setfunction

    def GetLabelText(self):
        return self.labeltext

    def GetText(self):
        return myrepr.repr(self.object)
예제 #28
0
class EnvDict(Mapping):
    """
    Implements the initialization functionality for Env, except it allows
    to more than one instance to exist, this is used internally to allow
    factory functions introspection without having to create an actual Env

    Parameters
    ----------
    source : dict or str
        If str, it will be interpreted as a path to a YAML file

    path_to_here : str or pathlib.Path
        Value used to expand the {{here}} placeholder. If None, it uses the
        location of the YAML spec. If initialized with a dict and None,
        the {{here}} placeholder is not available.

    defaults : dict, default=None
        Default values to use. If not None, it uses these as defaults and
        overwrites keys using values in source

    Notes
    -----
    By default, it includes the following placeholders (unless the passed
    dictionary already contains those keys): {{user}} (current user)
    {{cwd}} (working directory), {{here}} (env.yaml location, if any), {{root}}
    (project's root folder, if any)
    """
    def __init__(self, source, path_to_here=None, defaults=None):

        # if initialized from another EnvDict, copy the attributes to
        # initialize
        # this happens in the  CLI parser, which instanttiates the env
        # because it needs to create one and then replace cli args, then
        # passes this modified object to DAGSpec
        if isinstance(source, EnvDict):
            for attr in (
                    '_path_to_env',
                    '_preprocessed',
                    '_expander',
                    '_data',
                    '_repr',
                    '_default_keys',
            ):
                original = getattr(source, attr)
                setattr(self, attr, deepcopy(original))
        else:
            (
                # load data
                raw_data,
                # this will be None if source is a dict
                self._path_to_env) = load_from_source(source)

            if defaults:
                raw_data = {**defaults, **raw_data}

            # add default placeholders but override them if they are defined
            # in the raw data
            default = self._default_dict(include_here=path_to_here is not None)
            self._default_keys = set(default) - set(raw_data)
            raw_data = {**default, **raw_data}

            # check raw data is ok
            validate.raw_data_keys(raw_data)

            # expand _module special key, return its expanded value
            self._preprocessed = raw_preprocess(raw_data, self._path_to_env)

            # initialize expander, which converts placeholders to their values
            # we need to pass path_to_env since the {{here}} placeholder
            # resolves to its parent
            if path_to_here is None:
                # if no pat_to_here, use path_to_end
                path_to_here = (None if self._path_to_env is None else Path(
                    self._path_to_env).parent)
            else:
                path_to_here = Path(path_to_here).resolve()

            self._expander = EnvironmentExpander(self._preprocessed,
                                                 path_to_here=path_to_here)
            # now expand all values
            self._data = self._expander.expand_raw_dictionary(raw_data)

            self._repr = Repr()

    @classmethod
    def find(cls, source):
        """
        Find env file recursively, currently only used by the @with_env
        decorator
        """
        if not Path(source).exists():
            source_found, _ = default.find_file_recursively(source)

            if source_found is None:
                raise FileNotFoundError('Could not find file "{}" in the '
                                        'current working directory nor '
                                        '6 levels up'.format(source))
            else:
                source = source_found

        return cls(source, path_to_here=Path(source).parent)

    @property
    def default_keys(self):
        """Returns keys whose default value is used (i.e., if the user
        overrides them, they won't appear)
        """
        return self._default_keys

    @staticmethod
    def _default_dict(include_here):
        placeholders = {
            'user': '******',
            'cwd': '{{cwd}}',
            'now': '{{now}}',
        }

        if default.try_to_find_root_recursively() is not None:
            placeholders['root'] = '{{root}}'

        if include_here:
            placeholders['here'] = '{{here}}'

        return placeholders

    @property
    def path_to_env(self):
        return self._path_to_env

    def __getattr__(self, key):
        error = AttributeError("'{}' object has no attribute '{}'".format(
            type(self).__name__, key))
        # do not look up special atttributes this way!
        if key.startswith('__') and key.endswith('__'):
            raise error

        if key in self:
            return self[key]
        else:
            raise AttributeError("{} object has no atttribute '{}'".format(
                repr(self), key))

    def __getitem__(self, key):
        try:
            return self._getitem(key)
        except KeyError as e:
            # custom error will be displayed around quotes, but it's fine.
            # this is due to the KeyError.__str__ implementation
            msg = "{} object has no key '{}'".format(repr(self), key)
            e.args = (msg, )
            raise

    def _getitem(self, key):
        if key in self._preprocessed:
            return FrozenJSON(self._preprocessed[key])
        else:
            return FrozenJSON(self._data[key])

    def __setitem__(self, key, value):
        self._data[key] = value

    def __iter__(self):
        for k in self._data:
            yield k

    def __len__(self):
        return len(self._data)

    def __str__(self):
        return str(self._data)

    def __repr__(self):
        content = self._repr.repr_dict(self._data, level=2)
        return f'{type(self).__name__}({content})'

    def _replace_value(self, value, keys_all):
        """
        Replace a value in the underlying dictionary, by passing a value and
        a list of keys

        e.g. given {'a': {'b': 1}}, we can replace 1 by doing
        _replace_value(2, ['a', 'b'])
        """
        keys_to_final_dict = keys_all[:-1]
        key_to_edit = keys_all[-1]

        dict_to_edit = self._data

        for e in keys_to_final_dict:
            dict_to_edit = dict_to_edit[e]

        if dict_to_edit.get(key_to_edit) is None:
            dotted_path = '.'.join(keys_all)
            raise KeyError('Trying to replace key "{}" in env, '
                           'but it does not exist'.format(dotted_path))

        dict_to_edit[key_to_edit] = (self._expander.expand_raw_value(
            value, keys_all))

    def _inplace_replace_flatten_key(self, value, key_flatten):
        """
        Replace a value in the underlying dictionary, by passing a value and
        a list of keys

        e.g. given {'a': {'b': 1}}, we can replace 1 by doing
        _replace_flatten_keys(2, 'env__a__b'). This function is used
        internally to overrive env values when calling factories (functions
        decorated with @with_env or when doing so via the command line
        interface - ploomber build pipeline.yaml --env--a--b 2)

        Returns a copy
        """
        # convert env__a__b__c -> ['a', 'b', 'c']
        parts = key_flatten.split('__')

        if parts[0] != 'env':
            raise ValueError('keys_flatten must start with env__')

        keys_all = parts[1:]
        self._replace_value(value, keys_all)

    def _replace_flatten_key(self, value, key_flatten):
        obj = copy(self)
        obj._inplace_replace_flatten_key(value, key_flatten)
        return obj

    def _inplace_replace_flatten_keys(self, to_replace):
        """Replace multiple keys at once

        Returns a copy
        """
        for key, value in to_replace.items():
            self._inplace_replace_flatten_key(value, key)

    def _replace_flatten_keys(self, to_replace):
        obj = copy(self)
        obj._inplace_replace_flatten_keys(to_replace)
        return obj
예제 #29
0
class File(ProductWithClientMixin, os.PathLike, Product):
    """A file (or directory) in the local filesystem

    Parameters
    ----------
    identifier: str or pathlib.Path
        The path to the file (or directory), can contain placeholders
        (e.g. {{placeholder}})
    """
    def __init__(self, identifier, client=None):
        super().__init__(identifier)
        self._client = client
        self._repr = Repr()
        self._repr.maxstring = 40
        self._remote_ = _RemoteFile(self)

    def _init_identifier(self, identifier):
        if not isinstance(identifier, (str, Path)):
            raise TypeError('File must be initialized with a str or a '
                            'pathlib.Path')

        return Placeholder(str(identifier))

    @property
    def _path_to_file(self):
        return Path(str(self._identifier))

    @property
    def _path_to_metadata(self):
        name = f'.{self._path_to_file.name}.metadata'
        return self._path_to_file.with_name(name)

    @property
    def _remote(self):
        """
        RemoteFile for this File. Returns None if a
        File.client doesn't exist, remote file doesn't exist or remote
        metadata doesn't exist
        """
        return self._remote_

    @property
    def _remote_path_to_metadata(self):
        return self._remote._path_to_metadata

    def fetch_metadata(self):
        # migrate metadata file to keep compatibility with ploomber<0.10
        old_name = Path(str(self._path_to_file) + '.source')
        if old_name.is_file():
            shutil.move(old_name, self._path_to_metadata)

        return _fetch_metadata_from_file_product(self, check_file_exists=True)

    def save_metadata(self, metadata):
        self._path_to_metadata.write_text(json.dumps(metadata))

    def _delete_metadata(self):
        if self._path_to_metadata.exists():
            os.remove(str(self._path_to_metadata))

    def exists(self):
        return self._path_to_file.exists()

    def delete(self, force=False):
        # force is not used for this product but it is left for API
        # compatibility
        if self.exists():
            self.logger.debug('Deleting %s', self._path_to_file)
            if self._path_to_file.is_dir():
                shutil.rmtree(str(self._path_to_file))
            else:
                os.remove(str(self._path_to_file))
        else:
            self.logger.debug('%s does not exist ignoring...',
                              self._path_to_file)

    def __repr__(self):
        # do not shorten, we need to process the actual path
        path = Path(self._identifier.best_repr(shorten=False))

        # if absolute, try to show a shorter version, if possible
        if path.is_absolute():
            try:
                path = path.relative_to(Path('.').resolve())
            except ValueError:
                # happens if the path is not a file/folder within the current
                # working directory
                pass

        content = self._repr.repr(str(path))
        return f'{type(self).__name__}({content})'

    def _check_is_outdated(self, outdated_by_code):
        """
        Unlike other Product implementation that only have to check the
        current metadata, File has to check if there is a metadata remote copy
        and download it to decide outdated status, which yield to task
        execution or product downloading
        """
        should_download = False

        if self._remote.exists():
            if self._remote._is_equal_to_local_copy():
                return self._remote._is_outdated(with_respect_to_local=True)
            else:
                # download when doing so will bring the product
                # up-to-date (this takes into account upstream
                # timestamps)
                should_download = not self._remote._is_outdated(
                    with_respect_to_local=True,
                    outdated_by_code=outdated_by_code)

        if should_download:
            return TaskStatus.WaitingDownload

        # no need to download, check status using local metadata
        return super()._check_is_outdated(outdated_by_code=outdated_by_code)

    def _is_remote_outdated(self, outdated_by_code):
        """
        Check status using remote metadata, if no remote is available
        (or remote metadata is corrupted) returns True
        """
        if self._remote.exists():
            return self._remote._is_outdated(with_respect_to_local=False,
                                             outdated_by_code=outdated_by_code)
        else:
            # if no remote, return True. This is the least destructive option
            # since we don't know what will be available and what not when this
            # executes
            return True

    @property
    def client(self):
        try:
            client = super().client
        except MissingClientError:
            return None
        else:
            return client

    def download(self):
        self.logger.info('Downloading %s...', self._path_to_file)

        if self.client:
            self.client.download(str(self._path_to_file))
            self.client.download(str(self._path_to_metadata))

    def upload(self):
        if self.client:
            if not self._path_to_metadata.exists():
                raise RuntimeError(
                    f'Error uploading product {self!r}. '
                    f'Metadata {str(self._path_to_metadata)!r} does '
                    'not exist')

            if not self._path_to_file.exists():
                raise RuntimeError(f'Error uploading product {self!r}. '
                                   f'Product {str(self._path_to_file)!r} does '
                                   'not exist')

            self.logger.info('Uploading %s...', self._path_to_file)
            self.client.upload(self._path_to_metadata)
            self.client.upload(self._path_to_file)

    def __fspath__(self):
        """
        Abstract method defined in the os.PathLike interface, enables this
        to work: ``import pandas as pd; pd.read_csv(File('file.csv'))``
        """
        return str(self)

    def __eq__(self, other):
        return Path(str(self)).resolve() == Path(str(other)).resolve()

    def __hash__(self):
        return hash(Path(str(self)).resolve())
예제 #30
0
 def repr(self, object):
     return Repr.repr(self, object)
예제 #31
0
class CommandProcessor(Mprocessor.Processor):
    def __init__(self, core_obj, opts=None):
        get_option = lambda key: \
            Mmisc.option_set(opts, key, DEFAULT_PROC_OPTS)
        Mprocessor.Processor.__init__(self, core_obj)

        self.continue_running = False  # True if we should leave command loop
        self.event2short = dict(EVENT2SHORT)
        self.event2short['signal'] = '?!'
        self.event2short['brkpt'] = 'xx'

        self.optional_modules = ('ipython', 'bpy')
        self.cmd_instances = self._populate_commands()

        # command argument string. Is like current_command, but the part
        # after cmd_name has been removed.
        self.cmd_argstr = ''

        # command name before alias or macro resolution
        self.cmd_name = ''
        self.cmd_queue = []  # Queued debugger commands
        self.completer        = lambda text, state: \
          Mcomplete.completer(self, text, state)
        self.current_command = ''  # Current command getting run
        self.debug_nest = 1
        self.display_mgr = Mdisplay.DisplayMgr()
        self.intf = core_obj.debugger.intf
        self.last_command = None  # Initially a no-op
        self.precmd_hooks = []

        self.location = lambda: print_location(self)

        self.preloop_hooks = []
        self.postcmd_hooks = []

        self._populate_cmd_lists()
        self.prompt_str = '(trepan3k) '

        # Stop only if line/file is different from last time
        self.different_line = None

        # These values updated on entry. Set initial values.
        self.curframe = None
        self.event = None
        self.event_arg = None
        self.frame = None
        self.list_lineno = 0  # last list number used in "list"
        self.list_filename = None  # last filename used in list

        self.macros = {}  # Debugger Macros

        # Create a custom safe Repr instance and increase its maxstring.
        # The default of 30 truncates error messages too easily.
        self._repr = Repr()
        self._repr.maxstring = 100
        self._repr.maxother = 60
        self._repr.maxset = 10
        self._repr.maxfrozen = 10
        self._repr.array = 10
        self.stack = []
        self.thread_name = None
        self.frame_thread_name = None
        initfile_list = get_option('initfile_list')
        for init_cmdfile in initfile_list:
            self.queue_startfile(init_cmdfile)
        return

    def _saferepr(self, str, maxwidth=None):
        if maxwidth is None:
            maxwidth = self.debugger.settings['width']
        return self._repr.repr(str)[:maxwidth]

    def add_preloop_hook(self, hook, position=-1, nodups=True):
        if hook in self.preloop_hooks: return False
        self.preloop_hooks.insert(position, hook)
        return True

    # To be overridden in derived debuggers
    def defaultFile(self):
        """Produce a reasonable default."""
        filename = self.curframe.f_code.co_filename
        # Consider using is_exec_stmt(). I just don't understand
        # the conditions under which the below test is true.
        if filename == '<string>' and self.debugger.mainpyfile:
            filename = self.debugger.mainpyfile
            pass
        return filename

    def set_prompt(self, prompt='trepan3k'):
        if self.thread_name and self.thread_name != 'MainThread':
            prompt += ':' + self.thread_name
            pass
        self.prompt_str = '%s%s%s' % ('(' * self.debug_nest, prompt,
                                      ')' * self.debug_nest)
        highlight = self.debugger.settings['highlight']
        if highlight and highlight in ('light', 'dark'):
            self.prompt_str = colorize('underline', self.prompt_str)
        self.prompt_str += ' '

    def event_processor(self, frame, event, event_arg, prompt='trepan3k'):
        'command event processor: reading a commands do something with them.'
        self.frame = frame
        self.event = event
        self.event_arg = event_arg

        filename = frame.f_code.co_filename
        lineno = frame.f_lineno
        line = linecache.getline(filename, lineno, frame.f_globals)
        if not line:
            opts = {
                'output': 'plain',
                'reload_on_change': self.settings('reload'),
                'strip_nl': False
            }
            m = re.search('^<frozen (.*)>', filename)
            if m and m.group(1):
                filename = pyficache.unmap_file(m.group(1))
            line = pyficache.getline(filename, lineno, opts)
        self.current_source_text = line
        if self.settings('skip') is not None:
            if Mbytecode.is_def_stmt(line, frame):
                return True
            if Mbytecode.is_class_def(line, frame):
                return True
            pass
        self.thread_name = Mthread.current_thread_name()
        self.frame_thread_name = self.thread_name
        self.set_prompt(prompt)
        self.process_commands()
        if filename == '<string>': pyficache.remove_remap_file('<string>')
        return True

    def forget(self):
        """ Remove memory of state variables set in the command processor """
        self.stack = []
        self.curindex = 0
        self.curframe = None
        self.thread_name = None
        self.frame_thread_name = None
        return

    def eval(self, arg):
        """Eval string arg in the current frame context."""
        try:
            return eval(arg, self.curframe.f_globals, self.curframe.f_locals)
        except:
            t, v = sys.exc_info()[:2]
            if isinstance(t, str):
                exc_type_name = t
                pass
            else:
                exc_type_name = t.__name__
            self.errmsg(str("%s: %s" % (exc_type_name, arg)))
            raise
        return None  # Not reached

    def exec_line(self, line):
        if self.curframe:
            local_vars = self.curframe.f_locals
            global_vars = self.curframe.f_globals
        else:
            local_vars = None
            # FIXME: should probably have place where the
            # user can store variables inside the debug session.
            # The setup for this should be elsewhere. Possibly
            # in interaction.
            global_vars = None
        try:
            code = compile(line + '\n', '"%s"' % line, 'single')
            exec(code, global_vars, local_vars)
        except:
            t, v = sys.exc_info()[:2]
            if type(t) == bytes:
                exc_type_name = t
            else:
                exc_type_name = t.__name__
            self.errmsg('%s: %s' % (str(exc_type_name), str(v)))
            pass
        return

    def parse_position(self, arg, old_mod=None):
        """parse_position(self, arg)->(fn, name, lineno)

        Parse arg as [filename:]lineno | function | module
        Make sure it works for C:\foo\bar.py:12
        """
        colon = arg.rfind(':')
        if colon >= 0:
            # First handle part before the colon
            arg1 = arg[:colon].rstrip()
            lineno_str = arg[colon + 1:].lstrip()
            (mf, filename,
             lineno) = self.parse_position_one_arg(arg1, old_mod, False)
            if filename is None: filename = self.core.canonic(arg1)
            # Next handle part after the colon
            val = self.get_an_int(lineno_str,
                                  "Bad line number: %s" % lineno_str)
            if val is not None: lineno = val
        else:
            (mf, filename, lineno) = self.parse_position_one_arg(arg, old_mod)
            pass

        return mf, filename, lineno

    def parse_position_one_arg(self, arg, old_mod=None, show_errmsg=True):
        """parse_position_one_arg(self, arg, show_errmsg) ->
              (module/function, file, lineno)

        See if arg is a line number, function name, or module name.
        Return what we've found. None can be returned as a value in
        the triple.
        """
        modfunc, filename, lineno = (None, None, None)
        if self.curframe:
            g = self.curframe.f_globals
            l = self.curframe.f_locals
        else:
            g = globals()
            l = locals()
            pass
        try:
            # First see if argument is an integer
            lineno = int(eval(arg, g, l))
            if old_mod is None:
                filename = self.curframe.f_code.co_filename
                pass
        except:
            try:
                modfunc = eval(arg, g, l)
            except:
                modfunc = arg
                pass
            msg = ('Object %s is not known yet as a function, module, '
                   'or is not found along sys.path, '
                   'and not a line number.') % str(repr(arg))
            try:
                # See if argument is a module or function
                if inspect.isfunction(modfunc):
                    pass
                elif inspect.ismodule(modfunc):
                    filename = pyficache.pyc2py(modfunc.__file__)
                    filename = self.core.canonic(filename)
                    return (modfunc, filename, None)
                elif hasattr(modfunc, 'im_func'):
                    modfunc = modfunc.__func__
                    pass
                else:
                    if show_errmsg: self.errmsg(msg)
                    return (None, None, None)
                code = modfunc.__code__
                lineno = code.co_firstlineno
                filename = code.co_filename
            except:
                if show_errmsg: self.errmsg(msg)
                return (None, None, None)
            pass
        return (modfunc, self.core.canonic(filename), lineno)

    def get_an_int(self, arg, msg_on_error, min_value=None, max_value=None):
        """Like cmdfns.get_an_int(), but if there's a stack frame use that
        in evaluation."""
        ret_value = self.get_int_noerr(arg)
        if ret_value is None:
            if msg_on_error:
                self.errmsg(msg_on_error)
            else:
                self.errmsg('Expecting an integer, got: %s.' % str(arg))
                pass
            return None
        if min_value and ret_value < min_value:
            self.errmsg('Expecting integer value to be at least %d, got: %d.' %
                        (min_value, ret_value))
            return None
        elif max_value and ret_value > max_value:
            self.errmsg('Expecting integer value to be at most %d, got: %d.' %
                        (max_value, ret_value))
            return None
        return ret_value

    def get_int_noerr(self, arg):
        """Eval arg and it is an integer return the value. Otherwise
        return None"""
        if self.curframe:
            g = self.curframe.f_globals
            l = self.curframe.f_locals
        else:
            g = globals()
            l = locals()
            pass
        try:
            val = int(eval(arg, g, l))
        except (SyntaxError, NameError, ValueError, TypeError):
            return None
        return val

    def get_int(self, arg, min_value=0, default=1, cmdname=None, at_most=None):
        """If no argument use the default. If arg is a an integer between
        least min_value and at_most, use that. Otherwise report an error.
        If there's a stack frame use that in evaluation."""

        if arg is None: return default
        default = self.get_int_noerr(arg)
        if default is None:
            if cmdname:
                self.errmsg(
                    ("Command '%s' expects an integer; " + "got: %s.") %
                    (cmdname, str(arg)))
            else:
                self.errmsg('Expecting a positive integer, got: %s' % str(arg))
                pass
            return None
            pass
        if default < min_value:
            if cmdname:
                self.errmsg(("Command '%s' expects an integer at least" +
                             ' %d; got: %d.') % (cmdname, min_value, default))
            else:
                self.errmsg(
                    ("Expecting a positive integer at least" + ' %d; got: %d')
                    % (min_value, default))
                pass
            return None
        elif at_most and default > at_most:
            if cmdname:
                self.errmsg(("Command '%s' expects an integer at most" +
                             ' %d; got: %d.') % (cmdname, at_most, default))
            else:
                self.errmsg(("Expecting an integer at most %d; got: %d") %
                            (at_most, default))
                pass
            pass
        return default

    def getval(self, arg):
        try:
            return eval(arg, self.curframe.f_globals, self.curframe.f_locals)
        except:
            t, v = sys.exc_info()[:2]
            if isinstance(t, str):
                exc_type_name = t
            else:
                exc_type_name = t.__name__
            self.errmsg(str("%s: %s" % (exc_type_name, arg)))
            raise
        return

    def ok_for_running(self, cmd_obj, name, nargs):
        '''We separate some of the common debugger command checks here:
        whether it makes sense to run the command in this execution state,
        if the command has the right number of arguments and so on.
        '''
        if hasattr(cmd_obj, 'execution_set'):
            if not (self.core.execution_status in cmd_obj.execution_set):
                part1 = (
                    "Command '%s' is not available for execution status:" %
                    name)
                mess = Mmisc.wrapped_lines(part1, self.core.execution_status,
                                           self.debugger.settings['width'])
                self.errmsg(mess)
                return False
            pass
        if self.frame is None and cmd_obj.need_stack:
            self.intf[-1].errmsg("Command '%s' needs an execution stack." %
                                 name)
            return False
        if nargs < cmd_obj.min_args:
            self.errmsg(
                ("Command '%s' needs at least %d argument(s); " + "got %d.") %
                (name, cmd_obj.min_args, nargs))
            return False
        elif cmd_obj.max_args is not None and nargs > cmd_obj.max_args:
            self.errmsg(
                ("Command '%s' can take at most %d argument(s);" + " got %d.")
                % (name, cmd_obj.max_args, nargs))
            return False
        return True

    def process_commands(self):
        """Handle debugger commands."""
        if self.core.execution_status != 'No program':
            self.setup()
            self.location()
            pass
        leave_loop = run_hooks(self, self.preloop_hooks)
        self.continue_running = False

        while not leave_loop:
            try:
                run_hooks(self, self.precmd_hooks)
                # bdb had a True return to leave loop.
                # A more straight-forward way is to set
                # instance variable self.continue_running.
                leave_loop = self.process_command()
                if leave_loop or self.continue_running: break
            except EOFError:
                # If we have stacked interfaces, pop to the next
                # one.  If this is the last one however, we'll
                # just stick with that.  FIXME: Possibly we should
                # check to see if we are interactive.  and not
                # leave if that's the case. Is this the right
                # thing?  investigate and fix.
                if len(self.debugger.intf) > 1:
                    del self.debugger.intf[-1]
                    self.last_command = ''
                else:
                    if self.debugger.intf[-1].output:
                        self.debugger.intf[-1].output.writeline('Leaving')
                        raise Mexcept.DebuggerQuit
                        pass
                    break
                pass
            pass
        return run_hooks(self, self.postcmd_hooks)

    def process_command(self):
        # process command
        if len(self.cmd_queue) > 0:
            current_command = self.cmd_queue[0].strip()
            del self.cmd_queue[0]
        else:
            current_command = (self.intf[-1].read_command(
                self.prompt_str).strip())
            if '' == current_command and self.intf[-1].interactive:
                current_command = self.last_command
                pass
            pass
        # Look for comments
        if '' == current_command:
            if self.intf[-1].interactive:
                self.errmsg("No previous command registered, " +
                            "so this is a no-op.")
                pass
            return False
        if current_command is None or current_command[0] == '#':
            return False
        try:
            args_list = arg_split(current_command)
        except:
            self.errmsg("bad parse %s: %s" % sys.exc_info()[0:2])
            import traceback
            for s in traceback.format_tb(sys.exc_info()[2], limit=None):
                self.errmsg(s.strip())
            return False

        for args in args_list:
            if len(args):
                while True:
                    if len(args) == 0: return False
                    macro_cmd_name = args[0]
                    if macro_cmd_name not in self.macros: break
                    try:
                        current_command = \
                          self.macros[macro_cmd_name][0](*args[1:])
                    except TypeError:
                        t, v = sys.exc_info()[:2]
                        self.errmsg("Error expanding macro %s" %
                                    macro_cmd_name)
                        return False
                    if self.settings('debugmacro'):
                        print(current_command)
                        pass
                    if type(current_command) == list:
                        for x in current_command:
                            if str != type(x):
                                self.errmsg(("macro %s should return a List " +
                                             "of Strings. Has %s of type %s") %
                                            (macro_cmd_name, x,
                                             repr(current_command), type(x)))
                                return False
                            pass

                        first = current_command[0]
                        args = first.split()
                        self.cmd_queue + [current_command[1:]]
                        current_command = first
                    elif type(current_command) == str:
                        args = current_command.split()
                    else:
                        self.errmsg(("macro %s should return a List " +
                                     "of Strings or a String. Got %s") %
                                    (macro_cmd_name, repr(current_command)))
                        return False
                    pass

                self.cmd_name = args[0]
                cmd_name = resolve_name(self, self.cmd_name)
                self.cmd_argstr = current_command[len(self.cmd_name):].lstrip()
                if cmd_name:
                    self.last_command = current_command
                    cmd_obj = self.commands[cmd_name]
                    if self.ok_for_running(cmd_obj, cmd_name, len(args) - 1):
                        try:
                            self.current_command = current_command
                            result = cmd_obj.run(args)
                            if result: return result
                        except (Mexcept.DebuggerQuit, Mexcept.DebuggerRestart,
                                SystemExit):
                            # Let these exceptions propagate through
                            raise
                        except:
                            self.errmsg("INTERNAL ERROR: " +
                                        traceback.format_exc())
                            pass
                        pass
                    pass
                elif not self.settings('autoeval'):
                    self.undefined_cmd(current_command)
                else:
                    self.exec_line(current_command)
                    pass
                pass
            pass
        return False

    def remove_preloop_hook(self, hook):
        try:
            position = self.preloop_hooks.index(hook)
        except ValueError:
            return False
        del self.preloop_hooks[position]
        return True

    def setup(self):
        """Initialization done before entering the debugger-command
        loop. In particular we set up the call stack used for local
        variable lookup and frame/up/down commands.

        We return True if we should NOT enter the debugger-command
        loop."""
        self.forget()
        if self.settings('dbg_trepan'):
            self.frame = inspect.currentframe()
            pass
        if self.event in ['exception', 'c_exception']:
            exc_type, exc_value, exc_traceback = self.event_arg
        else:
            _, _, exc_traceback = (
                None,
                None,
                None,
            )  # NOQA
            pass
        if self.frame or exc_traceback:
            self.stack, self.curindex = \
                get_stack(self.frame, exc_traceback, None, self)
            self.curframe = self.stack[self.curindex][0]
            self.thread_name = Mthread.current_thread_name()
            if exc_traceback:
                self.list_lineno = traceback.extract_tb(exc_traceback, 1)[0][1]

        else:
            self.stack = self.curframe = \
                self.botframe = None
            pass
        if self.curframe:
            self.list_lineno = \
                max(1, inspect.getlineno(self.curframe)
                    - int(self.settings('listsize') / 2)) - 1
            self.list_filename = self.curframe.f_code.co_filename
        else:
            if not exc_traceback: self.list_lineno = None
            pass
        # if self.execRcLines()==1: return True
        return False

    def queue_startfile(self, cmdfile):
        '''Arrange for file of debugger commands to get read in the
        process-command loop.'''
        expanded_cmdfile = os.path.expanduser(cmdfile)
        is_readable = Mfile.readable(expanded_cmdfile)
        if is_readable:
            self.cmd_queue.append('source ' + expanded_cmdfile)
        elif is_readable is None:
            self.errmsg("source file '%s' doesn't exist" % expanded_cmdfile)
        else:
            self.errmsg("source file '%s' is not readable" % expanded_cmdfile)
            pass
        return

    def undefined_cmd(self, cmd):
        """Error message when a command doesn't exist"""
        self.errmsg('Undefined command: "%s". Try "help".' % cmd)
        return

    def read_history_file(self):
        """Read the command history file -- possibly."""
        histfile = self.debugger.intf[-1].histfile
        try:
            import readline
            readline.read_history_file(histfile)
        except IOError:
            pass
        except ImportError:
            pass
        return

    def write_history_file(self):
        """Write the command history file -- possibly."""
        settings = self.debugger.settings
        histfile = self.debugger.intf[-1].histfile
        if settings['hist_save']:
            try:
                import readline
                try:
                    readline.write_history_file(histfile)
                except IOError:
                    pass
            except ImportError:
                pass
            pass
        return

    def _populate_commands(self):
        """ Create an instance of each of the debugger
        commands. Commands are found by importing files in the
        directory 'command'. Some files are excluded via an array set
        in __init__.  For each of the remaining files, we import them
        and scan for class names inside those files and for each class
        name, we will create an instance of that class. The set of
        DebuggerCommand class instances form set of possible debugger
        commands."""
        from trepan.processor import command as Mcommand
        if hasattr(Mcommand, '__modules__'):
            return self.populate_commands_easy_install(Mcommand)
        else:
            return self.populate_commands_pip(Mcommand)

    def populate_commands_pip(self, Mcommand):
        cmd_instances = []
        eval_cmd_template = 'command_mod.%s(self)'
        for mod_name in Mcommand.__dict__.keys():
            if mod_name.startswith('__'): continue
            import_name = "trepan.processor.command." + mod_name
            imp = __import__(import_name)
            if imp.__name__ == 'trepan':
                command_mod = imp.processor.command
            else:
                if mod_name in (
                        'info_sub',
                        'set_sub',
                        'show_sub',
                ):
                    pass
                try:
                    command_mod = getattr(__import__(import_name), mod_name)
                except:
                    # Don't need to warn about optional modules
                    if mod_name not in self.optional_modules:
                        print('Error importing %s: %s' %
                              (mod_name, sys.exc_info()[0]))
                        pass
                    continue
                pass

            classnames = [
                tup[0]
                for tup in inspect.getmembers(command_mod, inspect.isclass)
                if ('DebuggerCommand' != tup[0] and tup[0].endswith('Command'))
            ]
            for classname in classnames:
                eval_cmd = eval_cmd_template % classname
                if False:
                    instance = eval(eval_cmd)
                    cmd_instances.append(instance)
                else:
                    try:
                        instance = eval(eval_cmd)
                        cmd_instances.append(instance)
                    except:
                        print('Error loading %s from %s: %s' %
                              (classname, mod_name, sys.exc_info()[0]))
                        pass
                    pass
                pass
            pass
        return cmd_instances

    def populate_commands_easy_install(self, Mcommand):
        cmd_instances = []
        srcdir = get_srcdir()
        sys.path.insert(0, srcdir)

        for mod_name in Mcommand.__modules__:
            if mod_name in (
                    'info_sub',
                    'set_sub',
                    'show_sub',
            ):
                pass
            import_name = "command." + mod_name
            try:
                command_mod = getattr(__import__(import_name), mod_name)
            except:
                if mod_name not in self.optional_modules:
                    print('Error importing %s: %s' %
                          (mod_name, sys.exc_info()[0]))
                    pass
                continue

            classnames = [
                tup[0]
                for tup in inspect.getmembers(command_mod, inspect.isclass)
                if ('DebuggerCommand' != tup[0] and tup[0].endswith('Command'))
            ]
            for classname in classnames:
                if False:
                    instance = getattr(command_mod, classname)(self)
                    cmd_instances.append(instance)
                else:
                    try:
                        instance = getattr(command_mod, classname)(self)
                        cmd_instances.append(instance)
                    except:
                        print('Error loading %s from %s: %s' %
                              (classname, mod_name, sys.exc_info()[0]))
                        pass
                    pass
                pass
            pass
        return cmd_instances

    def _populate_cmd_lists(self):
        """ Populate self.lists and hashes:
        self.commands, and self.aliases, self.category """
        self.commands = {}
        self.aliases = {}
        self.category = {}
        #         self.short_help = {}
        for cmd_instance in self.cmd_instances:
            if not hasattr(cmd_instance, 'aliases'): continue
            alias_names = cmd_instance.aliases
            cmd_name = cmd_instance.name
            self.commands[cmd_name] = cmd_instance
            for alias_name in alias_names:
                self.aliases[alias_name] = cmd_name
                pass
            cat = getattr(cmd_instance, 'category')
            if cat and self.category.get(cat):
                self.category[cat].append(cmd_name)
            else:
                self.category[cat] = [cmd_name]
                pass
#             sh = getattr(cmd_instance, 'short_help')
#             if sh:
#                 self.short_help[cmd_name] = getattr(c, 'short_help')
#                 pass
            pass
        for k in list(self.category.keys()):
            self.category[k].sort()
            pass

        return

    pass
예제 #32
0
def repr_values(condition: Callable[..., bool],
                lambda_inspection: Optional[ConditionLambdaInspection],
                resolved_kwargs: Mapping[str, Any],
                a_repr: reprlib.Repr) -> List[str]:
    """
    Represent function arguments and frame values in the error message on contract breach.

    :param condition: condition function of the contract
    :param lambda_inspection:
        inspected lambda AST node corresponding to the condition function (None if the condition was not given as a
        lambda function)
    :param resolved_kwargs: arguments put in the function call
    :param a_repr: representation instance that defines how the values are represented.
    :return: list of value representations
    """
    # Hide _ARGS and _KWARGS if they are not part of the condition for better readability
    if '_ARGS' in resolved_kwargs or '_KWARGS' in resolved_kwargs:
        parameters = inspect.signature(condition).parameters
        malleable_kwargs = cast(
            MutableMapping[str, Any],
            resolved_kwargs.copy()  # type: ignore
        )

        if '_ARGS' not in parameters:
            malleable_kwargs.pop('_ARGS', None)

        if '_KWARGS' not in parameters:
            malleable_kwargs.pop('_KWARGS', None)

        selected_kwargs = cast(Mapping[str, Any], malleable_kwargs)
    else:
        selected_kwargs = resolved_kwargs

    # Don't use ``resolved_kwargs`` from this point on.
    # ``selected_kwargs`` is meant to be used instead for better readability of error messages.

    if is_lambda(a_function=condition):
        assert lambda_inspection is not None, "Expected a lambda inspection when given a condition as a lambda function"
    else:
        assert lambda_inspection is None, "Expected no lambda inspection in a condition given as a non-lambda function"

    reprs = None  # type: Optional[MutableMapping[str, Any]]

    if lambda_inspection is not None:
        variable_lookup = collect_variable_lookup(
            condition=condition, resolved_kwargs=selected_kwargs)

        recompute_visitor = icontract._recompute.Visitor(
            variable_lookup=variable_lookup)

        recompute_visitor.visit(node=lambda_inspection.node.body)
        recomputed_values = recompute_visitor.recomputed_values

        repr_visitor = Visitor(recomputed_values=recomputed_values,
                               variable_lookup=variable_lookup,
                               atok=lambda_inspection.atok)
        repr_visitor.visit(node=lambda_inspection.node.body)

        reprs = repr_visitor.reprs

    # Add original arguments from the call unless they shadow a variable in the re-computation.
    #
    # The condition arguments are often not sufficient to figure out the error. The user usually needs
    # more context which is captured in the remainder of the call arguments.

    if reprs is None:
        reprs = dict()

    for key in sorted(selected_kwargs.keys()):
        val = selected_kwargs[key]
        if key not in reprs and _representable(value=val):
            reprs[key] = val

    parts = []  # type: List[str]

    # We need to sort in order to present the same violation error on repeated violations.
    # Otherwise, the order of the reported arguments may be arbitrary.
    for key in sorted(reprs.keys()):
        value = reprs[key]
        if isinstance(value, icontract._recompute.FirstExceptionInAll):
            writing = ['{} was False, e.g., with'.format(key)]
            for input_name, input_value in value.inputs:
                writing.append('\n')
                writing.append('  {} = {}'.format(input_name,
                                                  a_repr.repr(input_value)))

            parts.append(''.join(writing))
        else:
            parts.append('{} was {}'.format(key, a_repr.repr(value)))

    return parts
예제 #33
0
def safe_max(*args):
    iterable = args[0] if len(args) == 1 else args
    return max((value for value in iterable if value is not None),
               default=None)


def is_ipython():
    try:
        __IPYTHON__
        return True
    except NameError:
        return False


compact_repr = Repr()
compact_repr.maxstring = 80
compact_repr.maxother = 80
compact_repr = compact_repr.repr


class OmittedType:
    __slots__ = ()

    def __repr__(self):
        return "<...>"


Omitted = OmittedType()

예제 #34
0
standard_library.install_aliases()

from builtins import open

import inspect
import smtplib
import sys
import string
import tempfile
import traceback
import xmlrpc.client

from reprlib import Repr

_repr = Repr()
_repr.maxstring = 3000
_saferepr = _repr.repr


def printTraceBack(tb=None, output=sys.stderr, exc_type=None, exc_msg=None):
    if isinstance(output, str):
        output = open(output, 'w')

    exc_info = sys.exc_info()
    if tb is None:
        tb = exc_info[2]

    if exc_type is None:
        exc_type = exc_info[0]
예제 #35
0
import cmd
import bdb
from reprlib import Repr
import os
import re
import pprint
import traceback


class Restart(Exception):
    """Causes a debugger to be restarted for the debugged python program."""
    pass

# Create a custom safe Repr instance and increase its maxstring.
# The default of 30 truncates error messages too easily.
_repr = Repr()
_repr.maxstring = 200
_saferepr = _repr.repr

__all__ = ["run", "pm", "Pdb", "runeval", "runctx", "runcall", "set_trace",
           "post_mortem", "help"]

def find_function(funcname, filename):
    cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname))
    try:
        fp = open(filename)
    except IOError:
        return None
    # consumer of this info expects the first line to be 1
    lineno = 1
    answer = None
예제 #36
0
def repr_values(condition: Callable[..., bool],
                lambda_inspection: Optional[ConditionLambdaInspection],
                condition_kwargs: Mapping[str, Any],
                a_repr: reprlib.Repr) -> List[str]:
    # pylint: disable=too-many-locals
    """
    Represent function arguments and frame values in the error message on contract breach.

    :param condition: condition function of the contract
    :param lambda_inspection:
        inspected lambda AST node corresponding to the condition function (None if the condition was not given as a
        lambda function)
    :param condition_kwargs: condition arguments
    :param a_repr: representation instance that defines how the values are represented.
    :return: list of value representations
    """
    if _is_lambda(a_function=condition):
        assert lambda_inspection is not None, "Expected a lambda inspection when given a condition as a lambda function"
    else:
        assert lambda_inspection is None, "Expected no lambda inspection in a condition given as a non-lambda function"

    reprs = dict()  # type: MutableMapping[str, Any]

    if lambda_inspection is not None:
        # Collect the variable lookup of the condition function:
        variable_lookup = []  # type: List[Mapping[str, Any]]

        # Add condition arguments to the lookup
        variable_lookup.append(condition_kwargs)

        # Add closure to the lookup
        closure_dict = dict()  # type: Dict[str, Any]

        if condition.__closure__ is not None:  # type: ignore
            closure_cells = condition.__closure__  # type: ignore
            freevars = condition.__code__.co_freevars

            assert len(closure_cells) == len(freevars), \
                "Number of closure cells of a condition function ({}) == number of free vars ({})".format(
                    len(closure_cells), len(freevars))

            for cell, freevar in zip(closure_cells, freevars):
                closure_dict[freevar] = cell.cell_contents

        variable_lookup.append(closure_dict)

        # Add globals to the lookup
        if condition.__globals__ is not None:  # type: ignore
            variable_lookup.append(condition.__globals__)  # type: ignore

        # pylint: disable=protected-access
        recompute_visitor = icontract._recompute.Visitor(
            variable_lookup=variable_lookup)

        recompute_visitor.visit(node=lambda_inspection.node.body)
        recomputed_values = recompute_visitor.recomputed_values

        repr_visitor = Visitor(recomputed_values=recomputed_values,
                               variable_lookup=variable_lookup,
                               atok=lambda_inspection.atok)
        repr_visitor.visit(node=lambda_inspection.node.body)

        reprs = repr_visitor.reprs
    else:
        for key, val in condition_kwargs.items():
            if _representable(value=val):
                reprs[key] = val

    parts = []  # type: List[str]
    for key in sorted(reprs.keys()):
        parts.append('{} was {}'.format(key, a_repr.repr(reprs[key])))

    return parts