예제 #1
0
def _add_param_to_parser(name: str, param: BaseDescriptor,
                         parser: ArgumentParser) -> None:
    """
    Function to add a Param like IntParam, FloatParam, etc. called <name> to a parser

    :param name: The param name
    :param param: The param to add (IntParam, FloatParam, etc.)
    :param parser: The argument parser to add the param to.
    """
    argtype = _get_param_type(param)
    if argtype == type(None):
        raise NotImplementedError(
            f'Argparse type not implemented '
            f'for {param.__class__.__name__} and default not specifed')
    positional = getattr(param, 'positional', False)
    if (getattr(param, 'prefix', '') != ''
            and not getattr(param, 'expand', False)):
        raise ValueError(
            f'Failure with param {name}. Cannot add a prefix to a class without the'
            f' expand kwarg set to True')
    argname = name if positional else '--' + name
    required = True if getattr(param, 'required', False) else None
    default = param.default if required is None else None
    unit = getattr(param, 'unit', None)

    # format help nicely if default is specified and suppress is not set
    if positional or param.help == SUPPRESS:
        help = param.help
    else:
        help = f'{param.help} [default: {default} {unit}]' \
            if unit is not None else f'{param.help} [default: {default}]'
    if not required and positional:
        # TODO: use nargs='*' or nargs='?' to support not-required positional arguments
        raise ValueError(
            'Not-required positional arguments are currently not supported')
    elif positional:
        # positional arguments are required by default, and argparse complains if you specify
        # required = True
        required = None
        default = None
    action = match(param, BoolParam, lambda p: 'store_true'
                   if not p.default else 'store_false', BaseDescriptor,
                   lambda p: None)
    nargs = getattr(param, 'nargs', None)
    assert not (action is not None and nargs is not None)
    choices = match(param, EnumParam, lambda x: list(x.cls.__members__.keys()),
                    BaseDescriptor, lambda x: getattr(x, 'choices', None))
    kwargs = dict(action=action,
                  nargs=nargs,
                  default=default,
                  type=argtype,
                  required=required,
                  help=help,
                  choices=choices)
    # we delete all kwargs that are None to avoid hitting annoying action class initializations
    # such as when action is store_true and 'nargs' is in kwargs
    for kw in list(kwargs.keys()):
        if kwargs[kw] is None:
            del kwargs[kw]
    parser.add_argument(argname, **kwargs)
예제 #2
0
def test_sigusr2():
    """
    Sending SIGUSR2 to the process does an extra dump.
    """
    script = TEST_SCRIPTS / "sigusr2.py"
    output_dir = profile(script)

    # There are two dumps in the output directory, one for SIGUSR2, one for
    # shutdown.
    assert len(list(output_dir.iterdir())) == 2

    sigusr2, final = sorted(output_dir.glob("*/peak-memory.prof"))

    # SIGUSR2 dump only has allocations up to that point
    script = str(script)
    path1 = ((script, "<module>", 8), (numpy.core.numeric.__file__, "ones",
                                       ANY))
    path2 = ((script, "<module>", 11), (numpy.core.numeric.__file__, "ones",
                                        ANY))

    allocations_sigusr2 = get_allocations(sigusr2, direct=True)
    assert match(allocations_sigusr2, {path1: big},
                 as_mb) == pytest.approx(20, 0.1)
    with pytest.raises(MatchError):
        match(allocations_sigusr2, {path2: big}, as_mb)

    allocations_final = get_allocations(final, direct=True)
    assert match(allocations_final, {path1: big},
                 as_mb) == pytest.approx(20, 0.1)
    assert match(allocations_final, {path2: big},
                 as_mb) == pytest.approx(50, 0.1)
예제 #3
0
def test_malloc_in_c_extension():
    """
    Various malloc() and friends variants in C extension gets captured.
    """
    script = TEST_SCRIPTS / "malloc.py"
    output_dir = profile(script, "--size", "70")
    allocations = get_allocations(output_dir)

    script = str(script)

    # The realloc() in the scripts adds 10 to the 70:
    path = ((script, "<module>", 32), (script, "main", 28))
    assert match(allocations, {path: big},
                 as_mb) == pytest.approx(70 + 10, 0.1)

    # The C++ new allocation:
    path = ((script, "<module>", 32), (script, "main", 23))
    assert match(allocations, {path: big}, as_mb) == pytest.approx(40, 0.1)

    # C++ aligned_alloc(); not available on Conda, where it's just a macro
    # redirecting to posix_memalign.
    if not os.environ.get("CONDA_PREFIX"):
        path = ((script, "<module>", 32), (script, "main", 24))
        assert match(allocations, {path: big}, as_mb) == pytest.approx(90, 0.1)

    # Py*_*Malloc APIs:
    path = ((script, "<module>", 32), (script, "main", 25))
    assert match(allocations, {path: big}, as_mb) == pytest.approx(30, 0.1)

    # posix_memalign():
    path = ((script, "<module>", 32), (script, "main", 26))
    assert match(allocations, {path: big}, as_mb) == pytest.approx(15, 0.1)
예제 #4
0
def test_out_of_memory():
    """
    If an allocation is run that runs out of memory, current allocations are
    written out.
    """
    script = TEST_SCRIPTS / "oom.py"
    output_dir = profile(script, expect_exit_code=53)
    time.sleep(10)  # wait for child process to finish
    allocations = get_allocations(
        output_dir,
        [
            "out-of-memory.svg",
            "out-of-memory-reversed.svg",
            "out-of-memory.prof",
        ],
        "out-of-memory.prof",
    )

    ones = (numpy.core.numeric.__file__, "ones", ANY)
    script = str(script)
    expected_small_alloc = ((script, "<module>", 9), ones)
    toobig_alloc = ((script, "<module>", 12), ones)

    assert match(allocations, {expected_small_alloc: big},
                 as_mb) == pytest.approx(100, 0.1)
    assert match(allocations, {toobig_alloc: big},
                 as_mb) == pytest.approx(1024 * 1024 * 1024, 0.1)
예제 #5
0
    def test_dog(self):
        pet = {'type': 'dog', 'details': {'age': 3}}

        self.assertEqual(match(pet, {'details': {'age': _}},    lambda age: age),
                         3)
        self.assertEqual(match(pet, {_: {'age': _}},            lambda a, b: (a, b)),
                         ('details', 3))
예제 #6
0
def test_malloc_in_c_extension():
    """
    Various malloc() and friends variants in C extension gets captured.
    """
    script = Path("python-benchmarks") / "malloc.py"
    output_dir = profile(script, "--size", "70")
    allocations = get_allocations(output_dir)

    script = str(script)

    # The realloc() in the scripts adds 10 to the 70:
    path = ((script, "<module>", 32), (script, "main", 28))
    assert match(allocations, {path: big},
                 as_mb) == pytest.approx(70 + 10, 0.1)

    # The C++ new allocation:
    path = ((script, "<module>", 32), (script, "main", 23))
    assert match(allocations, {path: big}, as_mb) == pytest.approx(40, 0.1)

    # C++ aligned_alloc():
    path = ((script, "<module>", 32), (script, "main", 24))
    assert match(allocations, {path: big}, as_mb) == pytest.approx(90, 0.1)

    # Py*_*Malloc APIs:
    path = ((script, "<module>", 32), (script, "main", 25))
    assert match(allocations, {path: big}, as_mb) == pytest.approx(30, 0.1)

    # posix_memalign():
    path = ((script, "<module>", 32), (script, "main", 26))
    assert match(allocations, {path: big}, as_mb) == pytest.approx(15, 0.1)
예제 #7
0
    def test_match_no_run(self):
        self.assertEqual(match(2, 2, lambda: 0, run_callable=False)(), 0)

        def fn():
            return "xyz"

        self.assertEqual(match(2, 2, fn, run_callable=False), fn)
예제 #8
0
def test_jupyter(tmpdir):
    """Jupyter magic can run Fil."""
    shutil.copyfile(TEST_SCRIPTS / "jupyter.ipynb", tmpdir / "jupyter.ipynb")
    check_call(
        [
            "jupyter",
            "nbconvert",
            "--execute",
            "jupyter.ipynb",
            "--to",
            "html",
        ],
        cwd=tmpdir,
    )
    output_dir = tmpdir / "fil-result"

    # IFrame with SVG was included in output:
    with open(tmpdir / "jupyter.html") as f:
        html = f.read()
    assert "<iframe" in html
    [svg_path] = re.findall(r'src="([^"]*\.svg)"', html)
    assert svg_path.endswith("peak-memory.svg")
    assert Path(tmpdir / svg_path).exists()

    # Allocations were tracked:
    allocations = get_allocations(output_dir)
    path = (
        (re.compile(".*ipy*"), "__magic_run_with_fil", 3),
        (re.compile(".*ipy.*"), "alloc", 4),
        (numpy.core.numeric.__file__, "ones", ANY),
    )
    assert match(allocations, {path: big}, as_mb) == pytest.approx(48, 0.1)
    actual_path = None
    for key in allocations:
        try:
            match(key, path, lambda x: x)
        except MatchError:
            continue
        else:
            actual_path = key
    assert actual_path != None
    assert actual_path[0][0] != actual_path[1][0]  # code is in different cells
    path2 = (
        (re.compile(".*ipy.*"), "__magic_run_with_fil", 2),
        (numpy.core.numeric.__file__, "ones", ANY),
    )
    assert match(allocations, {path2: big}, as_mb) == pytest.approx(20, 0.1)
    # It's possible to run nbconvert again.
    check_call(
        [
            "jupyter",
            "nbconvert",
            "--execute",
            "jupyter.ipynb",
            "--to",
            "html",
        ],
        cwd=tmpdir,
    )
예제 #9
0
    def verify(self, dict_var: dict) -> JsonRspType:
        if self.base is not None:
            result = pampy.match(dict_var, *self.base, default=JsonRspType.UNDEFINED)
            if result != JsonRspType.UNDEFINED:
                return result

        result = pampy.match(dict_var, *self.extend, default=JsonRspType.UNDEFINED)
        return result if result != JsonRspType.UNDEFINED else self.default
예제 #10
0
 def avg_cuteness_pampy():
     cutenesses = []
     for pet in pets:
         match(pet,
               {_: {"cuteness": _}}, lambda key, x: cutenesses.append(x),
               {_: {"cuty": _}}, lambda key, x: cutenesses.append(x)
               )
     return sum(cutenesses) / len(cutenesses)
예제 #11
0
 def call_sync(self):
     match(
         self.call.get_result(),
         ("CALL_END", _),
         self.notify_call_end,
         _,
         lambda _e: None,
     )
예제 #12
0
 def event_loop(self):
     match(
         self.event_queue.get(),
         ("add_power_dialer", _),
         self.on_add_power_dialer,
         ("delete_power_dialer", _),
         self.on_delete_power_dialer,
     )
     self.event_loop()
예제 #13
0
    def test_match_child_matches_parent_class(self):
        class Parent:
            pass

        class Child(Parent):
            pass

        self.assertEqual(match(Child(), Child, 'Child', _, 'else'), 'Child')
        self.assertEqual(match(Child(), Parent, 'Parent', _, 'else'), 'Parent')
예제 #14
0
파일: gamesim.py 프로젝트: HarryR/gamesim
def simulate(objects, system, history, stochastic=None):
    if history is None:
        history = HistoryTracker()
    assert isinstance(history, HistoryTracker)

    for S, T in system:
        # Identify which items the transforms are run on
        # The selectors / filters are used to reduce the list of items
        sub_items = [list() for _ in range(len(S))]
        for i, s_i in enumerate(S):
            for o_i in objects:
                if match(o_i, s_i, True, default=False):
                    sub_items[i].append(o_i)

        # Then iterate through the permutations of all combinations of objects
        for p in yield_permuations(sub_items):
            # When doing stochastic simulation, call a function to exclude this branch
            if stochastic and not stochastic():
                continue

            # Then iterate through all of the applicable transforms
            for T_i in T:
                matchers, transforms = T_i(*p)
                failed = False
                for i, m_i in enumerate(matchers):
                    # All matchers for the rule must match, otherwise the combination is excluded
                    if not match(p[i], m_i, True, default=False):
                        failed = True
                        break
                if failed:
                    continue

                # Then need to apply the transforms to every object
                # And create a new state where all of the objects are replaced with the new ones
                replacement = list()
                for i, t_i in enumerate(transforms):
                    o_i = deepcopy(p[i])
                    # TODO: replace with `(t_i(o_i) or o_i) if t_i else o_i` ?
                    if t_i:
                        # If a transform is specified, apply it
                        # If transform returns None, use the object
                        replacement.append(t_i(o_i) or o_i)
                    else:
                        replacement.append(o_i)

                # Create a new object list with the result of the state transform applied
                new_objects = copy(objects)
                new_history = deepcopy(history)

                for before, after in zip(p, replacement):
                    i = objects.index(before)
                    new_objects[i] = after

                cycle = new_history.track(new_objects)

                yield new_objects, new_history, cycle
예제 #15
0
    def receive(self, next_entry: IQueue.QueueElement) -> None:
        """
        Main entry point of the DataProcessor. Upon receipt of an next_entry, process it respectfully.

        :param next_entry: An entry from input_queue, could be one of the followings:
                    1. a ControlElement;
                    2. a DataElement.
        """
        match(next_entry, DataElement, self._process_data_element,
              ControlElement, self._process_control_element)
예제 #16
0
    def test_match(self):
        self.assertTrue(match(3, 3, True))
        self.assertTrue(match(3, _, True))
        self.assertTrue(match(3,
                              1, False,
                              2, False,
                              _, True))

        self.assertTrue(match([1, 2],
                              [1], False,
                              [1, 2], True))
예제 #17
0
def test_result_error():
    error = CustomException("d'oh!")
    xs: Result[str, Exception] = Error(error)

    assert isinstance(xs, Result)
    assert not xs.is_ok()
    assert xs.is_error()
    assert str(xs) == f"Error {error}"

    with pytest.raises(CustomException):
        match(xs, Ok, lambda ok: ok.value, Error,
              lambda error: throw(error.error))
예제 #18
0
 def process_control_payload(self, tag: ActorVirtualIdentity,
                             payload: ControlPayloadV2) -> None:
     """
     Process the given ControlPayload with the tag.
     :param tag: ActorVirtualIdentity, the sender.
     :param payload: ControlPayloadV2 to be handled.
     """
     # logger.debug(f"processing one CONTROL: {payload} from {tag}")
     match((tag, get_one_of(payload)), typing.Tuple[ActorVirtualIdentity,
                                                    ControlInvocationV2],
           self._async_rpc_server.receive,
           typing.Tuple[ActorVirtualIdentity,
                        ReturnInvocationV2], self._async_rpc_client.receive)
예제 #19
0
def _expand_multi_arg_param(
        name: str, param: BaseDescriptor) -> Tuple[Tuple, Tuple, Tuple]:
    """
    Expand a parameter like GeomspaceParam or ArangeParam into seperate IntParams and FloatParams to
    parse as '--start X --stop X --num X' or '--start X --stop X --step X', etc.

    :param name: The param name
    :param param: The param to expand
    """
    new_arg_names = _expand_param_name(param)
    if getattr(param, 'positional', False):
        raise ValueError(
            f'Cannot expand positional {param.__class__.__name__} to {new_arg_names}'
        )
    expanded_types = match(param, GeomspaceParam,
                           [FloatParam, FloatParam, IntParam], ArangeParam,
                           [FloatParam, FloatParam, FloatParam], LinspaceParam,
                           [FloatParam, FloatParam, IntParam])
    unit = getattr(param, 'unit', None)
    expanded_units = match(param, GeomspaceParam, lambda p: [unit, unit, None],
                           ArangeParam, lambda p: [unit, unit, unit],
                           LinspaceParam, lambda p: [unit, unit, None])
    defaults = getattr(param, 'default', [None, None, None])
    if defaults is None:
        defaults = [None, None, None]
    choices = getattr(param, 'choices', None)
    expanded_choices = [[], [], []]
    if choices is not None:
        # transpose the choices for easy splitting but preserve type
        for c in choices:
            expanded_choices[0].append(c[0])
            expanded_choices[1].append(c[1])
            expanded_choices[2].append(c[2])
    expanded_params = []
    for i, (default, argtype,
            u) in enumerate(zip(defaults, expanded_types, expanded_units)):
        new_param = argtype(
            help=param.help + ' (expanded into three arguments)',
            default=default,
            required=getattr(param, 'required', False),
            choices=expanded_choices[i] if choices is not None else None,
            unit=u)
        expanded_params.append((new_arg_names[i], new_param))
    # we add the param we're expanding to the list as well to make sure the user does not pass it
    # Otherwise, it will get parsed silently, and the user will wonder what's happening
    copied_param = copy.deepcopy(param)
    setattr(copied_param, 'required', False)
    setattr(copied_param, 'help', SUPPRESS)
    setattr(copied_param, 'default', None)
    expanded_params.append((name, copied_param))
    return tuple(expanded_params)
예제 #20
0
def test_jupyter(tmpdir):
    """Jupyter magic can run Fil."""
    shutil.copyfile(TEST_SCRIPTS / "jupyter.ipynb", tmpdir / "jupyter.ipynb")
    check_call(
        [
            "jupyter",
            "nbconvert",
            "--execute",
            "jupyter.ipynb",
            "--to",
            "html",
        ],
        cwd=tmpdir,
    )
    output_dir = tmpdir / "fil-result"

    # IFrame with SVG was included in output:
    with open(tmpdir / "jupyter.html") as f:
        html = f.read()
    assert "<iframe" in html
    [svg_path] = re.findall(r'src="([^"]*\.svg)"', html)
    assert svg_path.endswith("peak-memory.svg")
    assert Path(tmpdir / svg_path).exists()

    # Allocations were tracked:
    allocations = get_allocations(output_dir)
    print(allocations)
    path = (
        (re.compile("<ipython-input-3-.*"), "__magic_run_with_fil", 3),
        (re.compile("<ipython-input-2-.*"), "alloc", 4),
        (numpy.core.numeric.__file__, "ones", ANY),
    )
    assert match(allocations, {path: big}, as_mb) == pytest.approx(48, 0.1)
    path2 = (
        (re.compile("<ipython-input-3-.*"), "__magic_run_with_fil", 2),
        (numpy.core.numeric.__file__, "ones", ANY),
    )
    assert match(allocations, {path2: big}, as_mb) == pytest.approx(20, 0.1)
    # It's possible to run nbconvert again.
    check_call(
        [
            "jupyter",
            "nbconvert",
            "--execute",
            "jupyter.ipynb",
            "--to",
            "html",
        ],
        cwd=tmpdir,
    )
예제 #21
0
def test_result_map_error_fluent(msg: str, y: int):
    xs: Result[int, str] = Error(msg)
    mapper: Callable[[int], int] = lambda x: x + y

    ys = xs.map(mapper)
    with pytest.raises(CustomException) as ex:
        match(
            ys,
            Ok,
            lambda ok: ok.value,
            Error,
            lambda error: throw(CustomException(error.error)),
        )
    assert ex.value.message == msg
예제 #22
0
def parse_args(parse, args):
    match(
        args,
        # argument define by a name
        str,
        lambda name: parse.add_argument(name),
        # argument with a name and options
        (str, dict),
        lambda name, options: parse.add_argument(name, **options),
        # Range of arguments : recursive parsing
        Iterable,
        lambda range: dump_map(parse_args << parse, range),
        # Unknown structure : ignore with an error message
        _,
        lambda value: print(f'Error, cannot parse : {value}'))
예제 #23
0
파일: 01.py 프로젝트: DavidsIT-Site/Python3
def pattern1():
    #item 586
    print("Pattern1")
    from pampy import match, HEAD, TAIL, _

    x = [1, 2, 3]

    match(x, [1, TAIL], lambda t: t)  # => [2, 3]

    print(match(x, [HEAD, TAIL], lambda h, t: (h, t)))  # => (1, [2, 3])
    #item 587
    input = x
    pattern = [1, 2, _]
    action = lambda x: print("it's {}".format(x))
    match(input, pattern, action)
예제 #24
0
    def test_match_enum(self):
        class Color(Enum):
            RED = 1
            GREEN = 2
            BLUE = 3

        self.assertEqual(
            match(Color.RED, Color.BLUE, "blue", Color.RED, "red", _, "else"),
            "red")
        self.assertEqual(
            match(Color.RED, Color.BLUE, "blue", Color.GREEN, "green", _,
                  "else"), "else")
        self.assertEqual(
            match(1, Color.BLUE, "blue", Color.GREEN, "green", _, "else"),
            "else")
예제 #25
0
def test_out_of_memory_slow_leak_cgroups():
    """
    If an allocation is run that runs out of memory slowly, hitting a cgroup
    limit that's lower than system memory, current allocations are written out.
    """
    available_memory = psutil.virtual_memory().available
    script = TEST_SCRIPTS / "oom-slow.py"
    output_dir = profile(
        script,
        expect_exit_code=53,
        argv_prefix=get_systemd_run_args(available_memory),
    )
    time.sleep(10)  # wait for child process to finish
    allocations = get_allocations(
        output_dir,
        [
            "out-of-memory.svg",
            "out-of-memory-reversed.svg",
            "out-of-memory.prof",
        ],
        "out-of-memory.prof",
    )

    expected_alloc = ((str(script), "<module>", 3), )

    # Should've allocated at least a little before running out, unless testing
    # environment is _really_ restricted, in which case other tests would've
    # failed.
    assert match(allocations, {expected_alloc: big}, as_mb) > 100
예제 #26
0
def test_minus_m_minus_m():
    """
    `python -m filprofiler -m package` runs the package.
    """
    dir = TEST_SCRIPTS
    script = (dir / "malloc.py").absolute()
    output_dir = Path(mkdtemp())
    check_call(
        [
            sys.executable,
            "-m",
            "filprofiler",
            "-o",
            str(output_dir),
            "run",
            "-m",
            "malloc",
            "--size",
            "50",
        ],
        cwd=dir,
    )
    allocations = get_allocations(output_dir)
    stripped_allocations = {k[3:]: v for (k, v) in allocations.items()}
    script = str(script)
    path = ((script, "<module>", 32), (script, "main", 28))

    assert match(stripped_allocations, {path: big},
                 as_mb) == pytest.approx(50 + 10, 0.1)
예제 #27
0
def test_python_objects():
    """
    Python objects gets detected and tracked.

    (NumPy uses Python memory APIs, so is not sufficient to test this.)
    """
    script = TEST_SCRIPTS / "pyobject.py"
    output_dir = profile(script)
    allocations = get_allocations(output_dir)

    script = str(script)
    path = ((script, "<module>", 1), )
    path2 = ((script, "<module>", 8), (script, "<genexpr>", 8))

    assert match(allocations, {path: big}, as_mb) == pytest.approx(34, 1)
    assert match(allocations, {path2: big}, as_mb) == pytest.approx(46, 1)
예제 #28
0
 def what_is(x):
     return match(x,
                  Dog(_, 0), 'good boy',
                  Dog(_, _), 'doggy!',
                  Cat(_, 0), 'tommy?',
                  Cat(_, _), 'a cat'
                  )
예제 #29
0
 def f(x):
     return match(x,
                  Point(1, 2), '1',
                  Point(_, 2), str,
                  Point(1, _), str,
                  Point(_, _), lambda a, b: str(a + b)
                  )
예제 #30
0
 def lisp(exp):
     return match(exp,
                  int, lambda x: x,
                  callable, lambda x: x,
                  (callable, REST), lambda f, rest: f(*map(lisp, rest)),
                  tuple, lambda t: list(map(lisp, t)),
                  )
예제 #31
0
파일: pampy.py 프로젝트: Aurametrix/Alg
from pampy import match, _

x = [1, 2, 3]

match(x, [1, TAIL],     lambda t: t)            # => [2, 3]

match(x, [HEAD, TAIL],  lambda h, t: (h, t))    # => (1, [2, 3])



match(x,
    3,              "this matches the number 3",

    int,            "matches any integer",

    (str, int),     lambda a, b: "a tuple (a, b) you can use in a function",

    [1, 2, _],      "any list of 3 elements that begins with [1, 2]",

    {'x': _},       "any dict with a key 'x' and any value associated",

    _,              "anything else"
)