def _add_param_to_parser(name: str, param: BaseDescriptor,
parser: ArgumentParser) -> None:
"""
Function to add a Param like IntParam, FloatParam, etc. called <name> to a parser
:param name: The param name
:param param: The param to add (IntParam, FloatParam, etc.)
:param parser: The argument parser to add the param to.
"""
argtype = _get_param_type(param)
if argtype == type(None):
raise NotImplementedError(
f'Argparse type not implemented '
f'for {param.__class__.__name__} and default not specifed')
positional = getattr(param, 'positional', False)
if (getattr(param, 'prefix', '') != ''
and not getattr(param, 'expand', False)):
raise ValueError(
f'Failure with param {name}. Cannot add a prefix to a class without the'
f' expand kwarg set to True')
argname = name if positional else '--' + name
required = True if getattr(param, 'required', False) else None
default = param.default if required is None else None
unit = getattr(param, 'unit', None)
# format help nicely if default is specified and suppress is not set
if positional or param.help == SUPPRESS:
help = param.help
else:
help = f'{param.help} [default: {default} {unit}]' \
if unit is not None else f'{param.help} [default: {default}]'
if not required and positional:
# TODO: use nargs='*' or nargs='?' to support not-required positional arguments
raise ValueError(
'Not-required positional arguments are currently not supported')
elif positional:
# positional arguments are required by default, and argparse complains if you specify
# required = True
required = None
default = None
action = match(param, BoolParam, lambda p: 'store_true'
if not p.default else 'store_false', BaseDescriptor,
lambda p: None)
nargs = getattr(param, 'nargs', None)
assert not (action is not None and nargs is not None)
choices = match(param, EnumParam, lambda x: list(x.cls.__members__.keys()),
BaseDescriptor, lambda x: getattr(x, 'choices', None))
kwargs = dict(action=action,
nargs=nargs,
default=default,
type=argtype,
required=required,
help=help,
choices=choices)
# we delete all kwargs that are None to avoid hitting annoying action class initializations
# such as when action is store_true and 'nargs' is in kwargs
for kw in list(kwargs.keys()):
if kwargs[kw] is None:
del kwargs[kw]
parser.add_argument(argname, **kwargs)
def test_sigusr2():
"""
Sending SIGUSR2 to the process does an extra dump.
"""
script = TEST_SCRIPTS / "sigusr2.py"
output_dir = profile(script)
# There are two dumps in the output directory, one for SIGUSR2, one for
# shutdown.
assert len(list(output_dir.iterdir())) == 2
sigusr2, final = sorted(output_dir.glob("*/peak-memory.prof"))
# SIGUSR2 dump only has allocations up to that point
script = str(script)
path1 = ((script, "<module>", 8), (numpy.core.numeric.__file__, "ones",
ANY))
path2 = ((script, "<module>", 11), (numpy.core.numeric.__file__, "ones",
ANY))
allocations_sigusr2 = get_allocations(sigusr2, direct=True)
assert match(allocations_sigusr2, {path1: big},
as_mb) == pytest.approx(20, 0.1)
with pytest.raises(MatchError):
match(allocations_sigusr2, {path2: big}, as_mb)
allocations_final = get_allocations(final, direct=True)
assert match(allocations_final, {path1: big},
as_mb) == pytest.approx(20, 0.1)
assert match(allocations_final, {path2: big},
as_mb) == pytest.approx(50, 0.1)
def test_malloc_in_c_extension():
"""
Various malloc() and friends variants in C extension gets captured.
"""
script = TEST_SCRIPTS / "malloc.py"
output_dir = profile(script, "--size", "70")
allocations = get_allocations(output_dir)
script = str(script)
# The realloc() in the scripts adds 10 to the 70:
path = ((script, "<module>", 32), (script, "main", 28))
assert match(allocations, {path: big},
as_mb) == pytest.approx(70 + 10, 0.1)
# The C++ new allocation:
path = ((script, "<module>", 32), (script, "main", 23))
assert match(allocations, {path: big}, as_mb) == pytest.approx(40, 0.1)
# C++ aligned_alloc(); not available on Conda, where it's just a macro
# redirecting to posix_memalign.
if not os.environ.get("CONDA_PREFIX"):
path = ((script, "<module>", 32), (script, "main", 24))
assert match(allocations, {path: big}, as_mb) == pytest.approx(90, 0.1)
# Py*_*Malloc APIs:
path = ((script, "<module>", 32), (script, "main", 25))
assert match(allocations, {path: big}, as_mb) == pytest.approx(30, 0.1)
# posix_memalign():
path = ((script, "<module>", 32), (script, "main", 26))
assert match(allocations, {path: big}, as_mb) == pytest.approx(15, 0.1)
def test_out_of_memory():
"""
If an allocation is run that runs out of memory, current allocations are
written out.
"""
script = TEST_SCRIPTS / "oom.py"
output_dir = profile(script, expect_exit_code=53)
time.sleep(10) # wait for child process to finish
allocations = get_allocations(
output_dir,
[
"out-of-memory.svg",
"out-of-memory-reversed.svg",
"out-of-memory.prof",
],
"out-of-memory.prof",
)
ones = (numpy.core.numeric.__file__, "ones", ANY)
script = str(script)
expected_small_alloc = ((script, "<module>", 9), ones)
toobig_alloc = ((script, "<module>", 12), ones)
assert match(allocations, {expected_small_alloc: big},
as_mb) == pytest.approx(100, 0.1)
assert match(allocations, {toobig_alloc: big},
as_mb) == pytest.approx(1024 * 1024 * 1024, 0.1)
def test_dog(self):
pet = {'type': 'dog', 'details': {'age': 3}}
self.assertEqual(match(pet, {'details': {'age': _}}, lambda age: age),
3)
self.assertEqual(match(pet, {_: {'age': _}}, lambda a, b: (a, b)),
('details', 3))
def test_malloc_in_c_extension():
"""
Various malloc() and friends variants in C extension gets captured.
"""
script = Path("python-benchmarks") / "malloc.py"
output_dir = profile(script, "--size", "70")
allocations = get_allocations(output_dir)
script = str(script)
# The realloc() in the scripts adds 10 to the 70:
path = ((script, "<module>", 32), (script, "main", 28))
assert match(allocations, {path: big},
as_mb) == pytest.approx(70 + 10, 0.1)
# The C++ new allocation:
path = ((script, "<module>", 32), (script, "main", 23))
assert match(allocations, {path: big}, as_mb) == pytest.approx(40, 0.1)
# C++ aligned_alloc():
path = ((script, "<module>", 32), (script, "main", 24))
assert match(allocations, {path: big}, as_mb) == pytest.approx(90, 0.1)
# Py*_*Malloc APIs:
path = ((script, "<module>", 32), (script, "main", 25))
assert match(allocations, {path: big}, as_mb) == pytest.approx(30, 0.1)
# posix_memalign():
path = ((script, "<module>", 32), (script, "main", 26))
assert match(allocations, {path: big}, as_mb) == pytest.approx(15, 0.1)
def test_match_no_run(self):
self.assertEqual(match(2, 2, lambda: 0, run_callable=False)(), 0)
def fn():
return "xyz"
self.assertEqual(match(2, 2, fn, run_callable=False), fn)
def test_jupyter(tmpdir):
"""Jupyter magic can run Fil."""
shutil.copyfile(TEST_SCRIPTS / "jupyter.ipynb", tmpdir / "jupyter.ipynb")
check_call(
[
"jupyter",
"nbconvert",
"--execute",
"jupyter.ipynb",
"--to",
"html",
],
cwd=tmpdir,
)
output_dir = tmpdir / "fil-result"
# IFrame with SVG was included in output:
with open(tmpdir / "jupyter.html") as f:
html = f.read()
assert "<iframe" in html
[svg_path] = re.findall(r'src="([^"]*\.svg)"', html)
assert svg_path.endswith("peak-memory.svg")
assert Path(tmpdir / svg_path).exists()
# Allocations were tracked:
allocations = get_allocations(output_dir)
path = (
(re.compile(".*ipy*"), "__magic_run_with_fil", 3),
(re.compile(".*ipy.*"), "alloc", 4),
(numpy.core.numeric.__file__, "ones", ANY),
)
assert match(allocations, {path: big}, as_mb) == pytest.approx(48, 0.1)
actual_path = None
for key in allocations:
try:
match(key, path, lambda x: x)
except MatchError:
continue
else:
actual_path = key
assert actual_path != None
assert actual_path[0][0] != actual_path[1][0] # code is in different cells
path2 = (
(re.compile(".*ipy.*"), "__magic_run_with_fil", 2),
(numpy.core.numeric.__file__, "ones", ANY),
)
assert match(allocations, {path2: big}, as_mb) == pytest.approx(20, 0.1)
# It's possible to run nbconvert again.
check_call(
[
"jupyter",
"nbconvert",
"--execute",
"jupyter.ipynb",
"--to",
"html",
],
cwd=tmpdir,
)
def verify(self, dict_var: dict) -> JsonRspType:
if self.base is not None:
result = pampy.match(dict_var, *self.base, default=JsonRspType.UNDEFINED)
if result != JsonRspType.UNDEFINED:
return result
result = pampy.match(dict_var, *self.extend, default=JsonRspType.UNDEFINED)
return result if result != JsonRspType.UNDEFINED else self.default
def avg_cuteness_pampy():
cutenesses = []
for pet in pets:
match(pet,
{_: {"cuteness": _}}, lambda key, x: cutenesses.append(x),
{_: {"cuty": _}}, lambda key, x: cutenesses.append(x)
)
return sum(cutenesses) / len(cutenesses)
def call_sync(self):
match(
self.call.get_result(),
("CALL_END", _),
self.notify_call_end,
_,
lambda _e: None,
)
def event_loop(self):
match(
self.event_queue.get(),
("add_power_dialer", _),
self.on_add_power_dialer,
("delete_power_dialer", _),
self.on_delete_power_dialer,
)
self.event_loop()
def test_match_child_matches_parent_class(self):
class Parent:
pass
class Child(Parent):
pass
self.assertEqual(match(Child(), Child, 'Child', _, 'else'), 'Child')
self.assertEqual(match(Child(), Parent, 'Parent', _, 'else'), 'Parent')
def simulate(objects, system, history, stochastic=None):
if history is None:
history = HistoryTracker()
assert isinstance(history, HistoryTracker)
for S, T in system:
# Identify which items the transforms are run on
# The selectors / filters are used to reduce the list of items
sub_items = [list() for _ in range(len(S))]
for i, s_i in enumerate(S):
for o_i in objects:
if match(o_i, s_i, True, default=False):
sub_items[i].append(o_i)
# Then iterate through the permutations of all combinations of objects
for p in yield_permuations(sub_items):
# When doing stochastic simulation, call a function to exclude this branch
if stochastic and not stochastic():
continue
# Then iterate through all of the applicable transforms
for T_i in T:
matchers, transforms = T_i(*p)
failed = False
for i, m_i in enumerate(matchers):
# All matchers for the rule must match, otherwise the combination is excluded
if not match(p[i], m_i, True, default=False):
failed = True
break
if failed:
continue
# Then need to apply the transforms to every object
# And create a new state where all of the objects are replaced with the new ones
replacement = list()
for i, t_i in enumerate(transforms):
o_i = deepcopy(p[i])
# TODO: replace with `(t_i(o_i) or o_i) if t_i else o_i` ?
if t_i:
# If a transform is specified, apply it
# If transform returns None, use the object
replacement.append(t_i(o_i) or o_i)
else:
replacement.append(o_i)
# Create a new object list with the result of the state transform applied
new_objects = copy(objects)
new_history = deepcopy(history)
for before, after in zip(p, replacement):
i = objects.index(before)
new_objects[i] = after
cycle = new_history.track(new_objects)
yield new_objects, new_history, cycle
def receive(self, next_entry: IQueue.QueueElement) -> None:
"""
Main entry point of the DataProcessor. Upon receipt of an next_entry, process it respectfully.
:param next_entry: An entry from input_queue, could be one of the followings:
1. a ControlElement;
2. a DataElement.
"""
match(next_entry, DataElement, self._process_data_element,
ControlElement, self._process_control_element)
def test_match(self):
self.assertTrue(match(3, 3, True))
self.assertTrue(match(3, _, True))
self.assertTrue(match(3,
1, False,
2, False,
_, True))
self.assertTrue(match([1, 2],
[1], False,
[1, 2], True))
def test_result_error():
error = CustomException("d'oh!")
xs: Result[str, Exception] = Error(error)
assert isinstance(xs, Result)
assert not xs.is_ok()
assert xs.is_error()
assert str(xs) == f"Error {error}"
with pytest.raises(CustomException):
match(xs, Ok, lambda ok: ok.value, Error,
lambda error: throw(error.error))
def process_control_payload(self, tag: ActorVirtualIdentity,
payload: ControlPayloadV2) -> None:
"""
Process the given ControlPayload with the tag.
:param tag: ActorVirtualIdentity, the sender.
:param payload: ControlPayloadV2 to be handled.
"""
# logger.debug(f"processing one CONTROL: {payload} from {tag}")
match((tag, get_one_of(payload)), typing.Tuple[ActorVirtualIdentity,
ControlInvocationV2],
self._async_rpc_server.receive,
typing.Tuple[ActorVirtualIdentity,
ReturnInvocationV2], self._async_rpc_client.receive)
def _expand_multi_arg_param(
name: str, param: BaseDescriptor) -> Tuple[Tuple, Tuple, Tuple]:
"""
Expand a parameter like GeomspaceParam or ArangeParam into seperate IntParams and FloatParams to
parse as '--start X --stop X --num X' or '--start X --stop X --step X', etc.
:param name: The param name
:param param: The param to expand
"""
new_arg_names = _expand_param_name(param)
if getattr(param, 'positional', False):
raise ValueError(
f'Cannot expand positional {param.__class__.__name__} to {new_arg_names}'
)
expanded_types = match(param, GeomspaceParam,
[FloatParam, FloatParam, IntParam], ArangeParam,
[FloatParam, FloatParam, FloatParam], LinspaceParam,
[FloatParam, FloatParam, IntParam])
unit = getattr(param, 'unit', None)
expanded_units = match(param, GeomspaceParam, lambda p: [unit, unit, None],
ArangeParam, lambda p: [unit, unit, unit],
LinspaceParam, lambda p: [unit, unit, None])
defaults = getattr(param, 'default', [None, None, None])
if defaults is None:
defaults = [None, None, None]
choices = getattr(param, 'choices', None)
expanded_choices = [[], [], []]
if choices is not None:
# transpose the choices for easy splitting but preserve type
for c in choices:
expanded_choices[0].append(c[0])
expanded_choices[1].append(c[1])
expanded_choices[2].append(c[2])
expanded_params = []
for i, (default, argtype,
u) in enumerate(zip(defaults, expanded_types, expanded_units)):
new_param = argtype(
help=param.help + ' (expanded into three arguments)',
default=default,
required=getattr(param, 'required', False),
choices=expanded_choices[i] if choices is not None else None,
unit=u)
expanded_params.append((new_arg_names[i], new_param))
# we add the param we're expanding to the list as well to make sure the user does not pass it
# Otherwise, it will get parsed silently, and the user will wonder what's happening
copied_param = copy.deepcopy(param)
setattr(copied_param, 'required', False)
setattr(copied_param, 'help', SUPPRESS)
setattr(copied_param, 'default', None)
expanded_params.append((name, copied_param))
return tuple(expanded_params)
def test_jupyter(tmpdir):
"""Jupyter magic can run Fil."""
shutil.copyfile(TEST_SCRIPTS / "jupyter.ipynb", tmpdir / "jupyter.ipynb")
check_call(
[
"jupyter",
"nbconvert",
"--execute",
"jupyter.ipynb",
"--to",
"html",
],
cwd=tmpdir,
)
output_dir = tmpdir / "fil-result"
# IFrame with SVG was included in output:
with open(tmpdir / "jupyter.html") as f:
html = f.read()
assert "<iframe" in html
[svg_path] = re.findall(r'src="([^"]*\.svg)"', html)
assert svg_path.endswith("peak-memory.svg")
assert Path(tmpdir / svg_path).exists()
# Allocations were tracked:
allocations = get_allocations(output_dir)
print(allocations)
path = (
(re.compile("<ipython-input-3-.*"), "__magic_run_with_fil", 3),
(re.compile("<ipython-input-2-.*"), "alloc", 4),
(numpy.core.numeric.__file__, "ones", ANY),
)
assert match(allocations, {path: big}, as_mb) == pytest.approx(48, 0.1)
path2 = (
(re.compile("<ipython-input-3-.*"), "__magic_run_with_fil", 2),
(numpy.core.numeric.__file__, "ones", ANY),
)
assert match(allocations, {path2: big}, as_mb) == pytest.approx(20, 0.1)
# It's possible to run nbconvert again.
check_call(
[
"jupyter",
"nbconvert",
"--execute",
"jupyter.ipynb",
"--to",
"html",
],
cwd=tmpdir,
)
def test_result_map_error_fluent(msg: str, y: int):
xs: Result[int, str] = Error(msg)
mapper: Callable[[int], int] = lambda x: x + y
ys = xs.map(mapper)
with pytest.raises(CustomException) as ex:
match(
ys,
Ok,
lambda ok: ok.value,
Error,
lambda error: throw(CustomException(error.error)),
)
assert ex.value.message == msg
def parse_args(parse, args):
match(
args,
# argument define by a name
str,
lambda name: parse.add_argument(name),
# argument with a name and options
(str, dict),
lambda name, options: parse.add_argument(name, **options),
# Range of arguments : recursive parsing
Iterable,
lambda range: dump_map(parse_args << parse, range),
# Unknown structure : ignore with an error message
_,
lambda value: print(f'Error, cannot parse : {value}'))
def pattern1():
#item 586
print("Pattern1")
from pampy import match, HEAD, TAIL, _
x = [1, 2, 3]
match(x, [1, TAIL], lambda t: t) # => [2, 3]
print(match(x, [HEAD, TAIL], lambda h, t: (h, t))) # => (1, [2, 3])
#item 587
input = x
pattern = [1, 2, _]
action = lambda x: print("it's {}".format(x))
match(input, pattern, action)
def test_match_enum(self):
class Color(Enum):
RED = 1
GREEN = 2
BLUE = 3
self.assertEqual(
match(Color.RED, Color.BLUE, "blue", Color.RED, "red", _, "else"),
"red")
self.assertEqual(
match(Color.RED, Color.BLUE, "blue", Color.GREEN, "green", _,
"else"), "else")
self.assertEqual(
match(1, Color.BLUE, "blue", Color.GREEN, "green", _, "else"),
"else")
def test_out_of_memory_slow_leak_cgroups():
"""
If an allocation is run that runs out of memory slowly, hitting a cgroup
limit that's lower than system memory, current allocations are written out.
"""
available_memory = psutil.virtual_memory().available
script = TEST_SCRIPTS / "oom-slow.py"
output_dir = profile(
script,
expect_exit_code=53,
argv_prefix=get_systemd_run_args(available_memory),
)
time.sleep(10) # wait for child process to finish
allocations = get_allocations(
output_dir,
[
"out-of-memory.svg",
"out-of-memory-reversed.svg",
"out-of-memory.prof",
],
"out-of-memory.prof",
)
expected_alloc = ((str(script), "<module>", 3), )
# Should've allocated at least a little before running out, unless testing
# environment is _really_ restricted, in which case other tests would've
# failed.
assert match(allocations, {expected_alloc: big}, as_mb) > 100
def test_minus_m_minus_m():
"""
`python -m filprofiler -m package` runs the package.
"""
dir = TEST_SCRIPTS
script = (dir / "malloc.py").absolute()
output_dir = Path(mkdtemp())
check_call(
[
sys.executable,
"-m",
"filprofiler",
"-o",
str(output_dir),
"run",
"-m",
"malloc",
"--size",
"50",
],
cwd=dir,
)
allocations = get_allocations(output_dir)
stripped_allocations = {k[3:]: v for (k, v) in allocations.items()}
script = str(script)
path = ((script, "<module>", 32), (script, "main", 28))
assert match(stripped_allocations, {path: big},
as_mb) == pytest.approx(50 + 10, 0.1)
def test_python_objects():
"""
Python objects gets detected and tracked.
(NumPy uses Python memory APIs, so is not sufficient to test this.)
"""
script = TEST_SCRIPTS / "pyobject.py"
output_dir = profile(script)
allocations = get_allocations(output_dir)
script = str(script)
path = ((script, "<module>", 1), )
path2 = ((script, "<module>", 8), (script, "<genexpr>", 8))
assert match(allocations, {path: big}, as_mb) == pytest.approx(34, 1)
assert match(allocations, {path2: big}, as_mb) == pytest.approx(46, 1)
def what_is(x):
return match(x,
Dog(_, 0), 'good boy',
Dog(_, _), 'doggy!',
Cat(_, 0), 'tommy?',
Cat(_, _), 'a cat'
)
def f(x):
return match(x,
Point(1, 2), '1',
Point(_, 2), str,
Point(1, _), str,
Point(_, _), lambda a, b: str(a + b)
)
def lisp(exp):
return match(exp,
int, lambda x: x,
callable, lambda x: x,
(callable, REST), lambda f, rest: f(*map(lisp, rest)),
tuple, lambda t: list(map(lisp, t)),
)
from pampy import match, _
x = [1, 2, 3]
match(x, [1, TAIL], lambda t: t) # => [2, 3]
match(x, [HEAD, TAIL], lambda h, t: (h, t)) # => (1, [2, 3])
match(x,
3, "this matches the number 3",
int, "matches any integer",
(str, int), lambda a, b: "a tuple (a, b) you can use in a function",
[1, 2, _], "any list of 3 elements that begins with [1, 2]",
{'x': _}, "any dict with a key 'x' and any value associated",
_, "anything else"
)