def grep(item, path, debug, **kwargs):
"""
Deep Grep Commandline
Grep through the contents of a file and find the path to the item.
It can read csv, tsv, json, yaml, and toml files.
"""
kwargs['case_sensitive'] = not kwargs.pop('ignore_case')
kwargs['match_string'] = kwargs.pop('exact_match')
try:
content = load_path_content(path)
except Exception as e: # pragma: no cover.
if debug: # pragma: no cover.
raise # pragma: no cover.
else: # pragma: no cover.
sys.exit(
str(f"Error when loading {path}: {e}")) # pragma: no cover.
try:
result = DeepSearch(content, item, **kwargs)
except Exception as e: # pragma: no cover.
if debug: # pragma: no cover.
raise # pragma: no cover.
else: # pragma: no cover.
sys.exit(str(f"Error when running deep search on {path}: {e}")
) # pragma: no cover.
pprint(result, indent=2)
def _validation_context(self,
instance: JSON,
ignore_in_doc_refs: bool = False):
"""
A context manager for building up and tearing down configuration for
a running our custom validator on a given instance.
"""
self._ignore_in_doc_refs = ignore_in_doc_refs
self._in_doc_refs_cache = dict()
# Build the in_doc_refs_cache if we're not ignoring in_doc_refs
if not ignore_in_doc_refs:
search = DeepSearch(self.schema, "in_doc_ref_pattern")
if "matched_paths" in search:
for path in search["matched_paths"]:
scope = {"root": self.schema}
exec(f"ref_path_pattern = {path}", scope)
ref_path_pattern = scope["ref_path_pattern"]
# If there are no cached values for this ref path pattern, collect them
if ref_path_pattern not in self._in_doc_refs_cache:
self._in_doc_refs_cache[
ref_path_pattern] = self._get_values_for_path_pattern(
ref_path_pattern, instance)
# see: https://docs.python.org/3/library/contextlib.html
try:
yield
finally:
self._in_doc_refs_cache = None
def test_string_in_dictionary_key_case_insensitive_partial(self):
obj = {"SOMEWHERE here": "around"}
result = {
'matched_paths': {"root['SOMEWHERE here']"}
}
ds = DeepSearch(obj, item, verbose_level=1, case_sensitive=False)
self.assertEqual(ds, result)
def test_loop_in_lists(self):
obj = [1, 2, 'somewhere']
obj.append(obj)
ds = DeepSearch(obj, item, verbose_level=1)
result = {'matched_values': {'root[2]'}}
assert ds == result
def test_skip_path1(self):
obj = {
"for life": "vegan",
"ingredients": ["no meat", "no eggs", "no dairy", "somewhere"]
}
ds = DeepSearch(obj, item, exclude_paths={"root['ingredients']"})
assert ds == {}
def _get_uuid_path_map(ct: dict) -> dict:
"""
Build a dictionary mapping upload placeholder UUIDs to a `deepdiff`-style
path to that artifact in the `ct` clinical trial metadata dictionary. This
will look something like:
```python
{
"uuuu-uuuu-iiii-dddd": "root['path']['to']['upload_placeholder']",
...
}
```
"""
def get_uuid_for_path(path: str):
# Spooky stuff: `exec` executes the string provided as its first argument as
# python code in a context populated with the global variables defined by
# the dictionary passed as its second argument. Since `path` looks like
# "root['path']['to']['upload_placeholder']", the following code looks up
# the UUID associated with `path` in the `ct` dict, whose value is assigned
# to "root" in the scope provided to `exec`, and stores that UUID in `scope['uuid']`.
scope = {"root": ct}
exec(f"uuid = {path}", scope)
return scope["uuid"]
return {
get_uuid_for_path(path): path
for path in DeepSearch(ct, "upload_placeholder")["matched_paths"]
}
def test_string_in_dictionary_case_insensitive(self):
obj = {"long": "Somewhere over there!", "string": 2, 0: 0, "SOMEWHERE": "around"}
result = {
'matched_paths': {"root['SOMEWHERE']"},
'matched_values': {"root['long']"}
}
ds = DeepSearch(obj, item, verbose_level=1, case_sensitive=False)
self.assertEqual(ds, result)
def test_bad_attribute(self):
class Bad:
__slots__ = ['x', 'y']
def __getattr__(self, key):
raise AttributeError("Bad item")
def __str__(self):
return "Bad Object"
obj = Bad()
ds = DeepSearch(obj, item, verbose_level=1)
result = {'unprocessed': ['root']}
assert ds == result
ds = DeepSearch(obj, item, verbose_level=2)
assert ds == result
def test_string_in_dictionary(self):
obj = {"long": "somewhere", "string": 2, 0: 0, "somewhere": "around"}
result = {
'matched_paths': {"root['somewhere']"},
'matched_values': {"root['long']"}
}
ds = DeepSearch(obj, item, verbose_level=1)
assert ds == result
def test_named_tuples_verbose(self):
from collections import namedtuple
Point = namedtuple('Point', ['x', 'somewhere_good'])
obj = Point(x="my keys are somewhere", somewhere_good=22)
ds = DeepSearch(obj, item, verbose_level=2)
result = {'matched_values': {'root.x': 'my keys are somewhere'},
'matched_paths': {'root.somewhere_good': 22}}
self.assertEqual(ds, result)
def test_regex_in_string_in_dictionary(self):
obj = {"long": "somewhere", "string": 2, 0: 0, "somewhere": "around"}
result = {
"matched_paths": {"root['somewhere']"},
"matched_values": {"root['long']"},
}
item = "some.*"
ds = DeepSearch(obj, item, verbose_level=1, use_regexp=True)
assert ds == result
def test_string_in_list_verbose3(self):
obj = ["long somewhere", "string", 0, "somewhere great!"]
result = {
"matched_values": {
'root[0]': 'long somewhere',
'root[3]': "somewhere great!"
}
}
assert DeepSearch(obj, item, verbose_level=2) == result
def test_regex_in_int_in_dictionary(self):
obj = {"long": "somewhere", "num": 232, 0: 0, "somewhere": "around"}
item = "2.*"
result = {"matched_values": {"root['num']"}}
ds = DeepSearch(obj,
item,
verbose_level=1,
use_regexp=True,
strict_checking=False)
assert ds == result
def test_search_inherited_attributes(self):
class Parent:
a = 1
class Child(Parent):
b = 2
obj = Child()
item = 1
result = {'matched_values': {'root.a'}}
assert DeepSearch(obj, item, verbose_level=1) == result
def test_loop(self):
class LoopTest:
def __init__(self, a):
self.loop = self
self.a = a
obj = LoopTest("somewhere around here.")
ds = DeepSearch(obj, item, verbose_level=1)
result = {'matched_values': {'root.a'}}
assert ds == result
def test_keep_searching_after_obj_match(self):
class AlwaysEqual:
def __init__(self, recurse=True):
if recurse:
self.some_attr = AlwaysEqual(recurse=False)
def __eq__(self, other):
return True
obj = AlwaysEqual()
item = AlwaysEqual()
result = {'matched_values': {'root', 'root.some_attr'}}
assert DeepSearch(obj, item, verbose_level=1) == result
def test_regex_in_named_tuples_verbose(self):
from collections import namedtuple
Point = namedtuple("Point", ["x", "somewhere_good"])
obj = Point(x="my keys are somewhere", somewhere_good=22)
item = "some.*"
ds = DeepSearch(obj, item, verbose_level=2, use_regexp=True)
result = {
"matched_values": {
"root.x": "my keys are somewhere"
},
"matched_paths": {
"root.somewhere_good": 22
},
}
assert ds == result
def test_int_cant_become_regex(self):
obj = {
"long": "somewhere",
"num": "1123456",
0: 0,
"somewhere": "around"
}
item = CustomClass(a=10)
with pytest.raises(TypeError) as exp:
DeepSearch(obj,
item,
verbose_level=1,
use_regexp=True,
strict_checking=False)
assert str(exp.value).startswith(
"The passed item of (10, None) is not usable for regex")
def search_in(obj, item, **kwargs):
if isinstance(obj, dict):
obj = obj
else:
try:
obj = json.loads(obj)
except (JSONDecodeError, TypeError) as error:
raise JsonCompareError(f"Only VALID JSON strings accepted! ERROR: {error}")
logger.debug(f"OBJ: {obj}")
logger.debug(f"ITEM: {item}")
logger.debug(f"KWARGS: {kwargs}")
ds = DeepSearch(obj, item, verbose_level=2, **kwargs)
logger.debug(f"DS: {type(ds)}")
print(ds)
return ds
def test_string_in_dictionary_in_list_verbose(self):
obj = [
"something somewhere", {
"long": "somewhere",
"string": 2,
0: 0,
"somewhere": "around"
}
]
result = {
'matched_paths': {
"root[1]['somewhere']": "around"
},
'matched_values': {
"root[1]['long']": "somewhere",
"root[0]": "something somewhere"
}
}
ds = DeepSearch(obj, item, verbose_level=2)
assert ds == result
def test_none(self):
obj = item = None
result = {'matched_values': {'root'}}
assert DeepSearch(obj, item, verbose_level=1) == result
def test_number_in_list(self):
obj = ["a", 10, 20]
item = 10
result = {"matched_values": {'root[1]'}}
assert DeepSearch(obj, item, verbose_level=1) == result
def test_int_in_dictionary(self):
obj = {"long": "somewhere", "num": 2, 0: 0, "somewhere": "around"}
item = 2
result = {'matched_values': {"root['num']"}}
ds = DeepSearch(obj, item, verbose_level=1)
assert ds == result
def test_unknown_parameters(self):
with pytest.raises(ValueError):
DeepSearch(1, 1, wrong_param=2)
def test_string_in_list(self):
obj = ["long", "string", 0, "somewhere"]
result = {"matched_values": {'root[3]'}}
assert DeepSearch(obj, item, verbose_level=1) == result
def test_string_in_root_verbose(self):
obj = "long string somewhere"
result = {"matched_values": {'root': "long string somewhere"}}
assert DeepSearch(obj, item, verbose_level=2) == result
def test_case_sensitive_of_str_in_list(self):
obj = ["a", "bb", "BBC", "aBbB"]
item = "BB"
result = {"matched_values": {'root[2]'}}
assert DeepSearch(obj, item, verbose_level=1,
case_sensitive=True) == result
def test_case_sensitive_of_str_in_one_liner(self):
obj = "Hello, what's up?"
item = "WHAT"
result = {}
assert DeepSearch(obj, item, verbose_level=1,
case_sensitive=True) == result
def test_complex_obj(self):
obj = datetime(2017, 5, 4, 1, 1, 1)
item = datetime(2017, 5, 4, 1, 1, 1)
result = {'matched_values': {'root'}}
assert DeepSearch(obj, item, verbose_level=1) == result
def test_case_insensitive_of_str_in_one_liner(self):
obj = "Hello, what's up?"
item = "WHAT"
result = {'matched_values': {'root'}}
assert DeepSearch(obj, item, verbose_level=1,
case_sensitive=False) == result
