def test_remap_set(self):
# explicit test for sets to make sure #84 is covered
s = set([1, 2, 3])
assert remap(s) == s
fs = frozenset([1, 2, 3])
assert remap(fs) == fs
def test_reraise_visit(self):
root = {'A': 'b', 1: 2}
key_to_lower = lambda p, k, v: (k.lower(), v)
with pytest.raises(AttributeError):
remap(root, key_to_lower)
remapped = remap(root, key_to_lower, reraise_visit=False)
assert remapped['a'] == 'b'
assert remapped[1] == 2
def test_drop_nones(self):
orig = {'a': 1, 'b': None, 'c': [3, None, 4, None]}
ref = {'a': 1, 'c': [3, 4]}
drop_none = lambda p, k, v: v is not None
remapped = remap(orig, visit=drop_none)
assert remapped == ref
orig = [None] * 100
remapped = remap(orig, drop_none)
assert not remapped
def test_root_selfref(self):
selfref = [0, 1, 2, 3]
selfref.append(selfref)
with pytest.raises(RuntimeError):
assert selfref == remap(selfref)
selfref2 = {}
selfref2['self'] = selfref2
with pytest.raises(RuntimeError):
assert selfref2 == remap(selfref2)
def test_root_selfref(self):
selfref = [0, 1, 2, 3]
selfref.append(selfref)
with pytest.raises(RuntimeError):
assert selfref == remap(selfref)
selfref2 = {}
selfref2['self'] = selfref2
with pytest.raises(RuntimeError):
assert selfref2 == remap(selfref2)
def test_drop_nones(self):
orig = {'a': 1, 'b': None, 'c': [3, None, 4, None]}
ref = {'a': 1, 'c': [3, 4]}
drop_none = lambda p, k, v: v is not None
remapped = remap(orig, visit=drop_none)
assert remapped == ref
orig = [None] * 100
remapped = remap(orig, drop_none)
assert not remapped
def post_save(cls, sender, document, **kwargs):
if kwargs.get("skip"):
return
# project is LazyReferenceField
project = document.project.fetch()
# set columns field for project
def update_columns(path, key, value):
path = delimiter.join(["data"] + list(path) + [key])
is_quantity = isinstance(value, dict) and quantity_keys.issubset(
value.keys()
)
is_text = bool(
not is_quantity and isinstance(value, str) and key not in quantity_keys
)
if is_quantity or is_text:
project.reload("columns")
try:
column = project.columns.get(path=path)
if is_quantity:
v = value["value"]
if isnan(column.max) or v > column.max:
column.max = v
if isnan(column.min) or v < column.min:
column.min = v
except DoesNotExist:
column = {"path": path}
if is_quantity:
column["unit"] = value["unit"]
column["min"] = column["max"] = value["value"]
project.columns.create(**column)
project.save().reload("columns")
ncolumns = len(project.columns)
if ncolumns > 50:
raise ValueError("Reached maximum number of columns (50)!")
return True
# run update_columns over document data
remap(document.data, visit=update_columns, enter=enter)
# add/remove columns for other components
for path in COMPONENTS.keys():
try:
project.columns.get(path=path)
except DoesNotExist:
if getattr(document, path):
project.columns.create(path=path)
project.save().reload("columns")
document.last_modified = datetime.utcnow()
def test_filter_map_block(self):
for config in self.nginx_conf_parse["config"]:
self.assertEqual(config["status"], "ok")
nginx_conf_parse = next(
config
for config in self.nginx_conf_parse["config"]
if path.basename(config["file"]) == "nginx.conf"
)
no_line = partial(del_keys_d, key="line")
self.assertDictEqual(
no_line(
get_dict_by_key_val(
deepcopy(nginx_conf_parse["parsed"]), "directive", "server_name"
)
),
{"directive": "server_name", "args": ["localhost"]},
)
self.assertDictEqual(
no_line(
get_dict_by_key_val(
deepcopy(nginx_conf_parse["parsed"]), "directive", "listen"
)
),
{"directive": "listen", "args": ["80"]},
)
nginx_conf_parse["parsed"] = remap(
nginx_conf_parse["parsed"],
visit=update_directive(
"server_name", ["localhost"], new_args=["example.com"]
),
)
nginx_conf_parse["parsed"] = remap(
nginx_conf_parse["parsed"],
visit=update_directive("listen", ["80"], new_args=["8080"]),
)
# pp(nginx_conf_parse['parsed'])
# print(crossplane.build(self.nginx_conf_parse['config'][0]['parsed']))
self.assertDictEqual(
no_line(
get_dict_by_key_val(
nginx_conf_parse["parsed"], "directive", "server_name"
)
),
{"directive": "server_name", "args": ["example.com"]},
)
self.assertDictEqual(
no_line(
get_dict_by_key_val(nginx_conf_parse["parsed"], "directive", "listen")
),
{"directive": "listen", "args": ["8080"]},
)
def test_remap_file(self):
with open(CUR_PATH, 'rb') as f:
x = {'a': [1, 2, 3], 'f': [f]}
assert remap(x) == x
f.read()
assert remap(x) == x
f.close() # see #146
assert remap(x) == x
return
def _sub(a, b):
'''
>>> _sub({'a': 3, 'b': [2, {'x': 'y'}], 'c': 3, 'd': 4},
{'a': 2, 'b': {'x': 'y'}, 'c': 3, 'd': 4})
{'a': 1, 'b': 2}
'''
a = deepcopy(a)
a['___previous_was_list___'] = False
def visit(path, key, value):
fpath = path + (key, )
vala = getx(a, fpath)
valb = value
if isinstance(getx(a, fpath[:-1]), list):
valA = getx(a, fpath, inany=True)
else:
valA = None
if vala is not None:
if not isinstance(vala, type(valb)):
if not isinstance(vala, list):
vala = [vala]
if not isinstance(valb, list):
valb = [valb]
new_val = [v for v in vala if v in valb]
if len(new_val) == 1:
new_val = new_val[0]
if not a['___previous_was_list___']:
setx(a, fpath, new_val, b)
a['___previous_was_list___'] = False
# print('>>', a, (vala, valb))
elif hasattr(vala, '__sub__'):
new_val = vala - valb
if not a['___previous_was_list___']:
if new_val:
setx(a, fpath, new_val, b)
else:
delx(a, fpath)
else:
if valA is not None:
# print(valA)
npath, _ = valA
# print('->', a)
delx(a, npath[:-1])
# print('-<', a)
a['___previous_was_list___'] = True
return key, value
remap(b, visit=visit)
del a['___previous_was_list___']
return a
def get_all_titles(items):
"""Get all main titles from a list of Zotero items."""
titles = []
def visit(path, key, value):
if key == 'title':
titles.append(value)
return False
remap(items, visit=visit)
return titles
def test_remap_file(self):
with open(CUR_PATH, 'rb') as f:
x = {'a': [1, 2, 3], 'f': [f]}
assert remap(x) == x
f.read()
assert remap(x) == x
f.close() # see #146
assert remap(x) == x
return
def get_valid_codes(data):
result = defaultdict(list)
def visit(path, key, value):
if isinstance(key, int):
if key % 100:
result[100 * (key // 100)].append(key)
return key, value
remap(data, visit=visit)
return dict(result)
async def _do_other(self,db):
seen = False
if self.options.last:
if self.options.layer < 0:
async for d in db.DoSelect("select * from data_log where data_type=${id} order by data_log.timestamp desc limit ${limit}", _dict=True, limit=self.options.last, id=dtid):
if self.root.verbose > 1:
if seen:
print("===")
if self.options.layer < -1:
await self.ext_layer(db,d)
add_human(d)
pprint(remap(d, lambda p, k, v: v is not None))
else:
print(d['id'],d['timestamp'],d['value'], sep='\t')
seen = True
else:
async for d in db.DoSelect("select data_agg.* from data_agg join data_agg_type on data_agg.data_agg_type=data_agg_type.id where data_agg_type.data_type=${id} and data_agg_type.layer=${layer} order by data_agg.timestamp desc limit ${limit}", _dict=True, id=dtid, layer=self.options.layer, limit=self.options.last):
if self.root.verbose > 1:
if seen:
print("===")
add_human(d)
pprint(remap(d, lambda p, k, v: v is not None))
else:
print(d['id'],d['data_agg_log.timestamp'],d['data_agg_log.value'], sep='\t')
seen = True
else:
if self.options.layer < 0:
async for d in db.DoSelect("select * from data_type order by n_values desc,tag", _dict=True):
add_human(d)
if self.root.verbose > 1:
if seen:
print("===")
pprint(remap(d, lambda p, k, v: v is not None))
else:
try:
m = d['human']['method'][0]
except KeyError:
m = "-"
print(d['n_values'],m,d['timestamp'],d['tag'], sep='\t')
seen = True
else:
async for d in db.DoSelect("select data_agg_type.*,data_type.tag from data_agg_type join data_type on data_type.id=data_agg_type.data_type where layer=${layer}", _dict=True, layer=self.options.layer):
if self.root.verbose > 1:
if seen:
print("===")
add_human(d)
pprint(remap(d, lambda p, k, v: v is not None))
else:
print(d['timestamp'],d['tag'], sep='\t')
seen = True
if not seen:
print("No data?")
def transform_data(data):
'''Transforms OpenWeatherMap data to match our schema'''
transformed_data = OpenWeatherMapTransformer.move_main(data)
transformed_data = remap(transformed_data,
visit=OpenWeatherMapTransformer.visit_move)
transformed_data = remap(transformed_data,
visit=OpenWeatherMapTransformer.visit_rename)
transformed_data = remap(transformed_data,
visit=OpenWeatherMapTransformer.visit_remove)
return transformed_data
def test_preprocessing():
for fixture in [
"tests/fixtures/inline_defined_objects.json",
"tests/fixtures/minimal_example.json",
]:
schema = load_base_schema(fixture).raw_schema
def visit(path, key, value):
if key == "schema" and "type" in value and value["type"] == "object":
assert "x-name" in value
return key, value
remap(schema, visit=visit)
def iterate_files(directory):
"""Itreate over files in the given directory and change values."""
for filename in os.listdir(directory):
if filename.endswith('.json'):
config_json = json.load(open(directory + filename, 'r'))
changed_values = remap(config_json, visit=change_val)
drop_keys = lambda path, key, value: key not in bad_keys
new_file = remap(changed_values, visit=drop_keys)
out_file = open(directory + filename,
'w') # Over-write the input file
# Separators added for no spaces b/w key/val in JSON.
out_file.write(
json.dumps(new_file, indent=2, separators=(',', ':')))
out_file.close()
def formatize(ndata, ignore=[], no_convert=[]):
'''
Applies converters, if they match the name after hash.
For example, if the key is '_:date#isoformat', then the
converters.isoformat() is applied to the value, if exists.
It returns keys without the # sign.
no_convert: list of types, not to apply conversion, e.g., ['url', 'decimal']
'''
def visit(path, key, value):
if isinstance(key, str):
if '#' in str(key)[1:-1:]:
if type(value) in [str, int, float]:
Format = key.rsplit('#', 1)[-1]
if hasattr(converters,
Format) and Format not in no_convert:
k = key.rsplit('#', 1)[0]
v = getattr(converters, key.rsplit('#', 1)[-1])(value)
if (k in ignore) or (key in ignore):
return k, value
return k, v
return key.rsplit('#', 1)[0], value
return key, value
result = remap(ndata, visit=visit)
if isinstance(ndata, list):
result = List(result)
return result
def by_tool_input(self, trans, tool_id, tool_version, param=None, param_dump=None, job_state='ok'):
"""Search for jobs producing same results using the 'inputs' part of a tool POST."""
user = trans.user
input_data = defaultdict(list)
def populate_input_data_input_id(path, key, value):
"""Traverses expanded incoming using remap and collects input_ids and input_data."""
if key == 'id':
path_key = get_path_key(path[:-2])
current_case = param_dump
for p in path:
current_case = current_case[p]
src = current_case['src']
current_case = param
for i, p in enumerate(path):
if p == 'values' and i == len(path) - 2:
continue
if isinstance(current_case, (list, dict)):
current_case = current_case[p]
identifier = getattr(current_case, "element_identifier", None)
input_data[path_key].append({'src': src,
'id': value,
'identifier': identifier,
})
return key, "__id_wildcard__"
return key, value
wildcard_param_dump = remap(param_dump, visit=populate_input_data_input_id)
return self.__search(tool_id=tool_id,
tool_version=tool_version,
user=user,
input_data=input_data,
job_state=job_state,
param_dump=param_dump,
wildcard_param_dump=wildcard_param_dump)
def _item_to_csv(self, item):
from boltons.iterutils import remap
output = io.BytesIO()
writer = self._create_csv_writer(output)
item = remap(item, visit=self._encode_string)
writer.writerow(item)
return output.getvalue().rstrip()
def on_data(self, data):
global path
global logfile
#print(json.loads(data)["text"])
tweet = json.loads(data)
id_json = tweet["id_str"]
json_doc = json.dumps(tweet, ensure_ascii=False)
#json_tweet = json_doc.encode('utf8')
#json_doc_v2 = json.dumps( (json.JSONDecoder().decode(data)).encode('utf8') )
print(type(json_doc))
#json_write = json.loads(json_doc,parse_float = Decimal)
json_write = json.loads(json_doc)
#dic_json = remap(dict(json_write), visit=drop_falsey)
dic_json = remap(dict(json_write), visit=drop_falsey2)
response = table.put_item(
Item = json.loads(json.dumps(dic_json),parse_float = Decimal)
)
print("PutItem succeeded:")
print(json.dumps(response))
#print(dic_json)
with open(path+logfile,"a+") as f:
f.write("PutItem succeeded:")
f.write(json.dumps(response)+"\n")
return True
def get_file_report(file_sha256, report_url, environment_id, type_, apikey,
secret, verify):
user_agent = {'User-agent': 'VxStream Sandbox'}
params = {
'apikey': apikey,
'secret': secret,
'environmentId': environment_id,
'type': type_
}
resource_url = '%s/%s' % (report_url, file_sha256)
try:
res = requests.get(resource_url,
headers=user_agent,
params=params,
verify=verify)
if res.status_code == 200:
# walk entire json blob to fix
# the keys known to cause issues
remapped = remap(res.json(), visit=visit)
return remapped
else:
print('Error code: {}, returned when getting report: {}'.format(
res.status_code, file_sha256))
return res
except requests.exceptions.HTTPError as err:
print(err)
def _deserialize(self, value, attr, obj, recurse_lvl=0):
def visit(parent, key, _value):
if isinstance(_value, dict) and self._BYTES_KEY in _value:
return key, self._hex_list_to_binary(_value[self._BYTES_KEY])
return True
return remap(value, visit=visit)
def preferred_locations(talent_pool_ids=None,
candidate_id=None,
preferred_location_id=None):
"""
:type talent_pool_ids: list[int]
:rtype: dict[list]
"""
data = {
'candidates': [{
'id':
candidate_id,
'talent_pool_ids': {
'add': talent_pool_ids
},
'preferred_locations': [{
'id': preferred_location_id,
'city': fake.city(),
'state': fake.state(),
'country_code': fake.country_code(),
'subdivision_code': 'US-CA'
}]
}]
}
# Recursively Remove keys with None values
data = remap(data, lambda p, k, v: v is not None)
return data
def military_services(talent_pool_ids=None,
candidate_id=None,
military_experience_id=None):
"""
:type talent_pool_ids: list[int]
:rtype: dict[list]
"""
data = {
'candidates': [{
'id':
candidate_id,
'talent_pool_ids': {
'add': talent_pool_ids
},
'military_services': [{
'id': military_experience_id,
'country_code': fake.country_code(),
'branch': fake.military_ship(),
'highest_rank': 'lieutenant',
'status': 'active',
'highest_grade': '0-1',
'comments': fake.bs(),
'from_date': '1974-5-25',
'to_date': '1996-12-12'
}]
}]
}
# Recursively Remove keys with None values
data = remap(data, lambda p, k, v: v is not None)
return data
def work_preference(talent_pool_ids=None,
candidate_id=None,
preference_id=None):
data = {
'candidates': [{
'id': candidate_id,
'talent_pool_ids': {
'add': talent_pool_ids
},
'work_preference': {
'id': preference_id,
"relocate": False,
"authorization": "US Citizen",
"telecommute": True,
"travel_percentage": randrange(0, 100),
"hourly_rate": float('%.2f' % random.uniform(20, 90)),
"salary": randrange(50000, 300000),
"employment_type": "full-time employment",
"security_clearance": None,
"third_party": False
}
}]
}
# Recursively Remove keys with None values
data = remap(data, lambda p, k, v: v is not None)
return data
def phones(talent_pool_ids=None,
candidate_id=None,
phone_id=None,
internationalize=False,
extension=False):
"""
:type talent_pool_ids: list[int]
:param internationalize: If True, the phone number value will be internationalized, e.g. +14085067789
:rtype: dict[list]
"""
# Generate phone number
value = generate_international_phone_number(
extension) if internationalize else sample_phone_number()
data = {
'candidates': [{
'id':
candidate_id,
'talent_pool_ids': {
'add': talent_pool_ids
},
'phones': [{
'id': phone_id,
'label': 'Work',
'value': value,
'is_default': False
}]
}]
}
# Recursively Remove keys with None values
data = remap(data, lambda p, k, v: v is not None)
return data
def round_floats(a_dict, precision):
"""Find all the floats in `a_dict` (recursive) and round them to `precision`."""
return remap(
a_dict,
lambda p, k, v: (k, round(v, precision))
if isinstance(v, float) else (k, v),
)
def _validate(args, app_desc):
if Core is None:
raise Exception("Cannot validate file, pykwalify is not installed.")
path = _find_config(args, app_desc)
# Allow empty mapping (not allowed by pykwalify)
raw_config = _order_load_path(path)
if raw_config.get(app_desc.app_name) is None:
raw_config[app_desc.app_name] = {}
# Rewrite the file any way to merge any duplicate keys
with tempfile.NamedTemporaryFile('w', delete=False,
suffix=".yml") as config_p:
ordered_dump(raw_config, config_p)
def _clean(p, k, v):
return k not in ['reloadable', 'path_resolves_to']
clean_schema = remap(app_desc.schema.raw_schema, _clean)
with tempfile.NamedTemporaryFile('w', suffix=".yml") as fp:
ordered_dump(clean_schema, fp)
fp.flush()
c = Core(
source_file=config_p.name,
schema_files=[fp.name],
)
os.remove(config_p.name)
c.validate()
def get_all_department_identifiers() -> [dict]:
"""
"""
url = 'https://api.maui.uiowa.edu/maui/api/pub/registrar/program-of' \
'-study/program'
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'application/json'
}
response = get(url=url, headers=headers)
if response.status_code != 200:
print('ERROR: HTTP{}'.format(response.status_code))
raise ValueError
data = response.json()
cleaned = remap(
data, lambda p, k, v: v is not None and v != [] and v != '' and v != [{
}] and v != {})
filtered = list(map(lambda p: filter_keys(p, ['academicUnit']), cleaned))
filtered = list(map(lambda p: p['academicUnit'], filtered))
departments = list(map(lambda p: filter_keys(p, ['naturalKey']), filtered))
departments = list(map(lambda p: p['naturalKey'], departments))
pprint(departments)
departments = list(sorted(list(set(departments))))
new_filtered = []
for item in departments:
if len(item) > 1:
new_filtered.append(item)
return new_filtered
def test_prepop(self):
"""Demonstrating normalization and ID addition through prepopulating
the objects wth an enter callback.
"""
base_obj = {'name': None,
'rank': None,
'id': 1}
def enter(path, key, value):
new_parent, new_items = default_enter(path, key, value)
try:
new_parent.update(base_obj)
base_obj['id'] += 1
except:
pass
return new_parent, new_items
orig = [{'name': 'Firefox', 'rank': 1},
{'name': 'Chrome', 'rank': 2},
{'name': 'IE'}]
ref = [{'name': 'Firefox', 'rank': 1, 'id': 1},
{'name': 'Chrome', 'rank': 2, 'id': 2},
{'name': 'IE', 'rank': None, 'id': 3}]
remapped = remap(orig, enter=enter)
assert remapped == ref
def _validate(args, app_desc):
path = _find_config(args, app_desc)
# Allow empty mapping (not allowed by pykawlify)
raw_config = _order_load_path(path)
if raw_config.get(app_desc.app_name, None) is None:
raw_config[app_desc.app_name] = {}
config_p = tempfile.NamedTemporaryFile('w',
delete=False,
suffix=".yml")
ordered_dump(raw_config, config_p)
config_p.flush()
path = config_p.name
fp = tempfile.NamedTemporaryFile('w', delete=False, suffix=".yml")
def _clean(p, k, v):
return k != 'reloadable'
clean_schema = remap(app_desc.schema.raw_schema, _clean)
ordered_dump(clean_schema, fp)
fp.flush()
name = fp.name
if Core is None:
raise Exception("Cannot validate file, pykwalify is not installed.")
c = Core(
source_file=path,
schema_files=[name],
)
c.validate()
def sanitize_strings(data):
def sanitize(p, k, v):
if isinstance(v, str):
return (k,
bleach.clean(v, [
'div',
'p',
'span',
'b',
'ul',
'li',
'ol',
'sub',
'sup',
'pre',
'a',
'blockquote',
'h1',
'h2',
'h3',
'h4',
'h5',
'strong',
'em',
'br',
],
strip=True,
strip_comments=True))
return k, v
return remap(data, sanitize)
def test_dict_to_omd(self):
def enter(path, key, value):
if isinstance(value, dict):
return OMD(), sorted(value.items())
return default_enter(path, key, value)
orig = [{
'title': 'Wild Palms',
'ratings': {
1: 1,
2: 3,
3: 5,
4: 6,
5: 3
}
}, {
'title': 'Twin Peaks',
'ratings': {
1: 3,
2: 2,
3: 8,
4: 12,
5: 15
}
}]
remapped = remap(orig, enter=enter)
assert remapped == orig
assert isinstance(remapped[0], OMD)
assert isinstance(remapped[0]['ratings'], OMD)
assert isinstance(remapped[1], OMD)
assert isinstance(remapped[1]['ratings'], OMD)
def members_dict(self, att_path_key: str, values: list):
m_dict = self.__members_dict
m_dict[att_path_key] = []
att_path_key_tuple = tuple(map(str, att_path_key.split('/')))
att_key = att_path_key_tuple[len(key_path_tuple) - 1]
att_path = key_path_tuple[0:(len(key_path_tuple) - 1)]
def visit(path, key, value):
if path == att_path:
if key == 'att_key':
m_dict[att_path_key] = m_dict[att_path_key].append(value)
for mm in self.members:
remap(mm.__dict__, visit=visit)
def untag(data, locale=None, params=None):
"""Untags fields, handling translation priority."""
updated_localized_paths = set()
paths_to_keep_tagged = set()
def visit(path, key, value):
"""Function for each key and value in the data."""
if not isinstance(key, basestring):
return key, value
if (path, key.rstrip('@')) in updated_localized_paths:
return False
if key.endswith('@#'):
return False
if key.endswith('@'):
if isinstance(value, list):
paths_to_keep_tagged.add((path, key))
key = key[:-1]
# Support <key>@<param key>.<param value>: <value>.
if params:
param_regex = re.compile(
r'(.*)@({})\.([^@]+)$'.format('|'.join(params.keys())))
param_match = param_regex.match(key)
if param_match:
untagged_key, param_key, param_value = param_match.groups()
param_value_regex = r'^{}$'.format(param_value)
if not params[param_key] or not re.match(param_value_regex, params[param_key]):
return False
updated_localized_paths.add((path, untagged_key.rstrip('@')))
return untagged_key, value
# Support <key>@<locale regex>: <value>.
match = LOCALIZED_KEY_REGEX.match(key)
if not match:
updated_localized_paths.add((path, key))
return key, value
untagged_key, locale_from_key = match.groups()
locale_regex = r'^{}$'.format(locale_from_key)
if not locale or not re.match(locale_regex, locale):
return False
updated_localized_paths.add((path, untagged_key.rstrip('@')))
return untagged_key, value
# Backwards compatibility for https://github.com/grow/grow/issues/95
def exit(path, key, old_parent, new_parent, new_items):
resp = iterutils.default_exit(path, key, old_parent,
new_parent, new_items)
if paths_to_keep_tagged and isinstance(resp, dict):
for sub_key, value in resp.items():
if not isinstance(value, list):
continue
new_key = '{}@'.format(sub_key)
resp[new_key] = value
try:
paths_to_keep_tagged.remove((path, key))
except KeyError:
pass
return resp
return iterutils.remap(data, visit=visit, exit=exit)
def addresses(talent_pool_ids=None,
candidate_id=None,
address_id=None,
is_default=False):
"""
:type talent_pool_ids: list[int]
:param talent_pool_ids is required for creating candidate, but not for updating
:type candidate_id: int | long
:rtype: dict
"""
data = {
'candidates': [{
'id':
candidate_id,
'talent_pool_ids': {
'add': talent_pool_ids
},
'addresses': [{
'id': address_id,
'address_line_1': fake.street_address(),
'city': fake.city(),
'subdivision_code': 'US-CA',
'zip_code': fake.zipcode(),
'country_code': fake.country_code(),
'is_default': is_default
}]
}]
}
# Remove keys with None values
data = remap(data, lambda p, k, v: v is not None)
return data
def test_sort_all_lists(self):
def exit(path, key, old_parent, new_parent, new_items):
# NB: in this case, I'd normally use *a, **kw
ret = default_exit(path, key, old_parent, new_parent, new_items)
if isinstance(ret, list):
ret.sort()
return ret
# NB: Airplane model numbers (Boeing and Airbus)
orig = [[[7, 0, 7],
[7, 2, 7],
[7, 7, 7],
[7, 3, 7]],
[[3, 8, 0],
[3, 2, 0],
[3, 1, 9],
[3, 5, 0]]]
ref = [[[0, 2, 3],
[0, 3, 5],
[0, 3, 8],
[1, 3, 9]],
[[0, 7, 7],
[2, 7, 7],
[3, 7, 7],
[7, 7, 7]]]
remapped = remap(orig, exit=exit)
assert remapped == ref
def test_prepop(self):
"""Demonstrating normalization and ID addition through prepopulating
the objects wth an enter callback.
"""
base_obj = {'name': None,
'rank': None,
'id': 1}
def enter(path, key, value):
new_parent, new_items = default_enter(path, key, value)
try:
new_parent.update(base_obj)
base_obj['id'] += 1
except:
pass
return new_parent, new_items
orig = [{'name': 'Firefox', 'rank': 1},
{'name': 'Chrome', 'rank': 2},
{'name': 'IE'}]
ref = [{'name': 'Firefox', 'rank': 1, 'id': 1},
{'name': 'Chrome', 'rank': 2, 'id': 2},
{'name': 'IE', 'rank': None, 'id': 3}]
remapped = remap(orig, enter=enter)
assert remapped == ref
def test_sort_all_lists(self):
def exit(path, key, old_parent, new_parent, new_items):
# NB: in this case, I'd normally use *a, **kw
ret = default_exit(path, key, old_parent, new_parent, new_items)
if isinstance(ret, list):
ret.sort()
return ret
# NB: Airplane model numbers (Boeing and Airbus)
orig = [[[7, 0, 7],
[7, 2, 7],
[7, 7, 7],
[7, 3, 7]],
[[3, 8, 0],
[3, 2, 0],
[3, 1, 9],
[3, 5, 0]]]
ref = [[[0, 2, 3],
[0, 3, 5],
[0, 3, 8],
[1, 3, 9]],
[[0, 7, 7],
[2, 7, 7],
[3, 7, 7],
[7, 7, 7]]]
remapped = remap(orig, exit=exit)
assert remapped == ref
def test_path(self):
path_map = {}
# test visit's path
target_str = 'test'
orig = [[[target_str]]]
ref_path = (0, 0, 0)
def visit(path, key, value):
if value is target_str:
path_map['target_str'] = path + (key,)
return key, value
remapped = remap(orig, visit=visit)
assert remapped == orig
assert path_map['target_str'] == ref_path
# test enter's path
target_obj = object()
orig = {'a': {'b': {'c': {'d': ['e', target_obj, 'f']}}}}
ref_path = ('a', 'b', 'c', 'd', 1)
def enter(path, key, value):
if value is target_obj:
path_map['target_obj'] = path + (key,)
return default_enter(path, key, value)
remapped = remap(orig, enter=enter)
assert remapped == orig
assert path_map['target_obj'] == ref_path
# test exit's path
target_set = frozenset([1, 7, 3, 8])
orig = [0, 1, 2, [3, 4, [5, target_set]]]
ref_path = (3, 2, 1)
def exit(path, key, old_parent, new_parent, new_items):
if old_parent is target_set:
path_map['target_set'] = path + (key,)
return default_exit(path, key, old_parent, new_parent, new_items)
remapped = remap(orig, exit=exit)
assert remapped == orig
assert path_map['target_set'] == ref_path
def test_sub_selfref(self):
coll = [0, 1, 2, 3]
sub = []
sub.append(sub)
coll.append(sub)
with pytest.raises(RuntimeError):
# if equal, should recurse infinitely
assert coll == remap(coll)
def gather_for_locale(front_matter, locale):
if not front_matter:
return ''
parsed = yaml.load(front_matter, Loader=PlainTextYamlLoader)
locale_extra = OrderedDict()
def visit(path, key, value):
if not isinstance(key, basestring):
return key, value
if key.endswith('@#'):
return key, value
match = LOCALIZED_KEY_REGEX.match(key)
if not match:
return key, value
base_key = match.group(1)
locale_from_key = match.group(2)
if locale_from_key == locale:
# If there is a key without the trailing @ then override it.
parent = parsed
locale_parent = locale_extra
for path_key in path:
parent = parent[path_key]
if isinstance(locale_parent, list):
locale_parent = locale_parent[path_key]
elif path_key not in locale_parent:
if isinstance(parent, list):
locale_parent[path_key] = copy.deepcopy(parent)
else:
locale_parent[path_key] = OrderedDict()
locale_parent = locale_parent[path_key]
else:
locale_parent = locale_parent[path_key]
if base_key in parent:
locale_parent[base_key] = value
else:
locale_parent['{}@'.format(base_key)] = value
if key in locale_parent:
locale_parent.pop(key, None)
return False
return key, value
parsed = iterutils.remap(parsed, visit=visit)
return (yaml.dump(
parsed, Dumper=PlainTextYamlDumper,
allow_unicode=True, default_flow_style=False).strip() if parsed else '',
locale_extra)
def untag(data, locale=None, env_name=None):
"""Untags fields, handling translation priority."""
updated_localized_paths = set()
paths_to_keep_tagged = set()
def visit(path, key, value):
if not isinstance(key, basestring):
return key, value
if (path, key.rstrip('@')) in updated_localized_paths:
return False
if key.endswith('@#'):
return False
if key.endswith('@'):
if isinstance(value, list):
paths_to_keep_tagged.add((path, key))
key = key[:-1]
# Support <key>@env.<name regex>: <value>.
env_match = ENV_KEY_REGEX.match(key)
if env_match:
untagged_key, env_name_from_key = env_match.groups()
env_regex = r'^{}$'.format(env_name_from_key)
if not env_name or not re.match(env_regex, env_name):
return False
updated_localized_paths.add((path, untagged_key.rstrip('@')))
return untagged_key, value
# Support <key>@<locale regex>: <value>.
match = LOCALIZED_KEY_REGEX.match(key)
if not match:
updated_localized_paths.add((path, key))
return key, value
untagged_key, locale_from_key = match.groups()
locale_regex = r'^{}$'.format(locale_from_key)
if not locale or not re.match(locale_regex, locale):
return False
updated_localized_paths.add((path, untagged_key.rstrip('@')))
return untagged_key, value
# Backwards compatibility for https://github.com/grow/grow/issues/95
def exit(path, key, old_parent, new_parent, new_items):
resp = iterutils.default_exit(path, key, old_parent,
new_parent, new_items)
if paths_to_keep_tagged and isinstance(resp, dict):
for sub_key, value in resp.items():
if not isinstance(value, list):
continue
new_key = '{}@'.format(sub_key)
resp[new_key] = value
try:
paths_to_keep_tagged.remove((path, key))
except KeyError:
pass
return resp
return iterutils.remap(data, visit=visit, exit=exit)
def test_noncallables(self):
with pytest.raises(TypeError):
remap([], visit='test')
with pytest.raises(TypeError):
remap([], enter='test')
with pytest.raises(TypeError):
remap([], exit='test')
async def _do_args(self,db, args):
seen = False
dtid, = await self.db.DoFn("select id from data_type where tag=${tag}", tag=' '.join(args))
if self.options.last:
if self.options.layer < 0:
async for d in db.DoSelect("select * from data_log where data_log.data_type=${id} order by data_log.timestamp desc limit ${limit}", _dict=True, id=dtid, limit=self.options.last):
if self.root.verbose > 1:
if seen:
print("===")
add_human(d)
pprint(remap(d, lambda p, k, v: v is not None))
else:
print(d['id'],d['timestamp'],d['value'], sep='\t')
seen = True
else:
async for d in db.DoSelect("select data_agg.* from data_agg join data_agg_type on data_agg.data_agg_type=data_agg_type.id where data_agg_type.data_type=${id} and data_agg_type.layer=${layer} order by data_agg.timestamp desc limit ${limit}", _dict=True, id=dtid, limit=self.options.last, layer=self.options.layer):
if self.root.verbose > 1:
if seen:
print("===")
add_human(d)
pprint(remap(d, lambda p, k, v: v is not None))
else:
print(d['id'],d['timestamp'],d['value'],d['n_values'], sep='\t')
seen = True
if not seen:
print("No data?")
else:
if self.options.layer < 0: # display this type
d = await self.db.DoFn("select * from data_type where id=${id}", _dict=True, id=dtid)
if self.options.layer < -1:
await self.ext_layer(db,d)
add_human(d)
pprint(remap(d, lambda p, k, v: v is not None))
else: # display this layer
d = await self.db.DoFn("select * from data_agg_type where data_type=${id} and layer=${layer}", _dict=True, id=dtid, layer=self.options.layer)
add_human(d)
pprint(remap(d, lambda p, k, v: v is not None))
def flatten_fields(msg_proto, **kwargs):
col_type_map = kwargs.pop('col_type_map', {})
default_col_type = kwargs.pop('default_col_type', 'text')
sep = kwargs.pop('sep', SEP)
if kwargs:
raise TypeError('unexpected keyword arguments: %r' % kwargs.keys())
ret = []
def visit(path, key, value):
if isinstance(value, (list, dict)):
return True # traverse containers, but don't no columns for them
full_path = path + (key,)
col_type = col_type_map.get(full_path, default_col_type)
full_path_str = sep.join([str(p) for p in full_path])
ret.append((full_path, full_path_str, col_type))
return True
# read more about remap here: http://sedimental.org/remap.html
remap(msg_proto, visit=visit)
return sorted(ret, key=lambda x: x[0])
async def _do_unassigned(self,db,method):
seen = False
if method is None:
mf = "IS NULL"
elif method == "all":
mf = "IS NOT NULL"
else:
mf = "= ${method}"
try:
method = modenames[method]
except KeyError:
print("Unknown method. Known:",' '.join(sorted(modenames.keys())), file=sys.stderr)
return
if self.options.last:
async for d in db.DoSelect("select data_log.*,data_type.tag from data_log join data_type on data_type.id=data_log.data_type where data_type.method %s order by data_log.timestamp desc limit ${limit}" % (mf,), _dict=True, limit=self.options.last, method=method):
if self.root.verbose > 1:
if seen:
print("===")
add_human(d)
pprint(remap(d, lambda p, k, v: v is not None))
else:
print(d['id'],d['timestamp'],d['tag'],d['value'], sep='\t')
seen = True
else:
async for d in db.DoSelect("select * from data_type where method %s and timestamp > '1999-12-31' order by n_values desc,tag" % (mf,), _dict=True, method=method):
add_human(d)
if self.root.verbose > 1:
if seen:
print("===")
pprint(remap(d, lambda p, k, v: v is not None))
else:
try:
m = d['human']['method'][0]
except KeyError:
m = "-"
print(d['n_values'],m,d['timestamp'],d['tag'], sep='\t')
seen = True
def update_dataset_ids(input_values, translate_values, src):
def replace_dataset_ids(path, key, value):
"""Exchanges dataset_ids (HDA, LDA, HDCA, not Dataset) in input_values with dataset ids used in job."""
current_case = input_values
if key == 'id':
for i, p in enumerate(path):
if isinstance(current_case, (list, dict)):
current_case = current_case[p]
if src == current_case.get('src'):
return key, translate_values.get(current_case['id'], value)
return key, value
return remap(input_values, visit=replace_dataset_ids)
def get_network_for_export(conn, network_id, include_data):
# QC the template
def visit(path, key, value):
if key in set(['cr_date']):
return False
elif key in set(['id', 'template_id', 'type_id', 'attr_id']):
return key, -1
return key, value
network = conn.get_network(network_id, include_data=include_data)
network = remap(dict(network), visit=visit)
filename = "%s.json" % network['name']
content = json.dumps(network, sort_keys=True, indent=4, separators=(',', ': '))
return filename, content
def test_collector_pattern(self):
all_interests = set()
def enter(path, key, value):
try:
all_interests.update(value['interests'])
except:
pass
return default_enter(path, key, value)
orig = [{'name': 'Kate',
'interests': ['theater', 'manga'],
'dads': [{'name': 'Chris',
'interests': ['biking', 'python']}]},
{'name': 'Avery',
'interests': ['museums', 'pears'],
'dads': [{'name': 'Kurt',
'interests': ['python', 'recursion']}]}]
ref = set(['python', 'recursion', 'biking', 'museums',
'pears', 'theater', 'manga'])
remap(orig, enter=enter)
assert all_interests == ref
def get_file_report(file_sha256, report_url, environment_id, type_, apikey, secret, verify):
user_agent = {'User-agent': 'VxStream Sandbox'}
params = {'apikey': apikey, 'secret': secret, 'environmentId': environment_id, 'type': type_}
resource_url = '%s/%s' % (report_url, file_sha256)
try:
res = requests.get(resource_url, headers=user_agent, params=params, verify=verify)
if res.status_code == 200:
# walk entire json blob to fix
# the keys known to cause issues
remapped = remap(res.json(), visit=visit)
return remapped
else:
print('Error code: {}, returned when getting report: {}'.format(res.status_code, file_sha256))
return res
except requests.exceptions.HTTPError as err:
print(err)
def test_dict_to_omd(self):
def enter(path, key, value):
if isinstance(value, dict):
return OMD(), sorted(value.items())
return default_enter(path, key, value)
orig = [{'title': 'Wild Palms',
'ratings': {1: 1, 2: 3, 3: 5, 4: 6, 5: 3}},
{'title': 'Twin Peaks',
'ratings': {1: 3, 2: 2, 3: 8, 4: 12, 5: 15}}]
remapped = remap(orig, enter=enter)
assert remapped == orig
assert isinstance(remapped[0], OMD)
assert isinstance(remapped[0]['ratings'], OMD)
assert isinstance(remapped[1], OMD)
assert isinstance(remapped[1]['ratings'], OMD)
def test_namedtuple(self):
"""TODO: this fails right now because namedtuples' __new__ is
overridden to accept arguments. remap's default_enter tries
to create an empty namedtuple and gets a TypeError.
Could make it so that immutable types actually don't create a
blank new parent and instead use the old_parent as a
placeholder, creating a new one at exit-time from the value's
__class__ (how default_exit works now). But even then it would
have to *args in the values, as namedtuple constructors don't
take an iterable.
"""
Point = namedtuple('Point', 'x y')
point_map = {'origin': [Point(0, 0)]}
with pytest.raises(TypeError):
remapped = remap(point_map)
assert isinstance(remapped['origin'][0], Point)
def modify_template():
if request.method == 'POST':
template = AttrDict(request.json['template'])
# QC the template
def visit(path, key, value):
if key in set(['cr_date']):
return False
#elif key in set(['id', 'template_id', 'type_id', 'attr_id']):
#return key, None
return key, value
template = remap(dict(template), visit=visit)
result = g.conn.call('update_template', {'tmpl': dict(template)})
if 'faultcode' in result:
result = -1
return jsonify(result=result)
return redirect(url_for('manager.manage_templates'))
def test_add_length(self):
def exit(path, key, old_parent, new_parent, new_items):
ret = default_exit(path, key, old_parent, new_parent, new_items)
try:
ret['review_length'] = len(ret['review'])
except:
pass
return ret
orig = {'Star Trek':
{'TNG': {'stars': 10,
'review': "Episodic AND deep. <3 Data."},
'DS9': {'stars': 8.5,
'review': "Like TNG, but with a story and no Data."},
'ENT': {'stars': None,
'review': "Can't review what you can't watch."}},
'Babylon 5': {'stars': 6,
'review': "Sophomoric, like a bitter laugh."},
'Dr. Who': {'stars': None,
'review': "800 episodes is too many to review."}}
remapped = remap(orig, exit=exit)
assert (remapped['Star Trek']['TNG']['review_length']
< remapped['Star Trek']['DS9']['review_length'])
def convert_for_locale(front_matter, locale, base=None):
if not front_matter:
parsed = {}
else:
parsed = yaml.load(front_matter, Loader=PlainTextYamlLoader)
def visit(path, key, value):
if not isinstance(key, basestring):
return key, value
if key.endswith('@#'):
return key, value
match = LOCALIZED_KEY_REGEX.match(key)
if not match:
return key, value
base_key = match.group(1)
locale_from_key = match.group(2)
if locale_from_key == locale:
# If there is a key without the trailing @ then override it.
parent = parsed
for path_key in path:
parent = parent[path_key]
if base_key in parent:
return base_key, value
return '{}@'.format(base_key), value
return False
parsed = iterutils.remap(parsed, visit=visit)
# If there are pre-existing fields, use them as a base for the locale
# specific values.
result = base or {}
_update_deep(result, parsed)
return result
def test_basic_upper(self):
orig = {'a': 1, 'b': object(), 'c': {'d': set()}}
remapped = remap(orig, lambda p, k, v: (k.upper(), v))
assert orig['a'] == remapped['A']
assert orig['b'] == remapped['B']
assert orig['c']['d'] == remapped['C']['D']
def test_item_drop(self):
orig = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
even_items = remap(orig, lambda p, k, v: not (v % 2))
assert even_items == [0, 2, 4, 6, 8]
