def test_remap_set(self): # explicit test for sets to make sure #84 is covered s = set([1, 2, 3]) assert remap(s) == s fs = frozenset([1, 2, 3]) assert remap(fs) == fs
def test_reraise_visit(self): root = {'A': 'b', 1: 2} key_to_lower = lambda p, k, v: (k.lower(), v) with pytest.raises(AttributeError): remap(root, key_to_lower) remapped = remap(root, key_to_lower, reraise_visit=False) assert remapped['a'] == 'b' assert remapped[1] == 2
def test_drop_nones(self): orig = {'a': 1, 'b': None, 'c': [3, None, 4, None]} ref = {'a': 1, 'c': [3, 4]} drop_none = lambda p, k, v: v is not None remapped = remap(orig, visit=drop_none) assert remapped == ref orig = [None] * 100 remapped = remap(orig, drop_none) assert not remapped
def test_root_selfref(self): selfref = [0, 1, 2, 3] selfref.append(selfref) with pytest.raises(RuntimeError): assert selfref == remap(selfref) selfref2 = {} selfref2['self'] = selfref2 with pytest.raises(RuntimeError): assert selfref2 == remap(selfref2)
def post_save(cls, sender, document, **kwargs): if kwargs.get("skip"): return # project is LazyReferenceField project = document.project.fetch() # set columns field for project def update_columns(path, key, value): path = delimiter.join(["data"] + list(path) + [key]) is_quantity = isinstance(value, dict) and quantity_keys.issubset( value.keys() ) is_text = bool( not is_quantity and isinstance(value, str) and key not in quantity_keys ) if is_quantity or is_text: project.reload("columns") try: column = project.columns.get(path=path) if is_quantity: v = value["value"] if isnan(column.max) or v > column.max: column.max = v if isnan(column.min) or v < column.min: column.min = v except DoesNotExist: column = {"path": path} if is_quantity: column["unit"] = value["unit"] column["min"] = column["max"] = value["value"] project.columns.create(**column) project.save().reload("columns") ncolumns = len(project.columns) if ncolumns > 50: raise ValueError("Reached maximum number of columns (50)!") return True # run update_columns over document data remap(document.data, visit=update_columns, enter=enter) # add/remove columns for other components for path in COMPONENTS.keys(): try: project.columns.get(path=path) except DoesNotExist: if getattr(document, path): project.columns.create(path=path) project.save().reload("columns") document.last_modified = datetime.utcnow()
def test_filter_map_block(self): for config in self.nginx_conf_parse["config"]: self.assertEqual(config["status"], "ok") nginx_conf_parse = next( config for config in self.nginx_conf_parse["config"] if path.basename(config["file"]) == "nginx.conf" ) no_line = partial(del_keys_d, key="line") self.assertDictEqual( no_line( get_dict_by_key_val( deepcopy(nginx_conf_parse["parsed"]), "directive", "server_name" ) ), {"directive": "server_name", "args": ["localhost"]}, ) self.assertDictEqual( no_line( get_dict_by_key_val( deepcopy(nginx_conf_parse["parsed"]), "directive", "listen" ) ), {"directive": "listen", "args": ["80"]}, ) nginx_conf_parse["parsed"] = remap( nginx_conf_parse["parsed"], visit=update_directive( "server_name", ["localhost"], new_args=["example.com"] ), ) nginx_conf_parse["parsed"] = remap( nginx_conf_parse["parsed"], visit=update_directive("listen", ["80"], new_args=["8080"]), ) # pp(nginx_conf_parse['parsed']) # print(crossplane.build(self.nginx_conf_parse['config'][0]['parsed'])) self.assertDictEqual( no_line( get_dict_by_key_val( nginx_conf_parse["parsed"], "directive", "server_name" ) ), {"directive": "server_name", "args": ["example.com"]}, ) self.assertDictEqual( no_line( get_dict_by_key_val(nginx_conf_parse["parsed"], "directive", "listen") ), {"directive": "listen", "args": ["8080"]}, )
def test_remap_file(self): with open(CUR_PATH, 'rb') as f: x = {'a': [1, 2, 3], 'f': [f]} assert remap(x) == x f.read() assert remap(x) == x f.close() # see #146 assert remap(x) == x return
def _sub(a, b): ''' >>> _sub({'a': 3, 'b': [2, {'x': 'y'}], 'c': 3, 'd': 4}, {'a': 2, 'b': {'x': 'y'}, 'c': 3, 'd': 4}) {'a': 1, 'b': 2} ''' a = deepcopy(a) a['___previous_was_list___'] = False def visit(path, key, value): fpath = path + (key, ) vala = getx(a, fpath) valb = value if isinstance(getx(a, fpath[:-1]), list): valA = getx(a, fpath, inany=True) else: valA = None if vala is not None: if not isinstance(vala, type(valb)): if not isinstance(vala, list): vala = [vala] if not isinstance(valb, list): valb = [valb] new_val = [v for v in vala if v in valb] if len(new_val) == 1: new_val = new_val[0] if not a['___previous_was_list___']: setx(a, fpath, new_val, b) a['___previous_was_list___'] = False # print('>>', a, (vala, valb)) elif hasattr(vala, '__sub__'): new_val = vala - valb if not a['___previous_was_list___']: if new_val: setx(a, fpath, new_val, b) else: delx(a, fpath) else: if valA is not None: # print(valA) npath, _ = valA # print('->', a) delx(a, npath[:-1]) # print('-<', a) a['___previous_was_list___'] = True return key, value remap(b, visit=visit) del a['___previous_was_list___'] return a
def get_all_titles(items): """Get all main titles from a list of Zotero items.""" titles = [] def visit(path, key, value): if key == 'title': titles.append(value) return False remap(items, visit=visit) return titles
def get_valid_codes(data): result = defaultdict(list) def visit(path, key, value): if isinstance(key, int): if key % 100: result[100 * (key // 100)].append(key) return key, value remap(data, visit=visit) return dict(result)
async def _do_other(self,db): seen = False if self.options.last: if self.options.layer < 0: async for d in db.DoSelect("select * from data_log where data_type=${id} order by data_log.timestamp desc limit ${limit}", _dict=True, limit=self.options.last, id=dtid): if self.root.verbose > 1: if seen: print("===") if self.options.layer < -1: await self.ext_layer(db,d) add_human(d) pprint(remap(d, lambda p, k, v: v is not None)) else: print(d['id'],d['timestamp'],d['value'], sep='\t') seen = True else: async for d in db.DoSelect("select data_agg.* from data_agg join data_agg_type on data_agg.data_agg_type=data_agg_type.id where data_agg_type.data_type=${id} and data_agg_type.layer=${layer} order by data_agg.timestamp desc limit ${limit}", _dict=True, id=dtid, layer=self.options.layer, limit=self.options.last): if self.root.verbose > 1: if seen: print("===") add_human(d) pprint(remap(d, lambda p, k, v: v is not None)) else: print(d['id'],d['data_agg_log.timestamp'],d['data_agg_log.value'], sep='\t') seen = True else: if self.options.layer < 0: async for d in db.DoSelect("select * from data_type order by n_values desc,tag", _dict=True): add_human(d) if self.root.verbose > 1: if seen: print("===") pprint(remap(d, lambda p, k, v: v is not None)) else: try: m = d['human']['method'][0] except KeyError: m = "-" print(d['n_values'],m,d['timestamp'],d['tag'], sep='\t') seen = True else: async for d in db.DoSelect("select data_agg_type.*,data_type.tag from data_agg_type join data_type on data_type.id=data_agg_type.data_type where layer=${layer}", _dict=True, layer=self.options.layer): if self.root.verbose > 1: if seen: print("===") add_human(d) pprint(remap(d, lambda p, k, v: v is not None)) else: print(d['timestamp'],d['tag'], sep='\t') seen = True if not seen: print("No data?")
def transform_data(data): '''Transforms OpenWeatherMap data to match our schema''' transformed_data = OpenWeatherMapTransformer.move_main(data) transformed_data = remap(transformed_data, visit=OpenWeatherMapTransformer.visit_move) transformed_data = remap(transformed_data, visit=OpenWeatherMapTransformer.visit_rename) transformed_data = remap(transformed_data, visit=OpenWeatherMapTransformer.visit_remove) return transformed_data
def test_preprocessing(): for fixture in [ "tests/fixtures/inline_defined_objects.json", "tests/fixtures/minimal_example.json", ]: schema = load_base_schema(fixture).raw_schema def visit(path, key, value): if key == "schema" and "type" in value and value["type"] == "object": assert "x-name" in value return key, value remap(schema, visit=visit)
def iterate_files(directory): """Itreate over files in the given directory and change values.""" for filename in os.listdir(directory): if filename.endswith('.json'): config_json = json.load(open(directory + filename, 'r')) changed_values = remap(config_json, visit=change_val) drop_keys = lambda path, key, value: key not in bad_keys new_file = remap(changed_values, visit=drop_keys) out_file = open(directory + filename, 'w') # Over-write the input file # Separators added for no spaces b/w key/val in JSON. out_file.write( json.dumps(new_file, indent=2, separators=(',', ':'))) out_file.close()
def formatize(ndata, ignore=[], no_convert=[]): ''' Applies converters, if they match the name after hash. For example, if the key is '_:date#isoformat', then the converters.isoformat() is applied to the value, if exists. It returns keys without the # sign. no_convert: list of types, not to apply conversion, e.g., ['url', 'decimal'] ''' def visit(path, key, value): if isinstance(key, str): if '#' in str(key)[1:-1:]: if type(value) in [str, int, float]: Format = key.rsplit('#', 1)[-1] if hasattr(converters, Format) and Format not in no_convert: k = key.rsplit('#', 1)[0] v = getattr(converters, key.rsplit('#', 1)[-1])(value) if (k in ignore) or (key in ignore): return k, value return k, v return key.rsplit('#', 1)[0], value return key, value result = remap(ndata, visit=visit) if isinstance(ndata, list): result = List(result) return result
def by_tool_input(self, trans, tool_id, tool_version, param=None, param_dump=None, job_state='ok'): """Search for jobs producing same results using the 'inputs' part of a tool POST.""" user = trans.user input_data = defaultdict(list) def populate_input_data_input_id(path, key, value): """Traverses expanded incoming using remap and collects input_ids and input_data.""" if key == 'id': path_key = get_path_key(path[:-2]) current_case = param_dump for p in path: current_case = current_case[p] src = current_case['src'] current_case = param for i, p in enumerate(path): if p == 'values' and i == len(path) - 2: continue if isinstance(current_case, (list, dict)): current_case = current_case[p] identifier = getattr(current_case, "element_identifier", None) input_data[path_key].append({'src': src, 'id': value, 'identifier': identifier, }) return key, "__id_wildcard__" return key, value wildcard_param_dump = remap(param_dump, visit=populate_input_data_input_id) return self.__search(tool_id=tool_id, tool_version=tool_version, user=user, input_data=input_data, job_state=job_state, param_dump=param_dump, wildcard_param_dump=wildcard_param_dump)
def _item_to_csv(self, item): from boltons.iterutils import remap output = io.BytesIO() writer = self._create_csv_writer(output) item = remap(item, visit=self._encode_string) writer.writerow(item) return output.getvalue().rstrip()
def on_data(self, data): global path global logfile #print(json.loads(data)["text"]) tweet = json.loads(data) id_json = tweet["id_str"] json_doc = json.dumps(tweet, ensure_ascii=False) #json_tweet = json_doc.encode('utf8') #json_doc_v2 = json.dumps( (json.JSONDecoder().decode(data)).encode('utf8') ) print(type(json_doc)) #json_write = json.loads(json_doc,parse_float = Decimal) json_write = json.loads(json_doc) #dic_json = remap(dict(json_write), visit=drop_falsey) dic_json = remap(dict(json_write), visit=drop_falsey2) response = table.put_item( Item = json.loads(json.dumps(dic_json),parse_float = Decimal) ) print("PutItem succeeded:") print(json.dumps(response)) #print(dic_json) with open(path+logfile,"a+") as f: f.write("PutItem succeeded:") f.write(json.dumps(response)+"\n") return True
def get_file_report(file_sha256, report_url, environment_id, type_, apikey, secret, verify): user_agent = {'User-agent': 'VxStream Sandbox'} params = { 'apikey': apikey, 'secret': secret, 'environmentId': environment_id, 'type': type_ } resource_url = '%s/%s' % (report_url, file_sha256) try: res = requests.get(resource_url, headers=user_agent, params=params, verify=verify) if res.status_code == 200: # walk entire json blob to fix # the keys known to cause issues remapped = remap(res.json(), visit=visit) return remapped else: print('Error code: {}, returned when getting report: {}'.format( res.status_code, file_sha256)) return res except requests.exceptions.HTTPError as err: print(err)
def _deserialize(self, value, attr, obj, recurse_lvl=0): def visit(parent, key, _value): if isinstance(_value, dict) and self._BYTES_KEY in _value: return key, self._hex_list_to_binary(_value[self._BYTES_KEY]) return True return remap(value, visit=visit)
def preferred_locations(talent_pool_ids=None, candidate_id=None, preferred_location_id=None): """ :type talent_pool_ids: list[int] :rtype: dict[list] """ data = { 'candidates': [{ 'id': candidate_id, 'talent_pool_ids': { 'add': talent_pool_ids }, 'preferred_locations': [{ 'id': preferred_location_id, 'city': fake.city(), 'state': fake.state(), 'country_code': fake.country_code(), 'subdivision_code': 'US-CA' }] }] } # Recursively Remove keys with None values data = remap(data, lambda p, k, v: v is not None) return data
def military_services(talent_pool_ids=None, candidate_id=None, military_experience_id=None): """ :type talent_pool_ids: list[int] :rtype: dict[list] """ data = { 'candidates': [{ 'id': candidate_id, 'talent_pool_ids': { 'add': talent_pool_ids }, 'military_services': [{ 'id': military_experience_id, 'country_code': fake.country_code(), 'branch': fake.military_ship(), 'highest_rank': 'lieutenant', 'status': 'active', 'highest_grade': '0-1', 'comments': fake.bs(), 'from_date': '1974-5-25', 'to_date': '1996-12-12' }] }] } # Recursively Remove keys with None values data = remap(data, lambda p, k, v: v is not None) return data
def work_preference(talent_pool_ids=None, candidate_id=None, preference_id=None): data = { 'candidates': [{ 'id': candidate_id, 'talent_pool_ids': { 'add': talent_pool_ids }, 'work_preference': { 'id': preference_id, "relocate": False, "authorization": "US Citizen", "telecommute": True, "travel_percentage": randrange(0, 100), "hourly_rate": float('%.2f' % random.uniform(20, 90)), "salary": randrange(50000, 300000), "employment_type": "full-time employment", "security_clearance": None, "third_party": False } }] } # Recursively Remove keys with None values data = remap(data, lambda p, k, v: v is not None) return data
def phones(talent_pool_ids=None, candidate_id=None, phone_id=None, internationalize=False, extension=False): """ :type talent_pool_ids: list[int] :param internationalize: If True, the phone number value will be internationalized, e.g. +14085067789 :rtype: dict[list] """ # Generate phone number value = generate_international_phone_number( extension) if internationalize else sample_phone_number() data = { 'candidates': [{ 'id': candidate_id, 'talent_pool_ids': { 'add': talent_pool_ids }, 'phones': [{ 'id': phone_id, 'label': 'Work', 'value': value, 'is_default': False }] }] } # Recursively Remove keys with None values data = remap(data, lambda p, k, v: v is not None) return data
def round_floats(a_dict, precision): """Find all the floats in `a_dict` (recursive) and round them to `precision`.""" return remap( a_dict, lambda p, k, v: (k, round(v, precision)) if isinstance(v, float) else (k, v), )
def _validate(args, app_desc): if Core is None: raise Exception("Cannot validate file, pykwalify is not installed.") path = _find_config(args, app_desc) # Allow empty mapping (not allowed by pykwalify) raw_config = _order_load_path(path) if raw_config.get(app_desc.app_name) is None: raw_config[app_desc.app_name] = {} # Rewrite the file any way to merge any duplicate keys with tempfile.NamedTemporaryFile('w', delete=False, suffix=".yml") as config_p: ordered_dump(raw_config, config_p) def _clean(p, k, v): return k not in ['reloadable', 'path_resolves_to'] clean_schema = remap(app_desc.schema.raw_schema, _clean) with tempfile.NamedTemporaryFile('w', suffix=".yml") as fp: ordered_dump(clean_schema, fp) fp.flush() c = Core( source_file=config_p.name, schema_files=[fp.name], ) os.remove(config_p.name) c.validate()
def get_all_department_identifiers() -> [dict]: """ """ url = 'https://api.maui.uiowa.edu/maui/api/pub/registrar/program-of' \ '-study/program' headers = { 'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'application/json' } response = get(url=url, headers=headers) if response.status_code != 200: print('ERROR: HTTP{}'.format(response.status_code)) raise ValueError data = response.json() cleaned = remap( data, lambda p, k, v: v is not None and v != [] and v != '' and v != [{ }] and v != {}) filtered = list(map(lambda p: filter_keys(p, ['academicUnit']), cleaned)) filtered = list(map(lambda p: p['academicUnit'], filtered)) departments = list(map(lambda p: filter_keys(p, ['naturalKey']), filtered)) departments = list(map(lambda p: p['naturalKey'], departments)) pprint(departments) departments = list(sorted(list(set(departments)))) new_filtered = [] for item in departments: if len(item) > 1: new_filtered.append(item) return new_filtered
def test_prepop(self): """Demonstrating normalization and ID addition through prepopulating the objects wth an enter callback. """ base_obj = {'name': None, 'rank': None, 'id': 1} def enter(path, key, value): new_parent, new_items = default_enter(path, key, value) try: new_parent.update(base_obj) base_obj['id'] += 1 except: pass return new_parent, new_items orig = [{'name': 'Firefox', 'rank': 1}, {'name': 'Chrome', 'rank': 2}, {'name': 'IE'}] ref = [{'name': 'Firefox', 'rank': 1, 'id': 1}, {'name': 'Chrome', 'rank': 2, 'id': 2}, {'name': 'IE', 'rank': None, 'id': 3}] remapped = remap(orig, enter=enter) assert remapped == ref
def _validate(args, app_desc): path = _find_config(args, app_desc) # Allow empty mapping (not allowed by pykawlify) raw_config = _order_load_path(path) if raw_config.get(app_desc.app_name, None) is None: raw_config[app_desc.app_name] = {} config_p = tempfile.NamedTemporaryFile('w', delete=False, suffix=".yml") ordered_dump(raw_config, config_p) config_p.flush() path = config_p.name fp = tempfile.NamedTemporaryFile('w', delete=False, suffix=".yml") def _clean(p, k, v): return k != 'reloadable' clean_schema = remap(app_desc.schema.raw_schema, _clean) ordered_dump(clean_schema, fp) fp.flush() name = fp.name if Core is None: raise Exception("Cannot validate file, pykwalify is not installed.") c = Core( source_file=path, schema_files=[name], ) c.validate()
def sanitize_strings(data): def sanitize(p, k, v): if isinstance(v, str): return (k, bleach.clean(v, [ 'div', 'p', 'span', 'b', 'ul', 'li', 'ol', 'sub', 'sup', 'pre', 'a', 'blockquote', 'h1', 'h2', 'h3', 'h4', 'h5', 'strong', 'em', 'br', ], strip=True, strip_comments=True)) return k, v return remap(data, sanitize)
def test_dict_to_omd(self): def enter(path, key, value): if isinstance(value, dict): return OMD(), sorted(value.items()) return default_enter(path, key, value) orig = [{ 'title': 'Wild Palms', 'ratings': { 1: 1, 2: 3, 3: 5, 4: 6, 5: 3 } }, { 'title': 'Twin Peaks', 'ratings': { 1: 3, 2: 2, 3: 8, 4: 12, 5: 15 } }] remapped = remap(orig, enter=enter) assert remapped == orig assert isinstance(remapped[0], OMD) assert isinstance(remapped[0]['ratings'], OMD) assert isinstance(remapped[1], OMD) assert isinstance(remapped[1]['ratings'], OMD)
def members_dict(self, att_path_key: str, values: list): m_dict = self.__members_dict m_dict[att_path_key] = [] att_path_key_tuple = tuple(map(str, att_path_key.split('/'))) att_key = att_path_key_tuple[len(key_path_tuple) - 1] att_path = key_path_tuple[0:(len(key_path_tuple) - 1)] def visit(path, key, value): if path == att_path: if key == 'att_key': m_dict[att_path_key] = m_dict[att_path_key].append(value) for mm in self.members: remap(mm.__dict__, visit=visit)
def untag(data, locale=None, params=None): """Untags fields, handling translation priority.""" updated_localized_paths = set() paths_to_keep_tagged = set() def visit(path, key, value): """Function for each key and value in the data.""" if not isinstance(key, basestring): return key, value if (path, key.rstrip('@')) in updated_localized_paths: return False if key.endswith('@#'): return False if key.endswith('@'): if isinstance(value, list): paths_to_keep_tagged.add((path, key)) key = key[:-1] # Support <key>@<param key>.<param value>: <value>. if params: param_regex = re.compile( r'(.*)@({})\.([^@]+)$'.format('|'.join(params.keys()))) param_match = param_regex.match(key) if param_match: untagged_key, param_key, param_value = param_match.groups() param_value_regex = r'^{}$'.format(param_value) if not params[param_key] or not re.match(param_value_regex, params[param_key]): return False updated_localized_paths.add((path, untagged_key.rstrip('@'))) return untagged_key, value # Support <key>@<locale regex>: <value>. match = LOCALIZED_KEY_REGEX.match(key) if not match: updated_localized_paths.add((path, key)) return key, value untagged_key, locale_from_key = match.groups() locale_regex = r'^{}$'.format(locale_from_key) if not locale or not re.match(locale_regex, locale): return False updated_localized_paths.add((path, untagged_key.rstrip('@'))) return untagged_key, value # Backwards compatibility for https://github.com/grow/grow/issues/95 def exit(path, key, old_parent, new_parent, new_items): resp = iterutils.default_exit(path, key, old_parent, new_parent, new_items) if paths_to_keep_tagged and isinstance(resp, dict): for sub_key, value in resp.items(): if not isinstance(value, list): continue new_key = '{}@'.format(sub_key) resp[new_key] = value try: paths_to_keep_tagged.remove((path, key)) except KeyError: pass return resp return iterutils.remap(data, visit=visit, exit=exit)
def addresses(talent_pool_ids=None, candidate_id=None, address_id=None, is_default=False): """ :type talent_pool_ids: list[int] :param talent_pool_ids is required for creating candidate, but not for updating :type candidate_id: int | long :rtype: dict """ data = { 'candidates': [{ 'id': candidate_id, 'talent_pool_ids': { 'add': talent_pool_ids }, 'addresses': [{ 'id': address_id, 'address_line_1': fake.street_address(), 'city': fake.city(), 'subdivision_code': 'US-CA', 'zip_code': fake.zipcode(), 'country_code': fake.country_code(), 'is_default': is_default }] }] } # Remove keys with None values data = remap(data, lambda p, k, v: v is not None) return data
def test_sort_all_lists(self): def exit(path, key, old_parent, new_parent, new_items): # NB: in this case, I'd normally use *a, **kw ret = default_exit(path, key, old_parent, new_parent, new_items) if isinstance(ret, list): ret.sort() return ret # NB: Airplane model numbers (Boeing and Airbus) orig = [[[7, 0, 7], [7, 2, 7], [7, 7, 7], [7, 3, 7]], [[3, 8, 0], [3, 2, 0], [3, 1, 9], [3, 5, 0]]] ref = [[[0, 2, 3], [0, 3, 5], [0, 3, 8], [1, 3, 9]], [[0, 7, 7], [2, 7, 7], [3, 7, 7], [7, 7, 7]]] remapped = remap(orig, exit=exit) assert remapped == ref
def test_path(self): path_map = {} # test visit's path target_str = 'test' orig = [[[target_str]]] ref_path = (0, 0, 0) def visit(path, key, value): if value is target_str: path_map['target_str'] = path + (key,) return key, value remapped = remap(orig, visit=visit) assert remapped == orig assert path_map['target_str'] == ref_path # test enter's path target_obj = object() orig = {'a': {'b': {'c': {'d': ['e', target_obj, 'f']}}}} ref_path = ('a', 'b', 'c', 'd', 1) def enter(path, key, value): if value is target_obj: path_map['target_obj'] = path + (key,) return default_enter(path, key, value) remapped = remap(orig, enter=enter) assert remapped == orig assert path_map['target_obj'] == ref_path # test exit's path target_set = frozenset([1, 7, 3, 8]) orig = [0, 1, 2, [3, 4, [5, target_set]]] ref_path = (3, 2, 1) def exit(path, key, old_parent, new_parent, new_items): if old_parent is target_set: path_map['target_set'] = path + (key,) return default_exit(path, key, old_parent, new_parent, new_items) remapped = remap(orig, exit=exit) assert remapped == orig assert path_map['target_set'] == ref_path
def test_sub_selfref(self): coll = [0, 1, 2, 3] sub = [] sub.append(sub) coll.append(sub) with pytest.raises(RuntimeError): # if equal, should recurse infinitely assert coll == remap(coll)
def gather_for_locale(front_matter, locale): if not front_matter: return '' parsed = yaml.load(front_matter, Loader=PlainTextYamlLoader) locale_extra = OrderedDict() def visit(path, key, value): if not isinstance(key, basestring): return key, value if key.endswith('@#'): return key, value match = LOCALIZED_KEY_REGEX.match(key) if not match: return key, value base_key = match.group(1) locale_from_key = match.group(2) if locale_from_key == locale: # If there is a key without the trailing @ then override it. parent = parsed locale_parent = locale_extra for path_key in path: parent = parent[path_key] if isinstance(locale_parent, list): locale_parent = locale_parent[path_key] elif path_key not in locale_parent: if isinstance(parent, list): locale_parent[path_key] = copy.deepcopy(parent) else: locale_parent[path_key] = OrderedDict() locale_parent = locale_parent[path_key] else: locale_parent = locale_parent[path_key] if base_key in parent: locale_parent[base_key] = value else: locale_parent['{}@'.format(base_key)] = value if key in locale_parent: locale_parent.pop(key, None) return False return key, value parsed = iterutils.remap(parsed, visit=visit) return (yaml.dump( parsed, Dumper=PlainTextYamlDumper, allow_unicode=True, default_flow_style=False).strip() if parsed else '', locale_extra)
def untag(data, locale=None, env_name=None): """Untags fields, handling translation priority.""" updated_localized_paths = set() paths_to_keep_tagged = set() def visit(path, key, value): if not isinstance(key, basestring): return key, value if (path, key.rstrip('@')) in updated_localized_paths: return False if key.endswith('@#'): return False if key.endswith('@'): if isinstance(value, list): paths_to_keep_tagged.add((path, key)) key = key[:-1] # Support <key>@env.<name regex>: <value>. env_match = ENV_KEY_REGEX.match(key) if env_match: untagged_key, env_name_from_key = env_match.groups() env_regex = r'^{}$'.format(env_name_from_key) if not env_name or not re.match(env_regex, env_name): return False updated_localized_paths.add((path, untagged_key.rstrip('@'))) return untagged_key, value # Support <key>@<locale regex>: <value>. match = LOCALIZED_KEY_REGEX.match(key) if not match: updated_localized_paths.add((path, key)) return key, value untagged_key, locale_from_key = match.groups() locale_regex = r'^{}$'.format(locale_from_key) if not locale or not re.match(locale_regex, locale): return False updated_localized_paths.add((path, untagged_key.rstrip('@'))) return untagged_key, value # Backwards compatibility for https://github.com/grow/grow/issues/95 def exit(path, key, old_parent, new_parent, new_items): resp = iterutils.default_exit(path, key, old_parent, new_parent, new_items) if paths_to_keep_tagged and isinstance(resp, dict): for sub_key, value in resp.items(): if not isinstance(value, list): continue new_key = '{}@'.format(sub_key) resp[new_key] = value try: paths_to_keep_tagged.remove((path, key)) except KeyError: pass return resp return iterutils.remap(data, visit=visit, exit=exit)
def test_noncallables(self): with pytest.raises(TypeError): remap([], visit='test') with pytest.raises(TypeError): remap([], enter='test') with pytest.raises(TypeError): remap([], exit='test')
async def _do_args(self,db, args): seen = False dtid, = await self.db.DoFn("select id from data_type where tag=${tag}", tag=' '.join(args)) if self.options.last: if self.options.layer < 0: async for d in db.DoSelect("select * from data_log where data_log.data_type=${id} order by data_log.timestamp desc limit ${limit}", _dict=True, id=dtid, limit=self.options.last): if self.root.verbose > 1: if seen: print("===") add_human(d) pprint(remap(d, lambda p, k, v: v is not None)) else: print(d['id'],d['timestamp'],d['value'], sep='\t') seen = True else: async for d in db.DoSelect("select data_agg.* from data_agg join data_agg_type on data_agg.data_agg_type=data_agg_type.id where data_agg_type.data_type=${id} and data_agg_type.layer=${layer} order by data_agg.timestamp desc limit ${limit}", _dict=True, id=dtid, limit=self.options.last, layer=self.options.layer): if self.root.verbose > 1: if seen: print("===") add_human(d) pprint(remap(d, lambda p, k, v: v is not None)) else: print(d['id'],d['timestamp'],d['value'],d['n_values'], sep='\t') seen = True if not seen: print("No data?") else: if self.options.layer < 0: # display this type d = await self.db.DoFn("select * from data_type where id=${id}", _dict=True, id=dtid) if self.options.layer < -1: await self.ext_layer(db,d) add_human(d) pprint(remap(d, lambda p, k, v: v is not None)) else: # display this layer d = await self.db.DoFn("select * from data_agg_type where data_type=${id} and layer=${layer}", _dict=True, id=dtid, layer=self.options.layer) add_human(d) pprint(remap(d, lambda p, k, v: v is not None))
def flatten_fields(msg_proto, **kwargs): col_type_map = kwargs.pop('col_type_map', {}) default_col_type = kwargs.pop('default_col_type', 'text') sep = kwargs.pop('sep', SEP) if kwargs: raise TypeError('unexpected keyword arguments: %r' % kwargs.keys()) ret = [] def visit(path, key, value): if isinstance(value, (list, dict)): return True # traverse containers, but don't no columns for them full_path = path + (key,) col_type = col_type_map.get(full_path, default_col_type) full_path_str = sep.join([str(p) for p in full_path]) ret.append((full_path, full_path_str, col_type)) return True # read more about remap here: http://sedimental.org/remap.html remap(msg_proto, visit=visit) return sorted(ret, key=lambda x: x[0])
async def _do_unassigned(self,db,method): seen = False if method is None: mf = "IS NULL" elif method == "all": mf = "IS NOT NULL" else: mf = "= ${method}" try: method = modenames[method] except KeyError: print("Unknown method. Known:",' '.join(sorted(modenames.keys())), file=sys.stderr) return if self.options.last: async for d in db.DoSelect("select data_log.*,data_type.tag from data_log join data_type on data_type.id=data_log.data_type where data_type.method %s order by data_log.timestamp desc limit ${limit}" % (mf,), _dict=True, limit=self.options.last, method=method): if self.root.verbose > 1: if seen: print("===") add_human(d) pprint(remap(d, lambda p, k, v: v is not None)) else: print(d['id'],d['timestamp'],d['tag'],d['value'], sep='\t') seen = True else: async for d in db.DoSelect("select * from data_type where method %s and timestamp > '1999-12-31' order by n_values desc,tag" % (mf,), _dict=True, method=method): add_human(d) if self.root.verbose > 1: if seen: print("===") pprint(remap(d, lambda p, k, v: v is not None)) else: try: m = d['human']['method'][0] except KeyError: m = "-" print(d['n_values'],m,d['timestamp'],d['tag'], sep='\t') seen = True
def update_dataset_ids(input_values, translate_values, src): def replace_dataset_ids(path, key, value): """Exchanges dataset_ids (HDA, LDA, HDCA, not Dataset) in input_values with dataset ids used in job.""" current_case = input_values if key == 'id': for i, p in enumerate(path): if isinstance(current_case, (list, dict)): current_case = current_case[p] if src == current_case.get('src'): return key, translate_values.get(current_case['id'], value) return key, value return remap(input_values, visit=replace_dataset_ids)
def get_network_for_export(conn, network_id, include_data): # QC the template def visit(path, key, value): if key in set(['cr_date']): return False elif key in set(['id', 'template_id', 'type_id', 'attr_id']): return key, -1 return key, value network = conn.get_network(network_id, include_data=include_data) network = remap(dict(network), visit=visit) filename = "%s.json" % network['name'] content = json.dumps(network, sort_keys=True, indent=4, separators=(',', ': ')) return filename, content
def test_collector_pattern(self): all_interests = set() def enter(path, key, value): try: all_interests.update(value['interests']) except: pass return default_enter(path, key, value) orig = [{'name': 'Kate', 'interests': ['theater', 'manga'], 'dads': [{'name': 'Chris', 'interests': ['biking', 'python']}]}, {'name': 'Avery', 'interests': ['museums', 'pears'], 'dads': [{'name': 'Kurt', 'interests': ['python', 'recursion']}]}] ref = set(['python', 'recursion', 'biking', 'museums', 'pears', 'theater', 'manga']) remap(orig, enter=enter) assert all_interests == ref
def get_file_report(file_sha256, report_url, environment_id, type_, apikey, secret, verify): user_agent = {'User-agent': 'VxStream Sandbox'} params = {'apikey': apikey, 'secret': secret, 'environmentId': environment_id, 'type': type_} resource_url = '%s/%s' % (report_url, file_sha256) try: res = requests.get(resource_url, headers=user_agent, params=params, verify=verify) if res.status_code == 200: # walk entire json blob to fix # the keys known to cause issues remapped = remap(res.json(), visit=visit) return remapped else: print('Error code: {}, returned when getting report: {}'.format(res.status_code, file_sha256)) return res except requests.exceptions.HTTPError as err: print(err)
def test_dict_to_omd(self): def enter(path, key, value): if isinstance(value, dict): return OMD(), sorted(value.items()) return default_enter(path, key, value) orig = [{'title': 'Wild Palms', 'ratings': {1: 1, 2: 3, 3: 5, 4: 6, 5: 3}}, {'title': 'Twin Peaks', 'ratings': {1: 3, 2: 2, 3: 8, 4: 12, 5: 15}}] remapped = remap(orig, enter=enter) assert remapped == orig assert isinstance(remapped[0], OMD) assert isinstance(remapped[0]['ratings'], OMD) assert isinstance(remapped[1], OMD) assert isinstance(remapped[1]['ratings'], OMD)
def test_namedtuple(self): """TODO: this fails right now because namedtuples' __new__ is overridden to accept arguments. remap's default_enter tries to create an empty namedtuple and gets a TypeError. Could make it so that immutable types actually don't create a blank new parent and instead use the old_parent as a placeholder, creating a new one at exit-time from the value's __class__ (how default_exit works now). But even then it would have to *args in the values, as namedtuple constructors don't take an iterable. """ Point = namedtuple('Point', 'x y') point_map = {'origin': [Point(0, 0)]} with pytest.raises(TypeError): remapped = remap(point_map) assert isinstance(remapped['origin'][0], Point)
def modify_template(): if request.method == 'POST': template = AttrDict(request.json['template']) # QC the template def visit(path, key, value): if key in set(['cr_date']): return False #elif key in set(['id', 'template_id', 'type_id', 'attr_id']): #return key, None return key, value template = remap(dict(template), visit=visit) result = g.conn.call('update_template', {'tmpl': dict(template)}) if 'faultcode' in result: result = -1 return jsonify(result=result) return redirect(url_for('manager.manage_templates'))
def test_add_length(self): def exit(path, key, old_parent, new_parent, new_items): ret = default_exit(path, key, old_parent, new_parent, new_items) try: ret['review_length'] = len(ret['review']) except: pass return ret orig = {'Star Trek': {'TNG': {'stars': 10, 'review': "Episodic AND deep. <3 Data."}, 'DS9': {'stars': 8.5, 'review': "Like TNG, but with a story and no Data."}, 'ENT': {'stars': None, 'review': "Can't review what you can't watch."}}, 'Babylon 5': {'stars': 6, 'review': "Sophomoric, like a bitter laugh."}, 'Dr. Who': {'stars': None, 'review': "800 episodes is too many to review."}} remapped = remap(orig, exit=exit) assert (remapped['Star Trek']['TNG']['review_length'] < remapped['Star Trek']['DS9']['review_length'])
def convert_for_locale(front_matter, locale, base=None): if not front_matter: parsed = {} else: parsed = yaml.load(front_matter, Loader=PlainTextYamlLoader) def visit(path, key, value): if not isinstance(key, basestring): return key, value if key.endswith('@#'): return key, value match = LOCALIZED_KEY_REGEX.match(key) if not match: return key, value base_key = match.group(1) locale_from_key = match.group(2) if locale_from_key == locale: # If there is a key without the trailing @ then override it. parent = parsed for path_key in path: parent = parent[path_key] if base_key in parent: return base_key, value return '{}@'.format(base_key), value return False parsed = iterutils.remap(parsed, visit=visit) # If there are pre-existing fields, use them as a base for the locale # specific values. result = base or {} _update_deep(result, parsed) return result
def test_basic_upper(self): orig = {'a': 1, 'b': object(), 'c': {'d': set()}} remapped = remap(orig, lambda p, k, v: (k.upper(), v)) assert orig['a'] == remapped['A'] assert orig['b'] == remapped['B'] assert orig['c']['d'] == remapped['C']['D']
def test_item_drop(self): orig = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] even_items = remap(orig, lambda p, k, v: not (v % 2)) assert even_items == [0, 2, 4, 6, 8]