def test_make_splitter(normal_dict, delimiter, delimiter_equivalent): splitter = make_splitter(delimiter) flat_dict = flatten(normal_dict, delimiter_equivalent) unflattened_dict_using_make_splitter = unflatten(flat_dict, splitter=splitter) unflattened_dict_using_equivalent_splitter = unflatten( flat_dict, splitter=delimiter_equivalent) assert unflattened_dict_using_make_splitter == unflattened_dict_using_equivalent_splitter
def load_from_config(find_dict, prefix=None, path=default_config_path, allow_sub_keys=False): if not prefix: prefix = tuple() config = load_config(path=path) if not config: return {} found_config_values = {} flat_find_dict = flatten_dict.flatten(find_dict, keep_empty_types=(dict, )) flat_config = flatten_dict.flatten(config, keep_empty_types=(dict, )) for find_key, _ in flat_find_dict.items(): for flat_key, flat_value in flat_config.items(): prefixed_key = prefix + find_key intersection = tuple([ v for i, v in enumerate(prefixed_key) if i < len(flat_key) and v == flat_key[i] ]) difference = tuple([v for v in flat_key if v not in intersection]) # HACK, Only append differences at the current depth of the config, # e.g. don't allow subdict append if prefixed_key == intersection and (allow_sub_keys or not difference): sub_key = find_key + difference found_config_values[sub_key] = flat_value return flatten_dict.unflatten(found_config_values)
def __init__(self, configfile: str, fix_values, fix_missing, write=False): """ Creates a config object from a json file :param configfile: the config filename (json) :param fix_values: a function to resolve values of the wrong type :param fix_missing: a function to resolve missing values :param write: writes changes (fix_values and fix_missing function results) to the file """ parsed = load_config(configfile) flat = flatten(parsed, 'path') alloweds = flatten(internalconf.CONFIG_TYPES, 'path') buffer = dict() for key, _type in alloweds.items(): if key not in flat.keys() or key is None: value = fix_missing(key) elif type(flat[key]) is not _type: value = fix_values(key) else: value = flat[key] buffer[key] = value if write: with open(config_path(configfile), 'w') as f: logging.info("Writing changes to the config file...") f.write(json.dumps(unflatten(buffer, 'path'), indent=4)) # On windows, flatten uses \\ for paths instead of /. Fix that. self._dict = {k.replace('\\','/'):v for k,v in buffer.items()}
def test_unflatten_dict_underscore(): flat_underscore_dict = { '_'.join(k): v for k, v in six.viewitems(flat_normal_dict) } assert unflatten(flat_underscore_dict, splitter='underscore') == normal_dict
def load_missing_action_kwargs(kwargs_configurations): flat_action_kwargs = {} for kwargs in kwargs_configurations: for group, args in kwargs.items(): # Set group prefixes kwargs_path = (group, ) if kwargs_path not in flat_action_kwargs: flat_action_kwargs[kwargs_path] = {} input_dict = find_value_in_dict(args, key="action_kwargs") required_fields = find_value_in_dict(args, key="valid_action_config") missing_dict = missing_fields(input_dict, required_fields) action_kwargs_flat_path = get_dict_path(args, key="action_kwargs", truncate=True) if action_kwargs_flat_path: dict_flat_path = action_kwargs_flat_path[:-1] if kwargs_path != dict_flat_path: kwargs_path = kwargs_path + dict_flat_path if kwargs_path not in flat_action_kwargs: flat_action_kwargs[kwargs_path] = {} # Fill in the provided arguments flat_action_kwargs[kwargs_path] = input_dict config_prefix = find_value_in_dict(args, key="config_prefix") # Update with missing arguments from config loaded_from_config = load_from_config(missing_dict, prefix=config_prefix) flat_action_kwargs[kwargs_path].update(loaded_from_config) return flatten_dict.unflatten(flat_action_kwargs)
def update(d: dict, path: str, value: Any): if not has(d, path): return d keys = tuple(path.split('.')) d_flat = flatten(d) d_flat[keys] = value return unflatten(d_flat)
def update(self, path, value): if not self.loaded: raise ConfigNotLoaded("Configuration file not loaded.") keys = path.split(':') flat = flatten(self.configuration) wlist = [] for key in keys: wlist.append(key) # # If a dictionary was passed in update all values contained in the dictionary # are updated in the configuration if not isinstance(value,dict): flat[ tuple(wlist)] = value else: for key in value: listkey = wlist.copy() listkey.append(key) flat[ tuple(listkey)] = value[key] self.configuration = unflatten(flat) self.modified = True self.save_config() return
def config_to_dict(self): """return the configuration as an unflattened dict""" server = self.server_config.create_mapping( self.server_config.default_config) dataset = self.default_dataset_config.create_mapping( self.default_dataset_config.default_config) external = self.external_config.create_mapping( self.external_config.default_config) config = dict(server={}, dataset={}) for attrname in server.keys(): config["server__" + attrname] = getattr(self.server_config, attrname) for attrname in dataset.keys(): config["dataset__" + attrname] = getattr( self.default_dataset_config, attrname) if self.dataroot_config: config["per_dataset_config"] = {} for dataroot_tag, dataroot_config in self.dataroot_config.items(): dataset = dataroot_config.create_mapping( dataroot_config.default_config) for attrname in dataset.keys(): config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname) for attrname in external.keys(): config["external__" + attrname] = getattr(self.external_config, attrname) config = unflatten(config, splitter=lambda key: key.split("__")) return config
def update_data(file_path_to_write: str, file_path_to_read: str, file_ending: str) -> None: """ Collects special chosen fields from the file_path_to_read and writes them into the file_path_to_write. :param file_path_to_write: The output file path to add the special fields to. :param file_path_to_read: The input file path to read the special fields from. :param file_ending: The files ending :return: None """ pack_obj_data, _ = get_dict_from_file(file_path_to_read) fields: list = DELETED_YML_FIELDS_BY_DEMISTO if file_ending == 'yml' else DELETED_JSON_FIELDS_BY_DEMISTO # Creates a nested-complex dict of all fields to be deleted by Demisto. # We need the dict to be nested, to easily merge it later to the file data. preserved_data: dict = unflatten( { field: dictor(pack_obj_data, field) for field in fields if dictor(pack_obj_data, field) }, splitter='dot') if file_ending == 'yml': with open(file_path_to_write, 'r') as yf: file_yaml_object = yaml.load(yf) if pack_obj_data: merge(file_yaml_object, preserved_data) with open(file_path_to_write, 'w') as yf: yaml.dump(file_yaml_object, yf) elif file_ending == 'json': file_data: dict = get_json(file_path_to_write) if pack_obj_data: merge(file_data, preserved_data) with open(file_path_to_write, 'w') as jf: json.dump(obj=file_data, fp=jf, indent=4)
def uncollate(tbl: dict) -> List[dict]: tbl = {k: list(v) for k, v in flatten(tbl).items()} return [ unflatten({k: v[i].unsqueeze(0) for k, v in tbl.items()}) for i in range(len(next(iter(tbl.values())))) ]
def delete(d: dict, path: str): if not has(d, path): return d keys = tuple(path.split('.')) d_flat = flatten(d) del d_flat[keys] return unflatten(d_flat)
def prepare_kwargs_configurations(provider, args, argument_groups, strip_group_prefix=True): """ Used to load missing arguments from the configuration file """ # Try to find all available args kwargs_configurations = [] for group in argument_groups: group_kwargs_config = {} name = group.lower() # Flat group_kwargs_config to do direct indexing if "_" in name: prefix = tuple(name.split("_")) group_kwargs_config = create_sub_dictionaries( group_kwargs_config, prefix) else: prefix = (name, ) group_kwargs_config[name] = {} prefix_action_kwargs = prefix + ("action_kwargs", ) prefix_action_config = prefix + ("valid_action_config", ) prefix_config_prefix = prefix + ("config_prefix", ) flat_group_kwargs_config = flatten_dict.flatten( group_kwargs_config, keep_empty_types=(dict, )) # TODO, subname on split prefix action_kwargs = vars(extract_arguments(args, [group])) if action_kwargs: # remove claimed action_kwargs from args args = remove_arguments(args, action_kwargs.keys(), prefix=name + "_") flat_group_kwargs_config[prefix_action_kwargs] = action_kwargs else: flat_group_kwargs_config[prefix_action_kwargs] = {} if group in corc_config_groups: valid_action_config = corc_config_groups[group] # gen config prefix config_prefix = gen_config_prefix(prefix) flat_group_kwargs_config[ prefix_action_config] = valid_action_config flat_group_kwargs_config[prefix_config_prefix] = config_prefix provider_groups = get_provider_config_groups(provider) if group in provider_groups: valid_action_config = provider_groups[group] prefix = (provider, ) + prefix config_prefix = gen_config_provider_prefix(prefix) flat_group_kwargs_config[ prefix_action_config] = valid_action_config flat_group_kwargs_config[prefix_config_prefix] = config_prefix if (prefix_action_config in flat_group_kwargs_config and flat_group_kwargs_config[prefix_action_config]): unflat_group_group_kwargs_config = flatten_dict.unflatten( flat_group_kwargs_config) kwargs_configurations.append(unflat_group_group_kwargs_config) return kwargs_configurations
def undo(self) -> None: section: TableauSection = self.get_section() cell = self.get_cell() # self.redo_command: str = "remove_one_column" prev = flatten(cell) content = flatten(self.content) to_update = unflatten({k: prev[k] for k in content}) section.update_cell(self.y, self.x, to_update, self.cursor_avant)
def _get_public_claims(self): public_claims = { k.replace(JWT.config.public_claim_namespace, ""): v for k, v in self.raw_data.items() if k.startswith(JWT.config.public_claim_namespace) and k not in ("iss", "sub", "aud", "exp", "nbf", "iat", "jti") } return unflatten(public_claims, splitter="path")
def sample(self): if self._current_observation is None: self._current_observation = self.env.reset() policy_input = flatten_input_structure({ key: self._current_observation[key][None, ...] for key in self.policy.observation_keys }) action = self.policy.actions_np(policy_input)[0] next_observation, reward, terminal, info = self.env.step(action) self._path_length += 1 self._path_return += reward self._total_samples += 1 processed_sample = self._process_sample( observation=self._current_observation, action=action, reward=reward, terminal=terminal, next_observation=next_observation, info=info, ) for key, value in flatten(processed_sample).items(): self._current_path[key].append(value) if terminal or self._path_length >= self._max_path_length: last_path = unflatten({ field_name: np.array(values) for field_name, values in self._current_path.items() }) self.pool.add_path({ key: value for key, value in last_path.items() if key != 'infos' }) self._last_n_paths.appendleft(last_path) self._max_path_return = max(self._max_path_return, self._path_return) self._last_path_return = self._path_return self.policy.reset() self.pool.terminate_episode() self._current_observation = None self._path_length = 0 self._path_return = 0 self._current_path = defaultdict(list) self._n_episodes += 1 else: self._current_observation = next_observation return next_observation, reward, terminal, info
def add_sample(self, sample): sample_flat = flatten(sample) samples_flat = type(sample)([ (field_name_flat, np.array(sample_flat[field_name_flat])[None, ...]) for field_name_flat in sample_flat.keys() ]) samples = unflatten(samples_flat) self.add_samples(samples)
def missing_fields(input_dict, required_fields, verbose=False, throw=False): missing = {} flat_input_fields = flatten_dict.flatten(input_dict) flat_required_fields = flatten_dict.flatten(required_fields) for k, v in flat_required_fields.items(): # Not present or not set if not present_in(k, flat_input_fields) or not flat_input_fields[k]: missing[k] = None return flatten_dict.unflatten(missing)
def REPLACE_FULL_OBSERVATION(original_batch, resampled_batch, where_resampled, environment): batch_flat = flatten(original_batch) resampled_batch_flat = flatten(resampled_batch) goal_keys = [key for key in batch_flat.keys() if key[0] == 'goals'] for key in goal_keys: assert (batch_flat[key][where_resampled].shape == resampled_batch_flat[key].shape) batch_flat[key][where_resampled] = (resampled_batch_flat[key]) return unflatten(batch_flat)
def _update_config(current_config_json, value={}): changes = False current_config = current_config_json.copy() flatten_config = flatten(current_config) for k, v in flatten(value).items(): if k in flatten_config: flatten_config[k] = v updated_config = unflatten(flatten_config) if flatten(current_config_json) != flatten_config: changes = True return updated_config, changes
def generate_oci_config(overwrite_with_empty=False, **kwargs): config = default_oci_config if kwargs: flat = flatten_dict.flatten(config) other_flat = flatten_dict.flatten(kwargs) for k, v in other_flat.items(): if not v and overwrite_with_empty: flat[k] = v if v: flat[k] = v config = flatten_dict.unflatten(flat) return config
def _update_config(config, update): config = flatten(config) update = flatten(update) for opt, val2 in update.items(): if isinstance(val2, list) and opt in config: val1 = config[opt] assert isinstance(val1, list) config[opt] = val1 + val2 # Append update for lists else: config[opt] = val2 return unflatten(config)
def _relabel_batch(self, batch, indices, her_strategy): batch_size = indices.size batch['resampled_distances'] = np.full((batch_size, 1), np.float('inf')) batch['resampled'] = np.zeros((batch_size, 1), dtype='bool') if her_strategy: her_strategy_type = self._her_strategy['type'] goal_resampling_probability = self._her_strategy[ 'resampling_probability'] to_resample_mask = (np.random.rand(batch_size) < goal_resampling_probability) where_resampled = np.flatnonzero(to_resample_mask) to_resample_indices = indices[where_resampled] episode_first_distances = -1 * batch['episode_index_forwards'][ where_resampled] episode_last_distances = batch['episode_index_backwards'][ where_resampled] resampled_indices, resampled_distances = self._resample_indices( to_resample_indices, episode_first_distances, episode_last_distances, her_strategy_type) resampled_batch_flat = flatten( super(HindsightExperienceReplayPool, self).batch_by_indices(indices=resampled_indices, field_name_filter=None)) batch_flat = flatten(batch) goal_keys = [key for key in batch_flat.keys() if key[0] == 'goals'] for key in goal_keys: assert (batch_flat[key][where_resampled].shape == resampled_batch_flat[key].shape) batch_flat[key][where_resampled] = (resampled_batch_flat[key]) if self._reward_function: batch_flat[('rewards', )][where_resampled] = ( self._reward_function(resampled_batch_flat)) if self._terminal_function: batch_flat[('terminals', )][where_resampled] = ( self._terminal_function(resampled_batch_flat)) batch = unflatten(batch_flat) batch['resampled_distances'][where_resampled] = ( resampled_distances) batch['resampled'][where_resampled] = True return batch
def load_manifest(self): containers_tuple = ('spec', 'template', 'spec', 'containers') init_containers_tuple = ('spec', 'template', 'spec', 'initContainers') self.logger.info(f"Loading manifest {self.generic_yaml.absolute()}") with open(self.generic_yaml, 'r') as stream: try: docs: List = list(yaml.safe_load_all(stream)) cont_docs = containers(docs) for i in range(0, len(docs)): doc = docs[i] if doc is not None and doc in cont_docs: self.logger.debug("Processing doc kind : {}".format( doc["kind"])) name = None try: doc_flat = flatten(doc) for c in doc_flat[containers_tuple]: name = c['name'] image = c["image"] image_inter = self.interpolate_image( self.variables, image) c['image'] = image_inter try: i_c = doc_flat[init_containers_tuple] for c in i_c: image = c['image'] image_inter = self.interpolate_image( self.variables, image) c['image'] = image_inter except KeyError as e: self.logger.error( f"initContainers not present for {name}") doc = unflatten(doc_flat) with open( Path(self.kustomize_builds_dir, name + '_inter.yaml'), 'w') as outfile: yaml.dump(doc, outfile, default_flow_style=False) except KeyError as e: self.logger.error(f"image is empty dict") docs[i] = doc with open(self.interpolated_yaml, 'w') as outfile: self.logger.info( f"Writing {self.interpolated_yaml.resolve()}") yaml.dump_all(docs, outfile, default_flow_style=False) except yaml.YAMLError as e: self.logger.error(e)
def REPLACE_FLAT_OBSERVATION(original_batch, resampled_batch, where_resampled, environment): batch_flat = flatten(original_batch) resampled_batch_flat = flatten(resampled_batch) observation_keys = [ key for key in batch_flat.keys() if key[0] == 'observations' or key[0] == 'next_observations' ] for key in observation_keys: state_size = int(batch_flat[key].shape[1] / 2) batch_flat[key][where_resampled][state_size:] = ( resampled_batch_flat[key][state_size:]) return unflatten(batch_flat)
def set_in_config(set_dict, prefix=None, path=default_config_path): if not prefix: prefix = tuple() config = load_config(path=path) if not config: return False flat_set_dict = flatten_dict.flatten(set_dict, keep_empty_types=(dict, )) flat_config = flatten_dict.flatten(config, keep_empty_types=(dict, )) for set_key, set_value in flat_set_dict.items(): flat_config[prefix + set_key] = set_value unflatten_dict = flatten_dict.unflatten(flat_config) return update_config(unflatten_dict, path=path)
def generate_atom(pages): # Reformat pages dict to be a list of every unique page that has a date, # sorted by date and capped to 10 entries a = flatten(pages, reducer=tab_reducer) a = { k.replace('\t', '_', k.count('\t') - 1): v for k, v in a.items() if k.count('\t') > 1 } a = unflatten(a, splitter=tab_splitter) a = list(a.values()) a = list(filter(lambda x: 'date' in x, a)) a = sorted(a, key=lambda x: x['date'], reverse=True) a = a[:10] return render_template('atom.xml', entries=a, lastupdated=a[0]['date'])
def merge_toml_dict(self) -> JsonDict: """Merge all included styles into a TOML (actually JSON) dictionary.""" merged_dict = unflatten(self._merged_styles, custom_splitter(SEPARATOR_FLATTEN)) # TODO: fix: check if the merged style file is still needed merged_style_path: Path = self.cache_dir / MERGED_STYLE_TOML toml = TomlDoc(obj=merged_dict) attempt = 1 while attempt < 5: try: merged_style_path.write_text(toml.reformatted) break except OSError: attempt += 1 return merged_dict
def config_file_update(): """ Function to determine if we need to update our yaml configuration file after an upgrade. """ log.debug('config_file_update() Started....') if os.path.isfile(skel_config_file): with open(config_file, 'r') as current_config: current_config = yaml.safe_load(current_config) with open(skel_config_file, 'r') as temp_config: temp_config = yaml.safe_load(temp_config) temp_current_config = flatten(current_config) temp_temp_config = flatten(temp_config) updates = (dict((k, v) for k, v in temp_temp_config.items() if k not in temp_current_config)) if updates != {}: copyfile(skel_config_file, (str(Path.home()) + '/.config/plot_manager/Config_Instructions.yaml')) copyfile(config_file, ( str(Path.home()) + f'/.config/plot_manager/plot_manager.yaml.{current_military_time}' )) temp_current_config.update(updates) new_config = (dict((k, v) for k, v in temp_current_config.items() if k in temp_temp_config)) else: new_config = (dict((k, v) for k, v in temp_current_config.items() if k not in temp_temp_config)) if new_config != {}: new_config = (dict((k, v) for k, v in temp_current_config.items() if k in temp_temp_config)) current_config = unflatten(new_config) current_config.update({'configured': False}) with open( (str(Path.home()) + '/.config/plot_manager/plot_manager.yaml'), 'w') as f: yaml.safe_dump(current_config, f) log.debug( f'Config File: {config_file} updated. Update as necessary to run this script.' ) exit() else: log.debug('No config file changes necessary! No changes made.') else: log.debug('New configuration file not found. No changes made.')
def update_shared_dict(shared_values_dict: Dict[str, Any], new_info_dict: Dict[str, Any]) -> None: """Update the dictionary and log any critical changes. Because this is a nested dictionary, make sure to flatten and then unflatten it to ensure full updates. """ flattened_new_dict = flatten(new_info_dict) flattened_shared_dict = flatten(shared_values_dict) flattened_shared_dict.update(flattened_new_dict) updated_shared_dict = unflatten(flattened_shared_dict) shared_values_dict.update(updated_shared_dict) new_recording_directory: Optional[str] = attempt_to_get_recording_directory_from_new_dict(new_info_dict) if new_recording_directory is not None: scrubbed_recordings_dir = redact_sensitive_info_from_path(new_recording_directory) msg = f"Using directory for recording files: {scrubbed_recordings_dir}" logger.info(msg)
def batch_by_indices(self, indices, field_name_filter=None): if np.any(indices % self._max_size > self.size): raise ValueError( "Tried to retrieve batch with indices greater than current" " size") field_names_flat = self.fields_flat.keys() if field_name_filter is not None: field_names_flat = self.filter_fields(field_names_flat, field_name_filter) batch_flat = { field_name: self.data[field_name][indices] for field_name in field_names_flat } batch = unflatten(batch_flat) return batch