def _parse_with_config(self, parser): config = confuse.Configuration('kafl', modname='kafl_fuzzer') # check default config search paths config.read(defaults=True, user=True) # local / workdir config workdir_config = os.path.join(os.getcwd(), 'kafl.yaml') if os.path.exists(workdir_config): config.set_file(workdir_config, base_for_paths=True) # ENV based config if 'KAFL_CONFIG' in os.environ: config.set_file(os.environ['KAFL_CONFIG'], base_for_paths=True) # merge all configs into a flat dictionary, delimiter = ':' config_values = FlatDict(config.flatten()) if 'KAFL_CONFIG_DEBUG' in os.environ: print("Options picked up from config: %s" % str(config_values)) # adopt defaults into parser, fixup 'required' and file/path fields for action in parser._actions: #print("action: %s" % repr(action)) if action.dest in config_values: if action.type == parse_is_file: action.default = config[action.dest].as_filename() elif isinstance(action, argparse._AppendAction): assert ("append are not supported in in yaml config") #action.default = [config[action.dest].as_str()] else: action.default = config[action.dest].get() action.required = False config_values.pop(action.dest) # remove options not defined in argparse (set_defaults() imports everything) for option in config_values: if 'KAFL_CONFIG_DEBUG' in os.environ: logger.warn("Dropping unrecognized option '%s'." % option) config_values.pop(option) # allow unrecognized options? #parser.set_defaults(**config_values) args = parser.parse_args() if 'KAFL_CONFIG_DEBUG' in os.environ: print("Final parsed args: %s" % repr(args)) return args
def save(cls, network, phases=[], filename=''): r""" Write Network to a Mat file for exporting to Matlab. Parameters ---------- network : OpenPNM Network Object filename : string Desired file name, defaults to network name if not given phases : list of phase objects ([]) Phases that have properties we want to write to file """ project, network, phases = cls._parse_args(network=network, phases=phases) network = network[0] # Write to file if filename == '': filename = project.name filename = cls._parse_filename(filename=filename, ext='mat') d = Dict.to_dict(network=network, phases=phases, interleave=True) d = FlatDict(d, delimiter='|') d = sanitize_dict(d) new_d = {} for key in list(d.keys()): new_key = key.replace('|', '_').replace('.', '_') new_d[new_key] = d.pop(key) spio.savemat(file_name=filename, mdict=new_d)
def from_dict(cls, dct, project=None, delim=' | '): r""" This method converts a correctly formatted dictionary into OpenPNM objects, and returns a handle to the *project* containing them. Parameters ---------- dct : dictionary The Python dictionary containing the data. The nesting and labeling of the dictionary is used to create the appropriate OpenPNM objects. project : OpenPNM Project Object The project with which the created objects should be associated. If not supplied, one will be created. Returns ------- An OpenPNM Project containing the objects created to store the given data. Notes ----- The requirement of a *correctly formed* dictionary is rather strict, and essentially means a dictionary produced by the ``to_dict`` method of this class. """ if project is None: project = ws.new_project() # Uncategorize pore/throat and labels/properties, if present fd = FlatDict(dct, delimiter=delim) # If . is the delimiter, replace with | otherwise things break if delim == '.': delim = ' | ' for key in list(fd.keys()): new_key = key.replace('.', delim) fd[new_key] = fd.pop(key) d = FlatDict(delimiter=delim) for key in list(fd.keys()): new_key = key.replace('pore' + delim, 'pore.') new_key = new_key.replace('throat' + delim, 'throat.') new_key = new_key.replace('labels' + delim, '') new_key = new_key.replace('properties' + delim, '') d[new_key] = fd.pop(key) # Plase data into correctly categorized dicts, for later handling objs = { 'network': NestedDict(), 'geometry': NestedDict(), 'physics': NestedDict(), 'phase': NestedDict(), 'algorithm': NestedDict(), 'base': NestedDict() } for item in d.keys(): path = item.split(delim) if len(path) > 2: if path[-3] in objs.keys(): # Item is categorized by type, so note it objs[path[-3]][path[-2]][path[-1]] = d[item] else: # item is nested, not categorized; make it a base objs['base'][path[-2]][path[-1]] = d[item] else: # If not categorized by type, make it a base objs['base'][path[-2]][path[-1]] = d[item] # Convert to OpenPNM Objects, attempting to infer type for objtype in objs.keys(): for name in objs[objtype].keys(): # Create empty object, using dummy name to avoid error obj = project._new_object(objtype=objtype, name='') # Overwrite name obj._set_name(name=name, validate=False) # Update new object with data from dict obj.update(objs[objtype][name]) return project
def from_dict(cls, dct, project=None, delim=' | '): r""" This method converts a correctly formatted dictionary into OpenPNM objects, and returns a handle to the *project* containing them. Parameters ---------- dct : dictionary The Python dictionary containing the data. The nesting and labeling of the dictionary is used to create the appropriate OpenPNM objects. project : OpenPNM Project Object The project with which the created objects should be associated. If not supplied, one will be created. Returns ------- An OpenPNM Project containing the objects created to store the given data. Notes ----- The requirement of a *correctly formed* dictionary is rather strict, and essentially means a dictionary produced by the ``to_dict`` method of this class. """ if project is None: project = ws.new_project() # Uncategorize pore/throat and labels/properties, if present fd = FlatDict(dct, delimiter=delim) # If . is the delimiter, replace with | otherwise things break if delim == '.': delim = ' | ' for key in list(fd.keys()): new_key = key.replace('.', delim) fd[new_key] = fd.pop(key) d = FlatDict(delimiter=delim) for key in list(fd.keys()): new_key = key.replace('pore' + delim, 'pore.') new_key = new_key.replace('throat' + delim, 'throat.') new_key = new_key.replace('labels' + delim, '') new_key = new_key.replace('properties' + delim, '') d[new_key] = fd.pop(key) # Plase data into correctly categorized dicts, for later handling objs = {'network': NestedDict(), 'geometry': NestedDict(), 'physics': NestedDict(), 'phase': NestedDict(), 'algorithm': NestedDict(), 'base': NestedDict()} for item in d.keys(): path = item.split(delim) if len(path) > 2: if path[-3] in objs.keys(): # Item is categorized by type, so note it objs[path[-3]][path[-2]][path[-1]] = d[item] else: # item is nested, not categorized; make it a base objs['base'][path[-2]][path[-1]] = d[item] else: # If not categorized by type, make it a base objs['base'][path[-2]][path[-1]] = d[item] # Convert to OpenPNM Objects, attempting to infer type for objtype in objs.keys(): for name in objs[objtype].keys(): # Create empty object, using dummy name to avoid error obj = project._new_object(objtype=objtype, name='') # Overwrite name obj._set_name(name=name, validate=False) # Update new object with data from dict obj.update(objs[objtype][name]) return project
def to_dataframe(cls, network=None, phases=[], join=False, delim=' | '): r""" Convert the Network (and optionally Phase) data to Pandas DataFrames. Parameters ---------- network: OpenPNM Network Object The network containing the data to be stored phases : list of OpenPNM Phase Objects The data on each supplied phase will be added to DataFrame join : boolean If ``False`` (default), two DataFrames are returned with *pore* data in one, and *throat* data in the other. If ``True`` the pore and throat data are combined into a single DataFrame. This can be problematic as it will put NaNs into all the *pore* columns which are shorter than the *throat* columns. Returns ------- Pandas ``DataFrame`` object containing property and label data in each column. If ``join`` was False (default) the two DataFrames are returned i a named tuple, or else a single DataFrame with pore and throat data in the same file, despite the column length being different. """ project, network, phases = cls._parse_args(network=network, phases=phases) # Initialize pore and throat data dictionary using Dict class pdata = Dict.to_dict(network=network, phases=phases, element='pore', interleave=True, flatten=True, categorize_by=['object']) tdata = Dict.to_dict(network=network, phases=phases, element='throat', interleave=True, flatten=True, categorize_by=['object']) pdata = FlatDict(pdata, delimiter=delim) tdata = FlatDict(tdata, delimiter=delim) # Scan data and convert non-1d arrays to multiple columns for key in list(pdata.keys()): if sp.shape(pdata[key]) != (network[0].Np, ): arr = pdata.pop(key) tmp = sp.split(arr, arr.shape[1], axis=1) cols = range(len(tmp)) pdata.update( {key + '[' + str(i) + ']': tmp[i].squeeze() for i in cols}) for key in list(tdata.keys()): if sp.shape(tdata[key]) != (network[0].Nt, ): arr = tdata.pop(key) tmp = sp.split(arr, arr.shape[1], axis=1) cols = range(len(tmp)) tdata.update( {key + '[' + str(i) + ']': tmp[i].squeeze() for i in cols}) # Convert sanitized dictionaries to DataFrames pdata = pd.DataFrame(sanitize_dict(pdata)) tdata = pd.DataFrame(sanitize_dict(tdata)) # Prepare DataFrames to be returned if join: data = tdata.join(other=pdata, how='left') else: nt = namedtuple('dataframes', ('pore', 'throat')) data = nt(pore=pdata, throat=tdata) return data