def _parse_with_config(self, parser):
config = confuse.Configuration('kafl', modname='kafl_fuzzer')
# check default config search paths
config.read(defaults=True, user=True)
# local / workdir config
workdir_config = os.path.join(os.getcwd(), 'kafl.yaml')
if os.path.exists(workdir_config):
config.set_file(workdir_config, base_for_paths=True)
# ENV based config
if 'KAFL_CONFIG' in os.environ:
config.set_file(os.environ['KAFL_CONFIG'], base_for_paths=True)
# merge all configs into a flat dictionary, delimiter = ':'
config_values = FlatDict(config.flatten())
if 'KAFL_CONFIG_DEBUG' in os.environ:
print("Options picked up from config: %s" % str(config_values))
# adopt defaults into parser, fixup 'required' and file/path fields
for action in parser._actions:
#print("action: %s" % repr(action))
if action.dest in config_values:
if action.type == parse_is_file:
action.default = config[action.dest].as_filename()
elif isinstance(action, argparse._AppendAction):
assert ("append are not supported in in yaml config")
#action.default = [config[action.dest].as_str()]
else:
action.default = config[action.dest].get()
action.required = False
config_values.pop(action.dest)
# remove options not defined in argparse (set_defaults() imports everything)
for option in config_values:
if 'KAFL_CONFIG_DEBUG' in os.environ:
logger.warn("Dropping unrecognized option '%s'." % option)
config_values.pop(option)
# allow unrecognized options?
#parser.set_defaults(**config_values)
args = parser.parse_args()
if 'KAFL_CONFIG_DEBUG' in os.environ:
print("Final parsed args: %s" % repr(args))
return args
def save(cls, network, phases=[], filename=''):
r"""
Write Network to a Mat file for exporting to Matlab.
Parameters
----------
network : OpenPNM Network Object
filename : string
Desired file name, defaults to network name if not given
phases : list of phase objects ([])
Phases that have properties we want to write to file
"""
project, network, phases = cls._parse_args(network=network,
phases=phases)
network = network[0]
# Write to file
if filename == '':
filename = project.name
filename = cls._parse_filename(filename=filename, ext='mat')
d = Dict.to_dict(network=network, phases=phases, interleave=True)
d = FlatDict(d, delimiter='|')
d = sanitize_dict(d)
new_d = {}
for key in list(d.keys()):
new_key = key.replace('|', '_').replace('.', '_')
new_d[new_key] = d.pop(key)
spio.savemat(file_name=filename, mdict=new_d)
def save(cls, network, phases=[], filename=''):
r"""
Write Network to a Mat file for exporting to Matlab.
Parameters
----------
network : OpenPNM Network Object
filename : string
Desired file name, defaults to network name if not given
phases : list of phase objects ([])
Phases that have properties we want to write to file
"""
project, network, phases = cls._parse_args(network=network,
phases=phases)
network = network[0]
# Write to file
if filename == '':
filename = project.name
filename = cls._parse_filename(filename=filename, ext='mat')
d = Dict.to_dict(network=network, phases=phases, interleave=True)
d = FlatDict(d, delimiter='|')
d = sanitize_dict(d)
new_d = {}
for key in list(d.keys()):
new_key = key.replace('|', '_').replace('.', '_')
new_d[new_key] = d.pop(key)
spio.savemat(file_name=filename, mdict=new_d)
def from_dict(cls, dct, project=None, delim=' | '):
r"""
This method converts a correctly formatted dictionary into OpenPNM
objects, and returns a handle to the *project* containing them.
Parameters
----------
dct : dictionary
The Python dictionary containing the data. The nesting and
labeling of the dictionary is used to create the appropriate
OpenPNM objects.
project : OpenPNM Project Object
The project with which the created objects should be associated.
If not supplied, one will be created.
Returns
-------
An OpenPNM Project containing the objects created to store the given
data.
Notes
-----
The requirement of a *correctly formed* dictionary is rather strict,
and essentially means a dictionary produced by the ``to_dict`` method
of this class.
"""
if project is None:
project = ws.new_project()
# Uncategorize pore/throat and labels/properties, if present
fd = FlatDict(dct, delimiter=delim)
# If . is the delimiter, replace with | otherwise things break
if delim == '.':
delim = ' | '
for key in list(fd.keys()):
new_key = key.replace('.', delim)
fd[new_key] = fd.pop(key)
d = FlatDict(delimiter=delim)
for key in list(fd.keys()):
new_key = key.replace('pore' + delim, 'pore.')
new_key = new_key.replace('throat' + delim, 'throat.')
new_key = new_key.replace('labels' + delim, '')
new_key = new_key.replace('properties' + delim, '')
d[new_key] = fd.pop(key)
# Plase data into correctly categorized dicts, for later handling
objs = {
'network': NestedDict(),
'geometry': NestedDict(),
'physics': NestedDict(),
'phase': NestedDict(),
'algorithm': NestedDict(),
'base': NestedDict()
}
for item in d.keys():
path = item.split(delim)
if len(path) > 2:
if path[-3] in objs.keys():
# Item is categorized by type, so note it
objs[path[-3]][path[-2]][path[-1]] = d[item]
else:
# item is nested, not categorized; make it a base
objs['base'][path[-2]][path[-1]] = d[item]
else:
# If not categorized by type, make it a base
objs['base'][path[-2]][path[-1]] = d[item]
# Convert to OpenPNM Objects, attempting to infer type
for objtype in objs.keys():
for name in objs[objtype].keys():
# Create empty object, using dummy name to avoid error
obj = project._new_object(objtype=objtype, name='')
# Overwrite name
obj._set_name(name=name, validate=False)
# Update new object with data from dict
obj.update(objs[objtype][name])
return project
def from_dict(cls, dct, project=None, delim=' | '):
r"""
This method converts a correctly formatted dictionary into OpenPNM
objects, and returns a handle to the *project* containing them.
Parameters
----------
dct : dictionary
The Python dictionary containing the data. The nesting and
labeling of the dictionary is used to create the appropriate
OpenPNM objects.
project : OpenPNM Project Object
The project with which the created objects should be associated.
If not supplied, one will be created.
Returns
-------
An OpenPNM Project containing the objects created to store the given
data.
Notes
-----
The requirement of a *correctly formed* dictionary is rather strict,
and essentially means a dictionary produced by the ``to_dict`` method
of this class.
"""
if project is None:
project = ws.new_project()
# Uncategorize pore/throat and labels/properties, if present
fd = FlatDict(dct, delimiter=delim)
# If . is the delimiter, replace with | otherwise things break
if delim == '.':
delim = ' | '
for key in list(fd.keys()):
new_key = key.replace('.', delim)
fd[new_key] = fd.pop(key)
d = FlatDict(delimiter=delim)
for key in list(fd.keys()):
new_key = key.replace('pore' + delim, 'pore.')
new_key = new_key.replace('throat' + delim, 'throat.')
new_key = new_key.replace('labels' + delim, '')
new_key = new_key.replace('properties' + delim, '')
d[new_key] = fd.pop(key)
# Plase data into correctly categorized dicts, for later handling
objs = {'network': NestedDict(),
'geometry': NestedDict(),
'physics': NestedDict(),
'phase': NestedDict(),
'algorithm': NestedDict(),
'base': NestedDict()}
for item in d.keys():
path = item.split(delim)
if len(path) > 2:
if path[-3] in objs.keys():
# Item is categorized by type, so note it
objs[path[-3]][path[-2]][path[-1]] = d[item]
else:
# item is nested, not categorized; make it a base
objs['base'][path[-2]][path[-1]] = d[item]
else:
# If not categorized by type, make it a base
objs['base'][path[-2]][path[-1]] = d[item]
# Convert to OpenPNM Objects, attempting to infer type
for objtype in objs.keys():
for name in objs[objtype].keys():
# Create empty object, using dummy name to avoid error
obj = project._new_object(objtype=objtype, name='')
# Overwrite name
obj._set_name(name=name, validate=False)
# Update new object with data from dict
obj.update(objs[objtype][name])
return project
def to_dataframe(cls, network=None, phases=[], join=False, delim=' | '):
r"""
Convert the Network (and optionally Phase) data to Pandas DataFrames.
Parameters
----------
network: OpenPNM Network Object
The network containing the data to be stored
phases : list of OpenPNM Phase Objects
The data on each supplied phase will be added to DataFrame
join : boolean
If ``False`` (default), two DataFrames are returned with *pore*
data in one, and *throat* data in the other. If ``True`` the pore
and throat data are combined into a single DataFrame. This can be
problematic as it will put NaNs into all the *pore* columns which
are shorter than the *throat* columns.
Returns
-------
Pandas ``DataFrame`` object containing property and label data in each
column. If ``join`` was False (default) the two DataFrames are
returned i a named tuple, or else a single DataFrame with pore and
throat data in the same file, despite the column length being
different.
"""
project, network, phases = cls._parse_args(network=network,
phases=phases)
# Initialize pore and throat data dictionary using Dict class
pdata = Dict.to_dict(network=network,
phases=phases,
element='pore',
interleave=True,
flatten=True,
categorize_by=['object'])
tdata = Dict.to_dict(network=network,
phases=phases,
element='throat',
interleave=True,
flatten=True,
categorize_by=['object'])
pdata = FlatDict(pdata, delimiter=delim)
tdata = FlatDict(tdata, delimiter=delim)
# Scan data and convert non-1d arrays to multiple columns
for key in list(pdata.keys()):
if sp.shape(pdata[key]) != (network[0].Np, ):
arr = pdata.pop(key)
tmp = sp.split(arr, arr.shape[1], axis=1)
cols = range(len(tmp))
pdata.update(
{key + '[' + str(i) + ']': tmp[i].squeeze()
for i in cols})
for key in list(tdata.keys()):
if sp.shape(tdata[key]) != (network[0].Nt, ):
arr = tdata.pop(key)
tmp = sp.split(arr, arr.shape[1], axis=1)
cols = range(len(tmp))
tdata.update(
{key + '[' + str(i) + ']': tmp[i].squeeze()
for i in cols})
# Convert sanitized dictionaries to DataFrames
pdata = pd.DataFrame(sanitize_dict(pdata))
tdata = pd.DataFrame(sanitize_dict(tdata))
# Prepare DataFrames to be returned
if join:
data = tdata.join(other=pdata, how='left')
else:
nt = namedtuple('dataframes', ('pore', 'throat'))
data = nt(pore=pdata, throat=tdata)
return data
