def __init__(self, defaults, config='config.yaml', verbose=True):
super().__init__()
self.__verbose = verbose
self.__settings = {'locations': get_locations(), 'files': dict()}
loaders = [(
'!str_join',
yaml_str_join,
), (
'!loc_join',
yaml_loc_join,
)]
defaults, sdict = ('startup import', defaults) if isinstance(
defaults, dict) else (search_location(defaults), None)
if not self.load(
'defaults', defaults, sdict=sdict, loaders=loaders,
merge=True):
shell_notify('could not load defaults',
state=True,
more=dict(defaults=defaults, sdict=sdict))
if config:
config = search_location(config, create_in='conf_dir')
if self.__settings != self.load(
'config', config, loaders=loaders, merge=True,
writeback=True):
shell_notify('settings config written',
more=config,
verbose=verbose)
def make_locations(locations=None, verbose=True):
'''
Creates folders
:param locations:
A list of folders to create (can be a dictionary, see note below)
:param verbose:
Warn if any folders were created
.. note::
* |params_locations_dict|
* |param_locations_none|
'''
from photon.util.structures import to_list
from photon.util.system import shell_notify
if not locations:
locations = get_locations().values()
locations = to_list(locations)
r = list()
for p in reversed(sorted(locations)):
if not _path.exists(p):
_makedirs(p)
r.append(p)
if verbose and r:
shell_notify('path created', state=None, more=r)
return r
def to_list(i, use_keys=False):
'''
Converts items to a list.
:param i: Item to convert
* If `i` is ``None``, the result is an empty list
* If `i` is 'string', the result won't be \
``['s', 't', 'r',...]`` rather more like ``['string']``
* If `i` is a nested dictionary, the result will be a flattened list.
:param use_keys:
If i is a dictionary, use the keys instead of values
:returns:
All items in i as list
'''
from photon.util.system import shell_notify
if not i:
return []
if isinstance(i, str):
return [i]
if isinstance(i, list):
return i
if isinstance(i, dict):
res = list()
for e in i.keys() if use_keys else i.values():
res.append(to_list(e)) if isinstance(e, dict) else res.append(e)
return res
shell_notify('type for %s uncovered' % (i), state=True, more=type(i))
def search_location(loc, locations=None,
critical=False, create_in=None, verbose=True):
'''
Locates files with a twist:
* Check the existence of a file using the full path in `loc`
* Search for the filename `loc` in `locations`
* Create it's enclosing folders if the file does not exist. \
use `create_in`
:param loc:
Filename to search
:param locations:
A list of possible locations to search within
(can be a dictionary, see note below)
:param critical:
|appteardown| if file was not found
:param create_in:
If `loc` was not found, the folder `create_in` is created.
If `locations` is a dictionary, `create_in` can also specify
a key of `locations`. The value will be used then.
:param verbose:
Pass verbose flag to :func:`make_locations`
:returns:
The full path of `loc` in matched location
.. note::
* |params_locations_dict|
* |param_locations_none|
'''
from photon.util.structures import to_list
from photon.util.system import shell_notify
if not locations:
locations = get_locations()
for p in reversed(sorted(to_list(locations))):
f = _path.join(p, loc)
if _path.exists(f):
return f
if _path.exists(_path.abspath(_path.expanduser(loc))):
return _path.abspath(_path.expanduser(loc))
if critical:
shell_notify('could not locate', state=True, more=dict(
file=loc, locations=locations
))
if create_in:
if isinstance(locations, dict):
create_in = locations.get(create_in, create_in)
make_locations(locations=[create_in], verbose=verbose)
return _path.join(create_in, loc)
def load(self,
skey,
sdesc,
sdict=None,
loaders=None,
merge=False,
writeback=False):
'''
Loads a dictionary into current settings
:param skey:
Type of data to load. Is be used to reference the data \
in the files sections within settings
:param sdesc:
Either filename of yaml-file to load or further description of \
imported data when `sdict` is used
:param dict sdict:
Directly pass data as dictionary instead of loading \
it from a yaml-file. \
Make sure to set `skey` and `sdesc` accordingly
:param list loaders:
Append custom loaders to the YAML-loader.
:param merge:
Merge received data into current settings or \
place it under `skey` within meta
:param writeback:
Write back loaded (and merged/imported) result back \
to the original file. \
This is used to generate the summary files
:returns:
The loaded (or directly passed) content
.. seealso:: |yaml_loaders|
'''
y = sdict if sdict else read_yaml(sdesc, add_constructor=loaders)
if y and isinstance(y, dict):
if not sdict:
self.__settings['files'].update({skey: sdesc})
if merge:
self.__settings = dict_merge(self.__settings, y)
else:
self.__settings[skey] = y
shell_notify('load %s data and %s it into settings' %
('got' if sdict else 'read',
'merged' if merge else 'imported'),
more=dict(skey=skey,
sdesc=sdesc,
merge=merge,
writeback=writeback),
verbose=self.__verbose)
if writeback and y != self.__settings:
write_yaml(sdesc, self.__settings)
return y
def change_location(src, tgt, move=False, verbose=True):
'''
Copies/moves/deletes locations
:param src:
Source location where to copy from
:param tgt:
Target location where to copy to
* To backup `src`, set `tgt` explicitly to ``True``. \
`tgt` will be set to `src` + '_backup_' + \
:func:`util.system.get_timestamp` then
:param move:
Deletes original location after copy (a.k.a. move)
* To delete `src` , set `tgt` explicitly to ``False`` \
and `move` to ``True`` (be careful!!1!)
:param verbose:
Show warnings
'''
from photon.util.system import shell_notify
if _path.exists(src):
if tgt:
if _path.isfile(src):
_copy2(src, search_location(
tgt, create_in=_path.dirname(tgt), verbose=verbose)
)
else:
for l in _listdir(src):
change_location(
_path.abspath(_path.join(src, l)),
_path.abspath(_path.join(tgt, l))
)
if move:
if _path.isdir(src) and not _path.islink(src):
_rmtree(src)
else:
_remove(src)
if verbose:
shell_notify(
'%s location' % (
'deleted'
if not tgt and move else
'moved'
if move else
'copied'
),
more=dict(src=src, tgt=tgt)
)
def __init__(self, defaults, config='config.yaml', verbose=True):
super().__init__()
self.__verbose = verbose
self.__settings = {
'locations': get_locations(),
'files': dict()
}
loaders = [
('!str_join', yaml_str_join,),
('!loc_join', yaml_loc_join,)
]
defaults, sdict = (
'startup import',
defaults
) if isinstance(defaults, dict) else (
search_location(defaults),
None
)
if not self.load(
'defaults',
defaults,
sdict=sdict,
loaders=loaders,
merge=True
):
shell_notify(
'could not load defaults',
state=True,
more=dict(defaults=defaults, sdict=sdict)
)
if config:
config = search_location(config, create_in='conf_dir')
if self.__settings != self.load(
'config',
config,
loaders=loaders,
merge=True,
writeback=True
):
shell_notify(
'settings config written',
more=config,
verbose=verbose
)
def load(self, skey, sdesc,
sdict=None, loaders=None, merge=False, writeback=False):
'''
Loads a dictionary into current settings
:param skey:
Type of data to load. Is be used to reference the data \
in the files sections within settings
:param sdesc:
Either filename of yaml-file to load or further description of \
imported data when `sdict` is used
:param dict sdict:
Directly pass data as dictionary instead of loading \
it from a yaml-file. \
Make sure to set `skey` and `sdesc` accordingly
:param list loaders:
Append custom loaders to the YAML-loader.
:param merge:
Merge received data into current settings or \
place it under `skey` within meta
:param writeback:
Write back loaded (and merged/imported) result back \
to the original file. \
This is used to generate the summary files
:returns:
The loaded (or directly passed) content
.. seealso:: |yaml_loaders|
'''
y = sdict if sdict else read_yaml(sdesc, add_constructor=loaders)
if y and isinstance(y, dict):
if not sdict:
self.__settings['files'].update({skey: sdesc})
if merge:
self.__settings = dict_merge(self.__settings, y)
else:
self.__settings[skey] = y
shell_notify(
'load %s data and %s it into settings' % (
'got' if sdict else 'read',
'merged' if merge else 'imported'
),
more=dict(skey=skey, sdesc=sdesc,
merge=merge, writeback=writeback),
verbose=self.__verbose
)
if writeback and y != self.__settings:
write_yaml(sdesc, self.__settings)
return y
def stage(self, name, clean=False):
'''
Switch stage
:param name:
Filename of new meta file. |filelocate|
* File must not already exist, will be created in 'data_dir' \
from :func:`util.locations.get_locations`
* Can also be a full path to place it anywhere desired
:param clean: What to do with preexisting meta files?
* ``False``: Merge current meta with preexisting one
* ``True``: Replace preexisting meta with current one
'''
name = search_location(name, create_in='data_dir')
if not clean:
self.load('stage', name, merge=True)
self.__meta['header'].update({'stage': name})
self.log = shell_notify('%s stage' %
('new clean' if clean else 'loaded'),
more=dict(meta=name, clean=clean),
verbose=self.__verbose)
def stage(self, name, clean=False):
'''
Switch stage
:param name:
Filename of new meta file. |filelocate|
* File must not already exist, will be created in 'data_dir' \
from :func:`util.locations.get_locations`
* Can also be a full path to place it anywhere desired
:param clean: What to do with preexisting meta files?
* ``False``: Merge current meta with preexisting one
* ``True``: Replace preexisting meta with current one
'''
name = search_location(name, create_in='data_dir')
if not clean:
self.load('stage', name, merge=True)
self.__meta['header'].update({'stage': name})
self.log = shell_notify(
'%s stage' % ('new clean' if clean else 'loaded'),
more=dict(meta=name, clean=clean),
verbose=self.__verbose
)
def check_m(pm):
'''
Shared helper function for all :ref:`tools` to check if the passed
m-function is indeed :func:`photon.Photon.m`
:params pm:
Suspected m-function
:returns:
Now to be proven correct m-function,
tears down whole application otherwise.
'''
if not any([
callable(pm), pm.__name__ != Photon.m.__name__,
pm.__doc__ != Photon.m.__doc__
]):
shell_notify('wrong "m-function" passed!',
state=True,
more=pm.__name__)
return pm
def check_m(pm):
'''
Shared helper function for all :ref:`tools` to check if the passed
m-function is indeed :func:`photon.Photon.m`
:params pm:
Suspected m-function
:returns:
Now to be proven correct m-function,
tears down whole application otherwise.
'''
if not any([
callable(pm),
pm.__name__ != Photon.m.__name__,
pm.__doc__ != Photon.m.__doc__
]):
shell_notify(
'wrong "m-function" passed!',
state=True,
more=pm.__name__
)
return pm
def __init__(self, defaults,
config='config.yaml', meta='meta.json', verbose=True):
super().__init__()
self.settings = Settings(defaults, config=config, verbose=verbose)
self.meta = Meta(meta=meta, verbose=verbose)
self.__verbose = verbose
self.s2m
self.meta.log = shell_notify(
'%s startup done' % (IDENT),
more=dict(defaults=defaults, config=config,
meta=meta, verbose=verbose),
verbose=False
)
def __init__(self,
defaults,
config='config.yaml',
meta='meta.json',
verbose=True):
super().__init__()
self.settings = Settings(defaults, config=config, verbose=verbose)
self.meta = Meta(meta=meta, verbose=verbose)
self.__verbose = verbose
self.s2m
self.meta.log = shell_notify('%s startup done' % (IDENT),
more=dict(defaults=defaults,
config=config,
meta=meta,
verbose=verbose),
verbose=False)
def load(self, mkey, mdesc, mdict=None, merge=False):
'''
Loads a dictionary into current meta
:param mkey:
Type of data to load.
Is be used to reference the data from the 'header' within meta
:param mdesc:
Either filename of json-file to load or further description
of imported data when `mdict` is used
:param dict mdict:
Directly pass data as dictionary instead of
loading it from a json-file.
Make sure to set `mkey` and `mdesc` accordingly
:param merge:
Merge received data into current meta or place it
under 'import' within meta
:returns:
The loaded (or directly passed) content
'''
j = mdict if mdict else read_json(mdesc)
if j and isinstance(j, dict):
self.__meta['header'].update({mkey: mdesc})
if merge:
self.__meta = dict_merge(self.__meta, j)
else:
self.__meta['import'][mkey] = j
self.log = shell_notify(
'load %s data and %s it into meta' % (
'got' if mdict else 'read',
'merged' if merge else 'imported'
),
more=dict(mkey=mkey, mdesc=mdesc, merge=merge),
verbose=self.__verbose
)
return j
def load(self, mkey, mdesc, mdict=None, merge=False):
'''
Loads a dictionary into current meta
:param mkey:
Type of data to load.
Is be used to reference the data from the 'header' within meta
:param mdesc:
Either filename of json-file to load or further description
of imported data when `mdict` is used
:param dict mdict:
Directly pass data as dictionary instead of
loading it from a json-file.
Make sure to set `mkey` and `mdesc` accordingly
:param merge:
Merge received data into current meta or place it
under 'import' within meta
:returns:
The loaded (or directly passed) content
'''
j = mdict if mdict else read_json(mdesc)
if j and isinstance(j, dict):
self.__meta['header'].update({mkey: mdesc})
if merge:
self.__meta = dict_merge(self.__meta, j)
else:
self.__meta['import'][mkey] = j
self.log = shell_notify('load %s data and %s it into meta' %
('got' if mdict else 'read',
'merged' if merge else 'imported'),
more=dict(mkey=mkey,
mdesc=mdesc,
merge=merge),
verbose=self.__verbose)
return j
def m(self,
msg,
state=False,
more=None,
cmdd=None,
critical=True,
verbose=None):
'''
Mysterious mega method managing multiple meshed modules magically
.. note:: If this function is used, the code contains facepalms: ``m(``
* It is possible to just show a message, \
or to run a command with message.
* But it is not possible to run a command without a message, \
use the `verbose`-flag to hide your debug message.
:param msg:
Add a message. Shown depending on `verbose` (see below)
:param state:
Pass `state` down to :func:`util.system.shell_notify`
:param more:
Pass `more` down to :func:`util.system.shell_notify`
:param dict cmdd:
If given, :func:`util.system.shell_run` is launched with
it's values
:param critical:
If set to ``True``: |appteardown| on failure of `cmdd` contents.
* Similar to :func:`util.system.shell_run` `critical`-flag
:param verbose:
Overrules parent's class `verbose`-flag.
* If left to ``None``, the verbose value Photon \
was started with is used
* Messages are shown/hidden if explicitly set to ``True``/``False``
:returns:
A dictionary specified the following:
* 'more':
`more` if it is not a dictionary otherwise \
it gets merged in if `more` is specified
* The output of :func:`util.system.shell_run` gets merged in \
if `cmdd` is specified
* 'failed': ``True`` if command failed
:func:`util.system.shell_notify` is used with this dictionary
to pipe it's output into :func:`meta.Meta.log` before returning.
'''
if verbose is None:
verbose = self.__verbose
res = dict()
if more:
res.update(more if isinstance(more, dict) else dict(more=more))
if cmdd and isinstance(cmdd, dict) and cmdd.get('cmd'):
res.update(
shell_run(cmdd.get('cmd'),
cin=cmdd.get('cin'),
cwd=cmdd.get('cwd'),
timeout=cmdd.get('timeout', 120),
critical=False,
verbose=cmdd.get('verbose', verbose)))
if res.get('returncode', -1) != 0:
res.update(dict(failed=True))
if state or critical and res.get('failed'):
self.meta.log = dict(message=msg, more=res, verbose=verbose)
shell_notify(msg, more=res, state=True)
self.meta.log = shell_notify(msg,
more=res,
state=state,
verbose=verbose)
return res
def m(self, msg,
state=False, more=None, cmdd=None, critical=True, verbose=None):
'''
Mysterious mega method managing multiple meshed modules magically
.. note:: If this function is used, the code contains facepalms: ``m(``
* It is possible to just show a message, \
or to run a command with message.
* But it is not possible to run a command without a message, \
use the `verbose`-flag to hide your debug message.
:param msg:
Add a message. Shown depending on `verbose` (see below)
:param state:
Pass `state` down to :func:`util.system.shell_notify`
:param more:
Pass `more` down to :func:`util.system.shell_notify`
:param dict cmdd:
If given, :func:`util.system.shell_run` is launched with
it's values
:param critical:
If set to ``True``: |appteardown| on failure of `cmdd` contents.
* Similar to :func:`util.system.shell_run` `critical`-flag
:param verbose:
Overrules parent's class `verbose`-flag.
* If left to ``None``, the verbose value Photon \
was started with is used
* Messages are shown/hidden if explicitly set to ``True``/``False``
:returns:
A dictionary specified the following:
* 'more':
`more` if it is not a dictionary otherwise \
it gets merged in if `more` is specified
* The output of :func:`util.system.shell_run` gets merged in \
if `cmdd` is specified
* 'failed': ``True`` if command failed
:func:`util.system.shell_notify` is used with this dictionary
to pipe it's output into :func:`meta.Meta.log` before returning.
'''
if verbose is None:
verbose = self.__verbose
res = dict()
if more:
res.update(more if isinstance(more, dict) else dict(more=more))
if cmdd and isinstance(cmdd, dict) and cmdd.get('cmd'):
res.update(shell_run(
cmdd.get('cmd'),
cin=cmdd.get('cin'),
cwd=cmdd.get('cwd'),
timeout=cmdd.get('timeout', 120),
critical=False,
verbose=cmdd.get('verbose', verbose)
))
if res.get('returncode', -1) != 0:
res.update(dict(failed=True))
if state or critical and res.get('failed'):
self.meta.log = dict(message=msg, more=res, verbose=verbose)
shell_notify(msg, more=res, state=True)
self.meta.log = shell_notify(msg, more=res,
state=state, verbose=verbose)
return res