def _resolve_dotfile_paths(self):
"""resolve dotfile paths"""
t = Templategen(variables=self.variables,
func_file=self.settings[Settings.key_func_file],
filter_file=self.settings[Settings.key_filter_file])
for dotfile in self.dotfiles.values():
# src
src = dotfile[self.key_dotfile_src]
if not src:
dotfile[self.key_dotfile_src] = ''
else:
new = t.generate_string(src)
if new != src and self.debug:
msg = 'dotfile src: \"{}\" -> \"{}\"'.format(src, new)
self.log.dbg(msg)
src = new
src = os.path.join(self.settings[self.key_settings_dotpath],
src)
dotfile[self.key_dotfile_src] = self._norm_path(src)
# dst
dst = dotfile[self.key_dotfile_dst]
if not dst:
dotfile[self.key_dotfile_dst] = ''
else:
new = t.generate_string(dst)
if new != dst and self.debug:
msg = 'dotfile dst: \"{}\" -> \"{}\"'.format(dst, new)
self.log.dbg(msg)
dst = new
dotfile[self.key_dotfile_dst] = self._norm_path(dst)
def eval_dotfiles(self, profile, variables, debug=False):
"""resolve dotfiles src/dst/actions templating for this profile"""
t = Templategen(variables=variables)
for d in self.get_dotfiles(profile):
# src and dst path
d.src = t.generate_string(d.src)
d.dst = t.generate_string(d.dst)
# pre actions
if self.key_actions_pre in d.actions:
for action in d.actions[self.key_actions_pre]:
action.action = t.generate_string(action.action)
# post actions
if self.key_actions_post in d.actions:
for action in d.actions[self.key_actions_post]:
action.action = t.generate_string(action.action)
def _get_imported_dotfiles_keys(self, profile):
"""import dotfiles from external file"""
keys = []
if self.key_profiles_imp not in self.lnk_profiles[profile]:
return keys
variables = self.get_variables(profile, debug=self.debug)
t = Templategen(variables=variables)
paths = self.lnk_profiles[profile][self.key_profiles_imp]
for path in paths:
path = self._abs_path(path)
path = t.generate_string(path)
if self.debug:
self.log.dbg('loading dotfiles from {}'.format(path))
content = self._load_yaml(path)
if not content:
self.log.warn('\"{}\" does not exist'.format(path))
continue
if self.key_profiles_dots not in content:
self.log.warn('not dotfiles in \"{}\"'.format(path))
continue
df = content[self.key_profiles_dots]
if self.debug:
self.log.dbg('imported dotfiles keys: {}'.format(df))
keys.extend(df)
return keys
def _load_ext_variables(self, paths, profile=None):
"""load external variables"""
variables = {}
dvariables = {}
cur_vars = self.get_variables(profile, debug=self.debug)
t = Templategen(variables=cur_vars)
for path in paths:
path = self._abs_path(path)
path = t.generate_string(path)
if self.debug:
self.log.dbg('loading variables from {}'.format(path))
content = self._load_yaml(path)
if not content:
self.log.warn('\"{}\" does not exist'.format(path))
continue
# variables
if self.key_variables in content:
variables.update(content[self.key_variables])
# dynamic variables
if self.key_dynvariables in content:
dvariables.update(content[self.key_dynvariables])
self.ext_variables = variables
if self.debug:
self.log.dbg('loaded ext variables: {}'.format(variables))
self.ext_dynvariables = dvariables
if self.debug:
self.log.dbg('loaded ext dynvariables: {}'.format(dvariables))
def eval_dotfiles(self, profile, debug=False):
"""resolve dotfiles src/dst templates"""
t = Templategen(profile=profile,
variables=self.get_variables(),
debug=debug)
for d in self.get_dotfiles(profile):
d.src = t.generate_string(d.src)
d.dst = t.generate_string(d.dst)
def _merge_variables(self):
"""
resolve all variables across the config
apply them to any needed entries
and return the full list of variables
"""
if self.debug:
self.log.dbg('get local variables')
# get all variables from local and resolve
var = self._get_variables_dict(self.profile)
# get all dynvariables from local and resolve
dvar = self._get_dvariables_dict()
# temporarly resolve all variables for "include"
merged = self._merge_dict(dvar, var)
merged = self._rec_resolve_vars(merged)
self._debug_vars(merged)
# exec dynvariables
self._shell_exec_dvars(dvar.keys(), merged)
if self.debug:
self.log.dbg('local variables resolved')
self._debug_vars(merged)
# resolve profile includes
t = Templategen(variables=merged,
func_file=self.settings[Settings.key_func_file],
filter_file=self.settings[Settings.key_filter_file])
for k, v in self.profiles.items():
if self.key_profile_include in v:
new = []
for k in v[self.key_profile_include]:
new.append(t.generate_string(k))
v[self.key_profile_include] = new
# now get the included ones
pro_var = self._get_included_variables(self.profile,
seen=[self.profile])
pro_dvar = self._get_included_dvariables(self.profile,
seen=[self.profile])
# exec incl dynvariables
self._shell_exec_dvars(pro_dvar.keys(), pro_dvar)
# merge all and resolve
merged = self._merge_dict(pro_var, merged)
merged = self._merge_dict(pro_dvar, merged)
merged = self._rec_resolve_vars(merged)
if self.debug:
self.log.dbg('resolve all uses of variables in config')
self._debug_vars(merged)
prokeys = list(pro_var.keys()) + list(pro_dvar.keys())
return merged, prokeys
def _rec_resolve_vars(self, variables):
"""recursive resolve variables"""
t = Templategen(variables=variables)
for k in variables.keys():
val = variables[k]
while Templategen.var_is_template(val):
val = t.generate_string(val)
variables[k] = val
t.update_variables(variables)
return variables
def _rec_resolve_vars(self, variables):
"""recursive resolve variables"""
default = self._get_variables_dict(self.profile)
t = Templategen(variables=self._merge_dict(default, variables))
for k in variables.keys():
val = variables[k]
while Templategen.var_is_template(val):
val = t.generate_string(val)
variables[k] = val
t.update_variables(variables)
return variables
def _resolve_dotfile_paths(self):
"""resolve dotfile paths"""
t = Templategen(variables=self.variables)
for dotfile in self.dotfiles.values():
# src
src = dotfile[self.key_dotfile_src]
new = t.generate_string(src)
if new != src and self.debug:
self.log.dbg('dotfile: {} -> {}'.format(src, new))
src = new
src = os.path.join(self.settings[self.key_settings_dotpath], src)
dotfile[self.key_dotfile_src] = self._norm_path(src)
# dst
dst = dotfile[self.key_dotfile_dst]
new = t.generate_string(dst)
if new != dst and self.debug:
self.log.dbg('dotfile: {} -> {}'.format(dst, new))
dst = new
dotfile[self.key_dotfile_dst] = self._norm_path(dst)
def _apply_variables(self):
"""template any needed parts of the config"""
t = Templategen(variables=self.variables)
# dotfiles src/dst/actions keys
for k, v in self.dotfiles.items():
# src
src = v.get(self.key_dotfile_src)
v[self.key_dotfile_src] = t.generate_string(src)
# dst
dst = v.get(self.key_dotfile_dst)
v[self.key_dotfile_dst] = t.generate_string(dst)
# import_actions
new = []
entries = self.settings.get(self.key_import_actions, [])
new = self._template_list(t, entries)
if new:
self.settings[self.key_import_actions] = new
# import_configs
entries = self.settings.get(self.key_import_configs, [])
new = self._template_list(t, entries)
if new:
self.settings[self.key_import_configs] = new
# import_variables
entries = self.settings.get(self.key_import_variables, [])
new = self._template_list(t, entries)
if new:
self.settings[self.key_import_variables] = new
# profile's import
for k, v in self.profiles.items():
entries = v.get(self.key_import_profile_dfs, [])
new = self._template_list(t, entries)
if new:
v[self.key_import_profile_dfs] = new
def _rec_resolve_variables(self, variables):
"""recursive resolve variables"""
var = self._enrich_vars(variables, self._profile)
# use a separated templategen to handle variables
# resolved outside the main config
t = Templategen(variables=var,
func_file=self.settings[Settings.key_func_file],
filter_file=self.settings[Settings.key_filter_file])
for k in variables.keys():
val = variables[k]
while Templategen.var_is_template(val):
val = t.generate_string(val)
variables[k] = val
t.update_variables(variables)
if variables is self.variables:
self._redefine_templater()
def _get_included_dotfiles(self, profile, seen=[]):
"""find all dotfiles for a specific profile
when using the include keyword"""
if profile in seen:
self.log.err('cyclic include in profile \"{}\"'.format(profile))
return False, []
if not self.lnk_profiles[profile]:
return True, []
dotfiles = self.prodots[profile]
if self.key_profiles_incl not in self.lnk_profiles[profile]:
# no include found
return True, dotfiles
if not self.lnk_profiles[profile][self.key_profiles_incl]:
# empty include found
return True, dotfiles
variables = self.get_variables(profile, debug=self.debug)
t = Templategen(variables=variables)
if self.debug:
self.log.dbg('handle includes for profile \"{}\"'.format(profile))
for other in self.lnk_profiles[profile][self.key_profiles_incl]:
# resolve include value
other = t.generate_string(other)
if other not in self.prodots:
# no such profile
self.log.warn('unknown included profile \"{}\"'.format(other))
continue
if self.debug:
msg = 'include dotfiles from \"{}\" into \"{}\"'
self.log.dbg(msg.format(other, profile))
lseen = seen.copy()
lseen.append(profile)
ret, recincludes = self._get_included_dotfiles(other, seen=lseen)
if not ret:
return False, []
dotfiles.extend(recincludes)
dotfiles.extend(self.prodots[other])
return True, dotfiles
def _get_profile_included_vars(self, tvars):
"""resolve profile included variables/dynvariables"""
t = Templategen(variables=tvars,
func_file=self.settings[Settings.key_func_file],
filter_file=self.settings[Settings.key_filter_file])
for k, v in self.profiles.items():
if self.key_profile_include in v:
new = []
for x in v[self.key_profile_include]:
new.append(t.generate_string(x))
v[self.key_profile_include] = new
# now get the included ones
pro_var = self._get_profile_included_item(self.profile,
self.key_profile_variables,
seen=[self.profile])
pro_dvar = self._get_profile_included_item(self.profile,
self.key_profile_dvariables,
seen=[self.profile])
# exec incl dynvariables
self._shell_exec_dvars(pro_dvar.keys(), pro_dvar)
return pro_var, pro_dvar
class CfgYaml:
# global entries
key_settings = Settings.key_yaml
key_dotfiles = 'dotfiles'
key_profiles = 'profiles'
key_actions = 'actions'
old_key_trans_r = 'trans'
key_trans_r = 'trans_read'
key_trans_w = 'trans_write'
key_variables = 'variables'
key_dvariables = 'dynvariables'
action_pre = 'pre'
action_post = 'post'
# profiles/dotfiles entries
key_dotfile_src = 'src'
key_dotfile_dst = 'dst'
key_dotfile_link = 'link'
key_dotfile_actions = 'actions'
key_dotfile_link_children = 'link_children'
key_dotfile_noempty = 'ignoreempty'
# profile
key_profile_dotfiles = 'dotfiles'
key_profile_include = 'include'
key_profile_variables = 'variables'
key_profile_dvariables = 'dynvariables'
key_profile_actions = 'actions'
key_all = 'ALL'
# import entries
key_import_actions = 'import_actions'
key_import_configs = 'import_configs'
key_import_variables = 'import_variables'
key_import_profile_dfs = 'import'
key_import_sep = ':'
key_import_ignore_key = 'optional'
key_import_fatal_not_found = True
# settings
key_settings_dotpath = Settings.key_dotpath
key_settings_workdir = Settings.key_workdir
key_settings_link_dotfile_default = Settings.key_link_dotfile_default
key_settings_noempty = Settings.key_ignoreempty
key_settings_minversion = Settings.key_minversion
key_imp_link = Settings.key_link_on_import
# link values
lnk_nolink = LinkTypes.NOLINK.name.lower()
lnk_link = LinkTypes.LINK.name.lower()
lnk_children = LinkTypes.LINK_CHILDREN.name.lower()
def __init__(self, path, profile=None, addprofiles=[], debug=False):
"""
config parser
@path: config file path
@profile: the selected profile
@addprofiles: included profiles
@debug: debug flag
"""
self._path = os.path.abspath(path)
self._profile = profile
self._debug = debug
self._log = Logger()
# config needs to be written
self._dirty = False
# indicates the config has been updated
self._dirty_deprecated = False
# profile variables
self._profilevarskeys = []
# included profiles
self._inc_profiles = addprofiles
# init the dictionaries
self.settings = {}
self.dotfiles = {}
self.profiles = {}
self.actions = {}
self.trans_r = {}
self.trans_w = {}
self.variables = {}
if not os.path.exists(self._path):
err = 'invalid config path: \"{}\"'.format(path)
if self._debug:
self._dbg(err)
raise YamlException(err)
self._yaml_dict = self._load_yaml(self._path)
# live patch deprecated entries
self._fix_deprecated(self._yaml_dict)
##################################################
# parse the config and variables
##################################################
# parse the "config" block
self.settings = self._parse_blk_settings(self._yaml_dict)
# base templater (when no vars/dvars exist)
self.variables = self._enrich_vars(self.variables, self._profile)
self._redefine_templater()
# variables and dynvariables need to be first merged
# before being templated in order to allow cyclic
# references between them
# parse the "variables" block
var = self._parse_blk_variables(self._yaml_dict)
self._add_variables(var, template=False)
# parse the "dynvariables" block
dvariables = self._parse_blk_dynvariables(self._yaml_dict)
self._add_variables(dvariables, template=False)
# now template variables and dynvariables from the same pool
self._rec_resolve_variables(self.variables)
# and execute dvariables
# since this is done after recursively resolving variables
# and dynvariables this means that variables referencing
# dynvariables will result with the not executed value
if dvariables.keys():
self._shell_exec_dvars(self.variables, keys=dvariables.keys())
# finally redefine the template
self._redefine_templater()
if self._debug:
self._debug_dict('current variables defined', self.variables)
# parse the "profiles" block
self.profiles = self._parse_blk_profiles(self._yaml_dict)
# include the profile's variables/dynvariables last
# as it overwrites existing ones
self._inc_profiles, pv, pvd = self._get_profile_included_vars()
self._add_variables(pv, prio=True)
self._add_variables(pvd, shell=True, prio=True)
self._profilevarskeys.extend(pv.keys())
self._profilevarskeys.extend(pvd.keys())
# template variables
self.variables = self._template_dict(self.variables)
if self._debug:
self._debug_dict('current variables defined', self.variables)
##################################################
# template the "include" entries
##################################################
self._template_include_entry()
if self._debug:
self._debug_dict('current variables defined', self.variables)
##################################################
# parse the other blocks
##################################################
# parse the "dotfiles" block
self.dotfiles = self._parse_blk_dotfiles(self._yaml_dict)
# parse the "actions" block
self.actions = self._parse_blk_actions(self._yaml_dict)
# parse the "trans_r" block
self.trans_r = self._parse_blk_trans_r(self._yaml_dict)
# parse the "trans_w" block
self.trans_w = self._parse_blk_trans_w(self._yaml_dict)
##################################################
# import elements
##################################################
# process imported variables (import_variables)
newvars = self._import_variables()
self._clear_profile_vars(newvars)
self._add_variables(newvars)
# process imported actions (import_actions)
self._import_actions()
# process imported profile dotfiles (import)
self._import_profiles_dotfiles()
# process imported configs (import_configs)
self._import_configs()
# process profile include
self._resolve_profile_includes()
# add the current profile variables
_, pv, pvd = self._get_profile_included_vars()
self._add_variables(pv, prio=True)
self._add_variables(pvd, shell=True, prio=True)
self._profilevarskeys.extend(pv.keys())
self._profilevarskeys.extend(pvd.keys())
# resolve variables
self._clear_profile_vars(newvars)
self._add_variables(newvars)
# process profile ALL
self._resolve_profile_all()
# patch dotfiles paths
self._template_dotfiles_paths()
if self._debug:
self._dbg('########### {} ###########'.format('final config'))
self._debug_entries()
########################################################
# outside available methods
########################################################
def resolve_dotfile_src(self, src, templater=None):
"""resolve dotfile src path"""
newsrc = ''
if src:
new = src
if templater:
new = templater.generate_string(src)
if new != src and self._debug:
msg = 'dotfile src: \"{}\" -> \"{}\"'.format(src, new)
self._dbg(msg)
src = new
src = os.path.join(self.settings[self.key_settings_dotpath], src)
newsrc = self._norm_path(src)
return newsrc
def resolve_dotfile_dst(self, dst, templater=None):
"""resolve dotfile dst path"""
newdst = ''
if dst:
new = dst
if templater:
new = templater.generate_string(dst)
if new != dst and self._debug:
msg = 'dotfile dst: \"{}\" -> \"{}\"'.format(dst, new)
self._dbg(msg)
dst = new
newdst = self._norm_path(dst)
return newdst
def add_dotfile_to_profile(self, dotfile_key, profile_key):
"""add an existing dotfile key to a profile_key"""
self._new_profile(profile_key)
profile = self._yaml_dict[self.key_profiles][profile_key]
if self.key_profile_dotfiles not in profile or \
profile[self.key_profile_dotfiles] is None:
profile[self.key_profile_dotfiles] = []
pdfs = profile[self.key_profile_dotfiles]
if self.key_all not in pdfs and \
dotfile_key not in pdfs:
profile[self.key_profile_dotfiles].append(dotfile_key)
if self._debug:
msg = 'add \"{}\" to profile \"{}\"'.format(
dotfile_key, profile_key)
msg.format(dotfile_key, profile_key)
self._dbg(msg)
self._dirty = True
return self._dirty
def get_all_dotfile_keys(self):
"""return all existing dotfile keys"""
return self.dotfiles.keys()
def add_dotfile(self, key, src, dst, link):
"""add a new dotfile"""
if key in self.dotfiles.keys():
return False
if self._debug:
self._dbg('adding new dotfile: {}'.format(key))
self._dbg('new dotfile src: {}'.format(src))
self._dbg('new dotfile dst: {}'.format(dst))
df_dict = {
self.key_dotfile_src: src,
self.key_dotfile_dst: dst,
}
dfl = self.settings[self.key_settings_link_dotfile_default]
if str(link) != dfl:
df_dict[self.key_dotfile_link] = str(link)
self._yaml_dict[self.key_dotfiles][key] = df_dict
self._dirty = True
def del_dotfile(self, key):
"""remove this dotfile from config"""
if key not in self._yaml_dict[self.key_dotfiles]:
self._log.err('key not in dotfiles: {}'.format(key))
return False
if self._debug:
self._dbg('remove dotfile: {}'.format(key))
del self._yaml_dict[self.key_dotfiles][key]
if self._debug:
dfs = self._yaml_dict[self.key_dotfiles]
self._dbg('new dotfiles: {}'.format(dfs))
self._dirty = True
return True
def del_dotfile_from_profile(self, df_key, pro_key):
"""remove this dotfile from that profile"""
if df_key not in self.dotfiles.keys():
self._log.err('key not in dotfiles: {}'.format(df_key))
return False
if pro_key not in self.profiles.keys():
self._log.err('key not in profile: {}'.format(pro_key))
return False
# get the profile dictionary
profile = self._yaml_dict[self.key_profiles][pro_key]
if df_key not in profile[self.key_profile_dotfiles]:
return True
if self._debug:
dfs = profile[self.key_profile_dotfiles]
self._dbg('{} profile dotfiles: {}'.format(pro_key, dfs))
self._dbg('remove {} from profile {}'.format(df_key, pro_key))
profile[self.key_profile_dotfiles].remove(df_key)
if self._debug:
dfs = profile[self.key_profile_dotfiles]
self._dbg('{} profile dotfiles: {}'.format(pro_key, dfs))
self._dirty = True
return True
def save(self):
"""save this instance and return True if saved"""
if not self._dirty:
return False
content = self._prepare_to_save(self._yaml_dict)
if self._dirty_deprecated:
# add minversion
settings = content[self.key_settings]
settings[self.key_settings_minversion] = VERSION
# save to file
if self._debug:
self._dbg('saving to {}'.format(self._path))
try:
with open(self._path, 'w') as f:
self._yaml_dump(content, f)
except Exception as e:
self._log.err(e)
raise YamlException('error saving config: {}'.format(self._path))
if self._dirty_deprecated:
warn = 'your config contained deprecated entries'
warn += ' and was updated'
self._log.warn(warn)
self._dirty = False
self.cfg_updated = False
return True
def dump(self):
"""dump the config dictionary"""
output = io.StringIO()
content = self._prepare_to_save(self._yaml_dict.copy())
self._yaml_dump(content, output)
return output.getvalue()
########################################################
# block parsing
########################################################
def _parse_blk_settings(self, dic):
"""parse the "config" block"""
block = self._get_entry(dic, self.key_settings).copy()
# set defaults
settings = Settings(None).serialize().get(self.key_settings)
settings.update(block)
# resolve minimum version
if self.key_settings_minversion in settings:
minversion = settings[self.key_settings_minversion]
self._check_minversion(minversion)
# normalize paths
p = self._norm_path(settings[self.key_settings_dotpath])
settings[self.key_settings_dotpath] = p
p = self._norm_path(settings[self.key_settings_workdir])
settings[self.key_settings_workdir] = p
p = [self._norm_path(p) for p in settings[Settings.key_filter_file]]
settings[Settings.key_filter_file] = p
p = [self._norm_path(p) for p in settings[Settings.key_func_file]]
settings[Settings.key_func_file] = p
if self._debug:
self._debug_dict('settings block:', settings)
return settings
def _parse_blk_dotfiles(self, dic):
"""parse the "dotfiles" block"""
dotfiles = self._get_entry(dic, self.key_dotfiles).copy()
keys = dotfiles.keys()
if len(keys) != len(list(set(keys))):
dups = [x for x in keys if x not in list(set(keys))]
err = 'duplicate dotfile keys found: {}'.format(dups)
raise YamlException(err)
dotfiles = self._norm_dotfiles(dotfiles)
if self._debug:
self._debug_dict('dotfiles block', dotfiles)
return dotfiles
def _parse_blk_profiles(self, dic):
"""parse the "profiles" block"""
profiles = self._get_entry(dic, self.key_profiles).copy()
profiles = self._norm_profiles(profiles)
if self._debug:
self._debug_dict('profiles block', profiles)
return profiles
def _parse_blk_actions(self, dic):
"""parse the "actions" block"""
actions = self._get_entry(dic, self.key_actions, mandatory=False)
if actions:
actions = actions.copy()
actions = self._norm_actions(actions)
if self._debug:
self._debug_dict('actions block', actions)
return actions
def _parse_blk_trans_r(self, dic):
"""parse the "trans_r" block"""
key = self.key_trans_r
if self.old_key_trans_r in dic:
msg = '\"trans\" is deprecated, please use \"trans_read\"'
self._log.warn(msg)
dic[self.key_trans_r] = dic[self.old_key_trans_r]
del dic[self.old_key_trans_r]
trans_r = self._get_entry(dic, key, mandatory=False)
if trans_r:
trans_r = trans_r.copy()
if self._debug:
self._debug_dict('trans_r block', trans_r)
return trans_r
def _parse_blk_trans_w(self, dic):
"""parse the "trans_w" block"""
trans_w = self._get_entry(dic, self.key_trans_w, mandatory=False)
if trans_w:
trans_w = trans_w.copy()
if self._debug:
self._debug_dict('trans_w block', trans_w)
return trans_w
def _parse_blk_variables(self, dic):
"""parse the "variables" block"""
variables = self._get_entry(dic, self.key_variables, mandatory=False)
if variables:
variables = variables.copy()
if self._debug:
self._debug_dict('variables block', variables)
return variables
def _parse_blk_dynvariables(self, dic):
"""parse the "dynvariables" block"""
dvariables = self._get_entry(dic, self.key_dvariables, mandatory=False)
if dvariables:
dvariables = dvariables.copy()
if self._debug:
self._debug_dict('dynvariables block', dvariables)
return dvariables
########################################################
# parsing helpers
########################################################
def _template_include_entry(self):
"""template all "include" entries"""
# import_actions
new = []
entries = self.settings.get(self.key_import_actions, [])
new = self._template_list(entries)
if new:
self.settings[self.key_import_actions] = new
# import_configs
entries = self.settings.get(self.key_import_configs, [])
new = self._template_list(entries)
if new:
self.settings[self.key_import_configs] = new
# import_variables
entries = self.settings.get(self.key_import_variables, [])
new = self._template_list(entries)
if new:
self.settings[self.key_import_variables] = new
# profile's import
for k, v in self.profiles.items():
entries = v.get(self.key_import_profile_dfs, [])
new = self._template_list(entries)
if new:
v[self.key_import_profile_dfs] = new
def _norm_actions(self, actions):
"""
ensure each action is either pre or post explicitely
action entry of the form {action_key: (pre|post, action)}
"""
if not actions:
return actions
new = {}
for k, v in actions.items():
if k == self.action_pre or k == self.action_post:
for key, action in v.items():
new[key] = (k, action)
else:
new[k] = (self.action_post, v)
return new
def _norm_profiles(self, profiles):
"""normalize profiles entries"""
if not profiles:
return profiles
new = {}
for k, v in profiles.items():
if not v:
# no dotfiles
continue
# add dotfiles entry if not present
if self.key_profile_dotfiles not in v:
v[self.key_profile_dotfiles] = []
new[k] = v
return new
def _norm_dotfiles(self, dotfiles):
"""normalize dotfiles entries"""
if not dotfiles:
return dotfiles
new = {}
for k, v in dotfiles.items():
# add 'src' as key' if not present
if self.key_dotfile_src not in v:
v[self.key_dotfile_src] = k
new[k] = v
else:
new[k] = v
# fix deprecated trans key
if self.old_key_trans_r in v:
msg = '\"trans\" is deprecated, please use \"trans_read\"'
self._log.warn(msg)
v[self.key_trans_r] = v[self.old_key_trans_r]
del v[self.old_key_trans_r]
new[k] = v
# apply link value
if self.key_dotfile_link not in v:
val = self.settings[self.key_settings_link_dotfile_default]
v[self.key_dotfile_link] = val
# apply noempty if undefined
if self.key_dotfile_noempty not in v:
val = self.settings.get(self.key_settings_noempty, False)
v[self.key_dotfile_noempty] = val
return new
def _add_variables(self, new, shell=False, template=True, prio=False):
"""
add new variables
@shell: execute the variable through the shell
@template: template the variable
@prio: new takes priority over existing variables
"""
if not new:
return
# merge
if prio:
self.variables = self._merge_dict(new, self.variables)
else:
self.variables = self._merge_dict(self.variables, new)
# ensure enriched variables are relative to this config
self.variables = self._enrich_vars(self.variables, self._profile)
# re-create the templater
self._redefine_templater()
if template:
# rec resolve variables with new ones
self._rec_resolve_variables(self.variables)
if shell:
# shell exec
self._shell_exec_dvars(self.variables, keys=new.keys())
# re-create the templater
self._redefine_templater()
def _enrich_vars(self, variables, profile):
"""return enriched variables"""
# add profile variable
if profile:
variables['profile'] = profile
# add some more variables
p = self.settings.get(self.key_settings_dotpath)
p = self._norm_path(p)
variables['_dotdrop_dotpath'] = p
variables['_dotdrop_cfgpath'] = self._norm_path(self._path)
p = self.settings.get(self.key_settings_workdir)
p = self._norm_path(p)
variables['_dotdrop_workdir'] = p
return variables
def _get_profile_included_item(self, keyitem):
"""recursively get included <keyitem> in profile"""
profiles = [self._profile] + self._inc_profiles
items = {}
for profile in profiles:
seen = [self._profile]
i = self.__get_profile_included_item(profile, keyitem, seen)
items = self._merge_dict(i, items)
return items
def __get_profile_included_item(self, profile, keyitem, seen):
"""recursively get included <keyitem> from profile"""
items = {}
if not profile or profile not in self.profiles.keys():
return items
# considered profile entry
pentry = self.profiles.get(profile)
# recursively get <keyitem> from inherited profile
for inherited_profile in pentry.get(self.key_profile_include, []):
if inherited_profile == profile or inherited_profile in seen:
raise YamlException('\"include\" loop')
seen.append(inherited_profile)
new = self.__get_profile_included_item(inherited_profile, keyitem,
seen)
if self._debug:
msg = 'included {} from {}: {}'
self._dbg(msg.format(keyitem, inherited_profile, new))
items.update(new)
cur = pentry.get(keyitem, {})
return self._merge_dict(cur, items)
def _resolve_profile_all(self):
"""resolve some other parts of the config"""
# profile -> ALL
for k, v in self.profiles.items():
dfs = v.get(self.key_profile_dotfiles, None)
if not dfs:
continue
if self.key_all in dfs:
if self._debug:
self._dbg('add ALL to profile \"{}\"'.format(k))
v[self.key_profile_dotfiles] = self.dotfiles.keys()
def _resolve_profile_includes(self):
"""resolve profile(s) including other profiles"""
for k, v in self.profiles.items():
self._rec_resolve_profile_include(k)
def _rec_resolve_profile_include(self, profile):
"""
recursively resolve include of other profiles's:
* dotfiles
* actions
returns dotfiles, actions
"""
this_profile = self.profiles[profile]
# considered profile content
dotfiles = this_profile.get(self.key_profile_dotfiles, []) or []
actions = this_profile.get(self.key_profile_actions, []) or []
includes = this_profile.get(self.key_profile_include, []) or []
if not includes:
# nothing to include
return dotfiles, actions
if self._debug:
self._dbg('{} includes {}'.format(profile, ','.join(includes)))
self._dbg('{} dotfiles before include: {}'.format(
profile, dotfiles))
self._dbg('{} actions before include: {}'.format(profile, actions))
seen = []
for i in uniq_list(includes):
if self._debug:
self._dbg('resolving includes "{}" <- "{}"'.format(profile, i))
# ensure no include loop occurs
if i in seen:
raise YamlException('\"include loop\"')
seen.append(i)
# included profile even exists
if i not in self.profiles.keys():
self._log.warn('include unknown profile: {}'.format(i))
continue
# recursive resolve
if self._debug:
self._dbg(
'recursively resolving includes for profile "{}"'.format(
i))
o_dfs, o_actions = self._rec_resolve_profile_include(i)
# merge dotfile keys
if self._debug:
self._dbg('Merging dotfiles {} <- {}: {} <- {}'.format(
profile, i, dotfiles, o_dfs))
dotfiles.extend(o_dfs)
this_profile[self.key_profile_dotfiles] = uniq_list(dotfiles)
# merge actions keys
if self._debug:
self._dbg('Merging actions {} <- {}: {} <- {}'.format(
profile, i, actions, o_actions))
actions.extend(o_actions)
this_profile[self.key_profile_actions] = uniq_list(actions)
dotfiles = this_profile.get(self.key_profile_dotfiles, [])
actions = this_profile.get(self.key_profile_actions, [])
if self._debug:
self._dbg('{} dotfiles after include: {}'.format(
profile, dotfiles))
self._dbg('{} actions after include: {}'.format(profile, actions))
# since included items are resolved here
# we can clear these include
self.profiles[profile][self.key_profile_include] = []
return dotfiles, actions
########################################################
# handle imported entries
########################################################
def _import_variables(self):
"""import external variables from paths"""
paths = self.settings.get(self.key_import_variables, None)
if not paths:
return
paths = self._resolve_paths(paths)
newvars = {}
for path in paths:
if self._debug:
self._dbg('import variables from {}'.format(path))
var = self._import_sub(path, self.key_variables, mandatory=False)
if self._debug:
self._dbg('import dynvariables from {}'.format(path))
dvar = self._import_sub(path, self.key_dvariables, mandatory=False)
merged = self._merge_dict(dvar, var)
self._rec_resolve_variables(merged)
if dvar.keys():
self._shell_exec_dvars(merged, keys=dvar.keys())
self._clear_profile_vars(merged)
newvars = self._merge_dict(newvars, merged)
if self._debug:
self._debug_dict('imported variables', newvars)
return newvars
def _import_actions(self):
"""import external actions from paths"""
paths = self.settings.get(self.key_import_actions, None)
if not paths:
return
paths = self._resolve_paths(paths)
for path in paths:
if self._debug:
self._dbg('import actions from {}'.format(path))
new = self._import_sub(path,
self.key_actions,
mandatory=False,
patch_func=self._norm_actions)
self.actions = self._merge_dict(new, self.actions)
def _import_profiles_dotfiles(self):
"""import profile dotfiles"""
for k, v in self.profiles.items():
imp = v.get(self.key_import_profile_dfs, None)
if not imp:
continue
if self._debug:
self._dbg('import dotfiles for profile {}'.format(k))
paths = self._resolve_paths(imp)
for path in paths:
current = v.get(self.key_dotfiles, [])
new = self._import_sub(path,
self.key_dotfiles,
mandatory=False)
v[self.key_dotfiles] = new + current
def _import_config(self, path):
"""import config from path"""
if self._debug:
self._dbg('import config from {}'.format(path))
sub = CfgYaml(path,
profile=self._profile,
addprofiles=self._inc_profiles,
debug=self._debug)
# settings are ignored from external file
# except for filter_file and func_file
self.settings[Settings.key_func_file] += [
self._norm_path(func_file)
for func_file in sub.settings[Settings.key_func_file]
]
self.settings[Settings.key_filter_file] += [
self._norm_path(func_file)
for func_file in sub.settings[Settings.key_filter_file]
]
# merge top entries
self.dotfiles = self._merge_dict(self.dotfiles, sub.dotfiles)
self.profiles = self._merge_dict(self.profiles, sub.profiles)
self.actions = self._merge_dict(self.actions, sub.actions)
self.trans_r = self._merge_dict(self.trans_r, sub.trans_r)
self.trans_w = self._merge_dict(self.trans_w, sub.trans_w)
self._clear_profile_vars(sub.variables)
if self._debug:
self._debug_dict('add import_configs var', sub.variables)
self._add_variables(sub.variables, prio=True)
def _import_configs(self):
"""import configs from external files"""
# settings -> import_configs
imp = self.settings.get(self.key_import_configs, None)
if not imp:
return
paths = self._resolve_paths(imp)
for path in paths:
self._import_config(path)
def _import_sub(self, path, key, mandatory=False, patch_func=None):
"""
import the block "key" from "path"
patch_func is applied to each element if defined
"""
if self._debug:
self._dbg('import \"{}\" from \"{}\"'.format(key, path))
extdict = self._load_yaml(path)
new = self._get_entry(extdict, key, mandatory=mandatory)
if patch_func:
if self._debug:
self._dbg('calling patch: {}'.format(patch_func))
new = patch_func(new)
if not new and mandatory:
err = 'no \"{}\" imported from \"{}\"'.format(key, path)
self._log.warn(err)
raise YamlException(err)
if self._debug:
self._dbg('imported \"{}\": {}'.format(key, new))
return new
########################################################
# add/remove entries
########################################################
def _new_profile(self, key):
"""add a new profile if it doesn't exist"""
if key not in self.profiles.keys():
# update yaml_dict
self._yaml_dict[self.key_profiles][key] = {
self.key_profile_dotfiles: []
}
if self._debug:
self._dbg('adding new profile: {}'.format(key))
self._dirty = True
########################################################
# handle deprecated entries
########################################################
def _fix_deprecated(self, yamldict):
"""fix deprecated entries"""
if not yamldict:
return
self._fix_deprecated_link_by_default(yamldict)
self._fix_deprecated_dotfile_link(yamldict)
return yamldict
def _fix_deprecated_link_by_default(self, yamldict):
"""fix deprecated link_by_default"""
key = 'link_by_default'
newkey = self.key_imp_link
if self.key_settings not in yamldict:
return
if not yamldict[self.key_settings]:
return
config = yamldict[self.key_settings]
if key not in config:
return
if config[key]:
config[newkey] = self.lnk_link
else:
config[newkey] = self.lnk_nolink
del config[key]
self._log.warn('deprecated \"link_by_default\"')
self._dirty = True
self._dirty_deprecated = True
def _fix_deprecated_dotfile_link(self, yamldict):
"""fix deprecated link in dotfiles"""
if self.key_dotfiles not in yamldict:
return
if not yamldict[self.key_dotfiles]:
return
for k, dotfile in yamldict[self.key_dotfiles].items():
new = self.lnk_nolink
if self.key_dotfile_link in dotfile and \
type(dotfile[self.key_dotfile_link]) is bool:
# patch link: <bool>
cur = dotfile[self.key_dotfile_link]
new = self.lnk_nolink
if cur:
new = self.lnk_link
dotfile[self.key_dotfile_link] = new
self._dirty = True
self._dirty_deprecated = True
self._log.warn('deprecated \"link\" value')
elif self.key_dotfile_link_children in dotfile and \
type(dotfile[self.key_dotfile_link_children]) is bool:
# patch link_children: <bool>
cur = dotfile[self.key_dotfile_link_children]
new = self.lnk_nolink
if cur:
new = self.lnk_children
del dotfile[self.key_dotfile_link_children]
dotfile[self.key_dotfile_link] = new
self._dirty = True
self._dirty_deprecated = True
self._log.warn('deprecated \"link_children\" value')
########################################################
# yaml utils
########################################################
def _prepare_to_save(self, content):
content = self._clear_none(content)
# make sure we have the base entries
if self.key_settings not in content:
content[self.key_settings] = None
if self.key_dotfiles not in content:
content[self.key_dotfiles] = None
if self.key_profiles not in content:
content[self.key_profiles] = None
return content
def _load_yaml(self, path):
"""load a yaml file to a dict"""
content = {}
if self._debug:
self._dbg('----------start:{}----------'.format(path))
cfg = '\n'
with open(path, 'r') as f:
for line in f:
cfg += line
self._dbg(cfg.rstrip())
self._dbg('----------end:{}----------'.format(path))
try:
content = self._yaml_load(path)
except Exception as e:
self._log.err(e)
raise YamlException('invalid config: {}'.format(path))
return content
def _yaml_load(self, path):
"""load from yaml"""
with open(path, 'r') as f:
y = yaml()
y.typ = 'rt'
content = y.load(f)
return content
def _yaml_dump(self, content, where):
"""dump to yaml"""
y = yaml()
y.default_flow_style = False
y.indent = 2
y.typ = 'rt'
y.dump(content, where)
########################################################
# templating
########################################################
def _redefine_templater(self):
"""create templater based on current variables"""
fufile = self.settings[Settings.key_func_file]
fifile = self.settings[Settings.key_filter_file]
self._tmpl = Templategen(variables=self.variables,
func_file=fufile,
filter_file=fifile)
def _template_item(self, item, exc_if_fail=True):
"""
template an item using the templategen
will raise an exception if template failed and exc_if_fail
"""
if not Templategen.var_is_template(item):
return item
try:
val = item
while Templategen.var_is_template(val):
val = self._tmpl.generate_string(val)
except UndefinedException as e:
if exc_if_fail:
raise e
return val
def _template_list(self, entries):
"""template a list of entries"""
new = []
if not entries:
return new
for e in entries:
et = self._template_item(e)
if self._debug and e != et:
self._dbg('resolved: {} -> {}'.format(e, et))
new.append(et)
return new
def _template_dict(self, entries):
"""template a dictionary of entries"""
new = {}
if not entries:
return new
for k, v in entries.items():
vt = self._template_item(v)
if self._debug and v != vt:
self._dbg('resolved: {} -> {}'.format(v, vt))
new[k] = vt
return new
def _template_dotfiles_paths(self):
"""template dotfiles paths"""
if self._debug:
self._dbg('templating dotfiles paths')
dotfiles = self.dotfiles.copy()
# make sure no dotfiles path is None
for dotfile in dotfiles.values():
src = dotfile[self.key_dotfile_src]
if src is None:
dotfile[self.key_dotfile_src] = ''
dst = dotfile[self.key_dotfile_dst]
if dst is None:
dotfile[self.key_dotfile_dst] = ''
# only keep dotfiles related to the selected profile
pdfs = []
pro = self.profiles.get(self._profile, [])
if pro:
pdfs = list(pro.get(self.key_profile_dotfiles, []))
for addpro in self._inc_profiles:
pro = self.profiles.get(addpro, [])
if not pro:
continue
pdfsalt = pro.get(self.key_profile_dotfiles, [])
pdfs.extend(pdfsalt)
pdfs = uniq_list(pdfs)
if self.key_all not in pdfs:
# take a subset of the dotfiles
newdotfiles = {}
for k, v in dotfiles.items():
if k in pdfs:
newdotfiles[k] = v
dotfiles = newdotfiles
for dotfile in dotfiles.values():
# src
src = dotfile[self.key_dotfile_src]
newsrc = self.resolve_dotfile_src(src, templater=self._tmpl)
dotfile[self.key_dotfile_src] = newsrc
# dst
dst = dotfile[self.key_dotfile_dst]
newdst = self.resolve_dotfile_dst(dst, templater=self._tmpl)
dotfile[self.key_dotfile_dst] = newdst
def _rec_resolve_variables(self, variables):
"""recursive resolve variables"""
var = self._enrich_vars(variables, self._profile)
# use a separated templategen to handle variables
# resolved outside the main config
t = Templategen(variables=var,
func_file=self.settings[Settings.key_func_file],
filter_file=self.settings[Settings.key_filter_file])
for k in variables.keys():
val = variables[k]
while Templategen.var_is_template(val):
val = t.generate_string(val)
variables[k] = val
t.update_variables(variables)
if variables is self.variables:
self._redefine_templater()
def _get_profile_included_vars(self):
"""resolve profile included variables/dynvariables"""
for k, v in self.profiles.items():
if self.key_profile_include in v and v[self.key_profile_include]:
new = []
for x in v[self.key_profile_include]:
new.append(self._tmpl.generate_string(x))
v[self.key_profile_include] = new
# now get the included ones
pro_var = self._get_profile_included_item(self.key_profile_variables)
pro_dvar = self._get_profile_included_item(self.key_profile_dvariables)
# the included profiles
inc_profiles = []
if self._profile and self._profile in self.profiles.keys():
pentry = self.profiles.get(self._profile)
inc_profiles = pentry.get(self.key_profile_include, [])
# exec incl dynvariables
return inc_profiles, pro_var, pro_dvar
########################################################
# helpers
########################################################
def _clear_profile_vars(self, dic):
"""
remove profile variables from dic if found inplace
to avoid profile variables being overwriten
"""
if not dic:
return
[dic.pop(k, None) for k in self._profilevarskeys]
def _parse_extended_import_path(self, path_entry):
"""Parse an import path in a tuple (path, fatal_not_found)."""
if self._debug:
self._dbg('parsing path entry {}'.format(path_entry))
path, _, attribute = path_entry.rpartition(self.key_import_sep)
fatal_not_found = attribute != self.key_import_ignore_key
is_valid_attribute = attribute in ('', self.key_import_ignore_key)
if not is_valid_attribute:
# If attribute is not valid it can mean that:
# - path_entry doesn't contain the separator, and attribute is set
# to the whole path by str.rpartition
# - path_entry contains a separator, but it's in the file path, so
# attribute is set to whatever comes after the separator by
# str.rpartition
# In both cases, path_entry is the path we're looking for.
if self._debug:
self._dbg('using attribute default values for path {}'.format(
path_entry))
path = path_entry
fatal_not_found = self.key_import_fatal_not_found
elif self._debug:
self._dbg(
'path entry {} has fatal_not_found flag set to {}'.format(
path_entry, fatal_not_found))
return path, fatal_not_found
def _handle_non_existing_path(self, path, fatal_not_found=True):
"""Raise an exception or log a warning to handle non-existing paths."""
error = 'bad path {}'.format(path)
if fatal_not_found:
raise YamlException(error)
self._log.warn(error)
def _check_path_existence(self, path, fatal_not_found=True):
"""Check if a path exists, raising if necessary."""
if os.path.exists(path):
if self._debug:
self._dbg('path {} exists'.format(path))
return path
self._handle_non_existing_path(path, fatal_not_found)
# Explicit return for readability. Anything evaluating to false is ok.
return None
def _process_path(self, path_entry):
"""
This method processed a path entry. Namely it:
- Normalizes the path.
- Expands globs.
- Checks for path existence, taking in account fatal_not_found.
This method always returns a list containing only absolute paths
existing on the filesystem. If the input is not a glob, the list
contains at most one element, otheriwse it could hold more.
"""
path, fatal_not_found = self._parse_extended_import_path(path_entry)
path = self._norm_path(path)
paths = self._glob_path(path) if self._is_glob(path) else [path]
if not paths:
if self._debug:
self._dbg("glob path {} didn't expand".format(path))
self._handle_non_existing_path(path, fatal_not_found)
return []
checked_paths = (self._check_path_existence(p, fatal_not_found)
for p in paths)
return [p for p in checked_paths if p]
def _resolve_paths(self, paths):
"""
This function resolves a list of paths. This means normalizing,
expanding globs and checking for existence, taking in account
fatal_not_found flags.
"""
processed_paths = (self._process_path(p) for p in paths)
return list(chain.from_iterable(processed_paths))
def _merge_dict(self, high, low):
"""merge high and low dict"""
if not high:
high = {}
if not low:
low = {}
return {**low, **high}
def _get_entry(self, dic, key, mandatory=True):
"""return copy of entry from yaml dictionary"""
if key not in dic:
if mandatory:
raise YamlException('invalid config: no {} found'.format(key))
dic[key] = {}
return deepcopy(dic[key])
if mandatory and not dic[key]:
# ensure is not none
dic[key] = {}
return deepcopy(dic[key])
def _clear_none(self, dic):
"""recursively delete all none/empty values in a dictionary."""
new = {}
for k, v in dic.items():
if k == self.key_dotfile_src:
# allow empty dotfile src
new[k] = v
continue
if k == self.key_dotfile_dst:
# allow empty dotfile dst
new[k] = v
continue
newv = v
if isinstance(v, dict):
# recursive travers dict
newv = self._clear_none(v)
if not newv:
# no empty dict
continue
if newv is None:
# no None value
continue
if isinstance(newv, list) and not newv:
# no empty list
continue
new[k] = newv
return new
def _is_glob(self, path):
"""Quick test if path is a glob."""
return '*' in path or '?' in path
def _glob_path(self, path):
"""Expand a glob."""
if self._debug:
self._dbg('expanding glob {}'.format(path))
expanded_path = os.path.expanduser(path)
return glob.glob(expanded_path, recursive=True)
def _norm_path(self, path):
"""Resolve a path either absolute or relative to config path"""
if not path:
return path
path = os.path.expanduser(path)
if not os.path.isabs(path):
d = os.path.dirname(self._path)
ret = os.path.join(d, path)
if self._debug:
msg = 'normalizing relative to cfg: {} -> {}'
self._dbg(msg.format(path, ret))
return ret
ret = os.path.normpath(path)
if self._debug and path != ret:
self._dbg('normalizing: {} -> {}'.format(path, ret))
return ret
def _shell_exec_dvars(self, dic, keys=[]):
"""shell execute dynvariables in-place"""
if not keys:
keys = dic.keys()
for k in keys:
v = dic[k]
ret, out = shell(v, debug=self._debug)
if not ret:
err = 'var \"{}: {}\" failed: {}'.format(k, v, out)
self._log.err(err)
raise YamlException(err)
if self._debug:
self._dbg('{}: `{}` -> {}'.format(k, v, out))
dic[k] = out
def _check_minversion(self, minversion):
if not minversion:
return
try:
cur = tuple([int(x) for x in VERSION.split('.')])
cfg = tuple([int(x) for x in minversion.split('.')])
except Exception:
err = 'bad version: \"{}\" VS \"{}\"'.format(VERSION, minversion)
raise YamlException(err)
if cur < cfg:
err = 'current dotdrop version is too old for that config file.'
err += ' Please update.'
raise YamlException(err)
def _debug_entries(self):
"""debug print all interesting entries"""
if not self._debug:
return
self._dbg('Current entries')
self._debug_dict('entry settings', self.settings)
self._debug_dict('entry dotfiles', self.dotfiles)
self._debug_dict('entry profiles', self.profiles)
self._debug_dict('entry actions', self.actions)
self._debug_dict('entry trans_r', self.trans_r)
self._debug_dict('entry trans_w', self.trans_w)
self._debug_dict('entry variables', self.variables)
def _debug_dict(self, title, elems):
"""pretty print dict"""
if not self._debug:
return
self._dbg('{}:'.format(title))
if not elems:
return
for k, v in elems.items():
self._dbg('\t- \"{}\": {}'.format(k, v))
def _dbg(self, content):
pre = os.path.basename(self._path)
self._log.dbg('[{}] {}'.format(pre, content))