def upgrade( self ):
warnings = OrderedDict()
do_warn = False
for vn, upgs in self.upgrades.items():
warnings[vn] = []
for u in upgs:
try:
exp = self.expand(u)
except:
continue
for upg in exp:
try:
old = self.get_item( upg['old'] )
except:
# OK: deprecated item not found
pass
else:
msg = self.show_keys(upg['old'])
if upg['new']:
msg += ' -> ' + self.show_keys(upg['new'])
else:
upg['new'] = upg['old']
msg += " - " + upg['cvt'].describe()
warnings[vn].append( msg )
do_warn = True
self.del_item( upg['old'] )
if upg['cvt'].describe() != "DELETED (OBSOLETE)":
self.put_item( upg['new'], upg['cvt'].convert(old) )
if do_warn and cylc.flags.verbose:
print >> sys.stderr, "WARNING: deprecated items were automatically upgraded in '" + self.descr + "':"
for vn,msgs in warnings.items():
for m in msgs:
print >> sys.stderr, " * (" + vn + ")", m
def upgrade(self):
warnings = OrderedDict()
for vn, upgs in self.upgrades.items():
for u in upgs:
try:
exp = self.expand(u)
except (KeyError, UpgradeError):
continue
for upg in exp:
try:
old = self.get_item(upg['old'])
except KeyError:
# OK: deprecated item not found
pass
else:
msg = self.show_keys(upg['old'])
if upg['new']:
msg += ' -> ' + self.show_keys(upg['new'])
else:
upg['new'] = upg['old']
msg += " - " + upg['cvt'].describe()
if not upg['silent']:
warnings.setdefault(vn, [])
warnings[vn].append(msg)
self.del_item(upg['old'])
if upg['cvt'].describe() != "DELETED (OBSOLETE)":
self.put_item(upg['new'], upg['cvt'].convert(old))
if warnings:
level = WARNING
if self.descr == self.SITE_CONFIG:
# Site level configuration, user cannot easily fix.
# Only log at debug level.
level = DEBUG
else:
# User level configuration, user should be able to fix.
# Log at warning level.
level = WARNING
LOG.log(
level,
"deprecated items were automatically upgraded in '%s':",
self.descr)
for vn, msgs in warnings.items():
for msg in msgs:
LOG.log(level, ' * (%s) %s', vn, msg)
def upgrade(self):
warnings = OrderedDict()
for vn, upgs in self.upgrades.items():
for u in upgs:
try:
exp = self.expand(u)
except (KeyError, UpgradeError):
continue
for upg in exp:
try:
old = self.get_item(upg['old'])
except KeyError:
# OK: deprecated item not found
pass
else:
msg = self.show_keys(upg['old'])
if upg['new']:
msg += ' -> ' + self.show_keys(upg['new'])
else:
upg['new'] = upg['old']
msg += " - " + upg['cvt'].describe()
if not upg['silent']:
warnings.setdefault(vn, [])
warnings[vn].append(msg)
self.del_item(upg['old'])
if upg['cvt'].describe() != "DELETED (OBSOLETE)":
self.put_item(upg['new'], upg['cvt'].convert(old))
if warnings:
level = WARNING
if self.descr == self.SITE_CONFIG:
# Site level configuration, user cannot easily fix.
# Only log at debug level.
level = DEBUG
else:
# User level configuration, user should be able to fix.
# Log at warning level.
level = WARNING
LOG.log(
level,
"deprecated items were automatically upgraded in '%s':",
self.descr)
for vn, msgs in warnings.items():
for msg in msgs:
LOG.log(level, ' * (%s) %s', vn, msg)
def get_summary_str(self):
"""Return the poll context as a summary string delimited by "|"."""
ret = OrderedDict()
for key in self.CONTEXT_ATTRIBUTES:
value = getattr(self, key)
if key == 'job_log_dir' or value is None:
continue
ret[key] = value
return '%s|%s' % (self.job_log_dir, json.dumps(ret))
def upgrade(self):
warnings = OrderedDict()
do_warn = False
for vn, upgs in self.upgrades.items():
warnings[vn] = []
for u in upgs:
try:
exp = self.expand(u)
except (KeyError, UpgradeError):
continue
for upg in exp:
try:
old = self.get_item(upg['old'])
except KeyError:
# OK: deprecated item not found
pass
else:
msg = self.show_keys(upg['old'])
if upg['new']:
msg += ' -> ' + self.show_keys(upg['new'])
else:
upg['new'] = upg['old']
msg += " - " + upg['cvt'].describe()
if not upg['silent']:
warnings[vn].append(msg)
do_warn = True
self.del_item(upg['old'])
if upg['cvt'].describe() != "DELETED (OBSOLETE)":
self.put_item(upg['new'], upg['cvt'].convert(old))
if do_warn and cylc.flags.verbose:
sys.stderr.write(
'WARNING: deprecated items were automatically upgraded in' +
" '%s':\n" % self.descr)
for vn, msgs in warnings.items():
for m in msgs:
sys.stderr.write(' * (%s) %s\n' % (vn, m))
def upgrade(self):
warnings = OrderedDict()
do_warn = False
for vn, upgs in self.upgrades.items():
warnings[vn] = []
for u in upgs:
try:
exp = self.expand(u)
except:
continue
for upg in exp:
try:
old = self.get_item(upg['old'])
except:
# OK: deprecated item not found
pass
else:
msg = self.show_keys(upg['old'])
if upg['new']:
msg += ' -> ' + self.show_keys(upg['new'])
else:
upg['new'] = upg['old']
msg += " - " + upg['cvt'].describe()
if not upg['silent']:
warnings[vn].append(msg)
do_warn = True
self.del_item(upg['old'])
if upg['cvt'].describe() != "DELETED (OBSOLETE)":
self.put_item(upg['new'], upg['cvt'].convert(old))
if do_warn and cylc.flags.verbose:
print >> sys.stderr, (
"WARNING: deprecated items were automatically upgraded in '" +
self.descr + "':")
for vn, msgs in warnings.items():
for m in msgs:
print >> sys.stderr, " * (" + vn + ")", m
def upgrade(self):
warnings = OrderedDict()
do_warn = False
for vn, upgs in self.upgrades.items():
warnings[vn] = []
for u in upgs:
try:
exp = self.expand(u)
except (KeyError, UpgradeError):
continue
for upg in exp:
try:
old = self.get_item(upg['old'])
except KeyError:
# OK: deprecated item not found
pass
else:
msg = self.show_keys(upg['old'])
if upg['new']:
msg += ' -> ' + self.show_keys(upg['new'])
else:
upg['new'] = upg['old']
msg += " - " + upg['cvt'].describe()
if not upg['silent']:
warnings[vn].append(msg)
do_warn = True
self.del_item(upg['old'])
if upg['cvt'].describe() != "DELETED (OBSOLETE)":
self.put_item(upg['new'], upg['cvt'].convert(old))
if do_warn and cylc.flags.verbose:
sys.stderr.write(
'WARNING: deprecated items were automatically upgraded in' +
" '%s':\n" % self.descr)
for vn, msgs in warnings.items():
for m in msgs:
sys.stderr.write(' * (%s) %s\n' % (vn, m))
def __init__(self, task_id, filenames, cmd_tmpls, init_active_index):
self.filenames = OrderedDict()
name_str, point_str = TaskID.split(task_id)
for filename in filenames:
try:
f_point_str, f_name_str, f_submit_num_str, f_base_name = (
filename.rsplit(os.sep, 4)[1:])
if (f_point_str == point_str and f_name_str == name_str
and int(f_submit_num_str) and f_base_name):
name = f_submit_num_str + os.sep + f_base_name
if ":" in filename:
name += " (%s)" % (filename.split(":", 1)[0])
except ValueError:
name = filename
self.filenames[name] = filename
self.init_active_index = init_active_index
self.cmd_tmpls = cmd_tmpls
logviewer.__init__(self, task_id, None,
filenames[self.init_active_index])
def __init__(self, cfg, descr):
"""Store the config dict to be upgraded if necessary."""
self.cfg = cfg
self.descr = descr
# upgrades must be ordered in case several act on the same item
self.upgrades = OrderedDict()
class upgrader(object):
"""Handles upgrading of deprecated config values."""
def __init__(self, cfg, descr):
"""Store the config dict to be upgraded if necessary."""
self.cfg = cfg
self.descr = descr
# upgrades must be ordered in case several act on the same item
self.upgrades = OrderedDict()
def deprecate(self, vn, oldkeys, newkeys=None, cvtr=None, silent=False):
if vn not in self.upgrades:
self.upgrades[vn] = []
if cvtr is None:
cvtr = converter(lambda x: x, "value unchanged") # identity
self.upgrades[vn].append({
'old': oldkeys,
'new': newkeys,
'cvt': cvtr,
'silent': silent
})
def obsolete(self, vn, oldkeys, newkeys=None, silent=False):
if vn not in self.upgrades:
self.upgrades[vn] = []
cvtr = converter(lambda x: x, "DELETED (OBSOLETE)") # identity
self.upgrades[vn].append({
'old': oldkeys,
'new': newkeys,
'cvt': cvtr,
'silent': silent
})
def get_item(self, keys):
item = self.cfg
for key in keys:
item = item[key]
return item
def put_item(self, keys, val):
item = self.cfg
for key in keys[:-1]:
if key not in item:
item[key] = {}
item = item[key]
item[keys[-1]] = val
def del_item(self, keys):
item = self.cfg
for key in keys[:-1]:
item = item[key]
del item[keys[-1]]
@staticmethod
def show_keys(keys):
return '[' + ']['.join(keys) + ']'
def expand(self, upg):
"""Expands __MANY__ items."""
if '__MANY__' not in upg['old']:
return [upg]
if upg['old'].count('__MANY__') > 1:
sys.stderr.write('%s\n' % upg['old'])
raise UpgradeError("Multiple simultaneous __MANY__ not supported")
exp_upgs = []
pre = []
post = []
many = []
i = -1
okeys = upg['old']
for k in okeys:
i += 1
if k == "__MANY__":
pre = okeys[:i]
post = okeys[i + 1:]
tmp = self.cfg
for j in pre:
tmp = tmp[j]
many = tmp.keys()
break
if not many:
exp_upgs.append(upg)
else:
i = -1
nkeys = upg['new']
if nkeys is None: # No new keys defined.
for m in many:
exp_upgs.append({
'old': pre + [m] + post,
'new': None,
'cvt': upg['cvt'],
'silent': upg['silent'],
})
return exp_upgs
npre = []
npost = []
for k in nkeys:
i += 1
if k == "__MANY__":
npre = nkeys[:i]
npost = nkeys[i + 1:]
if not npre or not npost:
raise UpgradeError('ERROR: __MANY__ mismatch')
for m in many:
exp_upgs.append({
'old': pre + [m] + post,
'new': npre + [m] + npost,
'cvt': upg['cvt'],
'silent': upg['silent'],
})
return exp_upgs
def upgrade(self):
warnings = OrderedDict()
do_warn = False
for vn, upgs in self.upgrades.items():
warnings[vn] = []
for u in upgs:
try:
exp = self.expand(u)
except (KeyError, UpgradeError):
continue
for upg in exp:
try:
old = self.get_item(upg['old'])
except KeyError:
# OK: deprecated item not found
pass
else:
msg = self.show_keys(upg['old'])
if upg['new']:
msg += ' -> ' + self.show_keys(upg['new'])
else:
upg['new'] = upg['old']
msg += " - " + upg['cvt'].describe()
if not upg['silent']:
warnings[vn].append(msg)
do_warn = True
self.del_item(upg['old'])
if upg['cvt'].describe() != "DELETED (OBSOLETE)":
self.put_item(upg['new'], upg['cvt'].convert(old))
if do_warn and cylc.flags.verbose:
sys.stderr.write(
'WARNING: deprecated items were automatically upgraded in' +
" '%s':\n" % self.descr)
for vn, msgs in warnings.items():
for m in msgs:
sys.stderr.write(' * (%s) %s\n' % (vn, m))
def get_task_class(self):
# return a task proxy class definition, to be used for
# instantiating objects of this particular task class.
base_types = []
for foo in self.modifiers + ['cycling']:
mod = __import__('cylc.task_types.' + foo, fromlist=[foo])
base_types.append(getattr(mod, foo))
tclass = type(self.name, tuple(base_types), dict())
# set class variables here
tclass.title = self.rtconfig['title']
tclass.description = self.rtconfig['description']
# For any instance-specific environment variables (note that
# [runtime][TASK][enviroment] is now held in a class variable).
tclass.env_vars = OrderedDict()
tclass.name = self.name
tclass.rtconfig = self.rtconfig
tclass.run_mode = self.run_mode
tclass.elapsed_times = []
tclass.mean_total_elapsed_time = None
tclass.intercycle = self.intercycle
tclass.max_future_prereq_offset = None
tclass.follow_on = self.follow_on_task
tclass.namespace_hierarchy = self.namespace_hierarchy
def tclass_add_prerequisites(sself, point):
# NOTE: Task objects hold all triggers defined for the task
# in all cycling graph sections in this data structure:
# self.triggers[sequence] = [list of triggers for this
# sequence]
# The list of triggers associated with sequenceX will only be
# used by a particular task if the task's cycle point is a
# valid member of sequenceX's sequence of cycle points.
# 1) non-conditional triggers
pp = plain_prerequisites(sself.id, self.start_point)
sp = plain_prerequisites(sself.id, self.start_point)
if self.sequential:
# For tasks declared 'sequential' we automatically add a
# previous-instance inter-cycle trigger, and adjust the
# cleanup cutoff (determined by inter-cycle triggers)
# accordingly.
p_next = None
adjusted = []
for seq in self.sequences:
nxt = seq.get_next_point(sself.point)
if nxt:
# may be None if beyond the sequence bounds
adjusted.append(nxt)
if adjusted:
p_next = min(adjusted)
if (sself.cleanup_cutoff is not None
and sself.cleanup_cutoff < p_next):
sself.cleanup_cutoff = p_next
p_prev = None
adjusted = []
for seq in self.sequences:
prv = seq.get_nearest_prev_point(sself.point)
if prv:
# may be None if out of sequence bounds
adjusted.append(prv)
if adjusted:
p_prev = max(adjusted)
pp.add(TaskID.get(sself.name, str(p_prev)) + ' succeeded')
for sequence in self.triggers:
for trig in self.triggers[sequence]:
if not sequence.is_valid(sself.point):
# This trigger is not used in current cycle
continue
if (trig.graph_offset_string is None or
(get_point_relative(trig.graph_offset_string, point) >=
self.start_point)):
# i.c.t. can be None after a restart, if one
# is not specified in the suite definition.
message, prereq_point = trig.get(point)
prereq_offset = prereq_point - point
if (prereq_offset > get_interval_cls().get_null() and
(sself.max_future_prereq_offset is None or
prereq_offset > sself.max_future_prereq_offset)):
sself.max_future_prereq_offset = prereq_offset
if trig.suicide:
sp.add(message)
else:
pp.add(message)
sself.prerequisites.add_requisites(pp)
sself.suicide_prerequisites.add_requisites(sp)
# 2) conditional triggers
for sequence in self.cond_triggers.keys():
for ctrig, exp in self.cond_triggers[sequence]:
foo = ctrig.keys()[0]
if not sequence.is_valid(sself.point):
# This trigger is not valid for current cycle (see NOTE just above)
continue
cp = conditional_prerequisites(sself.id, self.start_point)
for label in ctrig:
trig = ctrig[label]
if trig.graph_offset_string is not None:
is_less_than_start = (get_point_relative(
trig.graph_offset_string, point) <
self.start_point)
cp.add(
trig.get(point)[0], label, is_less_than_start)
else:
cp.add(trig.get(point)[0], label)
cp.set_condition(exp)
if ctrig[foo].suicide:
sself.suicide_prerequisites.add_requisites(cp)
else:
sself.prerequisites.add_requisites(cp)
tclass.add_prerequisites = tclass_add_prerequisites
# class init function
def tclass_init(sself,
start_point,
initial_state,
stop_point=None,
startup=False,
validate=False,
submit_num=0,
exists=False):
sself.__class__.sequences = self.sequences
sself.__class__.implicit_sequences = self.implicit_sequences
sself.startup = startup
sself.submit_num = submit_num
sself.exists = exists
sself.intercycle_offsets = self.intercycle_offsets
if startup:
# adjust up to the first on-sequence cycle point
adjusted = []
for seq in sself.__class__.sequences:
adj = seq.get_first_point(start_point)
if adj:
# may be None if out of sequence bounds
adjusted.append(adj)
if adjusted:
sself.point = min(adjusted)
sself.cleanup_cutoff = self.get_cleanup_cutoff_point(
sself.point, self.intercycle_offsets)
sself.id = TaskID.get(sself.name, str(sself.point))
else:
sself.point = None
# this task is out of sequence bounds (caller must
# check for a point of None)
return
else:
sself.point = start_point
sself.cleanup_cutoff = self.get_cleanup_cutoff_point(
sself.point, self.intercycle_offsets)
sself.id = TaskID.get(sself.name, str(sself.point))
if 'clocktriggered' in self.modifiers:
sself.clocktrigger_offset = self.clocktrigger_offset
# prerequisites
sself.prerequisites = prerequisites(self.start_point)
sself.suicide_prerequisites = prerequisites(self.start_point)
sself.add_prerequisites(sself.point)
sself.logfiles = logfiles()
for lfile in self.rtconfig['extra log files']:
sself.logfiles.add_path(lfile)
# outputs
sself.outputs = outputs(sself.id)
for outp in self.outputs:
msg = outp.get(sself.point)
if not sself.outputs.exists(msg):
sself.outputs.add(msg)
sself.outputs.register()
if stop_point:
# Manually inserted tasks may have a final cycle point set.
super(sself.__class__, sself).__init__(initial_state,
stop_point,
validate=validate)
else:
sself.stop_point = None
super(sself.__class__, sself).__init__(initial_state,
validate=validate)
sself.suite_polling_cfg = self.suite_polling_cfg
sself.reconfigure_me = False
sself.is_coldstart = self.is_coldstart
sself.set_from_rtconfig()
tclass.__init__ = tclass_init
return tclass
#!/usr/bin/env python
"""
An empty config file should successfully yield an empty sparse config dict.
"""
import os, sys
fpath = os.path.dirname(os.path.abspath(__file__))
# parsec
sys.path.append(fpath + '/../../..')
from parsec.config import config
from parsec.validate import validator as vdr
from parsec.OrderedDict import OrderedDict
SPEC = {'meta': {'title': vdr(vtype="string")}}
cfg = config(SPEC)
cfg.loadcfg("empty.rc")
if cfg.get(sparse=True) != OrderedDict():
sys.exit(1)
def __init__( self, cfg, descr ):
"""Store the config dict to be upgraded if necessary."""
self.cfg = cfg
self.descr = descr
# upgrades must be ordered in case several act on the same item
self.upgrades = OrderedDict()
class upgrader( object ):
"""Handles upgrading of deprecated config values."""
def __init__( self, cfg, descr ):
"""Store the config dict to be upgraded if necessary."""
self.cfg = cfg
self.descr = descr
# upgrades must be ordered in case several act on the same item
self.upgrades = OrderedDict()
def deprecate(self, vn, oldkeys, newkeys=None, cvtr=None, silent=False):
if vn not in self.upgrades:
self.upgrades[vn] = []
if cvtr is None:
cvtr = converter(lambda x: x, "value unchanged") # identity
self.upgrades[vn].append(
{
'old': oldkeys,
'new': newkeys,
'cvt': cvtr,
'silent': silent
}
)
def obsolete(self, vn, oldkeys, newkeys=None, silent=False):
if vn not in self.upgrades:
self.upgrades[vn] = []
cvtr = converter(lambda x: x, "DELETED (OBSOLETE)") # identity
self.upgrades[vn].append(
{
'old' : oldkeys,
'new' : newkeys,
'cvt' : cvtr,
'silent': silent
}
)
def get_item( self, keys ):
item = self.cfg
for key in keys:
item = item[key]
return item
def put_item( self, keys, val ):
item = self.cfg
for key in keys[:-1]:
if key not in item:
item[key] = {}
item = item[key]
item[keys[-1]] = val
def del_item( self, keys ):
item = self.cfg
for key in keys[:-1]:
item = item[key]
del item[keys[-1]]
def show_keys( self, keys ):
return '[' + ']['.join(keys) + ']'
def expand( self, upg ):
"""Expands __MANY__ items."""
if '__MANY__' not in upg['old']:
return [upg]
if upg['old'].count( '__MANY__' ) > 1:
print >> sys.stderr, upg['old']
raise UpgradeError( "Multiple simultaneous __MANY__ not supported" )
exp_upgs = []
pre = []
post = []
many = []
i = -1
okeys = upg['old']
for k in okeys:
i += 1
if k == "__MANY__":
pre = okeys[:i]
post = okeys[i+1:]
tmp = self.cfg
for j in pre:
tmp = tmp[j]
many = tmp.keys()
break
if not many:
exp_upgs.append( upg )
else:
i = -1
nkeys = upg['new']
npre = []
npost = []
for k in nkeys:
i += 1
if k == "__MANY__":
npre = nkeys[:i]
npost = nkeys[i+1:]
if not npre or not npost:
raise UpgradeError('ERROR: __MANY__ mismatch')
for m in many:
exp_upgs.append( {
'old': pre + [m] + post,
'new': npre + [m] + npost,
'cvt': upg['cvt'],
'silent': upg['silent']
})
return exp_upgs
def upgrade( self ):
warnings = OrderedDict()
do_warn = False
for vn, upgs in self.upgrades.items():
warnings[vn] = []
for u in upgs:
try:
exp = self.expand(u)
except:
continue
for upg in exp:
try:
old = self.get_item( upg['old'] )
except:
# OK: deprecated item not found
pass
else:
msg = self.show_keys(upg['old'])
if upg['new']:
msg += ' -> ' + self.show_keys(upg['new'])
else:
upg['new'] = upg['old']
msg += " - " + upg['cvt'].describe()
if not upg['silent']:
warnings[vn].append( msg )
do_warn = True
self.del_item( upg['old'] )
if upg['cvt'].describe() != "DELETED (OBSOLETE)":
self.put_item( upg['new'], upg['cvt'].convert(old) )
if do_warn and cylc.flags.verbose:
print >> sys.stderr, "WARNING: deprecated items were automatically upgraded in '" + self.descr + "':"
for vn,msgs in warnings.items():
for m in msgs:
print >> sys.stderr, " * (" + vn + ")", m
class upgrader(object):
"""Handles upgrading of deprecated config values."""
SITE_CONFIG = 'site config'
USER_CONFIG = 'user config'
def __init__(self, cfg, descr):
"""Store the config dict to be upgraded if necessary."""
self.cfg = cfg
self.descr = descr
# upgrades must be ordered in case several act on the same item
self.upgrades = OrderedDict()
def deprecate(self, vn, oldkeys, newkeys=None, cvtr=None, silent=False):
if vn not in self.upgrades:
self.upgrades[vn] = []
if cvtr is None:
cvtr = converter(lambda x: x, "value unchanged") # identity
self.upgrades[vn].append(
{'old': oldkeys, 'new': newkeys, 'cvt': cvtr, 'silent': silent})
def obsolete(self, vn, oldkeys, newkeys=None, silent=False):
if vn not in self.upgrades:
self.upgrades[vn] = []
cvtr = converter(lambda x: x, "DELETED (OBSOLETE)") # identity
self.upgrades[vn].append(
{'old': oldkeys, 'new': newkeys, 'cvt': cvtr, 'silent': silent})
def get_item(self, keys):
item = self.cfg
for key in keys:
item = item[key]
return item
def put_item(self, keys, val):
item = self.cfg
for key in keys[:-1]:
if key not in item:
item[key] = {}
item = item[key]
item[keys[-1]] = val
def del_item(self, keys):
item = self.cfg
for key in keys[:-1]:
item = item[key]
del item[keys[-1]]
@staticmethod
def show_keys(keys):
return '[' + ']['.join(keys) + ']'
def expand(self, upg):
"""Expands __MANY__ items."""
if '__MANY__' not in upg['old']:
return [upg]
if upg['old'].count('__MANY__') > 1:
raise UpgradeError(
'Multiple simultaneous __MANY__ not supported: %s' %
upg['old'])
exp_upgs = []
pre = []
post = []
many = []
i = -1
okeys = upg['old']
for k in okeys:
i += 1
if k == "__MANY__":
pre = okeys[:i]
post = okeys[i + 1:]
tmp = self.cfg
for j in pre:
tmp = tmp[j]
many = list(tmp.keys())
break
if not many:
exp_upgs.append(upg)
else:
i = -1
nkeys = upg['new']
if nkeys is None: # No new keys defined.
for m in many:
exp_upgs.append({
'old': pre + [m] + post,
'new': None,
'cvt': upg['cvt'],
'silent': upg['silent'],
})
return exp_upgs
npre = []
npost = []
for k in nkeys:
i += 1
if k == "__MANY__":
npre = nkeys[:i]
npost = nkeys[i + 1:]
if not npre or not npost:
raise UpgradeError('__MANY__ mismatch')
for m in many:
exp_upgs.append({
'old': pre + [m] + post,
'new': npre + [m] + npost,
'cvt': upg['cvt'],
'silent': upg['silent'],
})
return exp_upgs
def upgrade(self):
warnings = OrderedDict()
for vn, upgs in self.upgrades.items():
for u in upgs:
try:
exp = self.expand(u)
except (KeyError, UpgradeError):
continue
for upg in exp:
try:
old = self.get_item(upg['old'])
except KeyError:
# OK: deprecated item not found
pass
else:
msg = self.show_keys(upg['old'])
if upg['new']:
msg += ' -> ' + self.show_keys(upg['new'])
else:
upg['new'] = upg['old']
msg += " - " + upg['cvt'].describe()
if not upg['silent']:
warnings.setdefault(vn, [])
warnings[vn].append(msg)
self.del_item(upg['old'])
if upg['cvt'].describe() != "DELETED (OBSOLETE)":
self.put_item(upg['new'], upg['cvt'].convert(old))
if warnings:
level = WARNING
if self.descr == self.SITE_CONFIG:
# Site level configuration, user cannot easily fix.
# Only log at debug level.
level = DEBUG
else:
# User level configuration, user should be able to fix.
# Log at warning level.
level = WARNING
LOG.log(
level,
"deprecated items were automatically upgraded in '%s':",
self.descr)
for vn, msgs in warnings.items():
for msg in msgs:
LOG.log(level, ' * (%s) %s', vn, msg)
class upgrader(object):
"""Handles upgrading of deprecated config values."""
SITE_CONFIG = 'site config'
USER_CONFIG = 'user config'
def __init__(self, cfg, descr):
"""Store the config dict to be upgraded if necessary."""
self.cfg = cfg
self.descr = descr
# upgrades must be ordered in case several act on the same item
self.upgrades = OrderedDict()
def deprecate(self, vn, oldkeys, newkeys=None, cvtr=None, silent=False):
if vn not in self.upgrades:
self.upgrades[vn] = []
if cvtr is None:
cvtr = converter(lambda x: x, "value unchanged") # identity
self.upgrades[vn].append({
'old': oldkeys,
'new': newkeys,
'cvt': cvtr,
'silent': silent
})
def obsolete(self, vn, oldkeys, newkeys=None, silent=False):
if vn not in self.upgrades:
self.upgrades[vn] = []
cvtr = converter(lambda x: x, "DELETED (OBSOLETE)") # identity
self.upgrades[vn].append({
'old': oldkeys,
'new': newkeys,
'cvt': cvtr,
'silent': silent
})
def get_item(self, keys):
item = self.cfg
for key in keys:
item = item[key]
return item
def put_item(self, keys, val):
item = self.cfg
for key in keys[:-1]:
if key not in item:
item[key] = {}
item = item[key]
item[keys[-1]] = val
def del_item(self, keys):
item = self.cfg
for key in keys[:-1]:
item = item[key]
del item[keys[-1]]
@staticmethod
def show_keys(keys):
return '[' + ']['.join(keys) + ']'
def expand(self, upg):
"""Expands __MANY__ items."""
if '__MANY__' not in upg['old']:
return [upg]
if upg['old'].count('__MANY__') > 1:
raise UpgradeError(
'Multiple simultaneous __MANY__ not supported: %s' %
upg['old'])
exp_upgs = []
pre = []
post = []
many = []
i = -1
okeys = upg['old']
for k in okeys:
i += 1
if k == "__MANY__":
pre = okeys[:i]
post = okeys[i + 1:]
tmp = self.cfg
for j in pre:
tmp = tmp[j]
many = list(tmp.keys())
break
if not many:
exp_upgs.append(upg)
else:
i = -1
nkeys = upg['new']
if nkeys is None: # No new keys defined.
for m in many:
exp_upgs.append({
'old': pre + [m] + post,
'new': None,
'cvt': upg['cvt'],
'silent': upg['silent'],
})
return exp_upgs
npre = []
npost = []
for k in nkeys:
i += 1
if k == "__MANY__":
npre = nkeys[:i]
npost = nkeys[i + 1:]
if not npre or not npost:
raise UpgradeError('ERROR: __MANY__ mismatch')
for m in many:
exp_upgs.append({
'old': pre + [m] + post,
'new': npre + [m] + npost,
'cvt': upg['cvt'],
'silent': upg['silent'],
})
return exp_upgs
def upgrade(self):
warnings = OrderedDict()
for vn, upgs in self.upgrades.items():
for u in upgs:
try:
exp = self.expand(u)
except (KeyError, UpgradeError):
continue
for upg in exp:
try:
old = self.get_item(upg['old'])
except KeyError:
# OK: deprecated item not found
pass
else:
msg = self.show_keys(upg['old'])
if upg['new']:
msg += ' -> ' + self.show_keys(upg['new'])
else:
upg['new'] = upg['old']
msg += " - " + upg['cvt'].describe()
if not upg['silent']:
warnings.setdefault(vn, [])
warnings[vn].append(msg)
self.del_item(upg['old'])
if upg['cvt'].describe() != "DELETED (OBSOLETE)":
self.put_item(upg['new'], upg['cvt'].convert(old))
if warnings:
level = WARNING
if self.descr == self.SITE_CONFIG:
# Site level configuration, user cannot easily fix.
# Only log at debug level.
level = DEBUG
else:
# User level configuration, user should be able to fix.
# Log at warning level.
level = WARNING
LOG.log(level,
"deprecated items were automatically upgraded in '%s':",
self.descr)
for vn, msgs in warnings.items():
for msg in msgs:
LOG.log(level, ' * (%s) %s', vn, msg)
