def factory(handler, registry):
get = morph.pick(registry.settings, prefix=CONFIG_PREFIX).get
conf = aadict()
conf.enabled = asbool(get('enabled', True))
conf.include = [globre.compile(el, globre.EXACT)
for el in aslist(get('include', []))]
conf.exclude = [globre.compile(el, globre.EXACT)
for el in aslist(get('exclude', []))]
conf.reparse = aslist(get('reparse-methods', DEFAULT_REPARSE_METHODS))
conf.name = get('attribute-name', DEFAULT_ATTRIBUTE_NAME)
conf.deep = asbool(get('combine.deep', True))
conf.reqdict = asbool(get('require-dict', True))
conf.failunk = asbool(get('fail-unknown', True))
conf.ndict = asbool(get('native-dict', False))
conf.error = get('error-handler', None)
if conf.error:
conf.error = asset.symbol(conf.error)
conf.xfmt = asbool(get('xml.enable', True))
conf.jfmt = asbool(get('json.enable', True))
conf.yfmt = asbool(get('yaml.enable', bool(yaml or get('yaml.parser'))))
if conf.jfmt:
conf.jparser = get('json.parser', None)
if conf.jparser:
conf.jparser = asset.symbol(conf.jparser)
if conf.yfmt:
conf.yparser = asset.symbol(get('yaml.parser', 'yaml.load'))
if conf.xfmt:
conf.xparser = asset.symbol(get('xml.parser', 'xml.etree.ElementTree.fromstring'))
def input_tween(request):
return process(handler, request, conf)
return input_tween
def main(args=None):
cli = argparse.ArgumentParser(
description=__doc__,
epilog='ex: python cli.py train settings.ini -d train_set.csv')
cli.add_argument('command', metavar='COMMAND',
help='{ train | test | predict | purge }',
choices=(TRAIN, TEST, PREDICT, PURGE))
cli.add_argument('config', metavar='CONFIG', help='config file with settings')
cli.add_argument('-d', '--dataset', metavar='DATASET', help='dataset file')
options = cli.parse_args(args=args)
cp = ConfigParser.RawConfigParser()
cp.read(options.config)
settings = aadict(cp.items('nlu_trainer'))
trainer = asset.symbol(settings.driver)(settings)
predictions = []
if options.command in (TRAIN, TEST, PREDICT):
with open(options.dataset, 'rb') as csv_file:
r = csv.reader(csv_file)
if options.command == TRAIN:
trainer.train_set(settings.app_id, r)
elif options.command == TEST:
trainer.test_set(settings.app_id, r)
elif options.command == PREDICT:
predictions = trainer.predict_set(settings.app_id, r)
elif options.command == PURGE:
purge(trainer, settings)
if options.command == PREDICT:
w = csv.writer(sys.stdout)
w.writerows(predictions)
def execPublishingPointEngine(self, pubpnt, engconf):
engine = engconf.engine
engsym = str2sym(engine)
handler_options = self.adjustConfig(pubpnt, engconf, engsym)
# BIG TODO: there is currently a problem here. if the system
# configuration disables an engine, via say
# 'engines.email.overrides.enabled = false' but the
# pubpoint general config (not engine-specific) adds
# enabled=true, then the option is incorrectly set to
# true for context.enabled. note that this is currently
# not a problem for `enabled` as that gets checked
# early. *BUT* other options are not controlled like
# that...
if not handler_options.get('enabled', True):
self.log.info('engine "%s" disabled - skipping', engine)
return
# stopOnFail = pubobj.config.get('stopOnFail', False)
try:
pubname = engconf.get('engine-handler', 'svnpublish.engine.' + engsym + '.publish_' + engsym)
self.log.info('loading engine "%s" (callable: %s)', engine, pubname)
handler = symbol(pubname)
if self.options.dryrun:
pubname += '_dryrun'
try:
handler = symbol(pubname)
except ImportError:
self.log.warning('engine "%s" does not support dry-run mode - skipping', engine)
return
except ImportError:
raise api.EngineLoadError(
'svnpublish engine "%s" does not exist or could not be loaded' % (engine,),
cause=sys.exc_info())
context = aadict(pubpnt.config).update(handler_options).update({
'options' : handler_options,
'logger' : logging.getLogger('svnpublish.' + engine),
# BEGIN: backward compat,
'repository' : self.svnrev.svn.repos,
'revision' : self.svnrev.rev,
'revinfo' : pubpnt.revinfo,
'root' : pubpnt.root,
# END: backward compat,
'pubpoint' : pubpnt,
'svnpub' : self,
})
handler(context)
def _getOptions(context):
if __name__ in context:
return context[__name__]
options = aadict.d2ar(morph.pick(context.options, prefix='access.'))
# `options.groups` = all known groups LUT
options.groups = {
k : aadict({'docorator': v, 'class': list(_docorator2classes(v))[0]})
for k, v in morph.pick(options, prefix='group.').items()
}
# `options.default` = default access for endpoint, type, and attribute
options.default = aadict({
node : morph.tolist(options.get('default.' + node))
for node in ('endpoint', 'type', 'attribute')
})
# `options.rank.groups` = ordered list of group ranking (most-public to least-public)
options.rank = aadict(groups=morph.tolist(options.rank))
# `options.rank.classes` = ordered list of class ranking
options.rank.classes = [options.groups[grp]['class'] for grp in options.rank.groups]
# `options.rank.docorators` = ordered list of docorator ranking
options.rank.docorators = [options.groups[grp]['docorator'] for grp in options.rank.groups]
# `options.default[NODE].(groups|classes|docorators)`
for node, groups in list(options.default.items()):
options.default[node] = aadict(
groups = groups,
classes = [options.groups[grp]['class'] for grp in groups],
docorators = [options.groups[grp]['docorator'] for grp in groups],
)
# `options.classes` = all known access classes
options.classes = [
group['class'] for group in options.groups.values()]
# `options.docorators` = all known access docorators
options.docorators = [
group['docorator'] for group in options.groups.values()]
# `options.request` = current request information
options.request = aadict()
# `options.control` = request-to-group-access callback
if not options.control:
options.control = _defaultAccessControl
if options.control == GLOBAL_ACCESS:
options.request.groups = options.groups.keys()
else:
options.control = asset.symbol(options.control)
options.request.groups = options.control(context.request, context=context)
# `options.request.classes` = the classes this request has access to
options.request.classes = [
options.groups[group]['class']
for group in options.request.groups
if group in options.groups
]
# `options.request.docorators` = the docorators this request has access to
options.request.docorators = [
options.groups[group]['docorator']
for group in options.request.groups
if group in options.groups
]
context[__name__] = options
return options
def autoresolve(obj):
if isstr(obj):
return obj
if isinstance(obj, (list, tuple)):
return [autoresolve(e) for e in obj]
if not isinstance(obj, dict) or 'class' not in obj:
return obj
params = {k: autoresolve(v) for k, v in obj.items() if k != 'class'}
return symbol(obj['class'])(**params)
def setUp(self):
super(LuisApiTestCase, self).setUp()
inifile = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'../../test.ini')
cp = ConfigParser.RawConfigParser()
cp.read(inifile)
settings = aadict(cp.items('nlu_trainer'))
self.settings = settings
self.trainer = asset.symbol(settings.driver)(settings)
def loadExtension(self, spec):
log.debug('loading type registry extensions from: %r', spec)
try:
sym = asset.symbol(spec)
return sym(self)
except (ImportError, AttributeError):
pass
try:
return self.loadExtensionString(asset.load(spec).read(),
source=spec)
except (ImportError, AttributeError, ValueError):
pass
return self.loadExtensionString(spec)
def run_engines(params, paramkey, engtype, args=None,
prefix=None, evals=None, env=None):
if params.get(paramkey) is None:
return
if args is None:
args = []
engines = asList(params.get(paramkey))
engine_env = evalEnv(params, prefix,
params.get(paramkey + '-env'), params.get(paramkey + '-env+'),
evals, env)
for engcfg in engines:
if isstr(engcfg):
engcfg = aadict(engine='shell', command=engcfg)
engine = engcfg.engine
engsafe = str2sym(engine)
funcname = engcfg.get('engine-handler', 'svnpublish.engine.%s.%s_%s' % (engsafe, engtype, engsafe))
params.logger.info('loading %s engine "%s" (%s)', engtype, engine, funcname)
handler = symbol(funcname)
if params.dryrun:
funcname += '_dryrun'
try:
handler = symbol(funcname + '_dryrun')
except ImportError:
params.logger.debug('engine "%s" does not support dry-run mode - skipping', engine)
continue
engine_params = aadict(params).update(engcfg).update({
'options' : engcfg,
'logger' : logging.getLogger('.'.join((params.logger.name, engtype, engine))),
'prefix' : prefix,
'env' : engine_env,
'evals' : evals,
})
handler(engine_params, *args)
def __init__(self, settings=None):
if settings is None:
settings = dict()
self.threshold = float(settings.get('threshold', DEFAULT_INFLECT))
factors = settings.get('factors') or DEFAULT_FACTORS
if asset.isstr(factors):
factors = [s.strip() for s in factors.split(',')]
for key in settings.keys():
key = key.split('.')
if len(key) >= 3 \
and key[0] == 'factor' and key[2] == 'class' \
and key[1] not in factors:
factors.append(key[1])
self.factors = [self._load(factor, settings) for factor in factors]
self.logger = asset.symbol(settings.get('logger'))
self.pessimism = 1.0 / float(settings.get('pessimism', 10))
def __init__(self, settings=None):
if settings is None:
settings = dict()
self.threshold = float(settings.get('threshold', DEFAULT_INFLECT))
factors = settings.get('factors') or DEFAULT_FACTORS
if asset.isstr(factors):
factors = [s.strip() for s in factors.split(',')]
for key in settings.keys():
key = key.split('.')
if len(key) >= 3 \
and key[0] == 'factor' and key[2] == 'class' \
and key[1] not in factors:
factors.append(key[1])
self.factors = [self._load(factor, settings) for factor in factors]
self.logger = asset.symbol(settings.get('logger'))
self.pessimism = 1.0 / float(settings.get('pessimism', 10))
def _load(self, factor, settings):
predef = {
'length' : LengthFactor,
'charmix' : CharmixFactor,
'casemix' : CasemixFactor,
'variety' : VarietyFactor,
'notword' : NotWordFactor,
'phrase' : PhraseFactor,
}
params = dict()
if asset.isstr(factor):
for key, value in settings.items():
if key.startswith('factor.' + factor + '.'):
params[key[len(factor) + 8:]] = value
factor = params.pop('class', factor)
if factor in predef:
return predef[factor](**params)
return asset.symbol(factor)(**params)
def _load(self, factor, settings):
predef = {
'length': LengthFactor,
'charmix': CharmixFactor,
'casemix': CasemixFactor,
'variety': VarietyFactor,
'notword': NotWordFactor,
'phrase': PhraseFactor,
}
params = dict()
if asset.isstr(factor):
for key, value in settings.items():
if key.startswith('factor.' + factor + '.'):
params[key[len(factor) + 8:]] = value
factor = params.pop('class', factor)
if factor in predef:
return predef[factor](**params)
return asset.symbol(factor)(**params)
def publish_restsync(params):
'''
Configurable parameters:
:Parameters:
TODO: add docs...
urlHandlers : symbol-spec
request-filters : dict
callable : str
args : dict
'''
# steps:
# - get list of objects to PUT and DELETE
# - order the DELETEs depth-first (so that files/subdirectories get
# deleted before the containing directories)
# todo: should symlinks be copied?... the problem with that is that then
# any changes in the target need to be propagated into the symlink...
# todo: what about fixations?...
base = params.get('remote-base')
if not base.endswith('/'):
base += '/'
params.logger.info('synchronizing to REST server: ' + base)
dels = []
puts = []
for c in params.revinfo.changes:
# (NONE, ADDED, MODIFIED, DELETED, CONFLICT, MERGED, EXISTED) = range(7)
if c.content in [RepositoryChange.Content.ADDED,
RepositoryChange.Content.MODIFIED]:
puts.append(c)
continue
if c.content in [RepositoryChange.Content.DELETED]:
if not c.isdir:
dels.append(c.path)
continue
# TODO: replace this with the svnlook() method?...
# or better yet, create a RevisionInfo.changes_explicit
dels.extend(run('svnlook', 'tree', '--full-paths',
'--revision', str(int(params.revision) - 1),
params.repository, c.path).strip().split('\n'))
continue
params.logger.warn('ignoring unknown change type "%s" for entry "%s"'
% (RepositoryChange.Content.strmap[c.content], c.path))
continue
def delsort(a, b):
if len(a) != len(b):
if a.startswith(b):
return -1
if b.startswith(a):
return 1
if a.__gt__(b):
return 1
if a.__lt__(b):
return -1
return 0
dels.sort(cmp=delsort)
excludes = [re.compile(expr) for expr in asList(params.excludes)]
includes = [re.compile(expr) for expr in asList(params.includes)]
def syncPath(path):
if params.root == path:
relpath = ''
else:
if not path.startswith(params.root + '/'):
raise NotInContext(params.root, path)
relpath = path[len(params.root) + 1:]
ret = True
for e in excludes:
if not e.search(relpath):
continue
ret = False
break
if ret: return relpath
for i in includes:
if not i.search(relpath):
continue
return relpath
params.logger.debug('excluding %s%s' % (base, relpath))
return None
# load request decorator...
# todo: perhaps it would be better to use the default opener...
# that way the unittest could just use that instead of this
# bogus urlHandler param...
#opener = urllib2.OpenerDirector()
#opener.add_handler(urllib2.HTTPCookieProcessor(cookieJar))
opener = urllib2.build_opener(urllib2.HTTPHandler)
opener.add_handler(urllib2.HTTPSHandler())
for handler in asList(params.urlHandlers):
opener.add_handler(asset.symbol(handler)())
# TODO: i should enforce HTTPS certificate verification...
filters = [aadict(spec=s) for s in asList(params.get('request-filters'))]
for fltr in filters:
# TODO: convert to use asset.symbol()... i.e.:
# fltr.handler = asset.symbol(fltr.spec.callable)
modname = fltr.spec.callable.split(':')[0]
module = __import__(modname)
if modname.find('.') >= 0:
for n in modname.split('.')[1:]:
module = getattr(module, n)
fltr.handler = getattr(module, fltr.spec.callable.split(':')[1])
errors = []
for entry in dels:
path = syncPath(entry)
if path is None:
continue
url = '%s%s' % (base, path)
while url.endswith('/'):
url = url[:-1]
if params.dryrun:
params.logger.info('dry-run: DELETE: %s' % (url,))
continue
params.logger.debug('DELETE: %s' % (url,))
try:
request = urllib2.Request(url)
request.add_header('Content-Type', 'application/octet-stream')
request.get_method = lambda: 'DELETE'
for fltr in filters:
request = fltr.handler(request, **(fltr.spec.args))
resp = opener.open(request)
except urllib2.HTTPError, e:
errors.append(RestError('DELETE', url, str(e.code) + ':' + e.msg))
except urllib2.URLError, e:
errors.append(RestError('DELETE', url, e.reason))
def _getOptions(context):
if __name__ in context:
return context[__name__]
options = aadict.d2ar(morph.pick(context.options, prefix='access.'))
# `options.groups` = all known groups LUT
options.groups = {
k: aadict({
'docorator': v,
'class': list(_docorator2classes(v))[0]
})
for k, v in morph.pick(options, prefix='group.').items()
}
# `options.default` = default access for endpoint, type, and attribute
options.default = aadict({
node: morph.tolist(options.get('default.' + node))
for node in ('endpoint', 'type', 'attribute')
})
# `options.rank.groups` = ordered list of group ranking (most-public to least-public)
options.rank = aadict(groups=morph.tolist(options.rank))
# `options.rank.classes` = ordered list of class ranking
options.rank.classes = [
options.groups[grp]['class'] for grp in options.rank.groups
]
# `options.rank.docorators` = ordered list of docorator ranking
options.rank.docorators = [
options.groups[grp]['docorator'] for grp in options.rank.groups
]
# `options.default[NODE].(groups|classes|docorators)`
for node, groups in list(options.default.items()):
options.default[node] = aadict(
groups=groups,
classes=[options.groups[grp]['class'] for grp in groups],
docorators=[options.groups[grp]['docorator'] for grp in groups],
)
# `options.classes` = all known access classes
options.classes = [group['class'] for group in options.groups.values()]
# `options.docorators` = all known access docorators
options.docorators = [
group['docorator'] for group in options.groups.values()
]
# `options.request` = current request information
options.request = aadict()
# `options.control` = request-to-group-access callback
if not options.control:
options.control = _defaultAccessControl
if options.control == GLOBAL_ACCESS:
options.request.groups = options.groups.keys()
else:
options.control = asset.symbol(options.control)
options.request.groups = options.control(context.request,
context=context)
# `options.request.classes` = the classes this request has access to
options.request.classes = [
options.groups[group]['class'] for group in options.request.groups
if group in options.groups
]
# `options.request.docorators` = the docorators this request has access to
options.request.docorators = [
options.groups[group]['docorator'] for group in options.request.groups
if group in options.groups
]
context[__name__] = options
return options
