def factory(handler, registry): get = morph.pick(registry.settings, prefix=CONFIG_PREFIX).get conf = aadict() conf.enabled = asbool(get('enabled', True)) conf.include = [globre.compile(el, globre.EXACT) for el in aslist(get('include', []))] conf.exclude = [globre.compile(el, globre.EXACT) for el in aslist(get('exclude', []))] conf.reparse = aslist(get('reparse-methods', DEFAULT_REPARSE_METHODS)) conf.name = get('attribute-name', DEFAULT_ATTRIBUTE_NAME) conf.deep = asbool(get('combine.deep', True)) conf.reqdict = asbool(get('require-dict', True)) conf.failunk = asbool(get('fail-unknown', True)) conf.ndict = asbool(get('native-dict', False)) conf.error = get('error-handler', None) if conf.error: conf.error = asset.symbol(conf.error) conf.xfmt = asbool(get('xml.enable', True)) conf.jfmt = asbool(get('json.enable', True)) conf.yfmt = asbool(get('yaml.enable', bool(yaml or get('yaml.parser')))) if conf.jfmt: conf.jparser = get('json.parser', None) if conf.jparser: conf.jparser = asset.symbol(conf.jparser) if conf.yfmt: conf.yparser = asset.symbol(get('yaml.parser', 'yaml.load')) if conf.xfmt: conf.xparser = asset.symbol(get('xml.parser', 'xml.etree.ElementTree.fromstring')) def input_tween(request): return process(handler, request, conf) return input_tween
def main(args=None): cli = argparse.ArgumentParser( description=__doc__, epilog='ex: python cli.py train settings.ini -d train_set.csv') cli.add_argument('command', metavar='COMMAND', help='{ train | test | predict | purge }', choices=(TRAIN, TEST, PREDICT, PURGE)) cli.add_argument('config', metavar='CONFIG', help='config file with settings') cli.add_argument('-d', '--dataset', metavar='DATASET', help='dataset file') options = cli.parse_args(args=args) cp = ConfigParser.RawConfigParser() cp.read(options.config) settings = aadict(cp.items('nlu_trainer')) trainer = asset.symbol(settings.driver)(settings) predictions = [] if options.command in (TRAIN, TEST, PREDICT): with open(options.dataset, 'rb') as csv_file: r = csv.reader(csv_file) if options.command == TRAIN: trainer.train_set(settings.app_id, r) elif options.command == TEST: trainer.test_set(settings.app_id, r) elif options.command == PREDICT: predictions = trainer.predict_set(settings.app_id, r) elif options.command == PURGE: purge(trainer, settings) if options.command == PREDICT: w = csv.writer(sys.stdout) w.writerows(predictions)
def execPublishingPointEngine(self, pubpnt, engconf): engine = engconf.engine engsym = str2sym(engine) handler_options = self.adjustConfig(pubpnt, engconf, engsym) # BIG TODO: there is currently a problem here. if the system # configuration disables an engine, via say # 'engines.email.overrides.enabled = false' but the # pubpoint general config (not engine-specific) adds # enabled=true, then the option is incorrectly set to # true for context.enabled. note that this is currently # not a problem for `enabled` as that gets checked # early. *BUT* other options are not controlled like # that... if not handler_options.get('enabled', True): self.log.info('engine "%s" disabled - skipping', engine) return # stopOnFail = pubobj.config.get('stopOnFail', False) try: pubname = engconf.get('engine-handler', 'svnpublish.engine.' + engsym + '.publish_' + engsym) self.log.info('loading engine "%s" (callable: %s)', engine, pubname) handler = symbol(pubname) if self.options.dryrun: pubname += '_dryrun' try: handler = symbol(pubname) except ImportError: self.log.warning('engine "%s" does not support dry-run mode - skipping', engine) return except ImportError: raise api.EngineLoadError( 'svnpublish engine "%s" does not exist or could not be loaded' % (engine,), cause=sys.exc_info()) context = aadict(pubpnt.config).update(handler_options).update({ 'options' : handler_options, 'logger' : logging.getLogger('svnpublish.' + engine), # BEGIN: backward compat, 'repository' : self.svnrev.svn.repos, 'revision' : self.svnrev.rev, 'revinfo' : pubpnt.revinfo, 'root' : pubpnt.root, # END: backward compat, 'pubpoint' : pubpnt, 'svnpub' : self, }) handler(context)
def _getOptions(context): if __name__ in context: return context[__name__] options = aadict.d2ar(morph.pick(context.options, prefix='access.')) # `options.groups` = all known groups LUT options.groups = { k : aadict({'docorator': v, 'class': list(_docorator2classes(v))[0]}) for k, v in morph.pick(options, prefix='group.').items() } # `options.default` = default access for endpoint, type, and attribute options.default = aadict({ node : morph.tolist(options.get('default.' + node)) for node in ('endpoint', 'type', 'attribute') }) # `options.rank.groups` = ordered list of group ranking (most-public to least-public) options.rank = aadict(groups=morph.tolist(options.rank)) # `options.rank.classes` = ordered list of class ranking options.rank.classes = [options.groups[grp]['class'] for grp in options.rank.groups] # `options.rank.docorators` = ordered list of docorator ranking options.rank.docorators = [options.groups[grp]['docorator'] for grp in options.rank.groups] # `options.default[NODE].(groups|classes|docorators)` for node, groups in list(options.default.items()): options.default[node] = aadict( groups = groups, classes = [options.groups[grp]['class'] for grp in groups], docorators = [options.groups[grp]['docorator'] for grp in groups], ) # `options.classes` = all known access classes options.classes = [ group['class'] for group in options.groups.values()] # `options.docorators` = all known access docorators options.docorators = [ group['docorator'] for group in options.groups.values()] # `options.request` = current request information options.request = aadict() # `options.control` = request-to-group-access callback if not options.control: options.control = _defaultAccessControl if options.control == GLOBAL_ACCESS: options.request.groups = options.groups.keys() else: options.control = asset.symbol(options.control) options.request.groups = options.control(context.request, context=context) # `options.request.classes` = the classes this request has access to options.request.classes = [ options.groups[group]['class'] for group in options.request.groups if group in options.groups ] # `options.request.docorators` = the docorators this request has access to options.request.docorators = [ options.groups[group]['docorator'] for group in options.request.groups if group in options.groups ] context[__name__] = options return options
def autoresolve(obj): if isstr(obj): return obj if isinstance(obj, (list, tuple)): return [autoresolve(e) for e in obj] if not isinstance(obj, dict) or 'class' not in obj: return obj params = {k: autoresolve(v) for k, v in obj.items() if k != 'class'} return symbol(obj['class'])(**params)
def setUp(self): super(LuisApiTestCase, self).setUp() inifile = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../test.ini') cp = ConfigParser.RawConfigParser() cp.read(inifile) settings = aadict(cp.items('nlu_trainer')) self.settings = settings self.trainer = asset.symbol(settings.driver)(settings)
def loadExtension(self, spec): log.debug('loading type registry extensions from: %r', spec) try: sym = asset.symbol(spec) return sym(self) except (ImportError, AttributeError): pass try: return self.loadExtensionString(asset.load(spec).read(), source=spec) except (ImportError, AttributeError, ValueError): pass return self.loadExtensionString(spec)
def run_engines(params, paramkey, engtype, args=None, prefix=None, evals=None, env=None): if params.get(paramkey) is None: return if args is None: args = [] engines = asList(params.get(paramkey)) engine_env = evalEnv(params, prefix, params.get(paramkey + '-env'), params.get(paramkey + '-env+'), evals, env) for engcfg in engines: if isstr(engcfg): engcfg = aadict(engine='shell', command=engcfg) engine = engcfg.engine engsafe = str2sym(engine) funcname = engcfg.get('engine-handler', 'svnpublish.engine.%s.%s_%s' % (engsafe, engtype, engsafe)) params.logger.info('loading %s engine "%s" (%s)', engtype, engine, funcname) handler = symbol(funcname) if params.dryrun: funcname += '_dryrun' try: handler = symbol(funcname + '_dryrun') except ImportError: params.logger.debug('engine "%s" does not support dry-run mode - skipping', engine) continue engine_params = aadict(params).update(engcfg).update({ 'options' : engcfg, 'logger' : logging.getLogger('.'.join((params.logger.name, engtype, engine))), 'prefix' : prefix, 'env' : engine_env, 'evals' : evals, }) handler(engine_params, *args)
def __init__(self, settings=None): if settings is None: settings = dict() self.threshold = float(settings.get('threshold', DEFAULT_INFLECT)) factors = settings.get('factors') or DEFAULT_FACTORS if asset.isstr(factors): factors = [s.strip() for s in factors.split(',')] for key in settings.keys(): key = key.split('.') if len(key) >= 3 \ and key[0] == 'factor' and key[2] == 'class' \ and key[1] not in factors: factors.append(key[1]) self.factors = [self._load(factor, settings) for factor in factors] self.logger = asset.symbol(settings.get('logger')) self.pessimism = 1.0 / float(settings.get('pessimism', 10))
def _load(self, factor, settings): predef = { 'length' : LengthFactor, 'charmix' : CharmixFactor, 'casemix' : CasemixFactor, 'variety' : VarietyFactor, 'notword' : NotWordFactor, 'phrase' : PhraseFactor, } params = dict() if asset.isstr(factor): for key, value in settings.items(): if key.startswith('factor.' + factor + '.'): params[key[len(factor) + 8:]] = value factor = params.pop('class', factor) if factor in predef: return predef[factor](**params) return asset.symbol(factor)(**params)
def _load(self, factor, settings): predef = { 'length': LengthFactor, 'charmix': CharmixFactor, 'casemix': CasemixFactor, 'variety': VarietyFactor, 'notword': NotWordFactor, 'phrase': PhraseFactor, } params = dict() if asset.isstr(factor): for key, value in settings.items(): if key.startswith('factor.' + factor + '.'): params[key[len(factor) + 8:]] = value factor = params.pop('class', factor) if factor in predef: return predef[factor](**params) return asset.symbol(factor)(**params)
def publish_restsync(params): ''' Configurable parameters: :Parameters: TODO: add docs... urlHandlers : symbol-spec request-filters : dict callable : str args : dict ''' # steps: # - get list of objects to PUT and DELETE # - order the DELETEs depth-first (so that files/subdirectories get # deleted before the containing directories) # todo: should symlinks be copied?... the problem with that is that then # any changes in the target need to be propagated into the symlink... # todo: what about fixations?... base = params.get('remote-base') if not base.endswith('/'): base += '/' params.logger.info('synchronizing to REST server: ' + base) dels = [] puts = [] for c in params.revinfo.changes: # (NONE, ADDED, MODIFIED, DELETED, CONFLICT, MERGED, EXISTED) = range(7) if c.content in [RepositoryChange.Content.ADDED, RepositoryChange.Content.MODIFIED]: puts.append(c) continue if c.content in [RepositoryChange.Content.DELETED]: if not c.isdir: dels.append(c.path) continue # TODO: replace this with the svnlook() method?... # or better yet, create a RevisionInfo.changes_explicit dels.extend(run('svnlook', 'tree', '--full-paths', '--revision', str(int(params.revision) - 1), params.repository, c.path).strip().split('\n')) continue params.logger.warn('ignoring unknown change type "%s" for entry "%s"' % (RepositoryChange.Content.strmap[c.content], c.path)) continue def delsort(a, b): if len(a) != len(b): if a.startswith(b): return -1 if b.startswith(a): return 1 if a.__gt__(b): return 1 if a.__lt__(b): return -1 return 0 dels.sort(cmp=delsort) excludes = [re.compile(expr) for expr in asList(params.excludes)] includes = [re.compile(expr) for expr in asList(params.includes)] def syncPath(path): if params.root == path: relpath = '' else: if not path.startswith(params.root + '/'): raise NotInContext(params.root, path) relpath = path[len(params.root) + 1:] ret = True for e in excludes: if not e.search(relpath): continue ret = False break if ret: return relpath for i in includes: if not i.search(relpath): continue return relpath params.logger.debug('excluding %s%s' % (base, relpath)) return None # load request decorator... # todo: perhaps it would be better to use the default opener... # that way the unittest could just use that instead of this # bogus urlHandler param... #opener = urllib2.OpenerDirector() #opener.add_handler(urllib2.HTTPCookieProcessor(cookieJar)) opener = urllib2.build_opener(urllib2.HTTPHandler) opener.add_handler(urllib2.HTTPSHandler()) for handler in asList(params.urlHandlers): opener.add_handler(asset.symbol(handler)()) # TODO: i should enforce HTTPS certificate verification... filters = [aadict(spec=s) for s in asList(params.get('request-filters'))] for fltr in filters: # TODO: convert to use asset.symbol()... i.e.: # fltr.handler = asset.symbol(fltr.spec.callable) modname = fltr.spec.callable.split(':')[0] module = __import__(modname) if modname.find('.') >= 0: for n in modname.split('.')[1:]: module = getattr(module, n) fltr.handler = getattr(module, fltr.spec.callable.split(':')[1]) errors = [] for entry in dels: path = syncPath(entry) if path is None: continue url = '%s%s' % (base, path) while url.endswith('/'): url = url[:-1] if params.dryrun: params.logger.info('dry-run: DELETE: %s' % (url,)) continue params.logger.debug('DELETE: %s' % (url,)) try: request = urllib2.Request(url) request.add_header('Content-Type', 'application/octet-stream') request.get_method = lambda: 'DELETE' for fltr in filters: request = fltr.handler(request, **(fltr.spec.args)) resp = opener.open(request) except urllib2.HTTPError, e: errors.append(RestError('DELETE', url, str(e.code) + ':' + e.msg)) except urllib2.URLError, e: errors.append(RestError('DELETE', url, e.reason))
def _getOptions(context): if __name__ in context: return context[__name__] options = aadict.d2ar(morph.pick(context.options, prefix='access.')) # `options.groups` = all known groups LUT options.groups = { k: aadict({ 'docorator': v, 'class': list(_docorator2classes(v))[0] }) for k, v in morph.pick(options, prefix='group.').items() } # `options.default` = default access for endpoint, type, and attribute options.default = aadict({ node: morph.tolist(options.get('default.' + node)) for node in ('endpoint', 'type', 'attribute') }) # `options.rank.groups` = ordered list of group ranking (most-public to least-public) options.rank = aadict(groups=morph.tolist(options.rank)) # `options.rank.classes` = ordered list of class ranking options.rank.classes = [ options.groups[grp]['class'] for grp in options.rank.groups ] # `options.rank.docorators` = ordered list of docorator ranking options.rank.docorators = [ options.groups[grp]['docorator'] for grp in options.rank.groups ] # `options.default[NODE].(groups|classes|docorators)` for node, groups in list(options.default.items()): options.default[node] = aadict( groups=groups, classes=[options.groups[grp]['class'] for grp in groups], docorators=[options.groups[grp]['docorator'] for grp in groups], ) # `options.classes` = all known access classes options.classes = [group['class'] for group in options.groups.values()] # `options.docorators` = all known access docorators options.docorators = [ group['docorator'] for group in options.groups.values() ] # `options.request` = current request information options.request = aadict() # `options.control` = request-to-group-access callback if not options.control: options.control = _defaultAccessControl if options.control == GLOBAL_ACCESS: options.request.groups = options.groups.keys() else: options.control = asset.symbol(options.control) options.request.groups = options.control(context.request, context=context) # `options.request.classes` = the classes this request has access to options.request.classes = [ options.groups[group]['class'] for group in options.request.groups if group in options.groups ] # `options.request.docorators` = the docorators this request has access to options.request.docorators = [ options.groups[group]['docorator'] for group in options.request.groups if group in options.groups ] context[__name__] = options return options