def faq(self, *args, **kwargs): """ represent DAS FAQ. """ section = kwargs.get('section', None) highlight = kwargs.get('highlight', None) guide = self.templatepage('dbsql_vs_dasql', operators=', '.join(das_operators())) page = self.templatepage('das_faq', guide=guide, section=section, highlight=highlight, operators=', '.join(das_operators()), aggregators=', '.join(das_aggregators())) return self.page(page, response_div=False)
def faq(self, **kwargs): """ represent DAS FAQ. """ section = kwargs.get('section', None) highlight = kwargs.get('highlight', None) guide = self.templatepage('dbsql_vs_dasql', operators=', '.join(das_operators())) daskeys = self.templatepage('das_keys', daskeys=self.daskeyslist) page = self.templatepage('das_faq', guide=guide, daskeys=daskeys, section=section, highlight=highlight, operators=', '.join(das_operators()), aggregators=', '.join(das_aggregators())) return self.page(page, response_div=False)
def __init__(self, daskeys, dassystems, operators=None, specials=None, filters=None, aggregators=None, mapreduces=None, verbose=0): super(DASQueryParser, self).__init__() self.daskeys = daskeys + ['queries', 'records'] self.verbose = verbose self.dassystems = dassystems # test if we have been given a list of desired operators/filters # /aggregators, if not get the lists from das_ql self.operators = operators if operators else das_operators() self.filters = filters if filters else das_filters() self.aggregators = aggregators if aggregators else das_aggregators() self.mapreduces = mapreduces if mapreduces else das_mapreduces() self.specials = specials if specials else das_special_keys() if self.verbose: print("operators", self.operators) print('filters', self.filters) print('mapreduces', self.mapreduces) print('specials', self.specials) self.daskeys += self.specials
def error_msg(msg, show_kws=False, tmpl='das_ambiguous', **kwargs): """ Helper function which renders an error template, default is das_ambiguous, but can be overriden via tmpl param. Template has two versions: html and text for CLI. The template is passed with msg, base, guide, and **kwargs. """ # TODO: this shall be done by inheriting a parent template # TODO: no header/footer? guide = self.templatepage('dbsql_vs_dasql', operators=', '.join(das_operators())) # render keyword search loader, if needed kws = '' if show_kws: kws = self.templatepage('kwdsearch_via_ajax', uinput=uinput, jsonize=jsonize, url_extend_params_as_dict=url_extend_params_as_dict, inst=inst or self.dbs_global, kws_host=self._get_kws_host()) # render the appropriate template (html vs text mode) page = self.templatepage(tmpl + ('_txt' if not html_mode else ''), msg=msg, base=self.base, guide=guide, kws_enabled=show_kws, kws=kws, **kwargs) return page
def __init__(self, config=None): if not config: config = das_readconfig() if not config.has_key('dasmapping'): config['dasmapping'] = DASMapping(config) if not config.has_key('dasanalytics'): config['dasanalytics'] = DASAnalytics(config) if not config['dasmapping'].check_maps(): msg = "No DAS maps found in MappingDB" raise Exception(msg) self.map = config['dasmapping'] self.analytics = config['dasanalytics'] self.dasservices = config['services'] self.daskeysmap = self.map.daskeys() self.operators = list(das_operators()) self.daskeys = list(das_special_keys()) self.verbose = config['verbose'] self.logger = PrintManager('QLManger', self.verbose) for val in self.daskeysmap.values(): for item in val: self.daskeys.append(item) parserdir = config['das']['parserdir'] self.dasply = DASPLY(parserdir, self.daskeys, self.dasservices, verbose=self.verbose) self.enabledb = config['parserdb']['enable'] if self.enabledb: self.parserdb = DASParserDB(config)
def error_msg(msg, show_kws=False, tmpl="das_ambiguous", **kwargs): """ Helper function which renders an error template, default is das_ambiguous, but can be overriden via tmpl param. Template has two versions: html and text for CLI. The template is passed with msg, base, guide, and **kwargs. """ guide = self.templatepage("dbsql_vs_dasql", operators=", ".join(das_operators())) # render keyword search loader, if needed kws = "" if show_kws: kws = self.templatepage( "kwdsearch_via_ajax", uinput=uinput, inst=inst or self.dbs_global, kws_host=self._get_kws_host() ) # render the appropriate template (html vs text mode) page = self.templatepage( tmpl + ("_txt" if not html_mode else ""), msg=msg, base=self.base, guide=guide, kws_enabled=show_kws, kws=kws, **kwargs ) return page
def __init__(self, parserdir, daskeys, dassystems, operators=None, specials=None, filters=None, aggregators=None, mapreduces=None, verbose=0): self.daskeys = daskeys self.verbose = verbose self.lexer = None # defined at run time using self.build() self.parser = None # defined at run time using self.build() self.dassystems = dassystems self.parsertab_dir = parserdir if not os.path.isdir(self.parsertab_dir): msg = 'Directory %s does not exists' % self.parsertab_dir raise Exception(msg) # test if we have been given a list of desired operators/filters # /aggregators, if not get the lists from das_ql operators = operators if operators else das_operators() filters = filters if filters else das_filters() aggregators = aggregators if aggregators else das_aggregators() mapreduces = mapreduces if mapreduces else das_mapreduces() specials = specials if specials else das_special_keys() # build a map of token->token.type which we can use in the # enlarged VALUE rule self.tokenmap = {} for o in operators: self.tokenmap[o] = 'OPERATOR' for f in filters: self.tokenmap[f] = 'FILTER' for a in aggregators: self.tokenmap[a] = 'AGGREGATOR' for m in mapreduces: self.tokenmap[m] = 'MAPREDUCE' for s in specials: self.tokenmap[s] = 'SPECIALKEY'
def faq(self, *args, **kwargs): """ represent DAS FAQ. """ page = self.templatepage('das_faq', operators=', '.join(das_operators()), aggregators=', '.join(das_aggregators())) return self.page(page, response_div=False)
def helper(msg, html_error=None): """Helper function which provide error template""" if not html_error: return msg guide = self.templatepage('dbsql_vs_dasql', operators=', '.join(das_operators())) page = self.templatepage('das_ambiguous', msg=msg, base=self.base, guide=guide) return page
def faq(self, **kwargs): """ represent DAS FAQ. """ section = kwargs.get("section", None) highlight = kwargs.get("highlight", None) guide = self.templatepage("dbsql_vs_dasql", operators=", ".join(das_operators())) daskeys = self.templatepage("das_keys", daskeys=self.daskeyslist) page = self.templatepage( "das_faq", guide=guide, daskeys=daskeys, section=section, highlight=highlight, operators=", ".join(das_operators()), aggregators=", ".join(das_aggregators()), ) return self.page(page, response_div=False)
def __init__(self, config): self.map = config['dasmapping'] self.analytics = config['dasanalytics'] self.daskeysmap = self.map.daskeys() self.operators = das_operators() self.filters = ['%s ' % f for f in das_filters()] self.aggregators = das_aggregators() if not self.map.check_maps(): msg = "No DAS maps found in MappingDB" raise Exception(msg) self.daskeys = ['system', 'date'] # two reserved words for val in self.daskeysmap.values(): for item in val: self.daskeys.append(item)
def __init__(self, config=None): if not config: config = das_readconfig() self.dasmapping = DASMapping(config) if not self.dasmapping.check_maps(): msg = "No DAS maps found in MappingDB" raise Exception(msg) self.dasservices = config['services'] self.daskeysmap = self.dasmapping.daskeys() self.operators = list(das_operators()) self.daskeys = list(das_special_keys()) self.verbose = config['verbose'] self.logger = PrintManager('QLManger', self.verbose) for val in self.daskeysmap.values(): for item in val: self.daskeys.append(item)
def __init__(self, config=None, debug=0, nores=False, logger=None, engine=None, multitask=True): if config: dasconfig = config else: dasconfig = das_readconfig() verbose = dasconfig['verbose'] self.stdout = debug if isinstance(debug, int): self.verbose = debug dasconfig['verbose'] = debug else: self.verbose = verbose das_timer('DASCore::init', self.verbose) self.operators = das_operators() # set noresults option self.noresults = False if nores: dasconfig['write_cache'] = True self.noresults = nores self.multitask = dasconfig['das'].get('multitask', True) if debug or self.verbose: self.multitask = False # in verbose mode do not use multitask dasconfig['das']['multitask'] = False if not multitask: # explicitly call DASCore ctor, e.g. in analytics self.multitask = False dasconfig['das']['multitask'] = False dasconfig['engine'] = engine if self.multitask: nworkers = dasconfig['das'].get('core_workers', 5) if engine: thr_name = 'DASCore:PluginTaskManager' self.taskmgr = PluginTaskManager(\ engine, nworkers=nworkers, name=thr_name) self.taskmgr.subscribe() else: thr_name = 'DASCore:TaskManager' self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name) else: self.taskmgr = None if logger: self.logger = logger else: self.logger = PrintManager('DASCore', self.verbose) # define Mapping/Analytics/Parser in this order since Parser depends # on first two dasmapping = DASMapping(dasconfig) dasconfig['dasmapping'] = dasmapping self.mapping = dasmapping self.analytics = DASAnalytics(dasconfig) dasconfig['dasanalytics'] = self.analytics self.keylearning = DASKeyLearning(dasconfig) dasconfig['keylearning'] = self.keylearning # init DAS cache self.rawcache = DASMongocache(dasconfig) dasconfig['rawcache'] = self.rawcache # plug-in architecture: loop over registered data-services in # dasconfig; load appropriate module/class; register data # service with DASCore. self.systems = dasmapping.list_systems() # pointer to the DAS top level directory dasroot = '/'.join(__file__.split('/')[:-3]) for name in self.systems: try: klass = 'DAS/services/%s/%s_service.py' \ % (name, name) srvfile = os.path.join(dasroot, klass) with file(srvfile) as srvclass: for line in srvclass: if line.find('(DASAbstractService)') != -1: klass = line.split('(DASAbstractService)')[0] klass = klass.split('class ')[-1] break mname = 'DAS.services.%s.%s_service' % (name, name) module = __import__(mname, fromlist=[klass]) obj = getattr(module, klass)(dasconfig) setattr(self, name, obj) SERVICES[name] = obj except IOError as err: if debug > 1: # we have virtual services, so IOError can be correct print_exc(err) try: mname = 'DAS.services.generic_service' module = __import__(mname, fromlist=['GenericService']) obj = module.GenericService(name, dasconfig) setattr(self, name, obj) except Exception as exc: print_exc(exc) msg = "Unable to load %s data-service plugin" % name raise Exception(msg) except Exception as exc: print_exc(exc) msg = "Unable to load %s data-service plugin" % name raise Exception(msg) # loop over systems and get system keys, add mapping keys to final list self.service_keys = {} self.service_parameters = {} for name in self.systems: skeys = getattr(self, name).keys() self.service_keys[getattr(self, name).name] = skeys sparams = getattr(self, name).parameters() self.service_parameters[getattr(self, name).name] = sparams self.service_keys['special'] = das_special_keys() self.dasconfig = dasconfig das_timer('DASCore::init', self.verbose)
def __init__(self, config=None, debug=0, nores=False, logger=None, engine=None, multitask=True): if config: dasconfig = config else: dasconfig = das_readconfig() verbose = dasconfig['verbose'] self.stdout = debug if isinstance(debug, int) and debug: self.verbose = debug dasconfig['verbose'] = debug else: self.verbose = verbose das_timer('DASCore::init', self.verbose) self.operators = das_operators() self.collect_wait_time = dasconfig['das'].get('collect_wait_time', 120) # set noresults option self.noresults = False if nores: dasconfig['write_cache'] = True self.noresults = nores self.init_expire = dasconfig['das'].get('init_expire', 5 * 60) self.multitask = dasconfig['das'].get('multitask', True) if debug or self.verbose: self.multitask = False # in verbose mode do not use multitask dasconfig['das']['multitask'] = False if not multitask: # explicitly call DASCore ctor self.multitask = False dasconfig['das']['multitask'] = False dasconfig['engine'] = engine if self.multitask: nworkers = dasconfig['das'].get('core_workers', 5) # if engine: # thr_name = 'DASCore:PluginTaskManager' # self.taskmgr = PluginTaskManager(\ # engine, nworkers=nworkers, name=thr_name) # self.taskmgr.subscribe() # else: # thr_name = 'DASCore:TaskManager' # self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name) thr_name = 'DASCore:TaskManager' self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name) else: self.taskmgr = None if logger: self.logger = logger else: self.logger = PrintManager('DASCore', self.verbose) # define Mapping/Analytics/Parser in this order since Parser depends # on first two dasmapping = DASMapping(dasconfig) dasconfig['dasmapping'] = dasmapping self.mapping = dasmapping self.keylearning = DASKeyLearning(dasconfig) dasconfig['keylearning'] = self.keylearning # init DAS cache self.rawcache = DASMongocache(dasconfig) dasconfig['rawcache'] = self.rawcache # plug-in architecture: loop over registered data-services in # dasconfig; load appropriate module/class; register data # service with DASCore. self.systems = dasmapping.list_systems() # pointer to the DAS top level directory dasroot = '/'.join(__file__.split('/')[:-3]) for name in self.systems: try: klass = 'DAS/services/%s/%s_service.py' \ % (name, name) srvfile = os.path.join(dasroot, klass) with open(srvfile) as srvclass: for line in srvclass: if line.find('(DASAbstractService)') != -1: klass = line.split('(DASAbstractService)')[0] klass = klass.split('class ')[-1] break mname = 'DAS.services.%s.%s_service' % (name, name) module = __import__(mname, fromlist=[klass]) obj = getattr(module, klass)(dasconfig) setattr(self, name, obj) except IOError as err: if debug > 1: # we have virtual services, so IOError can be correct print_exc(err) try: mname = 'DAS.services.generic_service' module = __import__(mname, fromlist=['GenericService']) obj = module.GenericService(name, dasconfig) setattr(self, name, obj) except Exception as exc: print_exc(exc) msg = "Unable to load %s data-service plugin" % name raise Exception(msg) except Exception as exc: print_exc(exc) msg = "Unable to load %s data-service plugin" % name raise Exception(msg) # loop over systems and get system keys, add mapping keys to final list self.service_keys = {} self.service_parameters = {} for name in self.systems: skeys = list(getattr(self, name).keys()) self.service_keys[getattr(self, name).name] = skeys sparams = getattr(self, name).parameters() self.service_parameters[getattr(self, name).name] = sparams self.service_keys['special'] = das_special_keys() self.dasconfig = dasconfig das_timer('DASCore::init', self.verbose)