def faq(self, *args, **kwargs):
"""
represent DAS FAQ.
"""
section = kwargs.get('section', None)
highlight = kwargs.get('highlight', None)
guide = self.templatepage('dbsql_vs_dasql',
operators=', '.join(das_operators()))
page = self.templatepage('das_faq', guide=guide,
section=section, highlight=highlight,
operators=', '.join(das_operators()),
aggregators=', '.join(das_aggregators()))
return self.page(page, response_div=False)
def faq(self, **kwargs):
"""
represent DAS FAQ.
"""
section = kwargs.get('section', None)
highlight = kwargs.get('highlight', None)
guide = self.templatepage('dbsql_vs_dasql',
operators=', '.join(das_operators()))
daskeys = self.templatepage('das_keys', daskeys=self.daskeyslist)
page = self.templatepage('das_faq', guide=guide, daskeys=daskeys,
section=section, highlight=highlight,
operators=', '.join(das_operators()),
aggregators=', '.join(das_aggregators()))
return self.page(page, response_div=False)
def __init__(self,
daskeys,
dassystems,
operators=None,
specials=None,
filters=None,
aggregators=None,
mapreduces=None,
verbose=0):
super(DASQueryParser, self).__init__()
self.daskeys = daskeys + ['queries', 'records']
self.verbose = verbose
self.dassystems = dassystems
# test if we have been given a list of desired operators/filters
# /aggregators, if not get the lists from das_ql
self.operators = operators if operators else das_operators()
self.filters = filters if filters else das_filters()
self.aggregators = aggregators if aggregators else das_aggregators()
self.mapreduces = mapreduces if mapreduces else das_mapreduces()
self.specials = specials if specials else das_special_keys()
if self.verbose:
print("operators", self.operators)
print('filters', self.filters)
print('mapreduces', self.mapreduces)
print('specials', self.specials)
self.daskeys += self.specials
def error_msg(msg, show_kws=False, tmpl='das_ambiguous', **kwargs):
"""
Helper function which renders an error template, default is
das_ambiguous, but can be overriden via tmpl param.
Template has two versions: html and text for CLI.
The template is passed with msg, base, guide, and **kwargs. """
# TODO: this shall be done by inheriting a parent template
# TODO: no header/footer?
guide = self.templatepage('dbsql_vs_dasql',
operators=', '.join(das_operators()))
# render keyword search loader, if needed
kws = ''
if show_kws:
kws = self.templatepage('kwdsearch_via_ajax',
uinput=uinput,
jsonize=jsonize,
url_extend_params_as_dict=url_extend_params_as_dict,
inst=inst or self.dbs_global,
kws_host=self._get_kws_host())
# render the appropriate template (html vs text mode)
page = self.templatepage(tmpl + ('_txt' if not html_mode else ''),
msg=msg, base=self.base, guide=guide,
kws_enabled=show_kws, kws=kws, **kwargs)
return page
def __init__(self, config=None):
if not config:
config = das_readconfig()
if not config.has_key('dasmapping'):
config['dasmapping'] = DASMapping(config)
if not config.has_key('dasanalytics'):
config['dasanalytics'] = DASAnalytics(config)
if not config['dasmapping'].check_maps():
msg = "No DAS maps found in MappingDB"
raise Exception(msg)
self.map = config['dasmapping']
self.analytics = config['dasanalytics']
self.dasservices = config['services']
self.daskeysmap = self.map.daskeys()
self.operators = list(das_operators())
self.daskeys = list(das_special_keys())
self.verbose = config['verbose']
self.logger = PrintManager('QLManger', self.verbose)
for val in self.daskeysmap.values():
for item in val:
self.daskeys.append(item)
parserdir = config['das']['parserdir']
self.dasply = DASPLY(parserdir, self.daskeys, self.dasservices,
verbose=self.verbose)
self.enabledb = config['parserdb']['enable']
if self.enabledb:
self.parserdb = DASParserDB(config)
def error_msg(msg, show_kws=False, tmpl="das_ambiguous", **kwargs):
"""
Helper function which renders an error template, default is
das_ambiguous, but can be overriden via tmpl param.
Template has two versions: html and text for CLI.
The template is passed with msg, base, guide, and **kwargs. """
guide = self.templatepage("dbsql_vs_dasql", operators=", ".join(das_operators()))
# render keyword search loader, if needed
kws = ""
if show_kws:
kws = self.templatepage(
"kwdsearch_via_ajax", uinput=uinput, inst=inst or self.dbs_global, kws_host=self._get_kws_host()
)
# render the appropriate template (html vs text mode)
page = self.templatepage(
tmpl + ("_txt" if not html_mode else ""),
msg=msg,
base=self.base,
guide=guide,
kws_enabled=show_kws,
kws=kws,
**kwargs
)
return page
def __init__(self, parserdir, daskeys, dassystems,
operators=None, specials=None, filters=None,
aggregators=None, mapreduces=None, verbose=0):
self.daskeys = daskeys
self.verbose = verbose
self.lexer = None # defined at run time using self.build()
self.parser = None # defined at run time using self.build()
self.dassystems = dassystems
self.parsertab_dir = parserdir
if not os.path.isdir(self.parsertab_dir):
msg = 'Directory %s does not exists' % self.parsertab_dir
raise Exception(msg)
# test if we have been given a list of desired operators/filters
# /aggregators, if not get the lists from das_ql
operators = operators if operators else das_operators()
filters = filters if filters else das_filters()
aggregators = aggregators if aggregators else das_aggregators()
mapreduces = mapreduces if mapreduces else das_mapreduces()
specials = specials if specials else das_special_keys()
# build a map of token->token.type which we can use in the
# enlarged VALUE rule
self.tokenmap = {}
for o in operators:
self.tokenmap[o] = 'OPERATOR'
for f in filters:
self.tokenmap[f] = 'FILTER'
for a in aggregators:
self.tokenmap[a] = 'AGGREGATOR'
for m in mapreduces:
self.tokenmap[m] = 'MAPREDUCE'
for s in specials:
self.tokenmap[s] = 'SPECIALKEY'
def faq(self, *args, **kwargs):
"""
represent DAS FAQ.
"""
page = self.templatepage('das_faq',
operators=', '.join(das_operators()),
aggregators=', '.join(das_aggregators()))
return self.page(page, response_div=False)
def faq(self, *args, **kwargs):
"""
represent DAS FAQ.
"""
page = self.templatepage('das_faq',
operators=', '.join(das_operators()),
aggregators=', '.join(das_aggregators()))
return self.page(page, response_div=False)
def helper(msg, html_error=None):
"""Helper function which provide error template"""
if not html_error:
return msg
guide = self.templatepage('dbsql_vs_dasql',
operators=', '.join(das_operators()))
page = self.templatepage('das_ambiguous', msg=msg, base=self.base,
guide=guide)
return page
def faq(self, **kwargs):
"""
represent DAS FAQ.
"""
section = kwargs.get("section", None)
highlight = kwargs.get("highlight", None)
guide = self.templatepage("dbsql_vs_dasql", operators=", ".join(das_operators()))
daskeys = self.templatepage("das_keys", daskeys=self.daskeyslist)
page = self.templatepage(
"das_faq",
guide=guide,
daskeys=daskeys,
section=section,
highlight=highlight,
operators=", ".join(das_operators()),
aggregators=", ".join(das_aggregators()),
)
return self.page(page, response_div=False)
def __init__(self, config):
self.map = config['dasmapping']
self.analytics = config['dasanalytics']
self.daskeysmap = self.map.daskeys()
self.operators = das_operators()
self.filters = ['%s ' % f for f in das_filters()]
self.aggregators = das_aggregators()
if not self.map.check_maps():
msg = "No DAS maps found in MappingDB"
raise Exception(msg)
self.daskeys = ['system', 'date'] # two reserved words
for val in self.daskeysmap.values():
for item in val:
self.daskeys.append(item)
def __init__(self, config=None):
if not config:
config = das_readconfig()
self.dasmapping = DASMapping(config)
if not self.dasmapping.check_maps():
msg = "No DAS maps found in MappingDB"
raise Exception(msg)
self.dasservices = config['services']
self.daskeysmap = self.dasmapping.daskeys()
self.operators = list(das_operators())
self.daskeys = list(das_special_keys())
self.verbose = config['verbose']
self.logger = PrintManager('QLManger', self.verbose)
for val in self.daskeysmap.values():
for item in val:
self.daskeys.append(item)
def __init__(self, config=None):
if not config:
config = das_readconfig()
self.dasmapping = DASMapping(config)
if not self.dasmapping.check_maps():
msg = "No DAS maps found in MappingDB"
raise Exception(msg)
self.dasservices = config['services']
self.daskeysmap = self.dasmapping.daskeys()
self.operators = list(das_operators())
self.daskeys = list(das_special_keys())
self.verbose = config['verbose']
self.logger = PrintManager('QLManger', self.verbose)
for val in self.daskeysmap.values():
for item in val:
self.daskeys.append(item)
def __init__(self, config):
self.map = config['dasmapping']
self.analytics = config['dasanalytics']
self.daskeysmap = self.map.daskeys()
self.operators = das_operators()
self.filters = ['%s ' % f for f in das_filters()]
self.aggregators = das_aggregators()
if not self.map.check_maps():
msg = "No DAS maps found in MappingDB"
raise Exception(msg)
self.daskeys = ['system', 'date'] # two reserved words
for val in self.daskeysmap.values():
for item in val:
self.daskeys.append(item)
def __init__(self, daskeys, dassystems,
operators=None, specials=None, filters=None,
aggregators=None, mapreduces=None, verbose=0):
super(DASQueryParser, self).__init__()
self.daskeys = daskeys + ['queries', 'records']
self.verbose = verbose
self.dassystems = dassystems
# test if we have been given a list of desired operators/filters
# /aggregators, if not get the lists from das_ql
self.operators = operators if operators else das_operators()
self.filters = filters if filters else das_filters()
self.aggregators = aggregators if aggregators else das_aggregators()
self.mapreduces = mapreduces if mapreduces else das_mapreduces()
self.specials = specials if specials else das_special_keys()
if self.verbose:
print("operators", self.operators)
print('filters', self.filters)
print('mapreduces', self.mapreduces)
print('specials', self.specials)
self.daskeys += self.specials
def __init__(self, config=None, debug=0,
nores=False, logger=None, engine=None, multitask=True):
if config:
dasconfig = config
else:
dasconfig = das_readconfig()
verbose = dasconfig['verbose']
self.stdout = debug
if isinstance(debug, int):
self.verbose = debug
dasconfig['verbose'] = debug
else:
self.verbose = verbose
das_timer('DASCore::init', self.verbose)
self.operators = das_operators()
# set noresults option
self.noresults = False
if nores:
dasconfig['write_cache'] = True
self.noresults = nores
self.multitask = dasconfig['das'].get('multitask', True)
if debug or self.verbose:
self.multitask = False # in verbose mode do not use multitask
dasconfig['das']['multitask'] = False
if not multitask: # explicitly call DASCore ctor, e.g. in analytics
self.multitask = False
dasconfig['das']['multitask'] = False
dasconfig['engine'] = engine
if self.multitask:
nworkers = dasconfig['das'].get('core_workers', 5)
if engine:
thr_name = 'DASCore:PluginTaskManager'
self.taskmgr = PluginTaskManager(\
engine, nworkers=nworkers, name=thr_name)
self.taskmgr.subscribe()
else:
thr_name = 'DASCore:TaskManager'
self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name)
else:
self.taskmgr = None
if logger:
self.logger = logger
else:
self.logger = PrintManager('DASCore', self.verbose)
# define Mapping/Analytics/Parser in this order since Parser depends
# on first two
dasmapping = DASMapping(dasconfig)
dasconfig['dasmapping'] = dasmapping
self.mapping = dasmapping
self.analytics = DASAnalytics(dasconfig)
dasconfig['dasanalytics'] = self.analytics
self.keylearning = DASKeyLearning(dasconfig)
dasconfig['keylearning'] = self.keylearning
# init DAS cache
self.rawcache = DASMongocache(dasconfig)
dasconfig['rawcache'] = self.rawcache
# plug-in architecture: loop over registered data-services in
# dasconfig; load appropriate module/class; register data
# service with DASCore.
self.systems = dasmapping.list_systems()
# pointer to the DAS top level directory
dasroot = '/'.join(__file__.split('/')[:-3])
for name in self.systems:
try:
klass = 'DAS/services/%s/%s_service.py' \
% (name, name)
srvfile = os.path.join(dasroot, klass)
with file(srvfile) as srvclass:
for line in srvclass:
if line.find('(DASAbstractService)') != -1:
klass = line.split('(DASAbstractService)')[0]
klass = klass.split('class ')[-1]
break
mname = 'DAS.services.%s.%s_service' % (name, name)
module = __import__(mname, fromlist=[klass])
obj = getattr(module, klass)(dasconfig)
setattr(self, name, obj)
SERVICES[name] = obj
except IOError as err:
if debug > 1:
# we have virtual services, so IOError can be correct
print_exc(err)
try:
mname = 'DAS.services.generic_service'
module = __import__(mname, fromlist=['GenericService'])
obj = module.GenericService(name, dasconfig)
setattr(self, name, obj)
except Exception as exc:
print_exc(exc)
msg = "Unable to load %s data-service plugin" % name
raise Exception(msg)
except Exception as exc:
print_exc(exc)
msg = "Unable to load %s data-service plugin" % name
raise Exception(msg)
# loop over systems and get system keys, add mapping keys to final list
self.service_keys = {}
self.service_parameters = {}
for name in self.systems:
skeys = getattr(self, name).keys()
self.service_keys[getattr(self, name).name] = skeys
sparams = getattr(self, name).parameters()
self.service_parameters[getattr(self, name).name] = sparams
self.service_keys['special'] = das_special_keys()
self.dasconfig = dasconfig
das_timer('DASCore::init', self.verbose)
def __init__(self,
config=None,
debug=0,
nores=False,
logger=None,
engine=None,
multitask=True):
if config:
dasconfig = config
else:
dasconfig = das_readconfig()
verbose = dasconfig['verbose']
self.stdout = debug
if isinstance(debug, int) and debug:
self.verbose = debug
dasconfig['verbose'] = debug
else:
self.verbose = verbose
das_timer('DASCore::init', self.verbose)
self.operators = das_operators()
self.collect_wait_time = dasconfig['das'].get('collect_wait_time', 120)
# set noresults option
self.noresults = False
if nores:
dasconfig['write_cache'] = True
self.noresults = nores
self.init_expire = dasconfig['das'].get('init_expire', 5 * 60)
self.multitask = dasconfig['das'].get('multitask', True)
if debug or self.verbose:
self.multitask = False # in verbose mode do not use multitask
dasconfig['das']['multitask'] = False
if not multitask: # explicitly call DASCore ctor
self.multitask = False
dasconfig['das']['multitask'] = False
dasconfig['engine'] = engine
if self.multitask:
nworkers = dasconfig['das'].get('core_workers', 5)
# if engine:
# thr_name = 'DASCore:PluginTaskManager'
# self.taskmgr = PluginTaskManager(\
# engine, nworkers=nworkers, name=thr_name)
# self.taskmgr.subscribe()
# else:
# thr_name = 'DASCore:TaskManager'
# self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name)
thr_name = 'DASCore:TaskManager'
self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name)
else:
self.taskmgr = None
if logger:
self.logger = logger
else:
self.logger = PrintManager('DASCore', self.verbose)
# define Mapping/Analytics/Parser in this order since Parser depends
# on first two
dasmapping = DASMapping(dasconfig)
dasconfig['dasmapping'] = dasmapping
self.mapping = dasmapping
self.keylearning = DASKeyLearning(dasconfig)
dasconfig['keylearning'] = self.keylearning
# init DAS cache
self.rawcache = DASMongocache(dasconfig)
dasconfig['rawcache'] = self.rawcache
# plug-in architecture: loop over registered data-services in
# dasconfig; load appropriate module/class; register data
# service with DASCore.
self.systems = dasmapping.list_systems()
# pointer to the DAS top level directory
dasroot = '/'.join(__file__.split('/')[:-3])
for name in self.systems:
try:
klass = 'DAS/services/%s/%s_service.py' \
% (name, name)
srvfile = os.path.join(dasroot, klass)
with open(srvfile) as srvclass:
for line in srvclass:
if line.find('(DASAbstractService)') != -1:
klass = line.split('(DASAbstractService)')[0]
klass = klass.split('class ')[-1]
break
mname = 'DAS.services.%s.%s_service' % (name, name)
module = __import__(mname, fromlist=[klass])
obj = getattr(module, klass)(dasconfig)
setattr(self, name, obj)
except IOError as err:
if debug > 1:
# we have virtual services, so IOError can be correct
print_exc(err)
try:
mname = 'DAS.services.generic_service'
module = __import__(mname, fromlist=['GenericService'])
obj = module.GenericService(name, dasconfig)
setattr(self, name, obj)
except Exception as exc:
print_exc(exc)
msg = "Unable to load %s data-service plugin" % name
raise Exception(msg)
except Exception as exc:
print_exc(exc)
msg = "Unable to load %s data-service plugin" % name
raise Exception(msg)
# loop over systems and get system keys, add mapping keys to final list
self.service_keys = {}
self.service_parameters = {}
for name in self.systems:
skeys = list(getattr(self, name).keys())
self.service_keys[getattr(self, name).name] = skeys
sparams = getattr(self, name).parameters()
self.service_parameters[getattr(self, name).name] = sparams
self.service_keys['special'] = das_special_keys()
self.dasconfig = dasconfig
das_timer('DASCore::init', self.verbose)
