def add_query(self, query, mongoquery):
"""
Add DAS-QL/MongoDB-QL queries into analytics.
A unique record is contained for each (qhash, dhash) pair.
For each an array of call-times is contained.
"""
if isinstance(mongoquery, dict):
mongoquery = encode_mongo_query(mongoquery)
msg = 'query=%s, mongoquery=%s' % (query, mongoquery)
self.logger.debug(msg)
dhash = genkey(query)
qhash = genkey(mongoquery)
now = time.time()
existing = self.col.find_one({'qhash': qhash, 'dhash': dhash})
if existing:
# check if times contains very old timestamps
rec = self.col.find({'_id': ObjectId(existing['_id']),
'times':{'$lt' : now - self.history}})
if rec:
self.col.update({'_id': ObjectId(existing['_id'])},
{'$pull': {'times': {'$lt' : now - self.history}}})
# update times array with new timestamp
self.col.update({'_id': ObjectId(existing['_id'])},
{'$push': {'times': now}})
else:
record = dict(query=query, mongoquery=mongoquery,
qhash=qhash, dhash=dhash, times=[now])
self.col.insert(record)
index = [('qhash', DESCENDING),
('dhash', DESCENDING)]
create_indexes(self.col, index)
def create(self, viewname, query,
login='******', fullname='N/A', group='users'):
"""
create new view for given name and a query
"""
if viewname in self.map:
msg = "View '%s' already exists" % viewname
raise Exception(msg)
squery = strip_query(query)
vobj = View(genkey(squery), viewname, squery, datetime.today())
session = self.session()
try: # session transactions
try:
uobj = session.query(User).filter(User.login==login).one()
except:
uobj = User(login, fullname, contact="")
session.add(uobj)
pass
try:
gobj = session.query(Group).filter(Group.name==group).one()
except:
gobj = Group(group)
session.add(gobj)
pass
vobj.user = uobj
uobj.group = gobj
session.add(vobj)
session.commit()
except:
session.rollback()
self.logger.debug(traceback.format_exc())
traceback.print_exc()
msg = "Unable to commit DAS view DB"
raise Exception(msg)
def logdb(self, query):
"""
Make entry in Logging DB
"""
qhash = genkey(query)
args = cherrypy.request.params
doc = dict(qhash=qhash,
date=int(str(date.fromtimestamp(time.time())).replace('-', '')),
headers=cherrypy.request.headers,
method=cherrypy.request.method,
path=cherrypy.request.path_info,
args=args, ahash=genkey(args),
ip=cherrypy.request.remote.ip,
hostname=cherrypy.request.remote.name,
port=cherrypy.request.remote.port)
self.logcol.insert('web', doc)
def datasets_dbs(self):
"""
Retrieve a list of DBS datasets
"""
params = {'dataset_access_type':'VALID'}
encoded_data = urllib.urlencode(params, doseq=True)
url = self.dbs_url + '/datasets?' + encoded_data
req = urllib2.Request(url)
ckey, cert = get_key_cert()
handler = HTTPSClientAuthHandler(ckey, cert)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
stream = urllib2.urlopen(req)
gen = json.load(stream)
for row in gen:
dataset = row['dataset']
rec = {'dataset': dataset}
if self.write_hash:
storage_query = {"fields": ["dataset"],
"spec": [{"key": "dataset.name",
"value": "\"%s\"" % dataset}],
"instance": self.dbcoll}
rec.update({'qhash': genkey(storage_query)})
yield rec
stream.close()
def update_cache(self, query, results, expire):
"""
Insert results into cache.
We use set/get_multi memcache functions to reduce network latency.
- store each row to avoid max cache size limitation, 1MB.
- each row uses unique row_key which are stored into cache
within key namespace.
"""
self.logger.info("DASMemcache::result(%s) store to cache" % query)
if not results:
return
key = genkey(query)
rowdict = {}
rowid = 0
for row in results:
if type(row) is dict:
row['id'] = rowid
rowdict[rowid] = row
rowid += 1
if len(rowdict.keys()) == self.chunk_size:
self.memcache.set_multi(rowdict, time=expire, key_prefix=key)
rowdict = {}
yield row
self.memcache.set_multi(rowdict, time=expire, key_prefix=key)
self.memcache.set(key, rowid, expire)
def lookup_query(self, rawtext):
"""
Check the parser cache for a given rawtext query.
Search is done with the qhash of this string.
Returns a tuple (status, value) for the cases
(PARSERCACHE_VALID, mongo_query) - valid query found
(PARSERCACHE_INVALID, error) - error message for invalid query
(PARSERCACHE_NOTFOUND, None) - not in the cache
"""
result = find_one(self.col, {'qhash':genkey(rawtext)}, \
fields=['query', 'error'])
if result and result['query']:
if self.verbose:
self.logger.debug("DASParserCache: found valid %s->%s" %\
(rawtext, result['query']))
query = decode_mongo_query(result['query'])
return (PARSERCACHE_VALID, query)
elif result and result['error']:
if self.verbose:
self.logger.debug("DASParserCache: found invalid %s->%s" %\
(rawtext, result['error']))
return (PARSERCACHE_INVALID, result['error'])
else:
if self.verbose:
self.logger.debug("DASParserCache: not found %s" %\
(rawtext))
return (PARSERCACHE_NOTFOUND, None)
def add_api(self, system, query, api, args):
"""
Add API info to analytics DB.
Here args is a dict of API parameters.
"""
orig_query = query
if isinstance(query, dict):
query = encode_mongo_query(query)
msg = '(%s, %s, %s, %s)' % (system, query, api, args)
self.logger.debug(msg)
# find query record
qhash = genkey(query)
record = self.col.find_one({'qhash':qhash}, fields=['dasquery'])
if not record:
self.add_query("", orig_query)
# find api record
record = self.col.find_one({'qhash':qhash, 'system':system,
'api.name':api, 'api.params':args})
apidict = dict(name=api, params=args)
if record:
self.col.update({'_id':record['_id']}, {'$inc':{'counter':1}})
else:
record = dict(system=system, api=apidict, qhash=qhash, counter=1)
self.col.insert(record)
index = [('system', DESCENDING), ('dasquery', DESCENDING),
('api.name', DESCENDING), ('qhash', DESCENDING) ]
create_indexes(self.col, index)
def datasets_dbs3(self):
"""
Retrieve a list of DBS datasets (DBS3)
"""
params = {"dataset_access_type": "VALID"}
encoded_data = urllib.urlencode(params, doseq=True)
url = self.dbs_url + "/datasets?" + encoded_data
req = urllib2.Request(url)
ckey, cert = get_key_cert()
handler = HTTPSClientAuthHandler(ckey, cert)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
stream = urllib2.urlopen(req)
gen = json.load(stream)
for row in gen:
dataset = row["dataset"]
rec = {"dataset": dataset}
if self.write_hash:
storage_query = {
"fields": ["dataset"],
"spec": [{"key": "dataset.name", "value": '"%s"' % dataset}],
"instance": self.dbcoll,
}
rec.update({"qhash": genkey(storage_query)})
yield rec
stream.close()
def datasets_dbs(self):
"""
Retrieve a list of DBS datasets (DBS2)
"""
query = "find dataset,dataset.status"
params = {"api": "executeQuery", "apiversion": "DBS_2_0_9", "query": query}
encoded_data = urllib.urlencode(params, doseq=True)
url = self.dbs_url + "?" + encoded_data
req = urllib2.Request(url)
try:
stream = urllib2.urlopen(req)
except urllib2.HTTPError:
msg = "Fail to contact %s" % url
print dastimestamp("DAS ERROR"), msg
raise Exception(msg)
except Exception as exc:
print_exc(exc)
msg = "Fail to contact %s" % url
print dastimestamp("DAS ERROR"), msg
raise Exception(msg)
gen = qlxml_parser(stream, "dataset")
for row in gen:
dataset = row["dataset"]["dataset"]
rec = {"dataset": dataset}
if self.write_hash:
storage_query = {
"fields": ["dataset"],
"spec": [{"key": "dataset.name", "value": '"%s"' % dataset}],
"instance": self.dbcoll,
}
rec.update({"qhash": genkey(storage_query)})
if row["dataset"]["dataset.status"] == "VALID":
yield rec
stream.close()
def lookup_query(self, rawtext):
"""
Check the parser cache for a given rawtext query.
Search is done with the hash of this string.
Returns a tuple (status, value) for the cases
(PARSERCACHE_VALID, mongo_query) - valid query found
(PARSERCACHE_INVALID, error) - error message for invalid query
(PARSERCACHE_NOTFOUND, None) - not in the cache
"""
result = self.col.find_one({'hash':genkey(rawtext)},
fields=['query', 'error'])
if result and result['query']:
if self.verbose:
self.logger.debug("DASParserCache: found valid %s->%s" %\
(rawtext, result['query']))
query = decode_mongo_query(result['query'])
return (PARSERCACHE_VALID, query)
elif result and result['error']:
if self.verbose:
self.logger.debug("DASParserCache: found invalid %s->%s" %\
(rawtext, result['error']))
return (PARSERCACHE_INVALID, result['error'])
else:
if self.verbose:
self.logger.debug("DASParserCache: not found %s" %\
(rawtext))
return (PARSERCACHE_NOTFOUND, None)
def get_from_cache(self, query, idx=0, limit=0, skey=None, order='asc'):
"""
Retreieve results from cache, otherwise return null.
"""
id = 0
idx = int(idx)
limit = long(limit)
stop = idx + limit # get upper bound for range
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
key = genkey(query)
skey = ["%s" % key, timestamp()]
ekey = ["%s" % key, self.future]
options = {'startkey': skey, 'endkey': ekey}
results = cdb.loadView('dasviews', 'query', options)
try:
res = [row['value'] for row in results['rows']]
for row in results['rows']:
row['id'] = id
if limit:
if id >= idx and id <= stop:
yield row
else:
yield row
id += 1
except:
traceback.print_exc()
return
if res:
self.logger.info("DASCouchcache::get_from_cache for %s" % query)
def __init__(self, name, classname, interval,
kwargs, max_runs=None, only_before=None,
parent=None):
#name of the python class that will be run
self.classname = classname
#kwargs for the class instance
self.kwargs = kwargs
#desired frequency
self.interval = interval
#name by which this task shall be known
self.name = name
#tuple of master_ids, if any, that created this task
self.parent = parent if parent else tuple()
#master id is a static key for a given task
#whereas task_id is a UUID for each time the task runs
self.master_id = genkey({'hash':(name, classname, interval, kwargs,
max_runs, only_before, parent)})
self.run_count = 0
self.max_runs = max_runs
self.only_before = only_before
self.last_run_at = None
self.retries = 0
def update_cache(self, query, results, expire):
"""
Insert results into cache.
We use set/get_multi memcache functions to reduce network latency.
- store each row to avoid max cache size limitation, 1MB.
- each row uses unique row_key which are stored into cache
within key namespace.
"""
self.logger.info("DASMemcache::result(%s) store to cache" % query)
if not results:
return
key = genkey(query)
rowdict = {}
rowid = 0
for row in results:
if type(row) is types.DictType:
row["id"] = rowid
rowdict[rowid] = row
rowid += 1
if len(rowdict.keys()) == self.chunk_size:
self.memcache.set_multi(rowdict, time=expire, key_prefix=key)
rowdict = {}
yield row
self.memcache.set_multi(rowdict, time=expire, key_prefix=key)
self.memcache.set(key, rowid, expire)
def get_from_cache(self, query, idx=0, limit=0, skey=None, order='asc'):
"""
Retreieve results from cache, otherwise return null.
"""
id = 0
idx = int(idx)
limit = long(limit)
stop = idx + limit # get upper bound for range
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
key = genkey(query)
skey = ["%s" % key, timestamp()]
ekey = ["%s" % key, self.future]
options = {'startkey': skey, 'endkey': ekey}
results = cdb.loadView('dasviews', 'query', options)
try:
res = [row['value'] for row in results['rows']]
for row in results['rows']:
row['id'] = id
if limit:
if id >= idx and id <= stop:
yield row
else:
yield row
id += 1
except:
traceback.print_exc()
return
if res:
self.logger.info("DASCouchcache::get_from_cache for %s" % query)
def test_qhash(self):
"Test qhash property"
sq1 = {
'fields':
None,
'spec': [{
'key': 'a.b.c',
'value': '"ghi"'
}, {
'key': 'd.e.f',
'value': '"jkl"'
}]
}
sq2 = {
'fields':
None,
'spec': [{
'key': 'a.b.c',
'value': '"ghi"'
}, {
'key': 'd.e.f',
'value': '"jkl"'
}]
}
qh1 = DASQuery(sq1).qhash
qh2 = genkey(sq2)
self.assertEqual(qh1, qh2)
def test_genkey(self):
dataset = "/a/b/c"
d1 = {
'fields': None,
'spec': [{
'key': 'dataset.name',
'value': dataset
}]
}
d2 = {
'fields': None,
'spec': [{
'value': dataset,
'key': 'dataset.name'
}]
}
self.assertEqual(genkey(d1), genkey(d2))
def incache(self, query):
"""
Check if query exists in cache.
"""
key = genkey(query)
res = self.memcache.get(key)
if res and type(res) is types.IntType:
return True
return False
def ql_manager(config=None):
"Return QLManager instance"
key = genkey(str(config))
if key in QL_MANAGER_OBJECTS:
obj = QL_MANAGER_OBJECTS[key]
else:
obj = QLManager(config)
QL_MANAGER_OBJECTS[key] = obj
return obj
def create(self, **kwargs):
"Create DASCore object"
dashash = genkey(str(kwargs))
if dashash in self.params:
return self.params[dashash]
else:
das = DASCore(**kwargs)
self.params[dashash] = das
return das
def genkey(self, uri):
"Generate unique key"
if isinstance(uri, basestring):
key = uri
elif isinstance(uri, list) and len(uri) == 1:
key = uri[0]
else:
key = genkey(str(uri))
return key
def genkey(self, uri):
"Generate unique key"
if isinstance(uri, basestring):
key = uri
elif isinstance(uri, list) and len(uri) == 1:
key = uri[0]
else:
key = genkey(str(uri))
return key
def create(self, **kwargs):
"Create DASCore object"
dashash = genkey(str(kwargs))
if self.params.has_key(dashash):
return self.params[dashash]
else:
das = DASCore(**kwargs)
self.params[dashash] = das
return das
def incache(self, query):
"""
Check if query exists in cache.
"""
key = genkey(query)
res = self.memcache.get(key)
if res and type(res) is int:
return True
return False
def ql_manager(config=None):
"Return QLManager instance"
key = genkey(str(config))
if key in QL_MANAGER_OBJECTS:
obj = QL_MANAGER_OBJECTS[key]
else:
obj = QLManager(config)
QL_MANAGER_OBJECTS[key] = obj
return obj
def record(self, tag):
"""Record time for given tag"""
thash = genkey(tag)
if thash in self.timer:
time0 = self.timer[thash]['time']
self.timer[thash]['time'] = time.time() - time0
else:
self.timer[thash] = \
{'tag': tag, 'time': time.time(), 'counter': self.counter}
self.counter += 1
def record(self, tag):
"""Record time for given tag"""
thash = genkey(tag)
if thash in self.timer:
time0 = self.timer[thash]['time']
self.timer[thash]['time'] = time.time() - time0
else:
self.timer[thash] = \
{'tag': tag, 'time': time.time(), 'counter': self.counter}
self.counter += 1
def spawn(self, func, *args, **kwargs):
"""Spawn new process for given function"""
pid = kwargs.get('pid', genkey(str(args) + str(kwargs)))
if not pid in self._pids:
self._pids.add(pid)
ready = Event()
self._tasks.put((ready, pid, func, args, kwargs))
return ready, pid
else:
return 'in set', pid
def request(self, **kwargs):
"""
Request data from DAS cache.
"""
# remove expires records from merge collection
self.dasmgr.rawcache.remove_expired('merge')
# do not allow caching
cherrypy.response.headers['Cache-Control'] = 'no-cache'
cherrypy.response.headers['Pragma'] = 'no-cache'
uinput = kwargs.get('input', '').strip()
if not uinput:
kwargs['reason'] = 'No input found'
return self.redirect(**kwargs)
time0 = time.time()
self.adjust_input(kwargs)
view = kwargs.get('view', 'list')
inst = kwargs.get('instance', self.dbs_global)
uinput = kwargs.get('input', '')
if self.busy():
return self.busy_page(uinput)
ahash = genkey(cherrypy.request.params)
self.logdb(uinput)
form = self.form(uinput=uinput, instance=inst, view=view)
check, content = self.generate_dasquery(uinput, inst)
if check:
if view == 'list' or view == 'table':
return self.page(form + content, ctime=time.time()-time0)
else:
return content
dasquery = content # returned content is valid DAS query
status, qhash = self.dasmgr.get_status(dasquery)
if status == 'ok':
page = self.get_page_content(kwargs, complete_msg=False)
ctime = (time.time()-time0)
if view == 'list' or view == 'table':
return self.page(form + page, ctime=ctime)
return page
else:
kwargs['dasquery'] = dasquery.storage_query
addr = cherrypy.request.headers.get('Remote-Addr')
_evt, pid = self.taskmgr.spawn(self.dasmgr.call, dasquery, addr,
pid=dasquery.qhash)
self.reqmgr.add(pid, kwargs)
if self.taskmgr.is_alive(pid):
page = self.templatepage('das_check_pid', method='check_pid',
uinput=uinput, view=view, ahash=ahash,
base=self.base, pid=pid, interval=self.interval)
else:
page = self.get_page_content(kwargs)
self.reqmgr.remove(pid)
ctime = (time.time()-time0)
return self.page(form + page, ctime=ctime)
def qhash(self):
"""
Read only qhash, generated on demand.
"""
if not self._qhash:
sdict = deepcopy(self.storage_query)
for key in ['filters', 'aggregators', 'mapreduce']:
if key in sdict:
del sdict[key]
self._qhash = genkey(sdict)
return self._qhash
def qhash(self):
"""
Read only qhash, generated on demand.
"""
if not self._qhash:
sdict = deepcopy(self.storage_query)
for key in ['filters', 'aggregators', 'mapreduce']:
if key in sdict:
del sdict[key]
self._qhash = genkey(sdict)
return self._qhash
def update_cache(self, query, results, expire):
"""
Insert results into cache. Query provides a hash key which
becomes a file name.
"""
if not expire:
raise Exception('Expire parameter is null')
self.logger.info("DASFilecache::update_cache(%s) store to cache" \
% query)
if not results:
return
system = self.get_system(query)
key = genkey(query)
idir = create_dir(self.dir, system, self.base_dir, self.files_dir)
filename = os.path.join(idir, key)
fdr = open(filename, 'wb')
if type(results) is list or \
type(results) is types.GeneratorType:
for item in results:
marshal.dump(item, fdr)
yield item
else:
marshal.dump(results, fdr)
yield results
fdr.close()
create = time.time()
expire = create + expire
qobj = Query(key, query, str(create), str(expire))
session = self.session()
try: # session transactions
try:
sobj = session.query(System).filter(
System.name == system).one()
except:
sobj = System(system)
session.add(sobj)
pass
try:
dobj = session.query(Location).filter(
Location.dir == idir).one()
except:
dobj = Location(idir)
session.add(dobj)
pass
qobj.system = sobj
qobj.location = dobj
session.add(qobj)
session.commit()
except:
session.rollback()
self.logger.debug(traceback.format_exc())
msg = "Unable to commit DAS filecache DB"
raise Exception(msg)
def test_qhash(self):
"Test qhash property"
sq1 = {'fields':None,
'spec':[{'key':'a.b.c', 'value': '"ghi"'},
{'key':'d.e.f', 'value':'"jkl"'}]}
sq2 = {'fields':None,
'spec':[{'key':'a.b.c', 'value': '"ghi"'},
{'key':'d.e.f', 'value':'"jkl"'}]}
qh1 = DASQuery(sq1).qhash
qh2 = genkey(sq2)
self.assertEqual(qh1, qh2)
def test_key(self):
"""test DAS cache key generator"""
query = "find site where site=T2_UK"
result = genkey(query)
try:
hash = hashlib.md5()
except:
# prior python 2.5
hash = md5.new()
hash.update(query)
expect = hash.hexdigest()
self.assertEqual(expect, result)
def test_key(self):
"""test DAS cache key generator"""
query = "find site where site=T2_UK"
result = genkey(query)
try:
hash = hashlib.md5()
except:
# prior python 2.5
hash = md5.new()
hash.update(query)
expect = hash.hexdigest()
self.assertEqual(expect, result)
def update_cache(self, query, results, expire):
"""
Insert results into cache. Query provides a hash key which
becomes a file name.
"""
if not expire:
raise Exception('Expire parameter is null')
self.logger.info("DASFilecache::update_cache(%s) store to cache" \
% query)
if not results:
return
system = self.get_system(query)
key = genkey(query)
idir = create_dir(self.dir, system, self.base_dir, self.files_dir)
filename = os.path.join(idir, key)
fdr = open(filename, 'wb')
if type(results) is list or \
type(results) is types.GeneratorType:
for item in results:
marshal.dump(item, fdr)
yield item
else:
marshal.dump(results, fdr)
yield results
fdr.close()
create = time.time()
expire = create+expire
qobj = Query(key, query, str(create), str(expire))
session = self.session()
try: # session transactions
try:
sobj = session.query(System).filter(System.name==system).one()
except:
sobj = System(system)
session.add(sobj)
pass
try:
dobj = session.query(Location).filter(Location.dir==idir).one()
except:
dobj = Location(idir)
session.add(dobj)
pass
qobj.system = sobj
qobj.location = dobj
session.add(qobj)
session.commit()
except:
session.rollback()
self.logger.debug(traceback.format_exc())
msg = "Unable to commit DAS filecache DB"
raise Exception(msg)
def _insert_query(self, rawtext, query, error):
"""Internal method to insert a query"""
if self.verbose:
self.logger.debug("DASParserCache: insert %s->%s/%s" %\
(rawtext, query, error))
# since MongoDB does not support insertion of $ sign in queries
# we need to encode inserted query
if query:
encquery = encode_mongo_query(query)
else:
encquery = ""
self.col.insert({'raw':rawtext, 'hash':genkey(rawtext),
'query':encquery, 'error':str(error)})
def connection(self, uri):
"""Return MongoDB connection"""
key = genkey(str(uri))
if not self.conndict.has_key(key):
try:
dbinst = Connection(host=uri)
gfs = dbinst.gridfs
fsinst = gridfs.GridFS(gfs)
self.conndict[key] = (dbinst, fsinst)
except Exception as exc:
print_exc(exc)
return None
return self.conndict[key]
def update_apicall(self, query, das_dict):
"""
Update apicall record with provided DAS dict.
Moved from AbstractService
"""
msg = 'DBSAnalytics::update_apicall, query=%s, das_dict=%s'\
% (query, das_dict)
self.logger.debug(msg)
spec = {'apicall.qhash':genkey(encode_mongo_query(query))}
record = self.col.find_one(spec)
self.col.update({'_id':ObjectId(record['_id'])},
{'$set':{'dasapi':das_dict,
'apicall.expire':das_dict['response_expires']}})
def is_alive(self, uri):
"Check if DB connection is alive"
key = genkey(str(uri))
try:
conn, _ = self.connection(uri)
if conn:
_dbnames = conn.database_names()
else:
return False
except:
if self.conndict.has_key(key):
del self.conndict[key]
return False
return True
def update(self, system, query):
"""
Update records for given system/query.
"""
if isinstance(query, dict):
query = encode_mongo_query(query)
msg = 'system=%s, query=%s' % (system, query)
self.logger.debug(msg)
qhash = genkey(query)
if system:
cond = {'qhash':qhash, 'system':system}
else:
cond = {'qhash':qhash}
self.col.update(cond, {'$inc' : {'counter':1}}, multi=True)
def remove_from_cache(self, query):
"""
Remove query from cache
"""
key = genkey(query)
session = self.session()
try: # transactions
res = session.query(Query).filter(Query.hash == key).one()
session.delete(res)
session.commit()
except:
session.rollback()
self.logger.debug(traceback.format_exc())
return False
return True
def logdb(self, query):
"""
Make entry in Logging DB
"""
qhash = genkey(query)
headers = cherrypy.request.headers
doc = dict(qhash=qhash, timestamp=time.time(),
headers=cherrypy.request.headers,
method=cherrypy.request.method,
path=cherrypy.request.path_info,
args=cherrypy.request.params,
ip=cherrypy.request.remote.ip,
hostname=cherrypy.request.remote.name,
port=cherrypy.request.remote.port)
self.col.insert(doc)
def remove_from_cache(self, query):
"""
Remove query from cache
"""
key = genkey(query)
session = self.session()
try: # transactions
res = session.query(Query).filter(Query.hash == key).one()
session.delete(res)
session.commit()
except:
session.rollback()
self.logger.debug(traceback.format_exc())
return False
return True
def __init__(self, config=None, query=None, sleep=600):
self.dascore = DASCore(config, nores=True)
logdir = getarg(config, 'logdir', '/tmp')
self.pidfile = os.path.join(logdir, 'robot-%s.pid' % genkey(query))
if (hasattr(os, "devnull")):
devnull = os.devnull
else:
devnull = "/dev/null"
self.stdin = devnull # we do not read from stdinput
self.stdout = getarg(config, 'stdout', devnull)
self.stderr = getarg(config, 'stderr', devnull)
self.query = query
self.sleep = sleep
def is_expired(self, query):
"""
Check if we have query result is expired in cache.
"""
key = genkey(query)
curtime = time.time()
session = self.session()
sql_stm = session.query(Query).filter(and_(Query.hash == key, Query.expire < "%s" % curtime))
try:
res = sql_stm.one()
except Exception, exp:
msg = "query=%s\n%s %s %s\n%s" % (query, sql_stm, key, curtime, exp)
self.logger.debug(msg)
# print "\n### das_filecache:is_expired msg=", msg
return False
def __init__(self, config=None, query=None, sleep=600):
self.dascore = DASCore(config, nores=True)
logdir = getarg(config, 'logdir', '/tmp')
self.pidfile = os.path.join(logdir, 'robot-%s.pid' % genkey(query))
if (hasattr(os, "devnull")):
devnull = os.devnull
else:
devnull = "/dev/null"
self.stdin = devnull # we do not read from stdinput
self.stdout = getarg(config, 'stdout', devnull)
self.stderr = getarg(config, 'stderr', devnull)
self.query = query
self.sleep = sleep
def logdb(self, query):
"""
Make entry in Logging DB
"""
qhash = genkey(query)
headers = cherrypy.request.headers
doc = dict(qhash=qhash,
timestamp=time.time(),
headers=cherrypy.request.headers,
method=cherrypy.request.method,
path=cherrypy.request.path_info,
args=cherrypy.request.params,
ip=cherrypy.request.remote.ip,
hostname=cherrypy.request.remote.name,
port=cherrypy.request.remote.port)
self.col.insert(doc)
def _insert_query(self, rawtext, query, error):
"""Internal method to insert a query"""
if self.verbose:
self.logger.debug("DASParserCache: insert %s->%s/%s" %\
(rawtext, query, error))
# since MongoDB does not support insertion of $ sign in queries
# we need to encode inserted query
if query:
encquery = encode_mongo_query(query)
else:
encquery = ""
self.col.insert({
'raw': rawtext,
'qhash': genkey(rawtext),
'query': encquery,
'error': str(error)
})
def is_expired(self, query):
"""
Check if we have query result is expired in cache.
"""
key = genkey(query)
curtime = time.time()
session = self.session()
sql_stm = session.query(Query).filter(and_(Query.hash==key, \
Query.expire < '%s' % curtime ))
try:
res = sql_stm.one()
except Exception as exp:
msg = 'query=%s\n%s %s %s\n%s' % (query, sql_stm, key, curtime,
exp)
self.logger.debug(msg)
# print "\n### das_filecache:is_expired msg=", msg
return False
return True
def get_from_cache(self, query, idx=0, limit=0, skey=None, order='asc'):
"""
Retreieve results from cache, otherwise return null.
"""
idx = int(idx)
limit = long(limit)
stop = idx + limit
key = genkey(query)
res = self.memcache.get(key)
id = idx
if res and type(res) is int:
self.logger.info("DASMemcache::result(%s) using cache" % query)
if skey:
rowlist = [i for i in range(0, res)]
rowdict = self.memcache.get_multi(rowlist, key_prefix=key)
data = rowdict.values()
gendata = (i for i in sort_data(data, skey, order))
def subgroup(gen, idx, stop):
"""Extract sub-group of results from generator"""
id = 0
for item in gen:
if stop:
if id >= idx and id < stop:
yield item
else:
if id >= idx:
yield item
id += 1
items = subgroup(gendata, idx, stop)
else:
if limit:
if limit > res:
stop = res
rowlist = [i for i in range(idx, stop)]
else:
rowlist = [i for i in range(0, res)]
rowdict = self.memcache.get_multi(rowlist, key_prefix=key)
items = rowdict.values()
for item in items:
# item['id'] = id
yield item
id += 1
def spawn(self, func, *args, **kwargs):
"""Spawn new process for given function"""
pid = kwargs.get('pid', genkey(str(args) + str(kwargs)))
evt = Event()
if not pid in self._pids:
self._pids.add(pid)
task = (evt, pid, func, args, kwargs)
if isinstance(self._tasks, PriorityQueue):
uid = kwargs.get('uid', None)
self._uids.add(uid)
priority = self.assign_priority(uid)
self._tasks.put((priority, uid, task))
else:
self._tasks.put(task)
else:
# the event was not added to task list, invoke set()
# to pass it in wait() call, see joinall
evt.set()
return evt, pid
def incache(self, query):
"""
Check if we have query results in cache, otherwise return null.
"""
if self.is_expired(query):
self.remove_from_cache(query)
return False
key = genkey(query)
curtime = time.time()
session = self.session()
sql_stm = session.query(Query).filter(and_(Query.hash==key, \
Query.expire > '%s' % time.time() ))
try:
res = sql_stm.one()
except Exception as exp:
msg = 'query=%s\n%s %s %s\n%s' % (query, sql_stm, key, curtime,
exp)
self.logger.debug(msg)
# print "\n### das_filecache:incache msg=", msg
return False
return True
def create(self,
viewname,
query,
login='******',
fullname='N/A',
group='users'):
"""
create new view for given name and a query
"""
if viewname in self.map:
msg = "View '%s' already exists" % viewname
raise Exception(msg)
squery = strip_query(query)
vobj = View(genkey(squery), viewname, squery, datetime.today())
session = self.session()
try: # session transactions
try:
uobj = session.query(User).filter(User.login == login).one()
except:
uobj = User(login, fullname, contact="")
session.add(uobj)
pass
try:
gobj = session.query(Group).filter(Group.name == group).one()
except:
gobj = Group(group)
session.add(gobj)
pass
vobj.user = uobj
uobj.group = gobj
session.add(vobj)
session.commit()
except:
session.rollback()
self.logger.debug(traceback.format_exc())
traceback.print_exc()
msg = "Unable to commit DAS view DB"
raise Exception(msg)
def incache(self, query):
"""
Check if query exists in cache
"""
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
key = genkey(query)
#TODO:check how to query 1 result, I copied the way from get_from_cache
skey = ["%s" % key, timestamp()]
ekey = ["%s" % key, self.future]
options = {'startkey': skey, 'endkey': ekey}
# results = cdb.loadView('dasviews', 'incache', options)
results = cdb.loadView('dasviews', 'query', options)
try:
res = len(results['rows'])
except:
traceback.print_exc()
return
if res:
return True
return False
DAS.delete(opts.delete)
print(msg, end=' ')
sys.exit(0)
t1 = 0
t2 = timestamp()
if opts.fromtime:
t1 = opts.fromtime
if opts.totime:
t2 = opts.totime
options = {'group': 'true'}
view = 'timer'
design = 'dasviews'
if opts.query:
key = genkey(opts.query)
# skey = '["%s", %s ]' % (key, t1)
# ekey = '["%s", %s ]' % (key, t2)
skey = '["%s", %s ]' % (key, timestamp())
ekey = '["%s", %s ]' % (key, 9999999999)
options = {'startkey': skey, 'endkey': ekey}
view = 'query'
design = 'dasviews'
elif opts.system:
key = '"%s"' % opts.system
options = {'key': key}
view = 'system'
design = 'dasadmin'
elif opts.queries:
view = 'queries'
design = 'dasadmin'
def get_from_cache(self, query, idx=0, limit=0, skey=None, order='asc'):
"""
Retreieve results from cache, otherwise return null.
"""
# id = int(idx)
idx = int(idx)
stop = idx + long(limit)
key = genkey(query)
sysdir = os.path.join(self.dir, self.get_system(query))
session = self.session()
try: # transactions
res = session.query(Query, Location).\
filter(Query.dir_id==Location.id).\
filter(Query.hash==key)
session.commit()
except:
session.rollback()
self.logger.debug(traceback.format_exc())
pass
for qobj, dobj in res:
valid = eval(qobj.expire) - time.time()
timestring = eval(qobj.create)
idir = dobj.dir
filename = os.path.join(idir, key)
self.logger.info("DASFilecache::get_from_cache %s" % filename)
if valid > 0:
msg = "found valid query in cache, key=%s" % key
self.logger.debug("DASFilecache::get_from_cache %s" % msg)
if os.path.isfile(filename):
fdr = open(filename, 'rb')
if skey:
# first retrieve full list of results and sort it
data = []
id = 0
while 1:
try:
res = marshal.load(fdr)
if type(res) is dict:
res['id'] = id
data.append(res)
id += 1
except EOFError as err:
break
fdr.close()
sorted_data = sort_data(data, skey, order)
index = 0
for row in sorted_data:
if limit:
if index >= idx and index < stop:
yield row
else:
if index >= idx:
yield row
index += 1
else:
id = 0
while 1:
try:
res = marshal.load(fdr)
if type(res) is dict:
res['id'] = id
if limit:
if id >= idx and id < stop:
yield res
if id == stop:
break
else:
yield res
id += 1
except EOFError as err:
break
fdr.close()
else:
msg = "found expired query in cache, key=%s" % key
self.logger.debug("DASFilecache::get_from_cache %s" % msg)
fdir = os.path.split(filename)[0]
if os.path.isfile(filename):
os.remove(filename)
clean_dirs(fdir)
try: # session transactions
session.delete(qobj)
session.commit()
except:
session.rollback()
self.logger.debug(traceback.format_exc())
msg = "Unable to delete object from DAS filecache DB"
raise Exception(msg)
