def test_sorting(self):
"""Test sorting routines"""
data = [
{'id':6, 'dataset': 'bla6', 'run':200},
{'id':1, 'dataset': 'bla1', 'run':100},
{'id':2, 'dataset': 'bla2', 'run':700},
{'id':3, 'dataset': 'bla3', 'run':400},
{'id':4, 'dataset': 'bla4', 'run':300},
{'id':5, 'dataset': 'bla5', 'run':800},
]
sorted_data = [i for i in sort_data(data, 'dataset')]
expect = [
{'run': 100, 'id': 1, 'dataset': 'bla1'},
{'run': 700, 'id': 2, 'dataset': 'bla2'},
{'run': 400, 'id': 3, 'dataset': 'bla3'},
{'run': 300, 'id': 4, 'dataset': 'bla4'},
{'run': 800, 'id': 5, 'dataset': 'bla5'},
{'run': 200, 'id': 6, 'dataset': 'bla6'},
]
self.assertEqual(expect, sorted_data)
sorted_data = [i for i in sort_data(data, 'run')]
expect = [
{'run': 100, 'id': 1, 'dataset': 'bla1'},
{'run': 200, 'id': 6, 'dataset': 'bla6'},
{'run': 300, 'id': 4, 'dataset': 'bla4'},
{'run': 400, 'id': 3, 'dataset': 'bla3'},
{'run': 700, 'id': 2, 'dataset': 'bla2'},
{'run': 800, 'id': 5, 'dataset': 'bla5'},
]
self.assertEqual(expect, sorted_data)
sorted_data = [i for i in sort_data(data, 'run', 'desc')]
expect = [
{'run': 800, 'id': 5, 'dataset': 'bla5'},
{'run': 700, 'id': 2, 'dataset': 'bla2'},
{'run': 400, 'id': 3, 'dataset': 'bla3'},
{'run': 300, 'id': 4, 'dataset': 'bla4'},
{'run': 200, 'id': 6, 'dataset': 'bla6'},
{'run': 100, 'id': 1, 'dataset': 'bla1'},
]
self.assertEqual(expect, sorted_data)
def get_from_cache(self, query, idx=0, limit=0, skey=None, order='asc'):
"""
Retreieve results from cache, otherwise return null.
"""
idx = int(idx)
limit = long(limit)
stop = idx + limit
key = genkey(query)
res = self.memcache.get(key)
id = idx
if res and type(res) is int:
self.logger.info("DASMemcache::result(%s) using cache" % query)
if skey:
rowlist = [i for i in range(0, res)]
rowdict = self.memcache.get_multi(rowlist, key_prefix=key)
data = rowdict.values()
gendata = (i for i in sort_data(data, skey, order))
def subgroup(gen, idx, stop):
"""Extract sub-group of results from generator"""
id = 0
for item in gen:
if stop:
if id >= idx and id < stop:
yield item
else:
if id >= idx:
yield item
id += 1
items = subgroup(gendata, idx, stop)
else:
if limit:
if limit > res:
stop = res
rowlist = [i for i in range(idx, stop)]
else:
rowlist = [i for i in range(0, res)]
rowdict = self.memcache.get_multi(rowlist, key_prefix=key)
items = rowdict.values()
for item in items:
# item['id'] = id
yield item
id += 1
def get_from_cache(self, query, idx=0, limit=0, skey=None, order="asc"):
"""
Retreieve results from cache, otherwise return null.
"""
idx = int(idx)
limit = long(limit)
stop = idx + limit
key = genkey(query)
res = self.memcache.get(key)
id = idx
if res and type(res) is types.IntType:
self.logger.info("DASMemcache::result(%s) using cache" % query)
if skey:
rowlist = [i for i in range(0, res)]
rowdict = self.memcache.get_multi(rowlist, key_prefix=key)
data = rowdict.values()
gendata = (i for i in sort_data(data, skey, order))
def subgroup(gen, idx, stop):
"""Extract sub-group of results from generator"""
id = 0
for item in gen:
if stop:
if id >= idx and id < stop:
yield item
else:
if id >= idx:
yield item
id += 1
items = subgroup(gendata, idx, stop)
else:
if limit:
if limit > res:
stop = res
rowlist = [i for i in range(idx, stop)]
else:
rowlist = [i for i in range(0, res)]
rowdict = self.memcache.get_multi(rowlist, key_prefix=key)
items = rowdict.values()
for item in items:
# item['id'] = id
yield item
id += 1
def get_from_cache(self, query, idx=0, limit=0, skey=None, order='asc'):
"""
Retreieve results from cache, otherwise return null.
"""
# id = int(idx)
idx = int(idx)
stop = idx + long(limit)
key = genkey(query)
sysdir = os.path.join(self.dir, self.get_system(query))
session = self.session()
try: # transactions
res = session.query(Query, Location).\
filter(Query.dir_id==Location.id).\
filter(Query.hash==key)
session.commit()
except:
session.rollback()
self.logger.debug(traceback.format_exc())
pass
for qobj, dobj in res:
valid = eval(qobj.expire) - time.time()
timestring = eval(qobj.create)
idir = dobj.dir
filename = os.path.join(idir, key)
self.logger.info("DASFilecache::get_from_cache %s" % filename)
if valid > 0:
msg = "found valid query in cache, key=%s" % key
self.logger.debug("DASFilecache::get_from_cache %s" % msg)
if os.path.isfile(filename):
fdr = open(filename, 'rb')
if skey:
# first retrieve full list of results and sort it
data = []
id = 0
while 1:
try:
res = marshal.load(fdr)
if type(res) is dict:
res['id'] = id
data.append(res)
id += 1
except EOFError as err:
break
fdr.close()
sorted_data = sort_data(data, skey, order)
index = 0
for row in sorted_data:
if limit:
if index >= idx and index < stop:
yield row
else:
if index >= idx:
yield row
index += 1
else:
id = 0
while 1:
try:
res = marshal.load(fdr)
if type(res) is dict:
res['id'] = id
if limit:
if id >= idx and id < stop:
yield res
if id == stop:
break
else:
yield res
id += 1
except EOFError as err:
break
fdr.close()
else:
msg = "found expired query in cache, key=%s" % key
self.logger.debug("DASFilecache::get_from_cache %s" % msg)
fdir = os.path.split(filename)[0]
if os.path.isfile(filename):
os.remove(filename)
clean_dirs(fdir)
try: # session transactions
session.delete(qobj)
session.commit()
except:
session.rollback()
self.logger.debug(traceback.format_exc())
msg = "Unable to delete object from DAS filecache DB"
raise Exception(msg)
def test_sorting(self):
"""Test sorting routines"""
data = [
{
'id': 6,
'dataset': 'bla6',
'run': 200
},
{
'id': 1,
'dataset': 'bla1',
'run': 100
},
{
'id': 2,
'dataset': 'bla2',
'run': 700
},
{
'id': 3,
'dataset': 'bla3',
'run': 400
},
{
'id': 4,
'dataset': 'bla4',
'run': 300
},
{
'id': 5,
'dataset': 'bla5',
'run': 800
},
]
sorted_data = [i for i in sort_data(data, 'dataset')]
expect = [
{
'run': 100,
'id': 1,
'dataset': 'bla1'
},
{
'run': 700,
'id': 2,
'dataset': 'bla2'
},
{
'run': 400,
'id': 3,
'dataset': 'bla3'
},
{
'run': 300,
'id': 4,
'dataset': 'bla4'
},
{
'run': 800,
'id': 5,
'dataset': 'bla5'
},
{
'run': 200,
'id': 6,
'dataset': 'bla6'
},
]
self.assertEqual(expect, sorted_data)
sorted_data = [i for i in sort_data(data, 'run')]
expect = [
{
'run': 100,
'id': 1,
'dataset': 'bla1'
},
{
'run': 200,
'id': 6,
'dataset': 'bla6'
},
{
'run': 300,
'id': 4,
'dataset': 'bla4'
},
{
'run': 400,
'id': 3,
'dataset': 'bla3'
},
{
'run': 700,
'id': 2,
'dataset': 'bla2'
},
{
'run': 800,
'id': 5,
'dataset': 'bla5'
},
]
self.assertEqual(expect, sorted_data)
sorted_data = [i for i in sort_data(data, 'run', 'desc')]
expect = [
{
'run': 800,
'id': 5,
'dataset': 'bla5'
},
{
'run': 700,
'id': 2,
'dataset': 'bla2'
},
{
'run': 400,
'id': 3,
'dataset': 'bla3'
},
{
'run': 300,
'id': 4,
'dataset': 'bla4'
},
{
'run': 200,
'id': 6,
'dataset': 'bla6'
},
{
'run': 100,
'id': 1,
'dataset': 'bla1'
},
]
self.assertEqual(expect, sorted_data)
def get_from_cache(self, query, idx=0, limit=0, skey=None, order="asc"):
"""
Retreieve results from cache, otherwise return null.
"""
# id = int(idx)
idx = int(idx)
stop = idx + long(limit)
key = genkey(query)
sysdir = os.path.join(self.dir, self.get_system(query))
session = self.session()
try: # transactions
res = session.query(Query, Location).filter(Query.dir_id == Location.id).filter(Query.hash == key)
session.commit()
except:
session.rollback()
self.logger.debug(traceback.format_exc())
pass
for qobj, dobj in res:
valid = eval(qobj.expire) - time.time()
timestring = eval(qobj.create)
idir = dobj.dir
filename = os.path.join(idir, key)
self.logger.info("DASFilecache::get_from_cache %s" % filename)
if valid > 0:
msg = "found valid query in cache, key=%s" % key
self.logger.debug("DASFilecache::get_from_cache %s" % msg)
if os.path.isfile(filename):
fdr = open(filename, "rb")
if skey:
# first retrieve full list of results and sort it
data = []
id = 0
while 1:
try:
res = marshal.load(fdr)
if type(res) is types.DictType:
res["id"] = id
data.append(res)
id += 1
except EOFError, err:
break
fdr.close()
sorted_data = sort_data(data, skey, order)
index = 0
for row in sorted_data:
if limit:
if index >= idx and index < stop:
yield row
else:
if index >= idx:
yield row
index += 1
else:
id = 0
while 1:
try:
res = marshal.load(fdr)
if type(res) is types.DictType:
res["id"] = id
if limit:
if id >= idx and id < stop:
yield res
if id == stop:
break
else:
yield res
id += 1
except EOFError, err:
break
fdr.close()
