def test_das_dateformat(self):
"Test das_dateformat function"
obj1 = '20120101'
obj2 = '20120101 00:00:01'
time1 = das_dateformat(obj1)
time2 = das_dateformat(obj2)
self.assertEqual(time1, 1325376000)
self.assertEqual(time2, 1325376001)
wrong = '2012'
self.assertRaises(Exception, das_dateformat, wrong)
wrong = '20120101 12'
self.assertRaises(Exception, das_dateformat, wrong)
def test_das_dateformat(self):
"Test das_dateformat function"
obj1 = '20120101'
obj2 = '20120101 00:00:01'
time1 = das_dateformat(obj1)
time2 = das_dateformat(obj2)
self.assertEqual(time1, 1325376000)
self.assertEqual(time2, 1325376001)
wrong = '2012'
self.assertRaises(Exception, das_dateformat, wrong)
wrong = '20120101 12'
self.assertRaises(Exception, das_dateformat, wrong)
def adjust_value(val):
"Adjust value to DAS patterns"
if date_yyyymmdd_pattern.match(val):
return das_dateformat(val)
elif int_number_pattern.match(val):
return int(val)
return val
def adjust_value(val):
"Adjust value to DAS patterns"
if date_yyyymmdd_pattern.match(val):
return das_dateformat(val)
elif int_number_pattern.match(val):
return int(val)
return val
def p_date_for_filter(self, p):
"""oneexp : DASKEY EQUAL DATE
| DASKEY FILTER_OPERATOR DATE
| DASKEY_ATTR EQUAL DATE
| DASKEY_ATTR FILTER_OPERATOR DATE"""
val = ''
if len(p) == 4:
val += str(p[1]) + str(p[2]) + str(das_dateformat(p[3]))
p[0] = [val]
def spec_entry(key, oper, val):
"Convert key oper val triplet into MongoDB spec entry"
spec = {}
if oper == '=' or oper == 'last':
spec[key] = val
if key == 'date' and date_yyyymmdd_pattern.match(val):
spec[key] = das_dateformat(val)
elif oper == 'in' and isinstance(val, list):
spec[key] = {'$in': val}
if key == 'date':
out = [das_dateformat(d) for d in val]
spec[key] = {'$in': out}
elif oper == 'between' and isinstance(val, list):
spec[key] = {'$gte': min(val), '$lte': max(val)}
if key == 'date':
spec[key] = {'$gte': das_dateformat(str(min(val))),
'$lte': das_dateformat(str(max(val)))}
else:
Exception('Not implemented spec entry')
return spec
def spec_entry(key, oper, val):
"Convert key oper val triplet into MongoDB spec entry"
spec = {}
if oper == '=' or oper == 'last':
spec[key] = val
if key == 'date' and date_yyyymmdd_pattern.match(val):
spec[key] = das_dateformat(val)
elif oper == 'in' and isinstance(val, list):
spec[key] = {'$in': val}
if key == 'date':
out = [das_dateformat(d) for d in val]
spec[key] = {'$in': out}
elif oper == 'between' and isinstance(val, list):
spec[key] = {'$gte': min(val), '$lte': max(val)}
if key == 'date':
spec[key] = {
'$gte': das_dateformat(str(min(val))),
'$lte': das_dateformat(str(max(val)))
}
else:
Exception('Not implemented spec entry')
return spec
def convert_datetime(sec):
"""
Convert seconds since epoch or YYYYMMDD to date YYYY-MM-DD
"""
value = str(sec)
pat = date_yyyymmdd_pattern
pat2 = unix_time_pattern
if pat.match(value): # we accept YYYYMMDD
return das_dateformat(value)
elif pat2.match(value):
return value
else:
msg = 'Unacceptable date format'
raise Exception(msg)
def convert_datetime(sec):
"""
Convert seconds since epoch or YYYYMMDD to date YYYY-MM-DD
"""
value = str(sec)
pat = date_yyyymmdd_pattern
pat2 = unix_time_pattern
if pat.match(value): # we accept YYYYMMDD
return das_dateformat(value)
elif pat2.match(value):
return value
else:
msg = 'Unacceptable date format'
raise Exception(msg)
def ply2mongo(query):
"""
DAS-QL query : file block=123 | grep file.size | sum(file.size)
PLY query : {'keys': [('keyop', 'file', None, None),
('keyop', 'block', '=', 123)],
'pipe': [('filter', 'grep', ['file.size']),
('aggregators', ('aggregator', 'sum', 'file.size'))]}
Mongo query : {'fields': [u'file'],
'spec': {u'block.name': 123}, 'filters': ['file.size'],
'aggregators': [('sum', 'file.size')]}
"""
mongodict = {}
filters = {}
if 'pipe' in query:
for item in query['pipe']:
if item[0] == 'filter':
dasfilter, name, args = item
if dasfilter == 'filter' and name in ['grep', 'sort']:
if name in filters:
filters[name] += args
else:
filters[name] = args
if dasfilter == 'filter' and name == 'unique':
filters['unique'] = 1
if item[0] == 'aggregators':
aggs = [(k[1], k[2]) for k in item[1:]]
mongodict['aggregators'] = aggs
if item[0] == 'mapreduce':
_, name, _ = item
mongodict['mapreduce'] = name
if filters:
mongodict['filters'] = filters
fields = []
spec = {}
inst = None
system = None
for _, name, oper, val in query['keys']:
if name == 'instance':
inst = val
continue
if name == 'system':
system = val
continue
dasname = name
if (oper and val) or (oper and val==0): # real condition
value = val
if name == 'date' and oper == '=':
value = das_dateformat(value)
if oper == 'in':
vlist = list(val[1:])
if name == 'date':
vlist = [das_dateformat(x) for x in vlist]
value = {'$in' : vlist}
if oper == 'between':
vlist = list(val[1:])
if name == 'date':
vlist = [das_dateformat(x) for x in vlist]
vlist.sort()
value = {'$gte' : vlist[0], '$lte': vlist[-1]}
else: # selection field
if isinstance(name, list) or isinstance(name, tuple):
for item in name:
if item not in fields:
fields.append(item)
elif isinstance(name, basestring):
if name not in fields:
fields.append(name)
else:
msg = 'Unsupported data type, %s, type=%s' % (name, type(name))
raise Exception(msg)
value = '*'
if dasname in spec:
exist_value = spec[dasname]
if isinstance(exist_value, list) or isinstance(exist_value, tuple):
array = [r for r in exist_value] + [value]
elif exist_value == '*' and value != '*':
array = value
elif value == '*' and exist_value != '*':
array = exist_value
elif value == '*' and exist_value == '*':
array = value
else:
array = [exist_value, value]
spec[dasname] = array
else:
if isinstance(dasname, list) or isinstance(dasname, tuple):
for item in dasname:
spec[item] = value
else:
spec[dasname] = value
if fields:
mongodict['fields'] = fields
for key in fields:
if len(spec.keys()) != 1 and key in spec:
if spec[key] == '*' :
del spec[key]
else:
if len(spec.keys()) == 1:
mongodict['fields'] = [spec.keys()[0].split('.')[0]]
else:
mongodict['fields'] = None
mongodict['spec'] = spec
if inst:
mongodict['instance'] = inst
if system:
mongodict['system'] = system
if len(spec.keys()) == 1 and spec.values() == ['*'] and \
spec.keys()[0] not in ['records', 'queries', 'status']:
msg = 'Single DAS key with no conditions, mongodict=%s' \
% mongodict
raise Exception(msg)
return mongodict
