def test_das_dateformat(self): "Test das_dateformat function" obj1 = '20120101' obj2 = '20120101 00:00:01' time1 = das_dateformat(obj1) time2 = das_dateformat(obj2) self.assertEqual(time1, 1325376000) self.assertEqual(time2, 1325376001) wrong = '2012' self.assertRaises(Exception, das_dateformat, wrong) wrong = '20120101 12' self.assertRaises(Exception, das_dateformat, wrong)
def adjust_value(val): "Adjust value to DAS patterns" if date_yyyymmdd_pattern.match(val): return das_dateformat(val) elif int_number_pattern.match(val): return int(val) return val
def p_date_for_filter(self, p): """oneexp : DASKEY EQUAL DATE | DASKEY FILTER_OPERATOR DATE | DASKEY_ATTR EQUAL DATE | DASKEY_ATTR FILTER_OPERATOR DATE""" val = '' if len(p) == 4: val += str(p[1]) + str(p[2]) + str(das_dateformat(p[3])) p[0] = [val]
def spec_entry(key, oper, val): "Convert key oper val triplet into MongoDB spec entry" spec = {} if oper == '=' or oper == 'last': spec[key] = val if key == 'date' and date_yyyymmdd_pattern.match(val): spec[key] = das_dateformat(val) elif oper == 'in' and isinstance(val, list): spec[key] = {'$in': val} if key == 'date': out = [das_dateformat(d) for d in val] spec[key] = {'$in': out} elif oper == 'between' and isinstance(val, list): spec[key] = {'$gte': min(val), '$lte': max(val)} if key == 'date': spec[key] = {'$gte': das_dateformat(str(min(val))), '$lte': das_dateformat(str(max(val)))} else: Exception('Not implemented spec entry') return spec
def spec_entry(key, oper, val): "Convert key oper val triplet into MongoDB spec entry" spec = {} if oper == '=' or oper == 'last': spec[key] = val if key == 'date' and date_yyyymmdd_pattern.match(val): spec[key] = das_dateformat(val) elif oper == 'in' and isinstance(val, list): spec[key] = {'$in': val} if key == 'date': out = [das_dateformat(d) for d in val] spec[key] = {'$in': out} elif oper == 'between' and isinstance(val, list): spec[key] = {'$gte': min(val), '$lte': max(val)} if key == 'date': spec[key] = { '$gte': das_dateformat(str(min(val))), '$lte': das_dateformat(str(max(val))) } else: Exception('Not implemented spec entry') return spec
def convert_datetime(sec): """ Convert seconds since epoch or YYYYMMDD to date YYYY-MM-DD """ value = str(sec) pat = date_yyyymmdd_pattern pat2 = unix_time_pattern if pat.match(value): # we accept YYYYMMDD return das_dateformat(value) elif pat2.match(value): return value else: msg = 'Unacceptable date format' raise Exception(msg)
def ply2mongo(query): """ DAS-QL query : file block=123 | grep file.size | sum(file.size) PLY query : {'keys': [('keyop', 'file', None, None), ('keyop', 'block', '=', 123)], 'pipe': [('filter', 'grep', ['file.size']), ('aggregators', ('aggregator', 'sum', 'file.size'))]} Mongo query : {'fields': [u'file'], 'spec': {u'block.name': 123}, 'filters': ['file.size'], 'aggregators': [('sum', 'file.size')]} """ mongodict = {} filters = {} if 'pipe' in query: for item in query['pipe']: if item[0] == 'filter': dasfilter, name, args = item if dasfilter == 'filter' and name in ['grep', 'sort']: if name in filters: filters[name] += args else: filters[name] = args if dasfilter == 'filter' and name == 'unique': filters['unique'] = 1 if item[0] == 'aggregators': aggs = [(k[1], k[2]) for k in item[1:]] mongodict['aggregators'] = aggs if item[0] == 'mapreduce': _, name, _ = item mongodict['mapreduce'] = name if filters: mongodict['filters'] = filters fields = [] spec = {} inst = None system = None for _, name, oper, val in query['keys']: if name == 'instance': inst = val continue if name == 'system': system = val continue dasname = name if (oper and val) or (oper and val==0): # real condition value = val if name == 'date' and oper == '=': value = das_dateformat(value) if oper == 'in': vlist = list(val[1:]) if name == 'date': vlist = [das_dateformat(x) for x in vlist] value = {'$in' : vlist} if oper == 'between': vlist = list(val[1:]) if name == 'date': vlist = [das_dateformat(x) for x in vlist] vlist.sort() value = {'$gte' : vlist[0], '$lte': vlist[-1]} else: # selection field if isinstance(name, list) or isinstance(name, tuple): for item in name: if item not in fields: fields.append(item) elif isinstance(name, basestring): if name not in fields: fields.append(name) else: msg = 'Unsupported data type, %s, type=%s' % (name, type(name)) raise Exception(msg) value = '*' if dasname in spec: exist_value = spec[dasname] if isinstance(exist_value, list) or isinstance(exist_value, tuple): array = [r for r in exist_value] + [value] elif exist_value == '*' and value != '*': array = value elif value == '*' and exist_value != '*': array = exist_value elif value == '*' and exist_value == '*': array = value else: array = [exist_value, value] spec[dasname] = array else: if isinstance(dasname, list) or isinstance(dasname, tuple): for item in dasname: spec[item] = value else: spec[dasname] = value if fields: mongodict['fields'] = fields for key in fields: if len(spec.keys()) != 1 and key in spec: if spec[key] == '*' : del spec[key] else: if len(spec.keys()) == 1: mongodict['fields'] = [spec.keys()[0].split('.')[0]] else: mongodict['fields'] = None mongodict['spec'] = spec if inst: mongodict['instance'] = inst if system: mongodict['system'] = system if len(spec.keys()) == 1 and spec.values() == ['*'] and \ spec.keys()[0] not in ['records', 'queries', 'status']: msg = 'Single DAS key with no conditions, mongodict=%s' \ % mongodict raise Exception(msg) return mongodict