def adjust_value(val):
"Adjust value to DAS patterns"
if date_yyyymmdd_pattern.match(val):
return das_dateformat(val)
elif int_number_pattern.match(val):
return int(val)
return val
def parse_array(query, oper, daskey):
"""
Parse array boundaries in query and return array and new index
Return [('array', begin, end), shift]
"""
idx = query.index('[')
jdx = query.index(']')
try:
subq = ' '.join(query[idx:jdx+1])
if daskey == 'dataset':
subq = subq.replace('[', '["').replace(']', '"]').replace(',', '","')
subq = ''.join([s.strip() for s in subq.split()])
val = json.loads(subq)
except Exception as exc:
error(query, idx, 'Fail to extract value from square brackets, '+str(exc))
if oper == 'in':
out = val
elif oper == 'between':
out = [min(val), max(val)]
else:
error(query, idx, 'Fail to extract value from square brackets')
if daskey == 'date':
for item in val:
if not date_yyyymmdd_pattern.match(str(item)):
error(query, idx, 'Wrong date value, use YYYYMMDD format')
return out, jdx+1
def adjust_value(val):
"Adjust value to DAS patterns"
if date_yyyymmdd_pattern.match(val):
return das_dateformat(val)
elif int_number_pattern.match(val):
return int(val)
return val
def parse_array(query, oper, daskey):
"""
Parse array boundaries in query and return array and new index
Return [('array', begin, end), shift]
"""
idx = query.index('[')
jdx = query.index(']')
try:
subq = ' '.join(query[idx:jdx + 1])
if daskey == 'dataset':
subq = subq.replace('[', '["').replace(']',
'"]').replace(',', '","')
subq = ''.join([s.strip() for s in subq.split()])
val = json.loads(subq)
except Exception as exc:
error(query, idx,
'Fail to extract value from square brackets, ' + str(exc))
if oper == 'in':
out = val
elif oper == 'between':
out = [min(val), max(val)]
else:
error(query, idx, 'Fail to extract value from square brackets')
if daskey == 'date':
for item in val:
if not date_yyyymmdd_pattern.match(str(item)):
error(query, idx, 'Wrong date value, use YYYYMMDD format')
return out, jdx + 1
def spec_entry(key, oper, val):
"Convert key oper val triplet into MongoDB spec entry"
spec = {}
if oper == '=' or oper == 'last':
spec[key] = val
if key == 'date' and date_yyyymmdd_pattern.match(val):
spec[key] = das_dateformat(val)
elif oper == 'in' and isinstance(val, list):
spec[key] = {'$in': val}
if key == 'date':
out = [das_dateformat(d) for d in val]
spec[key] = {'$in': out}
elif oper == 'between' and isinstance(val, list):
spec[key] = {'$gte': min(val), '$lte': max(val)}
if key == 'date':
spec[key] = {'$gte': das_dateformat(str(min(val))),
'$lte': das_dateformat(str(max(val)))}
else:
Exception('Not implemented spec entry')
return spec
def spec_entry(key, oper, val):
"Convert key oper val triplet into MongoDB spec entry"
spec = {}
if oper == '=' or oper == 'last':
spec[key] = val
if key == 'date' and date_yyyymmdd_pattern.match(val):
spec[key] = das_dateformat(val)
elif oper == 'in' and isinstance(val, list):
spec[key] = {'$in': val}
if key == 'date':
out = [das_dateformat(d) for d in val]
spec[key] = {'$in': out}
elif oper == 'between' and isinstance(val, list):
spec[key] = {'$gte': min(val), '$lte': max(val)}
if key == 'date':
spec[key] = {
'$gte': das_dateformat(str(min(val))),
'$lte': das_dateformat(str(max(val)))
}
else:
Exception('Not implemented spec entry')
return spec
def parse(self, query):
"Parse input query"
spec = {}
filters = {}
aggregators = []
fields = []
keys = []
pipe = []
relaxed_query = relax(query, self.operators).split()
if self.verbose:
print("\n### input query=%s, relaxed=%s" % (query, relaxed_query))
tot = len(relaxed_query)
idx = 0
while idx < tot:
item = relaxed_query[idx]
if self.verbose > 1:
print("parse item", item)
if item == '|':
step = self.parse_pipe(relaxed_query[idx:], filters, aggregators)
idx += step
if item == ',':
idx += 1
continue
next_elem = relaxed_query[idx+1] if idx+1 < tot else None
next_next_elem = relaxed_query[idx+2] if idx+2 < tot else None
if self.verbose > 1:
print("### parse items", item, next_elem, next_next_elem)
if next_elem and (next_elem == ',' or next_elem in self.daskeys):
if item in self.daskeys:
fields.append(item)
idx += 1
continue
elif next_elem in self.operators:
oper = next_elem
if item not in self.daskeys+self.specials:
error(relaxed_query, idx, 'Wrong DAS key')
if next_next_elem.startswith('['):
val, step = parse_array(relaxed_query[idx:], next_elem, item)
spec.update(spec_entry(item, next_elem, val))
idx += step
elif next_elem in ['in', 'beetween'] and \
not next_next_elem.startswith('['):
msg = '"%s" operator ' % next_elem
msg += 'should be followed by square bracket value'
error(relaxed_query, idx, msg)
elif next_next_elem.startswith('"'):
val, step = parse_quotes(relaxed_query[idx:], '"')
spec.update(spec_entry(item, next_elem, val))
idx += step
elif next_next_elem.startswith("'"):
val, step = parse_quotes(relaxed_query[idx:], "'")
spec.update(spec_entry(item, next_elem, val))
idx += step
else:
if float_number_pattern.match(next_next_elem):
next_next_elem = float(next_next_elem)
elif int_number_pattern.match(next_next_elem) and \
not date_yyyymmdd_pattern.match(next_next_elem):
next_next_elem = int(next_next_elem)
elif next_next_elem in self.daskeys:
msg = 'daskey operator daskey structure is not allowed'
error(relaxed_query, idx, msg)
spec.update(spec_entry(item, next_elem, next_next_elem))
idx += 3
continue
elif item == '|':
step = self.parse_pipe(relaxed_query[idx:], filters, aggregators)
idx += step
elif not next_elem and not next_next_elem:
if item in self.daskeys:
fields.append(item)
idx += 1
else:
error(relaxed_query, idx, 'Not a DAS key')
else:
error(relaxed_query, idx)
out = {}
for word in ['instance', 'system']:
if word in spec:
out[word] = spec.pop(word)
if not fields:
fields = [k for k in spec.keys() if k in self.daskeys]
if len(fields) > 1:
fields = None # ambiguous spec, we don't know which field to look-up
if fields and not spec:
error(relaxed_query, 0, 'No conditition specified')
out['fields'] = fields
out['spec'] = spec
# perform cross-check of filter values
for key, item in filters.items():
if key not in ['grep', 'sort']:
continue
for val in item:
daskeyvalue_check(query, val, self.daskeys)
# perform cross-check of aggregator values
for _, val in aggregators:
daskeyvalue_check(query, val, self.daskeys)
if filters:
out['filters'] = filters
if aggregators:
out['aggregators'] = aggregators
if self.verbose:
print("MongoDB query: %s" % out)
return out
def parse_helper(self, query):
"Parse input query"
spec = {}
filters = {}
aggregators = []
fields = []
keys = []
pipe = []
relaxed_query = relax(query, self.operators).split()
if self.verbose:
print("\n### input query=%s, relaxed=%s" % (query, relaxed_query))
tot = len(relaxed_query)
idx = 0
while idx < tot:
item = relaxed_query[idx]
if self.verbose > 1:
print("parse item", item)
if item == '|':
step = self.parse_pipe(relaxed_query[idx:], filters,
aggregators)
idx += step
if item == ',':
idx += 1
continue
next_elem = relaxed_query[idx + 1] if idx + 1 < tot else None
next_next_elem = relaxed_query[idx + 2] if idx + 2 < tot else None
if self.verbose > 1:
print("### parse items", item, next_elem, next_next_elem)
if next_elem and (next_elem == ',' or next_elem in self.daskeys):
if item in self.daskeys:
fields.append(item)
idx += 1
continue
elif next_elem in self.operators:
oper = next_elem
if item not in self.daskeys + self.specials:
error(relaxed_query, idx, 'Wrong DAS key')
if next_next_elem.startswith('['):
val, step = parse_array(relaxed_query[idx:], next_elem,
item)
spec.update(spec_entry(item, next_elem, val))
idx += step
elif next_elem in ['in', 'beetween'] and \
not next_next_elem.startswith('['):
msg = '"%s" operator ' % next_elem
msg += 'should be followed by square bracket value'
error(relaxed_query, idx, msg)
elif next_next_elem.startswith('"'):
val, step = parse_quotes(relaxed_query[idx:], '"')
spec.update(spec_entry(item, next_elem, val))
idx += step
elif next_next_elem.startswith("'"):
val, step = parse_quotes(relaxed_query[idx:], "'")
spec.update(spec_entry(item, next_elem, val))
idx += step
else:
if float_number_pattern.match(next_next_elem):
next_next_elem = float(next_next_elem)
elif int_number_pattern.match(next_next_elem) and \
not date_yyyymmdd_pattern.match(next_next_elem):
next_next_elem = int(next_next_elem)
elif next_next_elem in self.daskeys:
msg = 'daskey operator daskey structure is not allowed'
error(relaxed_query, idx, msg)
spec.update(spec_entry(item, next_elem, next_next_elem))
idx += 3
continue
elif item == '|':
step = self.parse_pipe(relaxed_query[idx:], filters,
aggregators)
idx += step
elif not next_elem and not next_next_elem:
if item in self.daskeys:
fields.append(item)
idx += 1
else:
error(relaxed_query, idx, 'Not a DAS key')
else:
error(relaxed_query, idx)
out = {}
for word in ['instance', 'system']:
if word in spec:
out[word] = spec.pop(word)
if not fields:
fields = [k for k in spec.keys() if k in self.daskeys]
if len(fields) > 1:
fields = None # ambiguous spec, we don't know which field to look-up
if fields and not spec:
error(relaxed_query, 0, 'No conditition specified')
out['fields'] = fields
out['spec'] = spec
# perform cross-check of filter values
for key, item in filters.items():
if key not in ['grep', 'sort']:
continue
for val in item:
daskeyvalue_check(query, val, self.daskeys)
# perform cross-check of aggregator values
for _, val in aggregators:
daskeyvalue_check(query, val, self.daskeys)
if filters:
out['filters'] = filters
if aggregators:
out['aggregators'] = aggregators
if self.verbose:
print("MongoDB query: %s" % out)
return out
