def size_format(uinput):
"""
Format file size utility, it converts file size into KB, MB, GB, TB, PB units
"""
if not (float_number_pattern.match(str(uinput)) or \
int_number_pattern.match(str(uinput))):
return 'N/A'
try:
num = float(uinput)
except Exception as exc:
print_exc(exc)
return "N/A"
base = 1000. # power of 10, or use 1024. for power of 2
for xxx in ['', 'KB', 'MB', 'GB', 'TB', 'PB']:
if num < base:
return "%3.1f%s" % (num, xxx)
num /= base
def parse(self, query):
"Parse input query"
spec = {}
filters = {}
aggregators = []
fields = []
keys = []
pipe = []
relaxed_query = relax(query, self.operators).split()
if self.verbose:
print("\n### input query=%s, relaxed=%s" % (query, relaxed_query))
tot = len(relaxed_query)
idx = 0
while idx < tot:
item = relaxed_query[idx]
if self.verbose > 1:
print("parse item", item)
if item == '|':
step = self.parse_pipe(relaxed_query[idx:], filters, aggregators)
idx += step
if item == ',':
idx += 1
continue
next_elem = relaxed_query[idx+1] if idx+1 < tot else None
next_next_elem = relaxed_query[idx+2] if idx+2 < tot else None
if self.verbose > 1:
print("### parse items", item, next_elem, next_next_elem)
if next_elem and (next_elem == ',' or next_elem in self.daskeys):
if item in self.daskeys:
fields.append(item)
idx += 1
continue
elif next_elem in self.operators:
oper = next_elem
if item not in self.daskeys+self.specials:
error(relaxed_query, idx, 'Wrong DAS key')
if next_next_elem.startswith('['):
val, step = parse_array(relaxed_query[idx:], next_elem, item)
spec.update(spec_entry(item, next_elem, val))
idx += step
elif next_elem in ['in', 'beetween'] and \
not next_next_elem.startswith('['):
msg = '"%s" operator ' % next_elem
msg += 'should be followed by square bracket value'
error(relaxed_query, idx, msg)
elif next_next_elem.startswith('"'):
val, step = parse_quotes(relaxed_query[idx:], '"')
spec.update(spec_entry(item, next_elem, val))
idx += step
elif next_next_elem.startswith("'"):
val, step = parse_quotes(relaxed_query[idx:], "'")
spec.update(spec_entry(item, next_elem, val))
idx += step
else:
if float_number_pattern.match(next_next_elem):
next_next_elem = float(next_next_elem)
elif int_number_pattern.match(next_next_elem) and \
not date_yyyymmdd_pattern.match(next_next_elem):
next_next_elem = int(next_next_elem)
elif next_next_elem in self.daskeys:
msg = 'daskey operator daskey structure is not allowed'
error(relaxed_query, idx, msg)
spec.update(spec_entry(item, next_elem, next_next_elem))
idx += 3
continue
elif item == '|':
step = self.parse_pipe(relaxed_query[idx:], filters, aggregators)
idx += step
elif not next_elem and not next_next_elem:
if item in self.daskeys:
fields.append(item)
idx += 1
else:
error(relaxed_query, idx, 'Not a DAS key')
else:
error(relaxed_query, idx)
out = {}
for word in ['instance', 'system']:
if word in spec:
out[word] = spec.pop(word)
if not fields:
fields = [k for k in spec.keys() if k in self.daskeys]
if len(fields) > 1:
fields = None # ambiguous spec, we don't know which field to look-up
if fields and not spec:
error(relaxed_query, 0, 'No conditition specified')
out['fields'] = fields
out['spec'] = spec
# perform cross-check of filter values
for key, item in filters.items():
if key not in ['grep', 'sort']:
continue
for val in item:
daskeyvalue_check(query, val, self.daskeys)
# perform cross-check of aggregator values
for _, val in aggregators:
daskeyvalue_check(query, val, self.daskeys)
if filters:
out['filters'] = filters
if aggregators:
out['aggregators'] = aggregators
if self.verbose:
print("MongoDB query: %s" % out)
return out
def parse_filter(spec, flt):
"""
Parse given filter and return MongoDB key/value dictionary.
Be smart not to overwrite spec condition of DAS query.
"""
if flt.find('=') != -1 and flt.find('!=') == -1 and\
(flt.find('<') == -1 and flt.find('>') == -1):
key, val = flt.split('=')
if int_number_pattern.match(str(val)):
val = int(val)
elif float_number_pattern.match(str(val)):
val = float(val)
elif isinstance(val, str) or isinstance(val, unicode):
if val.find('*') != -1:
val = re.compile('%s' % val.replace('*', '.*'))
val = parse_filter_string(val)
return {key:val}
elif flt.find('!=') != -1 and \
(flt.find('<') == -1 and flt.find('>') == -1):
key, val = flt.split('!=')
if int_number_pattern.match(str(val)):
val = int(val)
elif float_number_pattern.match(str(val)):
val = float(val)
elif isinstance(val, str) or isinstance(val, unicode):
if val.find('*') != -1:
# val = re.compile('%s' % val.replace('*', '.*'))
val = re.compile('^(?:(?!%s).)*$' % val.replace('*', '.*'))
else:
val = re.compile('^(?:(?!%s).)*$' % val)
val = parse_filter_string(val)
return {key: val}
return {key: {'$ne': val}}
elif flt.find('<=') != -1:
key, val = flt.split('<=')
if int_number_pattern.match(str(val)):
val = int(val)
if float_number_pattern.match(str(val)):
val = float(val)
return {key: {'$lte': val}}
elif flt.find('<') != -1:
key, val = flt.split('<')
if int_number_pattern.match(str(val)):
val = int(val)
if float_number_pattern.match(str(val)):
val = float(val)
return {key: {'$lt': val}}
elif flt.find('>=') != -1:
key, val = flt.split('>=')
if int_number_pattern.match(str(val)):
val = int(val)
if float_number_pattern.match(str(val)):
val = float(val)
return {key: {'$gte': val}}
elif flt.find('>') != -1:
key, val = flt.split('>')
if int_number_pattern.match(str(val)):
val = int(val)
if float_number_pattern.match(str(val)):
val = float(val)
return {key: {'$gt': val}}
else:
if not spec.get(flt, None) and flt != 'unique':
return {flt:{'$exists':True}}
return {}
def parse_helper(self, query):
"Parse input query"
spec = {}
filters = {}
aggregators = []
fields = []
keys = []
pipe = []
relaxed_query = relax(query, self.operators).split()
if self.verbose:
print("\n### input query=%s, relaxed=%s" % (query, relaxed_query))
tot = len(relaxed_query)
idx = 0
while idx < tot:
item = relaxed_query[idx]
if self.verbose > 1:
print("parse item", item)
if item == '|':
step = self.parse_pipe(relaxed_query[idx:], filters,
aggregators)
idx += step
if item == ',':
idx += 1
continue
next_elem = relaxed_query[idx + 1] if idx + 1 < tot else None
next_next_elem = relaxed_query[idx + 2] if idx + 2 < tot else None
if self.verbose > 1:
print("### parse items", item, next_elem, next_next_elem)
if next_elem and (next_elem == ',' or next_elem in self.daskeys):
if item in self.daskeys:
fields.append(item)
idx += 1
continue
elif next_elem in self.operators:
oper = next_elem
if item not in self.daskeys + self.specials:
error(relaxed_query, idx, 'Wrong DAS key')
if next_next_elem.startswith('['):
val, step = parse_array(relaxed_query[idx:], next_elem,
item)
spec.update(spec_entry(item, next_elem, val))
idx += step
elif next_elem in ['in', 'beetween'] and \
not next_next_elem.startswith('['):
msg = '"%s" operator ' % next_elem
msg += 'should be followed by square bracket value'
error(relaxed_query, idx, msg)
elif next_next_elem.startswith('"'):
val, step = parse_quotes(relaxed_query[idx:], '"')
spec.update(spec_entry(item, next_elem, val))
idx += step
elif next_next_elem.startswith("'"):
val, step = parse_quotes(relaxed_query[idx:], "'")
spec.update(spec_entry(item, next_elem, val))
idx += step
else:
if float_number_pattern.match(next_next_elem):
next_next_elem = float(next_next_elem)
elif int_number_pattern.match(next_next_elem) and \
not date_yyyymmdd_pattern.match(next_next_elem):
next_next_elem = int(next_next_elem)
elif next_next_elem in self.daskeys:
msg = 'daskey operator daskey structure is not allowed'
error(relaxed_query, idx, msg)
spec.update(spec_entry(item, next_elem, next_next_elem))
idx += 3
continue
elif item == '|':
step = self.parse_pipe(relaxed_query[idx:], filters,
aggregators)
idx += step
elif not next_elem and not next_next_elem:
if item in self.daskeys:
fields.append(item)
idx += 1
else:
error(relaxed_query, idx, 'Not a DAS key')
else:
error(relaxed_query, idx)
out = {}
for word in ['instance', 'system']:
if word in spec:
out[word] = spec.pop(word)
if not fields:
fields = [k for k in spec.keys() if k in self.daskeys]
if len(fields) > 1:
fields = None # ambiguous spec, we don't know which field to look-up
if fields and not spec:
error(relaxed_query, 0, 'No conditition specified')
out['fields'] = fields
out['spec'] = spec
# perform cross-check of filter values
for key, item in filters.items():
if key not in ['grep', 'sort']:
continue
for val in item:
daskeyvalue_check(query, val, self.daskeys)
# perform cross-check of aggregator values
for _, val in aggregators:
daskeyvalue_check(query, val, self.daskeys)
if filters:
out['filters'] = filters
if aggregators:
out['aggregators'] = aggregators
if self.verbose:
print("MongoDB query: %s" % out)
return out
