def _normPyTuple(self, valu):
if len(valu) != 2:
raise s_exc.BadTypeValu(valu=valu, name=self.name,
mesg=f'Must be a 2-tuple of type {self.subtype.name}')
minv = self.subtype.norm(valu[0])[0]
maxv = self.subtype.norm(valu[1])[0]
if minv > maxv:
raise s_exc.BadTypeValu(valu=valu, name=self.name,
mesg='minval cannot be greater than maxval')
return (minv, maxv), {'subs': {'min': minv, 'max': maxv}}
def _normPyTuple(self, valu):
if len(valu) != 2:
raise s_exc.BadTypeValu(valu=valu, name=self.name,
mesg='Valu must contain valid latitude,longitude')
try:
latv = self.modl.type('geo:latitude').norm(valu[0])[0]
lonv = self.modl.type('geo:longitude').norm(valu[1])[0]
except Exception as e:
raise s_exc.BadTypeValu(valu=valu, name=self.name,
mesg=str(e)) from None
return (latv, lonv), {'subs': {'lat': latv, 'lon': lonv}}
def _normPyInt(self, valu):
if self.minval is not None and valu < self.minval:
mesg = f'value is below min={self.minval}'
raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg=mesg)
if self.maxval is not None and valu > self.maxval:
mesg = f'value is above max={self.maxval}'
raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg=mesg)
if self.enumrepr and valu not in self.enumrepr:
mesg = 'Value is not a valid enum value.'
raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg=mesg)
return valu, {}
async def nodesByPropArray(self, full, cmpr, valu):
prop = self.core.model.prop(full)
if prop is None:
mesg = f'No property named "{full}".'
raise s_exc.NoSuchProp(mesg=mesg)
if not isinstance(prop.type, s_types.Array):
mesg = f'Array synax is invalid on non array type: {prop.type.name}.'
raise s_exc.BadTypeValu(mesg=mesg)
cmprvals = prop.type.arraytype.getStorCmprs(cmpr, valu)
if prop.isform:
for layr in self.layers:
genr = layr.liftByPropArray(prop.name, None, cmprvals)
async for node in self._joinStorGenr(layr, genr):
if node.bylayer['ndef'] != layr:
continue
yield node
return
formname = None
if prop.form is not None:
formname = prop.form.name
for layr in self.layers:
genr = layr.liftByPropArray(formname, prop.name, cmprvals)
async for node in self._joinStorGenr(layr, genr):
if node.bylayer['props'].get(prop.name) != layr:
continue
yield node
def norm(self, valu):
try:
s_common.reqjsonsafe(valu)
except s_exc.MustBeJsonSafe as e:
raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg=str(e)) from None
byts = s_msgpack.en(valu)
return s_msgpack.un(byts), {}
async def nodesByPropArray(self, full, cmpr, valu):
prop = self.core.model.prop(full)
if prop is None:
mesg = f'No property named "{full}".'
raise s_exc.NoSuchProp(mesg=mesg)
if not isinstance(prop.type, s_types.Array):
mesg = f'Array synax is invalid on non array type: {prop.type.name}.'
raise s_exc.BadTypeValu(mesg=mesg)
cmprvals = prop.type.arraytype.getStorCmprs(cmpr, valu)
if prop.isform:
async for (buid, sodes) in self.core._liftByPropArray(
prop.name, None, cmprvals, self.layers):
node = await self._joinSodes(buid, sodes)
if node is not None:
yield node
return
formname = None
if prop.form is not None:
formname = prop.form.name
async for (buid, sodes) in self.core._liftByPropArray(
formname, prop.name, cmprvals, self.layers):
node = await self._joinSodes(buid, sodes)
if node is not None:
yield node
def bruteVersionStr(self, valu):
'''
Brute force the version out of a string.
Args:
valu (str): String to attempt to get version information for.
Notes:
This first attempts to parse strings using the it:semver normalization
before attempting to extract version parts out of the string.
Returns:
int, dict: The system normalized version integer and a subs dictionary.
'''
try:
valu, info = self.core.model.type('it:semver').norm(valu)
subs = info.get('subs')
return valu, subs
except s_exc.BadTypeValu:
# Try doing version part extraction by noming through the string
subs = s_version.parseVersionParts(valu)
if subs is None:
raise s_exc.BadTypeValu(valu=valu, name='bruteVersionStr',
mesg='Unable to brute force version parts out of the string')
if subs:
valu = s_version.packVersion(subs.get('major'),
subs.get('minor', 0),
subs.get('patch', 0))
return valu, subs
def _normPyStr(self, text):
try:
valu, off = s_grammar.parse_float(text, 0)
except Exception:
raise s_exc.BadTypeValu(valu=text, name=self.name,
mesg='Dist requires a valid float and dist '
'unit, no valid float found') from None
unit, off = s_grammar.nom(text, off, s_grammar.alphaset)
mult = units.get(unit.lower())
if mult is None:
raise s_exc.BadTypeValu(valu=text, name=self.name,
mesg='invalid/unknown dist unit: %s' % (unit,))
return int(valu * mult), {}
def _normPyStr(self, valu):
digs = digits(valu)
if not digs:
raise s_exc.BadTypeValu(valu=valu, name=self.name,
mesg='requires a digit string')
subs = {}
try:
info = s_l_phone.getPhoneInfo(int(digs))
except Exception as e: # pragma: no cover
raise s_exc.BadTypeValu(valu=valu, name=self.name,
mesg='Failed to get phone info')
cc = info.get('cc')
if cc is not None:
subs['loc'] = cc
# TODO prefix based validation?
return digs, {'subs': subs}
def _normPyStr(self, valu):
digs = digits(valu)
if not digs:
raise s_exc.BadTypeValu(valu=valu,
name=self.name,
mesg='requires a digit string')
return self._normPyInt(int(digs))
def _normPyTuple(self, valu):
fields = self.opts.get('fields')
if len(fields) != len(valu):
raise s_exc.BadTypeValu(
name=self.name,
valu=valu,
mesg='invalid number of fields given for norming')
subs = {}
adds = []
norms = []
for i, (name, typename) in enumerate(fields):
_type = self.tcache[name]
norm, info = _type.norm(valu[i])
subs[name] = norm
norms.append(norm)
for k, v in info.get('subs', {}).items():
subs[f'{name}:{k}'] = v
adds.extend(info.get('adds', ()))
norm = tuple(norms)
return norm, {'subs': subs, 'adds': adds}
def parsetz(text):
'''
Parse timezone from time string, with UTC as the default.
Args:
text (str): Time string
Returns:
tuple: A tuple of text with tz chars removed and base milliseconds to offset time.
'''
tz_hm = tz_hm_re.search(text)
if tz_hm is not None:
tzstr, rel, hrs, mins, _, _ = tz_hm.groups()
rel = 1 if rel == '-' else -1
base = rel * (onehour * int(hrs) + onemin * int(mins))
if abs(base) >= oneday:
raise s_exc.BadTypeValu(
valu=text,
name='time',
mesg=f'Timezone offset must be between +/- 24 hours')
return text.replace(tzstr, '', 1), base
return text, 0
def delta(text):
'''
Parse a simple time delta string and return the delta.
'''
text = text.strip().lower()
_, offs = _noms(text, 0, ' \t\r\n')
sign = '+'
if text and text[0] in ('+', '-'):
sign = text[0]
offs += 1
_, offs = _noms(text, offs, ' \t\r\n')
sizetext, offs = _noms(text, offs, '0123456789')
_, offs = _noms(text, offs, ' \t\r\n')
unittext = text[offs:]
size = int(sizetext, 0)
if sign == '-':
size = -size
base = timeunits.get(unittext)
if base is None:
mesg = f'unknown time delta units: {unittext}'
raise s_exc.BadTypeValu(name='time', valu=text, mesg=mesg)
return size * base
def wildrange(text):
'''
Parse an interval from a wild card time stamp: 2021/10/31*
'''
dttick, base, tlen = _rawparse(text)
if tlen not in (4, 6, 8, 10, 12, 14):
mesg = 'Time wild card position not supported.'
raise s_exc.BadTypeValu(mesg=mesg)
if tlen == 4:
dttock = dttick + relativedelta(years=1)
elif tlen == 6:
dttock = dttick + relativedelta(months=1)
elif tlen == 8:
dttock = dttick + relativedelta(days=1)
elif tlen == 10:
dttock = dttick + relativedelta(hours=1)
elif tlen == 12:
dttock = dttick + relativedelta(minutes=1)
else: # tlen = 14
dttock = dttick + relativedelta(seconds=1)
tick = int((dttick - EPOCH).total_seconds() * 1000 + base)
tock = int((dttock - EPOCH).total_seconds() * 1000 + base)
return (tick, tock)
def _normIntStr(self, valu):
try:
valu = float(valu)
except Exception:
raise s_exc.BadTypeValu(valu=valu, name=self.name,
mesg='Invalid float format')
return self._normFloat(valu)
def _normPyStr(self, valu):
try:
valu = float(valu)
except ValueError as e:
raise s_exc.BadTypeValu(name=self.name, valu=valu,
mesg=str(e)) from None
return self._normPyFloat(valu)
def _normFloat(self, valu):
if valu > 90.0 or valu < -90.0:
raise s_exc.BadTypeValu(valu=valu, name=self.name,
mesg='Latitude may only be -90.0 to 90.0')
valu = int(valu * Latitude.SCALE) / Latitude.SCALE
return valu, {}
def _normPyStr(self, valu):
valu = s_chop.hexstr(valu)
if self._size and len(valu) != self._size:
raise s_exc.BadTypeValu(valu=valu,
reqwidth=self._size,
name=self.name,
mesg='invalid width')
return valu, {}
def _normPyTuple(self, valu):
if len(valu) != 2:
mesg = 'edge requires (ndef, ndef)'
raise s_exc.BadTypeValu(mesg=mesg, name=self.name, valu=valu)
n1, n2 = valu
return self._initEdgeBase(n1, n2)
def getTickTock(self, vals):
'''
Get a tick, tock time pair.
Args:
vals (list): A pair of values to norm.
Returns:
(int, int): A ordered pair of integers.
'''
if len(vals) != 2:
mesg = 'Time range must have a length of 2: %r' % (vals,)
raise s_exc.BadTypeValu(mesg=mesg)
val0, val1 = vals
try:
_tick = self._getLiftValu(val0)
except ValueError:
mesg = 'Unable to process the value for val0 in _getLiftValu.'
raise s_exc.BadTypeValu(name=self.name, valu=val0,
mesg=mesg) from None
sortval = False
if isinstance(val1, str):
if val1.startswith(('+-', '-+')):
sortval = True
delt = s_time.delta(val1[2:])
# order matters
_tock = _tick + delt
_tick = _tick - delt
elif val1.startswith('-'):
sortval = True
_tock = self._getLiftValu(val1, relto=_tick)
else:
_tock = self._getLiftValu(val1, relto=_tick)
else:
_tock = self._getLiftValu(val1, relto=_tick)
if sortval and _tick >= _tock:
tick = min(_tick, _tock)
tock = max(_tick, _tock)
return tick, tock
return _tick, _tock
def _storLiftRange(self, cmpr, valu):
if type(valu) not in (list, tuple):
mesg = f'Range value must be a list: {valu!r}'
raise s_exc.BadTypeValu(mesg=mesg)
ticktock = self.getTickTock(valu)
return ((cmpr, ticktock, self.stortype), )
def _normPyIter(self, valu):
(minv, maxv), info = self._normByTickTock(valu)
# Norm via iter must produce an actual range.
if minv >= maxv:
raise s_exc.BadTypeValu(name=self.name, valu=valu,
mesg='Ival range must in (min, max) format')
return (minv, maxv), info
def _normPyInt(self, valu):
imsi = str(valu)
ilen = len(imsi)
if ilen > 15:
raise s_exc.BadTypeValu(valu=valu, name=self.name,
mesg='invalid imsi len: %d' % (ilen,))
mcc = imsi[0:3]
# TODO full imsi analysis tree
return valu, {'subs': {'mcc': mcc}}
def _normPyInt(self, valu):
imei = str(valu)
ilen = len(imei)
# we are missing our optional check digit
# lets add it for consistency...
if ilen == 14:
imei += imeicsum(imei)
return chop_imei(imei)
# if we *have* our check digit, lets check it
elif ilen == 15:
if imeicsum(imei) != imei[-1]:
raise s_exc.BadTypeValu(valu=valu, name=self.name,
mesg='invalid imei checksum byte')
return chop_imei(imei)
raise s_exc.BadTypeValu(valu=valu, name=self.name,
mesg='Failed to norm IMEI')
def _normByTickTock(self, valu):
if len(valu) != 2:
raise s_exc.BadTypeValu(name=self.name, valu=valu,
mesg='Ival _normPyIter requires 2 items')
tick, tock = self.timetype.getTickTock(valu)
minv, _ = self.timetype._normPyInt(tick)
maxv, _ = self.timetype._normPyInt(tock)
return (minv, maxv), {}
def _normPyStr(self, valu):
valu = valu.strip().lower()
if valu == '?':
raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg='interval requires begin time')
minv, _ = self.timetype.norm(valu)
# Norm is guaranteed to be a valid time value, but norm +1 may not be
maxv, info = self.timetype._normPyInt(minv + 1)
return (minv, maxv), info
def _normPyStr(self, text):
valu, off = s_grammar.parse_float(text, 0)
unit, off = s_grammar.nom(text, off, s_grammar.alphaset)
mult = units.get(unit.lower())
if mult is None:
raise s_exc.BadTypeValu(valu=text, name=self.name,
mesg='invalid/unknown dist unit: %s' % (unit,))
return int(valu * mult), {}
def _normPyStr(self, valu):
ival = self.enumnorm.get(valu)
if ival is not None:
return self._normPyInt(ival)
try:
valu = int(valu, 0)
except ValueError as e:
raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg=str(e))
return self._normPyInt(valu)
def _normPyStr(self, valu):
valu = valu.strip().lower()
if valu == '?':
raise s_exc.BadTypeValu(name=self.name,
valu=valu,
mesg='interval requires begin time')
norm, _ = self.timetype.norm(valu)
return (norm, norm + 1), {}
def _normPyStr(self, valu):
norm = valu.strip().lower().replace('\\', '/')
if norm.find('/') != -1:
mesg = 'file:base may not contain /'
raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg=mesg)
subs = {}
if norm.find('.') != -1:
subs['ext'] = norm.rsplit('.', 1)[1]
return norm, {'subs': subs}
