def _normPyTuple(self, valu): if len(valu) != 2: raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg=f'Must be a 2-tuple of type {self.subtype.name}') minv = self.subtype.norm(valu[0])[0] maxv = self.subtype.norm(valu[1])[0] if minv > maxv: raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg='minval cannot be greater than maxval') return (minv, maxv), {'subs': {'min': minv, 'max': maxv}}
def _normPyTuple(self, valu): if len(valu) != 2: raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg='Valu must contain valid latitude,longitude') try: latv = self.modl.type('geo:latitude').norm(valu[0])[0] lonv = self.modl.type('geo:longitude').norm(valu[1])[0] except Exception as e: raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg=str(e)) from None return (latv, lonv), {'subs': {'lat': latv, 'lon': lonv}}
def _normPyInt(self, valu): if self.minval is not None and valu < self.minval: mesg = f'value is below min={self.minval}' raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg=mesg) if self.maxval is not None and valu > self.maxval: mesg = f'value is above max={self.maxval}' raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg=mesg) if self.enumrepr and valu not in self.enumrepr: mesg = 'Value is not a valid enum value.' raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg=mesg) return valu, {}
async def nodesByPropArray(self, full, cmpr, valu): prop = self.core.model.prop(full) if prop is None: mesg = f'No property named "{full}".' raise s_exc.NoSuchProp(mesg=mesg) if not isinstance(prop.type, s_types.Array): mesg = f'Array synax is invalid on non array type: {prop.type.name}.' raise s_exc.BadTypeValu(mesg=mesg) cmprvals = prop.type.arraytype.getStorCmprs(cmpr, valu) if prop.isform: for layr in self.layers: genr = layr.liftByPropArray(prop.name, None, cmprvals) async for node in self._joinStorGenr(layr, genr): if node.bylayer['ndef'] != layr: continue yield node return formname = None if prop.form is not None: formname = prop.form.name for layr in self.layers: genr = layr.liftByPropArray(formname, prop.name, cmprvals) async for node in self._joinStorGenr(layr, genr): if node.bylayer['props'].get(prop.name) != layr: continue yield node
def norm(self, valu): try: s_common.reqjsonsafe(valu) except s_exc.MustBeJsonSafe as e: raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg=str(e)) from None byts = s_msgpack.en(valu) return s_msgpack.un(byts), {}
async def nodesByPropArray(self, full, cmpr, valu): prop = self.core.model.prop(full) if prop is None: mesg = f'No property named "{full}".' raise s_exc.NoSuchProp(mesg=mesg) if not isinstance(prop.type, s_types.Array): mesg = f'Array synax is invalid on non array type: {prop.type.name}.' raise s_exc.BadTypeValu(mesg=mesg) cmprvals = prop.type.arraytype.getStorCmprs(cmpr, valu) if prop.isform: async for (buid, sodes) in self.core._liftByPropArray( prop.name, None, cmprvals, self.layers): node = await self._joinSodes(buid, sodes) if node is not None: yield node return formname = None if prop.form is not None: formname = prop.form.name async for (buid, sodes) in self.core._liftByPropArray( formname, prop.name, cmprvals, self.layers): node = await self._joinSodes(buid, sodes) if node is not None: yield node
def bruteVersionStr(self, valu): ''' Brute force the version out of a string. Args: valu (str): String to attempt to get version information for. Notes: This first attempts to parse strings using the it:semver normalization before attempting to extract version parts out of the string. Returns: int, dict: The system normalized version integer and a subs dictionary. ''' try: valu, info = self.core.model.type('it:semver').norm(valu) subs = info.get('subs') return valu, subs except s_exc.BadTypeValu: # Try doing version part extraction by noming through the string subs = s_version.parseVersionParts(valu) if subs is None: raise s_exc.BadTypeValu(valu=valu, name='bruteVersionStr', mesg='Unable to brute force version parts out of the string') if subs: valu = s_version.packVersion(subs.get('major'), subs.get('minor', 0), subs.get('patch', 0)) return valu, subs
def _normPyStr(self, text): try: valu, off = s_grammar.parse_float(text, 0) except Exception: raise s_exc.BadTypeValu(valu=text, name=self.name, mesg='Dist requires a valid float and dist ' 'unit, no valid float found') from None unit, off = s_grammar.nom(text, off, s_grammar.alphaset) mult = units.get(unit.lower()) if mult is None: raise s_exc.BadTypeValu(valu=text, name=self.name, mesg='invalid/unknown dist unit: %s' % (unit,)) return int(valu * mult), {}
def _normPyStr(self, valu): digs = digits(valu) if not digs: raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg='requires a digit string') subs = {} try: info = s_l_phone.getPhoneInfo(int(digs)) except Exception as e: # pragma: no cover raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg='Failed to get phone info') cc = info.get('cc') if cc is not None: subs['loc'] = cc # TODO prefix based validation? return digs, {'subs': subs}
def _normPyStr(self, valu): digs = digits(valu) if not digs: raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg='requires a digit string') return self._normPyInt(int(digs))
def _normPyTuple(self, valu): fields = self.opts.get('fields') if len(fields) != len(valu): raise s_exc.BadTypeValu( name=self.name, valu=valu, mesg='invalid number of fields given for norming') subs = {} adds = [] norms = [] for i, (name, typename) in enumerate(fields): _type = self.tcache[name] norm, info = _type.norm(valu[i]) subs[name] = norm norms.append(norm) for k, v in info.get('subs', {}).items(): subs[f'{name}:{k}'] = v adds.extend(info.get('adds', ())) norm = tuple(norms) return norm, {'subs': subs, 'adds': adds}
def parsetz(text): ''' Parse timezone from time string, with UTC as the default. Args: text (str): Time string Returns: tuple: A tuple of text with tz chars removed and base milliseconds to offset time. ''' tz_hm = tz_hm_re.search(text) if tz_hm is not None: tzstr, rel, hrs, mins, _, _ = tz_hm.groups() rel = 1 if rel == '-' else -1 base = rel * (onehour * int(hrs) + onemin * int(mins)) if abs(base) >= oneday: raise s_exc.BadTypeValu( valu=text, name='time', mesg=f'Timezone offset must be between +/- 24 hours') return text.replace(tzstr, '', 1), base return text, 0
def delta(text): ''' Parse a simple time delta string and return the delta. ''' text = text.strip().lower() _, offs = _noms(text, 0, ' \t\r\n') sign = '+' if text and text[0] in ('+', '-'): sign = text[0] offs += 1 _, offs = _noms(text, offs, ' \t\r\n') sizetext, offs = _noms(text, offs, '0123456789') _, offs = _noms(text, offs, ' \t\r\n') unittext = text[offs:] size = int(sizetext, 0) if sign == '-': size = -size base = timeunits.get(unittext) if base is None: mesg = f'unknown time delta units: {unittext}' raise s_exc.BadTypeValu(name='time', valu=text, mesg=mesg) return size * base
def wildrange(text): ''' Parse an interval from a wild card time stamp: 2021/10/31* ''' dttick, base, tlen = _rawparse(text) if tlen not in (4, 6, 8, 10, 12, 14): mesg = 'Time wild card position not supported.' raise s_exc.BadTypeValu(mesg=mesg) if tlen == 4: dttock = dttick + relativedelta(years=1) elif tlen == 6: dttock = dttick + relativedelta(months=1) elif tlen == 8: dttock = dttick + relativedelta(days=1) elif tlen == 10: dttock = dttick + relativedelta(hours=1) elif tlen == 12: dttock = dttick + relativedelta(minutes=1) else: # tlen = 14 dttock = dttick + relativedelta(seconds=1) tick = int((dttick - EPOCH).total_seconds() * 1000 + base) tock = int((dttock - EPOCH).total_seconds() * 1000 + base) return (tick, tock)
def _normIntStr(self, valu): try: valu = float(valu) except Exception: raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg='Invalid float format') return self._normFloat(valu)
def _normPyStr(self, valu): try: valu = float(valu) except ValueError as e: raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg=str(e)) from None return self._normPyFloat(valu)
def _normFloat(self, valu): if valu > 90.0 or valu < -90.0: raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg='Latitude may only be -90.0 to 90.0') valu = int(valu * Latitude.SCALE) / Latitude.SCALE return valu, {}
def _normPyStr(self, valu): valu = s_chop.hexstr(valu) if self._size and len(valu) != self._size: raise s_exc.BadTypeValu(valu=valu, reqwidth=self._size, name=self.name, mesg='invalid width') return valu, {}
def _normPyTuple(self, valu): if len(valu) != 2: mesg = 'edge requires (ndef, ndef)' raise s_exc.BadTypeValu(mesg=mesg, name=self.name, valu=valu) n1, n2 = valu return self._initEdgeBase(n1, n2)
def getTickTock(self, vals): ''' Get a tick, tock time pair. Args: vals (list): A pair of values to norm. Returns: (int, int): A ordered pair of integers. ''' if len(vals) != 2: mesg = 'Time range must have a length of 2: %r' % (vals,) raise s_exc.BadTypeValu(mesg=mesg) val0, val1 = vals try: _tick = self._getLiftValu(val0) except ValueError: mesg = 'Unable to process the value for val0 in _getLiftValu.' raise s_exc.BadTypeValu(name=self.name, valu=val0, mesg=mesg) from None sortval = False if isinstance(val1, str): if val1.startswith(('+-', '-+')): sortval = True delt = s_time.delta(val1[2:]) # order matters _tock = _tick + delt _tick = _tick - delt elif val1.startswith('-'): sortval = True _tock = self._getLiftValu(val1, relto=_tick) else: _tock = self._getLiftValu(val1, relto=_tick) else: _tock = self._getLiftValu(val1, relto=_tick) if sortval and _tick >= _tock: tick = min(_tick, _tock) tock = max(_tick, _tock) return tick, tock return _tick, _tock
def _storLiftRange(self, cmpr, valu): if type(valu) not in (list, tuple): mesg = f'Range value must be a list: {valu!r}' raise s_exc.BadTypeValu(mesg=mesg) ticktock = self.getTickTock(valu) return ((cmpr, ticktock, self.stortype), )
def _normPyIter(self, valu): (minv, maxv), info = self._normByTickTock(valu) # Norm via iter must produce an actual range. if minv >= maxv: raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg='Ival range must in (min, max) format') return (minv, maxv), info
def _normPyInt(self, valu): imsi = str(valu) ilen = len(imsi) if ilen > 15: raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg='invalid imsi len: %d' % (ilen,)) mcc = imsi[0:3] # TODO full imsi analysis tree return valu, {'subs': {'mcc': mcc}}
def _normPyInt(self, valu): imei = str(valu) ilen = len(imei) # we are missing our optional check digit # lets add it for consistency... if ilen == 14: imei += imeicsum(imei) return chop_imei(imei) # if we *have* our check digit, lets check it elif ilen == 15: if imeicsum(imei) != imei[-1]: raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg='invalid imei checksum byte') return chop_imei(imei) raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg='Failed to norm IMEI')
def _normByTickTock(self, valu): if len(valu) != 2: raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg='Ival _normPyIter requires 2 items') tick, tock = self.timetype.getTickTock(valu) minv, _ = self.timetype._normPyInt(tick) maxv, _ = self.timetype._normPyInt(tock) return (minv, maxv), {}
def _normPyStr(self, valu): valu = valu.strip().lower() if valu == '?': raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg='interval requires begin time') minv, _ = self.timetype.norm(valu) # Norm is guaranteed to be a valid time value, but norm +1 may not be maxv, info = self.timetype._normPyInt(minv + 1) return (minv, maxv), info
def _normPyStr(self, text): valu, off = s_grammar.parse_float(text, 0) unit, off = s_grammar.nom(text, off, s_grammar.alphaset) mult = units.get(unit.lower()) if mult is None: raise s_exc.BadTypeValu(valu=text, name=self.name, mesg='invalid/unknown dist unit: %s' % (unit,)) return int(valu * mult), {}
def _normPyStr(self, valu): ival = self.enumnorm.get(valu) if ival is not None: return self._normPyInt(ival) try: valu = int(valu, 0) except ValueError as e: raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg=str(e)) return self._normPyInt(valu)
def _normPyStr(self, valu): valu = valu.strip().lower() if valu == '?': raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg='interval requires begin time') norm, _ = self.timetype.norm(valu) return (norm, norm + 1), {}
def _normPyStr(self, valu): norm = valu.strip().lower().replace('\\', '/') if norm.find('/') != -1: mesg = 'file:base may not contain /' raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg=mesg) subs = {} if norm.find('.') != -1: subs['ext'] = norm.rsplit('.', 1)[1] return norm, {'subs': subs}