def test_time_delta_2_second(delta, expect): ''' Test timeDelta2secs function, convert time delta object to seconds Normal cases: case 1: time delta is 1 day, expect is 86400 case 2: time delta is 1 day 3 hour 15 minutes 32 seconds, expect is 98132 case 3: time delta is less than 1 day, only 1 hour 10 minutes, expect is 4200 case 4: time delta is 1 hour ago, expect is -3600 Corner cases: case 1: delta object is None -- invalid input, expect is <TBD> ''' assert timeparser.timeDelta2secs(delta) == expect
def test_time_delta_2_second(delta, expect): ''' Test timeDelta2secs function, convert time delta object to seconds Normal cases: case 1: time delta is 1 day, expect is 86400 case 2: time delta is 1 day 3 hour 15 minutes 32 seconds, expect is 98132 case 3: time delta is less than 1 day, only 1 hour 10 minutes, expect is 4200 case 4: time delta is 1 hour ago, expect is -3600 Corner cases: case 1: delta object is None -- invalid input, expect is <TBD> ''' assert timeparser.timeDelta2secs(delta) == expect
def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, pivot_timestamp=None): if self.replacementType == 'static': return self.replacement # This logic is done in replay.py elif self.replacementType == 'replaytimestamp': pass elif self.replacementType == 'timestamp': if s.earliest and s.latest: if earliestTime and latestTime: if latestTime >= earliestTime: if pivot_timestamp: replacementTime = pivot_timestamp elif s.timestamp is None: minDelta = 0 # Compute timeDelta as total_seconds td = latestTime - earliestTime if not type(td) == float: maxDelta = timeDelta2secs(td) else: maxDelta = td # Get random timeDelta randomDelta = datetime.timedelta( seconds=random.randint(minDelta, maxDelta), microseconds=random.randint( 0, latestTime.microsecond if latestTime.microsecond > 0 else 999999)) # Compute replacmentTime replacementTime = latestTime - randomDelta s.timestamp = replacementTime else: replacementTime = s.timestamp replacement = self.replacement.replace( '%s', str(round(time.mktime( replacementTime.timetuple()))).rstrip( '0').rstrip('.')) replacementTime = replacementTime.strftime(replacement) # replacementTime == replacement for invalid strptime specifiers if replacementTime != self.replacement.replace( '%', ''): return replacementTime else: logger.error( "Invalid strptime specifier '%s' detected; will not replace" % (self.replacement)) return old # earliestTime/latestTime not proper else: logger.error(( "Earliest specifier '%s', value '%s' is greater than latest specifier '%s'" + "value '%s' for sample '%s'; will not replace") % (s.earliest, earliestTime, s.latest, latestTime, s.name)) return old # earliest/latest not proper else: logger.error( 'Earliest or latest specifier were not set; will not replace' ) return old elif self.replacementType in ('random', 'rated'): # Validations: if self._integerMatch is not None: integerMatch = self._integerMatch else: integerRE = re.compile(r'integer\[([-]?\d+):([-]?\d+)\]', re.I) integerMatch = integerRE.match(self.replacement) self._integerMatch = integerMatch if self._floatMatch is not None: floatMatch = self._floatMatch else: floatRE = re.compile( r'float\[(-?\d+|-?\d+\.(\d+)):(-?\d+|-?\d+\.(\d+))\]', re.I) floatMatch = floatRE.match(self.replacement) self._floatMatch = floatMatch if self._stringMatch is not None: stringMatch = self._stringMatch else: stringRE = re.compile(r'string\((\d+)\)', re.I) stringMatch = stringRE.match(self.replacement) self._stringMatch = stringMatch if self._hexMatch is not None: hexMatch = self._hexMatch else: hexRE = re.compile(r'hex\((\d+)\)', re.I) hexMatch = hexRE.match(self.replacement) self._hexMatch = hexMatch if self._listMatch is not None: listMatch = self._listMatch else: listRE = re.compile(r'list(\[[^\]]+\])', re.I) listMatch = listRE.match(self.replacement) self._listMatch = listMatch # Valid replacements: ipv4 | ipv6 | integer[<start>:<end>] | string(<i>) if self.replacement.lower() == 'ipv4': x = 0 replacement = '' while x < 4: replacement += str(random.randint(0, 255)) + '.' x += 1 replacement = replacement.strip('.') return replacement elif self.replacement.lower() == 'ipv6': x = 0 replacement = '' while x < 8: replacement += hex(random.randint(0, 65535))[2:] + ':' x += 1 replacement = replacement.strip(':') return replacement elif self.replacement.lower() == 'mac': x = 0 replacement = '' # Give me 6 blocks of 2 hex while x < 6: y = 0 while y < 2: replacement += hex(random.randint(0, 15))[2:] y += 1 replacement += ':' x += 1 replacement = replacement.strip(':') return replacement elif self.replacement.lower() == 'guid': return str(uuid.uuid4()) elif integerMatch: startInt = int(integerMatch.group(1)) endInt = int(integerMatch.group(2)) if endInt >= startInt: replacementInt = random.randint(startInt, endInt) if self.replacementType == 'rated': rateFactor = 1.0 if type(s.hourOfDayRate) == dict: try: rateFactor *= s.hourOfDayRate[str(s.now())] except KeyError: import traceback stack = traceback.format_exc() logger.error( "Hour of day rate failed for token %s. Stacktrace %s" % stack) if type(s.dayOfWeekRate) == dict: try: weekday = datetime.date.weekday(s.now()) if weekday == 6: weekday = 0 else: weekday += 1 rateFactor *= s.dayOfWeekRate[str(weekday)] except KeyError: import traceback stack = traceback.format_exc() logger.error( "Day of week rate failed. Stacktrace %s" % stack) replacementInt = int( round(replacementInt * rateFactor, 0)) replacement = str(replacementInt) return replacement else: logger.error( "Start integer %s greater than end integer %s; will not replace" % (startInt, endInt)) return old elif floatMatch: try: startFloat = float(floatMatch.group(1)) endFloat = float(floatMatch.group(3)) significance = 0 if floatMatch.group(2) is not None: significance = len(floatMatch.group(2)) if endFloat >= startFloat: floatret = round(random.uniform(startFloat, endFloat), significance) if self.replacementType == 'rated': rateFactor = 1.0 now = s.now() if type(s.hourOfDayRate) == dict: try: rateFactor *= s.hourOfDayRate[str( now.hour)] except KeyError: import traceback stack = traceback.format_exc() logger.error( "Hour of day rate failed for token %s. Stacktrace %s" % stack) if type(s.dayOfWeekRate) == dict: try: weekday = datetime.date.weekday(now) if weekday == 6: weekday = 0 else: weekday += 1 rateFactor *= s.dayOfWeekRate[str(weekday)] except KeyError: import traceback stack = traceback.format_exc() logger.error( "Day of week rate failed. Stacktrace %s" % stack) floatret = round(floatret * rateFactor, significance) floatret = str(floatret) return floatret else: logger.error( "Start float %s greater than end float %s; will not replace" % (startFloat, endFloat)) return old except ValueError: logger.error("Could not parse float[%s:%s]" % (floatMatch.group(1), floatMatch.group(4))) return old elif stringMatch: strLength = int(stringMatch.group(1)) if strLength == 0: return '' elif strLength > 0: replacement = '' while len(replacement) < strLength: # Generate a random ASCII between dec 33->126 replacement += chr(random.randint(33, 126)) # Practice safe strings replacement = re.sub('%[0-9a-fA-F]+', '', urllib.parse.quote(replacement)) return replacement else: logger.error( "Length specifier %s for string replacement must be greater than 0; will not replace" % (strLength)) return old elif hexMatch: strLength = int(hexMatch.group(1)) replacement = '' hexList = [ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' ] while len(replacement) < strLength: replacement += hexList[random.randint(0, 15)] return replacement elif listMatch: try: value = json.loads(listMatch.group(1)) except: logger.error( "Could not parse json for '%s' in sample '%s'" % (listMatch.group(1), s.name)) return old return random.SystemRandom().choice(value) else: logger.error( "Unknown replacement value '%s' for replacementType '%s'; will not replace" % (self.replacement, self.replacementType)) return old elif self.replacementType in ('file', 'mvfile', 'seqfile'): if self._replacementFile is not None: replacementFile = self._replacementFile replacementColumn = self._replacementColumn else: try: paths = self.replacement.split(':') if (len(paths) == 1): replacementColumn = 0 else: try: # When it's not a mvfile, there's no number on the end: replacementColumn = int(paths[-1]) except (ValueError): replacementColumn = 0 if (replacementColumn > 0): # This supports having a drive-letter colon replacementFile = s.pathParser(":".join(paths[0:-1])) else: replacementFile = s.pathParser(self.replacement) except ValueError: logger.error( "Replacement string '%s' improperly formatted. Should be /path/to/file or /path/to/file:column" % self.replacement) return old self._replacementFile = replacementFile self._replacementColumn = replacementColumn # If we've seen this file before, simply return already read results # This applies only if we're looking at a multivalue file and we want to # return the same random pick on every iteration if replacementColumn > 0 and replacementFile in self.mvhash: if replacementColumn > len(self.mvhash[replacementFile]): logger.error( "Index for column '%s' in replacement file '%s' is out of bounds" % (replacementColumn, replacementFile)) return old else: # logger.debug("Returning mvhash: %s" % self.mvhash[replacementFile][replacementColumn-1]) return self.mvhash[replacementFile][replacementColumn - 1] else: # Adding caching of the token file to avoid reading it every iteration if self._tokenfile is not None: replacementLines = self._tokenfile # Otherwise, lets read the file and build our cached results, pick a result and return it else: # logger.debug("replacementFile: %s replacementColumn: %s" % # (replacementFile, replacementColumn)) replacementFile = os.path.abspath(replacementFile) logger.debug("Normalized replacement file %s" % replacementFile) if os.path.exists(replacementFile) and os.path.isfile( replacementFile): replacementFH = open(replacementFile, 'rU') replacementLines = replacementFH.readlines() replacementFH.close() if len(replacementLines) == 0: logger.error( "Replacement file '%s' is empty; will not replace" % (replacementFile)) return old else: self._tokenfile = replacementLines else: logger.error("File '%s' does not exist" % (replacementFile)) return old if self.replacementType == 'seqfile': # pick value one by one from replacement file replacement = replacementLines[ self._tokenfilecounter % len(replacementLines)].strip() self._tokenfilecounter += 1 else: # pick value randomly from replacement file replacement = replacementLines[random.randint( 0, len(replacementLines) - 1)].strip() if replacementColumn > 0: self.mvhash[replacementFile] = replacement.split(',') if replacementColumn > len(self.mvhash[replacementFile]): logger.error( "Index for column '%s' in replacement file '%s' is out of bounds" % (replacementColumn, replacementFile)) return old else: return self.mvhash[replacementFile][replacementColumn - 1] else: return replacement elif self.replacementType == 'integerid': temp = self.replacement self.replacement = str(int(self.replacement) + 1) return temp else: logger.error("Unknown replacementType '%s'; will not replace" % self.replacementType) return old