def gen(self, count, earliest, latest, samplename=None): self.logger.debug("Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'" % (self._sample.name, self._sample.app, count, earliest, latest)) startTime = datetime.datetime.now() # If we're random, fill random events from sampleDict into eventsDict if self._sample.randomizeEvents: eventsDict = [] sdlen = len(self._sample.sampleDict) self.logger.debugv("Random filling eventsDict for sample '%s' in app '%s' with %d events" % (self._sample.name, self._sample.app, count)) # Count is -1, replay the whole file, but in randomizeEvents I think we'd want it to actually # just put as many events as there are in the file if count == -1: count = sdlen while len(eventsDict) < count: eventsDict.append(self._sample.sampleDict[random.randint(0, sdlen - 1)]) # If we're bundlelines, create count copies of the sampleDict elif self._sample.bundlelines: eventsDict = [] self.logger.debugv( "Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict" % (self._sample.name, self._sample.app, count)) for x in xrange(count): eventsDict.extend(self._sample.sampleDict) # Otherwise fill count events into eventsDict or keep making copies of events out of sampleDict until # eventsDict is as big as count else: # If count is -1, play the whole file, else grab a subset if count == -1: count = len(self._sample.sampleDict) eventsDict = self._sample.sampleDict[0:count] # Continue to fill events array until len(events) == count if len(eventsDict) < count: self.logger.debugv( "Events fill for sample '%s' in app '%s' less than count (%s vs. %s); continuing fill" % (self._sample.name, self._sample.app, len(eventsDict), count)) self.logger.debugv("Current eventsDict: %s" % eventsDict) # run a modulus on the size of the eventdict to figure out what the last event was. Populate to count # from there. while len(eventsDict) < count: if len(self._sample.sampleDict): nextEventToUse = self._sample.sampleDict[len(eventsDict) % len(self._sample.sampleDict)] self.logger.debugv("Next event to add: %s" % nextEventToUse) eventsDict.append(nextEventToUse) self.logger.debugv("Events fill complete for sample '%s' in app '%s' length %d" % (self._sample.name, self._sample.app, len(eventsDict))) GeneratorPlugin.build_events(self, eventsDict, startTime, earliest, latest)
def gen(self, count, earliest, latest, samplename=None): logger.debug("Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'" % (self._sample.name, self._sample.app, count, earliest, latest)) startTime = datetime.datetime.now() # If we're random, fill random events from sampleDict into eventsDict if self._sample.randomizeEvents: eventsDict = [] sdlen = len(self._sample.sampleDict) logger.debug("Random filling eventsDict for sample '%s' in app '%s' with %d events" % (self._sample.name, self._sample.app, count)) # Count is -1, replay the whole file, but in randomizeEvents I think we'd want it to actually # just put as many events as there are in the file if count == -1: count = sdlen while len(eventsDict) < count: eventsDict.append(self._sample.sampleDict[random.randint(0, sdlen - 1)]) # If we're bundlelines, create count copies of the sampleDict elif self._sample.bundlelines: eventsDict = [] logger.debug( "Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict" % (self._sample.name, self._sample.app, count)) for x in xrange(count): eventsDict.extend(self._sample.sampleDict) # Otherwise fill count events into eventsDict or keep making copies of events out of sampleDict until # eventsDict is as big as count else: # If count is -1, play the whole file, else grab a subset if count == -1: count = len(self._sample.sampleDict) eventsDict = self._sample.sampleDict[0:count] # Continue to fill events array until len(events) == count if len(eventsDict) < count: logger.debug( "Events fill for sample '%s' in app '%s' less than count (%s vs. %s); continuing fill" % (self._sample.name, self._sample.app, len(eventsDict), count)) logger.debug("Current eventsDict: %s" % eventsDict) # run a modulus on the size of the eventdict to figure out what the last event was. Populate to count # from there. while len(eventsDict) < count: if len(self._sample.sampleDict): nextEventToUse = self._sample.sampleDict[len(eventsDict) % len(self._sample.sampleDict)] logger.debug("Next event to add: %s" % nextEventToUse) eventsDict.append(nextEventToUse) logger.debug("Events fill complete for sample '%s' in app '%s' length %d" % (self._sample.name, self._sample.app, len(eventsDict))) GeneratorPlugin.build_events(self, eventsDict, startTime, earliest, latest)
def gen(self, count, earliest, latest, samplename=None): # count in this plugin is a measurement of byteself._sample. size = count self.logger.debug( "PerDayVolumeGenerator Called with a Size of: %s with Earliest: %s and Latest: %s" % (size, earliest, latest)) # very similar to the default generator. only difference is we go by size instead of count. try: self._sample.loadSample() self.logger.debug("File sample loaded successfully.") except TypeError: self.logger.error("Error loading sample file for sample '%s'" % self._sample.name) return self.logger.debug( "Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'" % (self._sample.name, self._sample.app, size, earliest, latest)) startTime = datetime.datetime.now() # Create a counter for the current byte size of the read in samples currentSize = 0 # If we're random, fill random events from sampleDict into eventsDict eventsDict = [] if self._sample.randomizeEvents: sdlen = len(self._sample.sampleDict) self.logger.debugv( "Random filling eventsDict for sample '%s' in app '%s' with %d bytes" % (self._sample.name, self._sample.app, size)) while currentSize < size: currentevent = self._sample.sampleDict[random.randint( 0, sdlen - 1)] eventsDict.append(currentevent) currentSize += len(currentevent['_raw']) # If we're bundlelines, create count copies of the sampleDict elif self._sample.bundlelines: self.logger.debugv( "Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict" % (self._sample.name, self._sample.app, size)) while currentSize <= size: sizeofsample = sum( len(sample['_raw']) for sample in self._sample.sampleDict) eventsDict.extend(self._sample.sampleDict) currentSize += sizeofsample # Otherwise fill count events into eventsDict or keep making copies of events out of sampleDict until # eventsDict is as big as count else: self.logger.debug("Simple replay in order, processing") # I need to check the sample and load events in order until the size is smaller than read events from file # or i've read the entire file. linecount = 0 currentreadsize = 0 linesinfile = len(self._sample.sampleDict) self.logger.debugv("Lines in files: %s " % linesinfile) while currentreadsize <= size: targetline = linecount % linesinfile sizeremaining = size - currentreadsize targetlinesize = len( self._sample.sampleDict[targetline]['_raw']) if targetlinesize <= sizeremaining or targetlinesize * .9 <= sizeremaining: currentreadsize += targetlinesize eventsDict.append(self._sample.sampleDict[targetline]) else: break linecount += 1 self.logger.debugv( "Events fill complete for sample '%s' in app '%s' length %d" % (self._sample.name, self._sample.app, len(eventsDict))) GeneratorPlugin.build_events(self, eventsDict, startTime, earliest, latest)