def gen(self, count, earliest, latest, samplename=None):
self.logger.debug("Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'" %
(self._sample.name, self._sample.app, count, earliest, latest))
startTime = datetime.datetime.now()
# If we're random, fill random events from sampleDict into eventsDict
if self._sample.randomizeEvents:
eventsDict = []
sdlen = len(self._sample.sampleDict)
self.logger.debugv("Random filling eventsDict for sample '%s' in app '%s' with %d events" %
(self._sample.name, self._sample.app, count))
# Count is -1, replay the whole file, but in randomizeEvents I think we'd want it to actually
# just put as many events as there are in the file
if count == -1:
count = sdlen
while len(eventsDict) < count:
eventsDict.append(self._sample.sampleDict[random.randint(0, sdlen - 1)])
# If we're bundlelines, create count copies of the sampleDict
elif self._sample.bundlelines:
eventsDict = []
self.logger.debugv(
"Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict" %
(self._sample.name, self._sample.app, count))
for x in xrange(count):
eventsDict.extend(self._sample.sampleDict)
# Otherwise fill count events into eventsDict or keep making copies of events out of sampleDict until
# eventsDict is as big as count
else:
# If count is -1, play the whole file, else grab a subset
if count == -1:
count = len(self._sample.sampleDict)
eventsDict = self._sample.sampleDict[0:count]
# Continue to fill events array until len(events) == count
if len(eventsDict) < count:
self.logger.debugv(
"Events fill for sample '%s' in app '%s' less than count (%s vs. %s); continuing fill" %
(self._sample.name, self._sample.app, len(eventsDict), count))
self.logger.debugv("Current eventsDict: %s" % eventsDict)
# run a modulus on the size of the eventdict to figure out what the last event was. Populate to count
# from there.
while len(eventsDict) < count:
if len(self._sample.sampleDict):
nextEventToUse = self._sample.sampleDict[len(eventsDict) % len(self._sample.sampleDict)]
self.logger.debugv("Next event to add: %s" % nextEventToUse)
eventsDict.append(nextEventToUse)
self.logger.debugv("Events fill complete for sample '%s' in app '%s' length %d" %
(self._sample.name, self._sample.app, len(eventsDict)))
GeneratorPlugin.build_events(self, eventsDict, startTime, earliest, latest)
def gen(self, count, earliest, latest, samplename=None):
logger.debug("Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'" %
(self._sample.name, self._sample.app, count, earliest, latest))
startTime = datetime.datetime.now()
# If we're random, fill random events from sampleDict into eventsDict
if self._sample.randomizeEvents:
eventsDict = []
sdlen = len(self._sample.sampleDict)
logger.debug("Random filling eventsDict for sample '%s' in app '%s' with %d events" %
(self._sample.name, self._sample.app, count))
# Count is -1, replay the whole file, but in randomizeEvents I think we'd want it to actually
# just put as many events as there are in the file
if count == -1:
count = sdlen
while len(eventsDict) < count:
eventsDict.append(self._sample.sampleDict[random.randint(0, sdlen - 1)])
# If we're bundlelines, create count copies of the sampleDict
elif self._sample.bundlelines:
eventsDict = []
logger.debug(
"Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict" %
(self._sample.name, self._sample.app, count))
for x in xrange(count):
eventsDict.extend(self._sample.sampleDict)
# Otherwise fill count events into eventsDict or keep making copies of events out of sampleDict until
# eventsDict is as big as count
else:
# If count is -1, play the whole file, else grab a subset
if count == -1:
count = len(self._sample.sampleDict)
eventsDict = self._sample.sampleDict[0:count]
# Continue to fill events array until len(events) == count
if len(eventsDict) < count:
logger.debug(
"Events fill for sample '%s' in app '%s' less than count (%s vs. %s); continuing fill" %
(self._sample.name, self._sample.app, len(eventsDict), count))
logger.debug("Current eventsDict: %s" % eventsDict)
# run a modulus on the size of the eventdict to figure out what the last event was. Populate to count
# from there.
while len(eventsDict) < count:
if len(self._sample.sampleDict):
nextEventToUse = self._sample.sampleDict[len(eventsDict) % len(self._sample.sampleDict)]
logger.debug("Next event to add: %s" % nextEventToUse)
eventsDict.append(nextEventToUse)
logger.debug("Events fill complete for sample '%s' in app '%s' length %d" %
(self._sample.name, self._sample.app, len(eventsDict)))
GeneratorPlugin.build_events(self, eventsDict, startTime, earliest, latest)
def gen(self, count, earliest, latest, samplename=None):
# count in this plugin is a measurement of byteself._sample.
size = count
self.logger.debug(
"PerDayVolumeGenerator Called with a Size of: %s with Earliest: %s and Latest: %s"
% (size, earliest, latest))
# very similar to the default generator. only difference is we go by size instead of count.
try:
self._sample.loadSample()
self.logger.debug("File sample loaded successfully.")
except TypeError:
self.logger.error("Error loading sample file for sample '%s'" %
self._sample.name)
return
self.logger.debug(
"Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'"
% (self._sample.name, self._sample.app, size, earliest, latest))
startTime = datetime.datetime.now()
# Create a counter for the current byte size of the read in samples
currentSize = 0
# If we're random, fill random events from sampleDict into eventsDict
eventsDict = []
if self._sample.randomizeEvents:
sdlen = len(self._sample.sampleDict)
self.logger.debugv(
"Random filling eventsDict for sample '%s' in app '%s' with %d bytes"
% (self._sample.name, self._sample.app, size))
while currentSize < size:
currentevent = self._sample.sampleDict[random.randint(
0, sdlen - 1)]
eventsDict.append(currentevent)
currentSize += len(currentevent['_raw'])
# If we're bundlelines, create count copies of the sampleDict
elif self._sample.bundlelines:
self.logger.debugv(
"Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict"
% (self._sample.name, self._sample.app, size))
while currentSize <= size:
sizeofsample = sum(
len(sample['_raw']) for sample in self._sample.sampleDict)
eventsDict.extend(self._sample.sampleDict)
currentSize += sizeofsample
# Otherwise fill count events into eventsDict or keep making copies of events out of sampleDict until
# eventsDict is as big as count
else:
self.logger.debug("Simple replay in order, processing")
# I need to check the sample and load events in order until the size is smaller than read events from file
# or i've read the entire file.
linecount = 0
currentreadsize = 0
linesinfile = len(self._sample.sampleDict)
self.logger.debugv("Lines in files: %s " % linesinfile)
while currentreadsize <= size:
targetline = linecount % linesinfile
sizeremaining = size - currentreadsize
targetlinesize = len(
self._sample.sampleDict[targetline]['_raw'])
if targetlinesize <= sizeremaining or targetlinesize * .9 <= sizeremaining:
currentreadsize += targetlinesize
eventsDict.append(self._sample.sampleDict[targetline])
else:
break
linecount += 1
self.logger.debugv(
"Events fill complete for sample '%s' in app '%s' length %d" %
(self._sample.name, self._sample.app, len(eventsDict)))
GeneratorPlugin.build_events(self, eventsDict, startTime, earliest,
latest)
