def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
self._currentevent = 0
self._timeSinceSleep = datetime.timedelta()
self._times = []
self.replayLock = None
def gen(self, count, earliest, latest, samplename=None):
logger.debug(
"Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'"
% (self._sample.name, self._sample.app, count, earliest, latest))
startTime = datetime.datetime.now()
# If we're random, fill random events from sampleDict into eventsDict
if self._sample.randomizeEvents:
eventsDict = []
sdlen = len(self._sample.sampleDict)
logger.debug(
"Random filling eventsDict for sample '%s' in app '%s' with %d events"
% (self._sample.name, self._sample.app, count))
# Count is -1, replay the whole file, but in randomizeEvents I think we'd want it to actually
# just put as many events as there are in the file
if count == -1:
count = sdlen
while len(eventsDict) < count:
eventsDict.append(self._sample.sampleDict[random.randint(
0, sdlen - 1)])
# If we're bundlelines, create count copies of the sampleDict
elif self._sample.bundlelines:
eventsDict = []
logger.debug(
"Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict"
% (self._sample.name, self._sample.app, count))
for x in range(count):
eventsDict.extend(self._sample.sampleDict)
# Otherwise fill count events into eventsDict or keep making copies of events out of sampleDict until
# eventsDict is as big as count
else:
# If count is -1, play the whole file, else grab a subset
if count == -1:
count = len(self._sample.sampleDict)
eventsDict = self._sample.sampleDict[0:count]
# Continue to fill events array until len(events) == count
if len(eventsDict) < count:
logger.debug(
"Events fill for sample '%s' in app '%s' less than count (%s vs. %s); continuing fill"
% (self._sample.name, self._sample.app, len(eventsDict),
count))
logger.debug("Current eventsDict: %s" % eventsDict)
# run a modulus on the size of the eventdict to figure out what the last event was. Populate to count
# from there.
while len(eventsDict) < count:
if len(self._sample.sampleDict):
nextEventToUse = self._sample.sampleDict[
len(eventsDict) % len(self._sample.sampleDict)]
logger.debug("Next event to add: %s" % nextEventToUse)
eventsDict.append(nextEventToUse)
logger.debug(
"Events fill complete for sample '%s' in app '%s' length %d"
% (self._sample.name, self._sample.app, len(eventsDict)))
GeneratorPlugin.build_events(self, eventsDict, startTime, earliest,
latest)
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
self.current_count = 0
self.target_count = 0
self.earliest = None
self.latest = None
self.jinja_count_type = "cycles"
self.end_of_cycle = False
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
self.start_count = 0.0
self.end_count = 0.0
self.count_by = 1.0
self.count_template = (
"{event_ts}-0700 Counter for sample:{samplename}, " +
"Now processing event counting {loop_count} of {max_loop} cycles. Counter Values:"
+ " Start_Count: {start_count} Current_Counter:{current_count}" +
" End_Count:{end_count} Counting_By: {count_by}")
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
f = open('tests/sample_eventgen_conf/perf/weblog/external_ips.sample')
self.external_ips = [x.strip() for x in f.readlines()]
self.external_ips_len = len(self.external_ips)
f.close()
f = open('tests/sample_eventgen_conf/perf/weblog/webhosts.sample')
self.webhosts = [x.strip() for x in f.readlines()]
f.close()
self.webhosts_len = len(self.webhosts)
f = open('tests/sample_eventgen_conf/perf/weblog/useragents.sample')
self.useragents = [x.strip() for x in f.readlines()]
f.close()
self.useragents_len = len(self.useragents)
f = open('tests/sample_eventgen_conf/perf/weblog/webserverstatus.sample')
self.webserverstatus = [x.strip() for x in f.readlines()]
f.close()
self.webserverstatus_len = len(self.webserverstatus)
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
def gen(self, count, earliest, latest, samplename=None):
# count in this plugin is a measurement of byteself._sample.
size = count
logger.debug(
"PerDayVolumeGenerator Called with a Size of: %s with Earliest: %s and Latest: %s"
% (size, earliest, latest))
# very similar to the default generator. only difference is we go by size instead of count.
try:
self._sample.loadSample()
logger.debug("File sample loaded successfully.")
except TypeError:
logger.error("Error loading sample file for sample '%s'" %
self._sample.name)
return
logger.debug(
"Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'"
% (self._sample.name, self._sample.app, size, earliest, latest))
startTime = datetime.datetime.now()
# Create a counter for the current byte size of the read in samples
currentSize = 0
# If we're random, fill random events from sampleDict into eventsDict
eventsDict = []
if self._sample.randomizeEvents:
sdlen = len(self._sample.sampleDict)
logger.debug(
"Random filling eventsDict for sample '%s' in app '%s' with %d bytes"
% (self._sample.name, self._sample.app, size))
while currentSize < size:
currentevent = self._sample.sampleDict[random.randint(
0, sdlen - 1)]
eventsDict.append(currentevent)
currentSize += len(currentevent['_raw'])
# If we're bundlelines, create count copies of the sampleDict
elif self._sample.bundlelines:
logger.debug(
"Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict"
% (self._sample.name, self._sample.app, size))
while currentSize <= size:
sizeofsample = sum(
len(sample['_raw']) for sample in self._sample.sampleDict)
eventsDict.extend(self._sample.sampleDict)
currentSize += sizeofsample
# Otherwise fill count events into eventsDict or keep making copies of events out of sampleDict until
# eventsDict is as big as count
else:
logger.debug("Simple replay in order, processing")
# I need to check the sample and load events in order until the size is smaller than read events from file
# or i've read the entire file.
linecount = 0
currentreadsize = 0
linesinfile = len(self._sample.sampleDict)
logger.debug("Lines in files: %s " % linesinfile)
while currentreadsize <= size:
targetline = linecount % linesinfile
sizeremaining = size - currentreadsize
targetlinesize = len(
self._sample.sampleDict[targetline]['_raw'])
if size < targetlinesize:
logger.error(
"Size is too small for sample {}. We need {} bytes but size of one event is {} bytes."
.format(self._sample.name, size, targetlinesize))
break
if targetlinesize <= sizeremaining:
currentreadsize += targetlinesize
eventsDict.append(self._sample.sampleDict[targetline])
else:
break
linecount += 1
logger.debug(
"Events fill complete for sample '%s' in app '%s' length %d" %
(self._sample.name, self._sample.app, len(eventsDict)))
# build the events and replace tokens
GeneratorPlugin.build_events(self, eventsDict, startTime, earliest,
latest)
