def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
# Logger already setup by config, just get an instance
logger = logging.getLogger('eventgen')
from eventgenconfig import EventgenAdapter
adapter = EventgenAdapter(logger, {'module': 'WeblogGenerator', 'sample': sample.name})
globals()['logger'] = adapter
from eventgenconfig import Config
globals()['c'] = Config()
f = open('tests/perf/weblog/external_ips.sample')
self.external_ips = [x.strip() for x in f.readlines()]
self.external_ips_len = len(self.external_ips)
f.close()
f = open('tests/perf/weblog/webhosts.sample')
self.webhosts = [x.strip() for x in f.readlines()]
f.close()
self.webhosts_len = len(self.webhosts)
f = open('tests/perf/weblog/useragents.sample')
self.useragents = [x.strip() for x in f.readlines()]
f.close()
self.useragents_len = len(self.useragents)
f = open('tests/perf/weblog/webserverstatus.sample')
self.webserverstatus = [x.strip() for x in f.readlines()]
f.close()
self.webserverstatus_len = len(self.webserverstatus)
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
# Logger already setup by config, just get an instance
logger = logging.getLogger('eventgen')
from eventgenconfig import EventgenAdapter
adapter = EventgenAdapter(logger, {'module': 'WeblogGenerator', 'sample': sample.name})
globals()['logger'] = adapter
from eventgenconfig import Config
globals()['c'] = Config()
f = open('tests/perf/weblog/external_ips.sample')
self.external_ips = [x.strip() for x in f.readlines()]
self.external_ips_len = len(self.external_ips)
f.close()
f = open('tests/perf/weblog/webhosts.sample')
self.webhosts = [x.strip() for x in f.readlines()]
f.close()
self.webhosts_len = len(self.webhosts)
f = open('tests/perf/weblog/useragents.sample')
self.useragents = [x.strip() for x in f.readlines()]
f.close()
self.useragents_len = len(self.useragents)
f = open('tests/perf/weblog/webserverstatus.sample')
self.webserverstatus = [x.strip() for x in f.readlines()]
f.close()
self.webserverstatus_len = len(self.webserverstatus)
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
# Logger already setup by config, just get an instance
logger = logging.getLogger('eventgen')
globals()['logger'] = logger
from eventgenconfig import Config
globals()['c'] = Config()
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
# Logger already setup by config, just get an instance
logger = logging.getLogger('eventgen')
globals()['logger'] = logger
from eventgenconfig import Config
globals()['c'] = Config()
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
self.current_count = 0
self.target_count = 0
self.earliest = None
self.latest = None
self.jinja_count_type = "cycles"
self.end_of_cycle = False
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
self.current_count = 0
self.target_count = 0
self.earliest = None
self.latest = None
self.jinja_count_type = "cycles"
self.end_of_cycle = False
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
self._sample = sample
# Logger already setup by config, just get an instance
logger = logging.getLogger('eventgen')
from eventgenconfig import EventgenAdapter
adapter = EventgenAdapter(logger, {'module': 'ReplayGenerator', 'sample': sample.name})
globals()['logger'] = adapter
from eventgenconfig import Config
globals()['c'] = Config()
self._currentevent = 0
self._timeSinceSleep = datetime.timedelta()
self._times = [ ]
s = self._sample
# Load sample from a file, using cache if possible, from superclass GeneratorPlugin
s.loadSample()
self._rpevents = s.sampleDict
self._currentevent = 0
# 8/18/15 CS Because this is not a queueable plugin, we can in a threadsafe way modify these data structures at init
# Iterate through events and remove any events which do not match a configured timestamp,
# log it and then continue on
for e in self._rpevents:
try:
s.getTSFromEvent(e[s.timeField])
except ValueError:
self._rpevents = [x for x in self._rpevents if x['_raw'] != e['_raw']]
# Quick check to see if we're sorted in time order, if not reverse
if len(self._rpevents) > 1:
ts1 = s.getTSFromEvent(self._rpevents[0][s.timeField])
ts2 = s.getTSFromEvent(self._rpevents[1][s.timeField])
td = ts2 - ts1
x = 2
# Make sure we're not all zero
while td.days == 0 and td.seconds == 0 and td.microseconds == 0 and x < len(self._rpevents):
ts2 = s.getTSFromEvent(self._rpevents[x][s.timeField])
td = ts2 - ts1
x += 1
self.logger.debug("Testing timestamps ts1: %s ts2: %s" % (ts1.strftime('%Y-%m-%d %H:%M:%S'), ts2.strftime('%Y-%m-%d %H:%M:%S')))
if td.days < 0:
self.logger.debug("Timestamp order seems to be reverse chronological, reversing")
self._rpevents.reverse()
try:
self.setupBackfill()
except ValueError as e:
self.logger.error("Exception during backfill for sample '%s': '%s'" % (s.name, str(e)))
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
# Logger already setup by config, just get an instance
logger = logging.getLogger('eventgen')
from eventgenconfig import EventgenAdapter
adapter = EventgenAdapter(logger, {'module': 'PerDayVolumeGenerator', 'sample': sample.name})
globals()['logger'] = adapter
from eventgenconfig import Config
globals()['c'] = Config()
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
# Logger already setup by config, just get an instance
logger = logging.getLogger('eventgen')
from eventgenconfig import EventgenAdapter
adapter = EventgenAdapter(logger, {'module': 'WindbagGenerator', 'sample': sample.name})
globals()['logger'] = adapter
from eventgenconfig import Config
globals()['c'] = Config()
def gen(self, count, earliest, latest, samplename=None):
logger.debug("Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'" %
(self._sample.name, self._sample.app, count, earliest, latest))
startTime = datetime.datetime.now()
# If we're random, fill random events from sampleDict into eventsDict
if self._sample.randomizeEvents:
eventsDict = []
sdlen = len(self._sample.sampleDict)
logger.debug("Random filling eventsDict for sample '%s' in app '%s' with %d events" %
(self._sample.name, self._sample.app, count))
# Count is -1, replay the whole file, but in randomizeEvents I think we'd want it to actually
# just put as many events as there are in the file
if count == -1:
count = sdlen
while len(eventsDict) < count:
eventsDict.append(self._sample.sampleDict[random.randint(0, sdlen - 1)])
# If we're bundlelines, create count copies of the sampleDict
elif self._sample.bundlelines:
eventsDict = []
logger.debug(
"Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict" %
(self._sample.name, self._sample.app, count))
for x in xrange(count):
eventsDict.extend(self._sample.sampleDict)
# Otherwise fill count events into eventsDict or keep making copies of events out of sampleDict until
# eventsDict is as big as count
else:
# If count is -1, play the whole file, else grab a subset
if count == -1:
count = len(self._sample.sampleDict)
eventsDict = self._sample.sampleDict[0:count]
# Continue to fill events array until len(events) == count
if len(eventsDict) < count:
logger.debug(
"Events fill for sample '%s' in app '%s' less than count (%s vs. %s); continuing fill" %
(self._sample.name, self._sample.app, len(eventsDict), count))
logger.debug("Current eventsDict: %s" % eventsDict)
# run a modulus on the size of the eventdict to figure out what the last event was. Populate to count
# from there.
while len(eventsDict) < count:
if len(self._sample.sampleDict):
nextEventToUse = self._sample.sampleDict[len(eventsDict) % len(self._sample.sampleDict)]
logger.debug("Next event to add: %s" % nextEventToUse)
eventsDict.append(nextEventToUse)
logger.debug("Events fill complete for sample '%s' in app '%s' length %d" %
(self._sample.name, self._sample.app, len(eventsDict)))
GeneratorPlugin.build_events(self, eventsDict, startTime, earliest, latest)
def gen(self, count, earliest, latest, samplename=None):
self.logger.debug("Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'" %
(self._sample.name, self._sample.app, count, earliest, latest))
startTime = datetime.datetime.now()
# If we're random, fill random events from sampleDict into eventsDict
if self._sample.randomizeEvents:
eventsDict = []
sdlen = len(self._sample.sampleDict)
self.logger.debugv("Random filling eventsDict for sample '%s' in app '%s' with %d events" %
(self._sample.name, self._sample.app, count))
# Count is -1, replay the whole file, but in randomizeEvents I think we'd want it to actually
# just put as many events as there are in the file
if count == -1:
count = sdlen
while len(eventsDict) < count:
eventsDict.append(self._sample.sampleDict[random.randint(0, sdlen - 1)])
# If we're bundlelines, create count copies of the sampleDict
elif self._sample.bundlelines:
eventsDict = []
self.logger.debugv(
"Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict" %
(self._sample.name, self._sample.app, count))
for x in xrange(count):
eventsDict.extend(self._sample.sampleDict)
# Otherwise fill count events into eventsDict or keep making copies of events out of sampleDict until
# eventsDict is as big as count
else:
# If count is -1, play the whole file, else grab a subset
if count == -1:
count = len(self._sample.sampleDict)
eventsDict = self._sample.sampleDict[0:count]
# Continue to fill events array until len(events) == count
if len(eventsDict) < count:
self.logger.debugv(
"Events fill for sample '%s' in app '%s' less than count (%s vs. %s); continuing fill" %
(self._sample.name, self._sample.app, len(eventsDict), count))
self.logger.debugv("Current eventsDict: %s" % eventsDict)
# run a modulus on the size of the eventdict to figure out what the last event was. Populate to count
# from there.
while len(eventsDict) < count:
if len(self._sample.sampleDict):
nextEventToUse = self._sample.sampleDict[len(eventsDict) % len(self._sample.sampleDict)]
self.logger.debugv("Next event to add: %s" % nextEventToUse)
eventsDict.append(nextEventToUse)
self.logger.debugv("Events fill complete for sample '%s' in app '%s' length %d" %
(self._sample.name, self._sample.app, len(eventsDict)))
GeneratorPlugin.build_events(self, eventsDict, startTime, earliest, latest)
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
f = open('tests/perf/weblog/external_ips.sample')
self.external_ips = [x.strip() for x in f.readlines()]
self.external_ips_len = len(self.external_ips)
f.close()
f = open('tests/perf/weblog/webhosts.sample')
self.webhosts = [x.strip() for x in f.readlines()]
f.close()
self.webhosts_len = len(self.webhosts)
f = open('tests/perf/weblog/useragents.sample')
self.useragents = [x.strip() for x in f.readlines()]
f.close()
self.useragents_len = len(self.useragents)
f = open('tests/perf/weblog/webserverstatus.sample')
self.webserverstatus = [x.strip() for x in f.readlines()]
f.close()
self.webserverstatus_len = len(self.webserverstatus)
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
f = open('tests/sample_eventgen_conf/perf/weblog/external_ips.sample')
self.external_ips = [x.strip() for x in f.readlines()]
self.external_ips_len = len(self.external_ips)
f.close()
f = open('tests/sample_eventgen_conf/perf/weblog/webhosts.sample')
self.webhosts = [x.strip() for x in f.readlines()]
f.close()
self.webhosts_len = len(self.webhosts)
f = open('tests/sample_eventgen_conf/perf/weblog/useragents.sample')
self.useragents = [x.strip() for x in f.readlines()]
f.close()
self.useragents_len = len(self.useragents)
f = open('tests/sample_eventgen_conf/perf/weblog/webserverstatus.sample')
self.webserverstatus = [x.strip() for x in f.readlines()]
f.close()
self.webserverstatus_len = len(self.webserverstatus)
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
self._currentevent = 0
self._timeSinceSleep = datetime.timedelta()
self._times = []
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
self._currentevent = 0
self._timeSinceSleep = datetime.timedelta()
self._times = []
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
def gen(self, count, earliest, latest, samplename=None):
# count in this plugin is a measurement of byteself._sample.
size = count
self.logger.debug(
"PerDayVolumeGenerator Called with a Size of: %s with Earliest: %s and Latest: %s"
% (size, earliest, latest))
# very similar to the default generator. only difference is we go by size instead of count.
try:
self._sample.loadSample()
self.logger.debug("File sample loaded successfully.")
except TypeError:
self.logger.error("Error loading sample file for sample '%s'" %
self._sample.name)
return
self.logger.debug(
"Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'"
% (self._sample.name, self._sample.app, size, earliest, latest))
startTime = datetime.datetime.now()
# Create a counter for the current byte size of the read in samples
currentSize = 0
# If we're random, fill random events from sampleDict into eventsDict
eventsDict = []
if self._sample.randomizeEvents:
sdlen = len(self._sample.sampleDict)
self.logger.debugv(
"Random filling eventsDict for sample '%s' in app '%s' with %d bytes"
% (self._sample.name, self._sample.app, size))
while currentSize < size:
currentevent = self._sample.sampleDict[random.randint(
0, sdlen - 1)]
eventsDict.append(currentevent)
currentSize += len(currentevent['_raw'])
# If we're bundlelines, create count copies of the sampleDict
elif self._sample.bundlelines:
self.logger.debugv(
"Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict"
% (self._sample.name, self._sample.app, size))
while currentSize <= size:
sizeofsample = sum(
len(sample['_raw']) for sample in self._sample.sampleDict)
eventsDict.extend(self._sample.sampleDict)
currentSize += sizeofsample
# Otherwise fill count events into eventsDict or keep making copies of events out of sampleDict until
# eventsDict is as big as count
else:
self.logger.debug("Simple replay in order, processing")
# I need to check the sample and load events in order until the size is smaller than read events from file
# or i've read the entire file.
linecount = 0
currentreadsize = 0
linesinfile = len(self._sample.sampleDict)
self.logger.debugv("Lines in files: %s " % linesinfile)
while currentreadsize <= size:
targetline = linecount % linesinfile
sizeremaining = size - currentreadsize
targetlinesize = len(
self._sample.sampleDict[targetline]['_raw'])
if targetlinesize <= sizeremaining or targetlinesize * .9 <= sizeremaining:
currentreadsize += targetlinesize
eventsDict.append(self._sample.sampleDict[targetline])
else:
break
linecount += 1
self.logger.debugv(
"Events fill complete for sample '%s' in app '%s' length %d" %
(self._sample.name, self._sample.app, len(eventsDict)))
GeneratorPlugin.build_events(self, eventsDict, startTime, earliest,
latest)
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
# Logger already setup by config, just get an instance
logger = logging.getLogger('eventgen')
from eventgenconfig import EventgenAdapter
adapter = EventgenAdapter(logger, {'module': 'WindbagGenerator', 'sample': sample.name})
globals()['logger'] = adapter
from eventgenconfig import Config
globals()['c'] = Config()
# Pull customers into a dictionary
fh = open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'samples', 'customer_master.sample'), 'r')
# fh = open('../samples/customer_master.sample', 'r')
self.customers = [ ]
csvReader = csv.DictReader(fh)
for line in csvReader:
newline = dict((k, line[k]) for k in ('Address', 'Age', 'Sex', 'accountNumber', 'customerCity', 'customerMDN', 'customerState', 'customerZip', 'firstName', 'lastName'))
newline['address'] = newline['Address']
del newline['Address']
newline['age'] = newline['Age']
del newline['Age']
newline['sex'] = newline['Sex']
del newline['Sex']
newline['city'] = newline['customerCity']
del newline['customerCity']
newline['phone'] = newline['customerMDN']
del newline['customerMDN']
newline['state'] = newline['customerState']
del newline['customerState']
newline['zip'] = newline['customerZip']
del newline['customerZip']
self.customers.append(newline)
# Bring items into a dictionary
fh = open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'samples', 'items.sample'), 'r')
self.items = [ ]
csvReader = csv.reader(fh)
for line in csvReader:
self.items.append({ 'category': line[0], 'itemid': line[1], 'description': line[2], 'price': float(line[3]) })
self.transType = [ 'purchase', 'purchase', 'purchase', 'purchase', 'purchase', 'purchase', 'sale' ]
self.characterType = [ 'Milk Maid', 'Masked Mouse', 'Curd Cobbler', 'Whey Warrior', 'Fermented Friar' ]
self.regions = ['Gorgonzolia', 'Camemberalot', 'Jarlsberg', 'Swiss Alps', 'Limburgerland' ]
self.servers = [ ]
for a in ['ace', 'bubbles', 'cupcake', 'dash']:
for b in xrange(0, random.randint(1, 12)):
self.servers.append('%s.%s.woc.com' % (a, b))
self.typeRate = { 'purchase': 1.0, 'sale': 0.2 }
self.maxItems = 12
self.tps = 5.0
self.customerslen = len(self.customers)
self.itemslen = len(self.items)
self.transtypelen = len(self.transType)
self.charactertypelen = len(self.characterType)
self.serverslen = len(self.servers)
self.regionslen = len(self.regions)
def __init__(self, sample):
GeneratorPlugin.__init__(self, sample)
# Logger already setup by config, just get an instance
logger = logging.getLogger('eventgen')
from eventgenconfig import EventgenAdapter
adapter = EventgenAdapter(logger, {
'module': 'WindbagGenerator',
'sample': sample.name
})
globals()['logger'] = adapter
from eventgenconfig import Config
globals()['c'] = Config()
# Pull customers into a dictionary
fh = open(
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'samples', 'customer_master.sample'), 'r')
# fh = open('../samples/customer_master.sample', 'r')
self.customers = []
csvReader = csv.DictReader(fh)
for line in csvReader:
newline = dict(
(k, line[k])
for k in ('Address', 'Age', 'Sex', 'accountNumber',
'customerCity', 'customerMDN', 'customerState',
'customerZip', 'firstName', 'lastName'))
newline['address'] = newline['Address']
del newline['Address']
newline['age'] = newline['Age']
del newline['Age']
newline['sex'] = newline['Sex']
del newline['Sex']
newline['city'] = newline['customerCity']
del newline['customerCity']
newline['phone'] = newline['customerMDN']
del newline['customerMDN']
newline['state'] = newline['customerState']
del newline['customerState']
newline['zip'] = newline['customerZip']
del newline['customerZip']
self.customers.append(newline)
# Bring items into a dictionary
fh = open(
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'samples', 'items.sample'), 'r')
self.items = []
csvReader = csv.reader(fh)
for line in csvReader:
self.items.append({
'category': line[0],
'itemid': line[1],
'description': line[2],
'price': float(line[3])
})
self.transType = [
'purchase', 'purchase', 'purchase', 'purchase', 'purchase',
'purchase', 'sale'
]
self.characterType = [
'Milk Maid', 'Masked Mouse', 'Curd Cobbler', 'Whey Warrior',
'Fermented Friar'
]
self.regions = [
'Gorgonzolia', 'Camemberalot', 'Jarlsberg', 'Swiss Alps',
'Limburgerland'
]
self.servers = []
for a in ['ace', 'bubbles', 'cupcake', 'dash']:
for b in xrange(0, random.randint(1, 12)):
self.servers.append('%s.%s.woc.com' % (a, b))
self.typeRate = {'purchase': 1.0, 'sale': 0.2}
self.maxItems = 12
self.tps = 5.0
self.customerslen = len(self.customers)
self.itemslen = len(self.items)
self.transtypelen = len(self.transType)
self.charactertypelen = len(self.characterType)
self.serverslen = len(self.servers)
self.regionslen = len(self.regions)
