def __init__(self, sample): GeneratorPlugin.__init__(self, sample) # Logger already setup by config, just get an instance logger = logging.getLogger('eventgen') from eventgenconfig import EventgenAdapter adapter = EventgenAdapter(logger, {'module': 'WeblogGenerator', 'sample': sample.name}) globals()['logger'] = adapter from eventgenconfig import Config globals()['c'] = Config() f = open('tests/perf/weblog/external_ips.sample') self.external_ips = [x.strip() for x in f.readlines()] self.external_ips_len = len(self.external_ips) f.close() f = open('tests/perf/weblog/webhosts.sample') self.webhosts = [x.strip() for x in f.readlines()] f.close() self.webhosts_len = len(self.webhosts) f = open('tests/perf/weblog/useragents.sample') self.useragents = [x.strip() for x in f.readlines()] f.close() self.useragents_len = len(self.useragents) f = open('tests/perf/weblog/webserverstatus.sample') self.webserverstatus = [x.strip() for x in f.readlines()] f.close() self.webserverstatus_len = len(self.webserverstatus)
def __init__(self, sample): GeneratorPlugin.__init__(self, sample) # Logger already setup by config, just get an instance logger = logging.getLogger('eventgen') globals()['logger'] = logger from eventgenconfig import Config globals()['c'] = Config()
def __init__(self, sample): GeneratorPlugin.__init__(self, sample) self.current_count = 0 self.target_count = 0 self.earliest = None self.latest = None self.jinja_count_type = "cycles" self.end_of_cycle = False
def __init__(self, sample): GeneratorPlugin.__init__(self, sample) self._sample = sample # Logger already setup by config, just get an instance logger = logging.getLogger('eventgen') from eventgenconfig import EventgenAdapter adapter = EventgenAdapter(logger, {'module': 'ReplayGenerator', 'sample': sample.name}) globals()['logger'] = adapter from eventgenconfig import Config globals()['c'] = Config() self._currentevent = 0 self._timeSinceSleep = datetime.timedelta() self._times = [ ] s = self._sample # Load sample from a file, using cache if possible, from superclass GeneratorPlugin s.loadSample() self._rpevents = s.sampleDict self._currentevent = 0 # 8/18/15 CS Because this is not a queueable plugin, we can in a threadsafe way modify these data structures at init # Iterate through events and remove any events which do not match a configured timestamp, # log it and then continue on for e in self._rpevents: try: s.getTSFromEvent(e[s.timeField]) except ValueError: self._rpevents = [x for x in self._rpevents if x['_raw'] != e['_raw']] # Quick check to see if we're sorted in time order, if not reverse if len(self._rpevents) > 1: ts1 = s.getTSFromEvent(self._rpevents[0][s.timeField]) ts2 = s.getTSFromEvent(self._rpevents[1][s.timeField]) td = ts2 - ts1 x = 2 # Make sure we're not all zero while td.days == 0 and td.seconds == 0 and td.microseconds == 0 and x < len(self._rpevents): ts2 = s.getTSFromEvent(self._rpevents[x][s.timeField]) td = ts2 - ts1 x += 1 self.logger.debug("Testing timestamps ts1: %s ts2: %s" % (ts1.strftime('%Y-%m-%d %H:%M:%S'), ts2.strftime('%Y-%m-%d %H:%M:%S'))) if td.days < 0: self.logger.debug("Timestamp order seems to be reverse chronological, reversing") self._rpevents.reverse() try: self.setupBackfill() except ValueError as e: self.logger.error("Exception during backfill for sample '%s': '%s'" % (s.name, str(e)))
def __init__(self, sample): GeneratorPlugin.__init__(self, sample) # Logger already setup by config, just get an instance logger = logging.getLogger('eventgen') from eventgenconfig import EventgenAdapter adapter = EventgenAdapter(logger, {'module': 'PerDayVolumeGenerator', 'sample': sample.name}) globals()['logger'] = adapter from eventgenconfig import Config globals()['c'] = Config()
def __init__(self, sample): GeneratorPlugin.__init__(self, sample) # Logger already setup by config, just get an instance logger = logging.getLogger('eventgen') from eventgenconfig import EventgenAdapter adapter = EventgenAdapter(logger, {'module': 'WindbagGenerator', 'sample': sample.name}) globals()['logger'] = adapter from eventgenconfig import Config globals()['c'] = Config()
def gen(self, count, earliest, latest, samplename=None): logger.debug("Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'" % (self._sample.name, self._sample.app, count, earliest, latest)) startTime = datetime.datetime.now() # If we're random, fill random events from sampleDict into eventsDict if self._sample.randomizeEvents: eventsDict = [] sdlen = len(self._sample.sampleDict) logger.debug("Random filling eventsDict for sample '%s' in app '%s' with %d events" % (self._sample.name, self._sample.app, count)) # Count is -1, replay the whole file, but in randomizeEvents I think we'd want it to actually # just put as many events as there are in the file if count == -1: count = sdlen while len(eventsDict) < count: eventsDict.append(self._sample.sampleDict[random.randint(0, sdlen - 1)]) # If we're bundlelines, create count copies of the sampleDict elif self._sample.bundlelines: eventsDict = [] logger.debug( "Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict" % (self._sample.name, self._sample.app, count)) for x in xrange(count): eventsDict.extend(self._sample.sampleDict) # Otherwise fill count events into eventsDict or keep making copies of events out of sampleDict until # eventsDict is as big as count else: # If count is -1, play the whole file, else grab a subset if count == -1: count = len(self._sample.sampleDict) eventsDict = self._sample.sampleDict[0:count] # Continue to fill events array until len(events) == count if len(eventsDict) < count: logger.debug( "Events fill for sample '%s' in app '%s' less than count (%s vs. %s); continuing fill" % (self._sample.name, self._sample.app, len(eventsDict), count)) logger.debug("Current eventsDict: %s" % eventsDict) # run a modulus on the size of the eventdict to figure out what the last event was. Populate to count # from there. while len(eventsDict) < count: if len(self._sample.sampleDict): nextEventToUse = self._sample.sampleDict[len(eventsDict) % len(self._sample.sampleDict)] logger.debug("Next event to add: %s" % nextEventToUse) eventsDict.append(nextEventToUse) logger.debug("Events fill complete for sample '%s' in app '%s' length %d" % (self._sample.name, self._sample.app, len(eventsDict))) GeneratorPlugin.build_events(self, eventsDict, startTime, earliest, latest)
def gen(self, count, earliest, latest, samplename=None): self.logger.debug("Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'" % (self._sample.name, self._sample.app, count, earliest, latest)) startTime = datetime.datetime.now() # If we're random, fill random events from sampleDict into eventsDict if self._sample.randomizeEvents: eventsDict = [] sdlen = len(self._sample.sampleDict) self.logger.debugv("Random filling eventsDict for sample '%s' in app '%s' with %d events" % (self._sample.name, self._sample.app, count)) # Count is -1, replay the whole file, but in randomizeEvents I think we'd want it to actually # just put as many events as there are in the file if count == -1: count = sdlen while len(eventsDict) < count: eventsDict.append(self._sample.sampleDict[random.randint(0, sdlen - 1)]) # If we're bundlelines, create count copies of the sampleDict elif self._sample.bundlelines: eventsDict = [] self.logger.debugv( "Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict" % (self._sample.name, self._sample.app, count)) for x in xrange(count): eventsDict.extend(self._sample.sampleDict) # Otherwise fill count events into eventsDict or keep making copies of events out of sampleDict until # eventsDict is as big as count else: # If count is -1, play the whole file, else grab a subset if count == -1: count = len(self._sample.sampleDict) eventsDict = self._sample.sampleDict[0:count] # Continue to fill events array until len(events) == count if len(eventsDict) < count: self.logger.debugv( "Events fill for sample '%s' in app '%s' less than count (%s vs. %s); continuing fill" % (self._sample.name, self._sample.app, len(eventsDict), count)) self.logger.debugv("Current eventsDict: %s" % eventsDict) # run a modulus on the size of the eventdict to figure out what the last event was. Populate to count # from there. while len(eventsDict) < count: if len(self._sample.sampleDict): nextEventToUse = self._sample.sampleDict[len(eventsDict) % len(self._sample.sampleDict)] self.logger.debugv("Next event to add: %s" % nextEventToUse) eventsDict.append(nextEventToUse) self.logger.debugv("Events fill complete for sample '%s' in app '%s' length %d" % (self._sample.name, self._sample.app, len(eventsDict))) GeneratorPlugin.build_events(self, eventsDict, startTime, earliest, latest)
def __init__(self, sample): GeneratorPlugin.__init__(self, sample) f = open('tests/perf/weblog/external_ips.sample') self.external_ips = [x.strip() for x in f.readlines()] self.external_ips_len = len(self.external_ips) f.close() f = open('tests/perf/weblog/webhosts.sample') self.webhosts = [x.strip() for x in f.readlines()] f.close() self.webhosts_len = len(self.webhosts) f = open('tests/perf/weblog/useragents.sample') self.useragents = [x.strip() for x in f.readlines()] f.close() self.useragents_len = len(self.useragents) f = open('tests/perf/weblog/webserverstatus.sample') self.webserverstatus = [x.strip() for x in f.readlines()] f.close() self.webserverstatus_len = len(self.webserverstatus)
def __init__(self, sample): GeneratorPlugin.__init__(self, sample) f = open('tests/sample_eventgen_conf/perf/weblog/external_ips.sample') self.external_ips = [x.strip() for x in f.readlines()] self.external_ips_len = len(self.external_ips) f.close() f = open('tests/sample_eventgen_conf/perf/weblog/webhosts.sample') self.webhosts = [x.strip() for x in f.readlines()] f.close() self.webhosts_len = len(self.webhosts) f = open('tests/sample_eventgen_conf/perf/weblog/useragents.sample') self.useragents = [x.strip() for x in f.readlines()] f.close() self.useragents_len = len(self.useragents) f = open('tests/sample_eventgen_conf/perf/weblog/webserverstatus.sample') self.webserverstatus = [x.strip() for x in f.readlines()] f.close() self.webserverstatus_len = len(self.webserverstatus)
def __init__(self, sample): GeneratorPlugin.__init__(self, sample) self._currentevent = 0 self._timeSinceSleep = datetime.timedelta() self._times = []
def __init__(self, sample): GeneratorPlugin.__init__(self, sample)
def gen(self, count, earliest, latest, samplename=None): # count in this plugin is a measurement of byteself._sample. size = count self.logger.debug( "PerDayVolumeGenerator Called with a Size of: %s with Earliest: %s and Latest: %s" % (size, earliest, latest)) # very similar to the default generator. only difference is we go by size instead of count. try: self._sample.loadSample() self.logger.debug("File sample loaded successfully.") except TypeError: self.logger.error("Error loading sample file for sample '%s'" % self._sample.name) return self.logger.debug( "Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'" % (self._sample.name, self._sample.app, size, earliest, latest)) startTime = datetime.datetime.now() # Create a counter for the current byte size of the read in samples currentSize = 0 # If we're random, fill random events from sampleDict into eventsDict eventsDict = [] if self._sample.randomizeEvents: sdlen = len(self._sample.sampleDict) self.logger.debugv( "Random filling eventsDict for sample '%s' in app '%s' with %d bytes" % (self._sample.name, self._sample.app, size)) while currentSize < size: currentevent = self._sample.sampleDict[random.randint( 0, sdlen - 1)] eventsDict.append(currentevent) currentSize += len(currentevent['_raw']) # If we're bundlelines, create count copies of the sampleDict elif self._sample.bundlelines: self.logger.debugv( "Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict" % (self._sample.name, self._sample.app, size)) while currentSize <= size: sizeofsample = sum( len(sample['_raw']) for sample in self._sample.sampleDict) eventsDict.extend(self._sample.sampleDict) currentSize += sizeofsample # Otherwise fill count events into eventsDict or keep making copies of events out of sampleDict until # eventsDict is as big as count else: self.logger.debug("Simple replay in order, processing") # I need to check the sample and load events in order until the size is smaller than read events from file # or i've read the entire file. linecount = 0 currentreadsize = 0 linesinfile = len(self._sample.sampleDict) self.logger.debugv("Lines in files: %s " % linesinfile) while currentreadsize <= size: targetline = linecount % linesinfile sizeremaining = size - currentreadsize targetlinesize = len( self._sample.sampleDict[targetline]['_raw']) if targetlinesize <= sizeremaining or targetlinesize * .9 <= sizeremaining: currentreadsize += targetlinesize eventsDict.append(self._sample.sampleDict[targetline]) else: break linecount += 1 self.logger.debugv( "Events fill complete for sample '%s' in app '%s' length %d" % (self._sample.name, self._sample.app, len(eventsDict))) GeneratorPlugin.build_events(self, eventsDict, startTime, earliest, latest)
def __init__(self, sample): GeneratorPlugin.__init__(self, sample) # Logger already setup by config, just get an instance logger = logging.getLogger('eventgen') from eventgenconfig import EventgenAdapter adapter = EventgenAdapter(logger, {'module': 'WindbagGenerator', 'sample': sample.name}) globals()['logger'] = adapter from eventgenconfig import Config globals()['c'] = Config() # Pull customers into a dictionary fh = open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'samples', 'customer_master.sample'), 'r') # fh = open('../samples/customer_master.sample', 'r') self.customers = [ ] csvReader = csv.DictReader(fh) for line in csvReader: newline = dict((k, line[k]) for k in ('Address', 'Age', 'Sex', 'accountNumber', 'customerCity', 'customerMDN', 'customerState', 'customerZip', 'firstName', 'lastName')) newline['address'] = newline['Address'] del newline['Address'] newline['age'] = newline['Age'] del newline['Age'] newline['sex'] = newline['Sex'] del newline['Sex'] newline['city'] = newline['customerCity'] del newline['customerCity'] newline['phone'] = newline['customerMDN'] del newline['customerMDN'] newline['state'] = newline['customerState'] del newline['customerState'] newline['zip'] = newline['customerZip'] del newline['customerZip'] self.customers.append(newline) # Bring items into a dictionary fh = open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'samples', 'items.sample'), 'r') self.items = [ ] csvReader = csv.reader(fh) for line in csvReader: self.items.append({ 'category': line[0], 'itemid': line[1], 'description': line[2], 'price': float(line[3]) }) self.transType = [ 'purchase', 'purchase', 'purchase', 'purchase', 'purchase', 'purchase', 'sale' ] self.characterType = [ 'Milk Maid', 'Masked Mouse', 'Curd Cobbler', 'Whey Warrior', 'Fermented Friar' ] self.regions = ['Gorgonzolia', 'Camemberalot', 'Jarlsberg', 'Swiss Alps', 'Limburgerland' ] self.servers = [ ] for a in ['ace', 'bubbles', 'cupcake', 'dash']: for b in xrange(0, random.randint(1, 12)): self.servers.append('%s.%s.woc.com' % (a, b)) self.typeRate = { 'purchase': 1.0, 'sale': 0.2 } self.maxItems = 12 self.tps = 5.0 self.customerslen = len(self.customers) self.itemslen = len(self.items) self.transtypelen = len(self.transType) self.charactertypelen = len(self.characterType) self.serverslen = len(self.servers) self.regionslen = len(self.regions)
def __init__(self, sample): GeneratorPlugin.__init__(self, sample) # Logger already setup by config, just get an instance logger = logging.getLogger('eventgen') from eventgenconfig import EventgenAdapter adapter = EventgenAdapter(logger, { 'module': 'WindbagGenerator', 'sample': sample.name }) globals()['logger'] = adapter from eventgenconfig import Config globals()['c'] = Config() # Pull customers into a dictionary fh = open( os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'samples', 'customer_master.sample'), 'r') # fh = open('../samples/customer_master.sample', 'r') self.customers = [] csvReader = csv.DictReader(fh) for line in csvReader: newline = dict( (k, line[k]) for k in ('Address', 'Age', 'Sex', 'accountNumber', 'customerCity', 'customerMDN', 'customerState', 'customerZip', 'firstName', 'lastName')) newline['address'] = newline['Address'] del newline['Address'] newline['age'] = newline['Age'] del newline['Age'] newline['sex'] = newline['Sex'] del newline['Sex'] newline['city'] = newline['customerCity'] del newline['customerCity'] newline['phone'] = newline['customerMDN'] del newline['customerMDN'] newline['state'] = newline['customerState'] del newline['customerState'] newline['zip'] = newline['customerZip'] del newline['customerZip'] self.customers.append(newline) # Bring items into a dictionary fh = open( os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'samples', 'items.sample'), 'r') self.items = [] csvReader = csv.reader(fh) for line in csvReader: self.items.append({ 'category': line[0], 'itemid': line[1], 'description': line[2], 'price': float(line[3]) }) self.transType = [ 'purchase', 'purchase', 'purchase', 'purchase', 'purchase', 'purchase', 'sale' ] self.characterType = [ 'Milk Maid', 'Masked Mouse', 'Curd Cobbler', 'Whey Warrior', 'Fermented Friar' ] self.regions = [ 'Gorgonzolia', 'Camemberalot', 'Jarlsberg', 'Swiss Alps', 'Limburgerland' ] self.servers = [] for a in ['ace', 'bubbles', 'cupcake', 'dash']: for b in xrange(0, random.randint(1, 12)): self.servers.append('%s.%s.woc.com' % (a, b)) self.typeRate = {'purchase': 1.0, 'sale': 0.2} self.maxItems = 12 self.tps = 5.0 self.customerslen = len(self.customers) self.itemslen = len(self.items) self.transtypelen = len(self.transType) self.charactertypelen = len(self.characterType) self.serverslen = len(self.servers) self.regionslen = len(self.regions)