def queue_it(self, count):
try:
realtime = self.sample.now(realnow=True)
if "-" in self.sample.backfill[0]:
mathsymbol = "-"
else:
mathsymbol = "+"
backfillnumber = ""
backfillletter = ""
for char in self.sample.backfill:
if char.isdigit():
backfillnumber += char
elif char != "-":
backfillletter += char
backfillearliest = timeParserTimeMath(
plusminus=mathsymbol,
num=backfillnumber,
unit=backfillletter,
ret=realtime,
)
while backfillearliest < realtime:
et = backfillearliest
lt = timeParserTimeMath(plusminus="+",
num=self.sample.interval,
unit="s",
ret=et)
genPlugin = self.generatorPlugin(sample=self.sample)
genPlugin.updateCounts(count=count, start_time=et, end_time=lt)
genPlugin.updateConfig(config=self.config,
outqueue=self.outputQueue)
try:
# Need to lock on replay mode since event duration is dynamic. Interval starts counting
# after the replay has finished.
if self.sample.generator == "replay":
genPlugin.run()
else:
self.generatorQueue.put(genPlugin)
except Full:
logger.warning(
"Generator Queue Full. Skipping current generation.")
# due to replays needing to iterate in reverse, it's more efficent to process backfill
# after the file has been parsed. This section is to allow replay mode to take
# care of all replays on it's first run. and sets backfilldone
if self.sample.generator == "replay":
backfillearliest = realtime
else:
backfillearliest = lt
if self.sample.generator != "replay":
self.sample.backfilldone = True
except Exception as e:
logger.error("Failed queuing backfill, exception: {0}".format(e))
def test_time_parser_time_math(plusminus, num, unit, ret, expect):
'''
test timeParserTimeMath function, parse the time modifier
Normal Case:
Case 1: input "+100s" -- the parser should translate it as 100 seconds later.
Case 2: input "-20m" -- the parser should handle the month larger than 12 and translate as 20 months ago
Case 3: input '3w' -- the parser should translate as 21 days later.
Corner Cases:
Case 1: input "0s" -- the time parser should return now
Case 2: input "123" -- unit is the empty string, behavior <TBD>
'''
check_datetime_equal(timeparser.timeParserTimeMath(plusminus, num, unichr, ret), expect)
def test_time_parser_time_math(plusminus, num, unit, ret, expect):
'''
test timeParserTimeMath function, parse the time modifier
Normal Case:
Case 1: input "+100s" -- the parser should translate it as 100 seconds later.
Case 2: input "-20m" -- the parser should handle the month larger than 12 and translate as 20 months ago
Case 3: input '3w' -- the parser should translate as 21 days later.
Corner Cases:
Case 1: input "0s" -- the time parser should return now
Case 2: input "123" -- unit is the empty string, behavior <TBD>
'''
check_datetime_equal(
timeparser.timeParserTimeMath(plusminus, num, chr, ret), expect)
def real_run(self):
"""
Worker function of the Timer class. Determine whether a plugin is queueable, and either
place an item in the generator queue for that plugin or call the plugin's gen method directly.
"""
if self.sample.delay > 0:
logger.info("Sample set to delay %s, sleeping." %
self.sample.delay)
time.sleep(self.sample.delay)
logger.debug("Timer creating plugin for '%s'" % self.sample.name)
end = False
previous_count_left = 0
raw_event_size = self.predict_event_size()
if self.end:
if int(self.end) == 0:
logger.info(
"End = 0, no events will be generated for sample '%s'" %
self.sample.name)
end = True
elif int(self.end) == -1:
logger.info(
"End is set to -1. Will be running without stopping for sample %s"
% self.sample.name)
while not end:
# Need to be able to stop threads by the main thread or this thread. self.config will stop all threads
# referenced in the config object, while, self.stopping will only stop this one.
if self.config.stopping or self.stopping:
end = True
continue
count = self.rater.rate()
# First run of the generator, see if we have any backfill work to do.
if self.countdown <= 0:
if self.sample.backfill and not self.sample.backfilldone:
realtime = self.sample.now(realnow=True)
if "-" in self.sample.backfill[0]:
mathsymbol = "-"
else:
mathsymbol = "+"
backfillnumber = ""
backfillletter = ""
for char in self.sample.backfill:
if char.isdigit():
backfillnumber += char
elif char != "-":
backfillletter += char
backfillearliest = timeParserTimeMath(plusminus=mathsymbol,
num=backfillnumber,
unit=backfillletter,
ret=realtime)
while backfillearliest < realtime:
if self.end and self.executions == int(self.end):
logger.info(
"End executions %d reached, ending generation of sample '%s'"
% (int(self.end), self.sample.name))
break
et = backfillearliest
lt = timeParserTimeMath(plusminus="+",
num=self.interval,
unit="s",
ret=et)
copy_sample = copy.copy(self.sample)
tokens = copy.deepcopy(self.sample.tokens)
copy_sample.tokens = tokens
genPlugin = self.generatorPlugin(sample=copy_sample)
# need to make sure we set the queue right if we're using multiprocessing or thread modes
genPlugin.updateConfig(config=self.config,
outqueue=self.outputQueue)
genPlugin.updateCounts(count=count,
start_time=et,
end_time=lt)
try:
self.generatorQueue.put(genPlugin, True, 3)
self.executions += 1
backfillearliest = lt
except Full:
logger.warning(
"Generator Queue Full. Reput the backfill generator task later. %d backfill generators are dispatched.",
self.executions)
backfillearliest = et
realtime = self.sample.now(realnow=True)
self.sample.backfilldone = True
else:
# 12/15/13 CS Moving the rating to a separate plugin architecture
# Save previous interval count left to avoid perdayvolumegenerator drop small tasks
if self.sample.generator == 'perdayvolumegenerator':
count = self.rater.rate() + previous_count_left
if 0 < count < raw_event_size:
logger.info(
"current interval size is {}, which is smaller than a raw event size {}."
.format(count, raw_event_size) +
"Wait for the next turn.")
previous_count_left = count
self.countdown = self.interval
self.executions += 1
continue
else:
previous_count_left = 0
else:
count = self.rater.rate()
et = self.sample.earliestTime()
lt = self.sample.latestTime()
try:
if count < 1 and count != -1:
logger.info(
"There is no data to be generated in worker {0} because the count is {1}."
.format(self.sample.config.generatorWorkers,
count))
else:
# Spawn workers at the beginning of job rather than wait for next interval
logger.info(
"Starting '%d' generatorWorkers for sample '%s'"
% (self.sample.config.generatorWorkers,
self.sample.name))
for worker_id in range(
self.config.generatorWorkers):
copy_sample = copy.copy(self.sample)
tokens = copy.deepcopy(self.sample.tokens)
copy_sample.tokens = tokens
genPlugin = self.generatorPlugin(
sample=copy_sample)
# Adjust queue for threading mode
genPlugin.updateConfig(
config=self.config,
outqueue=self.outputQueue)
genPlugin.updateCounts(count=count,
start_time=et,
end_time=lt)
try:
self.generatorQueue.put(genPlugin)
logger.debug((
"Worker# {0}: Put {1} MB of events in queue for sample '{2}'"
+ "with et '{3}' and lt '{4}'").format(
worker_id,
round((count / 1024.0 / 1024), 4),
self.sample.name, et, lt))
except Full:
logger.warning(
"Generator Queue Full. Skipping current generation."
)
self.executions += 1
except Exception as e:
logger.exception(str(e))
if self.stopping:
end = True
pass
# Sleep until we're supposed to wake up and generate more events
self.countdown = self.interval
# 8/20/15 CS Adding support for ending generation at a certain time
if self.end:
if int(self.end) == -1:
time.sleep(self.time)
self.countdown -= self.time
continue
# 3/16/16 CS Adding support for ending on a number of executions instead of time
# Should be fine with storing state in this sample object since each sample has it's own unique
# timer thread
if not self.endts:
if self.executions >= int(self.end):
logger.info(
"End executions %d reached, ending generation of sample '%s'"
% (int(self.end), self.sample.name))
self.stopping = True
end = True
elif lt >= self.endts:
logger.info(
"End Time '%s' reached, ending generation of sample '%s'"
% (self.sample.endts, self.sample.name))
self.stopping = True
end = True
else:
time.sleep(self.time)
self.countdown -= self.time