def real_run(self):
"""
Worker function of the Timer class. Determine whether a plugin is queueable, and either
place an item in the generator queue for that plugin or call the plugin's gen method directly.
"""
if self.sample.delay > 0:
self.logger.info("Sample set to delay %s, sleeping." %
self.sample.delay)
time.sleep(self.sample.delay)
self.logger.debug("Timer creating plugin for '%s'" % self.sample.name)
self.executions = 0
end = False
previous_count_left = 0
raw_event_size = self.predict_event_size()
while not end:
# Need to be able to stop threads by the main thread or this thread. self.config will stop all threads
# referenced in the config object, while, self.stopping will only stop this one.
if self.config.stopping or self.stopping:
end = True
count = self.rater.rate()
#First run of the generator, see if we have any backfill work to do.
if self.countdown <= 0:
if self.sample.backfill and not self.sample.backfilldone:
realtime = self.sample.now(realnow=True)
if "-" in self.sample.backfill[0]:
mathsymbol = "-"
else:
mathsymbol = "+"
backfillnumber = ""
backfillletter = ""
for char in self.sample.backfill:
if char.isdigit():
backfillnumber += char
elif char != "-":
backfillletter += char
backfillearliest = timeParserTimeMath(plusminus=mathsymbol,
num=backfillnumber,
unit=backfillletter,
ret=realtime)
while backfillearliest < realtime:
et = backfillearliest
lt = timeParserTimeMath(plusminus="+",
num=self.sample.interval,
unit="s",
ret=et)
genPlugin = self.generatorPlugin(sample=self.sample)
# need to make sure we set the queue right if we're using multiprocessing or thread modes
genPlugin.updateConfig(config=self.config,
outqueue=self.outputQueue)
genPlugin.updateCounts(count=count,
start_time=et,
end_time=lt)
try:
self.generatorQueue.put(genPlugin)
except Full:
self.logger.warning(
"Generator Queue Full. Skipping current generation."
)
backfillearliest = lt
self.sample.backfilldone = True
else:
# 12/15/13 CS Moving the rating to a separate plugin architecture
# Save previous interval count left to avoid perdayvolumegenerator drop small tasks
if self.sample.generator == 'perdayvolumegenerator':
count = self.rater.rate() + previous_count_left
if count < raw_event_size and count > 0:
self.logger.info(
"current interval size is {}, which is smaller than a raw event size {}. wait for the next turn."
.format(count, raw_event_size))
previous_count_left = count
self.countdown = self.sample.interval
self.executions += 1
continue
else:
previous_count_left = 0
else:
count = self.rater.rate()
et = self.sample.earliestTime()
lt = self.sample.latestTime()
try:
if count < 1 and count != -1:
self.logger.info(
"There is no data to be generated in worker {0} because the count is {1}."
.format(self.sample.config.generatorWorkers,
count))
else:
# Spawn workers at the beginning of job rather than wait for next interval
self.logger.info(
"Start '%d' generatorWorkers for sample '%s'" %
(self.sample.config.generatorWorkers,
self.sample.name))
for worker_id in range(
self.config.generatorWorkers):
# self.generatorPlugin is only an instance, now we need a real plugin. Make a copy of
# of the sample in case another generator corrupts it.
copy_sample = copy.copy(self.sample)
tokens = copy.deepcopy(self.sample.tokens)
copy_sample.tokens = tokens
genPlugin = self.generatorPlugin(
sample=copy_sample)
# Adjust queue for threading mode
genPlugin.updateConfig(
config=self.config,
outqueue=self.outputQueue)
genPlugin.updateCounts(count=count,
start_time=et,
end_time=lt)
try:
self.generatorQueue.put(genPlugin)
self.logger.info(
"Worker# {0}: Put {1} MB of events in queue for sample '{2}' with et '{3}' and lt '{4}'"
.format(
worker_id,
round((count / 1024.0 / 1024), 4),
self.sample.name, et, lt))
except Full:
self.logger.warning(
"Generator Queue Full. Skipping current generation."
)
except Exception as e:
self.logger.exception(e)
if self.stopping:
end = True
pass
# Sleep until we're supposed to wake up and generate more events
self.countdown = self.sample.interval
self.executions += 1
# 8/20/15 CS Adding support for ending generation at a certain time
if self.end:
# 3/16/16 CS Adding support for ending on a number of executions instead of time
# Should be fine with storing state in this sample object since each sample has it's own unique
# timer thread
if not self.endts:
if self.executions >= int(self.end):
self.logger.info(
"End executions %d reached, ending generation of sample '%s'"
% (int(self.end), self.sample.name))
self.stopping = True
end = True
elif lt >= self.endts:
self.logger.info(
"End Time '%s' reached, ending generation of sample '%s'"
% (self.sample.endts, self.sample.name))
self.stopping = True
end = True
else:
time.sleep(self.time)
self.countdown -= self.time
def real_run(self):
"""
Worker function of the Timer class. Determine whether a plugin is queueable, and either
place an item in the generator queue for that plugin or call the plugin's gen method directly.
"""
if self.sample.delay > 0:
self.logger.info("Sample set to delay %s, sleeping." % self.sample.delay)
time.sleep(self.sample.delay)
self.logger.debug("Timer creating plugin for '%s'" % self.sample.name)
end = False
previous_count_left = 0
raw_event_size = self.predict_event_size()
if self.end:
if int(self.end) == 0:
self.logger.info("End = 0, no events will be generated for sample '%s'" % self.sample.name)
end = True
elif int(self.end) == -1:
self.logger.info("End is set to -1. Will be running without stopping for sample %s" % self.sample.name)
while not end:
# Need to be able to stop threads by the main thread or this thread. self.config will stop all threads
# referenced in the config object, while, self.stopping will only stop this one.
if self.config.stopping or self.stopping:
end = True
count = self.rater.rate()
# First run of the generator, see if we have any backfill work to do.
if self.countdown <= 0:
if self.sample.backfill and not self.sample.backfilldone:
realtime = self.sample.now(realnow=True)
if "-" in self.sample.backfill[0]:
mathsymbol = "-"
else:
mathsymbol = "+"
backfillnumber = ""
backfillletter = ""
for char in self.sample.backfill:
if char.isdigit():
backfillnumber += char
elif char != "-":
backfillletter += char
backfillearliest = timeParserTimeMath(plusminus=mathsymbol, num=backfillnumber, unit=backfillletter,
ret=realtime)
while backfillearliest < realtime:
if self.executions == int(self.end):
self.logger.info("End executions %d reached, ending generation of sample '%s'" % (int(
self.end), self.sample.name))
break
et = backfillearliest
lt = timeParserTimeMath(plusminus="+", num=self.interval, unit="s", ret=et)
genPlugin = self.generatorPlugin(sample=self.sample)
# need to make sure we set the queue right if we're using multiprocessing or thread modes
genPlugin.updateConfig(config=self.config, outqueue=self.outputQueue)
genPlugin.updateCounts(count=count, start_time=et, end_time=lt)
try:
self.generatorQueue.put(genPlugin)
self.executions += 1
except Full:
self.logger.warning("Generator Queue Full. Skipping current generation.")
backfillearliest = lt
self.sample.backfilldone = True
else:
# 12/15/13 CS Moving the rating to a separate plugin architecture
# Save previous interval count left to avoid perdayvolumegenerator drop small tasks
if self.sample.generator == 'perdayvolumegenerator':
count = self.rater.rate() + previous_count_left
if 0 < count < raw_event_size:
self.logger.info("current interval size is {}, which is smaller than a raw event size {}.".
format(count, raw_event_size) + "Wait for the next turn.")
previous_count_left = count
self.countdown = self.interval
self.executions += 1
continue
else:
previous_count_left = 0
else:
count = self.rater.rate()
et = self.sample.earliestTime()
lt = self.sample.latestTime()
try:
if count < 1 and count != -1:
self.logger.info(
"There is no data to be generated in worker {0} because the count is {1}.".format(
self.sample.config.generatorWorkers, count))
else:
# Spawn workers at the beginning of job rather than wait for next interval
self.logger.info("Start '%d' generatorWorkers for sample '%s'" %
(self.sample.config.generatorWorkers, self.sample.name))
for worker_id in range(self.config.generatorWorkers):
# self.generatorPlugin is only an instance, now we need a real plugin. Make a copy of
# of the sample in case another generator corrupts it.
copy_sample = copy.copy(self.sample)
copy_tokens = []
for token in self.sample.tokens:
copy_tokens.append(token.deepcopy(self.sample))
copy_sample.tokens = copy_tokens
genPlugin = self.generatorPlugin(sample=copy_sample)
# Adjust queue for threading mode
genPlugin.updateConfig(config=self.config, outqueue=self.outputQueue)
genPlugin.updateCounts(count=count, start_time=et, end_time=lt)
try:
self.generatorQueue.put(genPlugin)
self.executions += 1
self.logger.info(("Worker# {0}: Put {1} MB of events in queue for sample '{2}'" +
"with et '{3}' and lt '{4}'").format(
worker_id, round((count / 1024.0 / 1024), 4),
self.sample.name, et, lt))
except Full:
self.logger.warning("Generator Queue Full. Skipping current generation.")
except Exception as e:
self.logger.exception(e)
if self.stopping:
end = True
pass
# Sleep until we're supposed to wake up and generate more events
self.countdown = self.interval
# 8/20/15 CS Adding support for ending generation at a certain time
if self.end:
if int(self.end) == -1:
time.sleep(self.time)
self.countdown -= self.time
continue
# 3/16/16 CS Adding support for ending on a number of executions instead of time
# Should be fine with storing state in this sample object since each sample has it's own unique
# timer thread
if not self.endts:
if self.executions >= int(self.end):
self.logger.info("End executions %d reached, ending generation of sample '%s'" % (int(
self.end), self.sample.name))
self.stopping = True
end = True
elif lt >= self.endts:
self.logger.info("End Time '%s' reached, ending generation of sample '%s'" % (self.sample.endts,
self.sample.name))
self.stopping = True
end = True
else:
time.sleep(self.time)
self.countdown -= self.time
def real_run(self):
"""
Worker function of the Timer class. Determine whether a plugin is queueable, and either
place an item in the generator queue for that plugin or call the plugin's gen method directly.
"""
if self.sample.delay > 0:
self.logger.info("Sample set to delay %s, sleeping." %
self.sample.delay)
time.sleep(self.sample.delay)
self.logger.debug("Timer creating plugin for '%s'" % self.sample.name)
self.executions = 0
end = False
while not end:
# Need to be able to stop threads by the main thread or this thread. self.config will stop all threads
# referenced in the config object, while, self.stopping will only stop this one.
if self.config.stopping or self.stopping:
end = True
count = self.rater.rate()
#First run of the generator, see if we have any backfill work to do.
#TODO: I think self.countdown can just go away.
if self.countdown <= 0:
if self.sample.backfill and not self.sample.backfilldone:
realtime = self.sample.now(realnow=True)
if "-" in self.sample.backfill[0]:
mathsymbol = "-"
else:
mathsymbol = "+"
backfillnumber = ""
backfillletter = ""
for char in self.sample.backfill:
if char.isdigit():
backfillnumber += char
elif char != "-":
backfillletter += char
backfillearliest = timeParserTimeMath(plusminus=mathsymbol,
num=backfillnumber,
unit=backfillletter,
ret=realtime)
while backfillearliest < realtime:
et = backfillearliest
lt = timeParserTimeMath(plusminus="+",
num=self.sample.interval,
unit="s",
ret=et)
genPlugin = self.generatorPlugin(sample=self.sample)
# need to make sure we set the queue right if we're using multiprocessing or thread modes
genPlugin.updateConfig(config=self.config,
outqueue=self.outputQueue)
genPlugin.updateCounts(count=count,
start_time=et,
end_time=lt)
try:
self.generatorQueue.put(genPlugin)
except Full:
self.logger.warning(
"Generator Queue Full. Skipping current generation."
)
backfillearliest = lt
self.sample.backfilldone = True
else:
# 12/15/13 CS Moving the rating to a separate plugin architecture
count = self.rater.rate()
et = self.sample.earliestTime()
lt = self.sample.latestTime()
try:
# Spawn workers at the beginning of job rather than wait for next interval
self.logger.info(
"Start '%d' generatorWorkers for sample '%s'" %
(self.sample.config.generatorWorkers,
self.sample.name))
for worker_id in range(self.config.generatorWorkers):
# self.generatorPlugin is only an instance, now we need a real plugin.
# make a copy of the sample so if it's mutated by another process, it won't mess up geeneration
# for this generator.
copy_sample = copy.copy(self.sample)
genPlugin = self.generatorPlugin(
sample=copy_sample)
# need to make sure we set the queue right if we're using multiprocessing or thread modes
genPlugin.updateConfig(config=self.config,
outqueue=self.outputQueue)
genPlugin.updateCounts(count=count,
start_time=et,
end_time=lt)
try:
self.generatorQueue.put(genPlugin)
except Full:
self.logger.warning(
"Generator Queue Full. Skipping current generation."
)
self.logger.info(
"Worker# %d: Put %d events in queue for sample '%s' with et '%s' and lt '%s'"
% (worker_id, count, self.sample.name, et, lt))
# TODO: put this back to just catching a full queue
except Exception as e:
self.logger.exception(e)
if self.stopping:
end = True
pass
# Sleep until we're supposed to wake up and generate more events
self.countdown = self.sample.interval
self.executions += 1
# 8/20/15 CS Adding support for ending generation at a certain time
if self.end != None:
# 3/16/16 CS Adding support for ending on a number of executions instead of time
# Should be fine with storing state in this sample object since each sample has it's own unique
# timer thread
if self.endts == None:
if self.executions >= int(self.end):
self.logger.info(
"End executions %d reached, ending generation of sample '%s'"
% (int(self.end), self.sample.name))
self.stopping = True
end = True
elif lt >= self.endts:
self.logger.info(
"End Time '%s' reached, ending generation of sample '%s'"
% (self.sample.endts, self.sample.name))
self.stopping = True
end = True
else:
self.countdown -= self.time
time.sleep(self.time)
