def loadProps():
d = dict(properties)
# For testing purpose
if isinstance(grinder, MockGrinder):
d.update({
'grinder.threads': 1,
'grinder.runNumber': 0,
'grinder.threadNumber': 0,
'grinder.processNumber': 0,
'AGENT': 0,
'RUN': 0,
'THREAD': 0,
'PROCESS': 0
})
return d
if grinder:
d.update({
'grinder.hostID': get_hostID(),
'grinder.runNumber': grinder.getRunNumber(),
'grinder.threadNumber': grinder.getThreadNumber(),
'grinder.processNumber': CoreGrinder.getRealProcessNumber(),
'grinder.agentNumber': '%02d' % (grinder.getAgentNumber()),
'HOST': socket.gethostname().split('.')[0],
'HOSTID': get_hostID(),
'AGENTID': get_agentId(get_hostID()),
'AGENT': '%02d' % (grinder.getAgentNumber()),
'RUN': grinder.getRunNumber(),
'THREAD': grinder.getThreadNumber(),
'PROCESS': CoreGrinder.getRealProcessNumber()
})
return d
def initClass(cls, strArgs):
cls.manageArgs(strArgs)
#========================
# TEMPORY PUT IN COMMENTS
# cls.checkArgs()
#========================
# to bypass (seek) N lines of the file
cls.startFrom = properties.getInt(
'throughput_start_from%d' % (CoreGrinder.getRealProcessNumber()),
0)
if cls.startFrom:
logger.info('We will ignore the first %d lines' % (cls.startFrom))
# The chunkGenerator will allow to minimize memory usage
cls.chunkGenerator = cls.getNextChunkFromFile()
cls.rows = cls.chunkGenerator.next()
def __manageClientReporter(cls):
logger.info('Reporting activation: %s' % (Configuration.use_reporter))
if Configuration.use_reporter:
#
# agentName: grinder.hostID is the standard way to set a naming for a specific agent
# by default, if we have 2 agents, the first one will get hostname-0, the second hostname-1 ...
# Warning: sometimes we get the full domain name separated with dot. (so we split and keep the first)
#
agentName = properties.get(
'grinder.hostID') or socket.gethostname().split('.')[0]
#
# report_show_process=True : you want to have all the per process metrics in the graphing tool
# report_show_process=False : you have metrics per agent (or hostname)
#
location = '%s.%d' % (agentName, CoreGrinder.getRealProcessNumber(
)) if properties.getBoolean('reporter_show_process',
False) else agentName
reporter_tool_name = properties.get('reporter_tool')
reporter_tool = (reporter_tool_name
or '').lower() or 'centralreporter'
reporter_target = {
'centralreporter': CentralReporterClient,
'carbonreporter': CarbonCacheClient,
'statsd': StatsdClient
}
reporterModule = None
if reporter_tool not in reporter_target:
try:
reporterModule = __import__(
'corelibs.stats.%s' % reporter_tool_name, globals(),
locals(), ['%s' % reporter_tool_name], -1)
except (Exception, JavaException), e:
logger.error(
'FAILED invalid property reporter_tool [corelibs.stats.%s], failed with reason: [%s]'
% (reporter_tool_name, e))
if isinstance(e, JavaException):
raise Exception(e)
raise
reporterHost = properties.get('reporter_host') or 'localhost'
reporterPort = properties.getInt('reporter_port', 1901)
# all the ordered testnames of all the scenarios
testnames = [
line.testName for scenario in cls.scenarioList.getList()
for line in scenario.lines
]
if reporter_tool == 'centralreporter':
# remove duplicates from the testname list
testnames = list(set(testnames))
try:
cls.clientReporter = CentralReporterClient(
reporterHost, reporterPort, location, testnames)
except:
logger.error(
'[reporter=%s][host=%s][port=%d][location=%s][testnames=%s]'
% (reporter_tool, reporterHost, reporterPort, location,
testnames))
elif reporter_tool == 'statsd':
try:
# reporter_aggregate_value aimed at grouping values (the machine location is absent)
# test names are indexed by the the process number index (process0=test0, ... )
location = '' if properties.getBoolean(
'reporter_aggregate_value', False) else '%s' % location
cls.clientReporter = StatsdClient(reporterHost,
reporterPort, location)
except Exception, e:
logger.error(
'statsd reporter - Exception=%s\n[reporter=%s][host=%s][port=%d][location=%s]'
% (str(e), reporter_tool, reporterHost, reporterPort,
location))
raise RuntimeError(
'Exception=%s\nUnable to start the statsd reporter ([reporter=%s][host=%s][port=%d][location=%s])'
% (str(e), reporter_tool, reporterHost, reporterPort,
location))
class Configuration:
# Some required declaration
dataFilePath = properties.get('dataFilePath')
templateFilePath = properties.get('templateFilePath')
cmdMgr = None
outTraceActivated = False
fOut = None
fOutExtra = None
cv_out = None
cv_ext = None
extraLevel = False
# get the delay time between commands
interMessageDelay = properties.getDouble('messageDelay', 0.0)
initialSleepTime = properties.getDouble('grinder.initialSleepTime', 0.0)
#
# because it is possible that each process (and therefore, each test_thread)
# could be kicked off numerous times by the console without exiting, get the total number of threads
# and processes, then modulo divide the process and test_thread number with the total (respective) number
# this will correctly create the correct file name (file.X.Y) on multiple runs
#
numberOfThreads = properties.getInt('grinder.threads', 0)
# numberOfRuns = properties.getInt('grinder.runs', 0)
asyncruns = 0
numberOfProcess = properties.getInt('grinder.processes', 0)
# When relaunching several times, the process number is incremented
processNumber = CoreGrinder.getRealProcessNumber()
processNumberPadding = properties.getInt('processNumberPadding', 2)
threadNumberPadding = properties.getInt('threadNumberPadding', 4)
runNumberPadding = properties.getInt('runNumberPadding', 7)
# ID format never changes, retrieve it once and for all.
idFormat = toolbox.getidFormat()
processIdFormat = '%%0%dd' % (processNumberPadding)
runIDPadding = toolbox.getRunIDPadding()
runIdFormat = '%%0%dd' % (runNumberPadding)
threadIdFormat = '%%0%dd' % (threadNumberPadding)
grinderStopScenarioOnError = properties.getBoolean(
'stop.scenario.on.error', properties.getBoolean('stopOnError', True))
#
# if displayReadResponse is set to 'True', then display:
# the first 256 bytes of the response received from the command
#
displayReadResponse = properties.getBoolean('displayReadResponse', False)
displayReadResponseMaxLength = properties.getInt(
'displayReadResponseMaxLength', 1024)
# The way inFile (scenario) are set to process, thread for different UC processing
oneFileByThread = properties.getBoolean(
'oneFileByThread',
properties.getBoolean('grindertool.test.scenarioPerThread', False))
oneFileByProcess = properties.getBoolean(
'oneFileByProcess',
properties.getBoolean('grindertool.test.scenarioPerProcess', False))
oneSingleList = not (oneFileByProcess or oneFileByThread)
shortFileName = properties.get('grinder.console.propertiesFile') or ''
if shortFileName:
shortFileName = shortFileName.split(os.sep)[-1]
cacheProtocols = properties.getBoolean('cacheProtocols', False)
# For Asynchronous management
async = False
routerContextClient = None
# Http server callback for asynchronous flows
http_graceful_period = properties.getInt('grindertool.http.stop.graceful',
5000)
async = False
pureThreadMode = False
validationMode = False
scenarioList = None
waiting_mode = False
# SmscDriver start if required
smpp_started = properties.getBoolean('grindertool.smsc.start', False)
# graceful period before stopping Smsc server (5 seconds by default)
smpp_graceful_period = properties.getInt('grindertool.smsc.stop.graceful',
5)
smscDriver = None
use_reporter = properties.getBoolean('reporter_activate', False)
clientReporter = None
metronom = None
monitor = None
use_throughput = properties.getBoolean('throughput_activate', False)
if numberOfThreads == 1:
properties.setBoolean('throughput_activate', False)
use_throughput = False
use_regulator = properties.getBoolean('regulator_activate', False)
listener_host = 'localhost'
listener_port = 9080
listener_poolSize = 32
listener_socketBacklog = 1024
# TODO
#-----------
# Add the grinder.threads as a Gauge
# Add the thread maximum from the parameter grindertool.threads.threshold_pct_active if thread_control_enable=True
#
# Active Session throttling
thread_control_enable = False
threshold_thread_active = 100
thread_wait_milli = 10
@classmethod
def initialize(cls):
logger.info(
"********************* STARTING INITIALIZATION ***********************"
)
# Check some parameters
cls.checkParameters()
# Immutable scenario loading and flags relative to the different mode
cls.manageScenarioAndFlags()
# Asynchronous configuration
if cls. async:
cls.__manageAsyncConfiguration()
# Asynchronous callback listener - must be started after the asynchronous configuration
cls.__startListener()
# Initialize the SMSCToolkit and wait after connection only for pureThreadMode
cls.__manageSMSCToolkit()
# Reporting monitoring data
cls.__manageClientReporter()
# Ramping up mode management & flow controller mode
cls.manageThroughput()
# thread throttling
cls.activateSessionThrottler()
logger.info(
"********************* END OF INITIALIZATION ***********************"
)
@classmethod
def activateSessionThrottler(cls):
# This is a throughput throttling mechanism
#-----------------------------------------------------
# Threshold of active threads
# thread throttling enable (True/False)
cls.thread_control_enable = properties.getBoolean(
'grindertool.threads.control_enable', False)
if cls.thread_control_enable:
cls.threshold_thread_active = int(
(properties.getInt('grindertool.threads.threshold_pct_active',
100) or 100) / 100) * cls.numberOfThreads
if cls.threshold_thread_active > cls.numberOfThreads:
raise SyntaxError(
'Number of threshold thread (%d) cannot be above number of grinder threads (%d)'
% (cls.threshold_thread_active, cls.numberOfThreads))
# When over threshold, time sleeping doing nothing
cls.thread_wait_milli = properties.getInt(
'grindertool.threads.sleep', 10)
logger.info(
'[CONTROL] Switching to controlled rampup ( [Threshold=%d][Threads=%d][Sleep=%d] ) ...'
% (cls.threshold_thread_active, cls.numberOfThreads,
cls.thread_wait_milli))
print(
'\t[CONTROL] Switching to controlled rampup ( [Threshold=%d][Threads=%d][Sleep=%d] ) ...'
% (cls.threshold_thread_active, cls.numberOfThreads,
cls.thread_wait_milli))
@classmethod
def checkParameters(cls):
# TODO : should be part of the scenario immutable object
# The template (payload) manager is a global instance
cls.cmdMgr = command.TemplateManager(cls.templateFilePath)
# Output trace for validation
cls.outTraceActivated = properties.getBoolean('outTraceActivated',
False)
if cls.outTraceActivated:
outFile = '%s%s%s.%s' % (properties.get('logFilePath')
or cls.dataFilePath, os.sep,
properties.get('fileout')
or 'default', toolbox.getFileTimeStamp())
cls.fOut = file(outFile, 'w')
logger.info('outFile "%s"' % outFile)
cls.extraLevel = properties.getBoolean('extraLevel', False)
if cls.extraLevel:
cls.fOutExtra = file('%s.EXTRA' % outFile, 'w')
logger.info('outExtraFile "%s"' % cls.fOutExtra)
# file lock for output and extra logging files
cls.cv_out = Condition()
cls.cv_ext = Condition()
@classmethod
def manageThroughput(cls):
# Throughput mode
logger.info('Throughput_mode: %s' % (str(cls.use_throughput)))
if (cls.use_regulator):
cls.use_throughput = True
Configuration.__launchRegulator(cls.clientReporter)
else:
# Thoughtput mode
if cls.use_throughput:
Configuration.__launchMetronom()
properties.setInt('grinder.runs', 0)
if cls.use_throughput:
logger.info(
'[throughput_mode activated, so forcing parameter grinder.runs=0'
)
properties.setInt('grinder.runs', 0)
if cls.numberOfThreads < 16:
logger.info(
'[throughput_mode activated, number of threads [current=%d] should be at at least greater than 16, setting to 16'
% (cls.numberOfThreads))
properties.setInt('grinder.threads', 16)
@classmethod
def manageScenarioAndFlags(cls):
cls.scenarioList = ScenarioList(cls)
cls. async = cls.scenarioList.isAsync()
# ValidationMode : in that case, the thread wait after the asynchronous operation termination
if cls. async:
if cls.numberOfThreads == 1 or cls.oneFileByThread or properties.getBoolean(
'grindertool.forceValidationMode', False):
cls.validationMode = True
cls.use_throughput = False
cls.graceful_async = 0
if cls. async:
cls.graceful_async = properties.getInt(
'grindertool.graceful_async_time', 5000)
# optimization for pure threading mode
cls.waiting_mode = properties.getBoolean('waiting_mode', False)
cls.pureThreadMode = not cls.use_throughput and not cls. async
from corelibs import command
from corelibs.contextCallback import HTTPServerCallback
from corelibs.contextIdentifier import ContextIdentifier
from corelibs.contextRouterClient import ContextRouterClient
from corelibs.metronom import Metronom
from corelibs.monitor import Monitor
from corelibs.grinderQueue import GrinderQueue
from corelibs.scenarioList import ScenarioList
from corelibs.stats.CarbonCacheClient import CarbonCacheClient
from corelibs.stats.CentralReporterClient import CentralReporterClient
from corelibs.stats.StatsdClient import StatsdClient
import toolbox
from corelibs.contextLock import ContextLock
grinder = CoreGrinder.getGrinder()
properties = CoreGrinder.getProperty()
logger = CoreGrinder.getLogger()
try:
from corelibs.SmscDriver import SMSCDriver
except ImportError, e:
if properties.getBoolean('grindertool.smsc.start', False):
logger.error(
'FATAL - are the libraries located in grindertool.smsc.libs loaded ?'
)
raise e
class Configuration:
'''
Created on 30 juil. 2014
@author: omerlin
'''
from __future__ import with_statement
from threading import Condition, Thread
import time
from corelibs.asynclog import asynclog
from corelibs.contextLock import ContextLock
from corelibs.coreGrinder import CoreGrinder
logger=CoreGrinder.getLogger()
'''
A context storage base on a hash of hash
first level is the identifier (contextKey)
The second level key is the value of the contextKey.
The second level value is a Context() object
'''
class ContextIdentifier:
hash={}
mutex=Condition()
reaperThread = None
stopReaperThread=False
threadStarted=False
