def __init__(self, num_runs, first, num_requests, increment, condition):
self.db = ProfilerDatabase()
self.num_runs = num_runs
self.first = first
self.num_requests = num_requests
self.increment = increment
self.condition = condition
self.protocol = settings.PROFILER.get('MY_SITE_PROTOCOL', 'http')
self.host = settings.PROFILER.get('MY_SITE_HOST', 'localhost')
self.port = settings.PROFILER.get('MY_SITE_PORT', '')
self.base_url = '%s://%s' % (self.protocol, self.host)
if self.port:
self.base_url += ':%s' % self.port
self.log_dir = settings.PROFILER.get('LOG_DIR', 'profiling_logs')
self.project_root = getattr(settings, 'PROJECT_ROOT')
self.profiler_tests = []
self.profiler_teardown_methods = {}
backend_methods = inspect.getmembers(self, inspect.ismethod)
for method in backend_methods:
if method[0].startswith('profile_'):
self.profiler_tests.append(method)
if method[0].startswith('teardown_'):
self.profiler_teardown_methods[method[0]] = method
self.previous_values = {}
def run(self):
db = ProfilerDatabase()
logger.debug('Process %s started' % self.name)
while True:
function = self.queue.get()
if function is not None:
try:
function_stats = db.get_function_stats(function['id'])
except Exception, e:
print str(e)
else:
self._plot_function_stats(function, function_stats)
self.queue.task_done()
# Poison pill means shutdown
else:
self.queue.task_done()
break
class StatsPlotter(object):
def __init__(self):
self.db = ProfilerDatabase()
self.workers = []
def plot(self, group_id, dir_name):
functions = self.db.get_functions(group_id)
profiler_group_run = self.db.get_profiler_group_run(group_id)
x_label = profiler_group_run['x_label']
num_profiler_runs = profiler_group_run['num_profiler_runs']
tasks = multiprocessing.JoinableQueue()
num_workers = multiprocessing.cpu_count() * 2
# Start workers
for i in xrange(num_workers):
w = PlotWorker(tasks, dir_name, x_label, num_profiler_runs)
self.workers.append(w)
w.start()
# Enqueue jobs
for function in functions:
data = {'id': function['id'], 'module': function['module'], 'name': function['name'], 'line': function['line']}
tasks.put(data)
# Add a poison pill for each worker
for i in xrange(num_workers):
tasks.put(None)
# Wait for all the tasks to finish
logger.debug('Waiting for worker processes')
tasks.join()
def stop(self):
logger.debug('\nTerminating processes')
for w in self.workers:
w.terminate()
w.join()
logger.debug('%s terminated' % w.name)
def handle_noargs(self, **options):
db = ProfilerDatabase()
profiler_group_runs = db.get_profiler_group_runs()
prompt_str = 'Which profiler run do you want to plot?\n'
for index, profiler_group__run in enumerate(profiler_group_runs):
prompt_str += '(%i) %s - (started: %s)\n' % (index, profiler_group__run['name'], profiler_group__run['started'])
prompt_str += '\nType your choice: '
profiler_group_run_index = int(raw_input(prompt_str))
profiler_group_run = profiler_group_runs[profiler_group_run_index]
try:
start_time = time.time()
dir_name = '%s_%s' % (profiler_group_run['name'], profiler_group_run['started'])
plotter = StatsPlotter()
plotter.plot(profiler_group_run['id'], dir_name)
except KeyboardInterrupt:
plotter.stop()
self.stdout.write('\nPlotting stopped.\n')
except Exception, e:
raise CommandError(e)
def __init__(self):
self.db = ProfilerDatabase()
self.workers = []
class DefaultProfiler(object):
def __init__(self, num_runs, first, num_requests, increment, condition):
self.db = ProfilerDatabase()
self.num_runs = num_runs
self.first = first
self.num_requests = num_requests
self.increment = increment
self.condition = condition
self.protocol = settings.PROFILER.get('MY_SITE_PROTOCOL', 'http')
self.host = settings.PROFILER.get('MY_SITE_HOST', 'localhost')
self.port = settings.PROFILER.get('MY_SITE_PORT', '')
self.base_url = '%s://%s' % (self.protocol, self.host)
if self.port:
self.base_url += ':%s' % self.port
self.log_dir = settings.PROFILER.get('LOG_DIR', 'profiling_logs')
self.project_root = getattr(settings, 'PROJECT_ROOT')
self.profiler_tests = []
self.profiler_teardown_methods = {}
backend_methods = inspect.getmembers(self, inspect.ismethod)
for method in backend_methods:
if method[0].startswith('profile_'):
self.profiler_tests.append(method)
if method[0].startswith('teardown_'):
self.profiler_teardown_methods[method[0]] = method
self.previous_values = {}
def start_profiling(self, test_index):
profiler_test = self.profiler_tests[test_index]
profiler_group_run_name = profiler_test[0].replace('profile_', '')
profiler_group_run_id = self.db.start_profiler_group_run(profiler_group_run_name, self.num_runs, self.first, self.num_requests, self.increment)
# Repoze profiler does not profile the first request
if self._first_request():
# After server restart, the python interpreter does some initialization
logger.debug('Starting initialization call')
successful = profiler_test[1](1)
if successful:
logger.debug('Initialization call was successful')
logger.debug('Starting initialization call teardown')
try:
teardown_method_name = 'teardown_%s' % profiler_group_run_name
self.profiler_teardown_methods[teardown_method_name][1](1)
except KeyError:
logger.error('No teardown method named %s' % teardown_method_name)
logger.debug('Initialization call teardown finished')
for i in range(self.num_runs):
#self._reset_profilers()
logger.debug('Started profiler run %i' % (i + 1))
profiler_run_id = self.db.start_profiler_run(profiler_group_run_id)
num_ok = 0
for j in range(1, self.num_requests + 1):
self._reset_profilers()
logger.debug('Starting profiling call %i' % j)
if j >= self.first and (j % self.increment) == 0:
request_id = self.db.start_request(profiler_run_id, j)
successful = profiler_test[1](j)
if successful:
num_ok += 1
logger.debug('Profiling call %i was successful' % j)
date = datetime.now()
ended = date.strftime('%Y-%m-%d %H:%M:%S')
if j >= self.first and (j % self.increment) == 0:
self.db.stop_request(request_id, successful, ended=ended)
self._save_stats(profiler_group_run_id, request_id)
else:
with self.condition:
while True:
lock_files = glob.glob(os.path.join(self.log_dir, '*.lock'))
if len(lock_files) == 0:
break
self.condition.wait()
logger.debug('%i/%i profiling calls finished successfully' % (num_ok, self.num_requests))
logger.debug('Starting teardown')
try:
teardown_method_name = 'teardown_%s' % profiler_group_run_name
self.profiler_teardown_methods[teardown_method_name][1](self.num_requests)
except KeyError:
logger.error('No teardown method named %s' % teardown_method_name)
logger.debug('Teardown finished')
self.db.stop_profiler_run(profiler_run_id)
logger.debug('Ended profiler run %i' % (i + 1))
#self._add_devices(profiler_run_id, self.num_requests)
#self._add_devices_and_interfaces(profiler_run_id, self.num_requests)
else:
logger.error('Error in first request')
self.db.stop_profiler_group_run(profiler_group_run_id)
def get_tests(self):
tests = []
for test in self.profiler_tests:
tests.append(test[0].replace('profile_', ''))
return tests
def _first_request(self):
pass
def _start_request(self, profiler_run_id, request_num):
request_id = self.db.start_request(profiler_run_id, request_num)
return request_id
def _reset_profilers(self):
# Reset the repoze.profiler
path = '/__profile__'
url = self.base_url + path
payload = {'clear': 'Clear'}
response = requests.post(url, data=payload)
if response.status_code != 200:
logger.error('Resetting repoze.profiler failed')
else:
logger.debug('Repoze.profiler resetted')
# Reset the profiler decorators
for filename in os.listdir(self.log_dir):
if filename.endswith('.prof'):
file_path = os.path.join(self.log_dir, filename)
try:
if os.path.isfile(file_path):
os.remove(file_path)
except Exception, e:
logger.error('Could not reset profile decorators')
logger.debug('Profile decorators resetted')
logger.debug('Profiler resetted')
