def __init__(self, num_runs, first, num_requests, increment, condition): self.db = ProfilerDatabase() self.num_runs = num_runs self.first = first self.num_requests = num_requests self.increment = increment self.condition = condition self.protocol = settings.PROFILER.get('MY_SITE_PROTOCOL', 'http') self.host = settings.PROFILER.get('MY_SITE_HOST', 'localhost') self.port = settings.PROFILER.get('MY_SITE_PORT', '') self.base_url = '%s://%s' % (self.protocol, self.host) if self.port: self.base_url += ':%s' % self.port self.log_dir = settings.PROFILER.get('LOG_DIR', 'profiling_logs') self.project_root = getattr(settings, 'PROJECT_ROOT') self.profiler_tests = [] self.profiler_teardown_methods = {} backend_methods = inspect.getmembers(self, inspect.ismethod) for method in backend_methods: if method[0].startswith('profile_'): self.profiler_tests.append(method) if method[0].startswith('teardown_'): self.profiler_teardown_methods[method[0]] = method self.previous_values = {}
def run(self): db = ProfilerDatabase() logger.debug('Process %s started' % self.name) while True: function = self.queue.get() if function is not None: try: function_stats = db.get_function_stats(function['id']) except Exception, e: print str(e) else: self._plot_function_stats(function, function_stats) self.queue.task_done() # Poison pill means shutdown else: self.queue.task_done() break
class StatsPlotter(object): def __init__(self): self.db = ProfilerDatabase() self.workers = [] def plot(self, group_id, dir_name): functions = self.db.get_functions(group_id) profiler_group_run = self.db.get_profiler_group_run(group_id) x_label = profiler_group_run['x_label'] num_profiler_runs = profiler_group_run['num_profiler_runs'] tasks = multiprocessing.JoinableQueue() num_workers = multiprocessing.cpu_count() * 2 # Start workers for i in xrange(num_workers): w = PlotWorker(tasks, dir_name, x_label, num_profiler_runs) self.workers.append(w) w.start() # Enqueue jobs for function in functions: data = {'id': function['id'], 'module': function['module'], 'name': function['name'], 'line': function['line']} tasks.put(data) # Add a poison pill for each worker for i in xrange(num_workers): tasks.put(None) # Wait for all the tasks to finish logger.debug('Waiting for worker processes') tasks.join() def stop(self): logger.debug('\nTerminating processes') for w in self.workers: w.terminate() w.join() logger.debug('%s terminated' % w.name)
def handle_noargs(self, **options): db = ProfilerDatabase() profiler_group_runs = db.get_profiler_group_runs() prompt_str = 'Which profiler run do you want to plot?\n' for index, profiler_group__run in enumerate(profiler_group_runs): prompt_str += '(%i) %s - (started: %s)\n' % (index, profiler_group__run['name'], profiler_group__run['started']) prompt_str += '\nType your choice: ' profiler_group_run_index = int(raw_input(prompt_str)) profiler_group_run = profiler_group_runs[profiler_group_run_index] try: start_time = time.time() dir_name = '%s_%s' % (profiler_group_run['name'], profiler_group_run['started']) plotter = StatsPlotter() plotter.plot(profiler_group_run['id'], dir_name) except KeyboardInterrupt: plotter.stop() self.stdout.write('\nPlotting stopped.\n') except Exception, e: raise CommandError(e)
def __init__(self): self.db = ProfilerDatabase() self.workers = []
class DefaultProfiler(object): def __init__(self, num_runs, first, num_requests, increment, condition): self.db = ProfilerDatabase() self.num_runs = num_runs self.first = first self.num_requests = num_requests self.increment = increment self.condition = condition self.protocol = settings.PROFILER.get('MY_SITE_PROTOCOL', 'http') self.host = settings.PROFILER.get('MY_SITE_HOST', 'localhost') self.port = settings.PROFILER.get('MY_SITE_PORT', '') self.base_url = '%s://%s' % (self.protocol, self.host) if self.port: self.base_url += ':%s' % self.port self.log_dir = settings.PROFILER.get('LOG_DIR', 'profiling_logs') self.project_root = getattr(settings, 'PROJECT_ROOT') self.profiler_tests = [] self.profiler_teardown_methods = {} backend_methods = inspect.getmembers(self, inspect.ismethod) for method in backend_methods: if method[0].startswith('profile_'): self.profiler_tests.append(method) if method[0].startswith('teardown_'): self.profiler_teardown_methods[method[0]] = method self.previous_values = {} def start_profiling(self, test_index): profiler_test = self.profiler_tests[test_index] profiler_group_run_name = profiler_test[0].replace('profile_', '') profiler_group_run_id = self.db.start_profiler_group_run(profiler_group_run_name, self.num_runs, self.first, self.num_requests, self.increment) # Repoze profiler does not profile the first request if self._first_request(): # After server restart, the python interpreter does some initialization logger.debug('Starting initialization call') successful = profiler_test[1](1) if successful: logger.debug('Initialization call was successful') logger.debug('Starting initialization call teardown') try: teardown_method_name = 'teardown_%s' % profiler_group_run_name self.profiler_teardown_methods[teardown_method_name][1](1) except KeyError: logger.error('No teardown method named %s' % teardown_method_name) logger.debug('Initialization call teardown finished') for i in range(self.num_runs): #self._reset_profilers() logger.debug('Started profiler run %i' % (i + 1)) profiler_run_id = self.db.start_profiler_run(profiler_group_run_id) num_ok = 0 for j in range(1, self.num_requests + 1): self._reset_profilers() logger.debug('Starting profiling call %i' % j) if j >= self.first and (j % self.increment) == 0: request_id = self.db.start_request(profiler_run_id, j) successful = profiler_test[1](j) if successful: num_ok += 1 logger.debug('Profiling call %i was successful' % j) date = datetime.now() ended = date.strftime('%Y-%m-%d %H:%M:%S') if j >= self.first and (j % self.increment) == 0: self.db.stop_request(request_id, successful, ended=ended) self._save_stats(profiler_group_run_id, request_id) else: with self.condition: while True: lock_files = glob.glob(os.path.join(self.log_dir, '*.lock')) if len(lock_files) == 0: break self.condition.wait() logger.debug('%i/%i profiling calls finished successfully' % (num_ok, self.num_requests)) logger.debug('Starting teardown') try: teardown_method_name = 'teardown_%s' % profiler_group_run_name self.profiler_teardown_methods[teardown_method_name][1](self.num_requests) except KeyError: logger.error('No teardown method named %s' % teardown_method_name) logger.debug('Teardown finished') self.db.stop_profiler_run(profiler_run_id) logger.debug('Ended profiler run %i' % (i + 1)) #self._add_devices(profiler_run_id, self.num_requests) #self._add_devices_and_interfaces(profiler_run_id, self.num_requests) else: logger.error('Error in first request') self.db.stop_profiler_group_run(profiler_group_run_id) def get_tests(self): tests = [] for test in self.profiler_tests: tests.append(test[0].replace('profile_', '')) return tests def _first_request(self): pass def _start_request(self, profiler_run_id, request_num): request_id = self.db.start_request(profiler_run_id, request_num) return request_id def _reset_profilers(self): # Reset the repoze.profiler path = '/__profile__' url = self.base_url + path payload = {'clear': 'Clear'} response = requests.post(url, data=payload) if response.status_code != 200: logger.error('Resetting repoze.profiler failed') else: logger.debug('Repoze.profiler resetted') # Reset the profiler decorators for filename in os.listdir(self.log_dir): if filename.endswith('.prof'): file_path = os.path.join(self.log_dir, filename) try: if os.path.isfile(file_path): os.remove(file_path) except Exception, e: logger.error('Could not reset profile decorators') logger.debug('Profile decorators resetted') logger.debug('Profiler resetted')