def run_swarming_tasks_parallel(swarming_server, isolate_server, extra_trigger_args, tasks): """Triggers swarming tasks in parallel and gets results. This is done by using one thread per task and shelling out swarming.py. Arguments: extra_trigger_args: list of additional flags to pass down to 'swarming.py trigger' tasks: list of tuple(task_name, isolated_hash, dimensions, caches, env) where dimension are --dimension flags to provide when triggering the task and caches are --named-cache flags. Yields: tuple(name, dimensions, stdout) for the tasks that failed. """ runs = len(tasks) # triger + collect total = 2 * runs failed_tasks = [] progress = threading_utils.Progress([('index', 0), ('size', total)]) progress.use_cr_only = False start = time.time() with threading_utils.ThreadPoolWithProgress(progress, runs, runs, total) as pool: runner = Runner(swarming_server, isolate_server, pool.add_task, progress, extra_trigger_args) for task_name, isolated_hash, dimensions, caches, env in tasks: pool.add_task(0, runner.trigger, task_name, isolated_hash, dimensions, caches, env) # Runner.collect() only return task failures. for failed_task in pool.iter_results(): task_name, dimensions, stdout = failed_task yield task_name, dimensions, stdout failed_tasks.append(task_name) duration = time.time() - start print('\nCompleted in %3.2fs' % duration) if failed_tasks: print('Detected the following failures:') for task in sorted(failed_tasks): print(' %s' % task)
def run_swarm_tests_on_swarm(oses, tests, logs, isolate_server, swarm_server): runs = len(tests) * len(oses) total = 3 * runs columns = [('index', 0), ('size', total)] progress = threading_utils.Progress(columns) progress.use_cr_only = False tempdir = tempfile.mkdtemp(prefix='swarm_client_tests') try: with threading_utils.ThreadPoolWithProgress(progress, runs, runs, total) as pool: start = time.time() runner = Runner(isolate_server, swarm_server, pool.add_task, progress, tempdir) for test in tests: for platform in oses: pool.add_task(0, runner.archive, test, platform) failed_tests = pool.join() duration = time.time() - start print('') finally: shutil.rmtree(tempdir) if logs: os.makedirs(logs) for test, platform, stdout in failed_tests: name = '%s_%s' % (platform, os.path.basename(test)) with open(os.path.join(logs, name + '.log'), 'wb') as f: f.write(stdout) print('Completed in %3.2fs' % duration) if failed_tests: failed_tests_per_os = {} for test, platform, _ in failed_tests: failed_tests_per_os.setdefault(test, []).append(platform) print('Detected the following failures:') for test, platforms in failed_tests_per_os.iteritems(): print(' %s on %s' % (test, ', '.join(sorted(platforms)))) return bool(failed_tests)
def trace_test_cases(cmd, cwd_dir, test_cases, jobs, logname): """Traces each test cases individually but all in parallel.""" assert os.path.isabs(cwd_dir) and os.path.isdir(cwd_dir), cwd_dir if not test_cases: return [] # Resolve any symlink. cwd_dir = os.path.realpath(cwd_dir) assert os.path.isdir(cwd_dir) api = trace_inputs.get_api() api.clean_trace(logname) jobs = jobs or multiprocessing.cpu_count() # Try to do black magic here by guessing a few of the run_test_cases.py # flags. It's cheezy but it works. for i, v in enumerate(cmd): if v.endswith('run_test_cases.py'): # Found it. Process the arguments here. _, options, _ = run_test_cases.process_args(cmd[i:]) # Always override with the lowest value. jobs = min(options.jobs, jobs) break columns = [('index', 0), ('size', len(test_cases))] progress = threading_utils.Progress(columns) with threading_utils.ThreadPoolWithProgress( progress, jobs, jobs, len(test_cases)) as pool: with api.get_tracer(logname) as tracer: function = Tracer(tracer, cmd, cwd_dir, progress).map for test_case in test_cases: pool.add_task(0, function, test_case) results = pool.join() print('') return results
def main(): colorama.init() parser = optparse.OptionParser(description=sys.modules[__name__].__doc__) parser.add_option('-S', '--swarming', metavar='URL', default='', help='Swarming server to use') swarming.add_filter_options(parser) parser.set_defaults(dimensions=[('os', swarming_load_test_bot.OS_NAME)]) group = optparse.OptionGroup(parser, 'Load generated') group.add_option( '-s', '--send-rate', type='float', default=16., metavar='RATE', help='Rate (item/s) of sending requests as a float, default: %default') group.add_option( '-D', '--duration', type='float', default=60., metavar='N', help='Duration (s) of the sending phase of the load test, ' 'default: %default') group.add_option( '-m', '--concurrent', type='int', default=200, metavar='N', help='Maximum concurrent on-going requests, default: %default') group.add_option( '-t', '--timeout', type='float', default=15 * 60., metavar='N', help='Task expiration and timeout to get results, the task itself will ' 'have %ds less than the value provided. Default: %%default' % TIMEOUT_OVERHEAD) group.add_option('-o', '--output-size', type='int', default=100, metavar='N', help='Bytes sent to stdout, default: %default') group.add_option( '--sleep', type='int', default=60, metavar='N', help='Amount of time the bot should sleep, e.g. faking work, ' 'default: %default') parser.add_option_group(group) group = optparse.OptionGroup(parser, 'Display options') group.add_option('--columns', type='int', default=graph.get_console_width(), metavar='N', help='For histogram display, default:%default') group.add_option( '--buckets', type='int', default=20, metavar='N', help='Number of buckets for histogram display, default:%default') parser.add_option_group(group) parser.add_option('--dump', metavar='FOO.JSON', help='Dumps to json file') parser.add_option('-v', '--verbose', action='store_true', help='Enables logging') options, args = parser.parse_args() logging.basicConfig( level=logging.INFO if options.verbose else logging.FATAL) if args: parser.error('Unsupported args: %s' % args) options.swarming = options.swarming.rstrip('/') if not options.swarming: parser.error('--swarming is required.') if options.duration <= 0: parser.error('Needs --duration > 0. 0.01 is a valid value.') swarming.process_filter_options(parser, options) total = int(round(options.send_rate * options.duration)) print( 'Sending %.1f i/s for %ds with max %d parallel requests; timeout %.1fs; ' 'total %d' % (options.send_rate, options.duration, options.concurrent, options.timeout, total)) print('[processing/processed/todo]') # This is used so there's no clash between runs and actual real usage. unique = ''.join(random.choice(string.ascii_letters) for _ in range(8)) columns = [('processing', 0), ('processed', 0), ('todo', 0)] progress = threading_utils.Progress(columns) index = 0 results = [] with threading_utils.ThreadPoolWithProgress(progress, 1, options.concurrent, 0) as pool: try: start = time.time() while True: duration = time.time() - start if duration > options.duration: break should_have_triggered_so_far = int( round(duration * options.send_rate)) while index < should_have_triggered_so_far: pool.add_task(0, trigger_task, options.swarming, options.dimensions, options.sleep, options.output_size, progress, unique, options.timeout, index) progress.update_item('', todo=1) index += 1 progress.print_update() time.sleep(0.01) progress.update_item('Getting results for on-going tasks.', raw=True) for i in pool.iter_results(): results.append(i) # This is a bit excessive but it's useful in the case where some tasks # hangs, so at least partial data is available. if options.dump: results.sort() if os.path.exists(options.dump): os.rename(options.dump, options.dump + '.old') with open(options.dump, 'wb') as f: json.dump(results, f, separators=(',', ':')) if not options.dump: results.sort() except KeyboardInterrupt: aborted = pool.abort() progress.update_item('Got Ctrl-C. Aborted %d unsent tasks.' % aborted, raw=True, todo=-aborted) progress.print_update() progress.print_update() # At this point, progress is not used anymore. print('') print(' - Took %.1fs.' % (time.time() - start)) print('') print_results(results, options.columns, options.buckets) return 0
def main(): colorama.init() parser = optparse.OptionParser(description=sys.modules[__name__].__doc__) parser.add_option( '-S', '--swarming', metavar='URL', default='', help='Swarming server to use') swarming.add_filter_options(parser) # Use improbable values to reduce the chance of interferring with real slaves. parser.set_defaults( dimensions=[ ('bits', '36'), ('machine', os.uname()[4] + '-experimental'), ('os', OS_NAME), ]) group = optparse.OptionGroup(parser, 'Load generated') group.add_option( '--slaves', type='int', default=300, metavar='N', help='Number of swarm bot slaves, default: %default') group.add_option( '-c', '--consume', type='float', default=60., metavar='N', help='Duration (s) for consuming a request, default: %default') parser.add_option_group(group) group = optparse.OptionGroup(parser, 'Display options') group.add_option( '--columns', type='int', default=graph.get_console_width(), metavar='N', help='For histogram display, default:%default') group.add_option( '--buckets', type='int', default=20, metavar='N', help='Number of buckets for histogram display, default:%default') parser.add_option_group(group) parser.add_option( '--dump', metavar='FOO.JSON', help='Dumps to json file') parser.add_option( '-v', '--verbose', action='store_true', help='Enables logging') options, args = parser.parse_args() logging.basicConfig(level=logging.INFO if options.verbose else logging.FATAL) if args: parser.error('Unsupported args: %s' % args) options.swarming = options.swarming.rstrip('/') if not options.swarming: parser.error('--swarming is required.') if options.consume <= 0: parser.error('Needs --consume > 0. 0.01 is a valid value.') swarming.process_filter_options(parser, options) print( 'Running %d slaves, each task lasting %.1fs' % ( options.slaves, options.consume)) print('Ctrl-C to exit.') print('[processing/processed/bots]') columns = [('processing', 0), ('processed', 0), ('bots', 0)] progress = threading_utils.Progress(columns) events = Queue.Queue() start = time.time() kill_event = threading.Event() swarm_bot_version_hash = calculate_version( options.swarming + '/get_slave_code') slaves = [ FakeSwarmBot( options.swarming, options.dimensions, swarm_bot_version_hash, i, progress, options.consume, events, kill_event) for i in range(options.slaves) ] try: # Wait for all the slaves to come alive. while not all(s.is_alive() for s in slaves): time.sleep(0.01) progress.update_item('Ready to run') while slaves: progress.print_update() time.sleep(0.01) # The slaves could be told to die. slaves = [s for s in slaves if s.is_alive()] except KeyboardInterrupt: kill_event.set() progress.update_item('Waiting for slaves to quit.', raw=True) progress.update_item('') while slaves: progress.print_update() slaves = [s for s in slaves if s.is_alive()] # At this point, progress is not used anymore. print('') print('Ran for %.1fs.' % (time.time() - start)) print('') results = events.queue print_results(results, options.columns, options.buckets) if options.dump: with open(options.dump, 'w') as f: json.dump(results, f, separators=(',',':')) return 0
def main(): colorama.init() parser = optparse.OptionParser(description=sys.modules[__name__].__doc__) parser.add_option('-S', '--swarming', metavar='URL', default='', help='Swarming server to use') swarming.add_filter_options(parser) parser.set_defaults(dimensions=[('os', swarming_load_test_bot.OS_NAME)]) group = optparse.OptionGroup(parser, 'Load generated') group.add_option( '-s', '--send-rate', type='float', default=16., metavar='RATE', help='Rate (item/s) of sending requests as a float, default: %default') group.add_option( '-D', '--duration', type='float', default=60., metavar='N', help='Duration (s) of the sending phase of the load test, ' 'default: %default') group.add_option( '-m', '--concurrent', type='int', default=200, metavar='N', help='Maximum concurrent on-going requests, default: %default') group.add_option('-t', '--timeout', type='float', default=3600., metavar='N', help='Timeout to get results, default: %default') parser.add_option_group(group) group = optparse.OptionGroup(parser, 'Display options') group.add_option('--columns', type='int', default=graph.get_console_width(), metavar='N', help='For histogram display, default:%default') group.add_option( '--buckets', type='int', default=20, metavar='N', help='Number of buckets for histogram display, default:%default') parser.add_option_group(group) parser.add_option('--dump', metavar='FOO.JSON', help='Dumps to json file') parser.add_option('-v', '--verbose', action='store_true', help='Enables logging') options, args = parser.parse_args() logging.basicConfig( level=logging.INFO if options.verbose else logging.FATAL) if args: parser.error('Unsupported args: %s' % args) options.swarming = options.swarming.rstrip('/') if not options.swarming: parser.error('--swarming is required.') if options.duration <= 0: parser.error('Needs --duration > 0. 0.01 is a valid value.') swarming.process_filter_options(parser, options) total = options.send_rate * options.duration print( 'Sending %.1f i/s for %ds with max %d parallel requests; timeout %.1fs; ' 'total %d' % (options.send_rate, options.duration, options.concurrent, options.timeout, total)) print('[processing/processed/todo]') # This is used so there's no clash between runs and actual real usage. unique = ''.join(random.choice(string.ascii_letters) for _ in range(8)) columns = [('processing', 0), ('processed', 0), ('todo', 0)] progress = threading_utils.Progress(columns) index = 0 with threading_utils.ThreadPoolWithProgress(progress, 1, options.concurrent, 0) as pool: try: start = time.time() while True: duration = time.time() - start if duration > options.duration: break should_have_triggered_so_far = int(duration * options.send_rate) while index < should_have_triggered_so_far: pool.add_task(0, trigger_task, options.swarming, options.dimensions, progress, unique, options.timeout, index) progress.update_item('', todo=1) index += 1 progress.print_update() time.sleep(0.01) except KeyboardInterrupt: aborted = pool.abort() progress.update_item('Got Ctrl-C. Aborted %d unsent tasks.' % aborted, raw=True, todo=-aborted) progress.print_update() finally: # TODO(maruel): We could give up on collecting results for the on-going # tasks but that would need to be optional. progress.update_item('Getting results for on-going tasks.', raw=True) results = sorted(pool.join()) progress.print_update() # At this point, progress is not used anymore. print('') print(' - Took %.1fs.' % (time.time() - start)) print('') print_results(results, options.columns, options.buckets) if options.dump: with open(options.dump, 'w') as f: json.dump(results, f, separators=(',', ':')) return 0