def main(): fix_encoding.fix_encoding() verbose = '-v' in sys.argv leak = bool('--leak' in sys.argv) if leak: sys.argv.remove('--leak') if verbose: logging.basicConfig(level=logging.INFO) Test.maxDiff = None else: logging.basicConfig(level=logging.ERROR) # Force language to be English, otherwise the error messages differ from # expectations. os.environ['LANG'] = 'en_US.UTF-8' os.environ['LANGUAGE'] = 'en_US.UTF-8' # So that we don't get goofed up when running this test on swarming :) os.environ.pop('SWARMING_TASK_ID', None) bot = None client = None servers = None failed = True try: servers = start_servers.LocalServers(False) servers.start() bot = start_bot.LocalBot(servers.swarming_server.url) Test.bot = bot bot.start() client = SwarmingClient(servers.swarming_server.url, servers.isolate_server.url) # Test cases only interract with the client; except for test_update_continue # which mutates the bot. Test.client = client Test.servers = servers failed = not unittest.main(exit=False).result.wasSuccessful() # Then try to terminate the bot sanely. After the terminate request # completed, the bot process should have terminated. Give it a few # seconds due to delay between sending the event that the process is # shutting down vs the process is shut down. if client.terminate(bot.bot_id) is not 0: print >> sys.stderr, 'swarming.py terminate failed' failed = True try: bot.wait(10) except subprocess42.TimeoutExpired: print >> sys.stderr, 'Bot is still alive after swarming.py terminate' failed = True except KeyboardInterrupt: print >> sys.stderr, '<Ctrl-C>' failed = True if bot is not None and bot.poll() is None: bot.kill() bot.wait() finally: cleanup(bot, client, servers, failed or verbose, leak) return int(failed)
def main(): if os.getenv('CHROME_REMOTE_DESKTOP_SESSION') == '1': # Disable itself when run under Google Chrome Remote Desktop, as it's # normally started at the console and starting up via Remote Desktop would # cause multiple bots to run concurrently on the host. print >> sys.stderr, ( 'Inhibiting Swarming bot under Google Chrome Remote Desktop.') return 0 # Always make the current working directory the directory containing this # file. It simplifies assumptions. os.chdir(os.path.dirname(THIS_FILE)) # Always create the logs dir first thing, before printing anything out. if not os.path.isdir('logs'): os.mkdir('logs') # This is necessary so os.path.join() works with unicode path. No kidding. # This must be done here as each of the command take wildly different code # path and this must be run in every case, as it causes really unexpected # issues otherwise, especially in module os.path. fix_encoding.fix_encoding() # This is extremely useful to debug hangs. signal_trace.register() if os.path.basename(THIS_FILE) == 'swarming_bot.zip': # Self-replicate itself right away as swarming_bot.1.zip and restart the bot # process as this copy. This enables LKGBC logic. print >> sys.stderr, 'Self replicating pid:%d.' % os.getpid() if os.path.isfile('swarming_bot.1.zip'): os.remove('swarming_bot.1.zip') shutil.copyfile('swarming_bot.zip', 'swarming_bot.1.zip') cmd = ['swarming_bot.1.zip'] + sys.argv[1:] print >> sys.stderr, 'cmd: %s' % cmd return common.exec_python(cmd) # sys.argv[0] is the zip file itself. cmd = 'start_slave' args = [] if len(sys.argv) > 1: cmd = sys.argv[1] args = sys.argv[2:] fn = getattr(sys.modules[__name__], 'CMD%s' % cmd, None) if fn: try: return fn(args) except ImportError: logging.exception('Failed to run %s', cmd) with zipfile.ZipFile(THIS_FILE, 'r') as f: logging.error('Files in %s:\n%s', THIS_FILE, f.namelist()) return 1 print >> sys.stderr, 'Unknown command %s' % cmd return 1
def main(): fix_encoding.fix_encoding() verbose = '-v' in sys.argv leak = bool('--leak' in sys.argv) if leak: sys.argv.remove('--leak') if verbose: logging.basicConfig(level=logging.INFO) Test.maxDiff = None else: logging.basicConfig(level=logging.ERROR) # Force language to be English, otherwise the error messages differ from # expectations. os.environ['LANG'] = 'en_US.UTF-8' os.environ['LANGUAGE'] = 'en_US.UTF-8' bot = None client = None servers = None failed = True try: servers = start_servers.LocalServers(False) servers.start() bot = start_bot.LocalBot(servers.swarming_server.url) bot.start() client = SwarmingClient( servers.swarming_server.url, servers.isolate_server.url) # Test cases only interract with the client; except for test_update_continue # which mutates the bot. Test.client = client Test.servers = servers failed = not unittest.main(exit=False).result.wasSuccessful() # Then try to terminate the bot sanely. After the terminate request # completed, the bot process should have terminated. Give it a few # seconds due to delay between sending the event that the process is # shutting down vs the process is shut down. if client.terminate(bot.bot_id) is not 0: print >> sys.stderr, 'swarming.py terminate failed' failed = True try: bot.wait(10) except subprocess42.TimeoutExpired: print >> sys.stderr, 'Bot is still alive after swarming.py terminate' failed = True except KeyboardInterrupt: print >> sys.stderr, '<Ctrl-C>' failed = True if bot.poll() is None: bot.kill() bot.wait() finally: cleanup(bot, client, servers, failed or verbose, leak) return int(failed)
def main(): fix_encoding.fix_encoding() # It's necessary for relative paths in .isolate. os.chdir(APP_DIR) parser = optparse.OptionParser() parser.add_option('-S', '--swarming', help='Swarming server') parser.add_option('-I', '--isolate-server', help='Isolate server') parser.add_option('-d', '--dimensions', nargs=2, default=[], action='append') parser.add_option('-v', '--verbose', action='store_true', help='Logs more') options, args = parser.parse_args() if args: parser.error('Unsupported args: %s' % args) if not options.swarming: parser.error('--swarming required') if not options.isolate_server: parser.error('--isolate-server required') if not os.path.isfile(SWARMING_SCRIPT): parser.error('Invalid checkout, %s does not exist' % SWARMING_SCRIPT) logging.basicConfig( level=logging.DEBUG if options.verbose else logging.ERROR) extra_flags = ['--priority', '5', '--tags', 'smoke_test:1'] for k, v in options.dimensions or [('os', 'Linux')]: extra_flags.extend(('-d', k, v)) # Run all the tests in parallel. tests = get_all_tests() results = Queue.Queue(maxsize=len(tests)) for name, fn in sorted(tests.iteritems()): logging.info('%s', name) t = threading.Thread(target=run_test, name=name, args=(results, options.swarming, options.isolate_server, extra_flags, name, fn)) t.start() print('%d tests started' % len(tests)) maxlen = max(len(name) for name in tests) for i in xrange(len(tests)): name, result, duration = results.get() print('[%d/%d] %-*s: %4.1fs: %s' % (i, len(tests), maxlen, name, duration, result)) return 0
def main(): if os.getenv('CHROME_REMOTE_DESKTOP_SESSION') == '1': # Disable itself when run under Google Chrome Remote Desktop, as it's # normally started at the console and starting up via Remote Desktop would # cause multiple bots to run concurrently on the host. print >> sys.stderr, ( 'Inhibiting Swarming bot under Google Chrome Remote Desktop.') return 0 # Always make the current working directory the directory containing this # file. It simplifies assumptions. os.chdir(os.path.dirname(THIS_FILE)) # Always create the logs dir first thing, before printing anything out. if not os.path.isdir('logs'): os.mkdir('logs') # This is necessary so os.path.join() works with unicode path. No kidding. # This must be done here as each of the command take wildly different code # path and this must be run in every case, as it causes really unexpected # issues otherwise, especially in module os.path. fix_encoding.fix_encoding() if os.path.basename(THIS_FILE) == 'swarming_bot.zip': # Self-replicate itself right away as swarming_bot.1.zip and restart as it. print >> sys.stderr, 'Self replicating pid:%d.' % os.getpid() if os.path.isfile('swarming_bot.1.zip'): os.remove('swarming_bot.1.zip') shutil.copyfile('swarming_bot.zip', 'swarming_bot.1.zip') cmd = ['swarming_bot.1.zip'] + sys.argv[1:] print >> sys.stderr, 'cmd: %s' % cmd return common.exec_python(cmd) # sys.argv[0] is the zip file itself. cmd = 'start_slave' args = [] if len(sys.argv) > 1: cmd = sys.argv[1] args = sys.argv[2:] fn = getattr(sys.modules[__name__], 'CMD%s' % cmd, None) if fn: try: return fn(args) except ImportError: logging.exception('Failed to run %s', cmd) with zipfile.ZipFile(THIS_FILE, 'r') as f: logging.error('Files in %s:\n%s', THIS_FILE, f.namelist()) return 1 print >> sys.stderr, 'Unknown command %s' % cmd return 1
def main(): fix_encoding.fix_encoding() if len(sys.argv) != 2: print >> sys.stderr, 'Specify url to Swarming server' return 1 bot = LocalBot(sys.argv[1], False) try: bot.start() bot.wait() bot.dump_log() except KeyboardInterrupt: print >> sys.stderr, '<Ctrl-C> received; stopping bot' finally: exit_code = bot.stop(False) return exit_code
def main(): fix_encoding.fix_encoding() parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__) parser.add_argument('-a', '--all', action='store_true') args = parser.parse_args() servers = LocalServers(args.all) try: servers.start() print('Swarming: %s' % servers.swarming_server.url) print('Isolate : %s' % servers.isolate_server.url) servers.wait() servers.dump_log() except KeyboardInterrupt: print >> sys.stderr, '<Ctrl-C> received; stopping servers' finally: exit_code = servers.stop(False) return exit_code
def main(): fix_encoding.fix_encoding() # It's necessary for relative paths in .isolate. os.chdir(APP_DIR) parser = optparse.OptionParser() parser.add_option('-S', '--swarming', help='Swarming server') parser.add_option('-I', '--isolate-server', help='Isolate server') parser.add_option('-d', '--dimensions', nargs=2, default=[], action='append') parser.add_option('-v', '--verbose', action='store_true', help='Logs more') options, args = parser.parse_args() if args: parser.error('Unsupported args: %s' % args) if not options.swarming: parser.error('--swarming required') if not options.isolate_server: parser.error('--isolate-server required') if not os.path.isfile(SWARMING_SCRIPT): parser.error('Invalid checkout, %s does not exist' % SWARMING_SCRIPT) logging.basicConfig(level=logging.DEBUG if options.verbose else logging.ERROR) extra_flags = ['--priority', '5', '--tags', 'smoke_test:1'] for k, v in options.dimensions or [('os', 'Linux')]: extra_flags.extend(('-d', k, v)) # Run all the tests in parallel. tests = get_all_tests() results = Queue.Queue(maxsize=len(tests)) for name, fn in sorted(tests.iteritems()): logging.info('%s', name) t = threading.Thread( target=run_test, name=name, args=(results, options.swarming, options.isolate_server, extra_flags, name, fn)) t.start() print('%d tests started' % len(tests)) maxlen = max(len(name) for name in tests) for i in xrange(len(tests)): name, result, duration = results.get() print('[%d/%d] %-*s: %4.1fs: %s' % (i, len(tests), maxlen, name, duration, result)) return 0
def main(): fix_encoding.fix_encoding() if len(sys.argv) != 2: print >> sys.stderr, 'Specify url to Swarming server' return 1 botdir = tempfile.mkdtemp(prefix='start_bot') try: bot = LocalBot(sys.argv[1], False, botdir) try: bot.start() bot.wait() bot.dump_log() except KeyboardInterrupt: print >> sys.stderr, '<Ctrl-C> received; stopping bot' finally: exit_code = bot.stop() finally: shutil.rmtree(botdir) return exit_code
def main(): fix_encoding.fix_encoding() parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__) parser.add_argument('-a', '--all', action='store_true') args = parser.parse_args() servers = LocalServers(args.all) dump_log = True try: servers.start() print('Swarming: %s' % servers.swarming_server.url) print('Isolate : %s' % servers.isolate_server.url) servers.wait() except KeyboardInterrupt: print >> sys.stderr, '<Ctrl-C> received; stopping servers' dump_log = False finally: exit_code = servers.stop(False) if dump_log: servers.dump_log() return exit_code
parser.error( 'Must pass one python script to run. Use --help for more details') # 1. Query the bots list. bots = get_bot_list(options.swarming, options.dimensions) print('Found %d bots to process' % len(bots)) if not bots: return 1 # 2. Archive the script to run. isolated_hash = archive(options.isolate_server, args[0]) print('Running %s' % isolated_hash) # 3. Trigger the tasks. name = os.path.basename(args[0]) if options.serial: return run_serial(options.swarming, options.isolate_server, str(options.priority), str(options.deadline), options.repeat, isolated_hash, name, bots) return run_parallel(options.swarming, options.isolate_server, str(options.priority), str(options.deadline), options.repeat, isolated_hash, name, bots) if __name__ == '__main__': fix_encoding.fix_encoding() tools.disable_buffering() colorama.init() sys.exit(main())
auth.process_auth_options(parser, options) isolateserver.process_isolate_server_options(data_group, options) if bool(options.isolated) == bool(options.hash): logging.debug('One and only one of --isolated or --hash is required.') parser.error('One and only one of --isolated or --hash is required.') options.cache = os.path.abspath(options.cache) policies = CachePolicies( options.max_cache_size, options.min_free_space, options.max_items) algo = isolateserver.get_hash_algo(options.namespace) try: # |options.cache| may not exist until DiskCache() instance is created. cache = DiskCache(options.cache, policies, algo) remote = options.isolate_server or options.indir with isolateserver.get_storage(remote, options.namespace) as storage: return run_tha_test( options.isolated or options.hash, storage, cache, algo, args) except Exception as e: # Make sure any exception is logged. tools.report_error(e) logging.exception(e) return 1 if __name__ == '__main__': # Ensure that we are always running with the correct encoding. fix_encoding.fix_encoding() sys.exit(main(sys.argv[1:]))