def main():
fix_encoding.fix_encoding()
verbose = '-v' in sys.argv
leak = bool('--leak' in sys.argv)
if leak:
sys.argv.remove('--leak')
if verbose:
logging.basicConfig(level=logging.INFO)
Test.maxDiff = None
else:
logging.basicConfig(level=logging.ERROR)
# Force language to be English, otherwise the error messages differ from
# expectations.
os.environ['LANG'] = 'en_US.UTF-8'
os.environ['LANGUAGE'] = 'en_US.UTF-8'
# So that we don't get goofed up when running this test on swarming :)
os.environ.pop('SWARMING_TASK_ID', None)
bot = None
client = None
servers = None
failed = True
try:
servers = start_servers.LocalServers(False)
servers.start()
bot = start_bot.LocalBot(servers.swarming_server.url)
Test.bot = bot
bot.start()
client = SwarmingClient(servers.swarming_server.url,
servers.isolate_server.url)
# Test cases only interract with the client; except for test_update_continue
# which mutates the bot.
Test.client = client
Test.servers = servers
failed = not unittest.main(exit=False).result.wasSuccessful()
# Then try to terminate the bot sanely. After the terminate request
# completed, the bot process should have terminated. Give it a few
# seconds due to delay between sending the event that the process is
# shutting down vs the process is shut down.
if client.terminate(bot.bot_id) is not 0:
print >> sys.stderr, 'swarming.py terminate failed'
failed = True
try:
bot.wait(10)
except subprocess42.TimeoutExpired:
print >> sys.stderr, 'Bot is still alive after swarming.py terminate'
failed = True
except KeyboardInterrupt:
print >> sys.stderr, '<Ctrl-C>'
failed = True
if bot is not None and bot.poll() is None:
bot.kill()
bot.wait()
finally:
cleanup(bot, client, servers, failed or verbose, leak)
return int(failed)
def main():
if os.getenv('CHROME_REMOTE_DESKTOP_SESSION') == '1':
# Disable itself when run under Google Chrome Remote Desktop, as it's
# normally started at the console and starting up via Remote Desktop would
# cause multiple bots to run concurrently on the host.
print >> sys.stderr, (
'Inhibiting Swarming bot under Google Chrome Remote Desktop.')
return 0
# Always make the current working directory the directory containing this
# file. It simplifies assumptions.
os.chdir(os.path.dirname(THIS_FILE))
# Always create the logs dir first thing, before printing anything out.
if not os.path.isdir('logs'):
os.mkdir('logs')
# This is necessary so os.path.join() works with unicode path. No kidding.
# This must be done here as each of the command take wildly different code
# path and this must be run in every case, as it causes really unexpected
# issues otherwise, especially in module os.path.
fix_encoding.fix_encoding()
# This is extremely useful to debug hangs.
signal_trace.register()
if os.path.basename(THIS_FILE) == 'swarming_bot.zip':
# Self-replicate itself right away as swarming_bot.1.zip and restart the bot
# process as this copy. This enables LKGBC logic.
print >> sys.stderr, 'Self replicating pid:%d.' % os.getpid()
if os.path.isfile('swarming_bot.1.zip'):
os.remove('swarming_bot.1.zip')
shutil.copyfile('swarming_bot.zip', 'swarming_bot.1.zip')
cmd = ['swarming_bot.1.zip'] + sys.argv[1:]
print >> sys.stderr, 'cmd: %s' % cmd
return common.exec_python(cmd)
# sys.argv[0] is the zip file itself.
cmd = 'start_slave'
args = []
if len(sys.argv) > 1:
cmd = sys.argv[1]
args = sys.argv[2:]
fn = getattr(sys.modules[__name__], 'CMD%s' % cmd, None)
if fn:
try:
return fn(args)
except ImportError:
logging.exception('Failed to run %s', cmd)
with zipfile.ZipFile(THIS_FILE, 'r') as f:
logging.error('Files in %s:\n%s', THIS_FILE, f.namelist())
return 1
print >> sys.stderr, 'Unknown command %s' % cmd
return 1
def main():
fix_encoding.fix_encoding()
verbose = '-v' in sys.argv
leak = bool('--leak' in sys.argv)
if leak:
sys.argv.remove('--leak')
if verbose:
logging.basicConfig(level=logging.INFO)
Test.maxDiff = None
else:
logging.basicConfig(level=logging.ERROR)
# Force language to be English, otherwise the error messages differ from
# expectations.
os.environ['LANG'] = 'en_US.UTF-8'
os.environ['LANGUAGE'] = 'en_US.UTF-8'
bot = None
client = None
servers = None
failed = True
try:
servers = start_servers.LocalServers(False)
servers.start()
bot = start_bot.LocalBot(servers.swarming_server.url)
bot.start()
client = SwarmingClient(
servers.swarming_server.url, servers.isolate_server.url)
# Test cases only interract with the client; except for test_update_continue
# which mutates the bot.
Test.client = client
Test.servers = servers
failed = not unittest.main(exit=False).result.wasSuccessful()
# Then try to terminate the bot sanely. After the terminate request
# completed, the bot process should have terminated. Give it a few
# seconds due to delay between sending the event that the process is
# shutting down vs the process is shut down.
if client.terminate(bot.bot_id) is not 0:
print >> sys.stderr, 'swarming.py terminate failed'
failed = True
try:
bot.wait(10)
except subprocess42.TimeoutExpired:
print >> sys.stderr, 'Bot is still alive after swarming.py terminate'
failed = True
except KeyboardInterrupt:
print >> sys.stderr, '<Ctrl-C>'
failed = True
if bot.poll() is None:
bot.kill()
bot.wait()
finally:
cleanup(bot, client, servers, failed or verbose, leak)
return int(failed)
def main():
fix_encoding.fix_encoding()
# It's necessary for relative paths in .isolate.
os.chdir(APP_DIR)
parser = optparse.OptionParser()
parser.add_option('-S', '--swarming', help='Swarming server')
parser.add_option('-I', '--isolate-server', help='Isolate server')
parser.add_option('-d',
'--dimensions',
nargs=2,
default=[],
action='append')
parser.add_option('-v', '--verbose', action='store_true', help='Logs more')
options, args = parser.parse_args()
if args:
parser.error('Unsupported args: %s' % args)
if not options.swarming:
parser.error('--swarming required')
if not options.isolate_server:
parser.error('--isolate-server required')
if not os.path.isfile(SWARMING_SCRIPT):
parser.error('Invalid checkout, %s does not exist' % SWARMING_SCRIPT)
logging.basicConfig(
level=logging.DEBUG if options.verbose else logging.ERROR)
extra_flags = ['--priority', '5', '--tags', 'smoke_test:1']
for k, v in options.dimensions or [('os', 'Linux')]:
extra_flags.extend(('-d', k, v))
# Run all the tests in parallel.
tests = get_all_tests()
results = Queue.Queue(maxsize=len(tests))
for name, fn in sorted(tests.iteritems()):
logging.info('%s', name)
t = threading.Thread(target=run_test,
name=name,
args=(results, options.swarming,
options.isolate_server, extra_flags, name,
fn))
t.start()
print('%d tests started' % len(tests))
maxlen = max(len(name) for name in tests)
for i in xrange(len(tests)):
name, result, duration = results.get()
print('[%d/%d] %-*s: %4.1fs: %s' %
(i, len(tests), maxlen, name, duration, result))
return 0
def main():
if os.getenv('CHROME_REMOTE_DESKTOP_SESSION') == '1':
# Disable itself when run under Google Chrome Remote Desktop, as it's
# normally started at the console and starting up via Remote Desktop would
# cause multiple bots to run concurrently on the host.
print >> sys.stderr, (
'Inhibiting Swarming bot under Google Chrome Remote Desktop.')
return 0
# Always make the current working directory the directory containing this
# file. It simplifies assumptions.
os.chdir(os.path.dirname(THIS_FILE))
# Always create the logs dir first thing, before printing anything out.
if not os.path.isdir('logs'):
os.mkdir('logs')
# This is necessary so os.path.join() works with unicode path. No kidding.
# This must be done here as each of the command take wildly different code
# path and this must be run in every case, as it causes really unexpected
# issues otherwise, especially in module os.path.
fix_encoding.fix_encoding()
if os.path.basename(THIS_FILE) == 'swarming_bot.zip':
# Self-replicate itself right away as swarming_bot.1.zip and restart as it.
print >> sys.stderr, 'Self replicating pid:%d.' % os.getpid()
if os.path.isfile('swarming_bot.1.zip'):
os.remove('swarming_bot.1.zip')
shutil.copyfile('swarming_bot.zip', 'swarming_bot.1.zip')
cmd = ['swarming_bot.1.zip'] + sys.argv[1:]
print >> sys.stderr, 'cmd: %s' % cmd
return common.exec_python(cmd)
# sys.argv[0] is the zip file itself.
cmd = 'start_slave'
args = []
if len(sys.argv) > 1:
cmd = sys.argv[1]
args = sys.argv[2:]
fn = getattr(sys.modules[__name__], 'CMD%s' % cmd, None)
if fn:
try:
return fn(args)
except ImportError:
logging.exception('Failed to run %s', cmd)
with zipfile.ZipFile(THIS_FILE, 'r') as f:
logging.error('Files in %s:\n%s', THIS_FILE, f.namelist())
return 1
print >> sys.stderr, 'Unknown command %s' % cmd
return 1
def main():
fix_encoding.fix_encoding()
if len(sys.argv) != 2:
print >> sys.stderr, 'Specify url to Swarming server'
return 1
bot = LocalBot(sys.argv[1], False)
try:
bot.start()
bot.wait()
bot.dump_log()
except KeyboardInterrupt:
print >> sys.stderr, '<Ctrl-C> received; stopping bot'
finally:
exit_code = bot.stop(False)
return exit_code
def main():
fix_encoding.fix_encoding()
if len(sys.argv) != 2:
print >> sys.stderr, 'Specify url to Swarming server'
return 1
bot = LocalBot(sys.argv[1], False)
try:
bot.start()
bot.wait()
bot.dump_log()
except KeyboardInterrupt:
print >> sys.stderr, '<Ctrl-C> received; stopping bot'
finally:
exit_code = bot.stop(False)
return exit_code
def main():
fix_encoding.fix_encoding()
parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
parser.add_argument('-a', '--all', action='store_true')
args = parser.parse_args()
servers = LocalServers(args.all)
try:
servers.start()
print('Swarming: %s' % servers.swarming_server.url)
print('Isolate : %s' % servers.isolate_server.url)
servers.wait()
servers.dump_log()
except KeyboardInterrupt:
print >> sys.stderr, '<Ctrl-C> received; stopping servers'
finally:
exit_code = servers.stop(False)
return exit_code
def main():
fix_encoding.fix_encoding()
# It's necessary for relative paths in .isolate.
os.chdir(APP_DIR)
parser = optparse.OptionParser()
parser.add_option('-S', '--swarming', help='Swarming server')
parser.add_option('-I', '--isolate-server', help='Isolate server')
parser.add_option('-d', '--dimensions', nargs=2, default=[], action='append')
parser.add_option('-v', '--verbose', action='store_true', help='Logs more')
options, args = parser.parse_args()
if args:
parser.error('Unsupported args: %s' % args)
if not options.swarming:
parser.error('--swarming required')
if not options.isolate_server:
parser.error('--isolate-server required')
if not os.path.isfile(SWARMING_SCRIPT):
parser.error('Invalid checkout, %s does not exist' % SWARMING_SCRIPT)
logging.basicConfig(level=logging.DEBUG if options.verbose else logging.ERROR)
extra_flags = ['--priority', '5', '--tags', 'smoke_test:1']
for k, v in options.dimensions or [('os', 'Linux')]:
extra_flags.extend(('-d', k, v))
# Run all the tests in parallel.
tests = get_all_tests()
results = Queue.Queue(maxsize=len(tests))
for name, fn in sorted(tests.iteritems()):
logging.info('%s', name)
t = threading.Thread(
target=run_test, name=name,
args=(results, options.swarming, options.isolate_server, extra_flags,
name, fn))
t.start()
print('%d tests started' % len(tests))
maxlen = max(len(name) for name in tests)
for i in xrange(len(tests)):
name, result, duration = results.get()
print('[%d/%d] %-*s: %4.1fs: %s' %
(i, len(tests), maxlen, name, duration, result))
return 0
def main():
fix_encoding.fix_encoding()
if len(sys.argv) != 2:
print >> sys.stderr, 'Specify url to Swarming server'
return 1
botdir = tempfile.mkdtemp(prefix='start_bot')
try:
bot = LocalBot(sys.argv[1], False, botdir)
try:
bot.start()
bot.wait()
bot.dump_log()
except KeyboardInterrupt:
print >> sys.stderr, '<Ctrl-C> received; stopping bot'
finally:
exit_code = bot.stop()
finally:
shutil.rmtree(botdir)
return exit_code
def main():
fix_encoding.fix_encoding()
parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
parser.add_argument('-a', '--all', action='store_true')
args = parser.parse_args()
servers = LocalServers(args.all)
dump_log = True
try:
servers.start()
print('Swarming: %s' % servers.swarming_server.url)
print('Isolate : %s' % servers.isolate_server.url)
servers.wait()
except KeyboardInterrupt:
print >> sys.stderr, '<Ctrl-C> received; stopping servers'
dump_log = False
finally:
exit_code = servers.stop(False)
if dump_log:
servers.dump_log()
return exit_code
parser.error(
'Must pass one python script to run. Use --help for more details')
# 1. Query the bots list.
bots = get_bot_list(options.swarming, options.dimensions)
print('Found %d bots to process' % len(bots))
if not bots:
return 1
# 2. Archive the script to run.
isolated_hash = archive(options.isolate_server, args[0])
print('Running %s' % isolated_hash)
# 3. Trigger the tasks.
name = os.path.basename(args[0])
if options.serial:
return run_serial(options.swarming, options.isolate_server,
str(options.priority), str(options.deadline),
options.repeat, isolated_hash, name, bots)
return run_parallel(options.swarming, options.isolate_server,
str(options.priority), str(options.deadline),
options.repeat, isolated_hash, name, bots)
if __name__ == '__main__':
fix_encoding.fix_encoding()
tools.disable_buffering()
colorama.init()
sys.exit(main())
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(data_group, options)
if bool(options.isolated) == bool(options.hash):
logging.debug('One and only one of --isolated or --hash is required.')
parser.error('One and only one of --isolated or --hash is required.')
options.cache = os.path.abspath(options.cache)
policies = CachePolicies(
options.max_cache_size, options.min_free_space, options.max_items)
algo = isolateserver.get_hash_algo(options.namespace)
try:
# |options.cache| may not exist until DiskCache() instance is created.
cache = DiskCache(options.cache, policies, algo)
remote = options.isolate_server or options.indir
with isolateserver.get_storage(remote, options.namespace) as storage:
return run_tha_test(
options.isolated or options.hash, storage, cache, algo, args)
except Exception as e:
# Make sure any exception is logged.
tools.report_error(e)
logging.exception(e)
return 1
if __name__ == '__main__':
# Ensure that we are always running with the correct encoding.
fix_encoding.fix_encoding()
sys.exit(main(sys.argv[1:]))
