def main(argv):
colorama.init()
if get_git_version() < MIN_UPSTREAM_TRACK_GIT_VERSION:
print >> sys.stderr, (
'This tool will not show all tracking information for git version '
'earlier than ' +
'.'.join(str(x) for x in MIN_UPSTREAM_TRACK_GIT_VERSION) +
'. Please consider upgrading.')
parser = argparse.ArgumentParser(
description='Print a a tree of all branches parented by their upstreams'
)
parser.add_argument('-v',
action='count',
help='Display branch hash and Rietveld URL')
parser.add_argument('--no-color',
action='store_true',
dest='nocolor',
help='Turn off colors.')
opts = parser.parse_args(argv[1:])
mapper = BranchMapper()
mapper.verbosity = opts.v
mapper.output.nocolor = opts.nocolor
mapper.start()
print mapper.output.as_formatted_string()
def main(argv):
colorama.init()
assert len(argv) == 1, "No arguments expected"
branch_map = {}
par_map = collections.defaultdict(list)
for branch in branches():
par = upstream(branch) or NO_UPSTREAM
branch_map[branch] = par
par_map[par].append(branch)
current = current_branch()
hashes = hash_multi(current, *branch_map.keys())
current_hash = hashes[0]
par_hashes = {
k: hashes[i + 1]
for i, k in enumerate(branch_map.iterkeys())
}
par_hashes[NO_UPSTREAM] = 0
tag_set = tags()
while par_map:
for parent in par_map:
if parent not in branch_map:
if parent not in par_hashes:
par_hashes[parent] = hash_one(parent)
print_branch(current, current_hash, parent, par_hashes,
par_map, branch_map, tag_set)
break
def main(argv):
colorama.init()
if get_git_version() < MIN_UPSTREAM_TRACK_GIT_VERSION:
print >> sys.stderr, (
'This tool will not show all tracking information for git version '
'earlier than ' +
'.'.join(str(x) for x in MIN_UPSTREAM_TRACK_GIT_VERSION) +
'. Please consider upgrading.')
parser = argparse.ArgumentParser(
description='Print a a tree of all branches parented by their upstreams')
parser.add_argument('-v', action='count',
help='Display branch hash and Rietveld URL')
parser.add_argument('--no-color', action='store_true', dest='nocolor',
help='Turn off colors.')
parser.add_argument(
'-j', '--maxjobs', action='store', type=int,
help='The number of jobs to use when retrieving review status')
parser.add_argument('--show-subject', action='store_true',
dest='show_subject', help='Show the commit subject.')
opts = parser.parse_args(argv)
mapper = BranchMapper()
mapper.verbosity = opts.v
mapper.output.nocolor = opts.nocolor
mapper.maxjobs = opts.maxjobs
mapper.show_subject = opts.show_subject
mapper.start()
print mapper.output.as_formatted_string()
return 0
def print_version_change(config_version):
"""Print a notice to let the user know we are collecting more metrics."""
colorama.init()
print(colorama.Fore.RED + '\033[1m', file=sys.stderr, end='')
print(NOTICE_VERSION_CHANGE_HEADER, file=sys.stderr)
print(EMPTY_LINE, file=sys.stderr)
for version in range(config_version + 1, CURRENT_VERSION + 1):
print(CHANGE_NOTICE[version], file=sys.stderr)
print(EMPTY_LINE, file=sys.stderr)
def print_notice(countdown):
"""Print a notice to let the user know the status of metrics collection."""
colorama.init()
print(colorama.Fore.RED + '\033[1m', file=sys.stderr)
if countdown:
print(NOTICE_COUNTDOWN_HEADER % countdown, file=sys.stderr)
else:
print(NOTICE_COLLECTION_HEADER, file=sys.stderr)
print(NOTICE_FOOTER + colorama.Style.RESET_ALL, file=sys.stderr)
def print_red_boxed_text(out, min_width, lines):
colorama.init()
if sys.platform == 'win32':
[EW, NS, SE, SW, NE, NW] = list('=|++++')
else:
[EW, NS, SE, SW, NE,
NW] = list(u'\u2501\u2503\u250F\u2513\u2517\u251B')
out(colorama.Fore.RED + colorama.Style.BRIGHT)
width = max(min_width, max(len(line) for line in lines))
out(SE + EW * (width + 2) + SW + '\n')
for line in lines:
out('%s %-*s %s\n' % (NS, width, line, NS))
out(NE + EW * (width + 2) + NW + '\n')
out(colorama.Style.RESET_ALL)
def Main(argv):
"""Doesn't parse the arguments here, just find the right subcommand to
execute."""
if sys.hexversion < 0x02060000:
print >> sys.stderr, (
'\nYour python version %s is unsupported, please upgrade.\n' %
sys.version.split(' ', 1)[0])
return 2
if not sys.executable:
print >> sys.stderr, (
'\nPython cannot find the location of it\'s own executable.\n')
return 2
fix_encoding.fix_encoding()
disable_buffering()
colorama.init()
dispatcher = subcommand.CommandDispatcher(__name__)
try:
return dispatcher.execute(OptionParser(), argv)
except KeyboardInterrupt:
gclient_utils.GClientChildren.KillAllRemainingChildren()
raise
except (gclient_utils.Error, subprocess2.CalledProcessError), e:
print >> sys.stderr, 'Error: %s' % str(e)
return 1
def main(argv):
colorama.init()
assert len(argv) == 1, "No arguments expected"
branch_map = {}
par_map = collections.defaultdict(list)
for branch in branches():
par = upstream(branch) or NO_UPSTREAM
branch_map[branch] = par
par_map[par].append(branch)
current = current_branch()
hashes = hash_multi(current, *branch_map.keys())
current_hash = hashes[0]
par_hashes = {k: hashes[i+1] for i, k in enumerate(branch_map.iterkeys())}
par_hashes[NO_UPSTREAM] = 0
tag_set = tags()
while par_map:
for parent in par_map:
if parent not in branch_map:
if parent not in par_hashes:
par_hashes[parent] = hash_one(parent)
print_branch(current, current_hash, parent, par_hashes, par_map,
branch_map, tag_set)
break
def Main(argv):
"""Doesn't parse the arguments here, just find the right subcommand to
execute."""
if sys.hexversion < 0x02060000:
print >> sys.stderr, (
'\nYour python version %s is unsupported, please upgrade.\n' %
sys.version.split(' ', 1)[0])
return 2
if not sys.executable:
print >> sys.stderr, (
'\nPython cannot find the location of it\'s own executable.\n')
return 2
fix_encoding.fix_encoding()
disable_buffering()
colorama.init()
dispatcher = subcommand.CommandDispatcher(__name__)
try:
return dispatcher.execute(OptionParser(), argv)
except KeyboardInterrupt:
gclient_utils.GClientChildren.KillAllRemainingChildren()
raise
except (gclient_utils.Error, subprocess2.CalledProcessError), e:
print >> sys.stderr, 'Error: %s' % str(e)
return 1
if options.os:
if options.os not in oses:
parser.error(
'--os %s is unknown. Valid values are %s' % (
options.os, ', '.join(sorted(oses))))
oses = [options.os]
if sys.platform in ('win32', 'cygwin'):
# If we are on Windows, don't generate the tests for Linux and Mac since
# they use symlinks and we can't create symlinks on windows.
oses = ['Windows']
if options.os != 'win32':
print('Linux and Mac tests skipped since running on Windows.')
return run_swarming_tests_on_swarming(
options.swarming,
options.isolate_server,
options.priority,
oses,
tests,
options.logs,
options.no_idempotent)
if __name__ == '__main__':
fix_encoding.fix_encoding()
tools.disable_buffering()
colorama.init()
sys.exit(main())
def main():
colorama.init()
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option(
'-S', '--swarming',
metavar='URL', default='',
help='Swarming server to use')
swarming.add_filter_options(parser)
parser.set_defaults(dimensions=[('os', swarming_load_test_bot.OS_NAME)])
group = optparse.OptionGroup(parser, 'Load generated')
group.add_option(
'-s', '--send-rate', type='float', default=16., metavar='RATE',
help='Rate (item/s) of sending requests as a float, default: %default')
group.add_option(
'-D', '--duration', type='float', default=60., metavar='N',
help='Duration (s) of the sending phase of the load test, '
'default: %default')
group.add_option(
'-m', '--concurrent', type='int', default=200, metavar='N',
help='Maximum concurrent on-going requests, default: %default')
group.add_option(
'-t', '--timeout', type='float', default=3600., metavar='N',
help='Timeout to get results, default: %default')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Display options')
group.add_option(
'--columns', type='int', default=graph.get_console_width(), metavar='N',
help='For histogram display, default:%default')
group.add_option(
'--buckets', type='int', default=20, metavar='N',
help='Number of buckets for histogram display, default:%default')
parser.add_option_group(group)
parser.add_option(
'--dump', metavar='FOO.JSON', help='Dumps to json file')
parser.add_option(
'-v', '--verbose', action='store_true', help='Enables logging')
options, args = parser.parse_args()
logging.basicConfig(level=logging.INFO if options.verbose else logging.FATAL)
if args:
parser.error('Unsupported args: %s' % args)
options.swarming = options.swarming.rstrip('/')
if not options.swarming:
parser.error('--swarming is required.')
if options.duration <= 0:
parser.error('Needs --duration > 0. 0.01 is a valid value.')
swarming.process_filter_options(parser, options)
total = options.send_rate * options.duration
print(
'Sending %.1f i/s for %ds with max %d parallel requests; timeout %.1fs; '
'total %d' %
(options.send_rate, options.duration, options.concurrent,
options.timeout, total))
print('[processing/processed/todo]')
# This is used so there's no clash between runs and actual real usage.
unique = ''.join(random.choice(string.ascii_letters) for _ in range(8))
columns = [('processing', 0), ('processed', 0), ('todo', 0)]
progress = threading_utils.Progress(columns)
index = 0
with threading_utils.ThreadPoolWithProgress(
progress, 1, options.concurrent, 0) as pool:
try:
start = time.time()
while True:
duration = time.time() - start
if duration > options.duration:
break
should_have_triggered_so_far = int(duration * options.send_rate)
while index < should_have_triggered_so_far:
pool.add_task(
0,
trigger_task,
options.swarming,
options.dimensions,
progress,
unique,
options.timeout,
index)
progress.update_item('', todo=1)
index += 1
progress.print_update()
time.sleep(0.01)
except KeyboardInterrupt:
aborted = pool.abort()
progress.update_item(
'Got Ctrl-C. Aborted %d unsent tasks.' % aborted,
raw=True,
todo=-aborted)
progress.print_update()
finally:
# TODO(maruel): We could give up on collecting results for the on-going
# tasks but that would need to be optional.
progress.update_item('Getting results for on-going tasks.', raw=True)
results = sorted(pool.join())
progress.print_update()
# At this point, progress is not used anymore.
print('')
print(' - Took %.1fs.' % (time.time() - start))
print('')
print_results(results, options.columns, options.buckets)
if options.dump:
with open(options.dump, 'w') as f:
json.dump(results, f, separators=(',',':'))
return 0
def main():
colorama.init()
parser = optparse.OptionParser(description=sys.modules['__main__'].__doc__)
tomorrow = datetime.datetime.utcnow().date() + datetime.timedelta(days=1)
year = datetime.datetime(tomorrow.year, 1, 1)
parser.add_option('-S',
'--swarming',
metavar='URL',
default=os.environ.get('SWARMING_SERVER', ''),
help='Swarming server to use')
group = optparse.OptionGroup(parser, 'Filtering')
group.add_option(
'--start',
default=year.strftime('%Y-%m-%d'),
help='Starting date in UTC; defaults to start of year: %default')
group.add_option('--end',
default=tomorrow.strftime('%Y-%m-%d'),
help='End date in UTC; defaults to tomorrow: %default')
group.add_option('--state',
default='ALL',
type='choice',
choices=STATES,
help='State to filter on')
group.add_option('--tags',
action='append',
default=[],
help='Tags to filter on')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Presentation')
group.add_option('--dimensions',
action='store_true',
help='Show the dimensions')
group.add_option(
'--daily-count',
action='store_true',
help='Show the daily count in raw number instead of histogram')
parser.add_option_group(group)
parser.add_option('--json',
default='counts.json',
help='File containing raw data; default: %default')
parser.add_option('-v', '--verbose', action='count', default=0, help='Log')
options, args = parser.parse_args()
if args:
parser.error('Unsupported argument %s' % args)
logging.basicConfig(
level=logging.DEBUG if options.verbose else logging.ERROR)
start = parse_time_option(options.start)
end = parse_time_option(options.end)
print('From %s (%d) to %s (%d)' %
(start, int((start - _EPOCH).total_seconds()), end,
int((end - _EPOCH).total_seconds())))
if options.swarming:
if options.dimensions:
data = fetch_tasks(options.swarming, start, end, options.state,
options.tags)
else:
data = fetch_counts(options.swarming, start, end, options.state,
options.tags)
with open(options.json, 'wb') as f:
json.dump(data, f)
elif not os.path.isfile(options.json):
parser.error('--swarming is required.')
else:
with open(options.json, 'rb') as f:
data = json.load(f)
print('')
if options.dimensions:
present_dimensions(data, options.daily_count)
else:
present_counts(data, options.daily_count)
return 0
def main():
colorama.init()
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option(
'-I', '--isolate-server',
metavar='URL', default='',
help='Isolate server to use')
parser.add_option(
'--namespace', default='temporary%d-gzip' % time.time(), metavar='XX',
help='Namespace to use on the server, default: %default')
parser.add_option(
'--threads', type='int', default=16, metavar='N',
help='Parallel worker threads to use, default:%default')
data_group = optparse.OptionGroup(parser, 'Amount of data')
graph.unit_option(
data_group, '--items', default=0, help='Number of items to upload')
graph.unit_option(
data_group, '--max-size', default=0,
help='Loop until this amount of data was transferred')
graph.unit_option(
data_group, '--mid-size', default=100*1024,
help='Rough average size of each item, default:%default')
parser.add_option_group(data_group)
ui_group = optparse.OptionGroup(parser, 'Result histogram')
ui_group.add_option(
'--columns', type='int', default=graph.get_console_width(), metavar='N',
help='Width of histogram, default:%default')
ui_group.add_option(
'--buckets', type='int', default=20, metavar='N',
help='Number of histogram\'s buckets, default:%default')
parser.add_option_group(ui_group)
log_group = optparse.OptionGroup(parser, 'Logging')
log_group.add_option(
'--dump', metavar='FOO.JSON', help='Dumps to json file')
log_group.add_option(
'-v', '--verbose', action='store_true', help='Enable logging')
parser.add_option_group(log_group)
options, args = parser.parse_args()
logging.basicConfig(level=logging.INFO if options.verbose else logging.FATAL)
if args:
parser.error('Unsupported args: %s' % args)
if bool(options.max_size) == bool(options.items):
parser.error(
'Use one of --max-size or --items.\n'
' Use --max-size if you want to run it until NN bytes where '
'transfered.\n'
' Otherwise use --items to run it for NN items.')
options.isolate_server = options.isolate_server.rstrip('/')
if not options.isolate_server:
parser.error('--isolate-server is required.')
print(
' - Using %d thread, items=%d, max-size=%d, mid-size=%d' % (
options.threads, options.items, options.max_size, options.mid_size))
start = time.time()
random_pool = Randomness()
print(' - Generated pool after %.1fs' % (time.time() - start))
columns = [('index', 0), ('data', 0), ('size', options.items)]
progress = Progress(columns)
storage = isolateserver.get_storage(options.isolate_server, options.namespace)
do_item = functools.partial(
send_and_receive,
random_pool,
storage,
progress)
# TODO(maruel): Handle Ctrl-C should:
# - Stop adding tasks.
# - Stop scheduling tasks in ThreadPool.
# - Wait for the remaining ungoing tasks to complete.
# - Still print details and write the json file.
with threading_utils.ThreadPoolWithProgress(
progress, options.threads, options.threads, 0) as pool:
if options.items:
for _ in xrange(options.items):
pool.add_task(0, do_item, gen_size(options.mid_size))
progress.print_update()
elif options.max_size:
# This one is approximate.
total = 0
while True:
size = gen_size(options.mid_size)
progress.update_item('', size=1)
progress.print_update()
pool.add_task(0, do_item, size)
total += size
if total >= options.max_size:
break
results = sorted(pool.join())
print('')
print(' - Took %.1fs.' % (time.time() - start))
print('')
print_results(results, options.columns, options.buckets)
if options.dump:
with open(options.dump, 'w') as f:
json.dump(results, f, separators=(',',':'))
return 0
def init():
# should_wrap instructs colorama to wrap stdout/stderr with an ANSI colorcode
# interpreter that converts them to SetConsoleTextAttribute calls. This only
# should be True in cases where we're connected to cmd.exe's console. Setting
# this to True on non-windows systems has no effect.
should_wrap = False
global IS_TTY, OUT_TYPE
IS_TTY = sys.stdout.isatty()
is_windows = sys.platform.startswith('win')
if IS_TTY:
# Yay! We detected a console in the normal way. It doesn't really matter
# if it's windows or not, we win.
OUT_TYPE = 'console'
should_wrap = True
elif is_windows:
# assume this is some sort of file
OUT_TYPE = 'file (win)'
import msvcrt
h = msvcrt.get_osfhandle(sys.stdout.fileno())
# h is the win32 HANDLE for stdout.
ftype = ctypes.windll.kernel32.GetFileType(h)
if ftype == 2: # FILE_TYPE_CHAR
# This is a normal cmd console, but we'll only get here if we're running
# inside a `git command` which is actually git->bash->command. Not sure
# why isatty doesn't detect this case.
OUT_TYPE = 'console (cmd via msys)'
IS_TTY = True
should_wrap = True
elif ftype == 3: # FILE_TYPE_PIPE
OUT_TYPE = 'pipe (win)'
# This is some kind of pipe on windows. This could either be a real pipe
# or this could be msys using a pipe to emulate a pty. We use the same
# algorithm that msys-git uses to determine if it's connected to a pty or
# not.
# This function and the structures are defined in the MSDN documentation
# using the same names.
def NT_SUCCESS(status):
# The first two bits of status are the severity. The success
# severities are 0 and 1, and the !success severities are 2 and 3.
# Therefore since ctypes interprets the default restype of the call
# to be an 'C int' (which is guaranteed to be signed 32 bits), All
# success codes are positive, and all !success codes are negative.
return status >= 0
class UNICODE_STRING(ctypes.Structure):
_fields_ = [('Length', ctypes.c_ushort),
('MaximumLength', ctypes.c_ushort),
('Buffer', ctypes.c_wchar_p)]
class OBJECT_NAME_INFORMATION(ctypes.Structure):
_fields_ = [('Name', UNICODE_STRING),
('NameBuffer', ctypes.c_wchar_p)]
buf = ctypes.create_string_buffer(1024)
# Ask NT what the name of the object our stdout HANDLE is. It would be
# possible to use GetFileInformationByHandleEx, but it's only available
# on Vista+. If you're reading this in 2017 or later, feel free to
# refactor this out.
#
# The '1' here is ObjectNameInformation
if NT_SUCCESS(
ctypes.windll.ntdll.NtQueryObject(h, 1, buf,
len(buf) - 2, None)):
out = OBJECT_NAME_INFORMATION.from_buffer(buf)
name = out.Name.Buffer.split('\\')[-1]
IS_TTY = name.startswith('msys-') and '-pty' in name
if IS_TTY:
OUT_TYPE = 'bash (msys)'
else:
# A normal file, or an unknown file type.
pass
else:
# This is non-windows, so we trust isatty.
OUT_TYPE = 'pipe or file'
if IS_TTY and is_windows:
# Wrapping may cause errors on some Windows versions (crbug.com/1114548).
if platform.release() != '10' or enable_native_ansi():
should_wrap = False
colorama.init(wrap=should_wrap)
def main():
colorama.init()
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option('-S',
'--swarming',
metavar='URL',
default='',
help='Swarming server to use')
swarming.add_filter_options(parser)
parser.set_defaults(dimensions=[('os', swarming_load_test_bot.OS_NAME)])
group = optparse.OptionGroup(parser, 'Load generated')
group.add_option(
'-s',
'--send-rate',
type='float',
default=16.,
metavar='RATE',
help='Rate (item/s) of sending requests as a float, default: %default')
group.add_option(
'-D',
'--duration',
type='float',
default=60.,
metavar='N',
help='Duration (s) of the sending phase of the load test, '
'default: %default')
group.add_option(
'-m',
'--concurrent',
type='int',
default=200,
metavar='N',
help='Maximum concurrent on-going requests, default: %default')
group.add_option(
'-t',
'--timeout',
type='float',
default=15 * 60.,
metavar='N',
help='Task expiration and timeout to get results, the task itself will '
'have %ds less than the value provided. Default: %%default' %
TIMEOUT_OVERHEAD)
group.add_option('-o',
'--output-size',
type='int',
default=100,
metavar='N',
help='Bytes sent to stdout, default: %default')
group.add_option(
'--sleep',
type='int',
default=60,
metavar='N',
help='Amount of time the bot should sleep, e.g. faking work, '
'default: %default')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Display options')
group.add_option('--columns',
type='int',
default=graph.get_console_width(),
metavar='N',
help='For histogram display, default:%default')
group.add_option(
'--buckets',
type='int',
default=20,
metavar='N',
help='Number of buckets for histogram display, default:%default')
parser.add_option_group(group)
parser.add_option('--dump', metavar='FOO.JSON', help='Dumps to json file')
parser.add_option('-v',
'--verbose',
action='store_true',
help='Enables logging')
options, args = parser.parse_args()
logging.basicConfig(
level=logging.INFO if options.verbose else logging.FATAL)
if args:
parser.error('Unsupported args: %s' % args)
options.swarming = options.swarming.rstrip('/')
if not options.swarming:
parser.error('--swarming is required.')
if options.duration <= 0:
parser.error('Needs --duration > 0. 0.01 is a valid value.')
swarming.process_filter_options(parser, options)
total = int(round(options.send_rate * options.duration))
print(
'Sending %.1f i/s for %ds with max %d parallel requests; timeout %.1fs; '
'total %d' % (options.send_rate, options.duration, options.concurrent,
options.timeout, total))
print('[processing/processed/todo]')
# This is used so there's no clash between runs and actual real usage.
unique = ''.join(random.choice(string.ascii_letters) for _ in range(8))
columns = [('processing', 0), ('processed', 0), ('todo', 0)]
progress = threading_utils.Progress(columns)
index = 0
results = []
with threading_utils.ThreadPoolWithProgress(progress, 1,
options.concurrent, 0) as pool:
try:
start = time.time()
while True:
duration = time.time() - start
if duration > options.duration:
break
should_have_triggered_so_far = int(
round(duration * options.send_rate))
while index < should_have_triggered_so_far:
pool.add_task(0, trigger_task, options.swarming,
options.dimensions, options.sleep,
options.output_size, progress, unique,
options.timeout, index)
progress.update_item('', todo=1)
index += 1
progress.print_update()
time.sleep(0.01)
progress.update_item('Getting results for on-going tasks.',
raw=True)
for i in pool.iter_results():
results.append(i)
# This is a bit excessive but it's useful in the case where some tasks
# hangs, so at least partial data is available.
if options.dump:
results.sort()
if os.path.exists(options.dump):
os.rename(options.dump, options.dump + '.old')
with open(options.dump, 'wb') as f:
json.dump(results, f, separators=(',', ':'))
if not options.dump:
results.sort()
except KeyboardInterrupt:
aborted = pool.abort()
progress.update_item('Got Ctrl-C. Aborted %d unsent tasks.' %
aborted,
raw=True,
todo=-aborted)
progress.print_update()
progress.print_update()
# At this point, progress is not used anymore.
print('')
print(' - Took %.1fs.' % (time.time() - start))
print('')
print_results(results, options.columns, options.buckets)
return 0
def main():
colorama.init()
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option(
'-S', '--swarming',
metavar='URL', default='',
help='Swarming server to use')
parser.add_option(
'--suffix', metavar='NAME', default='', help='Bot suffix name to use')
swarming.add_filter_options(parser)
# Use improbable values to reduce the chance of interferring with real slaves.
parser.set_defaults(
dimensions=[
('cpu', ['arm36']),
('hostname', socket.getfqdn()),
('os', OS_NAME),
])
group = optparse.OptionGroup(parser, 'Load generated')
group.add_option(
'--slaves', type='int', default=300, metavar='N',
help='Number of swarm bot slaves, default: %default')
group.add_option(
'-c', '--consume', type='float', default=60., metavar='N',
help='Duration (s) for consuming a request, default: %default')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Display options')
group.add_option(
'--columns', type='int', default=graph.get_console_width(), metavar='N',
help='For histogram display, default:%default')
group.add_option(
'--buckets', type='int', default=20, metavar='N',
help='Number of buckets for histogram display, default:%default')
parser.add_option_group(group)
parser.add_option(
'--dump', metavar='FOO.JSON', help='Dumps to json file')
parser.add_option(
'-v', '--verbose', action='store_true', help='Enables logging')
options, args = parser.parse_args()
logging.basicConfig(level=logging.INFO if options.verbose else logging.FATAL)
if args:
parser.error('Unsupported args: %s' % args)
options.swarming = options.swarming.rstrip('/')
if not options.swarming:
parser.error('--swarming is required.')
if options.consume <= 0:
parser.error('Needs --consume > 0. 0.01 is a valid value.')
swarming.process_filter_options(parser, options)
print(
'Running %d slaves, each task lasting %.1fs' % (
options.slaves, options.consume))
print('Ctrl-C to exit.')
print('[processing/processed/bots]')
columns = [('processing', 0), ('processed', 0), ('bots', 0)]
progress = threading_utils.Progress(columns)
events = Queue.Queue()
start = time.time()
kill_event = threading.Event()
swarm_bot_version_hash = calculate_version(options.swarming + '/bot_code')
hostname = get_hostname()
if options.suffix:
hostname += '-' + options.suffix
slaves = [
FakeSwarmBot(
options.swarming, options.dimensions, swarm_bot_version_hash, hostname, i,
progress, options.consume, events, kill_event)
for i in range(options.slaves)
]
try:
# Wait for all the slaves to come alive.
while not all(s.is_alive() for s in slaves):
time.sleep(0.01)
progress.update_item('Ready to run')
while slaves:
progress.print_update()
time.sleep(0.01)
# The slaves could be told to die.
slaves = [s for s in slaves if s.is_alive()]
except KeyboardInterrupt:
kill_event.set()
progress.update_item('Waiting for slaves to quit.', raw=True)
progress.update_item('')
while slaves:
progress.print_update()
slaves = [s for s in slaves if s.is_alive()]
# At this point, progress is not used anymore.
print('')
print('Ran for %.1fs.' % (time.time() - start))
print('')
results = list(events.queue)
print_results(results, options.columns, options.buckets)
if options.dump:
with open(options.dump, 'w') as f:
json.dump(results, f, separators=(',',':'))
return 0
def main():
colorama.init()
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option(
'-S', '--swarming',
metavar='URL', default='',
help='Swarming server to use')
swarming.add_filter_options(parser)
# Use improbable values to reduce the chance of interferring with real slaves.
parser.set_defaults(
dimensions=[
('bits', '36'),
('machine', os.uname()[4] + '-experimental'),
('os', OS_NAME),
])
group = optparse.OptionGroup(parser, 'Load generated')
group.add_option(
'--slaves', type='int', default=300, metavar='N',
help='Number of swarm bot slaves, default: %default')
group.add_option(
'-c', '--consume', type='float', default=60., metavar='N',
help='Duration (s) for consuming a request, default: %default')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Display options')
group.add_option(
'--columns', type='int', default=graph.get_console_width(), metavar='N',
help='For histogram display, default:%default')
group.add_option(
'--buckets', type='int', default=20, metavar='N',
help='Number of buckets for histogram display, default:%default')
parser.add_option_group(group)
parser.add_option(
'--dump', metavar='FOO.JSON', help='Dumps to json file')
parser.add_option(
'-v', '--verbose', action='store_true', help='Enables logging')
options, args = parser.parse_args()
logging.basicConfig(level=logging.INFO if options.verbose else logging.FATAL)
if args:
parser.error('Unsupported args: %s' % args)
options.swarming = options.swarming.rstrip('/')
if not options.swarming:
parser.error('--swarming is required.')
if options.consume <= 0:
parser.error('Needs --consume > 0. 0.01 is a valid value.')
swarming.process_filter_options(parser, options)
print(
'Running %d slaves, each task lasting %.1fs' % (
options.slaves, options.consume))
print('Ctrl-C to exit.')
print('[processing/processed/bots]')
columns = [('processing', 0), ('processed', 0), ('bots', 0)]
progress = threading_utils.Progress(columns)
events = Queue.Queue()
start = time.time()
kill_event = threading.Event()
swarm_bot_version_hash = calculate_version(
options.swarming + '/get_slave_code')
slaves = [
FakeSwarmBot(
options.swarming, options.dimensions, swarm_bot_version_hash, i, progress,
options.consume, events, kill_event)
for i in range(options.slaves)
]
try:
# Wait for all the slaves to come alive.
while not all(s.is_alive() for s in slaves):
time.sleep(0.01)
progress.update_item('Ready to run')
while slaves:
progress.print_update()
time.sleep(0.01)
# The slaves could be told to die.
slaves = [s for s in slaves if s.is_alive()]
except KeyboardInterrupt:
kill_event.set()
progress.update_item('Waiting for slaves to quit.', raw=True)
progress.update_item('')
while slaves:
progress.print_update()
slaves = [s for s in slaves if s.is_alive()]
# At this point, progress is not used anymore.
print('')
print('Ran for %.1fs.' % (time.time() - start))
print('')
results = events.queue
print_results(results, options.columns, options.buckets)
if options.dump:
with open(options.dump, 'w') as f:
json.dump(results, f, separators=(',',':'))
return 0
def main():
colorama.init()
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option('-S',
'--swarming',
metavar='URL',
default='',
help='Swarming server to use')
swarming.add_filter_options(parser)
parser.set_defaults(dimensions=[('os', swarming_load_test_bot.OS_NAME)])
group = optparse.OptionGroup(parser, 'Load generated')
group.add_option(
'-s',
'--send-rate',
type='float',
default=16.,
metavar='RATE',
help='Rate (item/s) of sending requests as a float, default: %default')
group.add_option(
'-D',
'--duration',
type='float',
default=60.,
metavar='N',
help='Duration (s) of the sending phase of the load test, '
'default: %default')
group.add_option(
'-m',
'--concurrent',
type='int',
default=200,
metavar='N',
help='Maximum concurrent on-going requests, default: %default')
group.add_option('-t',
'--timeout',
type='float',
default=3600.,
metavar='N',
help='Timeout to get results, default: %default')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Display options')
group.add_option('--columns',
type='int',
default=graph.get_console_width(),
metavar='N',
help='For histogram display, default:%default')
group.add_option(
'--buckets',
type='int',
default=20,
metavar='N',
help='Number of buckets for histogram display, default:%default')
parser.add_option_group(group)
parser.add_option('--dump', metavar='FOO.JSON', help='Dumps to json file')
parser.add_option('-v',
'--verbose',
action='store_true',
help='Enables logging')
options, args = parser.parse_args()
logging.basicConfig(
level=logging.INFO if options.verbose else logging.FATAL)
if args:
parser.error('Unsupported args: %s' % args)
options.swarming = options.swarming.rstrip('/')
if not options.swarming:
parser.error('--swarming is required.')
if options.duration <= 0:
parser.error('Needs --duration > 0. 0.01 is a valid value.')
swarming.process_filter_options(parser, options)
total = options.send_rate * options.duration
print(
'Sending %.1f i/s for %ds with max %d parallel requests; timeout %.1fs; '
'total %d' % (options.send_rate, options.duration, options.concurrent,
options.timeout, total))
print('[processing/processed/todo]')
# This is used so there's no clash between runs and actual real usage.
unique = ''.join(random.choice(string.ascii_letters) for _ in range(8))
columns = [('processing', 0), ('processed', 0), ('todo', 0)]
progress = threading_utils.Progress(columns)
index = 0
with threading_utils.ThreadPoolWithProgress(progress, 1,
options.concurrent, 0) as pool:
try:
start = time.time()
while True:
duration = time.time() - start
if duration > options.duration:
break
should_have_triggered_so_far = int(duration *
options.send_rate)
while index < should_have_triggered_so_far:
pool.add_task(0, trigger_task, options.swarming,
options.dimensions, progress, unique,
options.timeout, index)
progress.update_item('', todo=1)
index += 1
progress.print_update()
time.sleep(0.01)
except KeyboardInterrupt:
aborted = pool.abort()
progress.update_item('Got Ctrl-C. Aborted %d unsent tasks.' %
aborted,
raw=True,
todo=-aborted)
progress.print_update()
finally:
# TODO(maruel): We could give up on collecting results for the on-going
# tasks but that would need to be optional.
progress.update_item('Getting results for on-going tasks.',
raw=True)
results = sorted(pool.join())
progress.print_update()
# At this point, progress is not used anymore.
print('')
print(' - Took %.1fs.' % (time.time() - start))
print('')
print_results(results, options.columns, options.buckets)
if options.dump:
with open(options.dump, 'w') as f:
json.dump(results, f, separators=(',', ':'))
return 0
def init():
# should_wrap instructs colorama to wrap stdout/stderr with an ASNI colorcode
# interpreter that converts them to SetConsoleTextAttribute calls. This only
# should be True in cases where we're connected to cmd.exe's console. Setting
# this to True on non-windows systems has no effect.
should_wrap = False
global IS_TTY, OUT_TYPE
IS_TTY = sys.stdout.isatty()
if IS_TTY:
# Yay! We detected a console in the normal way. It doesn't really matter
# if it's windows or not, we win.
OUT_TYPE = 'console'
should_wrap = True
elif sys.platform.startswith('win'):
# assume this is some sort of file
OUT_TYPE = 'file (win)'
import msvcrt
import ctypes
h = msvcrt.get_osfhandle(sys.stdout.fileno())
# h is the win32 HANDLE for stdout.
ftype = ctypes.windll.kernel32.GetFileType(h)
if ftype == 2: # FILE_TYPE_CHAR
# This is a normal cmd console, but we'll only get here if we're running
# inside a `git command` which is actually git->bash->command. Not sure
# why isatty doesn't detect this case.
OUT_TYPE = 'console (cmd via msys)'
IS_TTY = True
should_wrap = True
elif ftype == 3: # FILE_TYPE_PIPE
OUT_TYPE = 'pipe (win)'
# This is some kind of pipe on windows. This could either be a real pipe
# or this could be msys using a pipe to emulate a pty. We use the same
# algorithm that msys-git uses to determine if it's connected to a pty or
# not.
# This function and the structures are defined in the MSDN documentation
# using the same names.
def NT_SUCCESS(status):
# The first two bits of status are the severity. The success
# severities are 0 and 1, and the !success severities are 2 and 3.
# Therefore since ctypes interprets the default restype of the call
# to be an 'C int' (which is guaranteed to be signed 32 bits), All
# success codes are positive, and all !success codes are negative.
return status >= 0
class UNICODE_STRING(ctypes.Structure):
_fields_ = [('Length', ctypes.c_ushort),
('MaximumLength', ctypes.c_ushort),
('Buffer', ctypes.c_wchar_p)]
class OBJECT_NAME_INFORMATION(ctypes.Structure):
_fields_ = [('Name', UNICODE_STRING),
('NameBuffer', ctypes.c_wchar_p)]
buf = ctypes.create_string_buffer('\0', 1024)
# Ask NT what the name of the object our stdout HANDLE is. It would be
# possible to use GetFileInformationByHandleEx, but it's only available
# on Vista+. If you're reading this in 2017 or later, feel free to
# refactor this out.
#
# The '1' here is ObjectNameInformation
if NT_SUCCESS(ctypes.windll.ntdll.NtQueryObject(h, 1, buf, len(buf)-2,
None)):
out = OBJECT_NAME_INFORMATION.from_buffer(buf)
name = out.Name.Buffer.split('\\')[-1]
IS_TTY = name.startswith('msys-') and '-pty' in name
if IS_TTY:
OUT_TYPE = 'bash (msys)'
else:
# A normal file, or an unknown file type.
pass
else:
# This is non-windows, so we trust isatty.
OUT_TYPE = 'pipe or file'
colorama.init(wrap=should_wrap)
def main():
colorama.init()
parser = optparse.OptionParser(description=sys.modules['__main__'].__doc__)
tomorrow = datetime.datetime.utcnow().date() + datetime.timedelta(days=1)
year = datetime.datetime(tomorrow.year, 1, 1)
parser.add_option(
'-S', '--swarming',
metavar='URL', default=os.environ.get('SWARMING_SERVER', ''),
help='Swarming server to use')
group = optparse.OptionGroup(parser, 'Filtering')
group.add_option(
'--start', default=year.strftime('%Y-%m-%d'),
help='Starting date in UTC; defaults to start of year: %default')
group.add_option(
'--end', default=tomorrow.strftime('%Y-%m-%d'),
help='End date in UTC; defaults to tomorrow: %default')
group.add_option(
'--state', default='ALL', type='choice', choices=STATES,
help='State to filter on. Values are: %s' % ', '.join(STATES))
group.add_option(
'--tags', action='append', default=[], help='Tags to filter on')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Presentation')
group.add_option(
'--dimensions', action='store_true', help='Show the dimensions')
group.add_option(
'--daily-count', action='store_true',
help='Show the daily count in raw number instead of histogram')
parser.add_option_group(group)
parser.add_option(
'--json', default='counts.json',
help='File containing raw data; default: %default')
parser.add_option(
'-v', '--verbose', action='count', default=0, help='Log')
options, args = parser.parse_args()
if args:
parser.error('Unsupported argument %s' % args)
logging.basicConfig(level=logging.DEBUG if options.verbose else logging.ERROR)
start = parse_time_option(options.start)
end = parse_time_option(options.end)
print('From %s (%d) to %s (%d)' % (
start, int((start- _EPOCH).total_seconds()),
end, int((end - _EPOCH).total_seconds())))
if options.swarming:
if options.dimensions:
data = fetch_tasks(
options.swarming, start, end, options.state, options.tags)
else:
data = fetch_counts(
options.swarming, start, end, options.state, options.tags)
with open(options.json, 'wb') as f:
json.dump(data, f)
elif not os.path.isfile(options.json):
parser.error('--swarming is required.')
else:
with open(options.json, 'rb') as f:
data = json.load(f)
print('')
if options.dimensions:
present_dimensions(data, options.daily_count)
else:
present_counts(data, options.daily_count)
return 0
def main():
colorama.init()
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option('-I',
'--isolate-server',
metavar='URL',
default='',
help='Isolate server to use')
parser.add_option('--namespace',
default='temporary%d-gzip' % time.time(),
metavar='XX',
help='Namespace to use on the server, default: %default')
parser.add_option('--threads',
type='int',
default=16,
metavar='N',
help='Parallel worker threads to use, default:%default')
graph.unit_option(parser,
'--items',
default=0,
help='Number of items to upload')
graph.unit_option(parser,
'--max-size',
default=0,
help='Loop until this amount of data was transferred')
graph.unit_option(parser,
'--mid-size',
default=100 * 1024,
help='Rough average size of each item, default:%default')
parser.add_option('--columns',
type='int',
default=graph.get_console_width(),
metavar='N',
help='For histogram display, default:%default')
parser.add_option(
'--buckets',
type='int',
default=20,
metavar='N',
help='Number of buckets for histogram display, default:%default')
parser.add_option('--dump', metavar='FOO.JSON', help='Dumps to json file')
parser.add_option('--dry-run',
action='store_true',
help='Do not send anything')
parser.add_option('-v',
'--verbose',
action='store_true',
help='Enable logging')
options, args = parser.parse_args()
logging.basicConfig(
level=logging.INFO if options.verbose else logging.FATAL)
if args:
parser.error('Unsupported args: %s' % args)
if bool(options.max_size) == bool(options.items):
parser.error(
'Use one of --max-size or --items.\n'
' Use --max-size if you want to run it until NN bytes where '
'transfered.\n'
' Otherwise use --items to run it for NN items.')
if not options.dry_run:
options.isolate_server = options.isolate_server.rstrip('/')
if not options.isolate_server:
parser.error('--isolate-server is required.')
print(' - Using %d thread, items=%d, max-size=%d, mid-size=%d' %
(options.threads, options.items, options.max_size, options.mid_size))
if options.dry_run:
print(' - %sDRY RUN MODE%s' %
(colorama.Fore.GREEN, colorama.Fore.RESET))
start = time.time()
random_pool = Randomness()
print(' - Generated pool after %.1fs' % (time.time() - start))
columns = [('index', 0), ('data', 0), ('size', options.items)]
progress = Progress(columns)
api = isolateserver.get_storage_api(options.isolate_server,
options.namespace)
do_item = functools.partial(
send_and_receive, random_pool, options.dry_run,
isolateserver.is_namespace_with_compression(options.namespace), api,
progress)
# TODO(maruel): Handle Ctrl-C should:
# - Stop adding tasks.
# - Stop scheduling tasks in ThreadPool.
# - Wait for the remaining ungoing tasks to complete.
# - Still print details and write the json file.
with threading_utils.ThreadPoolWithProgress(progress, options.threads,
options.threads, 0) as pool:
if options.items:
for _ in xrange(options.items):
pool.add_task(0, do_item, gen_size(options.mid_size))
progress.print_update()
elif options.max_size:
# This one is approximate.
total = 0
while True:
size = gen_size(options.mid_size)
progress.update_item('', size=1)
progress.print_update()
pool.add_task(0, do_item, size)
total += size
if total >= options.max_size:
break
results = sorted(pool.join())
print('')
print(' - Took %.1fs.' % (time.time() - start))
print('')
print_results(results, options.columns, options.buckets)
if options.dump:
with open(options.dump, 'w') as f:
json.dump(results, f, separators=(',', ':'))
return 0
parser.error(
'Must pass one python script to run. Use --help for more details')
# 1. Query the bots list.
bots = get_bot_list(options.swarming, options.dimensions)
print('Found %d bots to process' % len(bots))
if not bots:
return 1
# 2. Archive the script to run.
isolated_hash = archive(options.isolate_server, args[0])
print('Running %s' % isolated_hash)
# 3. Trigger the tasks.
name = os.path.basename(args[0])
if options.serial:
return run_serial(options.swarming, options.isolate_server,
str(options.priority), str(options.deadline),
options.repeat, isolated_hash, name, bots)
return run_parallel(options.swarming, options.isolate_server,
str(options.priority), str(options.deadline),
options.repeat, isolated_hash, name, bots)
if __name__ == '__main__':
fix_encoding.fix_encoding()
tools.disable_buffering()
colorama.init()
sys.exit(main())
def main():
colorama.init()
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option("-S", "--swarming", metavar="URL", default="", help="Swarming server to use")
swarming.add_filter_options(parser)
parser.set_defaults(dimensions=[("os", swarming_load_test_bot.OS_NAME)])
group = optparse.OptionGroup(parser, "Load generated")
group.add_option(
"-s",
"--send-rate",
type="float",
default=16.0,
metavar="RATE",
help="Rate (item/s) of sending requests as a float, default: %default",
)
group.add_option(
"-D",
"--duration",
type="float",
default=60.0,
metavar="N",
help="Duration (s) of the sending phase of the load test, " "default: %default",
)
group.add_option(
"-m",
"--concurrent",
type="int",
default=200,
metavar="N",
help="Maximum concurrent on-going requests, default: %default",
)
group.add_option(
"-t",
"--timeout",
type="float",
default=15 * 60.0,
metavar="N",
help="Task expiration and timeout to get results, the task itself will "
"have %ds less than the value provided. Default: %%default" % TIMEOUT_OVERHEAD,
)
group.add_option(
"-o", "--output-size", type="int", default=100, metavar="N", help="Bytes sent to stdout, default: %default"
)
group.add_option(
"--sleep",
type="int",
default=60,
metavar="N",
help="Amount of time the bot should sleep, e.g. faking work, " "default: %default",
)
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Display options")
group.add_option(
"--columns",
type="int",
default=graph.get_console_width(),
metavar="N",
help="For histogram display, default:%default",
)
group.add_option(
"--buckets",
type="int",
default=20,
metavar="N",
help="Number of buckets for histogram display, default:%default",
)
parser.add_option_group(group)
parser.add_option("--dump", metavar="FOO.JSON", help="Dumps to json file")
parser.add_option("-v", "--verbose", action="store_true", help="Enables logging")
options, args = parser.parse_args()
logging.basicConfig(level=logging.INFO if options.verbose else logging.FATAL)
if args:
parser.error("Unsupported args: %s" % args)
options.swarming = options.swarming.rstrip("/")
if not options.swarming:
parser.error("--swarming is required.")
if options.duration <= 0:
parser.error("Needs --duration > 0. 0.01 is a valid value.")
swarming.process_filter_options(parser, options)
total = int(round(options.send_rate * options.duration))
print(
"Sending %.1f i/s for %ds with max %d parallel requests; timeout %.1fs; "
"total %d" % (options.send_rate, options.duration, options.concurrent, options.timeout, total)
)
print("[processing/processed/todo]")
# This is used so there's no clash between runs and actual real usage.
unique = "".join(random.choice(string.ascii_letters) for _ in range(8))
columns = [("processing", 0), ("processed", 0), ("todo", 0)]
progress = threading_utils.Progress(columns)
index = 0
results = []
with threading_utils.ThreadPoolWithProgress(progress, 1, options.concurrent, 0) as pool:
try:
start = time.time()
while True:
duration = time.time() - start
if duration > options.duration:
break
should_have_triggered_so_far = int(round(duration * options.send_rate))
while index < should_have_triggered_so_far:
pool.add_task(
0,
trigger_task,
options.swarming,
options.dimensions,
options.sleep,
options.output_size,
progress,
unique,
options.timeout,
index,
)
progress.update_item("", todo=1)
index += 1
progress.print_update()
time.sleep(0.01)
progress.update_item("Getting results for on-going tasks.", raw=True)
for i in pool.iter_results():
results.append(i)
# This is a bit excessive but it's useful in the case where some tasks
# hangs, so at least partial data is available.
if options.dump:
results.sort()
if os.path.exists(options.dump):
os.rename(options.dump, options.dump + ".old")
with open(options.dump, "wb") as f:
json.dump(results, f, separators=(",", ":"))
if not options.dump:
results.sort()
except KeyboardInterrupt:
aborted = pool.abort()
progress.update_item("Got Ctrl-C. Aborted %d unsent tasks." % aborted, raw=True, todo=-aborted)
progress.print_update()
progress.print_update()
# At this point, progress is not used anymore.
print("")
print(" - Took %.1fs." % (time.time() - start))
print("")
print_results(results, options.columns, options.buckets)
return 0
