Example #1
0
def main():
  run_isolated.disable_buffering()
  parser = optparse.OptionParser(
      usage='%prog [options] test_name',
      description=sys.modules[__name__].__doc__)
  parser.add_option(
      '-u', '--url', default='http://localhost:8080',
      help='Specify the url of the Swarm server, defaults: %default')
  parser.add_option(
      '-v', '--verbose', action='store_true',
      help='Print verbose logging')
  parser.add_option(
      '-n', '--no-wait', action='store_true',
      help='Do not wait for completion')
  (options, args) = parser.parse_args()
  if not args:
    parser.error('Must specify one test name.')
  elif len(args) > 1:
    parser.error('Must specify only one test name.')

  url = options.url.rstrip('/')
  test_name = args[0]
  test_keys = get_test_keys(url, test_name)

  outputs = swarm_get_results(url, test_keys, not options.no_wait)
  print_results(outputs)

  exit_codes = []
  for output in outputs:
    exit_codes.extend(map(int, output['exit_codes'].split(',')))

  return max(exit_codes)
def parse_args():
    run_isolated.disable_buffering()
    parser = optparse.OptionParser(usage='%prog [options] test_name',
                                   description=sys.modules[__name__].__doc__)
    parser.add_option(
        '-u',
        '--url',
        default='http://localhost:8080',
        help='Specify the url of the Swarm server, defaults: %default')
    parser.add_option('-v',
                      '--verbose',
                      action='store_true',
                      help='Print verbose logging')
    parser.add_option(
        '-t',
        '--timeout',
        type='float',
        default=run_isolated.URL_OPEN_TIMEOUT,
        help='Timeout to wait for result, set to 0 for no timeout; default: '
        '%default s')
    # TODO(maruel): Remove once the masters have been updated.
    parser.add_option('-s', '--shards', help=optparse.SUPPRESS_HELP)

    (options, args) = parser.parse_args()
    if not args:
        parser.error('Must specify one test name.')
    elif len(args) > 1:
        parser.error('Must specify only one test name.')
    options.url = options.url.rstrip('/')
    logging.basicConfig(
        level=logging.DEBUG if options.verbose else logging.ERROR)
    return parser, options, args[0]
def main():
  """CLI frontend to validate arguments."""
  run_isolated.disable_buffering()
  parser = run_test_cases.OptionParserWithTestShardingAndFiltering(
      usage='%prog <options> [gtest]')

  # Override default seed value to default to 0.
  parser.set_defaults(seed=0)

  options, args = parser.parse_args()
  if not args:
    parser.error('Please provide the executable to run')

  cmd = run_isolated.fix_python_path(args)
  try:
    tests = run_test_cases.chromium_list_test_cases(
        cmd,
        os.getcwd(),
        index=options.index,
        shards=options.shards,
        seed=options.seed,
        disabled=options.disabled,
        fails=options.fails,
        flaky=options.flaky,
        pre=False,
        manual=options.manual)
    for test in tests:
      print test
  except run_test_cases.Failure, e:
    print e.args[0]
    return e.args[1]
Example #4
0
def main():
    run_isolated.disable_buffering()
    parser = optparse.OptionParser(usage='%prog [options] test_name',
                                   description=sys.modules[__name__].__doc__)
    parser.add_option(
        '-u',
        '--url',
        default='http://localhost:8080',
        help='Specify the url of the Swarm server, defaults: %default')
    parser.add_option('-v',
                      '--verbose',
                      action='store_true',
                      help='Print verbose logging')
    parser.add_option('-n',
                      '--no-wait',
                      action='store_true',
                      help='Do not wait for completion')
    (options, args) = parser.parse_args()
    if not args:
        parser.error('Must specify one test name.')
    elif len(args) > 1:
        parser.error('Must specify only one test name.')

    url = options.url.rstrip('/')
    test_name = args[0]
    test_keys = get_test_keys(url, test_name)

    outputs = swarm_get_results(url, test_keys, not options.no_wait)
    print_results(outputs)

    exit_codes = []
    for output in outputs:
        exit_codes.extend(map(int, output['exit_codes'].split(',')))

    return max(exit_codes)
Example #5
0
def parse_args():
  run_isolated.disable_buffering()
  parser = optparse.OptionParser(
      usage='%prog [options] test_name',
      description=sys.modules[__name__].__doc__)
  parser.add_option(
      '-u', '--url', default='http://localhost:8080',
      help='Specify the url of the Swarm server, defaults: %default')
  parser.add_option(
      '-v', '--verbose', action='store_true',
      help='Print verbose logging')
  parser.add_option(
      '-t', '--timeout',
      type='float',
      default=run_isolated.URL_OPEN_TIMEOUT,
      help='Timeout to wait for result, set to 0 for no timeout; default: '
           '%default s')
  # TODO(maruel): Remove once the masters have been updated.
  parser.add_option(
      '-s', '--shards',
      help=optparse.SUPPRESS_HELP)

  (options, args) = parser.parse_args()
  if not args:
    parser.error('Must specify one test name.')
  elif len(args) > 1:
    parser.error('Must specify only one test name.')
  options.url = options.url.rstrip('/')
  logging.basicConfig(level=logging.DEBUG if options.verbose else logging.ERROR)
  return parser, options, args[0]
Example #6
0
def main(args):
    run_isolated.disable_buffering()
    parser = optparse.OptionParser(
        usage="%prog [options] <file1..fileN> or - to read from stdin", description=sys.modules[__name__].__doc__
    )
    parser.add_option("-r", "--remote", help="Remote server to archive to")
    parser.add_option("-v", "--verbose", action="count", default=0, help="Use multiple times to increase verbosity")
    parser.add_option("--namespace", default="default-gzip", help="The namespace to use on the server.")

    options, files = parser.parse_args(args)

    levels = [logging.ERROR, logging.INFO, logging.DEBUG]
    logging.basicConfig(
        level=levels[min(len(levels) - 1, options.verbose)],
        format="[%(threadName)s] %(asctime)s,%(msecs)03d %(levelname)5s" " %(module)15s(%(lineno)3d): %(message)s",
        datefmt="%H:%M:%S",
    )
    if files == ["-"]:
        files = sys.stdin.readlines()

    if not files:
        parser.error("Nothing to upload")
    if not options.remote:
        parser.error("Nowhere to send. Please specify --remote")

    # Load the necessary metadata. This is going to be rewritten eventually to be
    # more efficient.
    infiles = dict((f, {"s": os.stat(f).st_size, "h": sha1_file(f)}) for f in files)

    with run_isolated.Profiler("Archive"):
        return upload_sha1_tree(
            base_url=options.remote, indir=os.getcwd(), infiles=infiles, namespace=options.namespace
        )
Example #7
0
def main(argv):
  run_isolated.disable_buffering()
  parser = optparse.OptionParser(
      usage='%prog [options]', description=sys.modules[__name__].__doc__)
  parser.add_option('-w', '--working_dir', default='swarm_tests',
                    help='Desired working direction on the swarm slave side. '
                    'Defaults to %default.')
  parser.add_option('-o', '--os_image',
                    help='Swarm OS image to request.')
  parser.add_option('-u', '--swarm-url', default='http://localhost:8080',
                    help='Specify the url of the Swarm server. '
                    'Defaults to %default')
  parser.add_option('-d', '--data-server',
                    help='The server where all the test data is stored.')
  parser.add_option('-t', '--test-name-prefix', default='',
                    help='Specify the prefix to give the swarm test request. '
                    'Defaults to %default')
  parser.add_option('--run_from_hash', nargs=4, action='append', default=[],
                    help='Specify a hash to run on swarm. The format is '
                    '(hash, hash_test_name, shards, test_filter). This may be '
                    'used multiple times to send multiple hashes.')
  parser.add_option('-v', '--verbose', action='store_true',
                    help='Print verbose logging')
  parser.add_option('--profile', action='store_true',
                    default=bool(os.environ.get('ISOLATE_DEBUG')),
                    help='Have run_isolated.py print profiling info')
  (options, args) = parser.parse_args(argv)

  if args:
    parser.error('Unknown args: %s' % args)

  if not options.os_image or options.os_image == 'None':
    # This means the Try Server/user wants to use the current OS.
    options.os_image = sys.platform
  if not options.data_server:
    parser.error('Must specify the data directory')

  if not options.run_from_hash:
    parser.error('At least one --run_from_hash is required.')

  highest_exit_code = 0
  try:
    # Send off the hash swarm test requests.
    for (file_sha1, test_name, shards, testfilter) in options.run_from_hash:
      exit_code = ProcessManifest(
          file_sha1,
          options.test_name_prefix + test_name,
          int(shards),
          testfilter,
          options)
      highest_exit_code = max(highest_exit_code, exit_code)
  except Failure as e:
    print >> sys.stderr, e.args[0]
    highest_exit_code = max(1, highest_exit_code)
  return highest_exit_code
Example #8
0
def main():
  run_isolated.disable_buffering()
  parser = optparse.OptionParser(
      usage='%prog [options]', description=sys.modules[__name__].__doc__)
  parser.add_option('-w', '--working_dir', default='swarm_tests',
                    help='Desired working direction on the swarm slave side. '
                    'Defaults to %default.')
  parser.add_option('-o', '--os_image',
                    help='Swarm OS image to request.')
  parser.add_option('-u', '--swarm-url', default='http://localhost:8080',
                    help='Specify the url of the Swarm server. '
                    'Defaults to %default')
  parser.add_option('-d', '--data-server',
                    help='The server where all the test data is stored.')
  parser.add_option('-t', '--test-name-prefix', default='',
                    help='Specify the prefix to give the swarm test request. '
                    'Defaults to %default')
  parser.add_option('--run_from_hash', nargs=4, action='append', default=[],
                    help='Specify a hash to run on swarm. The format is '
                    '(hash, hash_test_name, shards, test_filter). This may be '
                    'used multiple times to send multiple hashes.')
  parser.add_option('-v', '--verbose', action='store_true',
                    help='Print verbose logging')
  (options, args) = parser.parse_args()

  if args:
    parser.error('Unknown args: %s' % args)

  if not options.os_image or options.os_image == 'None':
    # This means the Try Server/user wants to use the current OS.
    options.os_image = sys.platform
  if not options.data_server:
    parser.error('Must specify the data directory')

  highest_exit_code = 0
  try:
    # Send off the hash swarm test requests.
    for (file_sha1, test_name, shards, testfilter) in options.run_from_hash:
      try:
        highest_exit_code = max(highest_exit_code,
                                ProcessManifest(
                                    file_sha1,
                                    options.test_name_prefix + test_name,
                                    int(shards),
                                    testfilter,
                                    options))
      except ValueError:
        print('Unable to process %s because integer not given for shard count' %
              test_name)
        highest_exit_code = max(1, highest_exit_code)
  except Failure as e:
    print >> sys.stderr, e.args[0]
    highest_exit_code = max(1, highest_exit_code)
  return highest_exit_code
Example #9
0
def main():
    run_isolated.disable_buffering()
    parser = optparse.OptionParser()
    parser.add_option('-s',
                      '--isolated',
                      help='.isolated file to profile with.')
    parser.add_option('--largest_files',
                      type='int',
                      help='If this is set, instead of compressing all the '
                      'files, only the large n files will be compressed')
    options, args = parser.parse_args()

    if args:
        parser.error('Unknown args passed in; %s' % args)
    if not options.isolated:
        parser.error('The .isolated file must be given.')

    temp_dir = None
    try:
        temp_dir = tempfile.mkdtemp()

        # Create a directory of the required files
        subprocess.check_call([
            os.path.join(BASE_DIR, 'isolate.py'), 'remap', '-s',
            options.isolated, '--outdir', temp_dir
        ])

        file_set = tree_files(temp_dir)

        if options.largest_files:
            sorted_by_size = sorted(file_set.iteritems(),
                                    key=lambda x: x[1],
                                    reverse=True)
            files_to_compress = sorted_by_size[:options.largest_files]

            for filename, size in files_to_compress:
                print('Compressing %s, uncompressed size %d' %
                      (filename, size))

                profile_compress('zlib', zlib.compressobj, range(10), zip_file,
                                 filename)
                profile_compress('bz2', bz2.BZ2Compressor, range(1, 10),
                                 zip_file, filename)
        else:
            print('Number of files: %s' % len(file_set))
            print('Total size: %s' % sum(file_set.itervalues()))

            # Profile!
            profile_compress('zlib', zlib.compressobj, range(10),
                             zip_directory, temp_dir)
            profile_compress('bz2', bz2.BZ2Compressor, range(1, 10),
                             zip_directory, temp_dir)
    finally:
        shutil.rmtree(temp_dir)
def main():
  """CLI frontend to validate arguments."""
  run_isolated.disable_buffering()
  parser = run_test_cases.OptionParserTestCases(
      usage='%prog <options> [gtest]')
  parser.format_description = lambda *_: parser.description
  parser.add_option(
      '-o', '--out',
      help='output file, defaults to <executable>.test_cases')
  parser.add_option(
      '-r', '--root-dir',
      help='Root directory under which file access should be noted')
  parser.add_option(
      '--trace-blacklist', action='append', default=[],
      help='List of regexp to use as blacklist filter')
  # TODO(maruel): Add support for options.timeout.
  parser.remove_option('--timeout')
  options, args = parser.parse_args()

  if not args:
    parser.error(
        'Please provide the executable line to run, if you need fancy things '
        'like xvfb, start this script from *inside* xvfb, it\'ll be much faster'
        '.')

  cmd = run_isolated.fix_python_path(args)
  cmd[0] = os.path.abspath(cmd[0])
  if not os.path.isfile(cmd[0]):
    parser.error('Tracing failed for: %s\nIt doesn\'t exit' % ' '.join(cmd))

  if not options.out:
    options.out = '%s.test_cases' % cmd[-1]
  options.out = os.path.abspath(options.out)
  if options.root_dir:
    options.root_dir = os.path.abspath(options.root_dir)
  logname = options.out + '.log'

  test_cases = parser.process_gtest_options(cmd, os.getcwd(), options)

  # Then run them.
  print('Tracing...')
  results = trace_test_cases(
      cmd,
      os.getcwd(),
      test_cases,
      options.jobs,
      logname)
  print('Reading trace logs...')
  blacklist = trace_inputs.gen_blacklist(options.trace_blacklist)
  write_details(logname, options.out, options.root_dir, blacklist, results)
  return 0
def main():
  run_isolated.disable_buffering()
  parser = optparse.OptionParser()
  parser.add_option('-s', '--isolated', help='.isolated file to profile with.')
  parser.add_option('--largest_files', type='int',
                    help='If this is set, instead of compressing all the '
                    'files, only the large n files will be compressed')
  options, args = parser.parse_args()

  if args:
    parser.error('Unknown args passed in; %s' % args)
  if not options.isolated:
    parser.error('The .isolated file must be given.')

  temp_dir = None
  try:
    temp_dir = tempfile.mkdtemp()

    # Create a directory of the required files
    subprocess.check_call([os.path.join(ROOT_DIR, 'isolate.py'),
                           'remap',
                           '-s', options.isolated,
                           '--outdir', temp_dir])

    file_set = tree_files(temp_dir)

    if options.largest_files:
      sorted_by_size = sorted(file_set.iteritems(),  key=lambda x: x[1],
                              reverse=True)
      files_to_compress = sorted_by_size[:options.largest_files]

      for filename, size in files_to_compress:
        print('Compressing %s, uncompressed size %d' % (filename, size))

        profile_compress('zlib', zlib.compressobj, range(10), zip_file,
                         filename)
        profile_compress('bz2', bz2.BZ2Compressor, range(1, 10), zip_file,
                         filename)
    else:
      print('Number of files: %s' % len(file_set))
      print('Total size: %s' % sum(file_set.itervalues()))

      # Profile!
      profile_compress('zlib', zlib.compressobj, range(10), zip_directory,
                       temp_dir)
      profile_compress('bz2', bz2.BZ2Compressor, range(1, 10), zip_directory,
                       temp_dir)
  finally:
    shutil.rmtree(temp_dir)
Example #12
0
def main(args=None):
    run_isolated.disable_buffering()
    parser = run_isolated.OptionParserWithLogging(usage="%prog <options> [file1] [file2] ...")
    parser.add_option("-o", "--output", help="Output to file instead of stdout")

    options, args = parser.parse_args(args)

    configs = load_isolates(args)
    data = configs.make_isolate_file()
    if options.output:
        with open(options.output, "wb") as f:
            print_all(configs.file_comment, data, f)
    else:
        print_all(configs.file_comment, data, sys.stdout)
    return 0
def main(args):
  run_isolated.disable_buffering()
  parser = optparse.OptionParser(
      usage='%prog [options] <file1..fileN> or - to read from stdin',
      description=sys.modules[__name__].__doc__)
  parser.add_option('-r', '--remote', help='Remote server to archive to')
  parser.add_option(
        '-v', '--verbose',
        action='count', default=0,
        help='Use multiple times to increase verbosity')
  parser.add_option('--namespace', default='default-gzip',
                    help='The namespace to use on the server.')

  options, files = parser.parse_args(args)

  levels = [logging.ERROR, logging.INFO, logging.DEBUG]
  logging.basicConfig(
      level=levels[min(len(levels)-1, options.verbose)],
      format='[%(threadName)s] %(asctime)s,%(msecs)03d %(levelname)5s'
             ' %(module)15s(%(lineno)3d): %(message)s',
      datefmt='%H:%M:%S')
  if files == ['-']:
    files = sys.stdin.readlines()

  if not files:
    parser.error('Nothing to upload')
  if not options.remote:
    parser.error('Nowhere to send. Please specify --remote')

  # Load the necessary metadata. This is going to be rewritten eventually to be
  # more efficient.
  infiles = dict(
      (
        f,
        {
          's': os.stat(f).st_size,
          'h': sha1_file(f),
        }
      )
      for f in files)

  with run_isolated.Profiler('Archive'):
    return upload_sha1_tree(
        base_url=options.remote,
        indir=os.getcwd(),
        infiles=infiles,
        namespace=options.namespace)
def main():
  run_isolated.disable_buffering()
  parser = optparse.OptionParser(usage='%prog <options> [gtest]')
  parser.disable_interspersed_args()
  parser.add_option(
      '-I', '--index',
      type='int',
      default=os.environ.get('GTEST_SHARD_INDEX'),
      help='Shard index to run')
  parser.add_option(
      '-S', '--shards',
      type='int',
      default=os.environ.get('GTEST_TOTAL_SHARDS'),
      help='Total number of shards to calculate from the --index to run')
  options, args = parser.parse_args()
  env = os.environ.copy()
  env['GTEST_TOTAL_SHARDS'] = str(options.shards)
  env['GTEST_SHARD_INDEX'] = str(options.index)
  return subprocess.call(run_isolated.fix_python_path(args), env=env)
Example #15
0
def main(args):
    run_isolated.disable_buffering()
    parser = optparse.OptionParser(
        usage='%prog [options] <file1..fileN> or - to read from stdin',
        description=sys.modules[__name__].__doc__)
    parser.add_option('-r', '--remote', help='Remote server to archive to')
    parser.add_option('-v',
                      '--verbose',
                      action='count',
                      default=0,
                      help='Use multiple times to increase verbosity')
    parser.add_option('--namespace',
                      default='default-gzip',
                      help='The namespace to use on the server.')

    options, files = parser.parse_args(args)

    levels = [logging.ERROR, logging.INFO, logging.DEBUG]
    logging.basicConfig(
        level=levels[min(len(levels) - 1, options.verbose)],
        format='[%(threadName)s] %(asctime)s,%(msecs)03d %(levelname)5s'
        ' %(module)15s(%(lineno)3d): %(message)s',
        datefmt='%H:%M:%S')
    if files == ['-']:
        files = sys.stdin.readlines()

    if not files:
        parser.error('Nothing to upload')
    if not options.remote:
        parser.error('Nowhere to send. Please specify --remote')

    # Load the necessary metadata. This is going to be rewritten eventually to be
    # more efficient.
    infiles = dict((f, {
        's': os.stat(f).st_size,
        'h': sha1_file(f),
    }) for f in files)

    with run_isolated.Profiler('Archive'):
        return upload_sha1_tree(base_url=options.remote,
                                indir=os.getcwd(),
                                infiles=infiles,
                                namespace=options.namespace)
Example #16
0
def main():
  run_isolated.disable_buffering()
  parser = optparse.OptionParser(usage='%prog <options> [gtest]')
  parser.disable_interspersed_args()
  parser.add_option(
      '-I', '--index',
      type='int',
      default=os.environ.get('GTEST_SHARD_INDEX'),
      help='Shard index to run')
  parser.add_option(
      '-S', '--shards',
      type='int',
      default=os.environ.get('GTEST_TOTAL_SHARDS'),
      help='Total number of shards to calculate from the --index to run')
  options, args = parser.parse_args()
  env = os.environ.copy()
  env['GTEST_TOTAL_SHARDS'] = str(options.shards)
  env['GTEST_SHARD_INDEX'] = str(options.index)
  return subprocess.call(fix_python_path(args), env=env)
def main():
  run_isolated.disable_buffering()
  parser = optparse.OptionParser(
      usage='%prog <options>',
      description=sys.modules[__name__].__doc__)
  parser.add_option('-i', '--isolate', help='.isolate file to use')
  parser.add_option(
      '-s', '--isolated',
      help='.isolated file to use. One of -i or -s must be used.')
  parser.add_option(
      '-o', '--os_image',
      metavar='OS',
      help='Swarm OS image to request. Should be one of the valid sys.platform '
           'values like darwin, linux2 or win32.')
  parser.add_option(
      '-u', '--swarm-url',
      metavar='URL',
      default=SWARM_SERVER,
      help='Specify the url of the Swarm server. Defaults to %default')
  parser.add_option(
      '-d', '--data-server',
      default=ISOLATE_SERVER,
      metavar='URL',
      help='The server where all the test data is stored. Defaults to %default')
  parser.add_option('-v', '--verbose', action='store_true')
  options, args = parser.parse_args()

  if args:
    parser.error('Use one of -i or -s but no unsupported arguments: %s' % args)
  if not options.isolate and not options.isolated:
    parser.error('Use one of -i or -s')

  return doall(
      options.isolate,
      options.isolated,
      options.swarm_url,
      options.data_server,
      options.os_image,
      options.verbose)
def main():
  """CLI frontend to validate arguments."""
  run_isolated.disable_buffering()
  parser = run_test_cases.OptionParserTestCases(
      usage='%prog <options> --isolated <.isolated>')
  parser.format_description = lambda *_: parser.description
  isolate.add_variable_option(parser)
  isolate.add_trace_option(parser)

  # TODO(maruel): Add support for options.timeout.
  parser.remove_option('--timeout')

  options, args = parser.parse_args()
  if args:
    parser.error('Unsupported arg: %s' % args)
  isolate.parse_isolated_option(parser, options, os.getcwd(), True)
  isolate.parse_variable_option(options)

  try:
    config, command, test_cases = safely_load_isolated(parser, options)
    if not command:
      parser.error('A command must be defined')
    if not test_cases:
      parser.error('No test case to run with command: %s' % ' '.join(command))

    config.saved_state.variables.update(options.variables)
    return isolate_test_cases(
        command,
        test_cases,
        options.jobs,
        config.isolated_filepath,
        config.saved_state.isolate_filepath,
        config.root_dir,
        config.saved_state.relative_cwd,
        config.saved_state.variables,
        options.trace_blacklist)
  except isolate.ExecutionError, e:
    print >> sys.stderr, str(e)
    return 1
def main():
  run_isolated.disable_buffering()
  parser = run_test_cases.OptionParserTestCases(
      usage='%prog <options> -s <something.isolated>')
  isolate.add_trace_option(parser)
  parser.add_option(
      '-s', '--isolated',
      help='The isolated file')
  options, args = parser.parse_args()

  if args:
    parser.error('Unsupported arg: %s' % args)
  isolate.parse_isolated_option(parser, options, os.getcwd(), True)

  _, command, test_cases = isolate_test_cases.safely_load_isolated(
      parser, options)
  if not command:
    parser.error('A command must be defined')
  if not test_cases:
    parser.error('No test case to run')
  return not fix_all(
      options.isolated, test_cases, options.trace_blacklist, options.verbose)