def CMDcollect(parser, args):
  """Retrieves results of a Swarming task.

  The result can be in multiple part if the execution was sharded. It can
  potentially have retries.
  """
  add_collect_options(parser)
  add_sharding_options(parser)
  (options, args) = parser.parse_args(args)
  if not args:
    parser.error('Must specify one task name.')
  elif len(args) > 1:
    parser.error('Must specify only one task name.')

  auth.ensure_logged_in(options.swarming)
  try:
    return collect(
        options.swarming,
        args[0],
        options.shards,
        options.timeout,
        options.decorate,
        options.print_status_updates,
        options.task_summary_json,
        options.task_output_dir)
  except Failure:
    on_error.report(None)
    return 1
Ejemplo n.º 2
0
def CMDquery(parser, args):
    """Returns information about the bots connected to the Swarming server."""
    add_filter_options(parser)
    parser.filter_group.add_option(
        '--dead-only',
        action='store_true',
        help='Only print dead bots, useful to reap them and reimage broken bots'
    )
    parser.filter_group.add_option('-k',
                                   '--keep-dead',
                                   action='store_true',
                                   help='Do not filter out dead bots')
    parser.filter_group.add_option('-b',
                                   '--bare',
                                   action='store_true',
                                   help='Do not print out dimensions')
    options, args = parser.parse_args(args)

    if options.keep_dead and options.dead_only:
        parser.error('Use only one of --keep-dead and --dead-only')

    auth.ensure_logged_in(options.swarming)
    service = net.get_http_service(options.swarming)

    data = service.json_request('GET', '/swarming/api/v1/bots')
    if data is None:
        print >> sys.stderr, 'Failed to access %s' % options.swarming
        return 1
    for machine in natsort.natsorted(data['machines'], key=lambda x: x['id']):
        if options.dead_only:
            if not machine['is_dead']:
                continue
        elif not options.keep_dead and machine['is_dead']:
            continue

        # If the user requested to filter on dimensions, ensure the bot has all the
        # dimensions requested.
        dimensions = machine['dimensions']
        for key, value in options.dimensions:
            if key not in dimensions:
                break
            # A bot can have multiple value for a key, for example,
            # {'os': ['Windows', 'Windows-6.1']}, so that --dimension os=Windows will
            # be accepted.
            if isinstance(dimensions[key], list):
                if value not in dimensions[key]:
                    break
            else:
                if value != dimensions[key]:
                    break
        else:
            print machine['id']
            if not options.bare:
                print '  %s' % dimensions
    return 0
def CMDrun(parser, args):
  """Triggers a task and wait for the results.

  Basically, does everything to run a command remotely.
  """
  add_trigger_options(parser)
  add_collect_options(parser)
  add_sharding_options(parser)
  args, isolated_cmd_args = extract_isolated_command_extra_args(args)
  options, args = parser.parse_args(args)
  process_trigger_options(parser, options, args)

  auth.ensure_logged_in(options.swarming)
  if file_path.is_url(options.isolate_server):
    auth.ensure_logged_in(options.isolate_server)
  try:
    tasks, task_name = trigger(
        swarming=options.swarming,
        isolate_server=options.isolate_server or options.indir,
        namespace=options.namespace,
        file_hash_or_isolated=args[0],
        task_name=options.task_name,
        extra_args=isolated_cmd_args,
        shards=options.shards,
        dimensions=options.dimensions,
        env=dict(options.env),
        deadline=options.deadline,
        verbose=options.verbose,
        profile=options.profile,
        priority=options.priority)
  except Failure as e:
    on_error.report(
        'Failed to trigger %s(%s): %s' %
        (options.task_name, args[0], e.args[0]))
    return 1
  if not tasks:
    on_error.report('Failed to trigger the task.')
    return 1
  if task_name != options.task_name:
    print('Triggered task: %s' % task_name)
  try:
    # TODO(maruel): Use task_ids, it's much more efficient!
    return collect(
        options.swarming,
        task_name,
        options.shards,
        options.timeout,
        options.decorate,
        options.print_status_updates,
        options.task_summary_json,
        options.task_output_dir)
  except Failure:
    on_error.report(None)
    return 1
Ejemplo n.º 4
0
def CMDreproduce(parser, args):
  """Runs a task locally that was triggered on the server.

  This running locally the same commands that have been run on the bot. The data
  downloaded will be in a subdirectory named 'work' of the current working
  directory.
  """
  options, args = parser.parse_args(args)
  if len(args) != 1:
    parser.error('Must specify exactly one task id.')

  url = options.swarming + '/swarming/api/v1/client/task/%s/request' % args[0]
  request = net.url_read_json(url)
  if not request:
    print >> sys.stderr, 'Failed to retrieve request data for the task'
    return 1

  if not os.path.isdir('work'):
    os.mkdir('work')

  swarming_host = urlparse.urlparse(options.swarming).netloc
  properties = request['properties']
  for data_url, _ in properties['data']:
    assert data_url.startswith('https://'), data_url
    data_host = urlparse.urlparse(data_url).netloc
    if data_host != swarming_host:
      auth.ensure_logged_in('https://' + data_host)

    content = net.url_read(data_url)
    if content is None:
      print >> sys.stderr, 'Failed to download %s' % data_url
      return 1
    with zipfile.ZipFile(StringIO.StringIO(content)) as zip_file:
      zip_file.extractall('work')

  env = None
  if properties['env']:
    env = os.environ.copy()
    logging.info('env: %r', properties['env'])
    env.update(
        (k.encode('utf-8'), v.encode('utf-8'))
        for k, v in properties['env'].iteritems())

  exit_code = 0
  for cmd in properties['commands']:
    try:
      c = subprocess.call(cmd, env=env, cwd='work')
    except OSError as e:
      print >> sys.stderr, 'Failed to run: %s' % ' '.join(cmd)
      print >> sys.stderr, str(e)
      c = 1
    if not exit_code:
      exit_code = c
  return exit_code
def CMDtrigger(parser, args):
  """Triggers a Swarming task.

  Accepts either the hash (sha1) of a .isolated file already uploaded or the
  path to an .isolated file to archive, packages it if needed and sends a
  Swarming manifest file to the Swarming server.

  If an .isolated file is specified instead of an hash, it is first archived.

  Passes all extra arguments provided after '--' as additional command line
  arguments for an isolated command specified in *.isolate file.
  """
  add_trigger_options(parser)
  add_sharding_options(parser)
  args, isolated_cmd_args = extract_isolated_command_extra_args(args)
  parser.add_option(
      '--dump-json',
      metavar='FILE',
      help='Dump details about the triggered task(s) to this file as json')
  options, args = parser.parse_args(args)
  process_trigger_options(parser, options, args)

  auth.ensure_logged_in(options.swarming)
  if file_path.is_url(options.isolate_server):
    auth.ensure_logged_in(options.isolate_server)
  try:
    tasks, task_name = trigger(
        swarming=options.swarming,
        isolate_server=options.isolate_server or options.indir,
        namespace=options.namespace,
        file_hash_or_isolated=args[0],
        task_name=options.task_name,
        extra_args=isolated_cmd_args,
        shards=options.shards,
        dimensions=options.dimensions,
        env=dict(options.env),
        deadline=options.deadline,
        verbose=options.verbose,
        profile=options.profile,
        priority=options.priority)
    if tasks:
      if task_name != options.task_name:
        print('Triggered task: %s' % task_name)
      if options.dump_json:
        data = {
          'base_task_name': task_name,
          'tasks': tasks,
        }
        tools.write_json(options.dump_json, data, True)
    return int(not tasks)
  except Failure:
    on_error.report(None)
    return 1
Ejemplo n.º 6
0
def CMDtrigger(parser, args):
    """Triggers a Swarming task.

  Accepts either the hash (sha1) of a .isolated file already uploaded or the
  path to an .isolated file to archive, packages it if needed and sends a
  Swarming manifest file to the Swarming server.

  If an .isolated file is specified instead of an hash, it is first archived.

  Passes all extra arguments provided after '--' as additional command line
  arguments for an isolated command specified in *.isolate file.
  """
    add_trigger_options(parser)
    add_sharding_options(parser)
    args, isolated_cmd_args = extract_isolated_command_extra_args(args)
    parser.add_option(
        '--dump-json',
        metavar='FILE',
        help='Dump details about the triggered task(s) to this file as json')
    options, args = parser.parse_args(args)
    process_trigger_options(parser, options, args)

    auth.ensure_logged_in(options.swarming)
    if file_path.is_url(options.isolate_server):
        auth.ensure_logged_in(options.isolate_server)
    try:
        tasks, task_name = trigger(swarming=options.swarming,
                                   isolate_server=options.isolate_server
                                   or options.indir,
                                   namespace=options.namespace,
                                   file_hash_or_isolated=args[0],
                                   task_name=options.task_name,
                                   extra_args=isolated_cmd_args,
                                   shards=options.shards,
                                   dimensions=options.dimensions,
                                   env=dict(options.env),
                                   deadline=options.deadline,
                                   verbose=options.verbose,
                                   profile=options.profile,
                                   priority=options.priority)
        if tasks:
            if task_name != options.task_name:
                print('Triggered task: %s' % task_name)
            if options.dump_json:
                data = {
                    'base_task_name': task_name,
                    'tasks': tasks,
                }
                tools.write_json(options.dump_json, data, True)
        return int(not tasks)
    except Failure:
        on_error.report(None)
        return 1
Ejemplo n.º 7
0
def CMDquery(parser, args):
  """Returns information about the bots connected to the Swarming server."""
  add_filter_options(parser)
  parser.filter_group.add_option(
      '--dead-only', action='store_true',
      help='Only print dead bots, useful to reap them and reimage broken bots')
  parser.filter_group.add_option(
      '-k', '--keep-dead', action='store_true',
      help='Do not filter out dead bots')
  parser.filter_group.add_option(
      '-b', '--bare', action='store_true',
      help='Do not print out dimensions')
  options, args = parser.parse_args(args)

  if options.keep_dead and options.dead_only:
    parser.error('Use only one of --keep-dead and --dead-only')

  auth.ensure_logged_in(options.swarming)
  service = net.get_http_service(options.swarming)

  data = service.json_request('GET', '/swarming/api/v1/bots')
  if data is None:
    print >> sys.stderr, 'Failed to access %s' % options.swarming
    return 1
  for machine in natsort.natsorted(data['machines'], key=lambda x: x['id']):
    if options.dead_only:
      if not machine['is_dead']:
        continue
    elif not options.keep_dead and machine['is_dead']:
      continue

    # If the user requested to filter on dimensions, ensure the bot has all the
    # dimensions requested.
    dimensions = machine['dimensions']
    for key, value in options.dimensions:
      if key not in dimensions:
        break
      # A bot can have multiple value for a key, for example,
      # {'os': ['Windows', 'Windows-6.1']}, so that --dimension os=Windows will
      # be accepted.
      if isinstance(dimensions[key], list):
        if value not in dimensions[key]:
          break
      else:
        if value != dimensions[key]:
          break
    else:
      print machine['id']
      if not options.bare:
        print '  %s' % dimensions
  return 0
Ejemplo n.º 8
0
def CMDrun(parser, args):
    """Triggers a task and wait for the results.

  Basically, does everything to run a command remotely.
  """
    add_trigger_options(parser)
    add_collect_options(parser)
    add_sharding_options(parser)
    args, isolated_cmd_args = extract_isolated_command_extra_args(args)
    options, args = parser.parse_args(args)
    process_trigger_options(parser, options, args)

    auth.ensure_logged_in(options.swarming)
    if file_path.is_url(options.isolate_server):
        auth.ensure_logged_in(options.isolate_server)
    try:
        tasks, task_name = trigger(swarming=options.swarming,
                                   isolate_server=options.isolate_server
                                   or options.indir,
                                   namespace=options.namespace,
                                   file_hash_or_isolated=args[0],
                                   task_name=options.task_name,
                                   extra_args=isolated_cmd_args,
                                   shards=options.shards,
                                   dimensions=options.dimensions,
                                   env=dict(options.env),
                                   deadline=options.deadline,
                                   verbose=options.verbose,
                                   profile=options.profile,
                                   priority=options.priority)
    except Failure as e:
        on_error.report('Failed to trigger %s(%s): %s' %
                        (options.task_name, args[0], e.args[0]))
        return 1
    if not tasks:
        on_error.report('Failed to trigger the task.')
        return 1
    if task_name != options.task_name:
        print('Triggered task: %s' % task_name)
    try:
        # TODO(maruel): Use task_ids, it's much more efficient!
        return collect(options.swarming, task_name, options.shards,
                       options.timeout, options.decorate,
                       options.print_status_updates, options.task_summary_json,
                       options.task_output_dir)
    except Failure:
        on_error.report(None)
        return 1
Ejemplo n.º 9
0
def CMDcollect(parser, args):
    """Retrieves results of a Swarming task.

  The result can be in multiple part if the execution was sharded. It can
  potentially have retries.
  """
    add_collect_options(parser)
    add_sharding_options(parser)
    (options, args) = parser.parse_args(args)
    if not args:
        parser.error('Must specify one task name.')
    elif len(args) > 1:
        parser.error('Must specify only one task name.')

    auth.ensure_logged_in(options.swarming)
    try:
        return collect(options.swarming, args[0], options.shards,
                       options.timeout, options.decorate,
                       options.print_status_updates, options.task_summary_json,
                       options.task_output_dir)
    except Failure:
        on_error.report(None)
        return 1
Ejemplo n.º 10
0
  def _process_swarming(self, options):
    """Processes the --swarming option and aborts if not specified.

    Returns the identity as determined by the server.
    """
    if not options.swarming:
      self.error('--swarming is required.')
    try:
      options.swarming = net.fix_url(options.swarming)
    except ValueError as e:
      self.error('--swarming %s' % e)
    on_error.report_on_exception_exit(options.swarming)
    try:
      user = auth.ensure_logged_in(options.swarming)
    except ValueError as e:
      self.error(str(e))
    return user
def main(args):
  tools.disable_buffering()
  parser = tools.OptionParserWithLogging(
      usage='%prog <options>',
      version=__version__,
      log_file=RUN_ISOLATED_LOG_FILE)

  data_group = optparse.OptionGroup(parser, 'Data source')
  data_group.add_option(
      '-s', '--isolated',
      metavar='FILE',
      help='File/url describing what to map or run')
  data_group.add_option(
      '-H', '--hash',
      help='Hash of the .isolated to grab from the hash table')
  isolateserver.add_isolate_server_options(data_group, True)
  parser.add_option_group(data_group)

  cache_group = optparse.OptionGroup(parser, 'Cache management')
  cache_group.add_option(
      '--cache',
      default='cache',
      metavar='DIR',
      help='Cache directory, default=%default')
  cache_group.add_option(
      '--max-cache-size',
      type='int',
      metavar='NNN',
      default=20*1024*1024*1024,
      help='Trim if the cache gets larger than this value, default=%default')
  cache_group.add_option(
      '--min-free-space',
      type='int',
      metavar='NNN',
      default=2*1024*1024*1024,
      help='Trim if disk free space becomes lower than this value, '
           'default=%default')
  cache_group.add_option(
      '--max-items',
      type='int',
      metavar='NNN',
      default=100000,
      help='Trim if more than this number of items are in the cache '
           'default=%default')
  parser.add_option_group(cache_group)

  auth.add_auth_options(parser)
  options, args = parser.parse_args(args)
  auth.process_auth_options(parser, options)
  isolateserver.process_isolate_server_options(data_group, options)

  if bool(options.isolated) == bool(options.hash):
    logging.debug('One and only one of --isolated or --hash is required.')
    parser.error('One and only one of --isolated or --hash is required.')

  options.cache = os.path.abspath(options.cache)
  policies = CachePolicies(
      options.max_cache_size, options.min_free_space, options.max_items)

  # |options.cache| path may not exist until DiskCache() instance is created.
  cache = DiskCache(
      options.cache, policies, isolateserver.get_hash_algo(options.namespace))

  remote = options.isolate_server or options.indir
  if file_path.is_url(remote):
    auth.ensure_logged_in(remote)

  with isolateserver.get_storage(remote, options.namespace) as storage:
    # Hashing schemes used by |storage| and |cache| MUST match.
    assert storage.hash_algo == cache.hash_algo
    return run_tha_test(
        options.isolated or options.hash, storage, cache, args)
Ejemplo n.º 12
0
def main(args):
    tools.disable_buffering()
    parser = tools.OptionParserWithLogging(usage='%prog <options>',
                                           version=__version__,
                                           log_file=RUN_ISOLATED_LOG_FILE)

    data_group = optparse.OptionGroup(parser, 'Data source')
    data_group.add_option('-s',
                          '--isolated',
                          metavar='FILE',
                          help='File/url describing what to map or run')
    data_group.add_option(
        '-H',
        '--hash',
        help='Hash of the .isolated to grab from the hash table')
    isolateserver.add_isolate_server_options(data_group, True)
    parser.add_option_group(data_group)

    cache_group = optparse.OptionGroup(parser, 'Cache management')
    cache_group.add_option('--cache',
                           default='cache',
                           metavar='DIR',
                           help='Cache directory, default=%default')
    cache_group.add_option(
        '--max-cache-size',
        type='int',
        metavar='NNN',
        default=20 * 1024 * 1024 * 1024,
        help='Trim if the cache gets larger than this value, default=%default')
    cache_group.add_option(
        '--min-free-space',
        type='int',
        metavar='NNN',
        default=2 * 1024 * 1024 * 1024,
        help='Trim if disk free space becomes lower than this value, '
        'default=%default')
    cache_group.add_option(
        '--max-items',
        type='int',
        metavar='NNN',
        default=100000,
        help='Trim if more than this number of items are in the cache '
        'default=%default')
    parser.add_option_group(cache_group)

    auth.add_auth_options(parser)
    options, args = parser.parse_args(args)
    auth.process_auth_options(parser, options)
    isolateserver.process_isolate_server_options(data_group, options)

    if bool(options.isolated) == bool(options.hash):
        logging.debug('One and only one of --isolated or --hash is required.')
        parser.error('One and only one of --isolated or --hash is required.')

    options.cache = os.path.abspath(options.cache)
    policies = CachePolicies(options.max_cache_size, options.min_free_space,
                             options.max_items)

    # |options.cache| path may not exist until DiskCache() instance is created.
    cache = DiskCache(options.cache, policies,
                      isolateserver.get_hash_algo(options.namespace))

    remote = options.isolate_server or options.indir
    if file_path.is_url(remote):
        auth.ensure_logged_in(remote)

    with isolateserver.get_storage(remote, options.namespace) as storage:
        # Hashing schemes used by |storage| and |cache| MUST match.
        assert storage.hash_algo == cache.hash_algo
        return run_tha_test(options.isolated or options.hash, storage, cache,
                            args)