def trigger_by_manifest(swarming, manifest):
  """Given a task manifest, triggers it for execution on swarming.

  Args:
    swarming: URL of a swarming service.
    manifest: instance of Manifest.

  Returns:
    tuple(Task id, priority) on success. tuple(None, None) on failure.
  """
  logging.info('Triggering: %s', manifest.task_name)
  manifest_text = manifest.to_json()
  result = net.url_read(swarming + '/test', data={'request': manifest_text})
  if not result:
    on_error.report('Failed to trigger task %s' % manifest.task_name)
    return None, None
  try:
    data = json.loads(result)
  except (ValueError, TypeError):
    msg = '\n'.join((
        'Failed to trigger task %s' % manifest.task_name,
        'Manifest: %s' % manifest_text,
        'Bad response: %s' % result))
    on_error.report(msg)
    return None, None
  if not data:
    return None, None
  return data['test_keys'][0]['test_key'], data['priority']
Esempio n. 2
0
def trigger_by_manifest(swarming, manifest):
    """Given a task manifest, triggers it for execution on swarming.

  Args:
    swarming: URL of a swarming service.
    manifest: instance of Manifest.

  Returns:
    tuple(Task id, priority) on success. tuple(None, None) on failure.
  """
    logging.info('Triggering: %s', manifest.task_name)
    manifest_text = manifest.to_json()
    result = net.url_read(swarming + '/test', data={'request': manifest_text})
    if not result:
        on_error.report('Failed to trigger task %s' % manifest.task_name)
        return None, None
    try:
        data = json.loads(result)
    except (ValueError, TypeError):
        msg = '\n'.join(
            ('Failed to trigger task %s' % manifest.task_name,
             'Manifest: %s' % manifest_text, 'Bad response: %s' % result))
        on_error.report(msg)
        return None, None
    if not data:
        return None, None
    return data['test_keys'][0]['test_key'], data['priority']
Esempio n. 3
0
def swarming_trigger(swarming, raw_request, xsrf_token):
  """Triggers a request on the Swarming server and returns the json data.

  It's the low-level function.

  Returns:
    {
      'request': {
        'created_ts': u'2010-01-02 03:04:05',
        'name': ..
      },
      'task_id': '12300',
    }
  """
  logging.info('Triggering: %s', raw_request['name'])

  headers = {'X-XSRF-Token': xsrf_token}
  result = net.url_read_json(
      swarming + '/swarming/api/v1/client/request',
      data=raw_request,
      headers=headers)
  if not result:
    on_error.report('Failed to trigger task %s' % raw_request['name'])
    return None
  return result
def CMDcollect(parser, args):
  """Retrieves results of a Swarming task.

  The result can be in multiple part if the execution was sharded. It can
  potentially have retries.
  """
  add_collect_options(parser)
  add_sharding_options(parser)
  (options, args) = parser.parse_args(args)
  if not args:
    parser.error('Must specify one task name.')
  elif len(args) > 1:
    parser.error('Must specify only one task name.')

  auth.ensure_logged_in(options.swarming)
  try:
    return collect(
        options.swarming,
        args[0],
        options.shards,
        options.timeout,
        options.decorate,
        options.print_status_updates,
        options.task_summary_json,
        options.task_output_dir)
  except Failure:
    on_error.report(None)
    return 1
Esempio n. 5
0
def CMDtrigger(parser, args):
    """Triggers a Swarming task.

  Accepts either the hash (sha1) of a .isolated file already uploaded or the
  path to an .isolated file to archive, packages it if needed and sends a
  Swarming manifest file to the Swarming server.

  If an .isolated file is specified instead of an hash, it is first archived.

  Passes all extra arguments provided after '--' as additional command line
  arguments for an isolated command specified in *.isolate file.
  """
    add_trigger_options(parser)
    add_sharding_options(parser)
    args, isolated_cmd_args = extract_isolated_command_extra_args(args)
    parser.add_option(
        '--dump-json',
        metavar='FILE',
        help='Dump details about the triggered task(s) to this file as json')
    options, args = parser.parse_args(args)
    process_trigger_options(parser, options, args)

    auth.ensure_logged_in(options.swarming)
    if file_path.is_url(options.isolate_server):
        auth.ensure_logged_in(options.isolate_server)
    try:
        tasks, task_name = trigger(swarming=options.swarming,
                                   isolate_server=options.isolate_server
                                   or options.indir,
                                   namespace=options.namespace,
                                   file_hash_or_isolated=args[0],
                                   task_name=options.task_name,
                                   extra_args=isolated_cmd_args,
                                   shards=options.shards,
                                   dimensions=options.dimensions,
                                   env=dict(options.env),
                                   deadline=options.deadline,
                                   verbose=options.verbose,
                                   profile=options.profile,
                                   priority=options.priority)
        if tasks:
            if task_name != options.task_name:
                print('Triggered task: %s' % task_name)
            if options.dump_json:
                data = {
                    'base_task_name': task_name,
                    'tasks': tasks,
                }
                tools.write_json(options.dump_json, data, True)
        return int(not tasks)
    except Failure:
        on_error.report(None)
        return 1
def CMDtrigger(parser, args):
  """Triggers a Swarming task.

  Accepts either the hash (sha1) of a .isolated file already uploaded or the
  path to an .isolated file to archive, packages it if needed and sends a
  Swarming manifest file to the Swarming server.

  If an .isolated file is specified instead of an hash, it is first archived.

  Passes all extra arguments provided after '--' as additional command line
  arguments for an isolated command specified in *.isolate file.
  """
  add_trigger_options(parser)
  add_sharding_options(parser)
  args, isolated_cmd_args = extract_isolated_command_extra_args(args)
  parser.add_option(
      '--dump-json',
      metavar='FILE',
      help='Dump details about the triggered task(s) to this file as json')
  options, args = parser.parse_args(args)
  process_trigger_options(parser, options, args)

  auth.ensure_logged_in(options.swarming)
  if file_path.is_url(options.isolate_server):
    auth.ensure_logged_in(options.isolate_server)
  try:
    tasks, task_name = trigger(
        swarming=options.swarming,
        isolate_server=options.isolate_server or options.indir,
        namespace=options.namespace,
        file_hash_or_isolated=args[0],
        task_name=options.task_name,
        extra_args=isolated_cmd_args,
        shards=options.shards,
        dimensions=options.dimensions,
        env=dict(options.env),
        deadline=options.deadline,
        verbose=options.verbose,
        profile=options.profile,
        priority=options.priority)
    if tasks:
      if task_name != options.task_name:
        print('Triggered task: %s' % task_name)
      if options.dump_json:
        data = {
          'base_task_name': task_name,
          'tasks': tasks,
        }
        tools.write_json(options.dump_json, data, True)
    return int(not tasks)
  except Failure:
    on_error.report(None)
    return 1
Esempio n. 7
0
def CMDcollect(parser, args):
  """Retrieves results of one or multiple Swarming task by its ID.

  The result can be in multiple part if the execution was sharded. It can
  potentially have retries.
  """
  add_collect_options(parser)
  parser.add_option(
      '-j', '--json',
      help='Load the task ids from .json as saved by trigger --dump-json')
  options, args = parser.parse_args(args)
  if not args and not options.json:
    parser.error('Must specify at least one task id or --json.')
  if args and options.json:
    parser.error('Only use one of task id or --json.')

  if options.json:
    options.json = unicode(os.path.abspath(options.json))
    try:
      with fs.open(options.json, 'rb') as f:
        data = json.load(f)
    except (IOError, ValueError):
      parser.error('Failed to open %s' % options.json)
    try:
      tasks = sorted(
          data['tasks'].itervalues(), key=lambda x: x['shard_index'])
      args = [t['task_id'] for t in tasks]
    except (KeyError, TypeError):
      parser.error('Failed to process %s' % options.json)
    if options.timeout is None:
      options.timeout = (
          data['request']['properties']['execution_timeout_secs'] +
          data['request']['expiration_secs'] + 10.)
  else:
    valid = frozenset('0123456789abcdef')
    if any(not valid.issuperset(task_id) for task_id in args):
      parser.error('Task ids are 0-9a-f.')

  try:
    return collect(
        options.swarming,
        args,
        options.timeout,
        options.decorate,
        options.print_status_updates,
        options.task_summary_json,
        options.task_output_dir,
        options.perf)
  except Failure:
    on_error.report(None)
    return 1
Esempio n. 8
0
def CMDtrigger(parser, args):
  """Triggers a Swarming task.

  Accepts either the hash (sha1) of a .isolated file already uploaded or the
  path to an .isolated file to archive.

  If an .isolated file is specified instead of an hash, it is first archived.

  Passes all extra arguments provided after '--' as additional command line
  arguments for an isolated command specified in *.isolate file.
  """
  add_trigger_options(parser)
  add_sharding_options(parser)
  parser.add_option(
      '--dump-json',
      metavar='FILE',
      help='Dump details about the triggered task(s) to this file as json')
  options, args = parser.parse_args(args)
  task_request = process_trigger_options(parser, options, args)
  try:
    tasks = trigger_task_shards(
        options.swarming, task_request, options.shards)
    if tasks:
      print('Triggered task: %s' % options.task_name)
      tasks_sorted = sorted(
          tasks.itervalues(), key=lambda x: x['shard_index'])
      if options.dump_json:
        data = {
          'base_task_name': options.task_name,
          'tasks': tasks,
          'request': task_request_to_raw_request(task_request),
        }
        tools.write_json(unicode(options.dump_json), data, True)
        print('To collect results, use:')
        print('  swarming.py collect -S %s --json %s' %
            (options.swarming, options.dump_json))
      else:
        print('To collect results, use:')
        print('  swarming.py collect -S %s %s' %
            (options.swarming, ' '.join(t['task_id'] for t in tasks_sorted)))
      print('Or visit:')
      for t in tasks_sorted:
        print('  ' + t['view_url'])
    return int(not tasks)
  except Failure:
    on_error.report(None)
    return 1
def isolated_to_hash(isolate_server, namespace, arg, algo, verbose):
  """Archives a .isolated file if needed.

  Returns the file hash to trigger and a bool specifying if it was a file (True)
  or a hash (False).
  """
  if arg.endswith('.isolated'):
    file_hash = archive(isolate_server, namespace, arg, algo, verbose)
    if not file_hash:
      on_error.report('Archival failure %s' % arg)
      return None, True
    return file_hash, True
  elif isolateserver.is_valid_hash(arg, algo):
    return arg, False
  else:
    on_error.report('Invalid hash %s' % arg)
    return None, False
Esempio n. 10
0
def isolated_to_hash(arg, algo):
    """Archives a .isolated file if needed.

  Returns the file hash to trigger and a bool specifying if it was a file (True)
  or a hash (False).
  """
    if arg.endswith(".isolated"):
        file_hash = isolated_format.hash_file(arg, algo)
        if not file_hash:
            on_error.report("Archival failure %s" % arg)
            return None, True
        return file_hash, True
    elif isolated_format.is_valid_hash(arg, algo):
        return arg, False
    else:
        on_error.report("Invalid hash %s" % arg)
        return None, False
Esempio n. 11
0
def isolated_to_hash(isolate_server, namespace, arg, algo, verbose):
    """Archives a .isolated file if needed.

  Returns the file hash to trigger and a bool specifying if it was a file (True)
  or a hash (False).
  """
    if arg.endswith('.isolated'):
        file_hash = archive(isolate_server, namespace, arg, algo, verbose)
        if not file_hash:
            on_error.report('Archival failure %s' % arg)
            return None, True
        return file_hash, True
    elif isolateserver.is_valid_hash(arg, algo):
        return arg, False
    else:
        on_error.report('Invalid hash %s' % arg)
        return None, False
Esempio n. 12
0
def swarming_trigger(swarming, raw_request):
    """Triggers a request on the Swarming server and returns the json data.

  It's the low-level function.

  Returns:
    {
      'request': {
        'created_ts': u'2010-01-02 03:04:05',
        'name': ..
      },
      'task_id': '12300',
    }
  """
    logging.info("Triggering: %s", raw_request["name"])

    result = net.url_read_json(swarming + "/_ah/api/swarming/v1/tasks/new", data=raw_request)
    if not result:
        on_error.report("Failed to trigger task %s" % raw_request["name"])
        return None
    return result
Esempio n. 13
0
def CMDtrigger(parser, args):
    """Triggers a Swarming task.

  Accepts either the hash (sha1) of a .isolated file already uploaded or the
  path to an .isolated file to archive.

  If an .isolated file is specified instead of an hash, it is first archived.

  Passes all extra arguments provided after '--' as additional command line
  arguments for an isolated command specified in *.isolate file.
  """
    add_trigger_options(parser)
    add_sharding_options(parser)
    parser.add_option(
        "--dump-json", metavar="FILE", help="Dump details about the triggered task(s) to this file as json"
    )
    options, args = parser.parse_args(args)
    task_request = process_trigger_options(parser, options, args)
    try:
        tasks = trigger_task_shards(options.swarming, task_request, options.shards)
        if tasks:
            print ("Triggered task: %s" % options.task_name)
            tasks_sorted = sorted(tasks.itervalues(), key=lambda x: x["shard_index"])
            if options.dump_json:
                data = {"base_task_name": options.task_name, "tasks": tasks}
                tools.write_json(options.dump_json, data, True)
                print ("To collect results, use:")
                print ("  swarming.py collect -S %s --json %s" % (options.swarming, options.dump_json))
            else:
                print ("To collect results, use:")
                print (
                    "  swarming.py collect -S %s %s" % (options.swarming, " ".join(t["task_id"] for t in tasks_sorted))
                )
            print ("Or visit:")
            for t in tasks_sorted:
                print ("  " + t["view_url"])
        return int(not tasks)
    except Failure:
        on_error.report(None)
        return 1
Esempio n. 14
0
def CMDcollect(parser, args):
    """Retrieves results of one or multiple Swarming task by its ID.

  The result can be in multiple part if the execution was sharded. It can
  potentially have retries.
  """
    add_collect_options(parser)
    parser.add_option("-j", "--json", help="Load the task ids from .json as saved by trigger --dump-json")
    options, args = parser.parse_args(args)
    if not args and not options.json:
        parser.error("Must specify at least one task id or --json.")
    if args and options.json:
        parser.error("Only use one of task id or --json.")

    if options.json:
        try:
            with open(options.json) as f:
                tasks = sorted(json.load(f)["tasks"].itervalues(), key=lambda x: x["shard_index"])
                args = [t["task_id"] for t in tasks]
        except (KeyError, IOError, TypeError, ValueError):
            parser.error("Failed to parse %s" % options.json)
    else:
        valid = frozenset("0123456789abcdef")
        if any(not valid.issuperset(task_id) for task_id in args):
            parser.error("Task ids are 0-9a-f.")

    try:
        return collect(
            options.swarming,
            None,
            args,
            options.timeout,
            options.decorate,
            options.print_status_updates,
            options.task_summary_json,
            options.task_output_dir,
        )
    except Failure:
        on_error.report(None)
        return 1
Esempio n. 15
0
def CMDrun(parser, args):
    """Triggers a task and wait for the results.

  Basically, does everything to run a command remotely.
  """
    add_trigger_options(parser)
    add_collect_options(parser)
    add_sharding_options(parser)
    options, args = parser.parse_args(args)
    task_request = process_trigger_options(parser, options, args)
    try:
        tasks = trigger_task_shards(options.swarming, task_request, options.shards)
    except Failure as e:
        on_error.report("Failed to trigger %s(%s): %s" % (options.task_name, args[0], e.args[0]))
        return 1
    if not tasks:
        on_error.report("Failed to trigger the task.")
        return 1
    print ("Triggered task: %s" % options.task_name)
    task_ids = [t["task_id"] for t in sorted(tasks.itervalues(), key=lambda x: x["shard_index"])]
    try:
        return collect(
            options.swarming,
            options.task_name,
            task_ids,
            options.timeout,
            options.decorate,
            options.print_status_updates,
            options.task_summary_json,
            options.task_output_dir,
        )
    except Failure:
        on_error.report(None)
        return 1
Esempio n. 16
0
def CMDcollect(parser, args):
    """Retrieves results of a Swarming task.

  The result can be in multiple part if the execution was sharded. It can
  potentially have retries.
  """
    add_collect_options(parser)
    add_sharding_options(parser)
    (options, args) = parser.parse_args(args)
    if not args:
        parser.error('Must specify one task name.')
    elif len(args) > 1:
        parser.error('Must specify only one task name.')

    auth.ensure_logged_in(options.swarming)
    try:
        return collect(options.swarming, args[0], options.shards,
                       options.timeout, options.decorate,
                       options.print_status_updates, options.task_summary_json,
                       options.task_output_dir)
    except Failure:
        on_error.report(None)
        return 1
Esempio n. 17
0
def swarming_trigger(swarming, raw_request):
  """Triggers a request on the Swarming server and returns the json data.

  It's the low-level function.

  Returns:
    {
      'request': {
        'created_ts': u'2010-01-02 03:04:05',
        'name': ..
      },
      'task_id': '12300',
    }
  """
  logging.info('Triggering: %s', raw_request['name'])

  result = net.url_read_json(
      swarming + '/_ah/api/swarming/v1/tasks/new', data=raw_request)
  if not result:
    on_error.report('Failed to trigger task %s' % raw_request['name'])
    return None
  if result.get('error'):
    # The reply is an error.
    msg = 'Failed to trigger task %s' % raw_request['name']
    if result['error'].get('errors'):
      for err in result['error']['errors']:
        if err.get('message'):
          msg += '\nMessage: %s' % err['message']
        if err.get('debugInfo'):
          msg += '\nDebug info:\n%s' % err['debugInfo']
    elif result['error'].get('message'):
      msg += '\nMessage: %s' % result['error']['message']

    on_error.report(msg)
    return None
  return result
def CMDrun(parser, args):
  """Triggers a task and wait for the results.

  Basically, does everything to run a command remotely.
  """
  add_trigger_options(parser)
  add_collect_options(parser)
  add_sharding_options(parser)
  args, isolated_cmd_args = extract_isolated_command_extra_args(args)
  options, args = parser.parse_args(args)
  process_trigger_options(parser, options, args)

  auth.ensure_logged_in(options.swarming)
  if file_path.is_url(options.isolate_server):
    auth.ensure_logged_in(options.isolate_server)
  try:
    tasks, task_name = trigger(
        swarming=options.swarming,
        isolate_server=options.isolate_server or options.indir,
        namespace=options.namespace,
        file_hash_or_isolated=args[0],
        task_name=options.task_name,
        extra_args=isolated_cmd_args,
        shards=options.shards,
        dimensions=options.dimensions,
        env=dict(options.env),
        deadline=options.deadline,
        verbose=options.verbose,
        profile=options.profile,
        priority=options.priority)
  except Failure as e:
    on_error.report(
        'Failed to trigger %s(%s): %s' %
        (options.task_name, args[0], e.args[0]))
    return 1
  if not tasks:
    on_error.report('Failed to trigger the task.')
    return 1
  if task_name != options.task_name:
    print('Triggered task: %s' % task_name)
  try:
    # TODO(maruel): Use task_ids, it's much more efficient!
    return collect(
        options.swarming,
        task_name,
        options.shards,
        options.timeout,
        options.decorate,
        options.print_status_updates,
        options.task_summary_json,
        options.task_output_dir)
  except Failure:
    on_error.report(None)
    return 1
Esempio n. 19
0
def CMDrun(parser, args):
    """Triggers a task and wait for the results.

  Basically, does everything to run a command remotely.
  """
    add_trigger_options(parser)
    add_collect_options(parser)
    add_sharding_options(parser)
    args, isolated_cmd_args = extract_isolated_command_extra_args(args)
    options, args = parser.parse_args(args)
    process_trigger_options(parser, options, args)

    auth.ensure_logged_in(options.swarming)
    if file_path.is_url(options.isolate_server):
        auth.ensure_logged_in(options.isolate_server)
    try:
        tasks, task_name = trigger(swarming=options.swarming,
                                   isolate_server=options.isolate_server
                                   or options.indir,
                                   namespace=options.namespace,
                                   file_hash_or_isolated=args[0],
                                   task_name=options.task_name,
                                   extra_args=isolated_cmd_args,
                                   shards=options.shards,
                                   dimensions=options.dimensions,
                                   env=dict(options.env),
                                   deadline=options.deadline,
                                   verbose=options.verbose,
                                   profile=options.profile,
                                   priority=options.priority)
    except Failure as e:
        on_error.report('Failed to trigger %s(%s): %s' %
                        (options.task_name, args[0], e.args[0]))
        return 1
    if not tasks:
        on_error.report('Failed to trigger the task.')
        return 1
    if task_name != options.task_name:
        print('Triggered task: %s' % task_name)
    try:
        # TODO(maruel): Use task_ids, it's much more efficient!
        return collect(options.swarming, task_name, options.shards,
                       options.timeout, options.decorate,
                       options.print_status_updates, options.task_summary_json,
                       options.task_output_dir)
    except Failure:
        on_error.report(None)
        return 1
Esempio n. 20
0
def run_shell_out(url, mode):
  # Enable 'report_on_exception_exit' even though main file is *_test.py.
  on_error._is_in_test = lambda: False

  # Hack it out so registering works.
  on_error._ENABLED_DOMAINS = (socket.getfqdn(),)

  # Don't try to authenticate into localhost.
  on_error.net.OAuthAuthenticator = lambda *_: None

  if not on_error.report_on_exception_exit(url):
    print 'Failure to register the handler'
    return 1

  # Hack out certificate verification because we are using a self-signed
  # certificate here. In practice, the SSL certificate is signed to guard
  # against MITM attacks.
  on_error._SERVER.engine.session.verify = False

  if mode == 'crash':
    # Sadly, net is a bit overly verbose, which breaks
    # test_shell_out_crash_server_down.
    logging.error = lambda *_, **_kwargs: None
    logging.warning = lambda *_, **_kwargs: None
    raise ValueError('Oops')

  if mode == 'report':
    # Generate a manual report without an exception frame. Also set the version
    # value.
    setattr(sys.modules['__main__'], '__version__', '123')
    on_error.report('Oh dang')

  if mode == 'exception':
    # Report from inside an exception frame.
    try:
      raise TypeError('You are not my type')
    except TypeError:
      on_error.report('Really')

  if mode == 'exception_no_msg':
    # Report from inside an exception frame.
    try:
      raise TypeError('You are not my type #2')
    except TypeError:
      on_error.report(None)
  return 0
Esempio n. 21
0
def run_shell_out(url, mode):
  # Enable 'report_on_exception_exit' even though main file is *_test.py.
  on_error._is_in_test = lambda: False

  # Hack it out so registering works.
  on_error._ENABLED_DOMAINS = (socket.getfqdn(),)

  # Don't try to authenticate into localhost.
  on_error.net.OAuthAuthenticator = lambda *_: None

  if not on_error.report_on_exception_exit(url):
    print 'Failure to register the handler'
    return 1

  # Hack out certificate verification because we are using a self-signed
  # certificate here. In practice, the SSL certificate is signed to guard
  # against MITM attacks.
  on_error._SERVER.engine.session.verify = False

  if mode == 'crash':
    # Sadly, net is a bit overly verbose, which breaks
    # test_shell_out_crash_server_down.
    logging.error = lambda *_, **_kwargs: None
    logging.warning = lambda *_, **_kwargs: None
    raise ValueError('Oops')

  if mode == 'report':
    # Generate a manual report without an exception frame. Also set the version
    # value.
    setattr(sys.modules['__main__'], '__version__', '123')
    on_error.report('Oh dang')

  if mode == 'exception':
    # Report from inside an exception frame.
    try:
      raise TypeError('You are not my type')
    except TypeError:
      on_error.report('Really')

  if mode == 'exception_no_msg':
    # Report from inside an exception frame.
    try:
      raise TypeError('You are not my type #2')
    except TypeError:
      on_error.report(None)
  return 0
Esempio n. 22
0
def isolated_handle_options(options, args):
  """Handles '--isolated <isolated>', '<isolated>' and '-- <args...>' arguments.

  Returns:
    tuple(command, data).
  """
  isolated_cmd_args = []
  if not options.isolated:
    if '--' in args:
      index = args.index('--')
      isolated_cmd_args = args[index+1:]
      args = args[:index]
    else:
      # optparse eats '--' sometimes.
      isolated_cmd_args = args[1:]
      args = args[:1]
    if len(args) != 1:
      raise ValueError(
          'Use --isolated, --raw-cmd or \'--\' to pass arguments to the called '
          'process.')
    # Old code. To be removed eventually.
    options.isolated, is_file = isolated_to_hash(
        options.isolate_server, options.namespace, args[0],
        isolated_format.get_hash_algo(options.namespace), options.verbose)
    if not options.isolated:
      raise ValueError('Invalid argument %s' % args[0])
  elif args:
    is_file = False
    if '--' in args:
      index = args.index('--')
      isolated_cmd_args = args[index+1:]
      if index != 0:
        raise ValueError('Unexpected arguments.')
    else:
      # optparse eats '--' sometimes.
      isolated_cmd_args = args

  command = isolated_get_run_commands(
      options.isolate_server, options.namespace, options.isolated,
      isolated_cmd_args, options.verbose)

  # If a file name was passed, use its base name of the isolated hash.
  # Otherwise, use user name as an approximation of a task name.
  if not options.task_name:
    if is_file:
      key = os.path.splitext(os.path.basename(args[0]))[0]
    else:
      key = options.user
    options.task_name = u'%s/%s/%s' % (
        key,
        '_'.join(
            '%s=%s' % (k, v)
            for k, v in sorted(options.dimensions.iteritems())),
        options.isolated)

  try:
    data = isolated_get_data(options.isolate_server)
  except (IOError, OSError):
    on_error.report('Failed to upload the zip file')
    raise ValueError('Failed to upload the zip file')

  return command, data
Esempio n. 23
0
def map_and_run(data, constant_run_path):
  """Runs a command with optional isolated input/output.

  Arguments:
  - data: TaskData instance.
  - constant_run_path: TODO

  Returns metadata about the result.
  """
  result = {
    'duration': None,
    'exit_code': None,
    'had_hard_timeout': False,
    'internal_failure': 'run_isolated did not complete properly',
    'stats': {
    # 'isolated': {
    #    'cipd': {
    #      'duration': 0.,
    #      'get_client_duration': 0.,
    #    },
    #    'download': {
    #      'duration': 0.,
    #      'initial_number_items': 0,
    #      'initial_size': 0,
    #      'items_cold': '<large.pack()>',
    #      'items_hot': '<large.pack()>',
    #    },
    #    'upload': {
    #      'duration': 0.,
    #      'items_cold': '<large.pack()>',
    #      'items_hot': '<large.pack()>',
    #    },
    #  },
    },
    # 'cipd_pins': {
    #   'packages': [
    #     {'package_name': ..., 'version': ..., 'path': ...},
    #     ...
    #   ],
    #  'client_package': {'package_name': ..., 'version': ...},
    # },
    'outputs_ref': None,
    'version': 5,
  }

  if data.root_dir:
    file_path.ensure_tree(data.root_dir, 0700)
  elif data.isolate_cache.cache_dir:
    data = data._replace(
        root_dir=os.path.dirname(data.isolate_cache.cache_dir))
  # See comment for these constants.
  # If root_dir is not specified, it is not constant.
  # TODO(maruel): This is not obvious. Change this to become an error once we
  # make the constant_run_path an exposed flag.
  if constant_run_path and data.root_dir:
    run_dir = os.path.join(data.root_dir, ISOLATED_RUN_DIR)
    if os.path.isdir(run_dir):
      file_path.rmtree(run_dir)
    os.mkdir(run_dir, 0700)
  else:
    run_dir = make_temp_dir(ISOLATED_RUN_DIR, data.root_dir)
  # storage should be normally set but don't crash if it is not. This can happen
  # as Swarming task can run without an isolate server.
  out_dir = make_temp_dir(
      ISOLATED_OUT_DIR, data.root_dir) if data.storage else None
  tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, data.root_dir)
  cwd = run_dir
  if data.relative_cwd:
    cwd = os.path.normpath(os.path.join(cwd, data.relative_cwd))
  command = data.command
  try:
    with data.install_packages_fn(run_dir) as cipd_info:
      if cipd_info:
        result['stats']['cipd'] = cipd_info.stats
        result['cipd_pins'] = cipd_info.pins

      if data.isolated_hash:
        isolated_stats = result['stats'].setdefault('isolated', {})
        bundle, isolated_stats['download'] = fetch_and_map(
            isolated_hash=data.isolated_hash,
            storage=data.storage,
            cache=data.isolate_cache,
            outdir=run_dir,
            use_symlinks=data.use_symlinks)
        change_tree_read_only(run_dir, bundle.read_only)
        # Inject the command
        if not command and bundle.command:
          command = bundle.command + data.extra_args
          # Only set the relative directory if the isolated file specified a
          # command, and no raw command was specified.
          if bundle.relative_cwd:
            cwd = os.path.normpath(os.path.join(cwd, bundle.relative_cwd))

      if not command:
        # Handle this as a task failure, not an internal failure.
        sys.stderr.write(
            '<No command was specified!>\n'
            '<Please secify a command when triggering your Swarming task>\n')
        result['exit_code'] = 1
        return result

      if not cwd.startswith(run_dir):
        # Handle this as a task failure, not an internal failure. This is a
        # 'last chance' way to gate against directory escape.
        sys.stderr.write('<Relative CWD is outside of run directory!>\n')
        result['exit_code'] = 1
        return result

      if not os.path.isdir(cwd):
        # Accepts relative_cwd that does not exist.
        os.makedirs(cwd, 0700)

      # If we have an explicit list of files to return, make sure their
      # directories exist now.
      if data.storage and data.outputs:
        isolateserver.create_directories(run_dir, data.outputs)

      command = tools.fix_python_path(command)
      command = process_command(command, out_dir, data.bot_file)
      file_path.ensure_command_has_abs_path(command, cwd)

      with data.install_named_caches(run_dir):
        sys.stdout.flush()
        start = time.time()
        try:
          # Need to switch the default account before 'get_command_env' call,
          # so it can grab correct value of LUCI_CONTEXT env var.
          with set_luci_context_account(data.switch_to_account, tmp_dir):
            env = get_command_env(
                tmp_dir, cipd_info, run_dir, data.env, data.env_prefix)
            result['exit_code'], result['had_hard_timeout'] = run_command(
                command, cwd, env, data.hard_timeout, data.grace_period)
        finally:
          result['duration'] = max(time.time() - start, 0)

    # We successfully ran the command, set internal_failure back to
    # None (even if the command failed, it's not an internal error).
    result['internal_failure'] = None
  except Exception as e:
    # An internal error occurred. Report accordingly so the swarming task will
    # be retried automatically.
    logging.exception('internal failure: %s', e)
    result['internal_failure'] = str(e)
    on_error.report(None)

  # Clean up
  finally:
    try:
      # Try to link files to the output directory, if specified.
      if out_dir:
        link_outputs_to_outdir(run_dir, out_dir, data.outputs)

      success = False
      if data.leak_temp_dir:
        success = True
        logging.warning(
            'Deliberately leaking %s for later examination', run_dir)
      else:
        # On Windows rmtree(run_dir) call above has a synchronization effect: it
        # finishes only when all task child processes terminate (since a running
        # process locks *.exe file). Examine out_dir only after that call
        # completes (since child processes may write to out_dir too and we need
        # to wait for them to finish).
        if fs.isdir(run_dir):
          try:
            success = file_path.rmtree(run_dir)
          except OSError as e:
            logging.error('Failure with %s', e)
            success = False
          if not success:
            sys.stderr.write(OUTLIVING_ZOMBIE_MSG % ('run', data.grace_period))
            if result['exit_code'] == 0:
              result['exit_code'] = 1
        if fs.isdir(tmp_dir):
          try:
            success = file_path.rmtree(tmp_dir)
          except OSError as e:
            logging.error('Failure with %s', e)
            success = False
          if not success:
            sys.stderr.write(OUTLIVING_ZOMBIE_MSG % ('temp', data.grace_period))
            if result['exit_code'] == 0:
              result['exit_code'] = 1

      # This deletes out_dir if leak_temp_dir is not set.
      if out_dir:
        isolated_stats = result['stats'].setdefault('isolated', {})
        result['outputs_ref'], success, isolated_stats['upload'] = (
            delete_and_upload(data.storage, out_dir, data.leak_temp_dir))
      if not success and result['exit_code'] == 0:
        result['exit_code'] = 1
    except Exception as e:
      # Swallow any exception in the main finally clause.
      if out_dir:
        logging.exception('Leaking out_dir %s: %s', out_dir, e)
      result['internal_failure'] = str(e)
  return result
Esempio n. 24
0
def map_and_run(
    isolated_hash, storage, cache, leak_temp_dir, root_dir, hard_timeout,
    grace_period, extra_args):
  """Maps and run the command. Returns metadata about the result."""
  # TODO(maruel): Include performance statistics.
  result = {
    'exit_code': None,
    'had_hard_timeout': False,
    'internal_failure': None,
    'outputs_ref': None,
    'version': 2,
  }
  if root_dir:
    if not fs.isdir(root_dir):
      fs.makedirs(root_dir, 0700)
    prefix = u''
  else:
    root_dir = os.path.dirname(cache.cache_dir) if cache.cache_dir else None
    prefix = u'isolated_'
  run_dir = make_temp_dir(prefix + u'run', root_dir)
  out_dir = make_temp_dir(prefix + u'out', root_dir)
  tmp_dir = make_temp_dir(prefix + u'tmp', root_dir)
  try:
    bundle = isolateserver.fetch_isolated(
        isolated_hash=isolated_hash,
        storage=storage,
        cache=cache,
        outdir=run_dir,
        require_command=True)

    change_tree_read_only(run_dir, bundle.read_only)
    cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
    command = bundle.command + extra_args
    file_path.ensure_command_has_abs_path(command, cwd)
    result['exit_code'], result['had_hard_timeout'] = run_command(
        process_command(command, out_dir), cwd, tmp_dir, hard_timeout,
        grace_period)
  except Exception as e:
    # An internal error occured. Report accordingly so the swarming task will be
    # retried automatically.
    logging.exception('internal failure: %s', e)
    result['internal_failure'] = str(e)
    on_error.report(None)
  finally:
    try:
      if leak_temp_dir:
        logging.warning(
            'Deliberately leaking %s for later examination', run_dir)
      else:
        # On Windows rmtree(run_dir) call above has a synchronization effect: it
        # finishes only when all task child processes terminate (since a running
        # process locks *.exe file). Examine out_dir only after that call
        # completes (since child processes may write to out_dir too and we need
        # to wait for them to finish).
        if fs.isdir(run_dir):
          try:
            success = file_path.rmtree(run_dir)
          except OSError as e:
            logging.error('Failure with %s', e)
            success = False
          if not success:
            print >> sys.stderr, (
                'Failed to delete the run directory, forcibly failing\n'
                'the task because of it. No zombie process can outlive a\n'
                'successful task run and still be marked as successful.\n'
                'Fix your stuff.')
            if result['exit_code'] == 0:
              result['exit_code'] = 1
        if fs.isdir(tmp_dir):
          try:
            success = file_path.rmtree(tmp_dir)
          except OSError as e:
            logging.error('Failure with %s', e)
            success = False
          if not success:
            print >> sys.stderr, (
                'Failed to delete the temporary directory, forcibly failing\n'
                'the task because of it. No zombie process can outlive a\n'
                'successful task run and still be marked as successful.\n'
                'Fix your stuff.')
            if result['exit_code'] == 0:
              result['exit_code'] = 1

      # This deletes out_dir if leak_temp_dir is not set.
      result['outputs_ref'], success = delete_and_upload(
          storage, out_dir, leak_temp_dir)
      if not success and result['exit_code'] == 0:
        result['exit_code'] = 1
    except Exception as e:
      # Swallow any exception in the main finally clause.
      logging.exception('Leaking out_dir %s: %s', out_dir, e)
      result['internal_failure'] = str(e)
  return result
Esempio n. 25
0
def map_and_run(command, isolated_hash, storage, isolate_cache, outputs,
                init_named_caches, leak_temp_dir, root_dir, hard_timeout,
                grace_period, bot_file, install_packages_fn, use_symlinks,
                constant_run_path):
    """Runs a command with optional isolated input/output.

  See run_tha_test for argument documentation.

  Returns metadata about the result.
  """
    assert isinstance(command, list), command
    assert root_dir or root_dir is None
    result = {
        'duration': None,
        'exit_code': None,
        'had_hard_timeout': False,
        'internal_failure': None,
        'stats': {
            # 'isolated': {
            #    'cipd': {
            #      'duration': 0.,
            #      'get_client_duration': 0.,
            #    },
            #    'download': {
            #      'duration': 0.,
            #      'initial_number_items': 0,
            #      'initial_size': 0,
            #      'items_cold': '<large.pack()>',
            #      'items_hot': '<large.pack()>',
            #    },
            #    'upload': {
            #      'duration': 0.,
            #      'items_cold': '<large.pack()>',
            #      'items_hot': '<large.pack()>',
            #    },
            #  },
        },
        # 'cipd_pins': {
        #   'packages': [
        #     {'package_name': ..., 'version': ..., 'path': ...},
        #     ...
        #   ],
        #  'client_package': {'package_name': ..., 'version': ...},
        # },
        'outputs_ref': None,
        'version': 5,
    }

    if root_dir:
        file_path.ensure_tree(root_dir, 0700)
    elif isolate_cache.cache_dir:
        root_dir = os.path.dirname(isolate_cache.cache_dir)
    # See comment for these constants.
    # If root_dir is not specified, it is not constant.
    # TODO(maruel): This is not obvious. Change this to become an error once we
    # make the constant_run_path an exposed flag.
    if constant_run_path and root_dir:
        run_dir = os.path.join(root_dir, ISOLATED_RUN_DIR)
        os.mkdir(run_dir)
    else:
        run_dir = make_temp_dir(ISOLATED_RUN_DIR, root_dir)
    # storage should be normally set but don't crash if it is not. This can happen
    # as Swarming task can run without an isolate server.
    out_dir = make_temp_dir(ISOLATED_OUT_DIR, root_dir) if storage else None
    tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, root_dir)
    cwd = run_dir

    try:
        with install_packages_fn(run_dir) as cipd_info:
            if cipd_info:
                result['stats']['cipd'] = cipd_info.stats
                result['cipd_pins'] = cipd_info.pins

            if isolated_hash:
                isolated_stats = result['stats'].setdefault('isolated', {})
                bundle, isolated_stats['download'] = fetch_and_map(
                    isolated_hash=isolated_hash,
                    storage=storage,
                    cache=isolate_cache,
                    outdir=run_dir,
                    use_symlinks=use_symlinks)
                change_tree_read_only(run_dir, bundle.read_only)
                cwd = os.path.normpath(os.path.join(cwd, bundle.relative_cwd))
                # Inject the command
                if bundle.command:
                    command = bundle.command + command

            if not command:
                # Handle this as a task failure, not an internal failure.
                sys.stderr.write(
                    '<No command was specified!>\n'
                    '<Please secify a command when triggering your Swarming task>\n'
                )
                result['exit_code'] = 1
                return result

            # If we have an explicit list of files to return, make sure their
            # directories exist now.
            if storage and outputs:
                isolateserver.create_directories(run_dir, outputs)

            command = tools.fix_python_path(command)
            command = process_command(command, out_dir, bot_file)
            file_path.ensure_command_has_abs_path(command, cwd)

            with init_named_caches(run_dir):
                sys.stdout.flush()
                start = time.time()
                try:
                    result['exit_code'], result[
                        'had_hard_timeout'] = run_command(
                            command, cwd, get_command_env(tmp_dir, cipd_info),
                            hard_timeout, grace_period)
                finally:
                    result['duration'] = max(time.time() - start, 0)
    except Exception as e:
        # An internal error occurred. Report accordingly so the swarming task will
        # be retried automatically.
        logging.exception('internal failure: %s', e)
        result['internal_failure'] = str(e)
        on_error.report(None)

    # Clean up
    finally:
        try:
            # Try to link files to the output directory, if specified.
            if out_dir:
                link_outputs_to_outdir(run_dir, out_dir, outputs)

            success = False
            if leak_temp_dir:
                success = True
                logging.warning(
                    'Deliberately leaking %s for later examination', run_dir)
            else:
                # On Windows rmtree(run_dir) call above has a synchronization effect: it
                # finishes only when all task child processes terminate (since a running
                # process locks *.exe file). Examine out_dir only after that call
                # completes (since child processes may write to out_dir too and we need
                # to wait for them to finish).
                if fs.isdir(run_dir):
                    try:
                        success = file_path.rmtree(run_dir)
                    except OSError as e:
                        logging.error('Failure with %s', e)
                        success = False
                    if not success:
                        print >> sys.stderr, (
                            'Failed to delete the run directory, thus failing the task.\n'
                            'This may be due to a subprocess outliving the main task\n'
                            'process, holding on to resources. Please fix the task so\n'
                            'that it releases resources and cleans up subprocesses.'
                        )
                        if result['exit_code'] == 0:
                            result['exit_code'] = 1
                if fs.isdir(tmp_dir):
                    try:
                        success = file_path.rmtree(tmp_dir)
                    except OSError as e:
                        logging.error('Failure with %s', e)
                        success = False
                    if not success:
                        print >> sys.stderr, (
                            'Failed to delete the temp directory, thus failing the task.\n'
                            'This may be due to a subprocess outliving the main task\n'
                            'process, holding on to resources. Please fix the task so\n'
                            'that it releases resources and cleans up subprocesses.'
                        )
                        if result['exit_code'] == 0:
                            result['exit_code'] = 1

            # This deletes out_dir if leak_temp_dir is not set.
            if out_dir:
                isolated_stats = result['stats'].setdefault('isolated', {})
                result['outputs_ref'], success, isolated_stats['upload'] = (
                    delete_and_upload(storage, out_dir, leak_temp_dir))
            if not success and result['exit_code'] == 0:
                result['exit_code'] = 1
        except Exception as e:
            # Swallow any exception in the main finally clause.
            if out_dir:
                logging.exception('Leaking out_dir %s: %s', out_dir, e)
            result['internal_failure'] = str(e)
    return result
Esempio n. 26
0
def map_and_run(
    isolated_hash, storage, cache, leak_temp_dir, root_dir, hard_timeout,
    grace_period, extra_args):
  """Maps and run the command. Returns metadata about the result."""
  result = {
    'duration': None,
    'exit_code': None,
    'had_hard_timeout': False,
    'internal_failure': None,
    'stats': {
    #  'download': {
    #    'duration': 0.,
    #    'initial_number_items': 0,
    #    'initial_size': 0,
    #    'items_cold': '<large.pack()>',
    #    'items_hot': '<large.pack()>',
    #  },
    #  'upload': {
    #    'duration': 0.,
    #    'items_cold': '<large.pack()>',
    #    'items_hot': '<large.pack()>',
    #  },
    },
    'outputs_ref': None,
    'version': 3,
  }
  if root_dir:
    if not fs.isdir(root_dir):
      fs.makedirs(root_dir, 0700)
    prefix = u''
  else:
    root_dir = os.path.dirname(cache.cache_dir) if cache.cache_dir else None
    prefix = u'isolated_'
  run_dir = make_temp_dir(prefix + u'run', root_dir)
  out_dir = make_temp_dir(prefix + u'out', root_dir)
  tmp_dir = make_temp_dir(prefix + u'tmp', root_dir)
  try:
    start = time.time()
    bundle = isolateserver.fetch_isolated(
        isolated_hash=isolated_hash,
        storage=storage,
        cache=cache,
        outdir=run_dir)
    if not bundle.command:
      # Handle this as a task failure, not an internal failure.
      sys.stderr.write(
          '<The .isolated doesn\'t declare any command to run!>\n'
          '<Check your .isolate for missing \'command\' variable>\n')
      if os.environ.get('SWARMING_TASK_ID'):
        # Give an additional hint when running as a swarming task.
        sys.stderr.write('<This occurs at the \'isolate\' step>\n')
      result['exit_code'] = 1
      return result
    result['stats']['download'] = {
      'duration': time.time() - start,
      'initial_number_items': cache.initial_number_items,
      'initial_size': cache.initial_size,
      'items_cold': base64.b64encode(large.pack(sorted(cache.added))),
      'items_hot': base64.b64encode(
          large.pack(sorted(set(cache.linked) - set(cache.added)))),
    }

    change_tree_read_only(run_dir, bundle.read_only)
    cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
    command = bundle.command + extra_args
    file_path.ensure_command_has_abs_path(command, cwd)
    sys.stdout.flush()
    start = time.time()
    try:
      result['exit_code'], result['had_hard_timeout'] = run_command(
          process_command(command, out_dir), cwd, tmp_dir, hard_timeout,
          grace_period)
    finally:
      result['duration'] = max(time.time() - start, 0)
  except Exception as e:
    # An internal error occured. Report accordingly so the swarming task will be
    # retried automatically.
    logging.exception('internal failure: %s', e)
    result['internal_failure'] = str(e)
    on_error.report(None)
  finally:
    try:
      if leak_temp_dir:
        logging.warning(
            'Deliberately leaking %s for later examination', run_dir)
      else:
        # On Windows rmtree(run_dir) call above has a synchronization effect: it
        # finishes only when all task child processes terminate (since a running
        # process locks *.exe file). Examine out_dir only after that call
        # completes (since child processes may write to out_dir too and we need
        # to wait for them to finish).
        if fs.isdir(run_dir):
          try:
            success = file_path.rmtree(run_dir)
          except OSError as e:
            logging.error('Failure with %s', e)
            success = False
          if not success:
            print >> sys.stderr, (
                'Failed to delete the run directory, forcibly failing\n'
                'the task because of it. No zombie process can outlive a\n'
                'successful task run and still be marked as successful.\n'
                'Fix your stuff.')
            if result['exit_code'] == 0:
              result['exit_code'] = 1
        if fs.isdir(tmp_dir):
          try:
            success = file_path.rmtree(tmp_dir)
          except OSError as e:
            logging.error('Failure with %s', e)
            success = False
          if not success:
            print >> sys.stderr, (
                'Failed to delete the temporary directory, forcibly failing\n'
                'the task because of it. No zombie process can outlive a\n'
                'successful task run and still be marked as successful.\n'
                'Fix your stuff.')
            if result['exit_code'] == 0:
              result['exit_code'] = 1

      # This deletes out_dir if leak_temp_dir is not set.
      start = time.time()
      result['outputs_ref'], success, cold, hot = delete_and_upload(
          storage, out_dir, leak_temp_dir)
      result['stats']['upload'] = {
        'duration': time.time() - start,
        'items_cold': base64.b64encode(large.pack(cold)),
        'items_hot': base64.b64encode(large.pack(hot)),
      }
      if not success and result['exit_code'] == 0:
        result['exit_code'] = 1
    except Exception as e:
      # Swallow any exception in the main finally clause.
      logging.exception('Leaking out_dir %s: %s', out_dir, e)
      result['internal_failure'] = str(e)
  return result
Esempio n. 27
0
def run_tha_test(isolated_hash, storage, cache, leak_temp_dir, extra_args):
  """Downloads the dependencies in the cache, hardlinks them into a temporary
  directory and runs the executable from there.

  A temporary directory is created to hold the output files. The content inside
  this directory will be uploaded back to |storage| packaged as a .isolated
  file.

  Arguments:
    isolated_hash: the SHA-1 of the .isolated file that must be retrieved to
                   recreate the tree of files to run the target executable.
    storage: an isolateserver.Storage object to retrieve remote objects. This
             object has a reference to an isolateserver.StorageApi, which does
             the actual I/O.
    cache: an isolateserver.LocalCache to keep from retrieving the same objects
           constantly by caching the objects retrieved. Can be on-disk or
           in-memory.
    leak_temp_dir: if true, the temporary directory will be deliberately leaked
                   for later examination.
    extra_args: optional arguments to add to the command stated in the .isolate
                file.
  """
  run_dir = make_temp_dir(u'run_tha_test', cache.cache_dir)
  out_dir = unicode(make_temp_dir(u'isolated_out', cache.cache_dir))
  result = 0
  try:
    try:
      bundle = isolateserver.fetch_isolated(
          isolated_hash=isolated_hash,
          storage=storage,
          cache=cache,
          outdir=run_dir,
          require_command=True)
    except isolated_format.IsolatedError:
      on_error.report(None)
      return 1

    change_tree_read_only(run_dir, bundle.read_only)
    cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
    command = bundle.command + extra_args

    file_path.ensure_command_has_abs_path(command, cwd)
    command = process_command(command, out_dir)
    logging.info('Running %s, cwd=%s' % (command, cwd))

    # TODO(csharp): This should be specified somewhere else.
    # TODO(vadimsh): Pass it via 'env_vars' in manifest.
    # Add a rotating log file if one doesn't already exist.
    env = os.environ.copy()
    if MAIN_DIR:
      env.setdefault('RUN_TEST_CASES_LOG_FILE',
          os.path.join(MAIN_DIR, RUN_TEST_CASES_LOG))
    sys.stdout.flush()
    with tools.Profiler('RunTest'):
      try:
        with subprocess42.Popen_with_handler(command, cwd=cwd, env=env) as p:
          p.communicate()
          result = p.returncode
      except OSError:
        on_error.report('Failed to run %s; cwd=%s' % (command, cwd))
        result = 1
    logging.info(
        'Command finished with exit code %d (%s)',
        result, hex(0xffffffff & result))
  finally:
    try:
      if leak_temp_dir:
        logging.warning('Deliberately leaking %s for later examination',
                        run_dir)
      else:
        try:
          if not file_path.rmtree(run_dir):
            print >> sys.stderr, (
                'Failed to delete the temporary directory, forcibly failing\n'
                'the task because of it. No zombie process can outlive a\n'
                'successful task run and still be marked as successful.\n'
                'Fix your stuff.')
            result = result or 1
        except OSError:
          logging.warning('Leaking %s', run_dir)
          result = 1

      # HACK(vadimsh): On Windows rmtree(run_dir) call above has
      # a synchronization effect: it finishes only when all task child processes
      # terminate (since a running process locks *.exe file). Examine out_dir
      # only after that call completes (since child processes may
      # write to out_dir too and we need to wait for them to finish).

      # Upload out_dir and generate a .isolated file out of this directory.
      # It is only done if files were written in the directory.
      if os.path.isdir(out_dir) and os.listdir(out_dir):
        with tools.Profiler('ArchiveOutput'):
          results = isolateserver.archive_files_to_storage(
              storage, [out_dir], None)
        # TODO(maruel): Implement side-channel to publish this information.
        output_data = {
          'hash': results[0][0],
          'namespace': storage.namespace,
          'storage': storage.location,
        }
        sys.stdout.flush()
        print(
            '[run_isolated_out_hack]%s[/run_isolated_out_hack]' %
            tools.format_json(output_data, dense=True))

    finally:
      try:
        if os.path.isdir(out_dir) and not file_path.rmtree(out_dir):
          result = result or 1
      except OSError:
        # The error was already printed out. Report it but that's it. Only
        # report on non-Windows or on Windows when the process had succeeded.
        # Due to the way file sharing works on Windows, it's sadly expected that
        # file deletion may fail when a test failed.
        if sys.platform != 'win32' or not result:
          on_error.report(None)
        result = 1

  return result
Esempio n. 28
0
def map_and_run(command, isolated_hash, storage, isolate_cache, outputs,
                init_name_caches, leak_temp_dir, root_dir, hard_timeout,
                grace_period, bot_file, extra_args, install_packages_fn,
                use_symlinks):
    """Runs a command with optional isolated input/output.

  See run_tha_test for argument documentation.

  Returns metadata about the result.
  """
    assert root_dir or root_dir is None
    assert bool(command) ^ bool(isolated_hash)
    result = {
        'duration': None,
        'exit_code': None,
        'had_hard_timeout': False,
        'internal_failure': None,
        'stats': {
            # 'isolated': {
            #    'cipd': {
            #      'duration': 0.,
            #      'get_client_duration': 0.,
            #    },
            #    'download': {
            #      'duration': 0.,
            #      'initial_number_items': 0,
            #      'initial_size': 0,
            #      'items_cold': '<large.pack()>',
            #      'items_hot': '<large.pack()>',
            #    },
            #    'upload': {
            #      'duration': 0.,
            #      'items_cold': '<large.pack()>',
            #      'items_hot': '<large.pack()>',
            #    },
            #  },
        },
        # 'cipd_pins': {
        #   'packages': [
        #     {'package_name': ..., 'version': ..., 'path': ...},
        #     ...
        #   ],
        #  'client_package': {'package_name': ..., 'version': ...},
        # },
        'outputs_ref': None,
        'version': 5,
    }

    if root_dir:
        file_path.ensure_tree(root_dir, 0700)
    elif isolate_cache.cache_dir:
        root_dir = os.path.dirname(isolate_cache.cache_dir)
    # See comment for these constants.
    run_dir = make_temp_dir(ISOLATED_RUN_DIR, root_dir)
    # storage should be normally set but don't crash if it is not. This can happen
    # as Swarming task can run without an isolate server.
    out_dir = make_temp_dir(ISOLATED_OUT_DIR, root_dir) if storage else None
    tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, root_dir)
    cwd = run_dir

    try:
        cipd_info = install_packages_fn(run_dir)
        if cipd_info:
            result['stats']['cipd'] = cipd_info['stats']
            result['cipd_pins'] = cipd_info['cipd_pins']

        if isolated_hash:
            isolated_stats = result['stats'].setdefault('isolated', {})
            bundle, isolated_stats['download'] = fetch_and_map(
                isolated_hash=isolated_hash,
                storage=storage,
                cache=isolate_cache,
                outdir=run_dir,
                use_symlinks=use_symlinks)
            if not bundle.command:
                # Handle this as a task failure, not an internal failure.
                sys.stderr.write(
                    '<The .isolated doesn\'t declare any command to run!>\n'
                    '<Check your .isolate for missing \'command\' variable>\n')
                if os.environ.get('SWARMING_TASK_ID'):
                    # Give an additional hint when running as a swarming task.
                    sys.stderr.write('<This occurs at the \'isolate\' step>\n')
                result['exit_code'] = 1
                return result

            change_tree_read_only(run_dir, bundle.read_only)
            cwd = os.path.normpath(os.path.join(cwd, bundle.relative_cwd))
            command = bundle.command + extra_args

        # If we have an explicit list of files to return, make sure their
        # directories exist now.
        if storage and outputs:
            isolateserver.create_directories(run_dir, outputs)

        command = tools.fix_python_path(command)
        command = process_command(command, out_dir, bot_file)
        file_path.ensure_command_has_abs_path(command, cwd)

        init_name_caches(run_dir)

        sys.stdout.flush()
        start = time.time()
        try:
            result['exit_code'], result['had_hard_timeout'] = run_command(
                command, cwd, tmp_dir, hard_timeout, grace_period)
        finally:
            result['duration'] = max(time.time() - start, 0)
    except Exception as e:
        # An internal error occurred. Report accordingly so the swarming task will
        # be retried automatically.
        logging.exception('internal failure: %s', e)
        result['internal_failure'] = str(e)
        on_error.report(None)

    # Clean up
    finally:
        try:
            # Try to link files to the output directory, if specified.
            if out_dir:
                link_outputs_to_outdir(run_dir, out_dir, outputs)

            success = False
            if leak_temp_dir:
                success = True
                logging.warning(
                    'Deliberately leaking %s for later examination', run_dir)
            else:
                # On Windows rmtree(run_dir) call above has a synchronization effect: it
                # finishes only when all task child processes terminate (since a running
                # process locks *.exe file). Examine out_dir only after that call
                # completes (since child processes may write to out_dir too and we need
                # to wait for them to finish).
                if fs.isdir(run_dir):
                    try:
                        success = file_path.rmtree(run_dir)
                    except OSError as e:
                        logging.error('Failure with %s', e)
                        success = False
                    if not success:
                        print >> sys.stderr, (
                            'Failed to delete the run directory, forcibly failing\n'
                            'the task because of it. No zombie process can outlive a\n'
                            'successful task run and still be marked as successful.\n'
                            'Fix your stuff.')
                        if result['exit_code'] == 0:
                            result['exit_code'] = 1
                if fs.isdir(tmp_dir):
                    try:
                        success = file_path.rmtree(tmp_dir)
                    except OSError as e:
                        logging.error('Failure with %s', e)
                        success = False
                    if not success:
                        print >> sys.stderr, (
                            'Failed to delete the temporary directory, forcibly failing\n'
                            'the task because of it. No zombie process can outlive a\n'
                            'successful task run and still be marked as successful.\n'
                            'Fix your stuff.')
                        if result['exit_code'] == 0:
                            result['exit_code'] = 1

            # This deletes out_dir if leak_temp_dir is not set.
            if out_dir:
                isolated_stats = result['stats'].setdefault('isolated', {})
                result['outputs_ref'], success, isolated_stats['upload'] = (
                    delete_and_upload(storage, out_dir, leak_temp_dir))
            if not success and result['exit_code'] == 0:
                result['exit_code'] = 1
        except Exception as e:
            # Swallow any exception in the main finally clause.
            if out_dir:
                logging.exception('Leaking out_dir %s: %s', out_dir, e)
            result['internal_failure'] = str(e)
    return result
def map_and_run(isolated_hash, storage, cache, leak_temp_dir, root_dir, extra_args):
    """Maps and run the command. Returns metadata about the result."""
    # TODO(maruel): Include performance statistics.
    result = {"exit_code": None, "internal_failure": None, "outputs_ref": None, "version": 1}
    if root_dir:
        if not os.path.isdir(root_dir):
            os.makedirs(root_dir, 0700)
        prefix = u""
    else:
        root_dir = os.path.dirname(cache.cache_dir) if cache.cache_dir else None
        prefix = u"isolated_"
    run_dir = make_temp_dir(prefix + u"run", root_dir)
    out_dir = make_temp_dir(prefix + u"out", root_dir)
    tmp_dir = make_temp_dir(prefix + u"tmp", root_dir)
    try:
        bundle = isolateserver.fetch_isolated(
            isolated_hash=isolated_hash, storage=storage, cache=cache, outdir=run_dir, require_command=True
        )

        change_tree_read_only(run_dir, bundle.read_only)
        cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
        command = bundle.command + extra_args
        file_path.ensure_command_has_abs_path(command, cwd)
        result["exit_code"] = run_command(process_command(command, out_dir), cwd, tmp_dir)
    except Exception as e:
        # An internal error occured. Report accordingly so the swarming task will be
        # retried automatically.
        logging.error("internal failure: %s", e)
        result["internal_failure"] = str(e)
        on_error.report(None)
    finally:
        try:
            if leak_temp_dir:
                logging.warning("Deliberately leaking %s for later examination", run_dir)
            else:
                if os.path.isdir(run_dir) and not file_path.rmtree(run_dir):
                    # On Windows rmtree(run_dir) call above has a synchronization effect:
                    # it finishes only when all task child processes terminate (since a
                    # running process locks *.exe file). Examine out_dir only after that
                    # call completes (since child processes may write to out_dir too and
                    # we need to wait for them to finish).
                    print >>sys.stderr, (
                        "Failed to delete the run directory, forcibly failing\n"
                        "the task because of it. No zombie process can outlive a\n"
                        "successful task run and still be marked as successful.\n"
                        "Fix your stuff."
                    )
                    if result["exit_code"] == 0:
                        result["exit_code"] = 1
                if os.path.isdir(tmp_dir) and not file_path.rmtree(tmp_dir):
                    print >>sys.stderr, (
                        "Failed to delete the temporary directory, forcibly failing\n"
                        "the task because of it. No zombie process can outlive a\n"
                        "successful task run and still be marked as successful.\n"
                        "Fix your stuff."
                    )
                    if result["exit_code"] == 0:
                        result["exit_code"] = 1

            # This deletes out_dir if leak_temp_dir is not set.
            result["outputs_ref"], success = delete_and_upload(storage, out_dir, leak_temp_dir)
            if not success and result["exit_code"] == 0:
                result["exit_code"] = 1
        except Exception as e:
            # Swallow any exception in the main finally clause.
            logging.error("Leaking out_dir %s: %s", out_dir, e)
            result["internal_failure"] = str(e)
    return result
def run_tha_test(isolated_hash, storage, cache, extra_args):
  """Downloads the dependencies in the cache, hardlinks them into a temporary
  directory and runs the executable from there.

  A temporary directory is created to hold the output files. The content inside
  this directory will be uploaded back to |storage| packaged as a .isolated
  file.

  Arguments:
    isolated_hash: the sha-1 of the .isolated file that must be retrieved to
                   recreate the tree of files to run the target executable.
    storage: an isolateserver.Storage object to retrieve remote objects. This
             object has a reference to an isolateserver.StorageApi, which does
             the actual I/O.
    cache: an isolateserver.LocalCache to keep from retrieving the same objects
           constantly by caching the objects retrieved. Can be on-disk or
           in-memory.
    extra_args: optional arguments to add to the command stated in the .isolate
                file.
  """
  run_dir = make_temp_dir('run_tha_test', cache.cache_dir)
  out_dir = unicode(make_temp_dir('isolated_out', cache.cache_dir))
  result = 0
  try:
    try:
      settings = isolateserver.fetch_isolated(
          isolated_hash=isolated_hash,
          storage=storage,
          cache=cache,
          outdir=run_dir,
          require_command=True)
    except isolateserver.ConfigError:
      on_error.report(None)
      return 1

    change_tree_read_only(run_dir, settings.read_only)
    cwd = os.path.normpath(os.path.join(run_dir, settings.relative_cwd))
    command = settings.command + extra_args

    # subprocess.call doesn't consider 'cwd' when searching for executable.
    # Yet isolate can specify command relative to 'cwd'. Convert it to absolute
    # path if necessary.
    if not os.path.isabs(command[0]):
      command[0] = os.path.abspath(os.path.join(cwd, command[0]))
    command = process_command(command, out_dir)
    logging.info('Running %s, cwd=%s' % (command, cwd))

    # TODO(csharp): This should be specified somewhere else.
    # TODO(vadimsh): Pass it via 'env_vars' in manifest.
    # Add a rotating log file if one doesn't already exist.
    env = os.environ.copy()
    if MAIN_DIR:
      env.setdefault('RUN_TEST_CASES_LOG_FILE',
          os.path.join(MAIN_DIR, RUN_TEST_CASES_LOG))
    try:
      sys.stdout.flush()
      with tools.Profiler('RunTest'):
        result = subprocess.call(command, cwd=cwd, env=env)
        logging.info(
            'Command finished with exit code %d (%s)',
            result, hex(0xffffffff & result))
    except OSError:
      on_error.report('Failed to run %s; cwd=%s' % (command, cwd))
      result = 1

  finally:
    try:
      try:
        if not rmtree(run_dir):
          print >> sys.stderr, (
              'Failed to delete the temporary directory, forcibly failing the\n'
              'task because of it. No zombie process can outlive a successful\n'
              'task run and still be marked as successful. Fix your stuff.')
          result = result or 1
      except OSError:
        logging.warning('Leaking %s', run_dir)
        result = 1

      # HACK(vadimsh): On Windows rmtree(run_dir) call above has
      # a synchronization effect: it finishes only when all task child processes
      # terminate (since a running process locks *.exe file). Examine out_dir
      # only after that call completes (since child processes may
      # write to out_dir too and we need to wait for them to finish).

      # Upload out_dir and generate a .isolated file out of this directory.
      # It is only done if files were written in the directory.
      if os.listdir(out_dir):
        with tools.Profiler('ArchiveOutput'):
          results = isolateserver.archive_files_to_storage(
              storage, [out_dir], None)
        # TODO(maruel): Implement side-channel to publish this information.
        output_data = {
          'hash': results[0][0],
          'namespace': storage.namespace,
          'storage': storage.location,
        }
        sys.stdout.flush()
        print(
            '[run_isolated_out_hack]%s[/run_isolated_out_hack]' %
            tools.format_json(output_data, dense=True))

    finally:
      try:
        if os.path.isdir(out_dir) and not rmtree(out_dir):
          result = result or 1
      except OSError:
        # The error was already printed out. Report it but that's it.
        on_error.report(None)
        result = 1

  return result
def trigger_task_shards(
    swarming, isolate_server, namespace, isolated_hash, task_name, extra_args,
    shards, dimensions, env, deadline, verbose, profile, priority):
  """Triggers multiple subtasks of a sharded task.

  Returns:
    Dict with task details, returned to caller as part of --dump-json output.
    None in case of failure.
  """
  # Collects all files that are necessary to bootstrap a task execution
  # on the bot. Usually it includes self contained run_isolated.zip and
  # a bunch of small other scripts. All heavy files are pulled
  # by run_isolated.zip. Updated in 'setup_run_isolated'.
  bundle = zip_package.ZipPackage(ROOT_DIR)

  # Make a separate Manifest for each shard, put shard index and number of
  # shards into env and subtask name.
  manifests = []
  for index in xrange(shards):
    manifest = Manifest(
        isolate_server=isolate_server,
        namespace=namespace,
        isolated_hash=isolated_hash,
        task_name=get_shard_task_name(task_name, shards, index),
        extra_args=extra_args,
        dimensions=dimensions,
        env=setup_googletest(env, shards, index),
        deadline=deadline,
        verbose=verbose,
        profile=profile,
        priority=priority)
    setup_run_isolated(manifest, bundle)
    manifests.append(manifest)

  # Upload zip bundle file to get its URL.
  try:
    bundle_url = upload_zip_bundle(isolate_server, bundle)
  except (IOError, OSError):
    on_error.report('Failed to upload the zip file for task %s' % task_name)
    return None, None

  # Attach that file to all manifests.
  for manifest in manifests:
    manifest.add_bundled_file('swarm_data.zip', bundle_url)

  # Trigger all the subtasks.
  tasks = {}
  priority_warning = False
  for index, manifest in enumerate(manifests):
    task_id, priority = trigger_by_manifest(swarming, manifest)
    if not task_id:
      break
    if not priority_warning and priority != manifest.priority:
      priority_warning = True
      print >> sys.stderr, 'Priority was reset to %s' % priority
    tasks[manifest.task_name] = {
      'shard_index': index,
      'task_id': task_id,
      'view_url': '%s/user/task/%s' % (swarming, task_id),
    }

  # Some shards weren't triggered. Abort everything.
  if len(tasks) != len(manifests):
    if tasks:
      print >> sys.stderr, 'Not all shards were triggered'
      for task_dict in tasks.itervalues():
        abort_task(swarming, task_dict['task_id'])
    return None

  return tasks
Esempio n. 32
0
def map_and_run(isolated_hash, storage, cache, leak_temp_dir, root_dir,
                extra_args):
    """Maps and run the command. Returns metadata about the result."""
    # TODO(maruel): Include performance statistics.
    result = {
        'exit_code': None,
        'internal_failure': None,
        'outputs_ref': None,
        'version': 1,
    }
    if root_dir:
        if not os.path.isdir(root_dir):
            os.makedirs(root_dir, 0700)
        prefix = u''
    else:
        root_dir = os.path.dirname(
            cache.cache_dir) if cache.cache_dir else None
        prefix = u'isolated_'
    run_dir = make_temp_dir(prefix + u'run', root_dir)
    out_dir = make_temp_dir(prefix + u'out', root_dir)
    tmp_dir = make_temp_dir(prefix + u'tmp', root_dir)
    try:
        bundle = isolateserver.fetch_isolated(isolated_hash=isolated_hash,
                                              storage=storage,
                                              cache=cache,
                                              outdir=run_dir,
                                              require_command=True)

        change_tree_read_only(run_dir, bundle.read_only)
        cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
        command = bundle.command + extra_args
        file_path.ensure_command_has_abs_path(command, cwd)
        result['exit_code'] = run_command(process_command(command, out_dir),
                                          cwd, tmp_dir)
    except Exception as e:
        # An internal error occured. Report accordingly so the swarming task will be
        # retried automatically.
        logging.error('internal failure: %s', e)
        result['internal_failure'] = str(e)
        on_error.report(None)
    finally:
        try:
            if leak_temp_dir:
                logging.warning(
                    'Deliberately leaking %s for later examination', run_dir)
            else:
                if os.path.isdir(run_dir) and not file_path.rmtree(run_dir):
                    # On Windows rmtree(run_dir) call above has a synchronization effect:
                    # it finishes only when all task child processes terminate (since a
                    # running process locks *.exe file). Examine out_dir only after that
                    # call completes (since child processes may write to out_dir too and
                    # we need to wait for them to finish).
                    print >> sys.stderr, (
                        'Failed to delete the run directory, forcibly failing\n'
                        'the task because of it. No zombie process can outlive a\n'
                        'successful task run and still be marked as successful.\n'
                        'Fix your stuff.')
                    if result['exit_code'] == 0:
                        result['exit_code'] = 1
                if os.path.isdir(tmp_dir) and not file_path.rmtree(tmp_dir):
                    print >> sys.stderr, (
                        'Failed to delete the temporary directory, forcibly failing\n'
                        'the task because of it. No zombie process can outlive a\n'
                        'successful task run and still be marked as successful.\n'
                        'Fix your stuff.')
                    if result['exit_code'] == 0:
                        result['exit_code'] = 1

            # This deletes out_dir if leak_temp_dir is not set.
            result['outputs_ref'], success = delete_and_upload(
                storage, out_dir, leak_temp_dir)
            if not success and result['exit_code'] == 0:
                result['exit_code'] = 1
        except Exception as e:
            # Swallow any exception in the main finally clause.
            logging.error('Leaking out_dir %s: %s', out_dir, e)
            result['internal_failure'] = str(e)
    return result
Esempio n. 33
0
def map_and_run(isolated_hash, storage, cache, leak_temp_dir, root_dir,
                hard_timeout, grace_period, extra_args):
    """Maps and run the command. Returns metadata about the result."""
    # TODO(maruel): Include performance statistics.
    result = {
        'exit_code': None,
        'had_hard_timeout': False,
        'internal_failure': None,
        'outputs_ref': None,
        'version': 2,
    }
    if root_dir:
        if not fs.isdir(root_dir):
            fs.makedirs(root_dir, 0700)
        prefix = u''
    else:
        root_dir = os.path.dirname(
            cache.cache_dir) if cache.cache_dir else None
        prefix = u'isolated_'
    run_dir = make_temp_dir(prefix + u'run', root_dir)
    out_dir = make_temp_dir(prefix + u'out', root_dir)
    tmp_dir = make_temp_dir(prefix + u'tmp', root_dir)
    try:
        try:
            bundle = isolateserver.fetch_isolated(isolated_hash=isolated_hash,
                                                  storage=storage,
                                                  cache=cache,
                                                  outdir=run_dir,
                                                  require_command=True)
        except isolateserver.IsolatedErrorNoCommand:
            # Handle this as a task failure, not an internal failure.
            sys.stderr.write(
                '<The .isolated doesn\'t declare any command to run!>\n'
                '<Check your .isolate for missing \'command\' variable>\n')
            if os.environ.get('SWARMING_TASK_ID'):
                # Give an additional hint when running as a swarming task.
                sys.stderr.write('<This occurs at the \'isolate\' step>\n')
            result['exit_code'] = 1
            return result

        change_tree_read_only(run_dir, bundle.read_only)
        cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
        command = bundle.command + extra_args
        file_path.ensure_command_has_abs_path(command, cwd)
        result['exit_code'], result['had_hard_timeout'] = run_command(
            process_command(command, out_dir), cwd, tmp_dir, hard_timeout,
            grace_period)
    except Exception as e:
        # An internal error occured. Report accordingly so the swarming task will be
        # retried automatically.
        logging.exception('internal failure: %s', e)
        result['internal_failure'] = str(e)
        on_error.report(None)
    finally:
        try:
            if leak_temp_dir:
                logging.warning(
                    'Deliberately leaking %s for later examination', run_dir)
            else:
                # On Windows rmtree(run_dir) call above has a synchronization effect: it
                # finishes only when all task child processes terminate (since a running
                # process locks *.exe file). Examine out_dir only after that call
                # completes (since child processes may write to out_dir too and we need
                # to wait for them to finish).
                if fs.isdir(run_dir):
                    try:
                        success = file_path.rmtree(run_dir)
                    except OSError as e:
                        logging.error('Failure with %s', e)
                        success = False
                    if not success:
                        print >> sys.stderr, (
                            'Failed to delete the run directory, forcibly failing\n'
                            'the task because of it. No zombie process can outlive a\n'
                            'successful task run and still be marked as successful.\n'
                            'Fix your stuff.')
                        if result['exit_code'] == 0:
                            result['exit_code'] = 1
                if fs.isdir(tmp_dir):
                    try:
                        success = file_path.rmtree(tmp_dir)
                    except OSError as e:
                        logging.error('Failure with %s', e)
                        success = False
                    if not success:
                        print >> sys.stderr, (
                            'Failed to delete the temporary directory, forcibly failing\n'
                            'the task because of it. No zombie process can outlive a\n'
                            'successful task run and still be marked as successful.\n'
                            'Fix your stuff.')
                        if result['exit_code'] == 0:
                            result['exit_code'] = 1

            # This deletes out_dir if leak_temp_dir is not set.
            result['outputs_ref'], success = delete_and_upload(
                storage, out_dir, leak_temp_dir)
            if not success and result['exit_code'] == 0:
                result['exit_code'] = 1
        except Exception as e:
            # Swallow any exception in the main finally clause.
            logging.exception('Leaking out_dir %s: %s', out_dir, e)
            result['internal_failure'] = str(e)
    return result
Esempio n. 34
0
def run_tha_test(isolated_hash, storage, cache, leak_temp_dir, extra_args):
  """Downloads the dependencies in the cache, hardlinks them into a temporary
  directory and runs the executable from there.

  A temporary directory is created to hold the output files. The content inside
  this directory will be uploaded back to |storage| packaged as a .isolated
  file.

  Arguments:
    isolated_hash: the SHA-1 of the .isolated file that must be retrieved to
                   recreate the tree of files to run the target executable.
    storage: an isolateserver.Storage object to retrieve remote objects. This
             object has a reference to an isolateserver.StorageApi, which does
             the actual I/O.
    cache: an isolateserver.LocalCache to keep from retrieving the same objects
           constantly by caching the objects retrieved. Can be on-disk or
           in-memory.
    leak_temp_dir: if true, the temporary directory will be deliberately leaked
                   for later examination.
    extra_args: optional arguments to add to the command stated in the .isolate
                file.
  """
  tmp_root = os.path.dirname(cache.cache_dir) if cache.cache_dir else None
  run_dir = make_temp_dir(u'run_tha_test', tmp_root)
  out_dir = unicode(make_temp_dir(u'isolated_out', tmp_root))
  result = 0
  try:
    try:
      bundle = isolateserver.fetch_isolated(
          isolated_hash=isolated_hash,
          storage=storage,
          cache=cache,
          outdir=run_dir,
          require_command=True)
    except isolated_format.IsolatedError:
      on_error.report(None)
      return 1

    change_tree_read_only(run_dir, bundle.read_only)
    cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
    command = bundle.command + extra_args

    file_path.ensure_command_has_abs_path(command, cwd)
    command = process_command(command, out_dir)
    logging.info('Running %s, cwd=%s' % (command, cwd))

    # TODO(csharp): This should be specified somewhere else.
    # TODO(vadimsh): Pass it via 'env_vars' in manifest.
    # Add a rotating log file if one doesn't already exist.
    env = os.environ.copy()
    if MAIN_DIR:
      env.setdefault('RUN_TEST_CASES_LOG_FILE',
          os.path.join(MAIN_DIR, RUN_TEST_CASES_LOG))
    sys.stdout.flush()
    with tools.Profiler('RunTest'):
      try:
        with subprocess42.Popen_with_handler(command, cwd=cwd, env=env) as p:
          p.communicate()
          result = p.returncode
      except OSError:
        on_error.report('Failed to run %s; cwd=%s' % (command, cwd))
        result = 1
    logging.info(
        'Command finished with exit code %d (%s)',
        result, hex(0xffffffff & result))
  finally:
    try:
      if leak_temp_dir:
        logging.warning('Deliberately leaking %s for later examination',
                        run_dir)
      else:
        try:
          if not file_path.rmtree(run_dir):
            print >> sys.stderr, (
                'Failed to delete the temporary directory, forcibly failing\n'
                'the task because of it. No zombie process can outlive a\n'
                'successful task run and still be marked as successful.\n'
                'Fix your stuff.')
            result = result or 1
        except OSError as exc:
          logging.error('Leaking run_dir %s: %s', run_dir, exc)
          result = 1

      # HACK(vadimsh): On Windows rmtree(run_dir) call above has
      # a synchronization effect: it finishes only when all task child processes
      # terminate (since a running process locks *.exe file). Examine out_dir
      # only after that call completes (since child processes may
      # write to out_dir too and we need to wait for them to finish).

      # Upload out_dir and generate a .isolated file out of this directory.
      # It is only done if files were written in the directory.
      if os.path.isdir(out_dir) and os.listdir(out_dir):
        with tools.Profiler('ArchiveOutput'):
          try:
            results = isolateserver.archive_files_to_storage(
                storage, [out_dir], None)
          except isolateserver.Aborted:
            # This happens when a signal SIGTERM was received while uploading
            # data. There is 2 causes:
            # - The task was too slow and was about to be killed anyway due to
            #   exceeding the hard timeout.
            # - The amount of data uploaded back is very large and took too much
            #   time to archive.
            #
            # There's 3 options to handle this:
            # - Ignore the upload failure as a silent failure. This can be
            #   detected client side by the fact no result file exists.
            # - Return as if the task failed. This is not factually correct.
            # - Return an internal failure. Sadly, it's impossible at this level
            #   at the moment.
            #
            # For now, silently drop the upload.
            #
            # In any case, the process only has a very short grace period so it
            # needs to exit right away.
            sys.stderr.write('Received SIGTERM while uploading')
            results = None

        if results:
          # TODO(maruel): Implement side-channel to publish this information.
          output_data = {
            'hash': results[0][0],
            'namespace': storage.namespace,
            'storage': storage.location,
          }
          sys.stdout.flush()
          print(
              '[run_isolated_out_hack]%s[/run_isolated_out_hack]' %
              tools.format_json(output_data, dense=True))

    finally:
      try:
        if os.path.isdir(out_dir) and not file_path.rmtree(out_dir):
          logging.error('Had difficulties removing out_dir %s', out_dir)
          result = result or 1
      except OSError as exc:
        # Only report on non-Windows or on Windows when the process had
        # succeeded. Due to the way file sharing works on Windows, it's sadly
        # expected that file deletion may fail when a test failed.
        logging.error('Failed to remove out_dir %s: %s', out_dir, exc)
        if sys.platform != 'win32' or not result:
          on_error.report(None)
        result = 1

  return result
Esempio n. 35
0
def trigger_task_shards(swarming, isolate_server, namespace, isolated_hash,
                        task_name, extra_args, shards, dimensions, env,
                        deadline, verbose, profile, priority):
    """Triggers multiple subtasks of a sharded task.

  Returns:
    Dict with task details, returned to caller as part of --dump-json output.
    None in case of failure.
  """
    # Collects all files that are necessary to bootstrap a task execution
    # on the bot. Usually it includes self contained run_isolated.zip and
    # a bunch of small other scripts. All heavy files are pulled
    # by run_isolated.zip. Updated in 'setup_run_isolated'.
    bundle = zip_package.ZipPackage(ROOT_DIR)

    # Make a separate Manifest for each shard, put shard index and number of
    # shards into env and subtask name.
    manifests = []
    for index in xrange(shards):
        manifest = Manifest(isolate_server=isolate_server,
                            namespace=namespace,
                            isolated_hash=isolated_hash,
                            task_name=get_shard_task_name(
                                task_name, shards, index),
                            extra_args=extra_args,
                            dimensions=dimensions,
                            env=setup_googletest(env, shards, index),
                            deadline=deadline,
                            verbose=verbose,
                            profile=profile,
                            priority=priority)
        setup_run_isolated(manifest, bundle)
        manifests.append(manifest)

    # Upload zip bundle file to get its URL.
    try:
        bundle_url = upload_zip_bundle(isolate_server, bundle)
    except (IOError, OSError):
        on_error.report('Failed to upload the zip file for task %s' %
                        task_name)
        return None, None

    # Attach that file to all manifests.
    for manifest in manifests:
        manifest.add_bundled_file('swarm_data.zip', bundle_url)

    # Trigger all the subtasks.
    tasks = {}
    priority_warning = False
    for index, manifest in enumerate(manifests):
        task_id, priority = trigger_by_manifest(swarming, manifest)
        if not task_id:
            break
        if not priority_warning and priority != manifest.priority:
            priority_warning = True
            print >> sys.stderr, 'Priority was reset to %s' % priority
        tasks[manifest.task_name] = {
            'shard_index': index,
            'task_id': task_id,
            'view_url': '%s/user/task/%s' % (swarming, task_id),
        }

    # Some shards weren't triggered. Abort everything.
    if len(tasks) != len(manifests):
        if tasks:
            print >> sys.stderr, 'Not all shards were triggered'
            for task_dict in tasks.itervalues():
                abort_task(swarming, task_dict['task_id'])
        return None

    return tasks