コード例 #1
0
 def test_pack_request_key(self):
   # Old style keys.
   self.assertEqual(
       '10',
      task_pack.pack_request_key(
          ndb.Key('TaskRequestShard', 'f71849', 'TaskRequest', 256)))
   # New style key.
   self.assertEqual(
       '11',
      task_pack.pack_request_key(ndb.Key('TaskRequest', 0x7fffffffffffffee)))
コード例 #2
0
 def test_pack_request_key(self):
     # Old style keys.
     self.assertEqual(
         '10',
         task_pack.pack_request_key(
             ndb.Key('TaskRequestShard', 'f71849', 'TaskRequest', 256)))
     # New style key.
     self.assertEqual(
         '11',
         task_pack.pack_request_key(
             ndb.Key('TaskRequest', 0x7fffffffffffffee)))
コード例 #3
0
ファイル: task_request_test.py プロジェクト: rmistry/luci-py
 def test_make_request_parent(self):
     parent = task_request.make_request(_gen_request(), True)
     # Hack: Would need to know about TaskResultSummary.
     parent_id = task_pack.pack_request_key(parent.key) + '1'
     child = task_request.make_request(
         _gen_request(parent_task_id=parent_id), True)
     self.assertEqual(parent_id, child.parent_task_id)
コード例 #4
0
 def test_make_request_parent(self):
   parent = task_request.make_request(_gen_request_data())
   # Hack: Would need to know about TaskResultSummary.
   parent_id = task_pack.pack_request_key(parent.key) + '1'
   child = task_request.make_request(
       _gen_request_data(parent_task_id=parent_id))
   self.assertEqual(parent_id, child.parent_task_id)
コード例 #5
0
 def test_make_request_clone(self):
   # Compare with test_make_request().
   parent = task_request.make_request(_gen_request_data())
   # Hack: Would need to know about TaskResultSummary.
   parent_id = task_pack.pack_request_key(parent.key) + '1'
   data = _gen_request_data(
       properties=dict(idempotent=True), parent_task_id=parent_id)
   request = task_request.make_request_clone(task_request.make_request(data))
   # Differences from make_request() are:
   # - idempotent was reset to False.
   # - parent_task_id was reset to None.
   expected_properties = {
     'commands': [[u'command1', u'arg1']],
     'data': [
       # Items were sorted.
       [u'http://localhost/bar', u'bar.zip'],
       [u'http://localhost/foo', u'foo.zip'],
     ],
     'dimensions': {u'OS': u'Windows-3.1.1', u'hostname': u'localhost'},
     'env': {u'foo': u'bar', u'joe': u'2'},
     'execution_timeout_secs': 30,
     'extra_args': None,
     'grace_period_secs': 30,
     'idempotent': False,
     'io_timeout_secs': None,
     'isolated': None,
     'isolatedserver': None,
     'namespace': None,
   }
   # Differences from make_request() are:
   # - parent_task_id was reset to None.
   # - tag 'user:'******'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
     'name': u'Request name (Retry #1)',
     'parent_task_id': None,
     'priority': 49,
     'properties': expected_properties,
     'properties_hash': None,
     'tags': [
       u'OS:Windows-3.1.1',
       u'hostname:localhost',
       u'priority:49',
       u'tag:1',
       u'user:[email protected]',
     ],
     'user': u'*****@*****.**',
   }
   actual = request.to_dict()
   # expiration_ts - created_ts == deadline_to_run.
   created = actual.pop('created_ts')
   expiration = actual.pop('expiration_ts')
   self.assertEqual(
       int(round((expiration - created).total_seconds())),
       data['scheduling_expiration_secs'])
   self.assertEqual(expected_request, actual)
   self.assertEqual(
       data['scheduling_expiration_secs'], request.scheduling_expiration_secs)
コード例 #6
0
ファイル: task_request_test.py プロジェクト: stefb965/luci-py
 def test_init_new_request_parent(self):
   parent = _gen_request()
   # Parent entity must have a valid key id and be stored.
   parent.key = task_request.new_request_key()
   parent.put()
   # The reference is to the TaskRunResult.
   parent_id = task_pack.pack_request_key(parent.key) + '1'
   child = _gen_request(parent_task_id=parent_id)
   self.assertEqual(parent_id, child.parent_task_id)
コード例 #7
0
 def test_make_request_clone(self):
   # Compare with test_make_request().
   parent = mkreq(_gen_request())
   # Hack: Would need to know about TaskResultSummary.
   parent_id = task_pack.pack_request_key(parent.key) + '1'
   data = _gen_request(
       properties=dict(idempotent=True), parent_task_id=parent_id)
   request = task_request.make_request_clone(mkreq(data))
   # Differences from make_request() are:
   # - idempotent was reset to False.
   # - parent_task_id was reset to None.
   expected_properties = {
     'command': [u'command1', u'arg1'],
     'dimensions': {
       u'OS': u'Windows-3.1.1',
       u'hostname': u'localhost',
       u'pool': u'default',
     },
     'env': {u'foo': u'bar', u'joe': u'2'},
     'execution_timeout_secs': 30,
     'extra_args': [],
     'grace_period_secs': 30,
     'idempotent': False,
     'inputs_ref': None,
     'io_timeout_secs': None,
     'packages': [{'package_name': 'rm', 'version': PINNED_PACKAGE_VERSION}],
   }
   # Differences from make_request() are:
   # - parent_task_id was reset to None.
   # - tag 'user:'******'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
     'name': u'Request name (Retry #1)',
     'parent_task_id': None,
     'priority': 49,
     'properties': expected_properties,
     'properties_hash': None,
     'pubsub_topic': None,
     'pubsub_userdata': None,
     'tags': [
       u'OS:Windows-3.1.1',
       u'hostname:localhost',
       u'pool:default',
       u'priority:49',
       u'tag:1',
       u'user:[email protected]',
     ],
     'user': u'*****@*****.**',
   }
   actual = request.to_dict()
   # expiration_ts - created_ts == deadline_to_run.
   actual.pop('created_ts')
   actual.pop('expiration_ts')
   self.assertEqual(expected_request, actual)
   self.assertEqual(30, request.expiration_secs)
コード例 #8
0
 def test_make_request_isolated(self):
   parent = mkreq(_gen_request(properties={
     'command': [],
     'inputs_ref': {
       'isolated': '0123456789012345678901234567890123456789',
       'isolatedserver': 'http://localhost:1',
       'namespace': 'default-gzip',
     },
   }))
   # Hack: Would need to know about TaskResultSummary.
   parent_id = task_pack.pack_request_key(parent.key) + '1'
   request = mkreq(_gen_request(
       properties={'idempotent':True}, parent_task_id=parent_id))
   expected_properties = {
     'command': [u'command1', u'arg1'],
     'dimensions': {
       u'OS': u'Windows-3.1.1',
       u'hostname': u'localhost',
       u'pool': u'default',
     },
     'env': {u'foo': u'bar', u'joe': u'2'},
     'extra_args': [],
     'execution_timeout_secs': 30,
     'grace_period_secs': 30,
     'idempotent': True,
     'inputs_ref': None,
     'io_timeout_secs': None,
     'packages': [{'package_name': 'rm', 'version': PINNED_PACKAGE_VERSION}],
   }
   expected_request = {
     'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
     'name': u'Request name',
     'parent_task_id': unicode(parent_id),
     'priority': 49,
     'properties': expected_properties,
     # Intentionally hard code the hash value since it has to be deterministic.
     # Other unit tests should use the calculated value.
     'properties_hash': '83b350298f05eff6072d54d2c6f031d06cc30449',
     'pubsub_topic': None,
     'pubsub_userdata': None,
     'tags': [
       u'OS:Windows-3.1.1',
       u'hostname:localhost',
       u'pool:default',
       u'priority:49',
       u'tag:1',
       u'user:Jesus',
     ],
     'user': u'Jesus',
   }
   actual = request.to_dict()
   # expiration_ts - created_ts == scheduling_expiration_secs.
   actual.pop('created_ts')
   actual.pop('expiration_ts')
   self.assertEqual(expected_request, actual)
   self.assertEqual(30, request.expiration_secs)
コード例 #9
0
 def test_make_request_isolated(self):
   parent = task_request.make_request(
       _gen_request(
           properties={
             'commands': None,
             'data': None,
             'inputs_ref': {
               'isolated': '0123456789012345678901234567890123456789',
               'isolatedserver': 'http://localhost:1',
               'namespace': 'default-gzip',
             },
           }),
       True)
   # Hack: Would need to know about TaskResultSummary.
   parent_id = task_pack.pack_request_key(parent.key) + '1'
   request = task_request.make_request(
       _gen_request(properties={'idempotent':True}, parent_task_id=parent_id),
       True)
   expected_properties = {
     'commands': [[u'command1', u'arg1']],
     'data': [
       # Items were sorted.
       [u'http://localhost/bar', u'bar.zip'],
       [u'http://localhost/foo', u'foo.zip'],
     ],
     'dimensions': {u'OS': u'Windows-3.1.1', u'hostname': u'localhost'},
     'env': {u'foo': u'bar', u'joe': u'2'},
     'extra_args': [],
     'execution_timeout_secs': 30,
     'grace_period_secs': 30,
     'idempotent': True,
     'inputs_ref': None,
     'io_timeout_secs': None,
   }
   expected_request = {
     'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
     'name': u'Request name',
     'parent_task_id': unicode(parent_id),
     'priority': 49,
     'properties': expected_properties,
     'properties_hash': 'b45f6f868f9227c3035cd82b4a5b0360f5ce6f61',
     'tags': [
       u'OS:Windows-3.1.1',
       u'hostname:localhost',
       u'priority:49',
       u'tag:1',
       u'user:Jesus',
     ],
     'user': u'Jesus',
   }
   actual = request.to_dict()
   # expiration_ts - created_ts == scheduling_expiration_secs.
   actual.pop('created_ts')
   actual.pop('expiration_ts')
   self.assertEqual(expected_request, actual)
   self.assertEqual(30, request.expiration_secs)
コード例 #10
0
ファイル: task_request_test.py プロジェクト: rmistry/luci-py
 def test_make_request_isolated(self):
     parent = task_request.make_request(
         _gen_request(
             properties={
                 "commands": None,
                 "data": None,
                 "inputs_ref": {
                     "isolated": "0123456789012345678901234567890123456789",
                     "isolatedserver": "http://localhost:1",
                     "namespace": "default-gzip",
                 },
             }
         ),
         True,
     )
     # Hack: Would need to know about TaskResultSummary.
     parent_id = task_pack.pack_request_key(parent.key) + "1"
     request = task_request.make_request(
         _gen_request(properties={"idempotent": True}, parent_task_id=parent_id), True
     )
     expected_properties = {
         "commands": [[u"command1", u"arg1"]],
         "data": [
             # Items were sorted.
             [u"http://localhost/bar", u"bar.zip"],
             [u"http://localhost/foo", u"foo.zip"],
         ],
         "dimensions": {u"OS": u"Windows-3.1.1", u"hostname": u"localhost"},
         "env": {u"foo": u"bar", u"joe": u"2"},
         "extra_args": [],
         "execution_timeout_secs": 30,
         "grace_period_secs": 30,
         "idempotent": True,
         "inputs_ref": None,
         "io_timeout_secs": None,
     }
     expected_request = {
         "authenticated": auth_testing.DEFAULT_MOCKED_IDENTITY,
         "name": u"Request name",
         "parent_task_id": unicode(parent_id),
         "priority": 49,
         "properties": expected_properties,
         "properties_hash": "b45f6f868f9227c3035cd82b4a5b0360f5ce6f61",
         "pubsub_topic": None,
         "pubsub_userdata": None,
         "tags": [u"OS:Windows-3.1.1", u"hostname:localhost", u"priority:49", u"tag:1", u"user:Jesus"],
         "user": u"Jesus",
     }
     actual = request.to_dict()
     # expiration_ts - created_ts == scheduling_expiration_secs.
     actual.pop("created_ts")
     actual.pop("expiration_ts")
     self.assertEqual(expected_request, actual)
     self.assertEqual(30, request.expiration_secs)
コード例 #11
0
 def test_make_request(self):
   # Compare with test_make_request_clone().
   parent = task_request.make_request(_gen_request_data())
   # Hack: Would need to know about TaskResultSummary.
   parent_id = task_pack.pack_request_key(parent.key) + '1'
   data = _gen_request_data(
       properties=dict(idempotent=True), parent_task_id=parent_id)
   request = task_request.make_request(data)
   expected_properties = {
     'commands': [[u'command1', u'arg1']],
     'data': [
       # Items were sorted.
       [u'http://localhost/bar', u'bar.zip'],
       [u'http://localhost/foo', u'foo.zip'],
     ],
     'dimensions': {u'OS': u'Windows-3.1.1', u'hostname': u'localhost'},
     'env': {u'foo': u'bar', u'joe': u'2'},
     'extra_args': None,
     'execution_timeout_secs': 30,
     'grace_period_secs': 30,
     'idempotent': True,
     'io_timeout_secs': None,
     'isolated': None,
     'isolatedserver': None,
     'namespace': None,
   }
   expected_request = {
     'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
     'name': u'Request name',
     'parent_task_id': unicode(parent_id),
     'priority': 49,
     'properties': expected_properties,
     'properties_hash': 'e7276ca9999756de386d26d39ae648e641ae1eac',
     'tags': [
       u'OS:Windows-3.1.1',
       u'hostname:localhost',
       u'priority:49',
       u'tag:1',
       u'user:Jesus',
     ],
     'user': u'Jesus',
   }
   actual = request.to_dict()
   # expiration_ts - created_ts == scheduling_expiration_secs.
   created = actual.pop('created_ts')
   expiration = actual.pop('expiration_ts')
   self.assertEqual(
       int(round((expiration - created).total_seconds())),
       data['scheduling_expiration_secs'])
   self.assertEqual(expected_request, actual)
   self.assertEqual(
       data['scheduling_expiration_secs'], request.scheduling_expiration_secs)
コード例 #12
0
ファイル: task_scheduler.py プロジェクト: rmoorman/luci-py
def bot_reap_task(bot_dimensions, bot_version, deadline):
  """Reaps a TaskToRun if one is available.

  The process is to find a TaskToRun where its .queue_number is set, then
  create a TaskRunResult for it.

  Returns:
    tuple of (TaskRequest, SecretBytes, TaskRunResult) for the task that was
    reaped. The TaskToRun involved is not returned.
  """
  start = time.time()
  bot_id = bot_dimensions[u'id'][0]
  iterated = 0
  reenqueued = 0
  expired = 0
  failures = 0
  stale_index = 0
  try:
    q = task_to_run.yield_next_available_task_to_dispatch(
        bot_dimensions, deadline)
    for request, to_run in q:
      iterated += 1
      if request.expiration_ts < utils.utcnow():
        s, r = _expire_task(to_run.key, request)
        if r:
          # Expiring a TaskToRun for TaskSlice may reenqueue a new TaskToRun.
          # It'll be processed accordingly but not handled here.
          reenqueued += 1
        elif s:
          expired += 1
        else:
          stale_index += 1
        continue
      run_result, secret_bytes = _reap_task(
          bot_dimensions, bot_version, to_run.key, request)
      if not run_result:
        failures += 1
        # Sad thing is that there is not way here to know the try number.
        logging.info(
            'failed to reap: %s0',
            task_pack.pack_request_key(to_run.request_key))
        continue

      logging.info('Reaped: %s', run_result.task_id)
      return request, secret_bytes, run_result
    return None, None, None
  finally:
    logging.debug(
        'bot_reap_task(%s) in %.3fs: %d iterated, %d reenqueued, %d expired, '
        '%d stale_index, %d failured',
        bot_id, time.time()-start, iterated, reenqueued, expired, stale_index,
        failures)
コード例 #13
0
ファイル: task_request_test.py プロジェクト: rmistry/luci-py
 def test_make_request_clone(self):
     # Compare with test_make_request().
     parent = task_request.make_request(_gen_request(), True)
     # Hack: Would need to know about TaskResultSummary.
     parent_id = task_pack.pack_request_key(parent.key) + "1"
     data = _gen_request(properties=dict(idempotent=True), parent_task_id=parent_id)
     request = task_request.make_request_clone(task_request.make_request(data, True))
     # Differences from make_request() are:
     # - idempotent was reset to False.
     # - parent_task_id was reset to None.
     expected_properties = {
         "commands": [[u"command1", u"arg1"]],
         "data": [
             # Items were sorted.
             [u"http://localhost/bar", u"bar.zip"],
             [u"http://localhost/foo", u"foo.zip"],
         ],
         "dimensions": {u"OS": u"Windows-3.1.1", u"hostname": u"localhost"},
         "env": {u"foo": u"bar", u"joe": u"2"},
         "execution_timeout_secs": 30,
         "extra_args": [],
         "grace_period_secs": 30,
         "idempotent": False,
         "inputs_ref": None,
         "io_timeout_secs": None,
     }
     # Differences from make_request() are:
     # - parent_task_id was reset to None.
     # - tag 'user:' was replaced
     # - user was replaced.
     expected_request = {
         "authenticated": auth_testing.DEFAULT_MOCKED_IDENTITY,
         "name": u"Request name (Retry #1)",
         "parent_task_id": None,
         "priority": 49,
         "properties": expected_properties,
         "properties_hash": None,
         "pubsub_topic": None,
         "pubsub_userdata": None,
         "tags": [u"OS:Windows-3.1.1", u"hostname:localhost", u"priority:49", u"tag:1", u"user:[email protected]"],
         "user": u"*****@*****.**",
     }
     actual = request.to_dict()
     # expiration_ts - created_ts == deadline_to_run.
     actual.pop("created_ts")
     actual.pop("expiration_ts")
     self.assertEqual(expected_request, actual)
     self.assertEqual(30, request.expiration_secs)
コード例 #14
0
def _validate_task_async(bot_dimensions, stats, now, to_run):
    """Validates the TaskToRun and updates stats.

  Returns:
    None if the task cannot be reaped by this bot.
    TaskRequest if this is a good candidate to reap.
  """
    # TODO(maruel): Create one TaskToRun per TaskRunResult.
    packed = task_pack.pack_request_key(
        task_to_run_key_to_request_key(to_run.key)) + '0'
    stats.total += 1

    # Do this after the basic weeding out but before fetching TaskRequest.
    neg = yield _lookup_cache_is_taken_async(to_run.key)
    if neg:
        logging.debug('_validate_task_async(%s): negative cache', packed)
        stats.cache_lookup += 1
        raise ndb.Return((None, None))

    # Ok, it's now worth taking a real look at the entity.
    request = yield task_to_run_key_to_request_key(to_run.key).get_async()
    props = request.task_slice(to_run.task_slice_index).properties

    # The hash may have conflicts. Ensure the dimensions actually match by
    # verifying the TaskRequest.
    #
    # There's a probability of 2**-31 of conflicts, which is low enough for our
    # purpose.
    if not match_dimensions(props.dimensions, bot_dimensions):
        logging.debug('_validate_task_async(%s): dimensions mismatch', packed)
        stats.real_mismatch += 1
        raise ndb.Return((None, None))

    # Expire as the bot polls by returning it, and task_scheduler will handle it.
    if to_run.expiration_ts < now:
        logging.debug('_validate_task_async(%s): expired %s < %s', packed,
                      to_run.expiration_ts, now)
        stats.expired += 1
    else:
        # It's a valid task! Note that in the meantime, another bot may have reaped
        # it. This is verified one last time in task_scheduler._reap_task() by
        # calling set_lookup_cache().
        logging.info('_validate_task_async(%s): ready to reap!', packed)
    raise ndb.Return((request, to_run))
コード例 #15
0
ファイル: task_request_test.py プロジェクト: molodiuc/luci-py
 def test_init_new_request_isolated(self):
   parent = mkreq(_gen_request(properties={
     'command': [],
     'inputs_ref': {
       'isolated': '0123456789012345678901234567890123456789',
       'isolatedserver': 'http://localhost:1',
       'namespace': 'default-gzip',
     },
   }))
   # Hack: Would need to know about TaskResultSummary.
   parent_id = task_pack.pack_request_key(parent.key) + '1'
   request = mkreq(_gen_request(
       properties={'idempotent':True}, parent_task_id=parent_id))
   expected_properties = {
     'caches': [],
     'cipd_input': {
       'client_package': {
         'package_name': 'infra/tools/cipd/${platform}',
         'path': None,
         'version': 'git_revision:deadbeef',
       },
       'packages': [{
         'package_name': 'rm',
         'path': 'bin',
         'version': 'git_revision:deadbeef',
       }],
       'server': 'https://chrome-infra-packages.appspot.com'
     },
     'command': [u'command1', u'arg1'],
     'dimensions': {
       u'OS': u'Windows-3.1.1',
       u'hostname': u'localhost',
       u'pool': u'default',
     },
     'env': {u'foo': u'bar', u'joe': u'2'},
     'extra_args': [],
     'execution_timeout_secs': 30,
     'grace_period_secs': 30,
     'idempotent': True,
     'inputs_ref': {
       'isolated': None,
       'isolatedserver': 'https://isolateserver.appspot.com',
       'namespace': 'default-gzip',
     },
     'io_timeout_secs': None,
   }
   expected_request = {
     'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
     'name': u'Request name',
     'parent_task_id': unicode(parent_id),
     'priority': 49,
     'properties': expected_properties,
     # Intentionally hard code the hash value since it has to be deterministic.
     # Other unit tests should use the calculated value.
     'properties_hash': '2202337f592f7e31b407e38832c35e23f306c6c8',
     'pubsub_topic': None,
     'pubsub_userdata': None,
     'service_account': u'none',
     'tags': [
       u'OS:Windows-3.1.1',
       u'hostname:localhost',
       u'pool:default',
       u'priority:49',
       u'service_account:none',
       u'tag:1',
       u'user:Jesus',
     ],
     'user': u'Jesus',
   }
   actual = request.to_dict()
   # expiration_ts - created_ts == scheduling_expiration_secs.
   actual.pop('created_ts')
   actual.pop('expiration_ts')
   self.assertEqual(expected_request, actual)
   self.assertEqual(30, request.expiration_secs)
コード例 #16
0
 def test_make_request(self):
     # Compare with test_make_request_clone().
     parent = task_request.make_request(_gen_request_data())
     # Hack: Would need to know about TaskResultSummary.
     parent_id = task_pack.pack_request_key(parent.key) + '1'
     data = _gen_request_data(properties=dict(idempotent=True),
                              parent_task_id=parent_id)
     request = task_request.make_request(data)
     expected_properties = {
         'commands': [[u'command1', u'arg1']],
         'data': [
             # Items were sorted.
             [u'http://localhost/bar', u'bar.zip'],
             [u'http://localhost/foo', u'foo.zip'],
         ],
         'dimensions': {
             u'OS': u'Windows-3.1.1',
             u'hostname': u'localhost'
         },
         'env': {
             u'foo': u'bar',
             u'joe': u'2'
         },
         'execution_timeout_secs':
         30,
         'grace_period_secs':
         30,
         'idempotent':
         True,
         'io_timeout_secs':
         None,
     }
     expected_request = {
         'authenticated':
         auth_testing.DEFAULT_MOCKED_IDENTITY,
         'name':
         u'Request name',
         'parent_task_id':
         unicode(parent_id),
         'priority':
         49,
         'properties':
         expected_properties,
         'properties_hash':
         '6ec96bdc40fad2bdaec3cbbe43594961ad72f02e',
         'tags': [
             u'OS:Windows-3.1.1',
             u'hostname:localhost',
             u'priority:49',
             u'tag:1',
             u'user:Jesus',
         ],
         'user':
         u'Jesus',
     }
     actual = request.to_dict()
     # expiration_ts - created_ts == scheduling_expiration_secs.
     created = actual.pop('created_ts')
     expiration = actual.pop('expiration_ts')
     self.assertEqual(int(round((expiration - created).total_seconds())),
                      data['scheduling_expiration_secs'])
     self.assertEqual(expected_request, actual)
     self.assertEqual(data['scheduling_expiration_secs'],
                      request.scheduling_expiration_secs)
コード例 #17
0
ファイル: task_request_test.py プロジェクト: rmistry/luci-py
 def test_make_request_isolated(self):
     parent = task_request.make_request(
         _gen_request(
             properties={
                 'commands': None,
                 'data': None,
                 'inputs_ref': {
                     'isolated': '0123456789012345678901234567890123456789',
                     'isolatedserver': 'http://localhost:1',
                     'namespace': 'default-gzip',
                 },
             }), True)
     # Hack: Would need to know about TaskResultSummary.
     parent_id = task_pack.pack_request_key(parent.key) + '1'
     request = task_request.make_request(
         _gen_request(properties={'idempotent': True},
                      parent_task_id=parent_id), True)
     expected_properties = {
         'commands': [[u'command1', u'arg1']],
         'data': [
             # Items were sorted.
             [u'http://localhost/bar', u'bar.zip'],
             [u'http://localhost/foo', u'foo.zip'],
         ],
         'dimensions': {
             u'OS': u'Windows-3.1.1',
             u'hostname': u'localhost'
         },
         'env': {
             u'foo': u'bar',
             u'joe': u'2'
         },
         'extra_args': [],
         'execution_timeout_secs':
         30,
         'grace_period_secs':
         30,
         'idempotent':
         True,
         'inputs_ref':
         None,
         'io_timeout_secs':
         None,
     }
     expected_request = {
         'authenticated':
         auth_testing.DEFAULT_MOCKED_IDENTITY,
         'name':
         u'Request name',
         'parent_task_id':
         unicode(parent_id),
         'priority':
         49,
         'properties':
         expected_properties,
         'properties_hash':
         'b45f6f868f9227c3035cd82b4a5b0360f5ce6f61',
         'pubsub_topic':
         None,
         'pubsub_userdata':
         None,
         'tags': [
             u'OS:Windows-3.1.1',
             u'hostname:localhost',
             u'priority:49',
             u'tag:1',
             u'user:Jesus',
         ],
         'user':
         u'Jesus',
     }
     actual = request.to_dict()
     # expiration_ts - created_ts == scheduling_expiration_secs.
     actual.pop('created_ts')
     actual.pop('expiration_ts')
     self.assertEqual(expected_request, actual)
     self.assertEqual(30, request.expiration_secs)
コード例 #18
0
 def test_pack_request_key(self):
     self.assertEqual(
         '11',
         task_pack.pack_request_key(
             ndb.Key('TaskRequest', 0x7fffffffffffffee)))
コード例 #19
0
 def test_make_request_clone(self):
     # Compare with test_make_request().
     parent = task_request.make_request(_gen_request_data())
     # Hack: Would need to know about TaskResultSummary.
     parent_id = task_pack.pack_request_key(parent.key) + '1'
     data = _gen_request_data(properties=dict(idempotent=True),
                              parent_task_id=parent_id)
     request = task_request.make_request_clone(
         task_request.make_request(data))
     # Differences from make_request() are:
     # - idempotent was reset to False.
     # - parent_task_id was reset to None.
     expected_properties = {
         'commands': [[u'command1', u'arg1']],
         'data': [
             # Items were sorted.
             [u'http://localhost/bar', u'bar.zip'],
             [u'http://localhost/foo', u'foo.zip'],
         ],
         'dimensions': {
             u'OS': u'Windows-3.1.1',
             u'hostname': u'localhost'
         },
         'env': {
             u'foo': u'bar',
             u'joe': u'2'
         },
         'execution_timeout_secs':
         30,
         'grace_period_secs':
         30,
         'idempotent':
         False,
         'io_timeout_secs':
         None,
     }
     # Differences from make_request() are:
     # - parent_task_id was reset to None.
     # - tag 'user:'******'authenticated':
         auth_testing.DEFAULT_MOCKED_IDENTITY,
         'name':
         u'Request name (Retry #1)',
         'parent_task_id':
         None,
         'priority':
         49,
         'properties':
         expected_properties,
         'properties_hash':
         None,
         'tags': [
             u'OS:Windows-3.1.1',
             u'hostname:localhost',
             u'priority:49',
             u'tag:1',
             u'user:[email protected]',
         ],
         'user':
         u'*****@*****.**',
     }
     actual = request.to_dict()
     # expiration_ts - created_ts == deadline_to_run.
     created = actual.pop('created_ts')
     expiration = actual.pop('expiration_ts')
     self.assertEqual(int(round((expiration - created).total_seconds())),
                      data['scheduling_expiration_secs'])
     self.assertEqual(expected_request, actual)
     self.assertEqual(data['scheduling_expiration_secs'],
                      request.scheduling_expiration_secs)
コード例 #20
0
ファイル: task_request_test.py プロジェクト: stefb965/luci-py
 def test_init_new_request_isolated(self):
   parent = _gen_request(
       properties=_gen_properties(
           command=[],
           inputs_ref={
             'isolated': '0123456789012345678901234567890123456789',
             'isolatedserver': 'http://localhost:1',
             'namespace': 'default-gzip',
           }))
   # Parent entity must have a valid key id and be stored.
   parent.key = task_request.new_request_key()
   parent.put()
   # The reference is to the TaskRunResult.
   parent_id = task_pack.pack_request_key(parent.key) + u'1'
   req = _gen_request(
       properties=_gen_properties(idempotent=True, has_secret_bytes=True),
       parent_task_id=parent_id)
   # TaskRequest with secret must have a valid key.
   req.key = task_request.new_request_key()
   # Needed for the get() call below.
   req.put()
   sb = _gen_secret(req, 'I am not a banana')
   # Needed for properties_hash() call.
   sb.put()
   expected_properties = {
     'caches': [],
     'cipd_input': {
       'client_package': {
         'package_name': u'infra/tools/cipd/${platform}',
         'path': None,
         'version': u'git_revision:deadbeef',
       },
       'packages': [{
         'package_name': u'rm',
         'path': u'bin',
         'version': u'git_revision:deadbeef',
       }],
       'server': u'https://chrome-infra-packages.appspot.com'
     },
     'command': [u'command1', u'arg1'],
     'relative_cwd': None,
     'dimensions': {
       u'OS': [u'Windows-3.1.1'],
       u'hostname': [u'localhost'],
       u'pool': [u'default'],
     },
     'env': {u'foo': u'bar', u'joe': u'2'},
     'env_prefixes': {u'PATH': [u'local/path']},
     'extra_args': [],
     'execution_timeout_secs': 30,
     'grace_period_secs': 30,
     'idempotent': True,
     'inputs_ref': {
       'isolated': None,
       'isolatedserver': u'https://isolateserver.appspot.com',
       'namespace': u'default-gzip',
     },
     'io_timeout_secs': None,
     'outputs': [],
     'has_secret_bytes': True,
   }
   expected_request = {
     'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
     'name': u'Request name',
     'parent_task_id': unicode(parent_id),
     'priority': 50,
     'pubsub_topic': None,
     'pubsub_userdata': None,
     'service_account': u'none',
     'tags': [
       u'OS:Windows-3.1.1',
       u'hostname:localhost',
       u'pool:default',
       u'priority:50',
       u'service_account:none',
       u'tag:1',
       u'user:Jesus',
     ],
     'task_slices': [
       {
         'expiration_secs': 30,
         'properties': expected_properties,
         'wait_for_capacity': False,
       },
     ],
     'user': u'Jesus',
   }
   actual = req.to_dict()
   # expiration_ts - created_ts == scheduling_expiration_secs.
   actual.pop('created_ts')
   actual.pop('expiration_ts')
   self.assertEqual(expected_request, actual)
   self.assertEqual(30, req.expiration_secs)
   # Intentionally hard code the hash value since it has to be deterministic.
   # Other unit tests should use the calculated value.
   self.assertEqual(
       '121c6bd6216a4cc9c4302a52da6292e5a240807ef13ace6f7f36a0c83aec6f55',
       req.task_slice(0).properties_hash().encode('hex'))
コード例 #21
0
ファイル: task_request_test.py プロジェクト: molodiuc/luci-py
 def test_new_request_clone(self):
   # Compare with test_init_new_request().
   parent = mkreq(_gen_request())
   # Hack: Would need to know about TaskResultSummary.
   parent_id = task_pack.pack_request_key(parent.key) + '1'
   data = _gen_request(
       properties=dict(idempotent=True), parent_task_id=parent_id)
   request = task_request.new_request_clone(mkreq(data), True)
   request.key = task_request.new_request_key()
   request.put()
   # Differences from init_new_request() are:
   # - idempotent was reset to False.
   # - parent_task_id was reset to None.
   expected_properties = {
     'caches': [],
     'cipd_input': {
       'client_package': {
         'package_name': 'infra/tools/cipd/${platform}',
         'path': None,
         'version': 'git_revision:deadbeef',
       },
       'packages': [{
         'package_name': 'rm',
         'path': 'bin',
         'version': 'git_revision:deadbeef',
       }],
       'server': 'https://chrome-infra-packages.appspot.com'
     },
     'command': [u'command1', u'arg1'],
     'dimensions': {
       u'OS': u'Windows-3.1.1',
       u'hostname': u'localhost',
       u'pool': u'default',
     },
     'env': {u'foo': u'bar', u'joe': u'2'},
     'execution_timeout_secs': 30,
     'extra_args': [],
     'grace_period_secs': 30,
     'idempotent': False,
     'inputs_ref': {
       'isolated': None,
       'isolatedserver': 'https://isolateserver.appspot.com',
       'namespace': 'default-gzip',
     },
     'io_timeout_secs': None,
   }
   # Differences from new_request() are:
   # - parent_task_id was reset to None.
   # - tag 'user:'******'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
     'name': u'Request name (Retry #1)',
     'parent_task_id': None,
     'priority': 49,
     'properties': expected_properties,
     'properties_hash': None,
     'pubsub_topic': None,
     'pubsub_userdata': None,
     'service_account': u'none',
     'tags': [
       u'OS:Windows-3.1.1',
       u'hostname:localhost',
       u'pool:default',
       u'priority:49',
       u'service_account:none',
       u'tag:1',
       u'user:[email protected]',
     ],
     'user': u'*****@*****.**',
   }
   actual = request.to_dict()
   # expiration_ts - created_ts == deadline_to_run.
   actual.pop('created_ts')
   actual.pop('expiration_ts')
   self.assertEqual(expected_request, actual)
   self.assertEqual(30, request.expiration_secs)
コード例 #22
0
 def test_make_request_isolated(self):
     parent = mkreq(
         _gen_request(
             properties={
                 'command': [],
                 'inputs_ref': {
                     'isolated': '0123456789012345678901234567890123456789',
                     'isolatedserver': 'http://localhost:1',
                     'namespace': 'default-gzip',
                 },
             }))
     # Hack: Would need to know about TaskResultSummary.
     parent_id = task_pack.pack_request_key(parent.key) + '1'
     request = mkreq(
         _gen_request(properties={'idempotent': True},
                      parent_task_id=parent_id))
     expected_properties = {
         'command': [u'command1', u'arg1'],
         'dimensions': {
             u'OS': u'Windows-3.1.1',
             u'hostname': u'localhost',
             u'pool': u'default',
         },
         'env': {
             u'foo': u'bar',
             u'joe': u'2'
         },
         'extra_args': [],
         'execution_timeout_secs':
         30,
         'grace_period_secs':
         30,
         'idempotent':
         True,
         'inputs_ref':
         None,
         'io_timeout_secs':
         None,
         'packages': [{
             'package_name': 'rm',
             'version': PINNED_PACKAGE_VERSION
         }],
     }
     expected_request = {
         'authenticated':
         auth_testing.DEFAULT_MOCKED_IDENTITY,
         'name':
         u'Request name',
         'parent_task_id':
         unicode(parent_id),
         'priority':
         49,
         'properties':
         expected_properties,
         # Intentionally hard code the hash value since it has to be deterministic.
         # Other unit tests should use the calculated value.
         'properties_hash':
         '83b350298f05eff6072d54d2c6f031d06cc30449',
         'pubsub_topic':
         None,
         'pubsub_userdata':
         None,
         'tags': [
             u'OS:Windows-3.1.1',
             u'hostname:localhost',
             u'pool:default',
             u'priority:49',
             u'tag:1',
             u'user:Jesus',
         ],
         'user':
         u'Jesus',
     }
     actual = request.to_dict()
     # expiration_ts - created_ts == scheduling_expiration_secs.
     actual.pop('created_ts')
     actual.pop('expiration_ts')
     self.assertEqual(expected_request, actual)
     self.assertEqual(30, request.expiration_secs)
コード例 #23
0
def _validate_task_async(bot_dimensions, deadline, stats, now, to_run):
    """Validates the TaskToRun and updates stats.

  Returns:
    None if the task cannot be reaped by this bot.
    TaskRequest if this is a good candidate to reap.
  """
    # TODO(maruel): Create one TaskToRun per TaskRunResult.
    packed = task_pack.pack_request_key(
        task_to_run_key_to_request_key(to_run.key)) + '0'
    stats.total += 1

    # Do this after the basic weeding out but before fetching TaskRequest.
    neg = yield _lookup_cache_is_taken_async(to_run.key)
    if neg:
        logging.debug('_validate_task_async(%s): negative cache', packed)
        stats.cache_lookup += 1
        raise ndb.Return((None, None))

    # Ok, it's now worth taking a real look at the entity.
    request = yield task_to_run_key_to_request_key(to_run.key).get_async()
    props = request.task_slice(to_run.task_slice_index).properties

    # The hash may have conflicts. Ensure the dimensions actually match by
    # verifying the TaskRequest.
    #
    # There's a probability of 2**-31 of conflicts, which is low enough for our
    # purpose.
    if not match_dimensions(props.dimensions, bot_dimensions):
        logging.debug('_validate_task_async(%s): dimensions mismatch', packed)
        stats.real_mismatch += 1
        raise ndb.Return((None, None))

    # If the bot has a deadline, don't allow it to reap the task unless it can be
    # completed before the deadline. We have to assume the task takes the
    # theoretical maximum amount of time possible, which is governed by
    # execution_timeout_secs. An isolated task's download phase is not subject to
    # this limit, so we need to add io_timeout_secs. When a task is signalled that
    # it's about to be killed, it receives a grace period as well.
    # grace_period_secs is given by run_isolated to the task execution process, by
    # task_runner to run_isolated, and by bot_main to the task_runner. Lastly, add
    # a few seconds to account for any overhead.
    #
    # Give an exemption to the special terminate task because it doesn't actually
    # run anything.
    if deadline is not None and not props.is_terminate:
        if not props.execution_timeout_secs:
            # Task never times out, so it cannot be accepted.
            logging.debug(
                '_validate_task_async(%s): deadline %s but no execution timeout',
                packed, deadline)
            stats.too_long += 1
            raise ndb.Return((None, None))
        hard = props.execution_timeout_secs
        grace = 3 * (props.grace_period_secs or 30)
        # Allowance buffer for overheads (scheduling and isolation)
        overhead = 300
        max_schedule = now + datetime.timedelta(seconds=hard + grace +
                                                overhead)
        if deadline <= max_schedule:
            logging.debug(
                '_validate_task_async(%s): deadline and too late %s > %s (%s + %d + '
                '%d + %d)', packed, deadline, max_schedule, now, hard, grace,
                overhead)
            stats.too_long += 1
            raise ndb.Return((None, None))

    # Expire as the bot polls by returning it, and task_scheduler will handle it.
    if to_run.expiration_ts < now:
        logging.debug('_validate_task_async(%s): expired %s < %s', packed,
                      to_run.expiration_ts, now)
        stats.expired += 1
    else:
        # It's a valid task! Note that in the meantime, another bot may have reaped
        # it. This is verified one last time in task_scheduler._reap_task() by
        # calling set_lookup_cache().
        logging.info('_validate_task_async(%s): ready to reap!', packed)
    raise ndb.Return((request, to_run))
コード例 #24
0
 def test_make_request_clone(self):
     # Compare with test_make_request().
     parent = mkreq(_gen_request())
     # Hack: Would need to know about TaskResultSummary.
     parent_id = task_pack.pack_request_key(parent.key) + '1'
     data = _gen_request(properties=dict(idempotent=True),
                         parent_task_id=parent_id)
     request = task_request.make_request_clone(mkreq(data))
     # Differences from make_request() are:
     # - idempotent was reset to False.
     # - parent_task_id was reset to None.
     expected_properties = {
         'command': [u'command1', u'arg1'],
         'dimensions': {
             u'OS': u'Windows-3.1.1',
             u'hostname': u'localhost',
             u'pool': u'default',
         },
         'env': {
             u'foo': u'bar',
             u'joe': u'2'
         },
         'execution_timeout_secs':
         30,
         'extra_args': [],
         'grace_period_secs':
         30,
         'idempotent':
         False,
         'inputs_ref':
         None,
         'io_timeout_secs':
         None,
         'packages': [{
             'package_name': 'rm',
             'version': PINNED_PACKAGE_VERSION
         }],
     }
     # Differences from make_request() are:
     # - parent_task_id was reset to None.
     # - tag 'user:'******'authenticated':
         auth_testing.DEFAULT_MOCKED_IDENTITY,
         'name':
         u'Request name (Retry #1)',
         'parent_task_id':
         None,
         'priority':
         49,
         'properties':
         expected_properties,
         'properties_hash':
         None,
         'pubsub_topic':
         None,
         'pubsub_userdata':
         None,
         'tags': [
             u'OS:Windows-3.1.1',
             u'hostname:localhost',
             u'pool:default',
             u'priority:49',
             u'tag:1',
             u'user:[email protected]',
         ],
         'user':
         u'*****@*****.**',
     }
     actual = request.to_dict()
     # expiration_ts - created_ts == deadline_to_run.
     actual.pop('created_ts')
     actual.pop('expiration_ts')
     self.assertEqual(expected_request, actual)
     self.assertEqual(30, request.expiration_secs)
コード例 #25
0
ファイル: task_request_test.py プロジェクト: molodiuc/luci-py
 def test_init_new_request_parent(self):
   parent = mkreq(_gen_request())
   # Hack: Would need to know about TaskResultSummary.
   parent_id = task_pack.pack_request_key(parent.key) + '1'
   child = mkreq(_gen_request(parent_task_id=parent_id))
   self.assertEqual(parent_id, child.parent_task_id)