def apply_server_property_defaults(properties):
"""Fills ndb task properties with default values read from server settings."""
cfg = config.settings()
if not cfg:
return
cfg = config.settings()
if cfg.isolate.default_server and cfg.isolate.default_namespace:
properties.inputs_ref = properties.inputs_ref or task_request.FilesRef(
)
properties.inputs_ref.isolatedserver = (
properties.inputs_ref.isolatedserver or cfg.isolate.default_server)
properties.inputs_ref.namespace = (properties.inputs_ref.namespace
or cfg.isolate.default_namespace)
if cfg.HasField('cipd') and properties.cipd_input:
properties.cipd_input.server = (properties.cipd_input.server
or cfg.cipd.default_server)
properties.cipd_input.client_package = (
properties.cipd_input.client_package or task_request.CipdPackage())
properties.cipd_input.client_package.package_name = (
properties.cipd_input.client_package.package_name
or cfg.cipd.default_client_package.package_name)
properties.cipd_input.client_package.version = (
properties.cipd_input.client_package.version
or cfg.cipd.default_client_package.version)
def _gen_cipd_input(**kwargs):
"""Creates a CipdInput."""
args = {
u'client_package': task_request.CipdPackage(
package_name=u'infra/tools/cipd/${platform}',
version=u'git_revision:deadbeef'),
u'packages': [
task_request.CipdPackage(
package_name=u'rm',
path=u'bin',
version=u'git_revision:deadbeef'),
],
u'server': u'https://chrome-infra-packages.appspot.com'
}
args.update(kwargs)
return task_request.CipdInput(**args)
def test_request_bad_named_cache_and_cipd_input(self):
# A CIPD package and named caches cannot be mapped to the same path.
req = _gen_request(properties=_gen_properties(
caches=[
task_request.CacheEntry(name='git_chromium', path='git_cache'),
],
cipd_input=_gen_cipd_input(packages=[
task_request.CipdPackage(
package_name='foo', path='git_cache', version='latest'),
])))
with self.assertRaises(datastore_errors.BadValueError):
req.put()
req = _gen_request(properties=_gen_properties(
caches=[
task_request.CacheEntry(name='git_chromium',
path='git_cache1'),
],
cipd_input=_gen_cipd_input(packages=[
task_request.CipdPackage(
package_name='foo', path='git_cache2', version='latest'),
]))).put()
def apply_server_property_defaults(properties):
"""Fills ndb task properties with default values read from server settings."""
settings = config.settings()
# TODO(iannucci): This was an artifact of the existing test harnesses;
# get_pool_config raises on None, but the way it's mocked in
# ./test_env_handlers.py allows `get_pool_config` to return None in this case.
# This try/except will be cleaned up in a subsequent CL, once I remove these
# default services from `config`.
try:
pool_cfg = pools_config.get_pool_config(properties.pool)
except ValueError:
pool_cfg = None
if not settings and not pool_cfg:
return
iso_server = settings.isolate.default_server
iso_ns = settings.isolate.default_namespace
if pool_cfg and pool_cfg.default_isolate:
iso_server = pool_cfg.default_isolate.server
iso_ns = pool_cfg.default_isolate.namespace
if iso_server and iso_ns:
properties.inputs_ref = properties.inputs_ref or task_request.FilesRef(
)
properties.inputs_ref.isolatedserver = (
properties.inputs_ref.isolatedserver or iso_server)
properties.inputs_ref.namespace = (properties.inputs_ref.namespace
or iso_ns)
cipd_server = settings.cipd.default_server
cipd_vers = settings.cipd.default_client_package.version
if pool_cfg and pool_cfg.default_cipd:
cipd_server = pool_cfg.default_cipd.server
cipd_vers = pool_cfg.default_cipd.client_version
if cipd_server and properties.cipd_input:
properties.cipd_input.server = (properties.cipd_input.server
or cipd_server)
properties.cipd_input.client_package = (
properties.cipd_input.client_package or task_request.CipdPackage())
# TODO(iannucci) - finish removing 'client_package' as a task-configurable
# setting.
properties.cipd_input.client_package.package_name = (
'infra/tools/cipd/${platform}')
properties.cipd_input.client_package.version = (
properties.cipd_input.client_package.version or cipd_vers)
def _gen_request(properties=None, **kwargs):
"""Creates a TaskRequest."""
properties = properties or {}
packages = properties.pop('packages', [{
'package_name': 'rm',
'version': PINNED_PACKAGE_VERSION,
}])
props = {
'command': [u'command1', u'arg1'],
'packages': [task_request.CipdPackage(**p) for p in packages],
'dimensions': {
u'OS': u'Windows-3.1.1',
u'hostname': u'localhost',
u'pool': u'default',
},
'env': {
u'foo': u'bar',
u'joe': u'2'
},
'execution_timeout_secs': 30,
'grace_period_secs': 30,
'idempotent': False,
'io_timeout_secs': None,
}
props.update(properties)
now = utils.utcnow()
args = {
'created_ts': now,
'name': 'Request name',
'priority': 50,
'properties': task_request.TaskProperties(**props),
'expiration_ts': now + datetime.timedelta(seconds=30),
'tags': [u'tag:1'],
'user': '******',
}
args.update(kwargs)
return task_request.TaskRequest(**args)
def post(self, task_id=None):
# Unlike handshake and poll, we do not accept invalid keys here. This code
# path is much more strict.
request = self.parse_body()
msg = log_unexpected_subset_keys(self.ACCEPTED_KEYS,
self.REQUIRED_KEYS, request,
self.request, 'bot', 'keys')
if msg:
self.abort_with_error(400, error=msg)
bot_id = request['id']
task_id = request['task_id']
machine_type = None
bot_info = bot_management.get_info_key(bot_id).get()
if bot_info:
machine_type = bot_info.machine_type
# Make sure bot self-reported ID matches the authentication token. Raises
# auth.AuthorizationError if not.
bot_auth.validate_bot_id_and_fetch_config(bot_id, machine_type)
bot_overhead = request.get('bot_overhead')
cipd_pins = request.get('cipd_pins')
cipd_stats = request.get('cipd_stats')
cost_usd = request.get('cost_usd', 0)
duration = request.get('duration')
exit_code = request.get('exit_code')
hard_timeout = request.get('hard_timeout')
io_timeout = request.get('io_timeout')
isolated_stats = request.get('isolated_stats')
output = request.get('output')
output_chunk_start = request.get('output_chunk_start')
outputs_ref = request.get('outputs_ref')
if (isolated_stats or cipd_stats) and bot_overhead is None:
ereporter2.log_request(request=self.request,
source='server',
category='task_failure',
message='Failed to update task: %s' %
task_id)
self.abort_with_error(
400,
error=
'isolated_stats and cipd_stats require bot_overhead to be set'
'\nbot_overhead: %s\nisolate_stats: %s' %
(bot_overhead, isolated_stats))
run_result_key = task_pack.unpack_run_result_key(task_id)
performance_stats = None
if bot_overhead is not None:
performance_stats = task_result.PerformanceStats(
bot_overhead=bot_overhead)
if isolated_stats:
download = isolated_stats.get('download') or {}
upload = isolated_stats.get('upload') or {}
def unpack_base64(d, k):
x = d.get(k)
if x:
return base64.b64decode(x)
performance_stats.isolated_download = task_result.OperationStats(
duration=download.get('duration'),
initial_number_items=download.get('initial_number_items'),
initial_size=download.get('initial_size'),
items_cold=unpack_base64(download, 'items_cold'),
items_hot=unpack_base64(download, 'items_hot'))
performance_stats.isolated_upload = task_result.OperationStats(
duration=upload.get('duration'),
items_cold=unpack_base64(upload, 'items_cold'),
items_hot=unpack_base64(upload, 'items_hot'))
if cipd_stats:
performance_stats.package_installation = task_result.OperationStats(
duration=cipd_stats.get('duration'))
if output is not None:
try:
output = base64.b64decode(output)
except UnicodeEncodeError as e:
logging.error('Failed to decode output\n%s\n%r', e, output)
output = output.encode('ascii', 'replace')
except TypeError as e:
# Save the output as-is instead. The error will be logged in ereporter2
# and returning a HTTP 500 would only force the bot to stay in a retry
# loop.
logging.error('Failed to decode output\n%s\n%r', e, output)
if outputs_ref:
outputs_ref = task_request.FilesRef(**outputs_ref)
if cipd_pins:
cipd_pins = task_result.CipdPins(
client_package=task_request.CipdPackage(
**cipd_pins['client_package']),
packages=[
task_request.CipdPackage(**args)
for args in cipd_pins['packages']
])
try:
state = task_scheduler.bot_update_task(
run_result_key=run_result_key,
bot_id=bot_id,
output=output,
output_chunk_start=output_chunk_start,
exit_code=exit_code,
duration=duration,
hard_timeout=hard_timeout,
io_timeout=io_timeout,
cost_usd=cost_usd,
outputs_ref=outputs_ref,
cipd_pins=cipd_pins,
performance_stats=performance_stats)
if not state:
logging.info('Failed to update, please retry')
self.abort_with_error(500,
error='Failed to update, please retry')
if state in (task_result.State.COMPLETED,
task_result.State.TIMED_OUT):
action = 'task_completed'
elif state == task_result.State.KILLED:
action = 'task_killed'
else:
assert state in (task_result.State.BOT_DIED,
task_result.State.RUNNING), state
action = 'task_update'
bot_management.bot_event(
event_type=action,
bot_id=bot_id,
external_ip=self.request.remote_addr,
authenticated_as=auth.get_peer_identity().to_bytes(),
dimensions=None,
state=None,
version=None,
quarantined=None,
maintenance_msg=None,
task_id=task_id,
task_name=None)
except ValueError as e:
ereporter2.log_request(request=self.request,
source='server',
category='task_failure',
message='Failed to update task: %s' % e)
self.abort_with_error(400, error=str(e))
except webob.exc.HTTPException:
raise
except Exception as e:
logging.exception('Internal error: %s', e)
self.abort_with_error(500, error=str(e))
self.send_response({
'must_stop': state == task_result.State.KILLED,
'ok': True
})
def test_request_bad_cipd_input(self):
def mkcipdreq(idempotent=False, **cipd_input):
return _gen_request(
properties=_gen_properties(
idempotent=idempotent,
cipd_input=_gen_cipd_input(**cipd_input)))
req = mkcipdreq(packages=[{}])
with self.assertRaises(datastore_errors.BadValueError):
req.put()
with self.assertRaises(datastore_errors.BadValueError):
mkcipdreq(
packages=[
task_request.CipdPackage(
package_name='infra|rm', path='.', version='latest'),
])
req = mkcipdreq(
packages=[task_request.CipdPackage(package_name='rm', path='.')])
with self.assertRaises(datastore_errors.BadValueError):
req.put()
req = mkcipdreq(
packages=[
task_request.CipdPackage(package_name='rm', version='latest'),
])
with self.assertRaises(datastore_errors.BadValueError):
req.put()
with self.assertRaises(datastore_errors.BadValueError):
mkcipdreq(
packages=[
task_request.CipdPackage(
package_name='rm', path='/', version='latest'),
])
with self.assertRaises(datastore_errors.BadValueError):
mkcipdreq(
packages=[
task_request.CipdPackage(
package_name='rm', path='/a', version='latest'),
])
with self.assertRaises(datastore_errors.BadValueError):
mkcipdreq(
packages=[
task_request.CipdPackage(
package_name='rm', path='a/..', version='latest'),
])
with self.assertRaises(datastore_errors.BadValueError):
mkcipdreq(
packages=[
task_request.CipdPackage(
package_name='rm', path='a/./b', version='latest'),
])
req = mkcipdreq(
packages=[
task_request.CipdPackage(
package_name='rm', path='.', version='latest'),
task_request.CipdPackage(
package_name='rm', path='.', version='canary'),
])
with self.assertRaises(datastore_errors.BadValueError):
req.put()
req = mkcipdreq(
idempotent=True,
packages=[
task_request.CipdPackage(
package_name='rm', path='.', version='latest'),
])
with self.assertRaises(datastore_errors.BadValueError):
req.put()
with self.assertRaises(datastore_errors.BadValueError):
mkcipdreq(server='abc')
with self.assertRaises(datastore_errors.BadValueError):
mkcipdreq(
client_package=task_request.CipdPackage(
package_name='--bad package--'))
mkcipdreq().put()
mkcipdreq(
packages=[
task_request.CipdPackage(
package_name='rm', path='.', version='latest'),
]).put()
mkcipdreq(
client_package=task_request.CipdPackage(
package_name='infra/tools/cipd/${platform}',
version='git_revision:daedbeef'),
packages=[
task_request.CipdPackage(
package_name='rm', path='.', version='latest'),
],
server='https://chrome-infra-packages.appspot.com').put()