Пример #1
0
 def test_unknown_bot_id(self):
     # Caller supplies bot_id not in the config.
     self.mock_config(TEST_CONFIG)
     self.mock_caller('anonymous:anonymous', '1.2.3.4')
     with self.assertRaises(auth.AuthorizationError):
         bot_auth.validate_bot_id_and_fetch_config('unknown_bot_id', None)
     self.assert_error_log('unknown bot_id, not in the config')
Пример #2
0
 def test_service_account_not_ok(self):
     # Caller is using wrong service account.
     self.mock_caller('user:[email protected]', '1.2.3.5')
     with self.assertRaises(auth.AuthorizationError):
         bot_auth.validate_bot_id_and_fetch_config(
             'bot_with_service_account')
     self.assert_error_log('bot is not using expected service account')
Пример #3
0
 def test_composite_machine_token_bad_id(self):
     # Caller is using machine token that doesn't match bot_id.
     self.mock_caller('bot:some-other-bot.domain', '1.2.3.5')
     with self.assertRaises(auth.AuthorizationError):
         bot_auth.validate_bot_id_and_fetch_config('bot_with_token--vm123')
     self.assert_error_log('bot ID doesn\'t match the machine token used')
     self.assert_error_log('bot_id: "bot_with_token"')
Пример #4
0
 def test_gce_token_not_present(self):
     self.mock_caller('bot:irrelevant',
                      '1.2.3.4',
                      gce_instance=None,
                      gce_project=None)
     with self.assertRaises(auth.AuthorizationError):
         bot_auth.validate_bot_id_and_fetch_config('bot_with_gce_token')
     self.assert_error_log('bot is not using X-Luci-Gce-Vm-Token')
Пример #5
0
 def test_service_account_ip_whitelist_not_ok(self):
     # Caller is using valid service account and doesn't belong to the IP
     # whitelist.
     self.mock_caller('user:[email protected]', '1.2.3.5')
     with self.assertRaises(auth.AuthorizationError):
         bot_auth.validate_bot_id_and_fetch_config(
             'bot_with_service_account_and_ip_whitelist')
     self.assert_error_log('bot IP is not whitelisted')
Пример #6
0
 def test_gce_token_wrong_instance(self):
     self.mock_caller('bot:irrelevant',
                      '1.2.3.4',
                      gce_instance='wrong_instance',
                      gce_project='expected_proj')
     with self.assertRaises(auth.AuthorizationError):
         bot_auth.validate_bot_id_and_fetch_config('bot_with_gce_token')
     self.assert_error_log('bot ID and GCE instance name do not match')
Пример #7
0
 def test_multiple_methods_fail(self):
     self.mock_caller('anonymous:anonymous', '2.2.2.2')
     with self.assertRaises(auth.AuthorizationError) as err:
         bot_auth.validate_bot_id_and_fetch_config(
             'bot_with_fallback_to_ip_wl')
     self.assertEqual(
         "All auth methods failed: Bot ID doesn't match the token used; "
         "Not IP whitelisted", err.exception.message)
Пример #8
0
 def test_gce_token_wrong_project(self):
     self.mock_caller('bot:irrelevant',
                      '1.2.3.4',
                      gce_instance='bot_with_gce_token',
                      gce_project='not_expected')
     with self.assertRaises(auth.AuthorizationError):
         bot_auth.validate_bot_id_and_fetch_config('bot_with_gce_token')
     self.assert_error_log('got GCE VM token from unexpected project')
Пример #9
0
 def test_broken_config_section(self):
     # This should not happen in practice, but test in case it somewhat happens.
     self.mock_config(TEST_CONFIG)
     self.mock_caller('anonymous:anonymous', '1.2.3.4')
     with self.assertRaises(auth.AuthorizationError):
         bot_auth.validate_bot_id_and_fetch_config('broken_config')
     self.assert_error_log(
         'invalid bot group config, no auth method defined')
Пример #10
0
 def test_machine_token_ip_whitelist_not_ok(self):
     # Caller is using valid machine token but doesn't belongs to the IP
     # whitelist.
     self.mock_caller('bot:bot_with_token_and_ip_whitelist.domain',
                      '1.2.3.5')
     with self.assertRaises(auth.AuthorizationError):
         bot_auth.validate_bot_id_and_fetch_config(
             'bot_with_token_and_ip_whitelist')
     self.assert_error_log('bot IP is not whitelisted')
Пример #11
0
    def test_unknown_bot_id(self):
        # Prepare bots.cfg with no default group
        cfg_without_default = copy.copy(TEST_CONFIG)
        cfg_without_default.bot_group.pop()  # last one is default

        # Caller supplies bot_id not in the config.
        self.mock_config(cfg_without_default)
        self.mock_caller('anonymous:anonymous', '1.2.3.4')
        with self.assertRaises(auth.AuthorizationError):
            bot_auth.validate_bot_id_and_fetch_config('unknown_bot_id')
        self.assert_error_log('unknown bot_id, not in the config')
Пример #12
0
 def test_containerized_bot_id(self):
     # Caller is using machine token that matches hostname
     self.mock_caller('bot:bot_host.domain', '1.2.3.5')
     cfg = bot_auth.validate_bot_id_and_fetch_config('bot_host--container1')
     self.assertEquals({u'pool': [u'container1']}, cfg.dimensions)
     cfg = bot_auth.validate_bot_id_and_fetch_config('bot_host--container2')
     self.assertEquals({u'pool': [u'container2']}, cfg.dimensions)
     cfg = bot_auth.validate_bot_id_and_fetch_config('bot_host--container3')
     self.assertEquals({u'pool': [u'container_range']}, cfg.dimensions)
     cfg = bot_auth.validate_bot_id_and_fetch_config(
         'bot_host--container99')
     self.assertEquals({u'pool': [u'bot_host']}, cfg.dimensions)
Пример #13
0
 def test_ip_whitelist_based_auth_ok(self):
     # Caller passes 'bots' IP whitelist and belongs to 'ip_whitelist'.
     self.mock_config(TEST_CONFIG)
     self.mock_caller('anonymous:anonymous', '1.2.3.4')
     cfg = bot_auth.validate_bot_id_and_fetch_config(
         'bot_with_ip_whitelist', None)
     self.assertEquals({u'pool': [u'with_ip_whitelist']}, cfg.dimensions)
Пример #14
0
 def test_composite_machine_token_ok(self):
     # Caller is using valid machine token.
     self.mock_config(TEST_CONFIG)
     self.mock_caller('bot:bot_with_token.domain', '1.2.3.5')
     cfg = bot_auth.validate_bot_id_and_fetch_config(
         'bot_with_token--vm123', None)
     self.assertEquals({u'pool': [u'with_token']}, cfg.dimensions)
Пример #15
0
 def test_alternative_service_account_ok(self):
     # Caller is using the second service account.
     self.mock_config(TEST_CONFIG)
     self.mock_caller('user:[email protected]', '1.2.3.5')
     cfg = bot_auth.validate_bot_id_and_fetch_config(
         'bot_with_service_account', None)
     self.assertEquals({u'pool': [u'with_service_account']}, cfg.dimensions)
Пример #16
0
 def test_gce_token_ok(self):
     self.mock_caller('bot:irrelevant',
                      '1.2.3.4',
                      gce_instance='bot_with_gce_token',
                      gce_project='expected_proj')
     cfg = bot_auth.validate_bot_id_and_fetch_config('bot_with_gce_token')
     self.assertEquals({u'pool': [u'bot_with_gce_token']}, cfg.dimensions)
Пример #17
0
    def post(self, task_id=None):
        request = self.parse_body()
        bot_id = request.get('id')
        task_id = request.get('task_id', '')
        message = request.get('message', 'unknown')

        machine_type = None
        bot_info = bot_management.get_info_key(bot_id).get()
        if bot_info:
            machine_type = bot_info.machine_type

        # Make sure bot self-reported ID matches the authentication token. Raises
        # auth.AuthorizationError if not.
        bot_auth.validate_bot_id_and_fetch_config(bot_id, machine_type)

        bot_management.bot_event(
            event_type='task_error',
            bot_id=bot_id,
            external_ip=self.request.remote_addr,
            authenticated_as=auth.get_peer_identity().to_bytes(),
            dimensions=None,
            state=None,
            version=None,
            quarantined=None,
            maintenance_msg=None,
            task_id=task_id,
            task_name=None,
            message=message)
        line = ('Bot: https://%s/restricted/bot/%s\n'
                'Task failed: https://%s/user/task/%s\n'
                '%s') % (app_identity.get_default_version_hostname(), bot_id,
                         app_identity.get_default_version_hostname(), task_id,
                         message)
        ereporter2.log_request(self.request, source='bot', message=line)

        msg = log_unexpected_keys(self.EXPECTED_KEYS, request, self.request,
                                  'bot', 'keys')
        if msg:
            self.abort_with_error(400, error=msg)

        msg = task_scheduler.bot_kill_task(
            task_pack.unpack_run_result_key(task_id), bot_id)
        if msg:
            logging.error(msg)
            self.abort_with_error(400, error=msg)
        self.send_response({})
Пример #18
0
 def test_machine_token_ip_whitelist_ok(self):
     # Caller is using valid machine token and belongs to the IP whitelist.
     self.mock_caller('bot:bot_with_token_and_ip_whitelist.domain',
                      '1.2.3.4')
     cfg = bot_auth.validate_bot_id_and_fetch_config(
         'bot_with_token_and_ip_whitelist')
     self.assertEquals({u'pool': [u'with_token_and_ip_whitelist']},
                       cfg.dimensions)
Пример #19
0
 def test_service_account_ip_whitelist_ok(self):
     # Caller is using valid service account and belongs to the IP whitelist.
     self.mock_caller('user:[email protected]', '1.2.3.4')
     cfg = bot_auth.validate_bot_id_and_fetch_config(
         'bot_with_service_account_and_ip_whitelist')
     self.assertEquals(
         {u'pool': [u'with_service_account_and_ip_whitelist']},
         cfg.dimensions)
Пример #20
0
    def post(self, task_id=None):
        # Unlike handshake and poll, we do not accept invalid keys here. This code
        # path is much more strict.
        request = self.parse_body()
        msg = log_unexpected_subset_keys(self.ACCEPTED_KEYS,
                                         self.REQUIRED_KEYS, request,
                                         self.request, 'bot', 'keys')
        if msg:
            self.abort_with_error(400, error=msg)

        bot_id = request['id']
        task_id = request['task_id']

        machine_type = None
        bot_info = bot_management.get_info_key(bot_id).get()
        if bot_info:
            machine_type = bot_info.machine_type

        # Make sure bot self-reported ID matches the authentication token. Raises
        # auth.AuthorizationError if not.
        bot_auth.validate_bot_id_and_fetch_config(bot_id, machine_type)

        bot_overhead = request.get('bot_overhead')
        cipd_pins = request.get('cipd_pins')
        cipd_stats = request.get('cipd_stats')
        cost_usd = request.get('cost_usd', 0)
        duration = request.get('duration')
        exit_code = request.get('exit_code')
        hard_timeout = request.get('hard_timeout')
        io_timeout = request.get('io_timeout')
        isolated_stats = request.get('isolated_stats')
        output = request.get('output')
        output_chunk_start = request.get('output_chunk_start')
        outputs_ref = request.get('outputs_ref')

        if (isolated_stats or cipd_stats) and bot_overhead is None:
            ereporter2.log_request(request=self.request,
                                   source='server',
                                   category='task_failure',
                                   message='Failed to update task: %s' %
                                   task_id)
            self.abort_with_error(
                400,
                error=
                'isolated_stats and cipd_stats require bot_overhead to be set'
                '\nbot_overhead: %s\nisolate_stats: %s' %
                (bot_overhead, isolated_stats))

        run_result_key = task_pack.unpack_run_result_key(task_id)
        performance_stats = None
        if bot_overhead is not None:
            performance_stats = task_result.PerformanceStats(
                bot_overhead=bot_overhead)
            if isolated_stats:
                download = isolated_stats.get('download') or {}
                upload = isolated_stats.get('upload') or {}

                def unpack_base64(d, k):
                    x = d.get(k)
                    if x:
                        return base64.b64decode(x)

                performance_stats.isolated_download = task_result.OperationStats(
                    duration=download.get('duration'),
                    initial_number_items=download.get('initial_number_items'),
                    initial_size=download.get('initial_size'),
                    items_cold=unpack_base64(download, 'items_cold'),
                    items_hot=unpack_base64(download, 'items_hot'))
                performance_stats.isolated_upload = task_result.OperationStats(
                    duration=upload.get('duration'),
                    items_cold=unpack_base64(upload, 'items_cold'),
                    items_hot=unpack_base64(upload, 'items_hot'))
            if cipd_stats:
                performance_stats.package_installation = task_result.OperationStats(
                    duration=cipd_stats.get('duration'))

        if output is not None:
            try:
                output = base64.b64decode(output)
            except UnicodeEncodeError as e:
                logging.error('Failed to decode output\n%s\n%r', e, output)
                output = output.encode('ascii', 'replace')
            except TypeError as e:
                # Save the output as-is instead. The error will be logged in ereporter2
                # and returning a HTTP 500 would only force the bot to stay in a retry
                # loop.
                logging.error('Failed to decode output\n%s\n%r', e, output)
        if outputs_ref:
            outputs_ref = task_request.FilesRef(**outputs_ref)

        if cipd_pins:
            cipd_pins = task_result.CipdPins(
                client_package=task_request.CipdPackage(
                    **cipd_pins['client_package']),
                packages=[
                    task_request.CipdPackage(**args)
                    for args in cipd_pins['packages']
                ])

        try:
            state = task_scheduler.bot_update_task(
                run_result_key=run_result_key,
                bot_id=bot_id,
                output=output,
                output_chunk_start=output_chunk_start,
                exit_code=exit_code,
                duration=duration,
                hard_timeout=hard_timeout,
                io_timeout=io_timeout,
                cost_usd=cost_usd,
                outputs_ref=outputs_ref,
                cipd_pins=cipd_pins,
                performance_stats=performance_stats)
            if not state:
                logging.info('Failed to update, please retry')
                self.abort_with_error(500,
                                      error='Failed to update, please retry')

            if state in (task_result.State.COMPLETED,
                         task_result.State.TIMED_OUT):
                action = 'task_completed'
            elif state == task_result.State.KILLED:
                action = 'task_killed'
            else:
                assert state in (task_result.State.BOT_DIED,
                                 task_result.State.RUNNING), state
                action = 'task_update'
            bot_management.bot_event(
                event_type=action,
                bot_id=bot_id,
                external_ip=self.request.remote_addr,
                authenticated_as=auth.get_peer_identity().to_bytes(),
                dimensions=None,
                state=None,
                version=None,
                quarantined=None,
                maintenance_msg=None,
                task_id=task_id,
                task_name=None)
        except ValueError as e:
            ereporter2.log_request(request=self.request,
                                   source='server',
                                   category='task_failure',
                                   message='Failed to update task: %s' % e)
            self.abort_with_error(400, error=str(e))
        except webob.exc.HTTPException:
            raise
        except Exception as e:
            logging.exception('Internal error: %s', e)
            self.abort_with_error(500, error=str(e))
        self.send_response({
            'must_stop': state == task_result.State.KILLED,
            'ok': True
        })
Пример #21
0
    def post(self):
        request = self.parse_body()
        logging.debug('Request body: %s', request)
        msg = log_unexpected_subset_keys(self.ACCEPTED_KEYS,
                                         self.REQUIRED_KEYS, request,
                                         self.request, 'bot', 'keys')
        if msg:
            self.abort_with_error(400, error=msg)

        account_id = request['account_id']
        bot_id = request['id']
        scopes = request['scopes']
        task_id = request.get('task_id')

        # Scopes should be a list of strings, always.
        if (not scopes or not isinstance(scopes, list)
                or not all(isinstance(s, basestring) for s in scopes)):
            self.abort_with_error(400,
                                  error='"scopes" must be a list of strings')

        # Only two flavors of accounts are supported.
        if account_id not in ('system', 'task'):
            self.abort_with_error(
                400,
                error='Unknown "account_id", expecting "task" or "system"')

        # If using 'task' account, task_id is required. We'll double check the bot
        # still executes this task (based on data in datastore), and if so, will
        # use a service account associated with this particular task.
        if account_id == 'task' and not task_id:
            self.abort_with_error(
                400,
                error='"task_id" is required when using "account_id" == "task"'
            )

        # Need machine type associated with the bot for bots.cfg query below.
        # BotInfo also contains ID of a task the bot currently executes (to compare
        # with 'task_id' request parameter).
        machine_type = None
        current_task_id = None
        bot_info = bot_management.get_info_key(bot_id).get()
        if bot_info:
            machine_type = bot_info.machine_type
            current_task_id = bot_info.task_id

        # Make sure bot self-reported ID matches the authentication token. Raises
        # auth.AuthorizationError if not. Also fetches corresponding BotGroupConfig
        # that contains system service account email for this bot.
        bot_group_cfg = bot_auth.validate_bot_id_and_fetch_config(
            bot_id, machine_type)

        # At this point, the request is valid structurally, and the bot used proper
        # authentication when making it.
        logging.info('Requesting a "%s" token with scopes %s', account_id,
                     scopes)

        # This is mostly a precaution against confused bot processes. We can always
        # just use 'current_task_id' to look up per-task service account. Datastore
        # is the source of truth here, not whatever bot reports.
        if account_id == 'task' and task_id != current_task_id:
            logging.error(
                'Bot %s requested "task" access token for task %s, but runs %s',
                bot_id, task_id, current_task_id)
            self.abort_with_error(
                400, error='Wrong task_id: the bot is not executing this task')

        account = None  # an email or 'bot' or 'none'
        token = None  # service_accounts.AccessToken
        try:
            if account_id == 'task':
                account, token = service_accounts.get_task_account_token(
                    task_id, bot_id, scopes)
            elif account_id == 'system':
                account, token = service_accounts.get_system_account_token(
                    bot_group_cfg.system_service_account, scopes)
            else:
                raise AssertionError('Impossible, there is a check above')
        except auth.AccessTokenError as exc:
            # Note: no need to log this, it is already logged at the source. Also
            # we cautiously do not return any error details to the bot, just in case
            # they may contain something we don't want to disclose.
            if exc.transient:
                self.abort_with_error(
                    500, error='Transient error when generating the token')
            self.abort_with_error(
                403,
                error='Fatal error when generating the token, see server logs')

        # Note: the token info is already logged by service_accounts.get_*_token.
        if token:
            self.send_response({
                'service_account': account,
                'access_token': token.access_token,
                'expiry': token.expiry,
            })
        else:
            assert account in ('bot', 'none'), account
            self.send_response({'service_account': account})
Пример #22
0
    def _process(self):
        """Fetches bot info and settings, does authorization and quarantine checks.

    Returns:
      _ProcessResult instance, see its fields for more info.

    Raises:
      auth.AuthorizationError if bot's credentials are invalid.
    """
        request = self.parse_body()
        version = request.get('version', None)

        dimensions = request.get('dimensions') or {}
        state = request.get('state') or {}
        bot_id = None
        if dimensions.get('id'):
            dimension_id = dimensions['id']
            if (isinstance(dimension_id, list) and len(dimension_id) == 1
                    and isinstance(dimension_id[0], unicode)):
                bot_id = dimensions['id'][0]

        lease_expiration_ts = None
        machine_type = None
        if bot_id:
            logging.debug('Fetching bot info and settings')
            bot_info, bot_settings = ndb.get_multi([
                bot_management.get_info_key(bot_id),
                bot_management.get_settings_key(bot_id)
            ])
            if bot_info:
                lease_expiration_ts = bot_info.lease_expiration_ts
                machine_type = bot_info.machine_type

        # Make sure bot self-reported ID matches the authentication token. Raises
        # auth.AuthorizationError if not.
        logging.debug('Fetching bot group config')
        bot_group_cfg = bot_auth.validate_bot_id_and_fetch_config(
            bot_id, machine_type)

        # The server side dimensions from bot_group_cfg override bot-provided ones.
        # If both server side config and bot report some dimension, server side
        # config wins. We still emit an warning if bot tries to supply the dimension
        # and it disagrees with the server defined one. Note that this may happen
        # on a first poll after server side config for a bot has changed. The bot
        # doesn't know about new server-assigned dimensions yet in this case. Also
        # don't report ['default'], bot sends it in the handshake before it knows
        # anything at all.
        for dim_key, from_cfg in bot_group_cfg.dimensions.iteritems():
            from_bot = sorted(dimensions.get(dim_key) or [])
            from_cfg = sorted(from_cfg)
            if from_bot and from_bot != ['default'] and from_bot != from_cfg:
                logging.warning(
                    'Dimensions in bots.cfg don\'t match ones provided by the bot\n'
                    'bot_id: "%s", key: "%s", from_bot: %s, from_cfg: %s',
                    bot_id, dim_key, from_bot, from_cfg)
            dimensions[dim_key] = from_cfg

        # Fill in all result fields except 'quarantined_msg'.
        result = _ProcessResult(request=request,
                                bot_id=bot_id,
                                version=version,
                                state=state,
                                dimensions=dimensions,
                                bot_group_cfg=bot_group_cfg,
                                lease_expiration_ts=lease_expiration_ts,
                                maintenance_msg=state.get('maintenance'))

        # The bot may decide to "self-quarantine" itself. Accept both via
        # dimensions or via state. See bot_management._BotCommon.quarantined for
        # more details.
        if (bool(dimensions.get('quarantined'))
                or bool(state.get('quarantined'))):
            result.quarantined_msg = 'Bot self-quarantined'
            return result

        quarantined_msg = None
        # Use a dummy 'for' to be able to break early from the block.
        for _ in [0]:

            quarantined_msg = has_unexpected_keys(self.EXPECTED_KEYS, request,
                                                  'keys')
            if quarantined_msg:
                break

            quarantined_msg = has_missing_keys(self.REQUIRED_STATE_KEYS, state,
                                               'state')
            if quarantined_msg:
                break

            if not bot_id:
                quarantined_msg = 'Missing bot id'
                break
            if not dimensions.get('pool'):
                quarantined_msg = 'Missing \'pool\' dimension'
                break

            if not all(
                    config.validate_dimension_key(key)
                    and isinstance(values, list) and all(
                        config.validate_dimension_value(value)
                        for value in values)
                    for key, values in dimensions.iteritems()):
                quarantined_msg = ('Invalid dimensions type:\n%s' %
                                   json.dumps(dimensions,
                                              sort_keys=True,
                                              indent=2,
                                              separators=(',', ': ')))
                break

        if quarantined_msg:
            line = 'Quarantined Bot\nhttps://%s/restricted/bot/%s\n%s' % (
                app_identity.get_default_version_hostname(), bot_id,
                quarantined_msg)
            ereporter2.log_request(self.request, source='bot', message=line)
            result.quarantined_msg = quarantined_msg
            return result

        # Look for admin enforced quarantine.
        if bool(bot_settings and bot_settings.quarantined):
            result.quarantined_msg = 'Quarantined by admin'
            return result

        # TODO(maruel): Parallelise.
        task_queues.assert_bot_async(dimensions).get_result()
        return result
Пример #23
0
    def _process(self):
        """Returns True if the bot has invalid parameter and should be automatically
    quarantined.

    Does one DB synchronous GET.

    Returns:
      _ProcessResult instance, see its fields for more info.

    Raises:
      auth.AuthorizationError if bot's credentials are invalid.
    """
        request = self.parse_body()
        version = request.get('version', None)

        dimensions = request.get('dimensions') or {}
        state = request.get('state') or {}
        bot_id = None
        if dimensions.get('id'):
            dimension_id = dimensions['id']
            if (isinstance(dimension_id, list) and len(dimension_id) == 1
                    and isinstance(dimension_id[0], unicode)):
                bot_id = dimensions['id'][0]

        # Make sure bot self-reported ID matches the authentication token. Raises
        # auth.AuthorizationError if not.
        bot_group_cfg = bot_auth.validate_bot_id_and_fetch_config(bot_id)

        # The server side dimensions from bot_group_cfg override bot-provided ones.
        # If both server side config and bot report some dimension, server side
        # config wins. We still emit an error if bot tries to supply the dimension
        # and it disagrees with the server defined one. Don't report ['default'] as
        # an error, bot sends it in the handshake before it knows anything at all.
        for dim_key, from_cfg in bot_group_cfg.dimensions.iteritems():
            from_bot = sorted(dimensions.get(dim_key) or [])
            from_cfg = sorted(from_cfg)
            if from_bot and from_bot != ['default'] and from_bot != from_cfg:
                logging.error(
                    'Dimensions in bots.cfg doesn\'t match ones provided by the bot\n'
                    'bot_id: "%s", key: "%s", from_bot: %s, from_cfg: %s',
                    bot_id, dim_key, from_bot, from_cfg)
            dimensions[dim_key] = from_cfg

        # Fill in all result fields except 'quarantined_msg'.
        result = _ProcessResult(request=request,
                                bot_id=bot_id,
                                version=version,
                                state=state,
                                dimensions=dimensions,
                                bot_group_cfg=bot_group_cfg)

        # The bot may decide to "self-quarantine" itself. Accept both via
        # dimensions or via state. See bot_management._BotCommon.quarantined for
        # more details.
        if (bool(dimensions.get('quarantined'))
                or bool(state.get('quarantined'))):
            result.quarantined_msg = 'Bot self-quarantined'
            return result

        quarantined_msg = None
        # Use a dummy 'for' to be able to break early from the block.
        for _ in [0]:

            quarantined_msg = has_unexpected_keys(self.EXPECTED_KEYS, request,
                                                  'keys')
            if quarantined_msg:
                break

            quarantined_msg = has_missing_keys(self.REQUIRED_STATE_KEYS, state,
                                               'state')
            if quarantined_msg:
                break

            if not bot_id:
                quarantined_msg = 'Missing bot id'
                break
            if not dimensions.get('pool'):
                quarantined_msg = 'Missing \'pool\' dimension'
                break

            if not all(
                    isinstance(key, unicode)
                    and re.match(task_request.DIMENSION_KEY_RE, key)
                    and isinstance(values, list) and all(
                        isinstance(value, unicode) for value in values)
                    for key, values in dimensions.iteritems()):
                quarantined_msg = ('Invalid dimensions type:\n%s' %
                                   json.dumps(dimensions,
                                              sort_keys=True,
                                              indent=2,
                                              separators=(',', ': ')))
                break

            dimensions_count = task_to_run.dimensions_powerset_count(
                dimensions)
            if dimensions_count > task_to_run.MAX_DIMENSIONS:
                quarantined_msg = 'Dimensions product %d is too high' % dimensions_count
                break

            if not isinstance(state.get('lease_expiration_ts'),
                              (None.__class__, int)):
                quarantined_msg = (
                    'lease_expiration_ts (%r) must be int or None' %
                    (state['lease_expiration_ts']))
                break

        if quarantined_msg:
            line = 'Quarantined Bot\nhttps://%s/restricted/bot/%s\n%s' % (
                app_identity.get_default_version_hostname(), bot_id,
                quarantined_msg)
            ereporter2.log_request(self.request, source='bot', message=line)
            result.quarantined_msg = quarantined_msg
            return result

        # Look for admin enforced quarantine.
        bot_settings = bot_management.get_settings_key(bot_id).get()
        if bool(bot_settings and bot_settings.quarantined):
            result.quarantined_msg = 'Quarantined by admin'
            return result

        return result
Пример #24
0
 def test_default_config_not_ip_whitelisted(self):
     # With default config, callers not in 'bots' IP whitelist are rejected.
     self.mock_config(None)
     self.mock_caller('anonymous:anonymous', '1.1.1.1')
     with self.assertRaises(auth.AuthorizationError):
         bot_auth.validate_bot_id_and_fetch_config('some-bot')
Пример #25
0
 def test_fallback_to_another_method(self):
     self.mock_caller('anonymous:anonymous', '1.1.1.1')
     cfg = bot_auth.validate_bot_id_and_fetch_config(
         'bot_with_fallback_to_ip_wl')
     self.assertEquals({u'pool': [u'with_fallback_to_ip_wl']},
                       cfg.dimensions)
Пример #26
0
 def test_first_method_is_used(self):
     self.mock_caller('bot:bot_with_fallback_to_ip_wl.domain', '2.2.2.2')
     cfg = bot_auth.validate_bot_id_and_fetch_config(
         'bot_with_fallback_to_ip_wl')
     self.assertEquals({u'pool': [u'with_fallback_to_ip_wl']},
                       cfg.dimensions)
Пример #27
0
 def test_default_config_ip_whitelisted(self):
     # With default config, callers in 'bots' IP whitelist are allowed.
     self.mock_config(None)
     self.mock_caller('anonymous:anonymous', '1.2.3.5')
     bot_auth.validate_bot_id_and_fetch_config('some-bot')