def test_dimensions_powerset_count(self): dimensions = { 'a': ['1', '2'], 'b': 'code', 'd': ['3', '4'], } self.assertEqual(task_to_run.dimensions_powerset_count(dimensions), len(list(task_to_run._powerset(dimensions))))
def test_dimensions_powerset_count(self): dimensions = { 'a': ['1', '2'], 'b': 'code', 'd': ['3', '4'], } self.assertEqual( task_to_run.dimensions_powerset_count(dimensions), len(list(task_to_run._powerset(dimensions))))
def _process(self): """Returns True if the bot has invalid parameter and should be automatically quarantined. Does one DB synchronous GET. Returns: tuple(request, bot_id, version, state, dimensions, quarantined_msg) """ request = self.parse_body() version = request.get('version', None) dimensions = request.get('dimensions', {}) state = request.get('state', {}) bot_id = None if dimensions.get('id'): dimension_id = dimensions['id'] if (isinstance(dimension_id, list) and len(dimension_id) == 1 and isinstance(dimension_id[0], unicode)): bot_id = dimensions['id'][0] # The bot may decide to "self-quarantine" itself. Accept both via # dimensions or via state. See bot_management._BotCommon.quarantined for # more details. if (bool(dimensions.get('quarantined')) or bool(state.get('quarantined'))): return request, bot_id, version, state, dimensions, 'Bot self-quarantined' quarantined_msg = None # Use a dummy 'for' to be able to break early from the block. for _ in [0]: quarantined_msg = has_unexpected_keys(self.EXPECTED_KEYS, request, 'keys') if quarantined_msg: break quarantined_msg = has_missing_keys(self.REQUIRED_STATE_KEYS, state, 'state') if quarantined_msg: break if not bot_id: quarantined_msg = 'Missing bot id' break if not all( isinstance(key, unicode) and isinstance(values, list) and all(isinstance(value, unicode) for value in values) for key, values in dimensions.iteritems()): quarantined_msg = ('Invalid dimensions type:\n%s' % json.dumps(dimensions, sort_keys=True, indent=2, separators=(',', ': '))) break dimensions_count = task_to_run.dimensions_powerset_count( dimensions) if dimensions_count > task_to_run.MAX_DIMENSIONS: quarantined_msg = 'Dimensions product %d is too high' % dimensions_count break if quarantined_msg: line = 'Quarantined Bot\nhttps://%s/restricted/bot/%s\n%s' % ( app_identity.get_default_version_hostname(), bot_id, quarantined_msg) ereporter2.log_request(self.request, source='bot', message=line) return request, bot_id, version, state, dimensions, quarantined_msg # Look for admin enforced quarantine. bot_settings = bot_management.get_settings_key(bot_id).get() if bool(bot_settings and bot_settings.quarantined): return request, bot_id, version, state, dimensions, 'Quarantined by admin' return request, bot_id, version, state, dimensions, None
def _process(self): """Returns True if the bot has invalid parameter and should be automatically quarantined. Does one DB synchronous GET. Returns: tuple(request, bot_id, version, state, dimensions, quarantined_msg) """ request = self.parse_body() version = request.get("version", None) dimensions = request.get("dimensions", {}) state = request.get("state", {}) bot_id = None if dimensions.get("id"): dimension_id = dimensions["id"] if isinstance(dimension_id, list) and len(dimension_id) == 1 and isinstance(dimension_id[0], unicode): bot_id = dimensions["id"][0] # The bot may decide to "self-quarantine" itself. Accept both via # dimensions or via state. See bot_management._BotCommon.quarantined for # more details. if bool(dimensions.get("quarantined")) or bool(state.get("quarantined")): return request, bot_id, version, state, dimensions, "Bot self-quarantined" quarantined_msg = None # Use a dummy 'for' to be able to break early from the block. for _ in [0]: quarantined_msg = has_unexpected_keys(self.EXPECTED_KEYS, request, "keys") if quarantined_msg: break quarantined_msg = has_missing_keys(self.REQUIRED_STATE_KEYS, state, "state") if quarantined_msg: break if not bot_id: quarantined_msg = "Missing bot id" break if not all( isinstance(key, unicode) and isinstance(values, list) and all(isinstance(value, unicode) for value in values) for key, values in dimensions.iteritems() ): quarantined_msg = "Invalid dimensions type:\n%s" % json.dumps( dimensions, sort_keys=True, indent=2, separators=(",", ": ") ) break dimensions_count = task_to_run.dimensions_powerset_count(dimensions) if dimensions_count > task_to_run.MAX_DIMENSIONS: quarantined_msg = "Dimensions product %d is too high" % dimensions_count break if quarantined_msg: line = "Quarantined Bot\nhttps://%s/restricted/bot/%s\n%s" % ( app_identity.get_default_version_hostname(), bot_id, quarantined_msg, ) ereporter2.log_request(self.request, source="bot", message=line) return request, bot_id, version, state, dimensions, quarantined_msg # Look for admin enforced quarantine. bot_settings = bot_management.get_settings_key(bot_id).get() if bool(bot_settings and bot_settings.quarantined): return request, bot_id, version, state, dimensions, "Quarantined by admin" return request, bot_id, version, state, dimensions, None
def _process(self): """Returns True if the bot has invalid parameter and should be automatically quarantined. Does one DB synchronous GET. Returns: _ProcessResult instance, see its fields for more info. Raises: auth.AuthorizationError if bot's credentials are invalid. """ request = self.parse_body() version = request.get('version', None) dimensions = request.get('dimensions') or {} state = request.get('state') or {} bot_id = None if dimensions.get('id'): dimension_id = dimensions['id'] if (isinstance(dimension_id, list) and len(dimension_id) == 1 and isinstance(dimension_id[0], unicode)): bot_id = dimensions['id'][0] # Make sure bot self-reported ID matches the authentication token. Raises # auth.AuthorizationError if not. bot_group_cfg = bot_auth.validate_bot_id_and_fetch_config(bot_id) # The server side dimensions from bot_group_cfg override bot-provided ones. # If both server side config and bot report some dimension, server side # config wins. We still emit an error if bot tries to supply the dimension # and it disagrees with the server defined one. Don't report ['default'] as # an error, bot sends it in the handshake before it knows anything at all. for dim_key, from_cfg in bot_group_cfg.dimensions.iteritems(): from_bot = sorted(dimensions.get(dim_key) or []) from_cfg = sorted(from_cfg) if from_bot and from_bot != ['default'] and from_bot != from_cfg: logging.error( 'Dimensions in bots.cfg doesn\'t match ones provided by the bot\n' 'bot_id: "%s", key: "%s", from_bot: %s, from_cfg: %s', bot_id, dim_key, from_bot, from_cfg) dimensions[dim_key] = from_cfg # Fill in all result fields except 'quarantined_msg'. result = _ProcessResult(request=request, bot_id=bot_id, version=version, state=state, dimensions=dimensions, bot_group_cfg=bot_group_cfg) # The bot may decide to "self-quarantine" itself. Accept both via # dimensions or via state. See bot_management._BotCommon.quarantined for # more details. if (bool(dimensions.get('quarantined')) or bool(state.get('quarantined'))): result.quarantined_msg = 'Bot self-quarantined' return result quarantined_msg = None # Use a dummy 'for' to be able to break early from the block. for _ in [0]: quarantined_msg = has_unexpected_keys(self.EXPECTED_KEYS, request, 'keys') if quarantined_msg: break quarantined_msg = has_missing_keys(self.REQUIRED_STATE_KEYS, state, 'state') if quarantined_msg: break if not bot_id: quarantined_msg = 'Missing bot id' break if not dimensions.get('pool'): quarantined_msg = 'Missing \'pool\' dimension' break if not all( isinstance(key, unicode) and re.match(task_request.DIMENSION_KEY_RE, key) and isinstance(values, list) and all( isinstance(value, unicode) for value in values) for key, values in dimensions.iteritems()): quarantined_msg = ('Invalid dimensions type:\n%s' % json.dumps(dimensions, sort_keys=True, indent=2, separators=(',', ': '))) break dimensions_count = task_to_run.dimensions_powerset_count( dimensions) if dimensions_count > task_to_run.MAX_DIMENSIONS: quarantined_msg = 'Dimensions product %d is too high' % dimensions_count break if not isinstance(state.get('lease_expiration_ts'), (None.__class__, int)): quarantined_msg = ( 'lease_expiration_ts (%r) must be int or None' % (state['lease_expiration_ts'])) break if quarantined_msg: line = 'Quarantined Bot\nhttps://%s/restricted/bot/%s\n%s' % ( app_identity.get_default_version_hostname(), bot_id, quarantined_msg) ereporter2.log_request(self.request, source='bot', message=line) result.quarantined_msg = quarantined_msg return result # Look for admin enforced quarantine. bot_settings = bot_management.get_settings_key(bot_id).get() if bool(bot_settings and bot_settings.quarantined): result.quarantined_msg = 'Quarantined by admin' return result return result
def test_dimensions_powerset_count(self): dimensions = {"a": ["1", "2"], "b": "code", "d": ["3", "4"]} self.assertEqual( task_to_run.dimensions_powerset_count(dimensions), len(list(task_to_run._powerset(dimensions))) )