def get(self):
changes = bit9.RuleChangeSet.query(projection=[
bit9.RuleChangeSet.blockable_key
],
distinct=True).fetch()
change_count = len(changes)
logging.info('Retrieved %d pending Bit9 change(s)', change_count)
monitoring.pending_changes.Set(change_count)
# Don't over-defer to the bit9-commit-change queue, otherwise it can back up
# real fast with duplicate tasks in the event of a large backlog.
queue_size = taskqueue_utils.QueueSize(
queue=constants.TASK_QUEUE.BIT9_COMMIT_CHANGE, deadline=30)
available = max(20 - queue_size, 0)
logging.info('Deferring %d Bit9 change(s)', available)
# Randomly sample from the outstanding changes in order to avoid
# head-of-the-line blocking due to unsynced hosts, for example.
sample_size = min(change_count, available)
selected_changes = random.sample(changes, sample_size)
blockable_keys = [change.blockable_key for change in selected_changes]
for blockable_key in blockable_keys:
change_set.DeferCommitBlockableChangeSet(blockable_key)
def testDeferCommitBlockableChangeSet_TailDefer_NoMoreChanges(self):
with mock.patch.object(change_set, 'CommitChangeSet') as mock_commit:
change_set.DeferCommitBlockableChangeSet(self.binary.key)
self.assertTaskCount(constants.TASK_QUEUE.BIT9_COMMIT_CHANGE, 1)
self.RunDeferredTasks(constants.TASK_QUEUE.BIT9_COMMIT_CHANGE)
self.assertTaskCount(constants.TASK_QUEUE.BIT9_COMMIT_CHANGE, 0)
mock_commit.assert_called_once_with(self.change.key)
def _CheckAndResolveAnomalousBlock(blockable_key, host_id):
"""Checks whether an unfulfilled rule already existed for this blockable.
If there are unfulfilled rules, triggers an attempt to commit them back to the
database.
Args:
blockable_key: The key of the blockable that was blocked.
host_id: The host on which the block occurred.
Returns:
Whether the block was anomalous (i.e. whether an unfulfilled rule existed
for the blockable-host pair).
"""
# Check and handle anomalous block events by detecting unfulfilled rules and,
# if present, attempting to commit them.
# pylint: disable=g-explicit-bool-comparison
unfulfilled_rule_query = bit9.Bit9Rule.query(
bit9.Bit9Rule.is_committed == True,
bit9.Bit9Rule.is_fulfilled == False,
bit9.Bit9Rule.host_id == host_id,
ancestor=blockable_key).order(bit9.Bit9Rule.updated_dt)
# pylint: enable=g-explicit-bool-comparison
unfulfilled_rules = unfulfilled_rule_query.fetch()
# Installer rules shouldn't be local (e.g. have host_id's) so they shouldn't
# have been returned by the query. Still, the sanity check couldn't hurt.
assert all(rule.policy in constants.RULE_POLICY.SET_EXECUTION
for rule in unfulfilled_rules)
if unfulfilled_rules:
logging.info('Processing %s unfulfilled rules for %s',
len(unfulfilled_rules), blockable_key.id())
# Mark all outstanding unfulfilled rules _except_ the most recent one as
# fulfilled as we're going to ignore them.
for rule in unfulfilled_rules[:-1]:
rule.is_fulfilled = True
# Mark the most recent unfulfilled rule as uncommitted as we're going to
# commit it.
unfulfilled_rules[-1].is_committed = False
# Create and trigger a change set to commit the most recent rule.
change = bit9.RuleChangeSet(rule_keys=[unfulfilled_rules[-1].key],
change_type=unfulfilled_rules[-1].policy,
parent=blockable_key)
ndb.put_multi(unfulfilled_rules + [change])
change_set.DeferCommitBlockableChangeSet(blockable_key)
return bool(unfulfilled_rules)
def get(self, blockable_id):
blockable = base_db.Blockable.get_by_id(blockable_id)
if blockable is None:
self.abort(httplib.NOT_FOUND,
explanation='Blockable does not exist')
platform = blockable.GetPlatformName()
if platform != constants.PLATFORM.WINDOWS:
self.abort(httplib.BAD_REQUEST,
explanation='Invalid Blockable platform: %s' %
(platform))
change_set.DeferCommitBlockableChangeSet(blockable.key)
def testDeferCommitBlockableChangeSet_TailDefer_MoreChanges(self):
test_utils.CreateRuleChangeSet(
self.binary.key,
rule_keys=[self.local_rule.key],
change_type=constants.RULE_POLICY.BLACKLIST)
with mock.patch.object(change_set, 'CommitChangeSet') as mock_commit:
change_set.DeferCommitBlockableChangeSet(self.binary.key)
self.assertTaskCount(constants.TASK_QUEUE.BIT9_COMMIT_CHANGE, 1)
self.RunDeferredTasks(constants.TASK_QUEUE.BIT9_COMMIT_CHANGE)
# Tail defer task for remaining change.
self.assertTaskCount(constants.TASK_QUEUE.BIT9_COMMIT_CHANGE, 1)
mock_commit.assert_called_once_with(self.change.key)
def _SetInstallerPolicy(self, blockable_id, new_policy):
blockable = base_db.Blockable.get_by_id(blockable_id)
# pylint: disable=g-explicit-bool-comparison
installer_rule_query = bit9_db.Bit9Rule.query(
bit9_db.Bit9Rule.in_effect == True,
bit9_db.Bit9Rule.policy.IN(constants.RULE_POLICY.SET_INSTALLER),
ancestor=blockable.key)
# pylint: enable=g-explicit-bool-comparison
existing_rule = installer_rule_query.get()
if existing_rule:
if existing_rule.policy == new_policy:
return blockable.is_installer
else:
existing_rule.in_effect = False
existing_rule.put()
# Create the Bit9Rule associated with the installer state and a change set
# to commit it.
new_rule = bit9_db.Bit9Rule(
rule_type=blockable.rule_type,
in_effect=True,
policy=new_policy,
parent=blockable.key)
new_rule.put()
change = bit9_db.RuleChangeSet(
rule_keys=[new_rule.key],
change_type=new_rule.policy,
parent=blockable.key)
change.put()
base_db.AuditLog.Create(
blockable,
'User %s changing installer state to %s' % (
self.user.key.id(), new_policy))
change_set.DeferCommitBlockableChangeSet(blockable.key)
# Update the blockable's is_installer property.
blockable.is_installer = new_policy == constants.RULE_POLICY.FORCE_INSTALLER
blockable.put()
return blockable.is_installer