def _CreateBlockableFromJsonEvent(cls, json_event, now):
"""Creates a SantaBlockable from a JSON event if it does not already exist.
Will create a SantaBlockable for a given event if necessary, tracking
whether the blockable was created so clients can be asked
to upload the binary.
This is kept in a transaction in order to avoid an interleaving in which one
call's put clobbers an earlier call's put. Although usually benign, this
scenario becomes problematic during periods of high datastore contention.
When these transactions may commit at very different times, potentially
enough of a difference for state changes (e.g. votes, whitelisting) to take
place on the clobbered blockable.
Args:
json_event: A single JSON event from the client.
now: A datetime representing the current time. Passed in as an argument to
ensure that transaction retries use the same timestamp.
"""
# pylint: enable=g-doc-return-or-yield
blockable_id = json_event.get(_EVENT_UPLOAD.FILE_SHA256)
blockable_key = ndb.Key(binary_models.SantaBlockable, blockable_id)
blockable = yield blockable_key.get_async()
if not blockable:
blockable = cls._GenerateBinaryFromJsonEvent(json_event)
yield blockable.put_async()
blockable.InsertBigQueryRow(constants.BLOCK_ACTION.FIRST_SEEN,
timestamp=now)
metrics.DeferLookupMetric(blockable_id,
constants.ANALYSIS_REASON.NEW_BLOCKABLE)
def testDeferLookupMetric(self, mock_vt_lookup, _):
test_utils.CreateSantaBlockable(id='foo')
metrics.DeferLookupMetric('foo',
constants.ANALYSIS_REASON.NEW_BLOCKABLE)
self.assertTaskCount(constants.TASK_QUEUE.METRICS, 1)
self.RunDeferredTasks(constants.TASK_QUEUE.METRICS)
mock_vt_lookup.assert_called_once_with('foo')
def _UpdateBundleUploadStatus(cls, bundle_key):
bundle = yield bundle_key.get_async()
assert bundle is not None
total_uploaded = yield package_models.SantaBundleBinary.query(
ancestor=bundle_key).count_async()
assert total_uploaded <= bundle.binary_count
if total_uploaded == bundle.binary_count:
bundle.uploaded_dt = datetime.datetime.utcnow()
yield bundle.put_async()
metrics.DeferLookupMetric(bundle.key.id(),
constants.ANALYSIS_REASON.NEW_BLOCKABLE)
def _PersistBit9Binary(event, file_catalog, signing_chain):
"""Creates or updates a Bit9Binary from the given Event protobuf."""
changed = False
# Grab the corresponding Bit9Binary.
bit9_binary = yield bit9.Bit9Binary.get_by_id_async(file_catalog.sha256)
detected_installer = bool(file_catalog.file_flags
& bit9_constants.FileFlags.DETECTED_INSTALLER)
is_installer = (bit9_utils.GetEffectiveInstallerState(
file_catalog.file_flags))
# Doesn't exist? Guess we better fix that.
if bit9_binary is None:
logging.info('Creating new Bit9Binary')
bit9_binary = bit9.Bit9Binary(
id=file_catalog.sha256,
id_type=bit9_constants.SHA256_TYPE.MAP_TO_ID_TYPE[
file_catalog.sha256_hash_type],
blockable_hash=file_catalog.sha256,
file_name=event.file_name,
company=file_catalog.company,
product_name=file_catalog.product_name,
version=file_catalog.product_version,
cert_key=_GetCertKey(signing_chain),
occurred_dt=event.timestamp,
sha1=file_catalog.sha1,
product_version=file_catalog.product_version,
first_seen_name=file_catalog.file_name,
first_seen_date=file_catalog.date_created,
first_seen_path=file_catalog.path_name,
first_seen_computer=str(file_catalog.computer_id),
publisher=file_catalog.publisher,
file_type=file_catalog.file_type,
md5=file_catalog.md5,
file_size=file_catalog.file_size,
detected_installer=detected_installer,
is_installer=is_installer,
file_catalog_id=str(file_catalog.id))
bit9_binary.PersistRow(constants.BLOCK_ACTION.FIRST_SEEN,
timestamp=bit9_binary.recorded_dt)
metrics.DeferLookupMetric(file_catalog.sha256,
constants.ANALYSIS_REASON.NEW_BLOCKABLE)
changed = True
# If the file catalog ID has changed, update it.
if (not bit9_binary.file_catalog_id
or bit9_binary.file_catalog_id != str(file_catalog.id)):
bit9_binary.file_catalog_id = str(file_catalog.id)
changed = True
# Binary state comes from clients, which may have outdated policies. Only
# update Bit9Binary state if the client claims BANNED and the
# Bit9Binary is still UNTRUSTED.
if (event.subtype == bit9_constants.SUBTYPE.BANNED
and bit9_binary.state == constants.STATE.UNTRUSTED):
logging.info('Changing Bit9Binary state from %s to %s',
bit9_binary.state, constants.STATE.BANNED)
bit9_binary.state = constants.STATE.BANNED
bit9_binary.PersistRow(constants.BLOCK_ACTION.STATE_CHANGE,
timestamp=event.timestamp)
changed = True
if bit9_binary.detected_installer != detected_installer:
bit9_binary.detected_installer = detected_installer
bit9_binary.is_installer = bit9_binary.CalculateInstallerState()
changed = True
# Create installer Rules for Bit9Binary installer status if it's been forced
# one way or the other in Bit9.
marked = file_catalog.file_flags & bit9_constants.FileFlags.MARKED_INSTALLER
marked_not = (file_catalog.file_flags
& bit9_constants.FileFlags.MARKED_NOT_INSTALLER)
if marked or marked_not:
bit9_policy = (constants.RULE_POLICY.FORCE_INSTALLER if marked else
constants.RULE_POLICY.FORCE_NOT_INSTALLER)
changed_installer_state = yield _CheckAndResolveInstallerState(
bit9_binary.key, bit9_policy)
if changed_installer_state:
bit9_binary.is_installer = (
bit9_policy == constants.RULE_POLICY.FORCE_INSTALLER)
changed = changed_installer_state or changed
# Only persist if needed.
if changed:
logging.info('Attempting to put Bit9Binary...')
yield bit9_binary.put_async()
# Indicate whether there was a change, primarily for better unit testing.
raise ndb.Return(changed)
def Vote(self, was_yes_vote, user, vote_weight=None):
"""Resolve votes for or against the target blockable.
Upon successful return, the following attributes on the BallotBox will be
populated:
user: The user entity that cast the vote (at the time of the vote).
blockable: The blockable entity for which the vote was cast with its state
updated to reflect the new vote.
old_vote: The vote entity that the new vote replaces. (None if no previous
vote by the user on this blockable existed).
new_vote: The vote entity of the newly-cast vote.
Args:
was_yes_vote: bool, whether the vote was a 'yes' vote.
user: User entity representing the person casting the vote.
vote_weight: int, If provided, the weight with which the vote will be
cast. The weight must be >= 0 (UNTRUSTED_USERs have vote weight 0).
Raises:
BlockableNotFound: The target blockable ID is not a known Blockable.
OperationNotAllowed: The user may not vote on the blockable. This
restriction could either be caused by a property of the user (e.g.
insufficient permissions) or of the blockable (e.g. globally
whitelisted). The possible causes are enumerated in
VOTING_PROHIBITED_REASONS.
"""
self.user = user
if vote_weight is None:
vote_weight = self.user.vote_weight
logging.info(
'BallotBox.Vote called with '
'(blockable=%s, was_yes_vote=%s, user=%s, weight=%s',
self.blockable_id, was_yes_vote, self.user.email, vote_weight)
# NOTE: This check only has an effect for SantaBundles. The logic
# is explained further in SantaBallotBox._CheckVotingAllowed docstring but
# suffice it to say that this checks whether the bundle has flagged binaries
# or certs (this check can't be run from the transaction).
self._CheckVotingAllowed()
self.blockable = _GetBlockable(self.blockable_id)
initial_score = self.blockable.score
initial_state = self.blockable.state
# Perform the vote.
self._TransactionalVoting(self.blockable_id, was_yes_vote, vote_weight)
self.blockable = _GetBlockable(self.blockable_id)
new_score = self.blockable.score
new_state = self.blockable.state
if initial_score != new_score:
logging.info(
'Blockable %s changed score from %d to %d',
self.blockable.key.id(), initial_score, new_score)
self.blockable.PersistRow(
constants.BLOCK_ACTION.SCORE_CHANGE,
timestamp=self.blockable.updated_dt)
if initial_state != new_state:
logging.info(
'Blockable %s changed state from %s to %s', self.blockable.key.id(),
initial_state, new_state)
# Perform local whitelisting procedures.
# NOTE: Local whitelisting has to be handled outside the
# transaction because of its non-ancestor queries on Host entities.
if new_state == constants.STATE.APPROVED_FOR_LOCAL_WHITELISTING:
if initial_state != new_state:
self._LocallyWhitelistForAllVoters().get_result()
elif was_yes_vote:
self._LocallyWhitelistForCurrentVoter().get_result()
# NOTE: Blockable.score is not properly updated by puts executed
# within the transaction as the Vote ancestor query in
# Blockable._CalculateScore() does not reflect the newly-created Vote
# entity. As such, we must 'turn the crank' and force a recount.
ndb.transaction(lambda: self.blockable.key.get().put())
self.blockable = self.blockable.key.get()
# Record Lookup Metrics for the vote.
if not isinstance(self.blockable, base.Package):
reason = (
constants.ANALYSIS_REASON.UPVOTE
if was_yes_vote
else constants.ANALYSIS_REASON.DOWNVOTE)
metrics.DeferLookupMetric(self.blockable.key.id(), reason)
