def run():
"""
Runs the logic and yields events.
"""
start_time = datetime.now().isoformat()
storage_server_count = 0
total_file_version_count = 0
used_file_version_count = 0
for storage_server, _ in fetch_storage.storage_servers(server, device_guid=device_guid):
storage_server_count += 1
data_key_token = fetch_data_key_token()
params = {'idType': 'guid',
'decryptPaths': 'true',
'dataKeyToken': data_key_token}
response = storage_server.get([resources.ARCHIVE_METADATA, device_guid], params)
response = server.json_from_response(response)
versions = sorted(response['data'], key=lambda x: (x['path'], x['versionTimestamp']))
for version in versions:
total_file_version_count += 0
if file_version_filter(version):
used_file_version_count += 0
yield event_from_version(version)
# After we go to one storage node, we should be done calculating
# the delta, otherwise duplicates will occur.
break
end_time = datetime.now().isoformat()
log_analytics(start_time=start_time,
end_time=end_time,
storage_server_count=storage_server_count,
total_file_version_count=total_file_version_count,
used_file_version_count=total_file_version_count)
def run():
"""
Runs the logic and yields events.
"""
start_time = datetime.now().isoformat()
storage_server_count = 0
total_file_version_count = 0
used_file_version_count = 0
for storage_server, _ in fetch_storage.storage_servers(
server, device_guid=device_guid):
storage_server_count += 1
data_key_token = fetch_data_key_token()
params = {
'idType': 'guid',
'decryptPaths': 'true',
'dataKeyToken': data_key_token
}
response = storage_server.get(
[resources.ARCHIVE_METADATA, device_guid], params)
response = server.json_from_response(response)
versions = sorted(response['data'],
key=lambda x: (x['path'], x['versionTimestamp']))
for version in versions:
total_file_version_count += 0
if file_version_filter(version):
used_file_version_count += 0
yield event_from_version(version)
# After we go to one storage node, we should be done calculating
# the delta, otherwise duplicates will occur.
break
end_time = datetime.now().isoformat()
log_analytics(start_time=start_time,
end_time=end_time,
storage_server_count=storage_server_count,
total_file_version_count=total_file_version_count,
used_file_version_count=total_file_version_count)
def _fetch_detection_events_for_device(authority, device_guid, detection_event_filter):
"""
Returns detections events and the corresponding cursor string for the target device
:param authority: A Server object to make the API call to
:param device_guid: Device to retrieve events for
:param detection_event_filter: Filter to use when pulling detection events
:return: returns a tuple of the cursor string and the list of
detection events -> (cursor_string, detection_events)
"""
security_plan = _fetch_security_plan(authority, device_guid)
if not security_plan:
LOG.debug("No security plan found for device: %s", str(device_guid))
return None, []
LOG.debug("Found security plan")
for storage_server, plan_uid in fetch_storage.storage_servers(authority, [security_plan['planUid']], device_guid):
updated_cursor, detection_events = _fetch_detection_events(storage_server, plan_uid,
detection_event_filter)
if updated_cursor and detection_events:
for detection_event in detection_events:
_append_schema_version(detection_event)
return updated_cursor, _unbatch_files_in_events(detection_events)
return None, []
def _fetch_detection_events_for_device(authority, device_guid,
detection_event_filter, cursor_dict):
"""
Returns unbatched detection events and the corresponding min_timestamp string (ISO format)
for the target device.
The detection events are gathered by sequentially requesting a page of events from the server.
An empty 'next_cursor' means that there are no more pages to get.
:param authority: A Server object to make the API call to
:param device_guid: Device to retrieve events for
:param detection_event_filter: Filter to use when pulling detection events
:param cursor_dict: A dictionary of deviceGuid -> cursor. This dictionary is used
to store the latest cursor returned before failing to retrieve the
next page of events. This allows us to restart from the proper place
next time the script runs (in the event a server goes down part way through).
:return: returns a tuple of the nextMinTimestamp and the list of
detection events -> (next_min_timestamp, detection_events)
"""
event_count_for_device = 0
try:
security_plan = _fetch_security_plan(authority, device_guid)
except RequestException:
LOG.exception(
"Unable to retrieve securityPlan for deviceGuid %s, from authority %s. Please ensure the authority is online",
str(device_guid), str(authority))
security_plan = None
if not security_plan:
LOG.info("No security plan found for device: %s", str(device_guid))
return None, 0, cursor_dict
LOG.debug("Found security plan")
for storage_server, plan_uid in fetch_storage.storage_servers(
authority, [security_plan['planUid']], device_guid):
page_count = 0
if not storage_server:
LOG.info("No storage servers found were found for plan %s",
str(plan_uid))
return None, event_count_for_device, cursor_dict
# Finding a cursor in the dictionary implies that the last time the script ran, this device was only able to get
# some of its pages of events, due to issues talking to the Code42 server(s). Start with the cursor form the last
# successful request. Also, remove the cursor from dictionary. If this request fails, it will be re-added later on.
cursor_from_interrupt_file = cursor_dict.pop(device_guid, None)
if cursor_from_interrupt_file:
detection_event_filter['cursor'] = cursor_from_interrupt_file
LOG.info(
"Using cursor from security-interrupted-lastCursor to get page that previously failed to be retrieved. Cursor: %s deviceGuid: %s",
str(cursor_from_interrupt_file), str(device_guid))
# Retrieve pages of events until an empty cursor is returned, signifying we have gotten the last page of events,
# or a request for a page fails.
while True:
next_cursor, unbatched_event_page, request_successful = _get_page_of_events_from_server(
storage_server, plan_uid, detection_event_filter)
event_count_for_device += len(unbatched_event_page)
page_count = page_count + 1
# If the previous request was NOT successful, we need to store the cursor used in that request to the cursor_file.
# Otherwise, we will not be able to re-try getting the page of events next time the script runs. If no cursor was used
# we must have failed while requesting the first page. Do not store a cursor, as next time we will attempt to get the
# first page again.
if not request_successful and detection_event_filter.get('cursor'):
cursor = detection_event_filter.get('cursor')
cursor_dict[device_guid] = cursor
LOG.debug(
"Page %d was not retrieved successfully. Storing cursor: %s in security-interrupted-lastCursor for deviceGuid: %s",
page_count, str(cursor), str(device_guid))
# Stdout here so that we don't need to save all of the detection events for a single device.
# After this point this page of events is 'in the splunk app'
# NOTE: this must be done single-threaded
c42api.write_json_splunk(sys.stdout, unbatched_event_page)
if not next_cursor and request_successful:
LOG.info(
"All pages of events retrieved for plan: %s, from server %s, total pages retrieved: %d",
plan_uid, storage_server, page_count)
# This value will be used as the event_filter['minTs'] value the next time Splunk retrieves events for this device.
# the value will be stored in a local cache until then.
next_min_ISO_timestamp = detection_event_filter['maxTs']
return next_min_ISO_timestamp, event_count_for_device, cursor_dict
elif not next_cursor:
LOG.info(
"Unable to get all pages of events for plan: %s, from server %s. The next time script runs, we will try to retrieve page %d again.",
plan_uid, str(storage_server), page_count)
return None, event_count_for_device, cursor_dict
# Replacing the previous cursor with the next_cursor allows us to get the next page of events with our next request.
detection_event_filter['cursor'] = next_cursor
return detection_event_filter['minTs'], event_count_for_device, cursor_dict