def test_atomic(self):
cache = TTLCache(maxsize=1, ttl=1, timer=Timer(auto=True))
cache[1] = 1
self.assertEqual(1, cache[1])
cache[1] = 1
self.assertEqual(1, cache.get(1))
cache[1] = 1
self.assertEqual(1, cache.pop(1))
cache[1] = 1
self.assertEqual(1, cache.setdefault(1))
def test_atomic(self):
cache = TTLCache(maxsize=1, ttl=1, timer=Timer(auto=True))
cache[1] = 1
self.assertEqual(1, cache[1])
cache[1] = 1
self.assertEqual(1, cache.get(1))
cache[1] = 1
self.assertEqual(1, cache.pop(1))
cache[1] = 1
self.assertEqual(1, cache.setdefault(1))
class CustomThreatService(BaseController):
"""
Implements a custom threat service for Resilient.
The root path (/xxx/ below) is configurable.
The service provides the following URLs:
OPTIONS /<root_path>/<any_sub_path>
POST /<root_path>/<any_sub_path>
GET /<root_path>/<any_sub_path>/<id>
"""
# Arbitrary constant
namespace = UUID('18222d9c-adf0-409c-aa19-beb27130ba12')
def __init__(self, opts):
super(CustomThreatService, self).__init__(**_make_args(opts))
# Configurable options
self.options = opts.get(CONFIG_SECTION, {})
# Do we support "file-content" artifacts? Default is no.
# TODO add implementation support to parse the file content
self.support_upload_file = strtobool(
self.options.get(CONFIG_UPLOAD_FILE.key,
CONFIG_UPLOAD_FILE.default))
# Default time that this service will tell Resilient to retry
self.first_retry_secs = int(
self.options.get(CONFIG_FIRST_RETRY_SECS.key,
CONFIG_FIRST_RETRY_SECS.default)) or 5
self.later_retry_secs = int(
self.options.get(CONFIG_LATER_RETRY_SECS.key,
CONFIG_LATER_RETRY_SECS.default)) or 60
# Size of the request cache
self.cache_size = int(
self.options.get(CONFIG_CACHE_SIZE.key, CONFIG_CACHE_SIZE.default))
# TTL of the request cache (millis before we give up on a request lookup)
self.cache_ttl = int(
self.options.get(CONFIG_CACHE_TTL.key, CONFIG_CACHE_TTL.default))
# Limit to the number of queries we'll answer for unfinished searchers (count before giving up on them)
self.max_retries = int(
self.options.get(CONFIG_MAX_RETRIES.key,
CONFIG_MAX_RETRIES.default))
# IDs and their results are maintained in a cache so that we can set
# an upper bound on the number of in-progress and recent lookups.
self.cache = TTLCache(maxsize=self.cache_size, ttl=self.cache_ttl)
# Helper component does event dispatch work
self.async_helper = CustomThreatServiceHelper(self)
(self.helper_thread, self.bridge) = self.async_helper.start()
urls = ["{0}/{1}".format(self.channel, e) for e in self.events()]
LOG.info("Web handler for %s", ", ".join(urls))
self.auth_user = self.options.get(CONFIG_AUTH_USER.key,
CONFIG_AUTH_USER.default)
self.auth_password = self.options.get(CONFIG_AUTH_PASSWORD.key,
CONFIG_AUTH_USER.default)
# Web endpoints
@exposeWeb("OPTIONS")
def _options_request(self, event, *args, **kwargs):
"""
Options indicate to Resilient whether file upload is supported.
"""
LOG.info(event.args[0])
options = {"upload_file": bool(self.support_upload_file)}
return options
@exposeWeb("POST")
def _post_request(self, event, *args, **kwargs):
LOG.info(event.args[0])
if not self.check_authentication(event.args[0]):
LOG.error("Custom Threat Service Authentication Error")
response = event.args[1]
response.status = 500
return {"id": None, "hits": []}
result = self._handle_post_request(event, *args, **kwargs)
LOG.info("%s: %s", event.args[1].status, json.dumps(result))
return result
def _handle_post_request(self, event, *args, **kwargs):
"""
Responds to POST /cts/<anything>
The URL below /cts/ is specific to this threat service. For example,
/cts/one and /cts/two can be registered as two separate threat sources.
The string 'one' or 'two' becomes the channel that searcher events are dispatched on.
Request is a ThreatServiceArtifactDTO containing the artifact to be scanned
Response is a ResponseDTO containing the response, or 'please retry' (HTTP status 303).
"""
request = event.args[0]
response = event.args[1]
# The channels that searchers are listening for events
cts_channel = searcher_channel(*args)
value = request.body.getvalue()
if not value:
err = "Empty request"
LOG.warn(err)
return {"id": str(uuid4()), "hits": []}
# Resilient sends artifacts in two formats: multi-part MIME, or plain JSON.
# server may send either, even for cases where there is no file content,
# so check content-type and decode appropriately.
try:
if request.headers and "form-data" in request.headers.get(
"Content-Type", ""):
multipart_data = decoder.MultipartDecoder(
value, request.headers["Content-Type"])
body = json.loads(multipart_data.parts[0].text)
LOG.debug(body)
else:
body = json.loads(value.decode("utf-8"))
LOG.debug(body)
except (ValueError, NonMultipartContentTypeException) as e:
err = "Can't handle request: {}".format(e)
LOG.warn(err)
LOG.debug(value)
return {"id": str(uuid4()), "hits": []}
if not isinstance(body, dict):
# Valid JSON but not a valid request.
err = "Invalid request: {}".format(json.dumps(body))
LOG.warn(err)
return {"id": str(uuid4()), "hits": []}
# Generate a request ID, derived from the artifact being requested.
request_id = str(uuid5(self.namespace, json.dumps(body)))
artifact_type = body.get("type", "unknown")
artifact_value = body.get("value")
response_object = {"id": request_id, "hits": []}
cache_key = (cts_channel, request_id)
if artifact_type == "net.name" and artifact_value == "localhost":
# Hard-coded response to 'net.name' of 'localhost'
# because this is used in 'resutil threatservicetest'
# and we want to return an immediate (not async) response
return response_object
# If we already have a completed query for this key, return it immmediately
request_data = self.cache.get(cache_key)
if request_data and request_data.get("complete"):
response_object["hits"] = request_data.get("hits", [])
return response_object
response.status = 303
response_object["retry_secs"] = self.first_retry_secs
# Add the request to the cache, then notify searchers that there's a new request
self.cache.setdefault(cache_key, {
"id": request_id,
"artifact": body,
"hits": [],
"complete": False
})
evt = ThreatServiceLookupEvent(request_id=request_id,
name=artifact_type,
artifact=body,
channel=cts_channel)
self.async_helper.fire(evt, HELPER_CHANNEL)
return response_object
@exposeWeb("GET")
def _get_request(self, event, *args, **kwargs):
LOG.info(event.args[0])
if not self.check_authentication(event.args[0]):
LOG.error("Custom Threat Service Authentication Error")
response = event.args[1]
response.status = 500
return {"id": None, "hits": []}
result = self._handle_get_request(event, *args, **kwargs)
LOG.info("%s: %s", event.args[1].status, json.dumps(result))
return result
def _handle_get_request(self, event, *args, **kwargs):
"""
Responds to GET /cts/<anything>/<request-id>
The URL below /cts/ is specific to this threat service. For example,
/cts/one and /cts/two are considered two separate threat sources.
Response is a ResponseDTO containing the response, or 'please retry'
"""
LOG.info(event.args[0])
response = event.args[1]
request_id = None
if not args:
return {"id": request_id, "hits": []}
# The ID of the lookup request
request_id = args[-1]
# The channel that searchers are listening for events
cts_channel = searcher_channel(*args[:-1])
response_object = {"id": request_id, "hits": []}
cache_key = (cts_channel, request_id)
request_data = self.cache.get(cache_key)
if not request_data:
# There's no record of this request in our cache, return empty hits
response.status = 200
return response_object
response_object["hits"] = request_data["hits"]
if not request_data["complete"]:
# The searchers haven't finished yet, return partial hits if available
response.status = 303
response_object["retry_secs"] = self.later_retry_secs
# Update the counter, so we can detect "stale" failures
request_data["count"] = request_data.get("count", 0) + 1
if request_data["count"] > self.max_retries:
LOG.info("Exceeded max retries for {}".format(cache_key))
try:
self.cache.pop(cache_key)
except KeyError:
pass
response.status = 200
return response_object
return response_object
# Remove the result from cache
# self.cache.pop(cache_key)
return response_object
@handler(channel=LOOKUP_COMPLETE_CHANNEL)
def _lookup_complete(self, event, *args, **kwargs):
"""
A lookup event was completed
"""
if not isinstance(event.parent, ThreatServiceLookupEvent):
return
results = event.parent.value.getValue()
artifact = event.parent.artifact
cts_channel = event.parent.cts_channel
request_id = event.parent.request_id
LOG.info("Lookup complete: %s, %s", event.parent, results)
# Depending on how many components handled this lookup event,
# the results can be a single value (dict), or an array, or None,
# or an exception, or a tuple (type, exception, traceback)
hits = []
complete = True
if isinstance(results, list):
for result in results:
if result:
if isinstance(result,
(tuple, ThreatLookupIncompleteException)):
LOG.info("Retry later!")
complete = False
elif isinstance(result, (tuple, Exception)):
LOG.error("No hits due to exception")
else:
hits.append(result)
elif results:
if isinstance(results, (tuple, ThreatLookupIncompleteException)):
LOG.info("Retry later!")
complete = False
elif isinstance(results, (tuple, Exception)):
LOG.error("No hits due to exception")
else:
hits.append(results)
# Store the result and mark as complete (or not)
cache_key = (cts_channel, request_id)
self.cache[cache_key] = {
"id": request_id,
"artifact": artifact,
"hits": hits,
"complete": complete
}
def _get_authentication_headers(self, request):
"""[extract user/password info in http header: Authentication Basic into a list]"""
if request.headers and "Basic" in request.headers.get(
"Authorization", ""):
auth = request.headers.get("Authorization", "").split(' ')
user_password = base64.b64decode(auth[1])
return b_to_s(user_password).split(":")
return [None, None]
def _is_authenticated(self, user_password_list):
"""[check if a user/password pair matches values set in app.config]"""
return self.auth_user == user_password_list[
0] and self.auth_password == user_password_list[1]
def check_authentication(self, request):
"""[check if the headers contain user/password information and they match the settings in app.config]"""
return self._is_authenticated(
self._get_authentication_headers(request))
class FrameReaderWorker(Thread):
log = logging.getLogger("events_processor.FrameReaderWorker")
@inject
def __init__(self, config: ConfigProvider, frame_queue: FrameQueue,
system_time: SystemTime, frame_reader: FrameReader):
super().__init__()
self._config = config
self._stop_requested = False
self._system_time = system_time
self._frame_queue = frame_queue
event_ttl = config.events_window_seconds + config.cache_seconds_buffer
self._recent_events = TTLCache(maxsize=10000000, ttl=event_ttl)
self._frame_reader = frame_reader
if config.event_ids:
self._events_iter = lambda: self._frame_reader.events_by_id_iter(
config.event_ids)
self._skip_mailed = False
else:
self._events_iter = self._frame_reader.events_iter
self._skip_mailed = True
def run(self) -> None:
while not self._stop_requested:
before = time.monotonic()
self._collect_events()
time_spent = (time.monotonic() - before)
self._system_time.sleep(
max(self._config.event_loop_seconds - time_spent, 0))
self.log.info("Terminating")
def stop(self) -> None:
self._stop_requested = True
def _collect_events(self) -> None:
self.log.info("Fetching event list")
for event_json in self._events_iter():
event_id = event_json['Id']
event_info = self._recent_events.setdefault(event_id, EventInfo())
event_info.event_json = event_json
if event_info.all_frames_were_read or event_info.notification_status.was_sending:
continue
if event_info.emailed and self._skip_mailed:
self.log.debug(
f'Skipping processing of event {event_info} as it was already mailed'
)
continue
self._collect_frames(event_info)
def _collect_frames(self, event_info: EventInfo):
self.log.info(f"Reading event frames: {event_info}")
frames = self._frame_reader.frames(event_info)
if frames:
pending_frames = False
for frame_info in frames:
if frame_info.type != 'Alarm':
continue
if frame_info.frame_id in event_info.retrieved_frame_ids:
continue
if self._frame_data_has_settled(frame_info):
self._frame_queue.put(frame_info)
event_info.retrieved_frame_ids.add(frame_info.frame_id)
else:
pending_frames = True
if not pending_frames and event_info.end_time is not None:
event_info.all_frames_were_read = True
def _frame_data_has_settled(self, frame_info):
frame_timestamp = datetime.strptime(frame_info.timestamp,
'%Y-%m-%d %H:%M:%S')
return datetime.now() >= frame_timestamp + timedelta(
seconds=self._config.frame_read_delay_seconds)
class AutoTTLCache(MutableMapping):
def __init__(self,
items=None,
*,
maxsize,
ttl,
timer=time.monotonic,
getsizeof=None):
self._cache_lock = threading.Lock()
self._cache = TTLCache(maxsize, ttl, timer=timer, getsizeof=getsizeof)
if items is not None:
self._cache.update(items)
self._monitor = CacheMonitor(self)
@property
def ttl(self):
with self._cache_lock:
return self._cache.ttl
@property
def maxsize(self):
with self._cache_lock:
return self._cache.maxsize
@property
def timer(self):
with self._cache_lock:
return self._cache.timer
def expire(self):
with self._cache_lock:
self._cache.expire()
def __contains__(self, key):
with self._cache_lock:
return key in self._cache
def __setitem__(self, k, v):
with self._cache_lock:
self._cache[k] = v
def __delitem__(self, k):
with self._cache_lock:
del self._cache[k]
def __getitem__(self, k):
with self._cache_lock:
return self._cache[k]
def __len__(self) -> int:
with self._cache_lock:
return len(self._cache)
def __iter__(self):
with self._cache_lock:
keys = list(self._cache)
yield from keys
# TODO: __reduce__ and __setstate__
def __repr__(self):
return f"{type(self).__name__}(max_size={self.maxsize}, ttl={self.ttl})"
def clear(self):
with self._cache_lock:
self._cache.clear()
def get(self, *args, **kwargs):
with self._cache_lock:
self._cache.get(*args, **kwargs)
def pop(self, *args, **kwargs):
with self._cache_lock:
self._cache.pop(*args, **kwargs)
def setdefault(self, *args, **kwargs):
with self._cache_lock:
self._cache.setdefault(*args, **kwargs)
def popitem(self):
with self._cache_lock:
self._cache.popitem()
