def test_atomic(self): cache = TTLCache(maxsize=1, ttl=1, timer=Timer(auto=True)) cache[1] = 1 self.assertEqual(1, cache[1]) cache[1] = 1 self.assertEqual(1, cache.get(1)) cache[1] = 1 self.assertEqual(1, cache.pop(1)) cache[1] = 1 self.assertEqual(1, cache.setdefault(1))
class CustomThreatService(BaseController): """ Implements a custom threat service for Resilient. The root path (/xxx/ below) is configurable. The service provides the following URLs: OPTIONS /<root_path>/<any_sub_path> POST /<root_path>/<any_sub_path> GET /<root_path>/<any_sub_path>/<id> """ # Arbitrary constant namespace = UUID('18222d9c-adf0-409c-aa19-beb27130ba12') def __init__(self, opts): super(CustomThreatService, self).__init__(**_make_args(opts)) # Configurable options self.options = opts.get(CONFIG_SECTION, {}) # Do we support "file-content" artifacts? Default is no. # TODO add implementation support to parse the file content self.support_upload_file = strtobool( self.options.get(CONFIG_UPLOAD_FILE.key, CONFIG_UPLOAD_FILE.default)) # Default time that this service will tell Resilient to retry self.first_retry_secs = int( self.options.get(CONFIG_FIRST_RETRY_SECS.key, CONFIG_FIRST_RETRY_SECS.default)) or 5 self.later_retry_secs = int( self.options.get(CONFIG_LATER_RETRY_SECS.key, CONFIG_LATER_RETRY_SECS.default)) or 60 # Size of the request cache self.cache_size = int( self.options.get(CONFIG_CACHE_SIZE.key, CONFIG_CACHE_SIZE.default)) # TTL of the request cache (millis before we give up on a request lookup) self.cache_ttl = int( self.options.get(CONFIG_CACHE_TTL.key, CONFIG_CACHE_TTL.default)) # Limit to the number of queries we'll answer for unfinished searchers (count before giving up on them) self.max_retries = int( self.options.get(CONFIG_MAX_RETRIES.key, CONFIG_MAX_RETRIES.default)) # IDs and their results are maintained in a cache so that we can set # an upper bound on the number of in-progress and recent lookups. self.cache = TTLCache(maxsize=self.cache_size, ttl=self.cache_ttl) # Helper component does event dispatch work self.async_helper = CustomThreatServiceHelper(self) (self.helper_thread, self.bridge) = self.async_helper.start() urls = ["{0}/{1}".format(self.channel, e) for e in self.events()] LOG.info("Web handler for %s", ", ".join(urls)) self.auth_user = self.options.get(CONFIG_AUTH_USER.key, CONFIG_AUTH_USER.default) self.auth_password = self.options.get(CONFIG_AUTH_PASSWORD.key, CONFIG_AUTH_USER.default) # Web endpoints @exposeWeb("OPTIONS") def _options_request(self, event, *args, **kwargs): """ Options indicate to Resilient whether file upload is supported. """ LOG.info(event.args[0]) options = {"upload_file": bool(self.support_upload_file)} return options @exposeWeb("POST") def _post_request(self, event, *args, **kwargs): LOG.info(event.args[0]) if not self.check_authentication(event.args[0]): LOG.error("Custom Threat Service Authentication Error") response = event.args[1] response.status = 500 return {"id": None, "hits": []} result = self._handle_post_request(event, *args, **kwargs) LOG.info("%s: %s", event.args[1].status, json.dumps(result)) return result def _handle_post_request(self, event, *args, **kwargs): """ Responds to POST /cts/<anything> The URL below /cts/ is specific to this threat service. For example, /cts/one and /cts/two can be registered as two separate threat sources. The string 'one' or 'two' becomes the channel that searcher events are dispatched on. Request is a ThreatServiceArtifactDTO containing the artifact to be scanned Response is a ResponseDTO containing the response, or 'please retry' (HTTP status 303). """ request = event.args[0] response = event.args[1] # The channels that searchers are listening for events cts_channel = searcher_channel(*args) value = request.body.getvalue() if not value: err = "Empty request" LOG.warn(err) return {"id": str(uuid4()), "hits": []} # Resilient sends artifacts in two formats: multi-part MIME, or plain JSON. # server may send either, even for cases where there is no file content, # so check content-type and decode appropriately. try: if request.headers and "form-data" in request.headers.get( "Content-Type", ""): multipart_data = decoder.MultipartDecoder( value, request.headers["Content-Type"]) body = json.loads(multipart_data.parts[0].text) LOG.debug(body) else: body = json.loads(value.decode("utf-8")) LOG.debug(body) except (ValueError, NonMultipartContentTypeException) as e: err = "Can't handle request: {}".format(e) LOG.warn(err) LOG.debug(value) return {"id": str(uuid4()), "hits": []} if not isinstance(body, dict): # Valid JSON but not a valid request. err = "Invalid request: {}".format(json.dumps(body)) LOG.warn(err) return {"id": str(uuid4()), "hits": []} # Generate a request ID, derived from the artifact being requested. request_id = str(uuid5(self.namespace, json.dumps(body))) artifact_type = body.get("type", "unknown") artifact_value = body.get("value") response_object = {"id": request_id, "hits": []} cache_key = (cts_channel, request_id) if artifact_type == "net.name" and artifact_value == "localhost": # Hard-coded response to 'net.name' of 'localhost' # because this is used in 'resutil threatservicetest' # and we want to return an immediate (not async) response return response_object # If we already have a completed query for this key, return it immmediately request_data = self.cache.get(cache_key) if request_data and request_data.get("complete"): response_object["hits"] = request_data.get("hits", []) return response_object response.status = 303 response_object["retry_secs"] = self.first_retry_secs # Add the request to the cache, then notify searchers that there's a new request self.cache.setdefault(cache_key, { "id": request_id, "artifact": body, "hits": [], "complete": False }) evt = ThreatServiceLookupEvent(request_id=request_id, name=artifact_type, artifact=body, channel=cts_channel) self.async_helper.fire(evt, HELPER_CHANNEL) return response_object @exposeWeb("GET") def _get_request(self, event, *args, **kwargs): LOG.info(event.args[0]) if not self.check_authentication(event.args[0]): LOG.error("Custom Threat Service Authentication Error") response = event.args[1] response.status = 500 return {"id": None, "hits": []} result = self._handle_get_request(event, *args, **kwargs) LOG.info("%s: %s", event.args[1].status, json.dumps(result)) return result def _handle_get_request(self, event, *args, **kwargs): """ Responds to GET /cts/<anything>/<request-id> The URL below /cts/ is specific to this threat service. For example, /cts/one and /cts/two are considered two separate threat sources. Response is a ResponseDTO containing the response, or 'please retry' """ LOG.info(event.args[0]) response = event.args[1] request_id = None if not args: return {"id": request_id, "hits": []} # The ID of the lookup request request_id = args[-1] # The channel that searchers are listening for events cts_channel = searcher_channel(*args[:-1]) response_object = {"id": request_id, "hits": []} cache_key = (cts_channel, request_id) request_data = self.cache.get(cache_key) if not request_data: # There's no record of this request in our cache, return empty hits response.status = 200 return response_object response_object["hits"] = request_data["hits"] if not request_data["complete"]: # The searchers haven't finished yet, return partial hits if available response.status = 303 response_object["retry_secs"] = self.later_retry_secs # Update the counter, so we can detect "stale" failures request_data["count"] = request_data.get("count", 0) + 1 if request_data["count"] > self.max_retries: LOG.info("Exceeded max retries for {}".format(cache_key)) try: self.cache.pop(cache_key) except KeyError: pass response.status = 200 return response_object return response_object # Remove the result from cache # self.cache.pop(cache_key) return response_object @handler(channel=LOOKUP_COMPLETE_CHANNEL) def _lookup_complete(self, event, *args, **kwargs): """ A lookup event was completed """ if not isinstance(event.parent, ThreatServiceLookupEvent): return results = event.parent.value.getValue() artifact = event.parent.artifact cts_channel = event.parent.cts_channel request_id = event.parent.request_id LOG.info("Lookup complete: %s, %s", event.parent, results) # Depending on how many components handled this lookup event, # the results can be a single value (dict), or an array, or None, # or an exception, or a tuple (type, exception, traceback) hits = [] complete = True if isinstance(results, list): for result in results: if result: if isinstance(result, (tuple, ThreatLookupIncompleteException)): LOG.info("Retry later!") complete = False elif isinstance(result, (tuple, Exception)): LOG.error("No hits due to exception") else: hits.append(result) elif results: if isinstance(results, (tuple, ThreatLookupIncompleteException)): LOG.info("Retry later!") complete = False elif isinstance(results, (tuple, Exception)): LOG.error("No hits due to exception") else: hits.append(results) # Store the result and mark as complete (or not) cache_key = (cts_channel, request_id) self.cache[cache_key] = { "id": request_id, "artifact": artifact, "hits": hits, "complete": complete } def _get_authentication_headers(self, request): """[extract user/password info in http header: Authentication Basic into a list]""" if request.headers and "Basic" in request.headers.get( "Authorization", ""): auth = request.headers.get("Authorization", "").split(' ') user_password = base64.b64decode(auth[1]) return b_to_s(user_password).split(":") return [None, None] def _is_authenticated(self, user_password_list): """[check if a user/password pair matches values set in app.config]""" return self.auth_user == user_password_list[ 0] and self.auth_password == user_password_list[1] def check_authentication(self, request): """[check if the headers contain user/password information and they match the settings in app.config]""" return self._is_authenticated( self._get_authentication_headers(request))
class FrameReaderWorker(Thread): log = logging.getLogger("events_processor.FrameReaderWorker") @inject def __init__(self, config: ConfigProvider, frame_queue: FrameQueue, system_time: SystemTime, frame_reader: FrameReader): super().__init__() self._config = config self._stop_requested = False self._system_time = system_time self._frame_queue = frame_queue event_ttl = config.events_window_seconds + config.cache_seconds_buffer self._recent_events = TTLCache(maxsize=10000000, ttl=event_ttl) self._frame_reader = frame_reader if config.event_ids: self._events_iter = lambda: self._frame_reader.events_by_id_iter( config.event_ids) self._skip_mailed = False else: self._events_iter = self._frame_reader.events_iter self._skip_mailed = True def run(self) -> None: while not self._stop_requested: before = time.monotonic() self._collect_events() time_spent = (time.monotonic() - before) self._system_time.sleep( max(self._config.event_loop_seconds - time_spent, 0)) self.log.info("Terminating") def stop(self) -> None: self._stop_requested = True def _collect_events(self) -> None: self.log.info("Fetching event list") for event_json in self._events_iter(): event_id = event_json['Id'] event_info = self._recent_events.setdefault(event_id, EventInfo()) event_info.event_json = event_json if event_info.all_frames_were_read or event_info.notification_status.was_sending: continue if event_info.emailed and self._skip_mailed: self.log.debug( f'Skipping processing of event {event_info} as it was already mailed' ) continue self._collect_frames(event_info) def _collect_frames(self, event_info: EventInfo): self.log.info(f"Reading event frames: {event_info}") frames = self._frame_reader.frames(event_info) if frames: pending_frames = False for frame_info in frames: if frame_info.type != 'Alarm': continue if frame_info.frame_id in event_info.retrieved_frame_ids: continue if self._frame_data_has_settled(frame_info): self._frame_queue.put(frame_info) event_info.retrieved_frame_ids.add(frame_info.frame_id) else: pending_frames = True if not pending_frames and event_info.end_time is not None: event_info.all_frames_were_read = True def _frame_data_has_settled(self, frame_info): frame_timestamp = datetime.strptime(frame_info.timestamp, '%Y-%m-%d %H:%M:%S') return datetime.now() >= frame_timestamp + timedelta( seconds=self._config.frame_read_delay_seconds)
class AutoTTLCache(MutableMapping): def __init__(self, items=None, *, maxsize, ttl, timer=time.monotonic, getsizeof=None): self._cache_lock = threading.Lock() self._cache = TTLCache(maxsize, ttl, timer=timer, getsizeof=getsizeof) if items is not None: self._cache.update(items) self._monitor = CacheMonitor(self) @property def ttl(self): with self._cache_lock: return self._cache.ttl @property def maxsize(self): with self._cache_lock: return self._cache.maxsize @property def timer(self): with self._cache_lock: return self._cache.timer def expire(self): with self._cache_lock: self._cache.expire() def __contains__(self, key): with self._cache_lock: return key in self._cache def __setitem__(self, k, v): with self._cache_lock: self._cache[k] = v def __delitem__(self, k): with self._cache_lock: del self._cache[k] def __getitem__(self, k): with self._cache_lock: return self._cache[k] def __len__(self) -> int: with self._cache_lock: return len(self._cache) def __iter__(self): with self._cache_lock: keys = list(self._cache) yield from keys # TODO: __reduce__ and __setstate__ def __repr__(self): return f"{type(self).__name__}(max_size={self.maxsize}, ttl={self.ttl})" def clear(self): with self._cache_lock: self._cache.clear() def get(self, *args, **kwargs): with self._cache_lock: self._cache.get(*args, **kwargs) def pop(self, *args, **kwargs): with self._cache_lock: self._cache.pop(*args, **kwargs) def setdefault(self, *args, **kwargs): with self._cache_lock: self._cache.setdefault(*args, **kwargs) def popitem(self): with self._cache_lock: self._cache.popitem()