class LoggerCountShipper(Shipper): PARSER = parsers.Options({ 'level': parsers.String(validators=[is_valid_level]), }) def __init__(self, level: int = logging.INFO, **kwargs): super().__init__(**kwargs) self.level: int = level def configure(self, config: Config) -> None: config = super().configure(config) if 'level' in config: self.level = logging.getLevelName(config['level'].value) def ship(self, investigation, anomalies: DataFrame) -> None: logger.log(self.level, "Received %d events", len(anomalies))
class LoggerShipper(Shipper): """Monitor that logs alerts to the standard logger.""" PARSER = parsers.Options({ 'level': parsers.String(validators=[is_valid_level]), }) def __init__(self, level: int = logging.INFO, **kwargs): super().__init__(**kwargs) self.level: int = level def configure(self, config: Config) -> None: config = super().configure(config) if 'level' in config: self.level = logging.getLevelName(config['level'].value) def ship(self, investigation, anomalies: DataFrame) -> None: for row in anomalies.iterrows(): logger.log(self.level, "\n%s", row)
class CSVShipper(Shipper): """Saves the malicious anomalies to a CSV. Appends results instead of writing over the file. Parameters ---------- filename: str Path to the CSV file. """ PARSER = parsers.Options({ 'filename': parsers.String(), }) """Monitor that writes alerts to a CSV file.""" def __init__(self, filename: str = 'alerts.csv', **kwargs): super().__init__(**kwargs) self.filename: str = filename def configure(self, config: Config) -> Config: config = super().configure(config) self.filename = config.get_value('filename', self.filename) return config def ship(self, investigation, anomalies: pa.DataFrame) -> None: logger.info( "Writing %d anomalies to '%s'", len(anomalies), self.filename, ) write_header = True try: write_header = os.path.getsize(self.filename) == 0 except FileNotFoundError: pass anomalies.to_csv( self.filename, mode='a+', header=write_header, index=False, )
from logging.config import dictConfig from scrywarden.config import parsers from scrywarden.config.base import Config LOGGING_PARSER = parsers.Options( { 'version': parsers.Integer(default=1), 'formatters': parsers.Dict( parsers.Options({ 'format': parsers.String(), 'datefmt': parsers.String(), })), 'handlers': parsers.Dict( parsers.Options({ 'class': parsers.String(required=True), 'level': parsers.String(), 'formatter': parsers.String(), 'filters': parsers.List(parsers.String()), })), 'loggers': parsers.Dict( parsers.Options({ 'level': parsers.String(), 'propagate': parsers.Boolean(), 'filters': parsers.List(parsers.String()), 'handlers': parsers.List(parsers.String()), })),
sa.Integer, sa.ForeignKey(Investigation.id, ondelete='CASCADE'), nullable=False, ), sa.Column( 'event_id', sa.BigInteger, sa.ForeignKey(Event.id, ondelete='CASCADE'), nullable=False, ), sa.UniqueConstraint('investigation_id', 'event_id'), ) """Many to many relation for investigations and events.""" PARSER = parsers.Options({ 'host': parsers.String(default='localhost'), 'port': parsers.Integer(default=5432), 'name': parsers.String(default='scrywarden'), 'user': parsers.String(default='scrywarden'), 'password': parsers.String(default='scrywarden'), }) def parse_engine(config: Config) -> Engine: """Parses an SQLAlchemy engine from a config object. Parameters ---------- config: Config Configuration object to parse engine from.
class CSVTransport(EphemeralTransport): """Transport that reads messages from a CSV file. By default it yields each row as a message with the message data as a dictionary containing each row value. The dictionary has each header as the key and each value as the string value from the CSV. Override the `transform` method to transform this dictionary value before setting it as the message value. Setting a `process_check` integer value will log a report message every number of rows to keep the user updated on it's progress. Parameters ---------- file: str Path the CSV is located at. headers: Iterable[str] Header names to use for the CSV if none are given in the file. process_check: int Log a processing check message after this values set value. """ PARSER = parsers.Options({ 'file': parsers.String(), 'headers': parsers.List(parsers.String()), 'process_check': parsers.Integer(), }) def __init__( self, file: str = '', headers: t.Iterable[str] = (), process_check: int = 0, **kwargs, ): super().__init__(**kwargs) self.file: str = file self.headers = headers or None self.process_check: int = process_check self._create_messages: t.Callable[ [csv.DictReader], t.Iterable[Message], ] = self._create_messages_without_log def process(self) -> t.Iterable[Message]: """Reads all rows as messages. Returns ------- Iterable[Messages] Iterable of each row as messages. """ if self.process_check: self._create_messages = self._create_messages_with_log with open(self.file) as file: reader = csv.DictReader(file, fieldnames=self.headers or None) yield from self._create_messages(reader) def configure(self, config: Config) -> Config: self.file = config.get_value('file', self.file) self.headers = config.get_value('headers', self.headers) self.process_check = config.get_value( 'process_check', self.process_check, ) return config def transform(self, row: t.Dict) -> t.Dict: """Overridable method that transforms the row value. This modifies the row data before setting it as the message data. This is useful for things like parsing numbers from the string values. Parameters ---------- row: Dict Row represented as a dictionary value. Returns ------- Dict Modified row dictionary. """ return row def _create_messages_without_log( self, reader: csv.DictReader, ) -> t.Iterable[Message]: for row in reader: yield self._create_message(row) def _create_message(self, row: t.Dict) -> Message: return Message.create(self.transform(row)) def _create_messages_with_log( self, reader: csv.DictReader, ) -> t.Iterable[Message]: for index, row in enumerate(reader, 1): if index % self.process_check == 0: logger.info("%d rows read from '%s'", index, self.file) yield self._create_message(row)