Beispiel #1
0
class DistributedQueryResult(SurrogatePK, Model):
    columns = Column(JSONB)
    timestamp = Column(db.DateTime, default=dt.datetime.utcnow)

    distributed_query_task_id = reference_col('distributed_query_task', nullable=False)
    distributed_query_task = relationship(
        'DistributedQueryTask',
        backref=db.backref('results',
                           cascade='all, delete-orphan',
                           lazy='joined'),
    )

    distributed_query_id = reference_col('distributed_query', nullable=False)
    distributed_query = relationship(
        'DistributedQuery',
        backref=db.backref('results',
                           cascade='all, delete-orphan',
                           lazy='joined'),
    )

    def __init__(self, columns, distributed_query=None, distributed_query_task=None):
        self.columns = columns
        self.distributed_query = distributed_query
        self.distributed_query_task = distributed_query_task

    def to_dict(self):
        dict = {c.name: getattr(self, c.name) for c in self.__table__.columns}
        # dict['tasks'] = self.tasks
        return dict

    def to_dict_obj(self):
        dict = {'columns': self.columns}
        # dict['tasks'] = self.tasks
        return dict
Beispiel #2
0
class AlertEmail(SurrogatePK, Model):
    alert_id = reference_col('alerts', nullable=False)
    alert = relationship(
        'Alerts',
        backref=db.backref('alert_email', lazy='dynamic'),
    )
    node = Column(db.String, nullable=False)
    status = Column(db.String, nullable=True)
    node_id = reference_col('node', nullable=False)

    node = relationship(
        'Node',
        backref=db.backref('alert_email', lazy='dynamic'),
    )
    body = Column(db.String, nullable=False)

    created_at = Column(db.DateTime, nullable=False, default=dt.datetime.utcnow)
    updated_at = Column(db.DateTime, nullable=False)

    def __init__(self, node=None, node_id=None, alert=None, alert_id=None, body=None,
                 status=None, updated_at=dt.datetime.utcnow()):

        if node:
            self.node = node
        elif node_id:
            self.node_id = node_id
        if alert:
            self.alert = alert
        elif alert_id:
            self.alert_id = alert_id
        self.updated_at = updated_at
        self.body = body
        self.status = status
Beispiel #3
0
class Query(SurrogatePK, Model):
    name = Column(db.String, nullable=False)
    sql = Column(db.String, nullable=False)
    interval = Column(db.Integer, default=3600)
    platform = Column(db.String)
    version = Column(db.String)
    description = Column(db.String)
    value = Column(db.String)
    removed = Column(db.Boolean, nullable=False, default=True)
    snapshot = Column(db.Boolean, nullable=False, default=False)
    shard = Column(db.Integer)

    packs = relationship(
        'Pack',
        secondary=querypacks,
        back_populates='queries',
    )

    tags = relationship(
        'Tag',
        secondary=query_tags,
        back_populates='queries',
        lazy='joined',
    )

    def __init__(self, name, query=None, sql=None, interval=3600, platform=None,packs=[],
                 version=None, description=None, value=None, removed=True,
                 shard=None, snapshot=False, **kwargs):
        self.name = name
        self.sql = query or sql
        self.interval = int(interval)
        self.platform = platform
        self.version = version
        self.description = description
        self.value = value
        self.removed = removed
        self.packs=packs
        self.snapshot = snapshot
        self.shard = shard

    def __repr__(self):
        return '<Query: {0.name}>'.format(self)

    def to_dict(self):
        return {
            'id': self.id,
            'query': self.sql,
            'interval': self.interval,
            'platform': self.platform,
            'version': self.version,
            'description': self.description,
            'value': self.value,
            'removed': self.removed,
            'shard': self.shard,
            'snapshot': self.snapshot,
            'tags': [r.to_dict() for r in self.tags]
        }
Beispiel #4
0
class Tag(SurrogatePK, Model):
    value = Column(db.String, nullable=False, unique=True)

    nodes = relationship(
        'Node',
        secondary=node_tags,
        back_populates='tags',
    )

    packs = relationship(
        'Pack',
        secondary=pack_tags,
        back_populates='tags',
    )

    queries = relationship(
        'Query',
        secondary=query_tags,
        back_populates='tags',
    )

    file_paths = relationship(
        'FilePath',
        secondary=file_path_tags,
        back_populates='tags',
    )

    def __init__(self, value, **kwargs):
        self.value = value

    def __repr__(self):
        return '<Tag: {0.value}>'.format(self)

    @property
    def packs_count(self):
        return db.session.object_session(self) \
            .query(Pack.id).with_parent(self, 'packs').count()

    @property
    def nodes_count(self):
        return db.session.object_session(self) \
            .query(Node.id).with_parent(self, 'nodes').count()

    @property
    def queries_count(self):
        return db.session.object_session(self) \
            .query(Query.id).with_parent(self, 'queries').count()

    @property
    def file_paths_count(self):
        return db.session.object_session(self) \
            .query(FilePath.id).with_parent(self, 'file_paths').count()

    def to_dict(self):
        return self.value
Beispiel #5
0
class FilePath(SurrogatePK, Model):
    category = Column(db.String, nullable=False, unique=True)
    target_paths = Column(db.String)

    tags = relationship(
        'Tag',
        secondary=file_path_tags,
        back_populates='file_paths',
        lazy='joined',
    )

    def __init__(self, category=None, target_paths=None, *args, **kwargs):
        self.category = category

        if target_paths is not None:
            self.set_paths(*target_paths)
        elif args:
            self.set_paths(*args)
        else:
            self.target_paths = ''

    def to_dict(self):
        return {self.category: self.get_paths()}

    def get_paths(self):
        return self.target_paths.split('!!')

    def set_paths(self, *target_paths):
        self.target_paths = '!!'.join(target_paths)
Beispiel #6
0
class DistributedQuery(SurrogatePK, Model):
    description = Column(db.String, nullable=True)
    sql = Column(db.String, nullable=False)
    timestamp = Column(db.DateTime, default=dt.datetime.utcnow)
    not_before = Column(db.DateTime, default=dt.datetime.utcnow)
    alert_id = reference_col('alerts', nullable=True)
    alert = relationship(
        'Alerts',
        backref=db.backref('distributed_query'),
    )

    # tasks = relationship(
    #     'DistributedQueryTask',
    #     backref=db.backref('distributed_query_task'),
    #
    # )

    def __init___(self, sql, description=None, not_before=None, alert_id=None):
        self.sql = sql
        self.alert_id = alert_id
        self.description = description
        self.not_before = not_before

    def to_dict(self):
        dict = {c.name: getattr(self, c.name) for c in self.__table__.columns}
        # dict['tasks'] = self.tasks
        return dict
Beispiel #7
0
class CarveSession(SurrogatePK, Model):
    # StatusQueried for queried carves that did not hit nodes yet
    StatusQueried = "QUERIED"
    # StatusInitialized for initialized carves
    StatusInitialized = "INITIALIZED"
    # StatusInProgress for carves that are on-going
    StatusInProgress = "IN PROGRESS"
    #  StatusCompleted for carves that finalized
    StatusCompleted = "COMPLETED"

    node_id = reference_col('node', nullable=False)
    session_id = Column(db.String, nullable=False)
    carve_guid = Column(db.String, nullable=False)

    carve_size = Column(db.Integer)
    block_size = Column(db.Integer)
    block_count = Column(db.Integer)
    completed_blocks = Column(db.Integer, default=0)
    archive = Column(db.String())

    request_id = Column(db.String, nullable=False)
    status = Column(db.String, nullable=False)

    created_at = Column(db.DateTime,
                        nullable=False,
                        default=dt.datetime.utcnow)
    updated_at = Column(db.DateTime, nullable=False)
    node = relationship('Node',
                        backref=db.backref('carve_session', lazy='dynamic'))

    def __init___(self,
                  node_id,
                  session_id=None,
                  carve_guid=None,
                  carve_size=0,
                  block_size=0,
                  block_count=0,
                  archive=None,
                  request_id=None):
        self.node_id = node_id
        self.session_id = session_id
        self.carve_guid = carve_guid
        self.carve_size = carve_size
        self.block_size = block_size
        self.block_count = block_count
        self.archive = archive
        self.request_id = request_id

    def to_dict(self):
        """Return object data in easily serializeable format"""
        return {
            'id': self.id,
            'node_id': self.node_id,
            'session_id': self.session_id,
            'carve_guid': self.carve_guid,
            'carve_size': self.carve_size,
            'block_count': self.block_count,
            'archive': self.archive,
            'created_at': dump_datetime(self.created_at),
        }
Beispiel #8
0
class StatusLog(SurrogatePK, Model):
    line = Column(db.Integer)
    message = Column(db.String)
    severity = Column(db.Integer)
    filename = Column(db.String)
    created = Column(db.DateTime, default=dt.datetime.utcnow)
    version = Column(db.String)

    node_id = reference_col('node', nullable=False)
    node = relationship(
        'Node',
        backref=db.backref('status_logs', lazy='dynamic')
    )

    def __init__(self, line=None, message=None, severity=None,
                 filename=None, created=None, node=None, node_id=None,
                 version=None, **kwargs):
        self.line = int(line)
        self.message = message
        self.severity = int(severity)
        self.filename = filename
        self.created = created
        self.version = version
        if node:
            self.node = node
        elif node_id:
            self.node_id = node_id

    @declared_attr
    def __table_args__(cls):
        return (
            Index('idx_%s_node_id_created_desc' % cls.__tablename__,
                  'node_id', cls.created.desc()),
        )
Beispiel #9
0
class AlertEmail(SurrogatePK, Model):
    alert_id = db.Column(db.Integer,
                         db.ForeignKey('alerts.id', ondelete='CASCADE'))
    alert = relationship(
        'Alerts',
        backref=db.backref('alert_email', lazy='dynamic',
                           passive_deletes=True),
    )

    status = Column(db.String, nullable=True)

    node_id = db.Column(db.Integer, db.ForeignKey('node.id',
                                                  ondelete='CASCADE'))
    node = relationship('Node',
                        backref=db.backref('alert_email',
                                           passive_deletes=True,
                                           lazy='dynamic'))

    body = Column(db.String, nullable=False)

    created_at = Column(db.DateTime,
                        nullable=False,
                        default=dt.datetime.utcnow)
    updated_at = Column(db.DateTime, nullable=False)

    def __init__(self,
                 node=None,
                 node_id=None,
                 alert=None,
                 alert_id=None,
                 body=None,
                 status=None,
                 updated_at=dt.datetime.utcnow()):

        if node:
            self.node = node
        elif node_id:
            self.node_id = node_id
        if alert:
            self.alert = alert
        elif alert_id:
            self.alert_id = alert_id
        self.updated_at = updated_at
        self.body = body
        self.status = status
Beispiel #10
0
class ResultLog(SurrogatePK, Model):
    NEW = 0
    PENDING = 1
    COMPLETE = 2
    name = Column(db.String, nullable=False)
    timestamp = Column(db.DateTime, default=dt.datetime.utcnow)
    action = Column(db.String)
    columns = Column(JSONB)
    node_id = reference_col('node', nullable=False)
    node = relationship('Node',
                        backref=db.backref('result_logs', lazy='dynamic'))
    uuid = Column(db.String, nullable=True)
    status = Column(db.Integer, default=NEW, nullable=False)
    task_id = Column(db.String, nullable=True)

    def __init__(self,
                 name=None,
                 action=None,
                 columns=None,
                 timestamp=None,
                 node=None,
                 node_id=None,
                 uuid=None,
                 **kwargs):
        self.name = name
        self.action = action
        self.columns = columns or {}
        self.timestamp = timestamp
        self.uuid = uuid
        if node:
            self.node = node
        elif node_id:
            self.node_id = node_id

    def to_json(self):
        return json.dumps(self,
                          default=lambda o: o.__dict__,
                          sort_keys=True,
                          indent=4)

    def as_dict(self):
        return {c.name: getattr(self, c.name) for c in self.__table__.columns}

    def to_dict(self):
        dictionary = {}
        for c in self.__table__.columns:
            if not c.name == "timestamp":
                dictionary[c.name] = getattr(self, c.name)
            else:
                dictionary[c.name] = getattr(
                    self, c.name).strftime('%m/%d/%Y %H/%M/%S')
        return dictionary

    @declared_attr
    def __table_args__(cls):
        return (Index('idx_%s_node_id_timestamp_desc' % cls.__tablename__,
                      'node_id', cls.timestamp.desc()), )
Beispiel #11
0
class DefaultFilters(SurrogatePK, Model):
    ARCH_x86 = "x86"
    ARCH_x64 = "x86_64"

    filters = Column(JSONB)
    platform = Column(db.String, nullable=False)
    arch = Column(db.String)
    apply_by_default = Column(db.Boolean, nullable=False, default=False)
    created_at = Column(db.DateTime, nullable=False)
    updated_at = Column(db.DateTime,
                        nullable=False,
                        default=dt.datetime.utcnow)
    config_id = reference_col('config', nullable=False)
    config = relationship('Config',
                          backref=db.backref('default_filters',
                                             lazy='dynamic'))

    def __init__(self,
                 filters,
                 platform,
                 created_at,
                 apply_by_default=False,
                 config_id=None,
                 arch=ARCH_x64,
                 **kwargs):
        self.filters = filters
        self.platform = platform
        self.apply_by_default = apply_by_default
        self.created_at = created_at
        self.updated_at = dt.datetime.utcnow()
        self.arch = arch
        self.config_id = config_id

    @property
    def serialize(self):
        """Return object data in easily serializeable format"""
        return {
            'id': self.id,
            'filters': json.loads(self.filters),
            'platform': self.platform,
            'arch': self.arch,
            'type': self.type,
            'created_at': dump_datetime(self.created_at),
            'updated_at': dump_datetime(self.updated_at)
        }
Beispiel #12
0
class NodeData(SurrogatePK, Model):
    # config = Column(JSONB)
    data = Column(JSONB, default={}, nullable=False)
    name = Column(db.String, nullable=False)
    node_id = reference_col('node', nullable=False)
    node = relationship(
        'Node',
        backref=db.backref('node_data',
                           cascade='all, delete-orphan',
                           lazy='dynamic'),
    )

    created_at = Column(db.DateTime,
                        nullable=False,
                        default=dt.datetime.utcnow)
    updated_at = Column(db.DateTime, nullable=False)

    def to_dict(self):
        return {
            'name': self.name,
            'data': json.dumps(self.data),
            'updated_at': dump_datetime(self.updated_at)
        }

    def to_dict_obj(self):
        return {
            'name': self.name,
            'data': self.data,
            'updated_at': dump_datetime(self.updated_at)
        }

    def __init__(self,
                 node=None,
                 node_id=None,
                 data=None,
                 name=False,
                 updated_at=None):

        if node:
            self.node = node
        elif node_id:
            self.node_id = node_id
        self.data = data
        self.name = name
        self.updated_at = updated_at
Beispiel #13
0
class NodeReconData(SurrogatePK, Model):
    columns = Column(JSONB, default={}, nullable=False)
    node_data_id = reference_col('node_data', nullable=False)
    node_data = relationship(
        'NodeData',
        backref=db.backref('node_recon_data',
                           cascade='all, delete-orphan',
                           lazy='dynamic'),
    )

    created_at = Column(db.DateTime,
                        nullable=False,
                        default=dt.datetime.utcnow)
    updated_at = Column(db.DateTime, nullable=False)

    def to_dict(self):
        return {
            'columns': json.dumps(self.columns),
            'updated_at': dump_datetime(self.updated_at)
        }

    def to_dict_obj(self):
        return {
            'columns': self.columns,
            'updated_at': dump_datetime(self.updated_at)
        }

    def __init__(self,
                 node_data=None,
                 node_data_id=None,
                 columns=None,
                 updated_at=None):

        if node_data:
            self.node_data = node_data
        elif node_data_id:
            self.node_data_id = node_data_id
        self.columns = columns
        self.updated_at = updated_at
Beispiel #14
0
class DistributedQueryTask(SurrogatePK, Model):
    NEW = 0
    PENDING = 1
    COMPLETE = 2
    FAILED = 3
    NOT_SENT = 4
    HIGH = 0

    LOW = 1

    save_results_in_db = Column(db.Boolean, nullable=False, default=False)
    guid = Column(db.String, nullable=False, unique=True)
    status = Column(db.Integer, default=0, nullable=False)
    timestamp = Column(db.DateTime)
    data = Column(JSONB)
    updated_at = Column(db.DateTime, nullable=True, default=None)
    viewed_at = Column(db.DateTime, nullable=True, default=None)
    priority = Column(db.Integer, default=0, nullable=False)
    sql = Column(db.String, nullable=True)

    distributed_query_id = reference_col('distributed_query', nullable=False)
    distributed_query = relationship(
        'DistributedQuery',
        backref=db.backref('tasks',
                           cascade='all, delete-orphan',
                           lazy='dynamic'),
    )

    node_id = db.Column(db.Integer, db.ForeignKey('node.id',
                                                  ondelete='CASCADE'))
    node = relationship('Node',
                        backref=db.backref('distributed_queries',
                                           passive_deletes=True,
                                           lazy='dynamic'))

    def __init__(self,
                 node=None,
                 node_id=None,
                 distributed_query=None,
                 save_results_in_db=False,
                 distributed_query_id=None,
                 updated_at=None,
                 priority=0,
                 viewed_at=None,
                 data=None):
        self.guid = str(uuid.uuid4())
        self.updated_at = updated_at
        self.viewed_at = viewed_at
        self.save_results_in_db = save_results_in_db
        self.data = data
        self.priority = priority
        if node:
            self.node = node
        elif node_id:
            self.node_id = node_id
        if distributed_query:
            self.distributed_query = distributed_query
        elif distributed_query_id:
            self.distributed_query_id = distributed_query_id

    @declared_attr
    def __table_args__(cls):
        return (Index('idx_%s_node_id_status' % cls.__tablename__, 'node_id',
                      'status'), )

    def to_dict_obj(self):
        return {
            'id': self.id,
            'distributed_query': {
                'description': self.distributed_query.description,
                'sql': self.distributed_query.sql
            },
            'results': self.data,
        }
Beispiel #15
0
class Alerts(SurrogatePK, Model):

    RULE = "rule"
    THREAT_INTEL = "Threat Intel"

    SOURCE_IOC = "IOC"

    RESOLVED = "RESOLVED"
    OPEN = "OPEN"

    query_name = Column(db.String, nullable=False)
    message = Column(JSONB)

    node_id = reference_col('node', nullable=False)
    rule_id = reference_col('rule', nullable=False)
    rule = relationship(
        'Rule',
        backref=db.backref('alerts', lazy='dynamic'),
    )
    severity = Column(db.String, nullable=True)
    type = Column(db.String, nullable=True)

    node = relationship(
        'Node',
        backref=db.backref('alerts', lazy='dynamic'),
    )

    created_at = Column(db.DateTime,
                        nullable=False,
                        default=dt.datetime.utcnow)

    severity = Column(db.String)
    recon_queries = Column(JSONB)
    result_log_uid = Column(db.String)
    type = Column(db.String)
    source = Column(db.String)
    source_data = Column(JSONB)
    status = Column(db.String, default=OPEN)

    def __init__(self, message, query_name, node_id, rule_id, recon_queries,
                 result_log_uid, type, source, source_data, severity):
        self.message = message
        self.query_name = query_name
        self.recon_queries = recon_queries
        self.node_id = node_id
        self.rule_id = rule_id

        self.type = type
        self.source = source
        self.source_data = source_data
        self.result_log_uid = result_log_uid
        self.severity = severity
        # self.node=node

    def to_dict(self):
        return {c.name: getattr(self, c.name) for c in self.__table__.columns}

    def as_dict(self):
        dictionary = {}
        for c in self.__table__.columns:
            if not c.name == "created_at":
                dictionary[c.name] = getattr(self, c.name)
            else:
                dictionary[c.name] = getattr(
                    self, c.name).strftime('%m/%d/%Y %H/%M/%S')
        return dictionary
Beispiel #16
0
class ResultLog(SurrogatePK, Model):
    NEW = 0
    PENDING = 1
    COMPLETE = 2
    name = Column(db.String, nullable=False)
    timestamp = Column(db.DateTime, default=dt.datetime.utcnow)
    action = Column(db.String)
    columns = Column(JSONB)
    node_id = reference_col('node', nullable=False)
    node = relationship('Node',
                        backref=db.backref('result_logs',
                                           cascade='all, delete-orphan',
                                           lazy='dynamic'))
    uuid = Column(db.String, nullable=True)
    status = Column(db.Integer, default=NEW, nullable=False)
    task_id = Column(db.String, nullable=True)

    def __init__(self,
                 name=None,
                 action=None,
                 columns=None,
                 timestamp=None,
                 node=None,
                 node_id=None,
                 uuid=None,
                 **kwargs):
        self.name = name
        self.action = action
        self.columns = columns or {}
        self.timestamp = timestamp
        self.uuid = uuid
        if node:
            self.node = node
        elif node_id:
            self.node_id = node_id

    def to_json(self):
        return json.dumps(self,
                          default=lambda o: o.__dict__,
                          sort_keys=True,
                          indent=4)

    def as_dict(self):
        return {c.name: getattr(self, c.name) for c in self.__table__.columns}

    def to_dict(self):
        dictionary = {}
        for c in self.__table__.columns:
            if not c.name == "timestamp":
                dictionary[c.name] = getattr(self, c.name)
            else:
                dictionary[c.name] = getattr(
                    self, c.name).strftime('%m/%d/%Y %H/%M/%S')
        return dictionary

    @declared_attr
    def __table_args__(cls):
        return (
            Index('idx_%s_node_id_timestamp_desc' % cls.__tablename__,
                  'node_id', cls.timestamp.desc()),
            Index('idx_%s_name' % cls.__tablename__, 'name'),
            Index('idx_%s_on_columns_md5' % cls.__tablename__,
                  sqlalchemy.text("(columns->'md5')"),
                  postgresql_using='btree'),
            Index('idx_%s_on_columns_domain_name' % cls.__tablename__,
                  sqlalchemy.text("(columns->'domain_name')"),
                  postgresql_using='btree'),
            Index('idx_%s_on_columns_ja3_md5' % cls.__tablename__,
                  sqlalchemy.text("(columns->'ja3_md5')"),
                  postgresql_using='btree'),
            Index('idx_%s_on_columns_sha256' % cls.__tablename__,
                  sqlalchemy.text("(columns->'sha256')"),
                  postgresql_using='btree'),
            Index('idx_%s_on_columns_process_guid' % cls.__tablename__,
                  sqlalchemy.text("(columns->'process_guid')"),
                  postgresql_using='btree'),
            Index('idx_%s_on_columns_parent_process_guid' % cls.__tablename__,
                  sqlalchemy.text("(columns->'parent_process_guid')"),
                  postgresql_using='btree'),
            Index('idx_%s_on_columns_time' % cls.__tablename__,
                  sqlalchemy.text("(columns->'time')"),
                  postgresql_using='btree'),
            Index('idx_%s_on_columns_target_path' % cls.__tablename__,
                  sqlalchemy.text("(columns->'target_path')"),
                  postgresql_using='btree'),
            Index('idx_%s_on_columns_target_name' % cls.__tablename__,
                  sqlalchemy.text("(columns->'target_name')"),
                  postgresql_using='btree'),
            Index('idx_%s_on_columns_process_name' % cls.__tablename__,
                  sqlalchemy.text("(columns->'process_name')"),
                  postgresql_using='btree'),
            Index('idx_%s_on_columns_remote_address' % cls.__tablename__,
                  sqlalchemy.text("(columns->'remote_address')"),
                  postgresql_using='btree'),
        )
Beispiel #17
0
class Alerts(SurrogatePK, Model):

    RULE = "rule"
    THREAT_INTEL = "Threat Intel"

    SOURCE_IOC = "IOC"

    RESOLVED = "RESOLVED"
    OPEN = "OPEN"

    CRITICAL = "CRITICAL"
    WARNING = "WARNING"
    LOW = "LOW"
    INFO = "INFO"

    query_name = Column(db.String, nullable=False)
    message = Column(JSONB)

    rule_id = reference_col('rule', nullable=True)
    rule = relationship(
        'Rule',
        backref=db.backref('alerts', lazy='dynamic'),
    )

    node_id = reference_col('node', nullable=False)
    node = relationship(
        'Node',
        backref=db.backref('alerts',
                           cascade='all, delete-orphan',
                           lazy='dynamic'),
    )

    severity = Column(db.String, nullable=True)
    type = Column(db.String, nullable=True)
    recon_queries = Column(JSONB)
    result_log_uid = Column(db.String)
    source = Column(db.String)
    source_data = Column(JSONB)
    status = Column(db.String, default=OPEN)

    created_at = Column(db.DateTime,
                        nullable=False,
                        default=dt.datetime.utcnow)

    def __init__(self, message, query_name, node_id, rule_id, recon_queries,
                 result_log_uid, type, source, source_data, severity):
        self.message = message
        self.query_name = query_name
        self.recon_queries = recon_queries
        self.node_id = node_id
        self.rule_id = rule_id
        self.type = type
        self.source = source
        self.source_data = source_data
        self.result_log_uid = result_log_uid
        self.severity = severity

    def to_dict(self):
        return {c.name: getattr(self, c.name) for c in self.__table__.columns}

    def as_dict(self):
        dictionary = {}
        for c in self.__table__.columns:
            if not c.name == "created_at":
                dictionary[c.name] = getattr(self, c.name)
            else:
                dictionary[c.name] = getattr(
                    self, c.name).strftime('%m/%d/%Y %H/%M/%S')
        return dictionary

    @declared_attr
    def __table_args__(cls):
        return (Index('idx_%s_created_at_desc' % cls.__tablename__,
                      'created_at', cls.created_at.desc()),
                Index('idx_%s_status' % cls.__tablename__, 'status'))
Beispiel #18
0
class Node(SurrogatePK, Model):

    ACTIVE = 0
    REMOVED = 1
    DELETED = 2

    NA = 0
    DISABLE = 1
    ENABLE = 2
    node_key = Column(db.String, nullable=False, unique=True)
    platform = Column(db.String)
    enroll_secret = Column(db.String)
    enrolled_on = Column(db.DateTime)
    host_identifier = Column(db.String)
    last_checkin = Column(db.DateTime)
    node_info = Column(JSONB, default={}, nullable=False)
    os_info = Column(JSONB, default={}, nullable=False)
    network_info = Column(JSONB, default={}, nullable=False)
    host_details = Column(JSONB, default={}, nullable=False)
    state = Column(db.Integer, default=ACTIVE, nullable=True)
    updated_at = Column(db.DateTime, nullable=True, default=dt.datetime.now())

    last_status = Column(db.DateTime)
    last_result = Column(db.DateTime)
    last_config = Column(db.DateTime)
    last_query_read = Column(db.DateTime)
    last_query_write = Column(db.DateTime)

    is_active = Column(db.Boolean, default=False, nullable=False)
    last_ip = Column(INET, nullable=True)

    tags = relationship(
        'Tag',
        secondary=node_tags,
        back_populates='nodes',
        lazy='joined',
    )

    def __init__(self,
                 host_identifier,
                 node_key=None,
                 enroll_secret=None,
                 enrolled_on=None,
                 last_checkin=None,
                 is_active=False,
                 last_ip=None,
                 last_status=None,
                 network_info=None,
                 os_info=None,
                 node_info=None,
                 last_result=None,
                 last_config=None,
                 last_query_read=None,
                 last_query_write=None,
                 state=None,
                 updated_at=None,
                 **kwargs):
        self.network_info = network_info
        self.node_info = node_info
        self.os_info = os_info
        self.node_key = node_key or str(uuid.uuid4())
        self.host_identifier = host_identifier
        self.enroll_secret = enroll_secret
        self.enrolled_on = enrolled_on
        self.last_checkin = last_checkin
        self.is_active = is_active
        self.last_ip = last_ip
        self.last_status = last_status
        self.last_result = last_result
        self.last_config = last_config
        self.last_query_read = last_query_read
        self.last_query_write = last_query_write
        self.state = state
        self.updated_at = updated_at

    def __repr__(self):
        return '<Node-{0.id}: node_key={0.node_key}, host_identifier={0.host_identifier}>'.format(
            self)

    def get_config(self, **kwargs):
        from polylogyx.utils import assemble_configuration
        return assemble_configuration(self)

    def get_new_queries(self, **kwargs):
        from polylogyx.utils import assemble_distributed_queries
        return assemble_distributed_queries(self)

    def node_is_active(self):
        checkin_interval = current_app.config['POLYLOGYX_CHECKIN_INTERVAL']
        if isinstance(checkin_interval, (int, float)):
            checkin_interval = dt.timedelta(seconds=checkin_interval)
        if (dt.datetime.utcnow() - self.last_checkin) < checkin_interval:
            return True
        return False

    @property
    def display_name(self):
        if 'display_name' in self.node_info and self.node_info['display_name']:
            return self.node_info['display_name']
        elif 'hostname' in self.node_info and self.node_info['hostname']:
            return self.node_info['hostname']
        elif 'computer_name' in self.node_info and self.node_info[
                'computer_name']:
            return self.node_info['computer_name']
        else:
            return self.host_identifier

    @property
    def packs(self):
        return db.session.object_session(self) \
            .query(Pack) \
            .join(pack_tags, pack_tags.c['pack.id'] == Pack.id) \
            .join(node_tags, node_tags.c['tag.id'] == pack_tags.c['tag.id']) \
            .filter(node_tags.c['node.id'] == self.id) \
            .options(db.lazyload('*'))

    @property
    def queries(self):
        return db.session.object_session(self) \
            .query(Query) \
            .join(query_tags, query_tags.c['query.id'] == Query.id) \
            .join(node_tags, node_tags.c['tag.id'] == query_tags.c['tag.id']) \
            .filter(node_tags.c['node.id'] == self.id) \
            .options(db.lazyload('*'))

    @hybrid_property
    def child_count(self):
        return db.session.query(db.func.count(DistributedQueryTask.id)).filter(
            DistributedQueryTask.node_id == self.id,
            DistributedQueryTask.viewed_at == None, DistributedQueryTask.status
            == DistributedQueryTask.COMPLETE).scalar()

    @property
    def file_paths(self):
        return db.session.object_session(self) \
            .query(FilePath) \
            .join(file_path_tags, file_path_tags.c['file_path.id'] == FilePath.id) \
            .join(node_tags, node_tags.c['tag.id'] == file_path_tags.c['tag.id']) \
            .filter(node_tags.c['node.id'] == self.id) \
            .options(db.lazyload('*'))

    def to_dict(self):
        # NOTE: deliberately not including any secret values in here, for now.
        if self.network_info is None:
            self.network_info = {}
        if self.os_info is None:
            self.os_info = {}
        if self.node_info is None:
            self.node_info = {}
        return {
            'id': self.id,
            'display_name': self.display_name,
            'enrolled_on': self.enrolled_on,
            'host_identifier': self.host_identifier,
            'last_checkin': self.last_checkin,
            'platform': self.platform,
            'os_info': self.os_info.copy(),
            'node_info': self.node_info.copy(),
            'network_info': self.network_info.copy(),
            'last_ip': self.last_ip,
            'is_active': self.is_active or self.node_is_active()
        }

    def as_dict(self):
        dictionary = {}
        for c in self.__table__.columns:
            if not (c.name == "enrolled_on" or c.name == "last_checkin"):
                dictionary[c.name] = getattr(self, c.name)
            else:
                if not getattr(self, c.name) == None:
                    dictionary[c.name] = getattr(
                        self, c.name).strftime('%m/%d/%Y %H/%M/%S')
                # else:
                #     dictionary[c.name] = '%0/%0/%0 %0/%0/%0'
        return dictionary
Beispiel #19
0
class Pack(SurrogatePK, Model):
    INTRUSION_DETECTION = "Intrusion Detection"
    MONITORING = "Monitoring"

    COMPLIANCE_MANAGEMENT = "Compliance and Management"
    FORENSICS_IR = "Forensics and Incident Response"
    GENERAL = "General"
    OTHERS = "Others"

    name = Column(db.String, nullable=False, unique=True)
    platform = Column(db.String)
    version = Column(db.String)
    description = Column(db.String)
    shard = Column(db.Integer)
    category = Column(db.String, default=GENERAL)

    queries = relationship(
        'Query',
        secondary=querypacks,
        back_populates='packs',
    )

    tags = relationship(
        'Tag',
        secondary=pack_tags,
        back_populates='packs',
    )

    def __init__(self,
                 name,
                 platform=None,
                 version=None,
                 description=None,
                 shard=None,
                 **kwargs):
        self.name = name
        self.platform = platform
        self.version = version
        self.description = description
        self.shard = shard

    def __repr__(self):
        return '<Pack: {0.name}>'.format(self)

    def to_dict(self):
        queries = {}
        discovery = []

        for query in self.queries:
            if 'discovery' in (t.value for t in query.tags):
                discovery.append(query.sql)
            else:
                queries[query.name] = query.to_dict()

        return {
            'id': self.id,
            'platform': self.platform,
            'version': self.version,
            'shard': self.shard,
            'discovery': discovery,
            'queries': queries,
            'name': self.name,
            'tags': [r.to_dict() for r in self.tags]
        }
Beispiel #20
0
class DefaultQuery(SurrogatePK, Model):
    ARCH_x86 = "x86"
    ARCH_x64 = "x86_64"

    name = Column(db.String, nullable=False)
    sql = Column(db.String, nullable=False)
    interval = Column(db.Integer, default=3600)
    platform = Column(db.String)
    arch = Column(db.String)
    version = Column(db.String)
    description = Column(db.String)
    value = Column(db.String)
    removed = Column(db.Boolean, nullable=False, default=True)
    snapshot = Column(db.Boolean, nullable=False, default=False)
    shard = Column(db.Integer)
    status = Column(db.Boolean, nullable=False, default=False)
    config_id = reference_col('config', nullable=True)
    config = relationship('Config',
                          backref=db.backref('default_query', lazy='dynamic'))

    def __init__(self,
                 name,
                 query=None,
                 sql=None,
                 interval=3600,
                 platform=None,
                 version=None,
                 description=None,
                 value=None,
                 removed=False,
                 config_id=None,
                 shard=None,
                 status=None,
                 snapshot=False,
                 arch=ARCH_x64,
                 **kwargs):
        self.name = name
        self.sql = query or sql
        self.interval = int(interval)
        self.platform = platform
        self.version = version
        self.description = description
        self.value = value
        self.removed = removed
        self.snapshot = snapshot
        self.shard = shard
        self.status = status
        self.arch = arch
        self.config_id = config_id

    def __repr__(self):
        return '<Query: {0.name}>'.format(self)

    def to_dict(self):
        return {
            'id': self.id,
            'query': self.sql,
            'interval': self.interval,
            'platform': self.platform,
            'version': self.version,
            'description': self.description,
            'value': self.value,
            'removed': False,
            'shard': self.shard,
            'snapshot': self.snapshot,
            'status': self.status
        }