class Telemetric(Document):
sensor = UUIDField(required=True, binary=False)
value = DecimalField(required=True)
date = DateTimeField(default=datetime.datetime.utcnow)
meta = {
'indexes': [
# Expires data in 1 day
{
'fields': ['date'],
'expireAfterSeconds': 86400
}
]
}
class Comparison(Model):
item_a = ReferenceField(Item)
item_b = ReferenceField(Item)
topic = ReferenceField(Topic)
address = StringField(max_length=50)
winning_item = ReferenceField(Item)
key = UUIDField(default=gen_comparison_key)
def serialize(self):
return {
'item_a': self.item_a.serialize(),
'item_b': self.item_b.serialize(),
'key': self.key
}
class UserName(CriptsBaseAttributes, CriptsSourceDocument,
CriptsActionsDocument, Document):
"""
UserName class.
"""
meta = {
"collection": settings.COL_USERNAMES,
"cripts_type": 'UserName',
"latest_schema_version": 1,
"schema_doc": {
'name':
'The actual username',
'username_id':
'An ID corresponding to the username since using the raw username as the key can run into little bobby tables issues',
'description':
'Description of the e-mail address',
'datasets': ('List [] of datasets this username'
' appeared in'),
'source': ('List [] of sources who provided information about this'
' username'),
},
"jtable_opts": {
'details_url':
'cripts.usernames.views.username_detail',
'details_url_key':
'username_id',
'default_sort':
"name",
'searchurl':
'cripts.usernames.views.usernames_listing',
'fields': ["name", "created", "source", "id", "username_id"],
'jtopts_fields':
["name", "created", "source", "favorite", "id", "username_id"],
'hidden_fields': ["username_id", "id"],
'linked_fields': [
"source",
],
'details_link':
'name',
'no_sort': []
}
}
name = StringField(required=True)
description = StringField(required=True)
username_id = UUIDField(binary=True, required=True, default=uuid.uuid4)
datasets = ListField(required=False)
class Signature(CritsBaseAttributes, CritsSourceDocument, CritsActionsDocument,
Document):
"""
Signature class.
"""
meta = {
"collection": settings.COL_SIGNATURES,
"auto_create_index": False,
"crits_type": 'Signature',
"latest_schema_version": 1,
"schema_doc": {},
"jtable_opts": {
'details_url':
'crits-signatures-views-signature_detail',
'details_url_key':
'id',
'default_sort':
"modified DESC",
'searchurl':
'crits-signatures-views-signatures_listing',
'fields': [
"title", "data_type", "data_type_min_version",
"data_type_max_version", "data_type_dependency", "version",
"modified", "source", "campaign", "id", "status"
],
'jtopts_fields': [
"details", "title", "data_type", "data_type_min_version",
"data_type_max_version", "data_type_dependency", "version",
"modified", "source", "campaign", "status", "favorite", "id"
],
'hidden_fields': [],
'linked_fields': ["source", "campaign"],
'details_link':
'details',
'no_sort': ['details']
}
}
data_type = StringField()
data_type_min_version = StringField()
data_type_max_version = StringField()
data_type_dependency = ListField()
data = StringField()
link_id = UUIDField(binary=True, required=True, default=uuid.uuid4)
md5 = StringField()
title = StringField()
version = IntField()
class UserToken(Document):
issued_at = DateTimeField(required=True, default=datetime.datetime.utcnow)
expires_at = DateTimeField(required=True)
user = LazyReferenceField("User",
required=True,
reverse_delete_rule=CASCADE)
uuid = UUIDField(binary=False, required=True, unique="True")
meta = {
"indexes": [
"user",
"uuid",
{
"fields": ["expires_at"],
"expireAfterSeconds": 0
},
]
}
class UserToken(HDocumentBase):
token = UUIDField(required=True)
user = ReferenceField(User)
issue_date = DateTimeField(default=get_now())
expire_date = DateTimeField(required=True)
meta = {
'indexes': [{
# See mongodb and mongo engine documentation for details
# by default, mongoengine will add a `_cls` field with the index as a compund index
# but mongodb only support Single Key Index on Hashed Token so far
# set the `cls` option to False can disable this beahviour on mongoengine
"fields": ["#token"],
"cls": False
}]
}
def __init__(self, **kwargs):
super(UserToken, self).__init__(**kwargs)
class Task(Document):
schema_version = StringField(max_length=10, default='1')
test = ReferenceField(Test)
test_suite = StringField(max_length=100) # embedded document from Test
testcases = ListField(StringField())
schedule_date = DateTimeField(default=datetime.datetime.utcnow)
run_date = DateTimeField()
status = StringField(max_length=50, default='waiting')
comment = StringField(max_length=1000)
kickedoff = IntField(min_value=0, default=0)
endpoint_list = ListField(UUIDField(binary=False))
parallelization = BooleanField(default=False)
endpoint_run = ReferenceField('Endpoint')
priority = IntField(min_value=QUEUE_PRIORITY_MIN, max_value=QUEUE_PRIORITY_MAX, default=QUEUE_PRIORITY_DEFAULT)
variables = DictField()
tester = ReferenceField(User)
upload_dir = StringField(max_length=100)
test_results = ListField(ReferenceField('TestResult'))
organization = ReferenceField(Organization) # embedded document from Test
team = ReferenceField(Team) # embedded document from Test
meta = {'collection': 'tasks'}
class OrderProducts(EmbeddedDocument):
id = UUIDField(primary_key=True, default=uuid4)
name = StringField()
description = StringField()
specifications = DynamicField()
images = ListField(EmbeddedDocumentField(ProductImage))
code = StringField()
quantity = IntField(default=1)
warranty = IntField() # Warranty in months
category = StringField(choices=ProductCategory)
warranty_expiry = DateTimeField()
warranty_claims = IntField(default=0)
condition = StringField()
price = FloatField()
discount_percentage = IntField(default=0)
def warranty_date_setter(self):
today = date.today()
month = today.month - 1 + self.warranty
year = int(today.year + self.warranty / 12)
month = month % 12 + 1
day = min(today.day, calendar.monthrange(year, month)[1])
return date(year, month, day)
class Note(Document):
id = UUIDField(primary_key=True)
owner = ReferenceField(User, required=True)
html = StringField(default='')
meta = {
'indexes': [
('owner', 'id'),
]
}
def clean(self):
# Generate the id on save, rather than on construction.
# This makes it easier to distinguish unsaved Notes.
if self.id is None:
self.id = uuid.uuid4()
def text(self):
# TODO use a library / do this correctly!
return re.sub('<[^>]*>', '', self.html)
def short_text(self, length=40):
return self.text()[:length]
class Order(Document):
meta = {
'indexes': [{
'fields': ['$order_id', "$payment_id"],
'default_language': 'english'
}]
}
id = UUIDField(primary_key=True, default=uuid4)
order_id = StringField(unique=True)
user = ReferenceField(User)
products = ListField(EmbeddedDocumentField(OrderProducts))
tax = FloatField()
discount = FloatField(default=0.0)
coupon_applied = StringField(default=None)
total = FloatField()
order_placed = DateTimeField(default=datetime.utcnow())
delivery_address = EmbeddedDocumentField(Address)
billing_address = EmbeddedDocumentField(Address)
order_canceled = DateTimeField()
status = StringField(chocies=OrderStates, default='Processing')
status_log = DictField()
payment_mode = StringField(chocies=PaymentModes)
payment_request_id = StringField()
payment_id = StringField()
class AnalysisResult(CritsDocument, CritsSchemaDocument,
CritsDocumentFormatter, Document):
"""
Analysis Result from running an analytic service.
"""
meta = {
"crits_type": "AnalysisResult",
"collection": settings.COL_ANALYSIS_RESULTS,
"latest_schema_version": 1,
"schema_doc": {
'analyst': 'Analyst who ran the service.',
'analysis_id': 'Unique ID for this service execution.',
'analysis_type': 'Type of analysis this is.',
'config': 'Configuration options used for this execution.',
'distributed': 'Distributed for this execution.',
'finish_date': 'Date execution finished.',
'log': 'Log entries for this execution.',
'object_type': 'Type of TLO this is for.',
'object_id': 'ObjectId of the TLO.',
'results': 'Analysis results.',
'service_name': 'Name of the service.',
'source': 'Source of the service.',
'start_date': 'Date execution started.',
'status': 'Status of the execution.',
'template': 'Custom template to render results.',
'version': 'Version of the service used.',
},
"jtable_opts": {
'details_url':
'crits.services.views.analysis_result',
'details_url_key':
'id',
'default_sort':
"start_date DESC",
'searchurl':
'crits.services.views.analysis_results_listing',
'fields': [
"object_type", "service_name", "version", "start_date",
"finish_date", "results", "object_id", "id"
],
'jtopts_fields': [
"details", "object_type", "service_name", "version",
"start_date", "finish_date", "results", "id"
],
'hidden_fields': ["object_id", "id"],
'linked_fields': ["object_type", "service_name"],
'details_link':
'details',
'no_sort': ['details']
}
}
#TODO: these should be datetime objects, not strings
analyst = StringField()
analysis_id = UUIDField(binary=False)
analysis_type = StringField(db_field="type")
config = EmbeddedDocumentField(AnalysisConfig)
distributed = BooleanField()
finish_date = StringField()
log = ListField(EmbeddedDocumentField(EmbeddedAnalysisResultLog))
object_type = StringField(required=True)
object_id = StringField(required=True)
results = ListField(DynamicField(DictField))
service_name = StringField()
source = StringField()
start_date = StringField()
status = StringField()
template = StringField()
version = StringField()
class Session(Document):
user = ReferenceField(User, required=True)
sessionId = UUIDField(binary=False, required=True)
dateCreated = DateTimeField(default=datetime.utcnow)
dateExpires = DateTimeField(default=datetime.utcnow() +
timedelta(days=30)) #1 month expiration
class Bed(EmbeddedDocument):
uuid = UUIDField(required=True, default=lambda: str(
uuid.uuid4()), binary=False)
plant = StringField()
sensors = ListField(EmbeddedDocumentField('Sensor'))
class EnterpriseModel(Document):
id = UUIDField(default=uuid.uuid4(), primary_key=True, editable=False)
name = StringField(editable=True, max_length=100, required=True)
projects = ListField(UUIDField(editable=False))
created_at = DateTimeField(default=datetime.datetime.now())
class UserIdMixin:
"""
Document mixin that defines a `userId` field to hold a user's UUID
"""
userId = UUIDField(required=True, custom_json=CustomJsonRules.HIDDEN)
class Order2(Document):
user = UUIDField(required=True)
items = ListField(EmbeddedDocumentField(OrderItem2))
class Event(CritsBaseAttributes, CritsSourceDocument, Document):
"""
Event class.
"""
meta = {
"collection": settings.COL_EVENTS,
"crits_type": 'Event',
"latest_schema_version": 2,
"schema_doc": {
'title': 'Title of this event',
'event_id': 'Unique event ID',
'event_type': 'Type of event based on Event Type options',
'description': 'Description of the event',
'source': ('List [] of sources who provided information about this'
' event')
},
"jtable_opts": {
'details_url': 'crits.events.views.view_event',
'details_url_key': 'id',
'default_sort': "created DESC",
'searchurl': 'crits.emails.views.emails_listing',
'fields': [ "title", "event_type", "created",
"source", "campaign", "status", "id"],
'jtopts_fields': [ "details",
"title",
"event_type",
"created",
"source",
"campaign",
"status",
"favorite",
"id"],
'hidden_fields': [],
'linked_fields': ["source", "campaign", "event_type"],
'details_link': 'details',
'no_sort': ['details']
}
}
title = StringField(required=True)
event_type = StringField(required=True)
description = StringField(required=True)
event_id = UUIDField(binary=True, required=True, default=uuid.uuid4())
def set_event_type(self, event_type):
"""
Set the Event Type.
:param event_type: The event type to set (must exist in DB).
:type event_type: str
"""
e = EventType.objects(name=event_type).first()
if e:
self.event_type = event_type
def to_stix(self, username=None):
"""
Converts a CRITs event to a STIX document.
The resulting document includes all related emails, samples, and
indicators converted to CybOX Observable objects.
Returns the STIX document and releasability constraints.
(NOTE: the following statement is untrue until the
releasability checking is finished, which includes setting
releasability on all CRITs objects.)
Raises UnreleasableEventError if the releasability on the
relationships and the event do not share any common releasability
sources.
"""
from crits.emails.email import Email
from crits.samples.sample import Sample
from crits.indicators.indicator import Indicator
from cybox.common import Time, ToolInformationList, ToolInformation
from cybox.core import Observables
from stix.common import StructuredText
from stix.core import STIXPackage, STIXHeader
from stix.common import InformationSource
from stix.common.identity import Identity
stix_indicators = []
stix_observables = []
final_objects = []
# create a list of sources to send as part of the results.
# list should be limited to the sources this user is allowed to use.
# this list should be used along with the list of objects to set the
# appropriate source's 'released' key to True for each object.
final_sources = []
user_source_list = user_sources(username)
for f in self.releasability:
if f.name in user_source_list:
final_sources.append(f.name)
final_sources = set(final_sources)
# TODO: eventually we can use class_from_id instead of the if block
# but only once we support all CRITs types.
for r in self.relationships:
obj = None
if r.rel_type == Email._meta['crits_type']:
obj = Email.objects(id=r.object_id,
source__name__in=user_source_list).first()
if obj:
ind, releas = obj.to_cybox()
stix_observables.append(ind[0])
elif r.rel_type == Sample._meta['crits_type']:
obj = Sample.objects(id=r.object_id,
source__name__in=user_source_list).first()
if obj:
ind, releas = obj.to_cybox()
for i in ind:
stix_observables.append(i)
elif r.rel_type == Indicator._meta['crits_type']:
#NOTE: Currently this will raise an exception if there
# are multiple indicators with the same value.
# Should be fixed automatically once we transition
# indicators to be related based on ObjectId rather
# than value.
obj = Indicator.objects(id=r.object_id,
source__name__in=user_source_list).first()
if obj:
ind, releas = obj.to_stix_indicator()
stix_indicators.append(ind)
else:
continue
#Create a releasability list that is the intersection of
# each related item's releasability with the event's
# releasability. If the resulting set is empty, raise exception
#TODO: Set releasability on all objects so that we actually
# get results here instead of always raising an exception.
if obj:
releas_sources = set([rel.name for rel in releas])
final_sources = final_sources.intersection(releas_sources)
#TODO: uncomment the following lines when objects have
# releasability set.
#if not final_sources:
# raise UnreleasableEventError(r.value)
# add to the final_objects list to send as part of the results
final_objects.append(obj)
tool_list = ToolInformationList()
tool = ToolInformation("CRITs", "MITRE")
tool.version = settings.CRITS_VERSION
tool_list.append(tool)
i_s = InformationSource(
time=Time(produced_time= datetime.datetime.now()),
identity = Identity(name=settings.COMPANY_NAME),
tools = tool_list
)
description = StructuredText(value=self.description)
header = STIXHeader(information_source=i_s,
description=description,
package_intent=self.event_type,
title=self.title)
return (STIXPackage(indicators=stix_indicators,
observables=Observables(stix_observables),
stix_header=header,
id_=self.event_id),
final_sources,
final_objects)
@classmethod
def from_stix(cls, stix_package, source):
"""
Converts a stix_package to a CRITs Event.
:param stix_package: A stix package.
:type stix_package: :class:`stix.core.STIXPackage`
:param source: The source list for this STIX package.
:type source: list
:returns: None, :class:`crits.events.event.Event'
"""
from stix.common import StructuredText
from stix.core import STIXPackage, STIXHeader
if isinstance(stix_package, STIXPackage):
stix_header = stix_package.stix_header
stix_id = stix_package.id_
event = cls(source=source)
event.title = "STIX Document %s" % stix_id
event.event_type = "Collective Threat Intelligence"
event.description = str(datetime.datetime.now())
try:
event.event_id = uuid.UUID(stix_id)
except:
if event.source[0].instances[0].reference:
event.source[0].instances[0].reference += ", STIX ID: %s" % stix_id
else:
event.source[0].instances[0].reference = "STIX ID: %s" % stix_id
if isinstance(stix_header, STIXHeader):
if stix_header.title:
event.title = stix_header.title
if stix_header.package_intent:
event.event_type = str(stix_header.package_intent)
description = stix_header.description
if isinstance(description, StructuredText):
try:
event.description = description.to_dict()
except:
pass
return event
else:
return None
def migrate(self):
"""
Migrate to the latest schema version.
"""
migrate_event(self)
class Event(CritsBaseAttributes, CritsSourceDocument, CritsActionsDocument,
Document):
"""
Event class.
"""
meta = {
"collection": settings.COL_EVENTS,
"crits_type": 'Event',
"latest_schema_version": 3,
"schema_doc": {
'title':
'Title of this event',
'event_id':
'Unique event ID',
'event_type':
'Type of event based on Event Type options',
'description':
'Description of the event',
'source': ('List [] of sources who provided information about this'
' event')
},
"jtable_opts": {
'details_url':
'crits.events.views.view_event',
'details_url_key':
'id',
'default_sort':
"created DESC",
'searchurl':
'crits.events.views.events_listing',
'fields': [
"title", "event_type", "created", "source", "campaign",
"status", "id"
],
'jtopts_fields': [
"details", "title", "event_type", "created", "source",
"campaign", "status", "favorite", "id"
],
'hidden_fields': [],
'linked_fields': ["source", "campaign", "event_type"],
'details_link':
'details',
'no_sort': ['details']
}
}
title = StringField(required=True)
event_type = StringField(required=True)
# description also exists in CritsBaseAttributes, but this one is required.
description = StringField(required=True)
event_id = UUIDField(binary=True, required=True, default=uuid.uuid4)
def set_event_type(self, event_type):
"""
Set the Event Type.
:param event_type: The event type to set (must exist in DB).
:type event_type: str
"""
if event_type in EventTypes.values():
self.event_type = event_type
def migrate(self):
"""
Migrate to the latest schema version.
"""
migrate_event(self)
class RawData(CritsBaseAttributes, CritsSourceDocument, Document):
"""
Raw Data class.
"""
meta = {
"collection": settings.COL_RAW_DATA,
"crits_type": 'RawData',
"latest_schema_version": 2,
"schema_doc": {},
"jtable_opts": {
'details_url':
'crits.raw_data.views.raw_data_details',
'details_url_key':
'id',
'default_sort':
"modified DESC",
'searchurl':
'crits.raw_data.views.raw_data_listing',
'fields': [
"title", "data_type", "version", "modified", "source",
"campaign", "id", "status"
],
'jtopts_fields': [
"details", "title", "data_type", "version", "modified",
"source", "campaign", "status", "favorite", "id"
],
'hidden_fields': [],
'linked_fields': ["source", "campaign"],
'details_link':
'details',
'no_sort': ['details']
}
}
data_type = StringField()
data = StringField()
highlights = ListField(EmbeddedDocumentField(EmbeddedHighlight))
inlines = ListField(EmbeddedDocumentField(EmbeddedInline))
link_id = UUIDField(binary=True, required=True, default=uuid.uuid4)
md5 = StringField()
title = StringField()
tool = EmbeddedDocumentField(EmbeddedTool)
version = IntField()
def migrate(self):
"""
Migrate to the latest schema version.
"""
migrate_raw_data(self)
def add_file_data(self, file_data):
"""
Add filedata to RawData.
:param file_data: The raw data to add.
:type file_data: str
"""
self._generate_file_metadata(file_data)
self.data = file_data
def add_file_obj(self, file_obj):
"""
Add filedata to RawData.
:param file_data: The raw data to add.
:type file_data: file handle
"""
data = file_obj.read()
self._generate_file_metadata(data)
self.data = data
def _generate_file_metadata(self, data):
"""
Generate metadata from the raw data. Uses the data to generate an MD5.
:param data: The data to generate metadata from.
"""
from hashlib import md5
if not self.md5:
self.md5 = md5(data).hexdigest()
def add_tool(self, name=None, version=None, details=None):
"""
Add a tool to RawData.
:param name: The name of the tool used to generate/acquire the raw data.
:type name: str
:param version: The version number of the tool.
:type version: str
:param details: Details about the tool.
:type details: str
"""
if name:
et = EmbeddedTool(name=name, version=version, details=details)
self.tool = et
def add_inline_comment(self, comment, line_num, analyst):
"""
Add an inline comment for RawData.
:param comment: The comment to add.
:type comment: str
:param line_num: The line number this comment is associated with.
:type line_Num: int
:param analyst: The user making the comment.
:type analyst: str
"""
if comment:
ei = EmbeddedInline(line=line_num,
comment=comment,
analyst=analyst)
c = 1
for i in self.inlines:
if i.line == int(line_num) and i.counter >= c:
c = i.counter + 1
ei.counter = c
self.inlines.append(ei)
def add_highlight(self, line_num, line_data, analyst):
"""
Highlight a specific line of the RawData.
:param line_num: The line number being highlighted.
:type line_num: str
:param line_data: The data on that line.
:type line_data: str
:param analyst: The user highlighting the line.
:type analyst: str
"""
eh = EmbeddedHighlight(line=line_num,
line_data=line_data,
analyst=analyst)
# determine line date
try:
pd = parse(line_data, fuzzy=True)
if pd:
eh.line_date = pd
except:
eh.line_date = datetime.datetime.now()
self.highlights.append(eh)
def remove_highlight(self, line_num, analyst):
"""
Remove highlight from a specific line of the RawData.
:param line_num: The line number being unhighlighted.
:type line_num: str
:param analyst: The user unhighlighting the line.
:type analyst: str
"""
highlights = []
for h in self.highlights:
if h.line == int(line_num) and h.analyst == analyst:
continue
else:
highlights.append(h)
self.highlights = highlights
class User(Document):
meta = {'strict': 'False'}
id = UUIDField(primary_key=True, default=uuid4)
name = StringField()
email = EmailField(unique=True)
password_hash = StringField()
confirmed = BooleanField(default=False)
contact_number = StringField()
conf_otp = IntField(default=000000)
contact_verified = BooleanField(default=False)
shipping_address = ListField(EmbeddedDocumentField(Address))
billing_address = EmbeddedDocumentField(Address)
referrals_list = ListField()
signup_timestamp = DateTimeField(default=datetime.utcnow())
login_log = DictField()
cart = ListField(DictField())
coupons = ListField(StringField())
def generate_auth_token(self, expiration=3600):
s = Serializer(current_app.config['SECRET_KEY'], expires_in=expiration)
return s.dumps({'email': self.email}).decode('utf-8')
def generate_confirm_token(self):
s = URLSerializer(current_app.config['SECRET_KEY'])
return s.dumps({'id': str(self.id)})
@property
def password(self):
raise AttributeError('password is not a readable attribute')
@password.setter
def password(self, password):
self.password_hash = generate_password_hash(password)
def verify_password(self, password):
return check_password_hash(self.password_hash, password)
def confirm(self, token):
s = URLSerializer(current_app.config['SECRET_KEY'])
try:
data = s.loads(token, max_age=3600)
except:
return False
if data['id'] == str(self.id):
self.confirmed = True
self.save()
return True
return False
@staticmethod
def verify_auth_token(token):
s = Serializer(current_app.config['SECRET_KEY'])
try:
data = s.loads(token)
except SignatureExpired:
return None
except BadSignature:
return None
except TypeError:
return None
user = User.objects.get(email=data['email'])
return user
class Address(EmbeddedDocument):
id = UUIDField(primary_key=True, default=uuid4)
address = StringField()
zip_code = StringField()
city = StringField()
class Coupon(Document):
id = UUIDField(primary_key=True, default=uuid4)
coupon_code = StringField(unique=True)
discount = IntField(default=0) # 0 to 100
discount_amount = IntField(default=0)
minimum_order_price = FloatField()
class RawData(CritsBaseAttributes, CritsSourceDocument, Document):
"""
Raw Data class.
"""
meta = {
"collection": settings.COL_RAW_DATA,
"crits_type": 'RawData',
"latest_schema_version": 1,
"schema_doc": {},
"jtable_opts": {
'details_url':
'crits.raw_data.views.raw_data_details',
'details_url_key':
'id',
'default_sort':
"modified DESC",
'searchurl':
'crits.raw_data.views.raw_data_listing',
'fields': [
"title", "data_type", "version", "modified", "source",
"campaign", "id", "status"
],
'jtopts_fields': [
"details", "title", "data_type", "version", "modified",
"source", "campaign", "status", "favorite", "id"
],
'hidden_fields': [],
'linked_fields': ["source", "campaign"],
'details_link':
'details',
'no_sort': ['details']
}
}
data_type = StringField()
description = StringField()
data = StringField()
highlights = ListField(EmbeddedDocumentField(EmbeddedHighlight))
inlines = ListField(EmbeddedDocumentField(EmbeddedInline))
link_id = UUIDField(binary=True, required=True, default=uuid.uuid4)
md5 = StringField()
title = StringField()
tool = EmbeddedDocumentField(EmbeddedTool)
version = IntField()
def migrate(self):
"""
Migrate to the latest schema version.
"""
pass
def add_file_data(self, file_data):
"""
Add filedata to RawData.
:param file_data: The raw data to add.
:type file_data: str
"""
self._generate_file_metadata(file_data)
self.data = file_data
def add_file_obj(self, file_obj):
"""
Add filedata to RawData.
:param file_data: The raw data to add.
:type file_data: file handle
"""
data = file_obj.read()
self._generate_file_metadata(data)
self.data = data
def _generate_file_metadata(self, data):
"""
Generate metadata from the raw data. Uses the data to generate an MD5.
:param data: The data to generate metadata from.
"""
from hashlib import md5
if not self.md5:
self.md5 = md5(data).hexdigest()
def add_tool(self, name=None, version=None, details=None):
"""
Add a tool to RawData.
:param name: The name of the tool used to generate/acquire the raw data.
:type name: str
:param version: The version number of the tool.
:type version: str
:param details: Details about the tool.
:type details: str
"""
if name:
et = EmbeddedTool(name=name, version=version, details=details)
self.tool = et
def add_inline_comment(self, comment, line_num, analyst):
"""
Add an inline comment for RawData.
:param comment: The comment to add.
:type comment: str
:param line_num: The line number this comment is associated with.
:type line_Num: int
:param analyst: The user making the comment.
:type analyst: str
"""
if comment:
ei = EmbeddedInline(line=line_num,
comment=comment,
analyst=analyst)
c = 1
for i in self.inlines:
if i.line == int(line_num) and i.counter >= c:
c = i.counter + 1
ei.counter = c
self.inlines.append(ei)
def add_highlight(self, line_num, line_data, analyst):
"""
Highlight a specific line of the RawData.
:param line_num: The line number being highlighted.
:type line_num: str
:param line_data: The data on that line.
:type line_data: str
:param analyst: The user highlighting the line.
:type analyst: str
"""
eh = EmbeddedHighlight(line=line_num,
line_data=line_data,
analyst=analyst)
# determine line date
try:
pd = parse(line_data, fuzzy=True)
if pd:
eh.line_date = pd
except:
eh.line_date = datetime.datetime.now()
self.highlights.append(eh)
def remove_highlight(self, line_num, analyst):
"""
Remove highlight from a specific line of the RawData.
:param line_num: The line number being unhighlighted.
:type line_num: str
:param analyst: The user unhighlighting the line.
:type analyst: str
"""
highlights = []
for h in self.highlights:
if h.line == int(line_num) and h.analyst == analyst:
continue
else:
highlights.append(h)
self.highlights = highlights
def to_cybox_observable(self):
"""
Convert a RawData to a CybOX Observables.
Returns a tuple of (CybOX object, releasability list).
To get the cybox object as xml or json, call to_xml() or
to_json(), respectively, on the resulting CybOX object.
"""
obj = Artifact(self.data, Artifact.TYPE_FILE)
obj.packaging.append(Base64Encoding())
obs = Observable(obj)
obs.description = self.description
return ([obs], self.releasability)
@classmethod
def from_cybox(cls, cybox_obs):
"""
Convert a Cybox DefinedObject to a MongoEngine RawData object.
:param cybox_obs: The cybox object to create the RawData from.
:type cybox_obs: :class:`cybox.core.Observable``
:returns: :class:`crits.raw_data.raw_data.RawData`
"""
cybox_object = cybox_obs.object_.properties
rawdata = cls()
rawdata.add_file_data(cybox_object.data)
db_obj = RawData.objects(md5=rawdata.md5).first()
if db_obj:
return db_obj
return rawdata
class DynDoc(DynamicDocument):
"""
Extensible document for diamonds
"""
document_id = UUIDField(unique=True)
record = DynamicField(required=True)
class Event(CritsBaseAttributes, CritsSourceDocument, Document):
"""
Event class.
"""
meta = {
"collection": settings.COL_EVENTS,
"crits_type": 'Event',
"latest_schema_version": 2,
"schema_doc": {
'title': 'Title of this event',
'event_id': 'Unique event ID',
'event_type': 'Type of event based on Event Type options',
'description': 'Description of the event',
'source': ('List [] of sources who provided information about this'
' event')
},
"jtable_opts": {
'details_url': 'crits.events.views.view_event',
'details_url_key': 'id',
'default_sort': "created DESC",
'searchurl': 'crits.events.views.events_listing',
'fields': [ "title", "event_type", "created",
"source", "campaign", "status", "id"],
'jtopts_fields': [ "details",
"title",
"event_type",
"created",
"source",
"campaign",
"status",
"favorite",
"id"],
'hidden_fields': [],
'linked_fields': ["source", "campaign", "event_type"],
'details_link': 'details',
'no_sort': ['details']
}
}
title = StringField(required=True)
event_type = StringField(required=True)
description = StringField(required=True)
event_id = UUIDField(binary=True, required=True, default=uuid.uuid4())
def set_event_type(self, event_type):
"""
Set the Event Type.
:param event_type: The event type to set (must exist in DB).
:type event_type: str
"""
e = EventType.objects(name=event_type).first()
if e:
self.event_type = event_type
def stix_description(self):
return self.description
def stix_intent(self):
return self.event_type
def stix_title(self):
return self.title
@classmethod
def from_stix(cls, stix_package, source):
"""
Converts a stix_package to a CRITs Event.
:param stix_package: A stix package.
:type stix_package: :class:`stix.core.STIXPackage`
:param source: The source list for this STIX package.
:type source: list
:returns: None, :class:`crits.events.event.Event'
"""
from stix.common import StructuredText
from stix.core import STIXPackage, STIXHeader
if isinstance(stix_package, STIXPackage):
stix_header = stix_package.stix_header
stix_id = stix_package.id_
event = cls(source=source)
event.title = "STIX Document %s" % stix_id
event.event_type = "Collective Threat Intelligence"
event.description = str(datetime.datetime.now())
eid = stix_package.id_
try:
uuid.UUID(eid)
except ValueError:
# The STIX package ID attribute is not a valid UUID
# so make one up.
eid = uuid.uuid4() # XXX: Log this somewhere?
event.event_id = eid
if isinstance(stix_header, STIXHeader):
if stix_header.title:
event.title = stix_header.title
#if stix_header.package_intents:
# package_intents are optional in the STIX Spec.. So we check for the attribute being present
# rather than the original check which causes an exception
if hasattr(stix_header,'package_intents'):
event.event_type = str(stix_header.package_intents[0])
description = stix_header.description
if isinstance(description, StructuredText):
try:
event.description = description.to_dict()
except:
pass
return event
else:
return None
def migrate(self):
"""
Migrate to the latest schema version.
"""
migrate_event(self)
class TeamWork(EmbeddedDocument):
id = UUIDField(required=True)
description = StringField()
type = IntField(required=True) # see TEAM_SHOW_TYPE
uri = StringField()
create_time = DateTimeField(default=get_now())
class TaskStatus(AutoRetryDocument, ReaperMixin):
"""
Represents a task.
Defines the schema for the documents in task_status collection. The documents in this
collection may be reaped, so it inherits from ReaperMixin.
:ivar task_id: identity of the task this status corresponds to
:type task_id: basestring
:ivar worker_name: The name of the worker that the Task is in
:type worker_name: basestring
:ivar tags: custom tags on the task
:type tags: list
:ivar state: state of callable in its lifecycle
:type state: basestring
:ivar error: Any errors or collections of errors that occurred while this task was running
:type error: dict (created from a PulpException)
:ivar spawned_tasks: List of tasks that were spawned during the running of this task
:type spawned_tasks: list of str
:ivar progress_report: A report containing information about task's progress
:type progress_report: dict
:ivar task_type: the fully qualified (package/method) type of the task
:type task_type: basestring
:ivar start_time: ISO8601 representation of the time the task started executing
:type start_time: basestring
:ivar finish_time: ISO8601 representation of the time the task completed
:type finish_time: basestring
:ivar group_id: The id used to identify which group of tasks a task belongs to
:type group_id: uuid.UUID
:ivar result: return value of the callable, if any
:type result: any
:ivar exception: Deprecated. This is always None.
:type exception: None
:ivar traceback: Deprecated. This is always None.
:type traceback: None
"""
task_id = StringField(required=True)
worker_name = StringField()
tags = ListField(StringField())
state = StringField(choices=constants.CALL_STATES, default=constants.CALL_WAITING_STATE)
error = DictField(default=None)
spawned_tasks = ListField(StringField())
progress_report = DictField()
task_type = StringField()
start_time = ISO8601StringField()
finish_time = ISO8601StringField()
result = DynamicField()
group_id = UUIDField(default=None)
# These are deprecated, and will always be None
exception = StringField()
traceback = StringField()
# For backward compatibility
_ns = StringField(default='task_status')
meta = {'collection': 'task_status',
'indexes': ['-tags', '-state', {'fields': ['-task_id'], 'unique': True}, '-group_id'],
'allow_inheritance': False,
'queryset_class': CriteriaQuerySet}
def save_with_set_on_insert(self, fields_to_set_on_insert):
"""
Save the current state of the TaskStatus to the database, using an upsert operation.
The upsert operation will only set those fields if this becomes an insert operation,
otherwise those fields will be ignored. This also validates the fields according to the
schema above.
This is required because the current mongoengine version we are using does not support
upsert with set_on_insert through mongoengine queries. Once we update to the version
which supports this, this method can be deleted and it's usages can be replaced
with mongoengine upsert queries.
:param fields_to_set_on_insert: A list of field names that should be updated with Mongo's
$setOnInsert operator.
:type fields_to_set_on_insert: list
"""
# If fields_to_set_on_insert is None or empty, just save
if not fields_to_set_on_insert:
self.save()
return
# This will be used in place of superclass' save method, so we need to call validate()
# explicitly.
self.validate()
stuff_to_update = dict(copy.deepcopy(self._data))
# Let's pop the $setOnInsert attributes out of the copy of self so that we can pass the
# remaining attributes to the $set operator in the query below.
set_on_insert = {}
for field in fields_to_set_on_insert:
set_on_insert[field] = stuff_to_update.pop(field)
task_id = stuff_to_update.pop('task_id')
update = {'$set': stuff_to_update,
'$setOnInsert': set_on_insert}
TaskStatus._get_collection().update({'task_id': task_id}, update, upsert=True)
@classmethod
def post_save(cls, sender, document, **kwargs):
"""
Send a taskstatus message on save.
:param sender: class of sender (unused)
:type sender: class
:param document: mongoengine document
:type document: mongoengine.Document
"""
send_taskstatus_message(document, routing_key="tasks.%s" % document['task_id'])
class BaseDocument(Document):
meta = {"abstract": True}
id = UUIDField(default=uuid4, primary_key=True)
created_at = DateTimeField(default=datetime.utcnow)
class Sensor(Document):
uuid = UUIDField(required=True, default=lambda: str(
uuid.uuid4()), binary=False)
telemetric = StringField(required=True, choices=sensorCatalog)
username = StringField(required=True)
class UuidMixin:
"""
Document mixin that defines a primary-key `id` field containing an auto-generated UUID
"""
id = UUIDField(default=uuid4, primary_key=True, custom_json=("id", str))
