class PCAP(CritsBaseAttributes, CritsSourceDocument, Document):
"""
PCAP class.
"""
meta = {
"collection": settings.COL_PCAPS,
"crits_type": 'PCAP',
"latest_schema_version": 2,
"shard_key": ('md5', ),
"schema_doc": {
'filename': 'The filename of the PCAP',
'md5': 'The MD5 of the PCAP file',
'length': 'The filesize of the PCAP',
'uploadDate': 'The ISODate when the PCAP was uploaded',
'description': 'Description of what the PCAP contains',
'contentType': 'The filetype of the PCAP',
'source':
'List [] of source information about who provided the PCAP'
},
"jtable_opts": {
'details_url':
'crits.pcaps.views.pcap_details',
'details_url_key':
'md5',
'default_sort':
"modified DESC",
'searchurl':
'crits.pcaps.views.pcaps_listing',
'fields': [
"filename", "description", "length", "modified", "source",
"campaign", "id", "md5", "status"
],
'jtopts_fields': [
"details", "filename", "description", "length", "modified",
"source", "campaign", "status", "md5", "favorite", "id"
],
'hidden_fields': ['md5'],
'linked_fields': ["source", "campaign"],
'details_link':
'details',
'no_sort': ['details']
}
}
contentType = StringField()
description = StringField()
filedata = getFileField(collection_name=settings.COL_PCAPS)
filename = StringField(required=True)
length = IntField(default=0)
md5 = StringField()
def migrate(self):
"""
Migrate to the latest schema version.
"""
migrate_pcap(self)
def add_file_data(self, file_data):
"""
Add filedata to this PCAP.
:param file_data: The filedata to add.
:type file_data: str
"""
self._generate_file_metadata(file_data)
self.filedata = file_data
def add_file_obj(self, file_obj):
"""
Add filedata to this PCAP.
:param file_data: The filedata to add.
:type file_data: file handle
"""
data = file_obj.read()
self._generate_file_metadata(data)
self.filedata = data
def _generate_file_metadata(self, data):
"""
Generate metadata from the file data. Will add content-type, length, and
MD5.
:param data: The data to generate metadata from.
:type data: str
"""
import magic
from hashlib import md5
self.contentType = magic.from_buffer(data)
self.length = len(data)
# this is a shard key. you can't modify it once it's set.
# MongoEngine will still mark the field as modified even if you set it
# to the same value.
if not self.md5:
self.md5 = md5(data).hexdigest()
def discover_binary(self):
"""
Queries GridFS for a matching binary to this pcap document.
"""
from crits.core.mongo_tools import mongo_connector
fm = mongo_connector("%s.files" % self._meta['collection'])
objectid = fm.find_one({'md5': self.md5}, {'_id': 1})
if objectid:
self.filedata.grid_id = objectid['_id']
self.filedata._mark_as_changed()
def to_cybox_observable(self):
"""
Convert a PCAP to a CybOX Observables.
Returns a tuple of (CybOX object, releasability list).
To get the cybox object as xml or json, call to_xml() or
to_json(), respectively, on the resulting CybOX object.
"""
obj = File()
obj.md5 = self.md5
obj.file_name = self.filename
obj.file_format = self.contentType
obj.size_in_bytes = self.length
obs = Observable(obj)
obs.description = self.description
art = Artifact(self.filedata.read(), Artifact.TYPE_NETWORK)
art.packaging.append(Base64Encoding())
obj.add_related(art, "Child_Of") # relate artifact to file
return ([obs], self.releasability)
@classmethod
def from_cybox(cls, cybox_obs):
"""
Convert a Cybox Artifact to a CRITs PCAP object.
:param cybox_obs: The cybox object to create the PCAP from.
:type cybox_obs: :class:`cybox.core.Observable`
:returns: :class:`crits.pcaps.pcap.PCAP`
"""
cybox_object = cybox_obs.object_.properties
if cybox_object.md5:
db_obj = PCAP.objects(md5=cybox_object.md5).first()
if db_obj:
return db_obj
pcap = cls()
pcap.description = str(cybox_obs.description)
pcap.md5 = cybox_object.md5
pcap.filename = str(cybox_object.file_name)
pcap.contentType = cybox_object.file_format
pcap.length = cybox_object.size_in_bytes.value if cybox_object.size_in_bytes else 0
for obj in cybox_object.parent.related_objects: # attempt to find data in cybox
if isinstance(obj.properties, Artifact
) and obj.properties.type_ == Artifact.TYPE_NETWORK:
pcap.add_file_data(obj.properties.data)
break
return pcap
class City(Document):
continent = StringField()
city_id = IntField(primary_key=True)
meta = {"abstract": True, "allow_inheritance": False}
class EBAP(Request):
full_name = 'Eliminación de la historia académica BAPI'
commite_cm = IntField(default=1, display='Acta de comité')
commite_cm_date = DateTimeField(display='Fecha acta de comité',
default=datetime.date.today)
regulation_list = ['008|2008|CSU'] # List of regulations
str_cm = [
'eliminar la historia académica BAPI, debido a que {}.',
]
str_pcm = [
'Modalidad de trabajo de grado: Asignaturas de posgrado. Acta de comité {}, del {} de {} '
+ 'del {}.'
]
def cm(self, docx):
paragraph = docx.add_paragraph()
paragraph.alignment = WD_ALIGN_PARAGRAPH.JUSTIFY
paragraph.paragraph_format.space_after = Pt(0)
paragraph.add_run(self.str_council_header + ' ')
self.cm_answer(paragraph)
paragraph.add_run(' ({}). '.format(
self.regulations[self.regulation_list[0]][0]))
def cm_answer(self, paragraph):
paragraph.add_run(
# pylint: disable=no-member
self.get_approval_status_display().upper() + ' ').font.bold = True
paragraph.add_run(self.str_cm[0].format(
'' if self.is_affirmative_response_approval_status() else 'no ') +
'.')
def pcm(self, docx):
self.pcm_analysis(docx)
paragraph = docx.add_paragraph()
paragraph.alignment = WD_ALIGN_PARAGRAPH.JUSTIFY
paragraph.paragraph_format.space_after = Pt(0)
paragraph.add_run(self.str_answer + ': ').bold = True
paragraph.add_run(self.str_comittee_header + ' ')
self.pcm_answer(paragraph)
def pcm_analysis(self, docx):
analysis_list = []
analysis_list += [
self.str_pcm[0].format(
# pylint: disable=no-member
self.commite_cm,
self.commite_cm_date.day,
num_to_month(self.commite_cm_date.month),
self.commite_cm_date.year)
]
analysis_list += self.extra_analysis
add_analysis_paragraph(docx, analysis_list)
def pcm_answer(self, paragraph):
paragraph.add_run(
# pylint: disable=no-member
self.get_advisor_response_display().upper() + ' ').font.bold = True
paragraph.add_run(self.str_cm[0].format(self.council_decision))
def resource_analysis(self, docx):
last_paragraph = docx.paragraphs[-1]
self.pcm_answer(last_paragraph)
def resource_pre_answer(self, docx):
last_paragraph = docx.paragraphs[-1]
self.pcm_answer(last_paragraph)
def resource_answer(self, docx):
last_paragraph = docx.paragraphs[-1]
self.cm_answer(last_paragraph)
class Information(Document):
name = StringField()
yob = IntField()
phone = StringField()
city = StringField()
class Person(Document):
name = StringField()
age = IntField()
meta = {"allow_inheritance": True}
class CountRecordAdapter(EmbeddedDocument):
"""
Adapter class to store Counts in MongoDB
:param valid (Bool): validity of the record.
:param counts_ch[X] (int): Counts in channel X
:param counts_trigger (int): trigger counts recieved
:param counts_time (Real): the time of the record
"""
valid = BooleanField()
"""
Set to true if the underlying record is valid
"""
counts_ch0 = IntField()
"""
Counts in channel 0
"""
counts_ch1 = IntField()
"""
Counts in channel 1
"""
counts_ch2 = IntField()
"""
Counts in channel 2
"""
counts_ch3 = IntField()
"""
Counts in channel 3
"""
counts_trigger = IntField()
"""
Trigger counts
"""
counters_time = DecimalField()
"""
Counts in the time register of the DAQ card. Basically a timestamp
"""
@staticmethod
def get(rec):
"""
Creates a CountRecordAdapter from a CountRecord
:param rec: CountRecord to convert
"""
return CountRecordAdapter(valid=rec.valid,
counts_ch0=rec.counts_ch0,
counts_ch1=rec.counts_ch1,
counts_ch2=rec.counts_ch2,
counts_ch3=rec.counts_ch3,
counts_trigger=rec.counts_trigger,
counters_time=rec.counters_time)
def createCount(self):
"""
Creates a CountRecord from the current object
:returns: CountRecord from the current CountRecordAdapter
"""
rec = CountRecord("")
rec.valid = self.valid
rec.counts_ch0 = self.counts_ch0
rec.counts_ch1 = self.counts_ch1
rec.counts_ch2 = self.counts_ch2
rec.counts_ch3 = self.counts_ch3
rec.counts_trigger = self.counts_trigger
rec.counters_time = self.counters_time
return rec
class Tag(InheritableDocument):
text = StringField(max_length=30)
count = IntField(default=0)
class Problem(Document):
problem = StringField(required=True)
key = StringField(required=True)
type_ = StringField(required=True,regex='(choice|blank)')
owner = ReferenceField(User,required=True)
difficulty = IntField(required=True,min_value=1,max_value=5)
class ScoredProblem(EmbeddedDocument):
problem = ReferenceField(Problem,required=True)
point = IntField(required=True)
class Service(Document):
account = StringField()
game = StringField()
price = StringField()
contact = IntField()
occupied = BooleanField()
class Trials(Document):
condition = StringField(required=True, unique=True)
studies = IntField()
nct_ids = ListField(StringField())
class River(Document):
name = StringField()
length = IntField()
continent = StringField()
class User(BaseDocument):
username = IntField(required=True, unique=True)
password = StringField(required=True)
is_enabled = BooleanField(default=True)
class Nonce(Document, NonceMixin):
"""One use numbers"""
server_url = StringField(max_length=255)
timestamp = IntField()
salt = StringField(max_length=40)
class Movie(Document):
title = StringField()
image = StringField()
year = IntField()
class AssignmentAnswer(Document):
assignment = ReferenceField(Assignment,required=True)
answers = ListField(StringField(),required=True)
points_get = ListField(IntField())
#作业上交时间
submit_time = DateTimeField(required=True)
class ActivityLog(Document):
# mongoDB will add a unique object '_id' for each 'Document'
user_id = IntField(required=True)
username = StringField(required=True, max_length=64)
timestamp = DateTimeField(default=datetime.utcnow())
details = StringField(required=True)
class Presentation(EmbeddedDocument):
gravity = IntField()
parity = EmbeddedDocumentField(Parity)
last_menstrual_period = StringField()
presenting_symptoms = ListField(StringField())
class Table(Document):
number = IntField()
number_of_guests = IntField()
class BetaReading(EmbeddedDocument):
date = StringField()
level = IntField()
class Character(Document):
"""Character class
Attributes:
member_id (str): Member ID.
name (str): Character name.
race (str): Character race.
sex (str): Character sex.
desc (str): Character description. Maximum length is 1500.
lvl (int): The current level of the character. Defaults to 1. Minimum
value is 1.
xp (int): The current experience of the character. Defaults to 0.
Minimum value is 0.
xp_factor (float): Multiplier experience for the character. Defaults
to 1.0.
avatar (str): Link to character avatar. Defaults to None.
inventory (Inventory): Character inventory.
attributes (Attributes): Character attributes.
equipment (Equipment): Character equipment.
"""
member_id = StringField(primary_key=True)
name = StringField(max_length=25)
race = StringField(choices=config.humanize.races.keys())
sex = StringField(choices=config.humanize.genders.keys())
desc = StringField(max_length=1500)
lvl = IntField(default=1, min_value=1)
xp = IntField(default=0, min_value=0)
xp_factor = FloatField(default=1.0)
avatar = URLField(default=None)
inventory = EmbeddedDocumentField(Inventory)
attributes = EmbeddedDocumentField(Attributes)
equipment = EmbeddedDocumentField(Equipment)
def __init__(
self,
member_id: str,
name: str,
race: str,
sex: str,
desc: str,
inventory: Inventory,
attributes: Attributes,
equipment: Equipment,
*args,
**values,
):
"""Character constructor
Args:
member_id: Member ID.
name: Character name. Maximum length is 25.
race: Character race.
sex: Character sex.
description: Character description. Maximum length is 1500.
attributes: Character attributes.
equipment: Character equipment.
"""
super().__init__(*args, **values)
self.member_id = member_id
self.name = name
self.race = race
self.sex = sex
self.desc = desc
self.inventory = inventory
self.attributes = attributes
self.equipment = equipment
@classmethod
def is_member_registered(cls, member_id: str) -> bool:
"""Returns whether the member has a character.
Args:
member_id: Member ID to check.
Returns:
bool: Character registered or not.
"""
if cls.objects(member_id=member_id):
return True
else:
return False
@classmethod
def get_char_by_id(cls, member_id: str):
"""Returns character object.
Args:
member_id: Member ID to get.
Returns:
Character: Character object.
Raises:
CharacterNotFound: If the member is not registered.
"""
chars = cls.objects(member_id=member_id)
if not chars:
raise CharacterNotFound
return chars.first()
class MethotrexateDose(EmbeddedDocument):
date = StringField()
quantity = IntField()
class Dish(EmbeddedDocument):
food = StringField(required=True)
number = IntField()
class Parity(EmbeddedDocument):
num_full_term_births = IntField()
num_pre_term_births = IntField()
num_abortions = IntField()
num_living_children = IntField()
class Employee(Person):
salary = IntField()
class Country(Document):
"""
CH4 - methane
GNI - gross national income
GINI - gini index
Agriculture_Percentage - percentage of land for agriculture
"""
id = IntField(required=True, primary_key=True)
Name = StringField(required=True, max_length=50)
Population = ListField(EmbeddedDocumentField(Year))
CO2 = ListField(EmbeddedDocumentField(Year))
CH4 = ListField(EmbeddedDocumentField(Year))
GNI = ListField(EmbeddedDocumentField(Year))
GINI = ListField(EmbeddedDocumentField(Year))
Agriculture_Percentage = ListField(EmbeddedDocumentField(Year))
Renewable_Percentage = ListField(EmbeddedDocumentField(Year))
Fossil_Fuel_Percentage = ListField(EmbeddedDocumentField(Year))
def __init__(self, *args, **kwargs):
super(Country, self).__init__(*args, **kwargs)
def to_dict(self, indicators, start_year, end_year):
global ALL_INDICATORS
r = {
'Name': self.Name
}
if indicators is None or len(indicators) == 0:
indicators = ALL_INDICATORS
for i in indicators:
if i in CALC_INDICATORS.keys():
params = {}
ind = CALC_INDICATORS[i]
for param in ind.params:
params[param] = {}
for param in ind.params:
for year in getattr(self, param):
if start_year <= year.Year <= end_year:
params[param][year.Year] = year.Value
r[i] = []
has_results = True
for p in params.values():
if len(p) == 0:
has_results = False
break
if not has_results:
continue
year = start_year
while year <= end_year:
if reduce((lambda x, y: x and y), map((lambda p: year in params[p]), ind.params)):
p = [params[p][year] for p in ind.params]
r[i].append({'year': year, 'value': ind.run(p)})
else:
r[i].append({'year': year, 'value': -1})
year += 1
else:
r[i] = [yr.to_dict() for yr in getattr(self, i) if start_year <= yr.Year <= end_year]
return r
def get_values_list(self, indicator, start_year, end_year):
'''
year = end_year - index
list is in reverse order (from end year to start year)
:param indicator:
:param start_year:
:param end_year:
:return:
'''
data = []
if getattr(self, indicator) == []:
data = [-1] * (end_year-start_year)
else:
[data.append(yr.Value) for yr in getattr(self, indicator) if start_year <= yr.Year <= end_year]
output = {'start_year': start_year, 'end_year': end_year, 'data': data}
return output
class City(Document):
continent = StringField()
id = IntField()
meta = {"abstract": True, "allow_inheritance": False}
class Bike(Document):
Model = StringField()
Dailyfee = IntField()
Image = StringField()
Year = IntField()
class Supplier(Document):
"""
Suppliers data schema: https://github.com/nyu-devops-fall19-suppliers/suppliers/issues/21
"""
logger = logging.getLogger('flask.app')
app = None
# Table Schema
supplierName = StringField(required=True)
address = StringField(required=False)
productIdList = ListField(StringField(), required=False)
averageRating = IntField(required=False)
def __repr__(self):
return '<Supplier %r>' % (self.supplierName)
# def save(self):
# """
# Saves a Supplier to the data store
# """
# Supplier.logger.info('Saving %s', self.supplierName)
# self.save()
# Deprecated function. Use supplier.to_json() instead
@classmethod
def init_db(cls, app):
""" Initializes the database session """
cls.logger.info('Initializing database')
cls.app = app
DB_URI = "mongodb+srv://suppliers:[email protected]/myDatabase?retryWrites=true&w=majority"
# This is where we initialize mongoDB from the Flask app
connect('myDatabase', host=DB_URI)
app.app_context().push()
@classmethod
def all(cls):
"""This is a function to return all suppliers"""
cls.logger.info('Processing all suppliers')
return cls.objects()
@classmethod
def find_by_name(cls, supplier_name):
""" Find a supplier by its name """
cls.logger.info('Processing looking for name %s', supplier_name)
try:
res = cls.objects.get(supplierName=supplier_name)
except DoesNotExist:
return None
return res
@classmethod
def find(cls, supplier_id):
"""Retrieves a single supplier with a given id (supplierID) """
cls.logger.info('Getting supplier with id: %s', supplier_id)
try:
res = cls.objects(id=supplier_id).first()
except ValidationError:
return None
return res
@classmethod
def find_by_product(cls, product_id):
"""Retrieves a list of supplier with a given product id """
cls.logger.info(
"Getting suppliers with product id: %s".format(product_id))
res = cls.objects(productIdList__in=product_id)
return res
@classmethod
def find_by_rating(cls, rating):
"""Retrieves a list of supplier with a given rating score """
cls.logger.info(
"Getting suppliers with ratting score greater than: %d".format(
rating))
res = cls.objects(averageRating__gte=rating)
return res
@classmethod
def find_by_equals_to_rating(cls, rating):
"""Retrieves a list of supplier with a given rating score """
cls.logger.info(
"Getting suppliers with ratting score equals to: %d".format(
rating))
res = cls.objects(averageRating=rating)
return res
@classmethod
def action_make_recommendation(cls, product_id):
"""Retrieves a list of supplier with a given rating score and product id """
cls.logger.info(
"Getting suppliers with ratting score greater than: %s".format(
product_id))
res = cls.objects(
Q(productIdList__in=product_id)
& Q(averageRating__gte=3))
return res
class Sample(CritsBaseAttributes, CritsSourceDocument, CritsActionsDocument,
Document):
"""Sample object"""
meta = {
"collection": settings.COL_SAMPLES,
"crits_type": 'Sample',
"latest_schema_version": 5,
"shard_key": ('md5',),
"schema_doc": {
'filename': 'The name of the last file that was uploaded with this'\
'MD5',
'filenames': 'A list of filenames this binary has gone by.',
'filetype': 'The filetype of the file',
'mimetype': 'The mimetype of the file',
'size': 'The size of the file',
'md5': 'The MD5 of the file',
'sha1': 'The SHA1 of the file',
'sha256': 'The SHA256 of the file',
'ssdeep': 'The ssdeep of the file',
'impfuzzy': 'The impfuzzy of the executable file',
'campaign': 'List [] of campaigns using this file',
'source': 'List [] of sources that provided this file',
'created': 'ISODate of when this file was uploaded',
'modified': 'ISODate of when the file metadata was last modified',
'filedata': 'The ObjectId of the file in GridFS'
},
"jtable_opts": {
'details_url': 'crits.samples.views.detail',
'details_url_key': 'md5',
'default_sort': "created DESC",
'searchurl': 'crits.samples.views.samples_listing',
'fields': [ "filename", "size", "filetype",
"created", "modified", "campaign",
"source", "md5", "id", "status"],
'jtopts_fields': [ "details",
"filename",
"size",
"filetype",
"created",
"campaign",
"source",
"md5",
"status",
"favorite",
"id"],
'hidden_fields': ["md5"],
'linked_fields': ["filename", "source", "campaign",
"filetype"],
'details_link': 'details',
'no_sort': ['details']
},
}
filedata = getFileField(collection_name=settings.COL_SAMPLES)
filename = StringField(required=True)
filenames = ListField(StringField())
filetype = StringField()
md5 = StringField(required=True)
mimetype = StringField()
sha1 = StringField()
sha256 = StringField()
size = IntField(default=0)
ssdeep = StringField()
impfuzzy = StringField()
def migrate(self):
migrate_sample(self)
def add_file_data(self, file_data):
self._generate_file_metadata(file_data)
self.filedata = file_data
def add_file_obj(self, file_obj):
data = file_obj.read()
self._generate_file_metadata(data)
self.filedata = data
def _generate_file_metadata(self, data):
import pydeep
import magic
from hashlib import md5, sha1, sha256
try:
import pyimpfuzzy
except ImportError:
pass
try:
self.filetype = magic.from_buffer(data)
if len(self.filetype) > 1000:
self.filetype = self.filetype[0:1000] + '<TRUNCATED>'
except:
self.filetype = "Unavailable"
try:
mimetype = magic.from_buffer(data, mime=True)
if mimetype:
self.mimetype = mimetype.split(";")[0]
if not mimetype:
self.mimetype = "unknown"
except:
self.mimetype = "Unavailable"
self.size = len(data)
# this is a shard key. you can't modify it once it's set.
# MongoEngine will still mark the field as modified even if you set it
# to the same value.
if not self.md5:
self.md5 = md5(data).hexdigest()
self.sha1 = sha1(data).hexdigest()
self.sha256 = sha256(data).hexdigest()
try:
self.ssdeep = pydeep.hash_bytes(data)
except:
self.ssdeep = None
try:
self.impfuzzy = pyimpfuzzy.get_impfuzzy_data(data)
except:
self.impfuzzy = None
def is_office(self):
"""
Is this a Office file.
"""
ret = self.filedata.grid_id != None and self.filedata.read(
8) == "\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
if self.filedata.grid_id:
self.filedata.seek(0)
return ret
def is_pe(self):
"""
Is this a PE file.
"""
ret = self.filedata.grid_id != None and self.filedata.read(2) == "MZ"
if self.filedata.grid_id:
self.filedata.seek(0)
return ret
def is_pdf(self):
"""
Is this a PDF.
"""
ret = self.filedata.grid_id != None and "%PDF-" in self.filedata.read(
1024)
if self.filedata.grid_id:
self.filedata.seek(0)
return ret
def discover_binary(self):
"""
Queries GridFS for a matching binary to this sample document.
"""
from crits.core.mongo_tools import mongo_connector
fm = mongo_connector("%s.files" % self._meta['collection'])
objectid = fm.find_one({'md5': self.md5}, {'_id': 1})
if objectid:
self.filedata.grid_id = objectid['_id']
self.filedata._mark_as_changed()
def set_filenames(self, filenames):
"""
Set the Sample filenames to a specified list.
:param filenames: The filenames to set.
:type filenames: list
"""
if isinstance(filenames, list):
self.filenames = filenames
def _json_yaml_convert(self, exclude=[]):
"""
Helper to convert to a dict before converting to JSON.
:param exclude: list of fields to exclude.
:type exclude: list
:returns: json
"""
d = self.to_dict(exclude)
if 'filedata' not in exclude:
(d['filedata'], ext) = format_file(self.filedata.read(), 'base64')
return json.dumps(d, default=json_handler)
