def clean(self):
permalink = self.get_permalink()
if Project.objects.filter(permalink=permalink).count() > 0:
raise ValidationError('Duplicate project')
if self.start_date > self.end_date:
raise ValidationError(
'start date should be greater than end date.')
def clean(self):
if len(User.objects.filter(username=self.username)) > 0:
raise ValidationError('Username already exists')
if len(User.objects.filter(email=self.email)) > 0:
raise ValidationError('Email already exists')
if USERNAME_REGX is not None and USERNAME_REGX.match(
self.username) is None:
raise ValidationError('Special characters not allowed in username.')
def pre_save(cls, sender, document, **kwargs):
'''
validating max length and mentioned user
'''
mentions = re.findall(MENTION_REGEX, document.text)
if len(mentions) > 0:
document.mentions = [mention[1:] for mention in mentions]
if document.created_by not in document.project.members:
raise ValidationError('You are not a member of this project')
if len(document.text) > 140:
raise ValidationError('Update exceeds 140 characters')
def save(self, *args, **kwargs):
if self.token:
raise ValidationError(
"Token needs to be generated on save")
if self.valid_until:
raise ValidationError(
"Validity needs to be generated on save")
self.token = generate_token()
self.valid_until = self.created + td(days=1)
super(PasswordResetToken, self).save(*args, **kwargs)
def add_sprint(self, user):
last_sprint = self.get_last_sprint()
duration = self.duration
if not last_sprint:
start_date = self.start_date
end_date = self.start_date + timedelta(days=duration)
sequence = 0
else:
if last_sprint.sequence != 0:
start_date = last_sprint.end_date + timedelta(days=1)
else:
start_date = self.start_date
end_date = start_date + timedelta(days=duration)
sequence = last_sprint.sequence + 1
try:
sprint = Sprint(start_date=start_date,
end_date=end_date,
sequence=sequence,
project=self,
created_by=user,
updated_by=user)
sprint.save()
self.sprints.append('Sprint ' + str(sprint.sequence))
self.update(set__sprints=self.sprints, set__end_date=end_date)
return sprint
except Exception, error:
raise ValidationError(error)
def get_project_object(cls, sequence=None, organization=None):
if not organization:
try:
project = Project.objects.get(sequence=sequence,
is_active=True)
except Project.DoesNotExist, error:
raise ValidationError(error.message)
def pre_save(cls, sender, document, **kwargs):
'''
1. check if group already exists,
2. create default data for groups
'''
if len(document.name) > 250:
raise ValidationError('Group exceeds 250 characters')
def pre_save(cls, sender, document, **kwargs):
'''
1. check if project already exists,
2. create id for project
'''
if document.organization and \
document.organization not in document.created_by.belongs_to:
raise ValidationError('You do not belong to this organization.')
last_sequence = cls.last_project_id(document.organization)
if last_sequence:
document.sequence = last_sequence + 1
else:
document.sequence = 1
if len(document.title) > 64:
raise ValidationError('Title exceeds 64 characters')
if len(document.description) > 500:
raise ValidationError('Description exceeds 500 characters')
def validate_unique(self):
errors = []
for form in self.forms:
if not hasattr(form, 'cleaned_data'):
continue
errors += form.validate_unique()
if errors:
raise ValidationError(errors)
def pre_save(cls, sender, document, **kwargs):
'''
1. check if Story already exists,
2. create id for Story
'''
last_sequence = cls.last_story_id(document.project)
if last_sequence:
document.sequence = last_sequence + 1
else:
document.sequence = 1
# if document.created_by not in document.project.admin:
# raise ValidationError('You are not admin of this project')
if len(document.title) > 128:
raise ValidationError('Story exceeds 128 characters')
if document.description and len(document.description) > 500:
raise ValidationError('Story description exceeds 500 characters')
def inner_validate(value, *args, **kwargs):
value = old_clean(value, *args, **kwargs)
if value is None and field.required:
raise ValidationError("This field is required")
elif value is None:
return value
try:
new_clean(value)
return value
except ValidationError, e:
raise forms.ValidationError(e)
def update_sprint_metadata(self):
from datamodels.analytics import ProjectMetadata
'''
function to update project metadata in case a sprint is being
added.
'''
project_metadata = None
try:
project_metadata = ProjectMetadata.objects.get(
project=self.project)
except ProjectMetadata.DoesNotExist:
raise ValidationError('No data available')
metadata = project_metadata.metadata
sprint_metadata = metadata[self.project.permalink]
sprint_metadata.update({str(self.sequence): {}})
project_metadata.update(set__metadata=metadata)
def update_sprint_metadata(self, old_sprint=None):
'''
function to update project metadata in case a story is being
moved to different sprint.
'''
project_metadata = None
try:
project_metadata = ProjectMetadata.objects.get(
project=self.project)
except ProjectMetadata.DoesNotExist:
raise ValidationError('No data available')
metadata = project_metadata.metadata
sprint_metadata = metadata[self.project.permalink][str(
old_sprint.sequence)]
story_metadata = sprint_metadata[str(self.sequence)]
sprint_metadata.pop(str(self.sequence))
for sprint in metadata[self.project.permalink]:
if int(sprint) == self.sprint.sequence:
sprint_dict = metadata[self.project.permalink][sprint]
sprint_dict.update({str(self.sequence): story_metadata})
project_metadata.update(set__metadata=metadata)
def clean(self):
if Group.objects.filter(name=self.name,
project=self.project).count() > 0:
raise ValidationError('Duplicate group')
errors = {}
for field, value in fields:
if value is not None:
try:
field._validate(value)
# need to capture field-level errors!
if hasattr(field, 'myError'):
errors[field.name] = field.myError
# del it not that we have captured the error
del field.myError
except ValidationError, error:
errors[field.name] = error.errors or error
except (ValueError, AttributeError, AssertionError), error:
errors[field.name] = error
elif field.required:
errors[field.name] = ValidationError('Field is required',
field_name=field.name)
return errors
# inspired by http://stackoverflow.com/questions/6102103/using-mongoengine-document-class-methods-for-custom-validation-and-pre-save-hook
class MyDoc(app.db.Document):
def fieldsToHandle(self):
return fieldsToHandle(self)
def validate(self):
return validate(self)
class MyEmbedDoc(app.db.EmbeddedDocument):
def fieldsToHandle(self):
return fieldsToHandle(self)
def add_object(type_,
oid,
object_type,
name,
source,
method,
reference,
analyst,
value=None,
file_=None,
add_indicator=False,
get_objects=True,
obj=None,
is_sort_relationships=False,
is_validate_only=False,
is_validate_locally=False,
cache={}):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param name: The name of the ObjectType being added.
:type name: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param analyst: The user adding this object.
:type analyst: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_validate_only: Only validate, do not add.
:type is_validate_only: bool
:param is_validate_locally: Only validate, do not add.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
if oid == None:
oid = ""
if obj == None:
obj = class_from_id(type_, oid)
if not obj:
if is_validate_locally == True:
# TODO: Perform some form of validation
results['success'] = True
return results
else:
results['message'] = "Could not find item to add object to."
results['success'] = False
return results
if name == "URL" and "://" not in value.split('.')[0]:
return {
"success":
False,
"message":
"URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)"
}
elif object_type == "Address":
if "ipv4" in name:
try:
validate_ipv4_address(value)
except DjangoValidationError:
return {"success": False, "message": "Invalid IPv4 address. "}
elif "ipv6" in name:
try:
validate_ipv6_address(value)
except DjangoValidationError:
return {"success": False, "message": "Invalid IPv6 address. "}
elif "cidr" in name:
try:
if '/' not in value:
raise ValidationError("")
cidr_parts = value.split('/')
if int(cidr_parts[1]) < 0 or int(cidr_parts[1]) > 128:
raise ValidationError("")
if ':' not in cidr_parts[0] and int(cidr_parts[1]) > 32:
raise ValidationError("")
validate_ipv46_address(cidr_parts[0])
except (ValidationError, ValueError) as cidr_error:
return {"success": False, "message": "Invalid CIDR address. "}
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type, name, value, source, method, reference,
analyst)
if is_validate_only == False:
obj.save(username=analyst)
new_len = len(obj.obj)
if new_len > cur_len:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=analyst,
related_id=oid,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
if object_type != name:
object_type = "%s - %s" % (object_type, name)
campaign = obj.campaign if hasattr(obj, 'campaign') else None
ind_res = handle_indicator_ind(value,
source,
reference,
object_type,
analyst,
method,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
forge_relationship(left_class=obj,
right_class=ind,
rel_type="Related_To",
analyst=analyst,
get_rels=is_sort_relationships)
else:
results['message'] = "Object was added, but failed to add Indicator." \
"<br>Error: " + ind_res.get('message')
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(
analyst, meta=True)
else:
results['relationships'] = obj.sort_relationships(
analyst, meta=True)
else:
results[
'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
results['id'] = str(obj.id)
return results
except ValidationError, e:
return {'success': False, 'message': str(e)}
def validate(self, value):
if not SlugField.SLUG_REGEX.match(value):
raise ValidationError('This string is not a slug: %s' % value)
def get_organization_object(cls, name):
try:
org = Organization.objects.get(name=name)
except Organization.DoesNotExist, error:
raise ValidationError(error.message)
def clean(self):
if Organization.objects.filter(name=self.name).count() > 0:
raise ValidationError('Duplicate Organization')
if len(self.name) > 30:
raise ValidationError('Maximum 30 characters allowed')
def myError(field, message="", errors=None, field_name=None):
field_name = field_name if field_name else field.name
field.myError = ValidationError(message,
errors=errors,
field_name=field_name)
def clean(self):
if Story.objects.filter(title=self.title,
project=self.project,
is_active=True).count() > 0:
raise ValidationError('Duplicate Story')
class Project(me.Document):
organization = me.ReferenceField('Organization',
required=False,
reverse_delete_rule=me.CASCADE,
default=None)
# these four will come in request data for put
title = me.StringField(max_length=64, required=True)
permalink = me.StringField(required=True, unique=True)
start_date = me.DateTimeField(default=datetime.utcnow())
end_date = me.DateTimeField(default=datetime.utcnow())
duration = me.IntField() # sprint duration
updates = me.ListField()
hashtags = me.ListField()
# following has to be updated at time of saving object
sequence = me.IntField()
sprints = me.ListField()
members = me.ListField()
admin = me.ListField(me.ReferenceField('User'))
is_active = me.BooleanField(default=True)
description = me.StringField(max_length=500)
roles = me.ListField(default=PROJECT_ROLES)
is_todo = me.BooleanField(default=False)
# these have to be moved to base model
created_by = me.ReferenceField('User', required=True, dbref=True)
updated_by = me.ReferenceField('User', required=True, dbref=True)
created_at = me.DateTimeField(default=datetime.utcnow())
updated_at = me.DateTimeField(default=datetime.utcnow())
deleted_at = me.DateTimeField()
deleted_by = me.ReferenceField('User', dbref=True)
meta = {'indexes': ['title', 'permalink']}
def __init__(self, *args, **values):
super(Project, self).__init__(*args, **values)
def __str__(self):
return self.title
def to_json(self, fields=None, exclude=None):
return json_dumper(self, fields, exclude)
def clean(self):
permalink = self.get_permalink()
if Project.objects.filter(permalink=permalink).count() > 0:
raise ValidationError('Duplicate project')
if self.start_date > self.end_date:
raise ValidationError(
'start date should be greater than end date.')
def get_permalink(self):
owner = self.organization.name if self.organization else\
self.created_by.username
permalink = slugify(owner) + '/' + slugify(self.title)
return permalink
@classmethod
def last_project_id(cls, organization=None):
sequence = None
if not organization:
if Project.objects:
sequence = list(
Project.objects.order_by('sequence'))[-1].sequence
elif organization:
projects = Project.objects.filter(
organization=organization).order_by('sequence')
if projects:
sequence = list(projects)[-1].sequence
return sequence
@classmethod
def pre_save(cls, sender, document, **kwargs):
'''
1. check if project already exists,
2. create id for project
'''
if document.organization and \
document.organization not in document.created_by.belongs_to:
raise ValidationError('You do not belong to this organization.')
last_sequence = cls.last_project_id(document.organization)
if last_sequence:
document.sequence = last_sequence + 1
else:
document.sequence = 1
if len(document.title) > 64:
raise ValidationError('Title exceeds 64 characters')
if len(document.description) > 500:
raise ValidationError('Description exceeds 500 characters')
@classmethod
def post_save(cls, sender, document, **kwargs):
'''
1. update sequence value
'''
if document.sequence:
document.update(set__sequence=document.sequence,
set__members=[document.created_by],
set__admin=[document.created_by])
def create_sprints(self):
# for todo
try:
Sprint.objects.get(sequence=-1, project=self)
except Sprint.DoesNotExist:
Sprint.objects.create(project=self,
created_by=self.created_by,
updated_by=self.updated_by,
sequence=-1,
start_date=self.start_date,
end_date=self.end_date)
for (idx, sprint) in enumerate(self.sprints):
if idx != 0:
proj_start_date = self.start_date
sprint_duration = int(self.duration)
sprint_start_date = proj_start_date + timedelta(days=(
(idx - 1) * sprint_duration))
sprint_end_date = sprint_start_date + timedelta(
days=sprint_duration)
try:
Sprint.objects.get(sequence=idx, project=self)
except Sprint.DoesNotExist:
Sprint.objects.create(sequence=idx,
project=self,
start_date=sprint_start_date,
end_date=sprint_end_date,
created_by=self.created_by,
updated_by=self.updated_by)
elif idx == 0: # for backlog
try:
Sprint.objects.get(sequence=idx, project=self)
except Sprint.DoesNotExist:
Sprint.objects.create(project=self,
created_by=self.created_by,
updated_by=self.updated_by)
def calculate_sprints(self):
project_duration = (self.end_date - self.start_date).days
if not self.duration:
self.duration = project_duration
self.update(set__duration=self.duration)
sprints = project_duration / self.duration
# in case duration is of form 7K + 1, one sprint has to be added
self.update(set__sprints=['Sprint -1'])
self.create_sprints()
else:
self.update(set__duration=self.duration)
sprints = project_duration / self.duration
# in case duration is of form 7K + 1, one sprint has to be added
difference = project_duration - (self.duration * sprints)
if difference > 0 and difference < self.duration:
sprints += 1
sprint_list = ['Backlog']
sprint_list.extend(
['Sprint ' + str(idx) for idx in range(1, sprints + 1)])
self.update(set__sprints=sprint_list)
self.create_sprints()
def get_current_sprint(self):
now = datetime.utcnow()
sprints = Sprint.objects.filter(project=self,
sequence__nin=[0, -1],
start_date__lte=now,
end_date__gte=now)
if len(sprints) == 1:
curr_sprint = sprints[0]
elif self.start_date > now:
curr_sprint = Sprint.objects.get(project=self, sequence=0)
else:
curr_sprint = Sprint.objects.get(project=self,
sequence=(len(self.sprints) - 2))
return curr_sprint
@classmethod
def get_project_object(cls, sequence=None, organization=None):
if not organization:
try:
project = Project.objects.get(sequence=sequence,
is_active=True)
except Project.DoesNotExist, error:
raise ValidationError(error.message)
elif organization:
try:
project = Project.objects.get(sequence=sequence,
organization=organization,
is_active=True)
except Project.DoesNotExist, error:
raise ValidationError(error.message)
def get_sprint_object(self, sequence):
try:
sprint = Sprint.objects.get(sequence=sequence, project=self)
except Sprint.DoesNotExist, error:
raise ValidationError(error)