class DirectCampaignReport(Document):
campaign_id = fields.LongField(required=True)
campaign_name = fields.StringField(required=True)
date = fields.DateTimeField()
clicks = fields.LongField()
total_cost = fields.FloatField()
type_name = fields.StringField()
class DatasetBase(EmbeddedDocument):
owner = fields.StringField(required=True)
hash = fields.StringField()
size = fields.LongField()
date = fields.LongField()
title = fields.StringField(required=True)
description = fields.StringField()
category = fields.StringField(choices=DATASET_CATEGORIES)
labels = fields.ListField(fields.StringField())
shape = fields.ListField(fields.IntField())
class Certification(Document):
_id = fields.ObjectIdField(require=True)
user_id = fields.ObjectIdField(require=True)
org = fields.StringField(required=True)
name = fields.StringField(required=True)
tags = fields.ListField(required=True)
apply_time = fields.LongField(required=True)
deal_time = fields.LongField(required=True)
state = fields.StringField(require=True) # 通过的状态
_class = fields.StringField()
meta = {'collection' : 'Certification'}
class MetrikaSourceSummaryReport(Document):
source_id = fields.LongField(required=True)
date = fields.DateTimeField()
rid = fields.StringField()
name = fields.StringField()
group_id = fields.StringField()
group_name = fields.StringField()
visits = fields.LongField()
users_count = fields.LongField()
bounce_rate = fields.FloatField()
page_depth = fields.FloatField()
avg_visit_duration = fields.FloatField()
class Prep(Document):
name=fields.StringField(primary_key=True)
campaign = fields.StringField()
priority = fields.LongField(default=0)
updated = fields.DateTimeField()
workflows=fields.EmbeddedDocumentListField(Workflow)
class TaskMetadata(Document, MetadataMixin):
id = fields.StringField(primary_key=True,
default=lambda: str(uuid.uuid4()))
status = fields.StringField(default=PENDING,
choices=TASK_STATUS_CODES,
required=True)
owner = fields.StringField(required=True)
command = fields.StringField(choices=TASK_COMMANDS, required=True)
date = fields.LongField(default=lambda: int(time.time()))
config = fields.DictField(default=lambda: dict())
history = fields.DictField()
meta = {
'allow_inheritance': True,
'db_alias': 'metadata',
'collection': 'tasks'
}
def to_dict(self):
meta = self.to_mongo().to_dict()
if '_id' in meta:
meta['id'] = meta['_id']
del meta['_id']
del meta['_cls']
return meta
def from_dict(self, meta):
for name in self._fields:
if name in meta:
setattr(self, name, meta[name])
class RegularModel(Document):
"""
A model class for testing regular flat fields.
"""
str_field = fields.StringField()
str_regex_field = fields.StringField(regex="^valid_regex")
url_field = fields.URLField()
email_field = fields.EmailField()
int_field = fields.IntField()
long_field = fields.LongField()
float_field = fields.FloatField()
boolean_field = fields.BooleanField()
nullboolean_field = fields.BooleanField(null=True)
date_field = fields.DateTimeField()
complexdate_field = fields.ComplexDateTimeField()
uuid_field = fields.UUIDField()
id_field = fields.ObjectIdField()
decimal_field = fields.DecimalField()
custom_field = CustomField()
# TODO
# dynamic_field = fields.DynamicField()
# bin_field = fields.BinaryField()
# file_field = fields.FileField()
# image_field = fields.ImageField()
def method(self):
return 'method'
class CharacterSheet(mongoengine.Document):
name = fields.StringField()
description = fields.StringField()
hit_points = fields.IntField()
level = fields.IntField()
experience = fields.LongField()
strength = fields.FloatField()
dexterity = fields.FloatField()
constitution = fields.FloatField()
inteligence = fields.FloatField()
wisdom = fields.FloatField()
charisma = fields.FloatField()
armor_class = fields.FloatField()
fortitude = fields.FloatField()
reflex = fields.FloatField()
will = fields.FloatField()
race = fields.StringField()
klass = fields.StringField()
skills = fields.ListField(fields.StringField())
items = fields.ListField(fields.StringField())
owner = fields.StringField()
# race = fields.ReferenceField('Race')
# klass = fields.ReferenceField('Klass')
# skills = fields.ListField(fields.ReferenceField('Skill'))
# items = fields.ListField(fields.ReferenceField('Item'))
# owner = fields.ReferenceField('User')
def restore(self, memento):
"""
Restores the Originator's state from a memento object.
"""
self.hit_points = memento.get_hit_points()
self.update()
class Foo(Document):
string = fields.StringField()
required = fields.StringField(required=True)
choices = fields.StringField(choices=('foo', 'bar', 'baz'))
regex = fields.StringField(regex=r'^[a-z]*$')
length = fields.StringField(min_length=1, max_length=3)
strings = fields.ListField(fields.StringField())
sorted_strings = fields.SortedListField(fields.StringField())
integer = fields.IntField()
bounded_int = fields.IntField(min_value=0, max_value=10)
longeger = fields.LongField()
bounded_long = fields.LongField(min_value=0, max_value=10)
floating = fields.FloatField()
bounded_float = fields.FloatField(min_value=0.0, max_value=1.0)
boolean = fields.BooleanField()
datetime = fields.DateTimeField()
complex_datetime = fields.ComplexDateTimeField()
binary = fields.BinaryField()
bounded_binary = fields.BinaryField(max_bytes=8)
mapping = fields.MapField(fields.StringField())
uuid = fields.UUIDField()
old_geopoint = fields.GeoPointField()
point = fields.PointField()
line = fields.LineStringField()
polygon = fields.PolygonField()
points = fields.MultiPointField()
lines = fields.MultiLineStringField()
polygons = fields.MultiPolygonField()
even_length_string = fields.StringField(
validation=lambda s: len(s) % 2 == 0)
@fields.EmbeddedDocumentField
class embedded_bar(EmbeddedDocument):
bar = fields.StringField()
@fields.EmbeddedDocumentListField
class embedded_baz(EmbeddedDocument):
baz = fields.StringField()
class Task(EmbeddedDocument):
name = fields.StringField(primary_key=True)
short_name = fields.StringField()
workflow = fields.StringField()
parent_workflow = fields.StringField()
job_type = fields.StringField()
failures_count = fields.LongField()
prep = fields.EmbeddedDocumentField(TaskPrep)
statuses = fields.EmbeddedDocumentListField(TaskSiteStatus)
task_action = fields.ReferenceField(TaskAction)
class Article(Document):
title = fields.StringField(unique=True)
text = fields.StringField()
comments = fields.ListField(fields.EmbeddedDocumentField(Comment))
top_comment = fields.EmbeddedDocumentField(Comment)
tags = fields.ListField(fields.StringField())
publish = fields.BooleanField()
publish_date = fields.DateTimeField()
version = fields.FloatField()
order = fields.IntField()
serial_number = fields.LongField()
class ModelBase(EmbeddedDocument):
owner = fields.StringField(required=True)
hash = fields.StringField()
size = fields.LongField()
date = fields.LongField()
title = fields.StringField(required=True)
description = fields.StringField()
labels = fields.ListField(field=fields.StringField())
metrics = fields.DictField(default=lambda: dict())
architecture = fields.ReferenceField(ArchitectureMetadata, required=True)
dataset = fields.ReferenceField(DatasetMetadata, required=True)
# TODO: parent = fields.ReferenceField(Model)
shape = fields.ListField(field=fields.IntField())
# category = fields.StringField()
@property
def category(self):
"""Return category from dataset"""
return self.dataset.base.category
class SimpleDoc(Document):
f_str = fields.StringField()
f_url = fields.URLField()
f_eml = fields.EmailField()
f_int = fields.IntField()
f_lng = fields.LongField()
f_flt = fields.FloatField()
f_dec = fields.DecimalField()
f_bool = fields.BooleanField()
f_dt = fields.DateTimeField()
f_oid = fields.ObjectIdField()
f_uuid = fields.UUIDField()
class CeleryTaskControl(Document):
"""celery的控制模型"""
queue = fields.StringField(verbose_name='任务队列名, eg: inner_quick',
require=True)
control_type = fields.StringField(verbose_name='控制类别',
require=True,
default=ControlTypeEnum.func)
func = fields.StringField(verbose_name='方法名: test_1, 当控制类别为func时必填')
start_time = fields.LongField(verbose_name='开始生效时间', default=get_timestamp)
end_time = fields.LongField(verbose_name='结束生效时间', default=0)
rm_current_tasks = fields.StringField(
verbose_name='删除现有未执行的任务，当控制类别为queue时有效',
default=ControlRmCurrentEnum.no)
status = fields.IntField(verbose_name='启用状态',
require=True,
default=ControlValidEnum.valid)
meta = {
'collection': 'hdy_celery_control',
'indexes': ['start_time', 'end_time', 'status']
}
class Character(mongoengine.Document):
# Maybe will be necessary to switch ReferenceFields for LazyReferenceFields
name = fields.StringField()
description = fields.StringField()
hit_points = fields.IntField()
level = fields.IntField()
experience = fields.LongField()
attributes = fields.ReferenceField('Attributes')
race = fields.ReferenceField('Race')
klass = fields.ReferenceField('Klass')
skills = fields.ListField(fields.ReferenceField('Skill'))
items = fields.ListField(fields.ReferenceField('Item'))
owner = fields.ReferenceField('User')
class ConcreteCharacterMemento(mongoengine.Document):
hit_points = fields.IntField()
level = fields.IntField()
experience = fields.LongField()
strength = fields.FloatField()
desterity = fields.FloatField()
costitution = fields.FloatField()
intelligence = fields.FloatField()
wisdom = fields.FloatField()
charisma = fields.FloatField()
skills = fields.ListField(fields.StringField())
items = fields.ListField(fields.StringField())
date = fields.DateTimeField()
class Schema1EmbDoc2(EmbeddedDocument):
embdoc2_int = fields.IntField()
embdoc2_str = fields.StringField()
embdoc2_str_empty = fields.StringField()
embdoc2_str_ten = fields.StringField(choices=[str(x) for x in range(11)])
embdoc2_float = fields.FloatField()
embdoc2_int_empty = fields.IntField()
embdoc2_long = fields.LongField()
embdoc2_decimal = fields.DecimalField()
embdoc2_complex_datetime = fields.ComplexDateTimeField()
embdoc2_list = fields.ListField()
embdoc2_ref_doc1 = fields.ReferenceField('Schema1Doc1')
embdoc2_emb_embdoc2 = fields.EmbeddedDocumentField('self')
embdoc2_emblist_embdoc2 = fields.EmbeddedDocumentListField('self')
class User(db.Document, BaseModelMixin):
username = fields.StringField()
_password = fields.StringField(db_field='password')
email = fields.EmailField(unique=True)
avatar = fields.StringField(default=None)
position = fields.StringField() # 职位
department = fields.StringField() # 部门
phone = fields.StringField()
equipment_num = fields.LongField(default=0) # 装置数量
device_num = fields.LongField(default=0) # 设备数量
interface_num = fields.LongField(default=0) # 接口数量
cable_num = fields.LongField(default=0) # 线缆数量
login_num = fields.LongField(default=0)
search_num = fields.LongField(default=0)
author_type = fields.IntField(default=0)
author_editable = fields.BooleanField(default=True)
deletable = fields.BooleanField(default=True)
is_active = fields.BooleanField(default=True)
token = fields.EmbeddedDocumentField(Token, default=None)
is_anonymous = False
def get_id(self):
return str(self.id)
@property
def is_authenticated(self):
return self.is_active and self.token is not None and self.token.expired_at > datetime.now(
)
@property
def password(self):
return self._password
@password.setter
def password(self, pwd):
self._password = pbkdf2_sha256.hash(pwd)
def check_password(self, pwd):
return pbkdf2_sha256.verify(pwd, self._password)
def refresh_token(self, token_expired_time):
self.token.expired_at = datetime.now() + timedelta(
seconds=token_expired_time)
return self.save()
def generate_token(self, token_expired_time):
token = binascii.hexlify(
os.urandom(20) + str(self.id).encode('utf-8')).decode()
expired_at = datetime.now() + timedelta(seconds=token_expired_time)
self.token = Token(value=token, expired_at=expired_at)
self.login_num += 1
self.save()
return self.token.value
class CovidPersonalUser(Document):
user = fields.LongField(required=True)
date_birth = fields.DateField(required=True)
last_send_geo = fields.DateTimeField()
last_update_status = fields.DateTimeField(default=datetime.now)
created_at = fields.DateTimeField(default=datetime.now)
status = fields.StringField(required=True)
observation = fields.StringField()
coordinates = fields.PointField()
class Meta:
verbose_name = 'CovidPersonalUser'
verbose_name_plural = 'CovidPersonalUsers'
def __str__(self):
return User.objects.get(pk=self.user).name
class URLStat(db.Document):
"""Statistic for redirect usage"""
#User.email - no direct reference
author = fields.EmailField(required=True)
#URL.target - no direct reference
target = fields.StringField(required=True)
date = fields.DateTimeField(default=utils.utcnow)
count = fields.LongField(default=0, required=True)
#Json Text ?
useragent = fields.StringField()
def __unicode__(self):
return "%s - %s" % (self.author, self.target)
class CeleryErrorRecord(Document):
"""celery执行的错误记录"""
task_id = fields.StringField(verbose_name='celery任务id',
require=True,
primary_key=True)
full_func = fields.StringField(
verbose_name='带路径的方法名，eg: tasks.task_1.test_1')
func_params = fields.DictField(verbose_name='方法参数 {args:(), kwargs: {}}')
exc = fields.StringField(verbose_name='简短异常说明')
error_info = fields.StringField(verbose_name='异常堆栈详细信息')
create_time = fields.LongField(verbose_name='创建时间', default=get_timestamp)
create_time_human = fields.StringField(verbose_name='创建时间,人看的',
default=now_yyyymmddhhmmss)
meta = {
'collection': 'hdy_celery_error',
'indexes': ['full_func', 'create_time', 'create_time_human']
}
class TaskAction(Document):
name = fields.StringField()
workflow = fields.StringField()
acted = fields.IntField()
timestamp = fields.LongField()
parameters = fields.EmbeddedDocumentField(TaskActionParameters)
class AlarmClass(nosql.Document):
"""
Alarm class
"""
meta = {
"collection": "noc.alarmclasses",
"strict": False,
"auto_create_index": False,
"json_collection": "fm.alarmclasses",
"json_depends_on": [
"fm.alarmseverities"
],
}
name = fields.StringField(required=True, unique=True)
uuid = fields.UUIDField(binary=True)
description = fields.StringField(required=False)
# Create or not create separate Alarm
# if is_unique is True and there is active alarm
# Do not create separate alarm if is_unique set
is_unique = fields.BooleanField(default=False)
# List of var names to be used as discriminator key
discriminator = fields.ListField(nosql.StringField())
# Can alarm status be cleared by user
user_clearable = fields.BooleanField(default=True)
# Default alarm severity
default_severity = nosql.PlainReferenceField(AlarmSeverity)
#
datasources = fields.ListField(fields.EmbeddedDocumentField(DataSource))
vars = fields.ListField(fields.EmbeddedDocumentField(AlarmClassVar))
# Text messages
subject_template = fields.StringField()
body_template = fields.StringField()
symptoms = fields.StringField()
probable_causes = fields.StringField()
recommended_actions = fields.StringField()
# Flap detection
flap_condition = fields.StringField(
required=False,
choices=[("none", "none"), ("count", "count")],
default="none")
flap_window = fields.IntField(required=False, default=0)
flap_threshold = fields.FloatField(required=False, default=0)
# RCA
root_cause = fields.ListField(
fields.EmbeddedDocumentField(AlarmRootCauseCondition))
topology_rca = fields.BooleanField(default=False)
# List of handlers to be called on alarm raising
handlers = fields.ListField(fields.StringField())
# List of handlers to be called on alarm clear
clear_handlers = fields.ListField(fields.StringField())
# Plugin settings
plugins = fields.ListField(fields.EmbeddedDocumentField(AlarmPlugin))
# Time in seconds to delay alarm risen notification
notification_delay = fields.IntField(required=False)
# Control time to reopen alarm instead of creating new
control_time0 = fields.IntField(required=False)
# Control time to reopen alarm after 1 reopen
control_time1 = fields.IntField(required=False)
# Control time to reopen alarm after >1 reopen
control_timeN = fields.IntField(required=False)
# Consequence recover time
# Root cause will be detached if consequence alarm
# will not clear itself in *recover_time*
recover_time = fields.IntField(required=False, default=300)
#
bi_id = fields.LongField(unique=True)
#
category = nosql.ObjectIdField()
_id_cache = cachetools.TTLCache(maxsize=1000, ttl=60)
_bi_id_cache = cachetools.TTLCache(maxsize=1000, ttl=60)
_name_cache = cachetools.TTLCache(maxsize=1000, ttl=60)
_handlers_cache = {}
_clear_handlers_cache = {}
def __unicode__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock)
def get_by_id(cls, id):
return AlarmClass.objects.filter(id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock)
def get_by_bi_id(cls, id):
return AlarmClass.objects.filter(bi_id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_name_cache"), lock=lambda _: id_lock)
def get_by_name(cls, name):
return AlarmClass.objects.filter(name=name).first()
def get_handlers(self):
@cachetools.cached(self._handlers_cache, key=lambda x: x.id, lock=handlers_lock)
def _get_handlers(alarm_class):
handlers = []
for hh in alarm_class.handlers:
try:
h = get_handler(hh)
except ImportError:
h = None
if h:
handlers += [h]
return handlers
return _get_handlers(self)
def get_clear_handlers(self):
@cachetools.cached(self._clear_handlers_cache, key=lambda x: x.id, lock=handlers_lock)
def _get_handlers(alarm_class):
handlers = []
for hh in alarm_class.clear_handlers:
try:
h = get_handler(hh)
except ImportError:
h = None
if h:
handlers += [h]
return handlers
return _get_handlers(self)
def save(self, *args, **kwargs):
c_name = " | ".join(self.name.split(" | ")[:-1])
c = AlarmClassCategory.objects.filter(name=c_name).first()
if not c:
c = AlarmClassCategory(name=c_name)
c.save()
self.category = c.id
super(AlarmClass, self).save(*args, **kwargs)
def get_discriminator(self, vars):
"""
Calculate discriminator hash
:param vars: Dict of vars
:returns: Discriminator hash
"""
if vars:
ds = sorted(str(vars[n]) for n in self.discriminator)
return hashlib.sha1("\x00".join(ds)).hexdigest()
else:
return hashlib.sha1("").hexdigest()
def to_json(self):
c = self
r = ["{"]
r += [" \"name\": \"%s\"," % q(c.name)]
r += [" \"$collection\": \"%s\"," % self._meta["json_collection"]]
r += [" \"uuid\": \"%s\"," % c.uuid]
if c.description:
r += [" \"desciption\": \"%s\"," % q(c.description)]
r += [" \"is_unique\": %s," % q(c.is_unique)]
if c.is_unique and c.discriminator:
r += [" \"discriminator\": [%s]," % ", ".join(["\"%s\"" % q(d) for d in c.discriminator])]
r += [" \"user_clearable\": %s," % q(c.user_clearable)]
r += [" \"default_severity__name\": \"%s\"," % q(c.default_severity.name)]
# datasources
if c.datasources:
r += [" \"datasources\": ["]
jds = []
for ds in c.datasources:
x = []
x += [" \"name\": \"%s\"" % q(ds.name)]
x += [" \"datasource\": \"%s\"" % q(ds.datasource)]
ss = []
for k in sorted(ds.search):
ss += [" \"%s\": \"%s\"" % (q(k), q(ds.search[k]))]
x += [" \"search\": {\n%s\n }" % (",\n".join(ss))]
jds += [" {\n%s\n }" % ",\n".join(x)]
r += [",\n\n".join(jds)]
r += [" ],"]
# vars
vars = []
for v in c.vars:
vd = [" {"]
vd += [" \"name\": \"%s\"," % q(v.name)]
vd += [" \"description\": \"%s\"" % q(v.description)]
if v.default:
vd[-1] += ","
vd += [" \"default\": \"%s\"" % q(v.default)]
vd += [" }"]
vars += ["\n".join(vd)]
r += [" \"vars\": ["]
r += [",\n".join(vars)]
r += [" ],"]
# Handlers
if self.handlers:
hh = [" \"%s\"" % h for h in self.handlers]
r += [" \"handlers\": ["]
r += [",\n\n".join(hh)]
r += [" ],"]
if self.clear_handlers:
hh = [" \"%s\"" % h for h in self.clear_handlers]
r += [" \"clear_handlers\": ["]
r += [",\n\n".join(hh)]
r += [" ],"]
# Text
r += [" \"subject_template\": \"%s\"," % q(c.subject_template)]
r += [" \"body_template\": \"%s\"," % q(c.body_template)]
r += [" \"symptoms\": \"%s\"," % q(c.symptoms if c.symptoms else "")]
r += [" \"probable_causes\": \"%s\"," % q(c.probable_causes if c.probable_causes else "")]
r += [" \"recommended_actions\": \"%s\"," % q(c.recommended_actions if c.recommended_actions else "")]
# Root cause
if self.root_cause:
rc = []
for rr in self.root_cause:
rcd = [" {"]
rcd += [" \"name\": \"%s\"," % rr.name]
rcd += [" \"root__name\": \"%s\"," % rr.root.name]
rcd += [" \"window\": %d," % rr.window]
if rr.condition:
rcd += [" \"condition\": \"%s\"," % rr.condition]
rcd += [" \"match_condition\": {"]
mcv = []
for v in rr.match_condition:
mcv += [" \"%s\": \"%s\"" % (v, rr.match_condition[v])]
rcd += [",\n".join(mcv)]
rcd += [" }"]
rcd += [" }"]
rc += ["\n".join(rcd)]
if r[-1][-1] != ",":
r[-1] += ","
r += [" \"root_cause\": ["]
r += [",\n".join(rc)]
r += [" ]"]
if self.topology_rca:
if r[-1][-1] != ",":
r[-1] += ","
r += [" \"topology_rca\": true"]
# Plugins
if self.plugins:
if r[-1][-1] != ",":
r[-1] += ","
plugins = []
for p in self.plugins:
pd = [" {"]
pd += [" \"name\": \"%s\"" % p.name]
if p.config:
pd[-1] += ","
pc = []
for v in p.config:
pc += [" \"%s\": \"%s\"" % (v, p.config.vars[v])]
pd += [" \"config\": {"]
pd += [",\n".join(pc)]
pd += [" }"]
pd += [" }"]
plugins += ["\n".join(pd)]
r += [" \"plugins\": ["]
r += [",\n".join(plugins)]
r += [" ]"]
if self.notification_delay:
if r[-1][-1] != ",":
r[-1] += ","
r += [" \"notification_delay\": %d" % self.notification_delay]
if self.control_time0:
if r[-1][-1] != ",":
r[-1] += ","
r += [" \"control_time0\": %d" % self.control_time0]
if self.control_time1:
r[-1] += ","
r += [" \"control_time1\": %d" % self.control_time1]
if self.control_timeN:
r[-1] += ","
r += [" \"control_timeN\": %d" % self.control_timeN]
if self.recover_time:
if r[-1][-1] != ",":
r[-1] += ","
r += [" \"recover_time\": %d" % self.recover_time]
# Close
if r[-1].endswith(","):
r[-1] = r[-1][:-1]
r += ["}", ""]
return "\n".join(r)
def get_json_path(self):
p = [quote_safe_path(n.strip()) for n in self.name.split("|")]
return os.path.join(*p) + ".json"
@property
def config(self):
if not hasattr(self, "_config"):
self._config = AlarmClassConfig.objects.filter(alarm_class=self.id).first()
return self._config
def get_notification_delay(self):
if self.config:
return self.config.notification_delay or None
else:
return self.notification_delay or None
def get_control_time(self, reopens):
if reopens == 0:
if self.config:
return self.config.control_time0 or None
else:
return self.control_time0 or None
elif reopens == 1:
if self.config:
return self.config.control_time1 or None
else:
return self.control_time1 or None
else:
if self.config:
return self.config.control_timeN or None
else:
return self.control_timeN or None
class TaskSiteStatus(EmbeddedDocument):
site = fields.StringField()
dataset = fields.StringField()
success_count = fields.LongField(default=0)
failed_count = fields.LongField(default=0)
class TaskPrep(EmbeddedDocument):
name = fields.StringField()
campaign = fields.StringField()
priority = fields.LongField(default=0)
class MetricKeys(EmbeddedDocument):
count = fields.LongField(default=0)
mail_in = fields.LongField(default=0)
mail_out = fields.LongField(default=0)
clean = fields.LongField(default=0)
spam = fields.LongField(default=0)
virus = fields.LongField(default=0)
banned = fields.LongField(default=0)
unchecked = fields.LongField(default=0)
bounce = fields.LongField(default=0)
files_count = fields.LongField(default=0)
total_size = fields.LongField(default=0.0)
class MessageStore(BaseDocument):
"""
TODO: is_spam, is_virus, ... en champs numérique: 0 ou 1 pour stats
TODO: is_ham, is_spam, ...
0-prod\rs-admin\archives\rs-admin\rs_admin\mail_parser_utils.py
TODO: ('X-Originating-IP', '[88.175.183.38]')
"""
completed = fields.IntField(default=0)
group_name = fields.StringField(required=True,
max_length=80,
default=constants.GROUP_DEFAULT,
verbose_name=gettext(u"Group"))
domain_name = fields.StringField(max_length=63,
verbose_name=gettext(u"Domain"))
policy_uid = fields.StringField()
quarantine_id = fields.StringField()
is_in = fields.IntField(default=1,
verbose_name=gettext(u"Incoming message"),
help_text=gettext(u"Incoming or Outgoing Message"))
store_key = fields.StringField(required=True, unique=True)
sent = fields.DateTimeField(verbose_name=gettext(u"Sent Date"))
#sent_origin = fields.StringField()
received = fields.DateTimeField(default=utils.timestamp,
verbose_name=gettext(u"Received Date"))
headers = fields.DictField()
message = fields.FileField()
#FIXME: ne pas utiliser le nom size !!!!
size = fields.LongField(default=0, verbose_name=gettext(u"Size"))
subject = fields.StringField(verbose_name=gettext(u"Subject"))
message_id = fields.StringField()
sender = fields.StringField(verbose_name=gettext(u"Sender"))
#Pas de EmailField
rcpt = fields.ListField(fields.StringField(),
default=[],
verbose_name=gettext(u"Recipients(s)"))
rcpt_count = fields.IntField(default=0,
verbose_name=gettext(u"Rcpts"),
help_text=gettext(u"Number of recipients"))
#mode proxy ou autre pour rcpt refusé à la livraison - refus partiel seulement
rcpt_refused = fields.DictField(default={})
#IP du client original par xforward
client_address = fields.StringField(verbose_name=gettext(u"IP Address"))
country = fields.StringField(
verbose_name=gettext(u"Country"),
help_text=gettext(u"Country based on ip address of sender smtp"))
#receiveds_header = fields.SortedListField(fields.StringField(), default=[])
is_bounce = fields.IntField(default=0, verbose_name=gettext(u"Bounce ?"))
is_spam = fields.IntField(default=0, verbose_name=gettext(u"Spam ?"))
is_virus = fields.IntField(default=0, verbose_name=gettext(u"Infected ?"))
is_banned = fields.IntField(default=0, verbose_name=gettext(u"Banned ?"))
is_unchecked = fields.IntField(default=0,
verbose_name=gettext(u"Checked ?"))
xforward = fields.DictField()
#IP du serveur SMTP postfix
server = fields.StringField()
queue = fields.IntField(choices=constants.MESSAGE_QUEUE_CHOICES,
default=constants.MESSAGE_QUEUE_INCOMING)
files_count = fields.IntField(
default=0,
verbose_name=gettext(u"Files"),
help_text=gettext(u"Number of attachments in message"))
files = fields.EmbeddedDocumentListField(MessageAttachment)
events = fields.EmbeddedDocumentListField(MessageEvent)
tags = fields.ListField(fields.StringField(), default=[])
parsing_errors = fields.ListField(fields.StringField(), default=[])
errors_count = fields.IntField(default=0)
turing = fields.EmbeddedDocumentField(MessageTuring, required=False)
metric = fields.IntField(default=0)
def get_filter_result(self):
if self.is_virus:
return "VIRUS"
elif self.is_spam:
return "SPAM"
elif self.is_banned:
return "BANNED"
elif self.is_unchecked:
return "UNCHECKED"
else:
return "CLEAN"
filter_result = property(fget=get_filter_result)
def _complete(self):
"""
Complete parsing message
"""
values = {'completed': 1}
other_fields = message_complete(self.store_key,
self.parse_message(pure_string=True),
sender=self.sender)
values.update(other_fields)
values.update(
identify(sender=self.sender,
client_address=self.client_address,
rcpt=self.rcpt))
fields = MessageStore._fields.keys()
for key, value in values.iteritems():
if key in fields:
setattr(self, key, value)
#TODO: validate / clean
return self.save(
force_insert=False
) #, validate, clean, write_concern, cascade, cascade_kwargs, _refs, save_condition)
def parse_message(self, pure_string=False):
msg_string = utils.uncompress(self.message.read())
try:
if pure_string:
return msg_string
return from_string(msg_string)
except Exception, err:
logger.error(str(err))
return recover(msg_string)
