class Descriptor(MeshTerm):
"""Model for a MeSH Descriptor record"""
allowable_qualifiers = ListField(ReferenceField(Qualifier))
# reference uid from descriptor
# update to ReferenceField if create collection for PharmacologicalAction
pharmacological_actions = ListField()
# only name. does not correspond to uid
previous_indexings = ListField()
tree_numbers = ListField()
# used for querying tree ex. db.descriptor.find({parents:"A02"})
parents = ListField()
meta = {"indexes": ["tree_numbers"]}
def __init__(self, record: OrderedDict = None, *args, **kwargs):
"""Constructor for the MeSH Descriptor Record.
Takes OrderedDict object from xmltodict output.
Original source is xml download of descXXXX
renamed and compressed to descXXXX.xml.gz"""
qualifers = kwargs.pop("qualifiers", None)
super(Descriptor, self).__init__(record, *args, **kwargs)
if record is not None:
if "AllowableQualifiersList" in record:
if "qualifiers" is not None:
if isinstance(
record["AllowableQualifiersList"]
["AllowableQualifier"], list):
self.allowable_qualifiers = [
qualifers[q["QualifierReferredTo"]["QualifierUI"]]
for q in record["AllowableQualifiersList"]
["AllowableQualifier"]
]
else:
self.allowable_qualifiers = [
qualifers[record["AllowableQualifiersList"]
["AllowableQualifier"]
["QualifierReferredTo"]["QualifierUI"]]
]
if "PharmacologicalActionList" in record:
if isinstance(
record["PharmacologicalActionList"]
["PharmacologicalAction"], list):
self.pharmacological_actions = [
pa["DescriptorReferredTo"]["DescriptorUI"]
for pa in record["PharmacologicalActionList"]
["PharmacologicalAction"]
]
else:
self.pharmacological_actions = [
record["PharmacologicalActionList"]
["PharmacologicalAction"]["DescriptorReferredTo"]
["DescriptorUI"]
]
if "PreviousIndexingList" in record:
if isinstance(record["PreviousIndexingList"], list):
self.previous_indexings = record["PreviousIndexingList"][
"PreviousIndexing"]
else:
self.previous_indexings = [
record["PreviousIndexingList"]["PreviousIndexing"]
]
if "TreeNumberList" in record:
if isinstance(record["TreeNumberList"]["TreeNumber"], list):
self.tree_numbers = record["TreeNumberList"]["TreeNumber"]
else:
self.tree_numbers = [
record["TreeNumberList"]["TreeNumber"]
]
self.parents = []
for tn in self.tree_numbers:
previous_node = None
for node in tn.split("."):
if previous_node is not None:
previous_node = ".".join([previous_node, node])
self.parents.append(previous_node)
else:
self.parents.append(node)
previous_node = node
class Cluster(EmbeddedDocument):
centroid = ListField(required=True)
num_possible_abnormal_events = IntField(required=True)
possible_abnormal_events = DictField()
class Task(AttributedDocument):
meta = {
"db_alias":
Database.backend,
"strict":
strict,
"indexes": [
"created",
"started",
"completed",
"parent",
"project",
("company", "name"),
("company", "type", "system_tags", "status"),
("company", "project", "type", "system_tags", "status"),
("status", "last_update"), # for maintenance tasks
{
"name":
"%s.task.main_text_index" % Database.backend,
"fields": [
"$name",
"$id",
"$comment",
"$execution.model",
"$output.model",
"$script.repository",
"$script.entry_point",
],
"default_language":
"english",
"weights": {
"name": 10,
"id": 10,
"comment": 10,
"execution.model": 2,
"output.model": 2,
"script.repository": 1,
"script.entry_point": 1,
},
},
],
}
id = StringField(primary_key=True)
name = StrippedStringField(required=True,
user_set_allowed=True,
sparse=False,
min_length=3)
type = StringField(required=True, choices=get_options(TaskType))
status = StringField(default=TaskStatus.created,
choices=get_options(TaskStatus))
status_reason = StringField()
status_message = StringField()
status_changed = DateTimeField()
comment = StringField(user_set_allowed=True)
created = DateTimeField(required=True, user_set_allowed=True)
started = DateTimeField()
completed = DateTimeField()
published = DateTimeField()
parent = StringField()
project = StringField(reference_field=Project, user_set_allowed=True)
output = EmbeddedDocumentField(Output, default=Output)
execution: Execution = EmbeddedDocumentField(Execution, default=Execution)
tags = ListField(StringField(required=True), user_set_allowed=True)
system_tags = ListField(StringField(required=True), user_set_allowed=True)
script = EmbeddedDocumentField(Script)
last_worker = StringField()
last_worker_report = DateTimeField()
last_update = DateTimeField()
last_iteration = IntField(default=DEFAULT_LAST_ITERATION)
last_metrics = SafeMapField(
field=SafeMapField(EmbeddedDocumentField(MetricEvent)))
metric_stats = SafeMapField(field=EmbeddedDocumentField(MetricEventStats))
class Article(Document):
""" 一篇文章内容，可以是 seeking alpha 中的一篇专栏文章，也可以是 twitter 上的一篇 post，
也可以是 google news search 的一篇结果等 """
uuid = StringField(max_length=32, primary_key=True)
"""各采编的数据处理函数自行决定该 hash 算法，只要全局不冲撞即可"""
title = StringField(required=True)
"""文章的标题，或者是 post 的贴文等"""
title_ana = EmbeddedDocumentField(AzureTextAna)
abstract = StringField()
"""新闻的摘要，或 搜索，或者 twitter 的短文内容 """
abstract_ana = EmbeddedDocumentField(AzureTextAna)
full_text_url = StringField()
"""全文的访问地址"""
related_image_url = StringField()
"""如果是 twitter ， 可能会有关联的图片 """
rating = IntField()
"""一些站点，如 seeking alpha 是有 rating 数的，这里记录该 rating 值"""
rating_change = IntField()
"""评级变动"""
pred_ret_this_yr = FloatField()
"""当年度return预测"""
pred_pe_this_yr = FloatField()
"""当年度pe预测"""
pred_ret_next_yr = FloatField()
"""下一年度return预测"""
pred_pe_next_yr = FloatField()
"""下一年度pe预测"""
publish_time = DateTimeField()
related_symbols = ListField(LazyReferenceField(FinancialInstrumentSymbol),
default=None)
engine_site = StringField()
""" 数据来自于， SeekingAlpha / GoogleNews / Twitter """
channel_in_site = StringField()
""" 文章属于站点中的哪个栏目下的 ， 包括可以使用 Search """
batch_action_uuid = StringField()
action_uuid = StringField()
from_searching_phase = LazyReferenceField(SearchingPhrase)
"""如果文章来源于一个搜索结果，则这里提供其中的一个搜索词"""
seeking_alpha_author = LazyReferenceField(AuthorInSeekingAlpha)
""" 文章如果来自于 seeking alpha ， 则这里是 seeking alpha 的作者信息 """
seeking_alpha_extra = EmbeddedDocumentField(SeekingAlphaArticleExtra)
""" 文章如果来自于 seeking alpha ， 是一些 comments 等信息 """
twitter_poster = LazyReferenceField(UserInTwitter)
"""article如果来自于 twitter , 则 填入"""
tweet_extra = EmbeddedDocumentField(TweetExtra)
"""tweet 的 comment / like / follow 等数据"""
meta = {
"indexes": [
"$title", # text index
"#batch_action_uuid",
"#action_uuid",
"-publish_time", # follower 降序
"-rating", # follower 降序
"#engine_site", # hash index site 可以快速的选择不同的源
"#channel_in_site", # 用于快速过滤来源
]
}
class Ticket(Document):
meta = dict(
collection='Tickets',
allow_inheritance=False,
indexes=['character.id'],
)
token = StringField(db_field='t')
character = EmbeddedDocumentField(Entity,
db_field='c',
default=lambda: Entity())
corporation = EmbeddedDocumentField(Entity,
db_field='o',
default=lambda: Entity())
alliance = EmbeddedDocumentField(Entity,
db_field='a',
default=lambda: Entity())
tags = ListField(StringField(), db_field='g', default=list)
password = PasswordField(db_field='pw', difficulty=0.25)
expires = DateTimeField(db_field='e')
seen = DateTimeField(db_field='s')
@property
def has_password(self):
return bool(self.password)
def __repr__(self):
return "<Ticket {0.id} \"{0.character.name}\">".format(self)
@classmethod
def authenticate(cls, identifier, password=None):
"""Validate the given identifier; password is ignored."""
api = API(config['api.endpoint'], config['api.identity'],
config['api.private'], config['api.public'])
result = api.core.info(identifier)
user = cls.objects(character__id=result.character.id).first()
if not user:
user = cls(token=identifier,
expires=result.expires,
seen=datetime.utcnow())
user.character.id = result.character.id
user.character.name = result.character.name
user.corporation.id = result.corporation.id
user.corporation.name = result.corporation.name
if result.alliance:
user.alliance.id = result.alliance.id
user.alliance.name = result.alliance.name
user.tags = result.tags if 'tags' in result else []
user.save()
else:
# TODO: Also update membership information.
user.update(set__token=identifier,
set__seen=datetime.utcnow(),
set__tags=result.get('tags', []))
return user.id, user
@classmethod
def lookup(cls, identifier):
"""Thaw current user data based on session-stored user ID."""
user = cls.objects(id=identifier).first()
if user:
user.update(set__seen=datetime.utcnow())
return user
class UserRoleMap(SimpleBaseDocument):
RoleName = StringField(required=True)
IdsAllowedToRead = ListField(StringField(required=True), default=[])
meta = {'collection': 'UserRoleMaps'}
class TaskStatus(Document, ReaperMixin):
"""
Represents a task.
This inherits from mongoengine.Document and defines the schema for the documents
in task_status collection. The documents in this collection may be reaped,
so it inherits from ReaperMixin.
:ivar task_id: identity of the task this status corresponds to
:type task_id: basestring
:ivar worker_name: The name of the worker that the Task is in
:type worker_name: basestring
:ivar tags: custom tags on the task
:type tags: list
:ivar state: state of callable in its lifecycle
:type state: basestring
:ivar error: Any errors or collections of errors that occurred while this task was running
:type error: dict (created from a PulpException)
:ivar spawned_tasks: List of tasks that were spawned during the running of this task
:type spawned_tasks: list of str
:ivar progress_report: A report containing information about task's progress
:type progress_report: dict
:ivar task_type: the fully qualified (package/method) type of the task
:type task_type: basestring
:ivar start_time: ISO8601 representation of the time the task started executing
:type start_time: basestring
:ivar finish_time: ISO8601 representation of the time the task completed
:type finish_time: basestring
:ivar result: return value of the callable, if any
:type result: any
:ivar exception: Deprecated. This is always None.
:type exception: None
:ivar traceback: Deprecated. This is always None.
:type traceback: None
"""
task_id = StringField(required=True)
worker_name = StringField()
tags = ListField(StringField())
state = StringField(choices=constants.CALL_STATES,
default=constants.CALL_WAITING_STATE)
error = DictField(default=None)
spawned_tasks = ListField(StringField())
progress_report = DictField()
task_type = StringField()
start_time = ISO8601StringField()
finish_time = ISO8601StringField()
result = DynamicField()
# These are deprecated, and will always be None
exception = StringField()
traceback = StringField()
# For backward compatibility
_ns = StringField(default='task_status')
meta = {
'collection': 'task_status',
'indexes':
['-tags', '-state', {
'fields': ['-task_id'],
'unique': True
}],
'allow_inheritance': False,
'queryset_class': CriteriaQuerySet
}
def save_with_set_on_insert(self, fields_to_set_on_insert):
"""
Save the current state of the TaskStatus to the database, using an upsert operation.
The upsert operation will only set those fields if this becomes an insert operation,
otherwise those fields will be ignored. This also validates the fields according to the
schema above.
This is required because the current mongoengine version we are using does not support
upsert with set_on_insert through mongoengine queries. Once we update to the version
which supports this, this method can be deleted and it's usages can be replaced
with mongoengine upsert queries.
:param fields_to_set_on_insert: A list of field names that should be updated with Mongo's
$setOnInsert operator.
:type fields_to_set_on_insert: list
"""
# If fields_to_set_on_insert is None or empty, just save
if not fields_to_set_on_insert:
self.save()
return
# This will be used in place of superclass' save method, so we need to call validate()
# explicitly.
self.validate()
stuff_to_update = dict(copy.deepcopy(self._data))
# Let's pop the $setOnInsert attributes out of the copy of self so that we can pass the
# remaining attributes to the $set operator in the query below.
set_on_insert = {}
for field in fields_to_set_on_insert:
set_on_insert[field] = stuff_to_update.pop(field)
task_id = stuff_to_update.pop('task_id')
update = {'$set': stuff_to_update, '$setOnInsert': set_on_insert}
TaskStatus._get_collection().update({'task_id': task_id},
update,
upsert=True)
@classmethod
def post_save(cls, sender, document, **kwargs):
"""
Send a taskstatus message on save.
:param sender: class of sender (unused)
:type sender: class
:param document: mongoengine document
:type document: mongoengine.Document
"""
send_taskstatus_message(document,
routing_key="tasks.%s" % document['task_id'])
class User(JsonMixin, DynamicDocument):
""" An account on 1base. """
email = EmailField(max_length=1024, required=True, unique=True)
password = StringField(max_length=1027, required=True)
username = StringField(max_length=120,
required=False,
unique=True,
sparse=True)
groups = ListField(ReferenceField(Group), default=list())
# attributes required by flask-login
# https://flask-login.readthedocs.io/en/latest/
# `is_authenticated` and `is_anonymous` are attibutes in constructor
is_active = BooleanField(default=False)
verification = StringField(default=token_urlsafe())
# Personal "business" information
sex = StringField(choices=SEX_CHOICES)
first_name = StringField(max_length=1024)
middle_name = StringField(max_length=1024)
last_name = StringField(max_length=1024)
birth_date = DateTimeField()
phone_number = StringField(max_length=48)
# Developer information
api_key = StringField()
# Frilly information
avatar = ForgivingURLField()
def __init__(self, *args, **kwargs):
""" Construct a new user. """
self.is_authenticated = False
self.is_anonymous = False
super(User, self).__init__(*args, **kwargs)
def get_id(self):
""" Get ID of the user.
Required by
`flask-login <https://flask-login.readthedocs.io/en/latest/>`_
"""
return self.id.encode('unicode')
def generate_api_key(self):
self.api_key = token_urlsafe()
@property
def all_permissions(self):
""" Return all permissions used by the user. """
perms = []
for g in self.groups:
perms.extend(g.permissions)
return list(tuple(perms))
@property
def is_admin(self):
""" Return True if user is in any ADMIN_GROUPS group. """
return len([g for g in self.groups if g.name in ADMIN_GROUPS]) > 0
def can_any(self, *permissions):
""" Return True if user can use any of the permissions.
NOTE: By default will return True if group name is listed in
onebase_common.settings.ADMIN_GROUPS
"""
if self.is_admin:
return True
ap = self.all_permissions
for p in permissions:
if p in ap:
return True
return False
def can_all(self, *permissions):
""" Return True if user can use *all* permissions listed.
NOTE: By default will return True if group name is listed in
onebase_common.settings.ADMIN_GROUPS
:param permissions: Permissions for the user.
"""
if self.is_admin:
return True
ap = self.all_permissions
for p in permissions:
if p not in ap:
return False
return True
def to_json(self):
return super(User, self).to_json(omit=['password'])
class SocialData(Document):
obs = StringField()
tags = ListField(StringField())
person = CachedReferenceField(Person, fields=("group", ))
class Exchange(Document):
parent_id = StringField()
name = StringField(required=True)
guild = StringField(required=True)
description = StringField()
channel_id = StringField()
engagement_id = StringField()
type_name = StringField()
characters = ListField(StringField())
opposition = ListField(StringField())
active_turn = StringField()
started_on = DateTimeField()
ended_on = DateTimeField()
archived = BooleanField(default=False)
history_id = StringField()
created_by = StringField()
created = DateTimeField(required=True)
updated_by = StringField()
updated = DateTimeField(required=True)
@classmethod
def post_save(cls, sender, document, **kwargs):
if document.history_id:
user = User().get_by_id(document.updated_by)
user.history_id = document.history_id
user.updated_by = document.updated_by
user.updated = document.updated
user.save()
print({'history_id': document.history_id})
else:
changes = document._delta()[0]
action = 'updated'
if 'created' in kwargs:
action = 'created' if kwargs['created'] else action
if action == 'updated' and 'archived' in changes:
action = 'archived' if changes['archived'] else 'restored'
Log().create_new(str(document.id), document.name,
document.updated_by, document.guild, 'Exchange',
changes, action)
user = User().get_by_id(document.updated_by)
if user.history_id:
user.history_id = None
user.updated_by = document.updated_by
user.updated = document.updated
user.save()
print(changes)
@staticmethod
def query():
return Exchange.objects
@staticmethod
def filter(**params):
return Exchange.objects.filter(**params)
def create_new(self, user, guild, channel_id, engagement_id, name,
archived):
self.name = name
self.guild = guild
self.parent_id = engagement_id
self.channel_id = channel_id
self.engagement_id = engagement_id
self.created_by = str(user.id)
self.created = T.now()
self.updated_by = str(user.id)
self.updated = T.now()
self.save()
return self
def find(self, guild, channel_id, engagement_id, name, archived=False):
filter = Exchange.objects(guild=guild,
channel_id=channel_id,
engagement_id=engagement_id,
name__icontains=name,
archived=archived)
exchange = filter.first()
return exchange
def get_or_create(self,
user,
guild,
channel,
engagement,
name,
archived=False):
exchange = self.find(guild, str(channel.id), str(engagement.id), name,
archived)
if exchange is None:
exchange = self.create_new(user, guild, str(channel.id),
str(engagement.id), name, archived)
exchange.character = Character().get_or_create(
user, name, guild, exchange, 'Exchange', archived)
exchange.save()
return exchange
def get_by_id(self, id):
exchange = Exchange.objects(id=id).first()
return exchange
@classmethod
def get_by_channel(cls, channel, archived=False, page_num=1, page_size=5):
if page_num:
offset = (page_num - 1) * page_size
items = cls.filter(
channel_id=str(channel.id),
archived=archived).skip(offset).limit(page_size).all()
else:
items = cls.filter(channel_id=str(channel.id),
archived=archived).order_by('name',
'created').all()
return items
@classmethod
def get_by_engagement(cls,
engagement,
archived=False,
page_num=1,
page_size=5):
if page_num:
offset = (page_num - 1) * page_size
items = cls.filter(
engagement_id=str(engagement.id),
archived=archived).skip(offset).limit(page_size).all()
else:
items = cls.filter(engagement_id=str(engagement.id),
archived=archived).order_by('name',
'created').all()
return items
@classmethod
def get_by_page(cls, params, page_num=1, page_size=5):
if page_num:
offset = (page_num - 1) * page_size
logs = cls.filter(**params).order_by(
'name', 'created').skip(offset).limit(page_size).all()
else:
logs = cls.filter(**params).order_by('name', 'created').all()
return logs
@classmethod
def get_by_parent(cls, **params):
items = cls.filter(**params).all()
return [items] if items else []
def archive(self, user):
self.reverse_archive(user)
self.archived = True
self.updated_by = str(user.id)
self.updated = T.now()
self.save()
def reverse_archive(self, user):
for z in Exchange().get_by_parent(parent_id=str(self.id)):
z.reverse_archive(user)
z.archived = True
z.updated_by = str(user.id)
z.updated = T.now()
z.save()
def restore(self, user):
self.reverse_restore(user)
self.archived = False
self.updated_by = str(user.id)
self.updated = T.now()
self.save()
def reverse_restore(self, user):
for z in Exchange().get_by_parent(parent_id=str(self.id)):
z.reverse_restore(user)
z.archived = False
z.updated_by = str(user.id)
z.updated = T.now()
z.save()
def get_string_characters(self, channel=None):
characters = [Character.get_by_id(id) for id in self.characters]
characters = '***\n ***'.join(c.name for c in characters
if c)
return f' _Characters:_\n ***{characters}***'
def get_short_string_characters(self, channel=None):
characters = [Character.get_by_id(id) for id in self.characters]
characters = ', '.join(c.name for c in characters if c)
return f' _({characters})_'
def get_string_opposition(self, channel=None):
opposition = [Character.get_by_id(id) for id in self.opposition]
opposition = '***\n ***'.join(c.name for c in opposition
if c)
return f' _Opposition:_\n ***{opposition}***'
def get_short_string_opposition(self, channel=None):
opposition = [Character.get_by_id(id) for id in self.opposition]
opposition = ', '.join(c.name for c in opposition if c)
return f' _({opposition})_'
def get_string(self, channel, user=None):
name = f'***{self.name}***'
active = ''
if channel:
active = ' _(Active Exchange)_ ' if str(
self.id) == channel.active_exchange else ''
start = ''
if self.started_on:
start = f'\n_Started On:_ ***{T.to(self.started_on, user)}***' if self.started_on else ''
end = ''
if self.ended_on:
end = f'\n_Ended On:_ ***{T.to(self.ended_on, user)}***' if self.ended_on else ''
description = f' - "{self.description}"' if self.description else ''
characters = f'\n\n{self.get_string_characters()}' if self.characters else ''
opposition = f'\n\n{self.get_string_opposition()}' if self.opposition else ''
aspects = ''
stress = ''
if self.character:
name = f'***{self.character.name}***' if self.character.name else name
description = f' - "{self.character.description}"' if self.character.description else description
aspects = self.character.get_string_aspects()
stress = self.character.get_string_stress(
) if self.character.has_stress else ''
return f' {name}{active}{start}{end}{description}{characters}{opposition}{aspects}{stress}'
def get_short_string(self, channel=None):
name = f'***{self.name}***'
active = ''
if channel:
active = f' _(Active {str(self.type_name).title()})_ ' if str(
self.id
) == channel.active_exchange else f' _({str(self.type_name).title()})_ '
characters = f'\n{self.get_short_string_characters()}' if self.characters else ''
opposition = f' v. {self.get_short_string_opposition()}' if self.opposition else ''
description = f' - "{self.description}"' if self.description else ''
if self.character:
name = f'***{self.character.name}***' if self.character.name else name
description = f' - "{self.character.description}"' if self.character.description else ''
return f' {name}{active}{description}{characters}{opposition}'
class Export(ScheduleEntry):
SCHEDULED_TASK = 'core.exports.export.execute_export'
CUSTOM_FILTER = Q()
include_tags = ListField(ReferenceField(Tag, reverse_delete_rule=PULL))
exclude_tags = ListField(ReferenceField(Tag, reverse_delete_rule=PULL))
ignore_tags = ListField(ReferenceField(Tag, reverse_delete_rule=PULL))
output_dir = StringField(default='exports')
acts_on = StringField(verbose_name="Acts on", required=True)
template = ReferenceField(ExportTemplate)
hash_md5 = StringField(max_length=32)
def __init__(self, *args, **kwargs):
super(Export, self).__init__(*args, **kwargs)
if not os.path.exists(self.output_dir):
os.makedirs(self.output_dir)
@property
def output_file(self):
return os.path.abspath(os.path.join(self.output_dir, self.name))
@property
def content_uri(self):
return url_for("api.Export:content", id=str(self.id))
def execute(self):
q_include = Q()
for t in self.include_tags:
q_include |= Q(tags__match={'name': t.name, 'fresh': True})
q_exclude = Q(tags__name__nin=[t.name for t in self.exclude_tags])
q = Q(tags__not__size=0,
tags__match={'fresh': True}) & q_include & q_exclude & Q(
_cls="Observable.{}".format(self.acts_on))
return self.template.render(
self.filter_ignore_tags(Observable.objects(q).no_cache()),
self.output_file)
def filter_ignore_tags(self, elements):
ignore = set([t.name for t in self.ignore_tags])
for e in elements:
if set([t.name for t in e.tags]) - ignore:
yield e
def info(self):
i = {
k: v
for k, v in self._data.items() if k in [
"name", "output_dir", "enabled", "description", "status",
"last_run", "ignore_tags", "include_tags", "exclude_tags"
]
}
i['frequency'] = str(self.frequency)
i['id'] = str(self.id)
i['ignore_tags'] = [tag.name for tag in self.ignore_tags]
i['include_tags'] = [tag.name for tag in self.include_tags]
i['exclude_tags'] = [tag.name for tag in self.exclude_tags]
i['template'] = self.template.name
i['acts_on'] = self.acts_on
i['content_uri'] = self.content_uri
return i
class Tool(Document):
name = StringField(required=True, unique=True)
description = StringField()
display_name = StringField()
versions = ListField(ReferenceField(ToolVersion))
edam_operations = ListField(ReferenceField(EDAMOperation))
meta = {'collection': 'tools'}
@classmethod
def retrieve_tools_from_instance(cls, instance):
galaxy_instance = GalaxyInstance(url=instance.url)
tool_client = ToolClient(galaxy_instance)
for element in tool_client.get_tools():
if element['model_class'] == 'Tool':
tool_name = element['id']
if '/' in tool_name:
tool_name = tool_name.split('/')[-2]
try:
tool = Tool.objects.get(name=tool_name)
except Tool.DoesNotExist:
tool = Tool(name=tool_name)
tool.description = element['description']
tool.display_name = element['name']
for edam_opetation_id in element['edam_operations']:
edam_operation = EDAMOperation.get_from_id(
edam_opetation_id, allow_creation=True)
if edam_operation is not None and edam_operation not in tool.edam_operations:
tool.edam_operations.append(edam_operation)
try:
if 'tool_shed_repository' in element:
tool_version = ToolVersion.objects.get(
name=tool_name,
changeset=element['tool_shed_repository']
['changeset_revision'],
tool_shed=element['tool_shed_repository']
['tool_shed'],
owner=element['tool_shed_repository']['owner'])
else:
tool_version = ToolVersion.objects.get(
name=tool_name,
version=element['version'],
tool_shed=None,
owner=None)
except ToolVersion.DoesNotExist:
tool_version = ToolVersion(name=tool_name,
version=element['version'])
if 'tool_shed_repository' in element:
tool_version.changeset = element['tool_shed_repository'][
'changeset_revision']
tool_version.tool_shed = element['tool_shed_repository'][
'tool_shed']
tool_version.owner = element['tool_shed_repository'][
'owner']
if instance not in tool_version.instances:
tool_version.instances.append(instance)
tool_version.save()
if tool_version not in tool.versions:
tool.versions.append(tool_version)
tool.save()
@classmethod
def search(cls, search):
if search is None or len(search) == 0:
return []
q = None
instances = []
nodes = parse_search_query(search)
for node in nodes:
q_part = None
if type(node) == ComparisonNode:
key = node[0]
value = u" ".join(node[2])
if key == u"topic":
try:
edam_operation = EDAMOperation.objects().get(
label=value)
except EDAMOperation.DoesNotExist:
return []
else:
q_part = Q(edam_operations=edam_operation)
elif key == u"instance":
try:
instance = Instance.objects().get(brand=value)
except Instance.DoesNotExist:
return []
else:
instances.append(instance)
else:
# unknown key
return []
else:
term = " ".join(node)
q_part = (Q(name__icontains=term)
| Q(description__icontains=term)
| Q(display_name__icontains=term))
if q_part is not None:
if q is None:
q = q_part
else:
q = q & q_part
tools = Tool.objects(q)
selected_tools = []
if len(instances) > 0:
for tool in tools:
for version in tool.versions:
if len(filter(lambda x: x in version.instances,
instances)) == len(instances):
selected_tools.append(tool)
else:
selected_tools = tools
return selected_tools
@classmethod
def update_catalog(cls):
# delete all Tool and ToolVersion documents
ToolVersion.objects().delete()
Tool.objects().delete()
for instance in Instance.objects():
Instance.add_instance(url=instance.url)
class Entity(Node):
SEARCH_ALIASES = {
"name": "aliases",
}
VERB_DICT = {
"Malware": {"Actor": "Used by", "TTP": "Leverages"},
"Actor": {"Malware": "Uses", "TTP": "Leverages"},
"Company": {},
"TTP": {"Actor": "Leveraged by", "Malware": "Observed in"},
}
DISPLAY_FIELDS = [("name", "Name"), ("tags", "Tags")]
name = StringField(
verbose_name="Name",
required=True,
unique_with="_cls",
sparse=True,
max_length=1024,
)
description = StringField(verbose_name="Description")
tags = ListField(StringField(), verbose_name="Relevant tags")
meta = {
"allow_inheritance": True,
"indexes": ["tags"],
"ordering": ["name"],
}
def clean(self):
tags = []
for t in self.tags:
if t:
tags.append(Tag.get_or_create(name=t.lower().strip()))
self.tags = [t.name for t in tags]
@classmethod
def get_form(klass, override=None):
if override:
klass = override
form = model_form(klass, exclude=klass.exclude_fields)
form.tags = TagListField("Tags that will link to this entity")
form.links = EntityListField("Bind to entities")
return form
def __unicode__(self):
return "{}".format(self.name)
def action(self, target, source, verb=None):
if not verb:
if self.__class__.name == target.__class__.__name__:
verb = "Related {}".format(self.__class__.__name__)
else:
verb = Entity.VERB_DICT.get(self.__class__.__name__, {}).get(
target.__class__.__name__, "Relates to"
)
self.active_link_to(target, verb, source)
def generate_tags(self):
return []
def info(self):
"""Object info.
When there is no Flask context, url and human_url are not returned and the
object id is returned instead.
"""
i = {
"name": self.name,
"description": self.description,
"tags": self.tags,
"id": str(self.id),
}
try:
i["url"] = url_for("api.Entity:post", id=str(self.id), _external=True)
i["human_url"] = url_for(
"frontend.EntityView:get", id=str(self.id), _external=True
)
except RuntimeError:
# No flask context, so we can't generate links
pass
return i
class Country(Document):
"""
CH4 - methane
GNI - gross national income
GINI - gini index
Agriculture_Percentage - percentage of land for agriculture
"""
id = IntField(required=True, primary_key=True)
Name = StringField(required=True, max_length=50)
Population = ListField(EmbeddedDocumentField(Year))
CO2 = ListField(EmbeddedDocumentField(Year))
CH4 = ListField(EmbeddedDocumentField(Year))
GNI = ListField(EmbeddedDocumentField(Year))
GINI = ListField(EmbeddedDocumentField(Year))
Agriculture_Percentage = ListField(EmbeddedDocumentField(Year))
Renewable_Percentage = ListField(EmbeddedDocumentField(Year))
Fossil_Fuel_Percentage = ListField(EmbeddedDocumentField(Year))
def __init__(self, *args, **kwargs):
super(Country, self).__init__(*args, **kwargs)
def to_dict(self, indicators, start_year, end_year):
global ALL_INDICATORS
r = {
'Name': self.Name
}
if indicators is None or len(indicators) == 0:
indicators = ALL_INDICATORS
for i in indicators:
if i in CALC_INDICATORS.keys():
params = {}
ind = CALC_INDICATORS[i]
for param in ind.params:
params[param] = {}
for param in ind.params:
for year in getattr(self, param):
if start_year <= year.Year <= end_year:
params[param][year.Year] = year.Value
r[i] = []
has_results = True
for p in params.values():
if len(p) == 0:
has_results = False
break
if not has_results:
continue
year = start_year
while year <= end_year:
if reduce((lambda x, y: x and y), map((lambda p: year in params[p]), ind.params)):
p = [params[p][year] for p in ind.params]
r[i].append({'year': year, 'value': ind.run(p)})
else:
r[i].append({'year': year, 'value': -1})
year += 1
else:
r[i] = [yr.to_dict() for yr in getattr(self, i) if start_year <= yr.Year <= end_year]
return r
def get_values_list(self, indicator, start_year, end_year):
'''
year = end_year - index
list is in reverse order (from end year to start year)
:param indicator:
:param start_year:
:param end_year:
:return:
'''
data = []
if getattr(self, indicator) == []:
data = [-1] * (end_year-start_year)
else:
[data.append(yr.Value) for yr in getattr(self, indicator) if start_year <= yr.Year <= end_year]
output = {'start_year': start_year, 'end_year': end_year, 'data': data}
return output
class BlogPost(Document):
comments = ListField(
GenericEmbeddedDocumentField(choices=(Comments, )))
class Basket(Document):
products = ListField(CachedReferenceField(Product,
fields=["name"]))
class Supplier(Document):
"""
Suppliers data schema: https://github.com/nyu-devops-fall19-suppliers/suppliers/issues/21
"""
logger = logging.getLogger('flask.app')
app = None
# Table Schema
supplierName = StringField(required=True)
address = StringField(required=False)
productIdList = ListField(StringField(), required=False)
averageRating = IntField(required=False)
def __repr__(self):
return '<Supplier %r>' % (self.supplierName)
# def save(self):
# """
# Saves a Supplier to the data store
# """
# Supplier.logger.info('Saving %s', self.supplierName)
# self.save()
# Deprecated function. Use supplier.to_json() instead
@classmethod
def init_db(cls, app):
""" Initializes the database session """
cls.logger.info('Initializing database')
cls.app = app
DB_URI = "mongodb+srv://suppliers:[email protected]/myDatabase?retryWrites=true&w=majority"
# This is where we initialize mongoDB from the Flask app
connect('myDatabase', host=DB_URI)
app.app_context().push()
@classmethod
def all(cls):
"""This is a function to return all suppliers"""
cls.logger.info('Processing all suppliers')
return cls.objects()
@classmethod
def find_by_name(cls, supplier_name):
""" Find a supplier by its name """
cls.logger.info('Processing looking for name %s', supplier_name)
try:
res = cls.objects.get(supplierName=supplier_name)
except DoesNotExist:
return None
return res
@classmethod
def find(cls, supplier_id):
"""Retrieves a single supplier with a given id (supplierID) """
cls.logger.info('Getting supplier with id: %s', supplier_id)
try:
res = cls.objects(id=supplier_id).first()
except ValidationError:
return None
return res
@classmethod
def find_by_product(cls, product_id):
"""Retrieves a list of supplier with a given product id """
cls.logger.info(
"Getting suppliers with product id: %s".format(product_id))
res = cls.objects(productIdList__in=product_id)
return res
@classmethod
def find_by_rating(cls, rating):
"""Retrieves a list of supplier with a given rating score """
cls.logger.info(
"Getting suppliers with ratting score greater than: %d".format(
rating))
res = cls.objects(averageRating__gte=rating)
return res
@classmethod
def find_by_equals_to_rating(cls, rating):
"""Retrieves a list of supplier with a given rating score """
cls.logger.info(
"Getting suppliers with ratting score equals to: %d".format(
rating))
res = cls.objects(averageRating=rating)
return res
@classmethod
def action_make_recommendation(cls, product_id):
"""Retrieves a list of supplier with a given rating score and product id """
cls.logger.info(
"Getting suppliers with ratting score greater than: %s".format(
product_id))
res = cls.objects(
Q(productIdList__in=product_id)
& Q(averageRating__gte=3))
return res
class Owner(EmbeddedDocument):
name = StringField()
tags = ListField(StringField())
class User(Document):
username = StringField(required=True)
website = URLField()
tags = ListField(StringField(max_length=16))
# 添加到User
profile = EmbeddedDocumentField(Profile)
class Job(db.Document):
created_at = DateTimeField(required=True)
last_modified_at = DateTimeField(required=True,
default=datetime.datetime.now)
job_id = StringField(required=True)
executions = ListField(EmbeddedDocumentField(JobExecution))
task = ReferenceField("Task",
required=True,
reverse_delete_rule=mongoengine.CASCADE)
metadata = DictField(required=False)
scheduled = BooleanField(required=True, default=False)
def save(self, *args, **kwargs):
if self.executions is None:
self.executions = []
if not self.created_at:
self.created_at = datetime.datetime.utcnow()
self.last_modified_at = datetime.datetime.utcnow()
return super(Job, self).save(*args, **kwargs)
def create_execution(self, image, command):
ex_id = str(uuid4())
ex = JobExecution(
execution_id=ex_id,
image=image,
command=command,
created_at=datetime.datetime.utcnow(),
)
self.executions.append(ex)
self.save()
return ex
def get_metadata(self, blacklist):
if "envs" in self.metadata:
envs = {}
for key, val in self.metadata["envs"].items():
for word in blacklist:
if word in key.lower():
val = "*" * len(str(val))
break
envs[key] = val
self.metadata["envs"] = envs
return self.metadata
def to_dict(
self,
include_log=False,
include_error=False,
include_executions=True,
blacklist=None,
):
if blacklist is None:
blacklist = []
meta = self.get_metadata(blacklist)
res = {
"createdAt": self.created_at.isoformat(),
"lastModifiedAt": self.last_modified_at.isoformat(),
"taskId": self.task.task_id,
"scheduled": self.scheduled,
"executionCount": len(self.executions),
"metadata": meta,
}
if include_executions:
executions = [
ex.to_dict(include_log, include_error)
for ex in self.executions
]
res["executions"] = executions
return res
@classmethod
def get_by_id(cls, task_id, job_id):
from fastlane.models.task import Task
if task_id is None or task_id == "" or job_id is None or job_id == "":
raise RuntimeError(
"Task ID and Job ID are required and can't be None or empty.")
t = Task.objects(task_id=task_id).first()
j = cls.objects(task=t, job_id=job_id).first()
return j
def get_execution_by_id(self, execution_id):
for job_execution in self.executions:
if job_execution.execution_id == execution_id:
return job_execution
return None
def get_last_execution(self):
if not self.executions:
return None
return self.executions[-1]
class Garden(MongoModel, Document):
brewtils_model = brewtils.models.Garden
name = StringField(required=True, default="default")
status = StringField(default="INITIALIZING")
status_info = EmbeddedDocumentField("StatusInfo", default=StatusInfo())
namespaces = ListField()
connection_type = StringField()
connection_params = DictField()
systems = ListField(ReferenceField(System, reverse_delete_rule=PULL))
meta = {
"auto_create_index":
False, # We need to manage this ourselves
"index_background":
True,
"indexes": [
{
"name": "unique_index",
"fields": ["name"],
"unique": True
},
{
"name": "local_unique_index",
"fields": ["connection_type"],
"unique": True,
"partialFilterExpression": {
"connection_type": "LOCAL"
},
},
],
}
def deep_save(self):
if self.connection_type != "LOCAL":
self._update_associated_systems()
self.save()
def _update_associated_systems(self):
"""If the call to the `deep_save` method is on a child garden object, we ensure
that when saving the systems, unknowns are deleted."""
# import moved here to avoid a circular import loop
from beer_garden.systems import get_systems, remove_system
logger = logging.getLogger(self.__class__.__name__)
def _get_system_triple(system: System) -> Tuple[str, str, str]:
return (
system.namespace,
system.name,
system.version,
)
our_namespaces = set(self.namespaces).union(
set(map(attrgetter("namespace"), self.systems)))
# we leverage the fact that systems must be unique up to the triple of their
# namespaces, names and versions
child_systems_already_known = {
_get_system_triple(system): str(system.id)
for system in get_systems(filter_params={
"local": False,
"namespace__in": our_namespaces
})
}
for system in self.systems:
triple = _get_system_triple(system)
if triple in child_systems_already_known:
system_id_to_remove = child_systems_already_known.pop(triple)
if system_id_to_remove != str(system.id):
# remove the system from before this update with the same triple
logger.debug(
f"Removing System <{triple[0]}"
f", {triple[1]}"
f", {triple[2]}> with ID={system_id_to_remove}"
f"; doesn't match ID={str(system.id)}"
" for known system with same attributes")
remove_system(system_id=system_id_to_remove)
system.save()
# if there's anything left over, delete those too; this could occur, e.g.,
# if a child system deleted a particular version of a plugin and installed
# another version of the same plugin
for bad_system_id in child_systems_already_known.values():
logger.debug(
f"Removing System with ID={str(bad_system_id)} because it "
f"matches no known system in child garden ({self.name})")
remove_system(system_id=bad_system_id)
class FeatureModel(BaseModel):
"""
待办跟踪
"""
title = StringField(required=True,
verbose_name='标题',
max_length=50,
min_length=2)
desc = StringField(required=True, verbose_name='需求描述', min_length=2)
demander = StringField(required=True, verbose_name='需求方', min_length=2)
priority = StringField(required=True,
verbose_name='优先级',
choices=PriorityEnum.choices(),
default=PriorityEnum.LOW)
status = StringField(required=True,
verbose_name='需求状态',
choices=ProcessStatusEnum.choices(),
default=ProcessStatusEnum.UN_STARTED)
expect_deadline = DateTimeField(required=True, verbose_name='预计完成时间')
actual_deadline = DateTimeField(required=False,
verbose_name='实际完成时间',
null=True)
submitter = ListField(required=True, verbose_name='提交人')
implementer = StringField(required=True, verbose_name='执行人')
meta = {
'collection': 'feature',
'verbose_name': '待办项跟踪',
'indexes': [
'submitter',
'implementer',
'updated_time',
'created_time',
]
}
@property
def status_render(self):
return ProcessStatusEnum[self.status].desc
@property
def priority_render(self):
return PriorityEnum[self.priority].desc
@property
def expect_deadline_render(self):
return time2str(self.expect_deadline)
@property
def actual_deadline_render(self):
if self.actual_deadline:
return time2str(self.actual_deadline)
return '-'
@property
def submitter_render(self):
if not self.submitter:
return '-'
user_info = UserService.batch_get_user_by_email(self.submitter)
names = []
for email in self.submitter:
name = (user_info.get(email) or {}).get('name')
if name:
names.append(name)
return '，'.join(names) or '-'
class Movie(Document):
info = DictField()
tags = ListField(DictField())
class HasItems(Document):
item_ids = ListField(ObjectIdField(), db_field='itemIds')
meta = {'allow_inheritance': True, 'abstract': True}
class MovingAverage(Document):
title = collection
window = sorted(window_list)
redis_key_for_latest = "moving_average_%s_latest" % collection
meta = {"db_alias": "moving_average", "collection": collection}
time = DateTimeField(primary_key=True)
ma = ListField(FloatField())
value = FloatField()
@classmethod
def generate_from_timeline(cls, timeline, time_begin, time_end):
value_list = pandas.Series([v for t, v in timeline])
window_ma = {w: value_list.rolling(w, min_periods=1).mean() for w in cls.window}
assert(timeline[0][0] <= time_begin)
assert(timeline[-1][0] >= time_end - datetime.timedelta(minutes=1))
for index, (t, v) in enumerate(timeline):
if time_begin <= t < time_end:
cls(time=t, value=v, ma=[window_ma[w][index] for w in cls.window]).save()
redis_client.set(cls.redis_key_for_latest, simplejson.dumps(get_latest(cls).ma), ex=90)
@classmethod
def get_prediction(cls, wait_model, time_begin, time_end):
data = list(cls.objects(time__gte=time_begin, time__lt=time_end).order_by("time"))
window_diff = numpy.array([[r.value / r.ma[win_index] - 1 for r in data] for win_index, w in enumerate(cls.window)])
return numpy.matmul(wait_model["reversion"], window_diff)
@classmethod
def test_reversion(cls, model, time_begin, time_end):
data = list(cls.objects(time__gte=time_begin, time__lt=time_end).order_by("time"))
window_diff = numpy.array([[r.value / r.ma[win_index] - 1 for r in data] for win_index, w in enumerate(cls.window)])
for wait in TRAIN_WAIT_LIST:
wait_model = model.get(str(wait))
if not wait_model:
wait_model = model[wait]
logging.info("%s wait:%s %s", cls.title, wait, wait_model)
label = numpy.array([after.value / before.value - 1 for before, after in zip(data, data[wait:])])
predict = sum(alpha * diff[:-wait] for alpha, diff in zip(wait_model["reversion"], window_diff)) * 0.5
label_len = numpy.linalg.norm(label)
after_len = numpy.linalg.norm(label + predict)
logging.info("%s wait:%s before:%s / after:%s = %s", cls.title, wait, label_len, after_len, after_len / label_len)
@classmethod
def generate_reversion(cls, time_begin, time_end):
data = list(cls.objects(time__gte=time_begin, time__lt=time_end).order_by("time"))
model = {}
window_diff = numpy.array([[r.value / r.ma[win_index] - 1 for r in data] for win_index, w in enumerate(cls.window)])
for wait in TRAIN_WAIT_LIST:
label = numpy.array([after.value / before.value - 1 for before, after in zip(data, data[wait:])])
local_diff = window_diff[:, :-wait]
window_weight = lsq_linear(-local_diff.transpose(), label, bounds=(0, 1)).x
label_diff = window_diff[:, wait:]
local_predict = numpy.matmul(window_weight, local_diff)
label_predict = numpy.matmul(window_weight, label_diff)
remain = numpy.dot(local_predict, label_predict) / (numpy.linalg.norm(local_predict) ** 2)
_, std = norm.fit(label_predict - local_predict * remain)
model[wait] = {"reversion": list(window_weight), "std": std, "remain": remain}
cls.test_reversion(model, time_begin, time_end)
return model
@classmethod
def time_dict(cls, time_begin, time_end):
return {o.time: o for o in cls.objects(time__gte=time_begin, time__lt=time_end)}
@classmethod
def latest(cls):
return cls(ma=simplejson.loads(redis_client.get(cls.redis_key_for_latest)))
def numpy_ma(self):
try:
return self._numpy_ma
except AttributeError:
self._numpy_ma = numpy.array(self.ma)
return self._numpy_ma
def calculate_offset(self, mid, wait_model):
return numpy.matmul(mid - self.numpy_ma(), wait_model["reversion"])
class Record(Document):
posts = ListField(EmbeddedDocumentField(Post))
class MyCustomers(Document):
"""每日度量数据快照"""
date_str = StringField(verbose_name = "日期字符串", required=True)
psuser_id = IntField(verbose_name = "员工ID", required = True)
rjjh_sub = ListField(verbose_name = '当日服务类目订单', field = EmbeddedDocumentField(SubscribeAmount), required = True)
ztc_sub = ListField(verbose_name = '当日服务直通车订单', field = EmbeddedDocumentField(SubscribeAmount), required = True)
zz_sub = ListField(verbose_name = '当日服务钻展订单', field = EmbeddedDocumentField(SubscribeAmount), required = True)
zx_sub = ListField(verbose_name = '当日服务装修订单', field = EmbeddedDocumentField(SubscribeAmount), required = True)
dyy_sub = ListField(verbose_name = '当日服务代运营订单', field = EmbeddedDocumentField(SubscribeAmount), required = True)
seo_sub = ListField(verbose_name = '当日服务seo订单', field = EmbeddedDocumentField(SubscribeAmount), required = True)
new_cust_sub = ListField(verbose_name = '当日新增账户订单', field = EmbeddedDocumentField(SubscribeAmount), required = True)
pause_sub = ListField(verbose_name = '当日暂停订单', field = EmbeddedDocumentField(SubscribeAmount), required = True)
unsub_sub = ListField(verbose_name = '当日退款订单', field = EmbeddedDocumentField(SubscribeAmount), required = True)
expire_sub = ListField(verbose_name = '当日流失订单', field = EmbeddedDocumentField(SubscribeAmount), required = True)
change_sub = ListField(verbose_name = '当日更换操作人订单', field = EmbeddedDocumentField(SubscribeAmount), required = True)
create_time = DateTimeField(verbose_name = "创建时间", default = datetime.datetime.now)
meta = {'collection': 'ncrm_my_customers', 'shard_key': ['date_str'], 'indexes': ['psuser_id']}
class Person(Document):
name = StringField()
likes = ListField(GenericEmbeddedDocumentField(choices=(Dish, )))
class Monkey(Document):
"""
This class has 2 main section:
* The schema section defines the DB fields in the document. This is the data of the
object.
* The logic section defines complex questions we can ask about a single document which
are asked multiple
times, somewhat like an API.
"""
# SCHEMA
guid = StringField(required=True)
config = EmbeddedDocumentField("Config")
creds = ListField(EmbeddedDocumentField("Creds"))
dead = BooleanField()
description = StringField()
hostname = StringField()
ip_addresses = ListField(StringField())
launch_time = StringField()
keepalive = DateTimeField()
modifytime = DateTimeField()
# TODO make "parent" an embedded document, so this can be removed and the schema explained (
# and validated) verbosely.
# This is a temporary fix, since mongoengine doesn't allow for lists of strings to be null
# (even with required=False of null=True).
# See relevant issue: https://github.com/MongoEngine/mongoengine/issues/1904
parent = ListField(ListField(DynamicField()))
config_error = BooleanField()
critical_services = ListField(StringField())
pba_results = ListField()
ttl_ref = ReferenceField(MonkeyTtl)
tunnel = ReferenceField("self")
command_control_channel = EmbeddedDocumentField(CommandControlChannel)
# Environment related fields
environment = StringField(
default=environment_names.Environment.UNKNOWN.value,
choices=environment_names.ALL_ENVIRONMENTS_NAMES,
)
aws_instance_id = StringField(
required=False
) # This field only exists when the monkey is running on an AWS
# instance. See https://github.com/guardicore/monkey/issues/426.
# LOGIC
@staticmethod
def get_single_monkey_by_id(db_id):
try:
return Monkey.objects.get(id=db_id)
except DoesNotExist as ex:
raise MonkeyNotFoundError("info: {0} | id: {1}".format(ex, str(db_id)))
@staticmethod
# See https://www.python.org/dev/peps/pep-0484/#forward-references
def get_single_monkey_by_guid(monkey_guid) -> "Monkey":
try:
return Monkey.objects.get(guid=monkey_guid)
except DoesNotExist as ex:
raise MonkeyNotFoundError("info: {0} | guid: {1}".format(ex, str(monkey_guid)))
@staticmethod
def get_latest_modifytime():
if Monkey.objects.count() > 0:
return Monkey.objects.order_by("-modifytime").first().modifytime
return None
def is_dead(self):
monkey_is_dead = False
if self.dead:
monkey_is_dead = True
else:
try:
if MonkeyTtl.objects(id=self.ttl_ref.id).count() == 0:
# No TTLs - monkey has timed out. The monkey is MIA.
monkey_is_dead = True
except (DoesNotExist, AttributeError):
# Trying to dereference unknown document - the monkey is MIA.
monkey_is_dead = True
return monkey_is_dead
def get_os(self):
os = "unknown"
if self.description.lower().find("linux") != -1:
os = "linux"
elif self.description.lower().find("windows") != -1:
os = "windows"
return os
@ring.lru()
@staticmethod
def get_label_by_id(object_id):
current_monkey = Monkey.get_single_monkey_by_id(object_id)
label = Monkey.get_hostname_by_id(object_id) + " : " + current_monkey.ip_addresses[0]
if len(set(current_monkey.ip_addresses).intersection(local_ip_addresses())) > 0:
label = "MonkeyIsland - " + label
return label
@ring.lru()
@staticmethod
def get_hostname_by_id(object_id):
"""
:param object_id: the object ID of a Monkey in the database.
:return: The hostname of that machine.
:note: Use this and not monkey.hostname for performance - this is lru-cached.
"""
return Monkey.get_single_monkey_by_id(object_id).hostname
def set_hostname(self, hostname):
"""
Sets a new hostname for a machine and clears the cache for getting it.
:param hostname: The new hostname for the machine.
"""
self.hostname = hostname
self.save()
Monkey.get_hostname_by_id.delete(self.id)
Monkey.get_label_by_id.delete(self.id)
def get_network_info(self):
"""
Formats network info from monkey's model
:return: dictionary with an array of IP's and a hostname
"""
return {"ips": self.ip_addresses, "hostname": self.hostname}
@ring.lru(
# data has TTL of 1 second. This is useful for rapid calls for report generation.
expire=1
)
@staticmethod
def is_monkey(object_id):
try:
_ = Monkey.get_single_monkey_by_id(object_id)
return True
except: # noqa: E722
return False
@staticmethod
def get_tunneled_monkeys():
return Monkey.objects(tunnel__exists=True)
def renew_ttl(self, duration=DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS):
self.ttl_ref = create_monkey_ttl_document(duration)
self.save()
class Presentation(EmbeddedDocument):
gravity = IntField()
parity = EmbeddedDocumentField(Parity)
last_menstrual_period = StringField()
presenting_symptoms = ListField(StringField())
