class Stores(me.Document):
created_at = me.DateTimeField(default=datetime.now())
name = me.StringField(required=True)
phone = me.StringField(require=True)
type_ = me.StringField(required=True)
sub_type = me.StringField(required=True)
is_active = me.BooleanField(default=True)
location = me.MapField(me.EmbeddedDocumentListField(Locations))
owner_id = me.StringField()
products = me.EmbeddedDocumentListField(Product)
web_info = me.MapField(me.EmbeddedDocumentListField(Web))
schedule = me.StringField()
meta = {'db_alias': 'core', 'collection': 'stores'}
class User(document.Document):
meta = {'allow_inheritance': True}
_hidden = ['password', 'completion_token']
type = orm.StringField()
name = orm.StringField(required=True)
image = orm.URLField()
email = orm.EmailField(required=True, unique=True)
password = orm.StringField()
gender = orm.StringField()
phone = orm.StringField()
complete = orm.BooleanField(default=True)
completion_token = orm.StringField()
# Different based on context of user - whether events attended, mentored, or organized
checkins = orm.ListField(orm.ReferenceField(events.Event))
events = orm.ListField(orm.ReferenceField(events.Event))
shirt_type = orm.StringField()
shirt_size = orm.StringField()
dietary = orm.StringField()
notes = orm.MapField(orm.StringField())
stripe_id = orm.StringField()
class BaseConfig(me.EmbeddedDocument):
meta = {'allow_inheritance': True}
player_opts_class = BasePlayerOpts
game_type = me.StringField(required=True, choices=[])
gameid = me.StringField(required=True) # TODO get rid of this
players = me.ListField(me.ReferenceField('User'))
player_opts = me.MapField(me.EmbeddedDocumentField(BasePlayerOpts))
class Owner(me.Document):
id = me.StringField(primary_key=True,
default=lambda: uuid4().hex)
activation_date = me.FloatField() # TODO: Use datetime object
# this exists for preventing conflicting rule id's
rule_counter = me.IntField(default=0)
total_machine_count = me.IntField()
rules = me.MapField(field=me.ReferenceField(Rule))
alerts_email = me.ListField(me.StringField(), default=[])
avatar = me.StringField(default='')
last_active = me.DateTimeField()
meta = {
'allow_inheritance': True,
'ordering': ['-activation_date'],
'strict': False,
}
def count_mon_machines(self):
from mist.api.clouds.models import Cloud
from mist.api.machines.models import Machine
clouds = Cloud.objects(owner=self, deleted=None)
return Machine.objects(cloud__in=clouds,
monitoring__hasmonitoring=True).count()
def get_id(self):
# TODO: This must be deprecated
if not self.id:
self.id = lambda: uuid.uuid4().hex
return self.id
def get_external_id(self, service):
import mist.api.helpers
return mist.api.helpers.encrypt(self.id, key_salt=service, no_iv=True)
def as_dict(self):
# FIXME: Now, this is just silly
return json.loads(self.to_json())
def clean(self):
# TODO: check if these are valid email addresses,
# to avoid possible spam
if self.alerts_email:
if isinstance(self.alerts_email, basestring):
emails = []
for email in self.alerts_email.split(','):
if re.match("[^@]+@[^@]+\.[^@]+", email):
emails.append(email.replace(' ', ''))
self.emails = emails
super(Owner, self).clean()
class Snapshot(me.Document):
id = me.SequenceField(primary_key=True)
created_at = me.DateTimeField(default=datetime.now(), required=True)
trained_at = me.DateTimeField()
types = me.MapField(me.EmbeddedDocumentField(Type), required=True)
semaphore = me.IntField(default=0)
meta = {'indexes': ['semaphore']}
@staticmethod
def from_string(version_string):
if 'CURRENT' in version_string:
return Snapshot.current()
return Snapshot.objects.get(id=int(version_string[1:]))
@staticmethod
def current():
return Snapshot.objects.get(id=CURRENT_ID)
def __str__(self):
return f'v{"CURRENT" if self.id == CURRENT_ID else self.id}'
@contextmanager
def training_lock(self, un_train: bool = False):
if Snapshot.objects(id=self.id,
semaphore=0).update_one(dec__semaphore=1) == 1:
try:
yield self
finally:
if Snapshot.objects(id=self.id, semaphore=-1).update_one(
inc__semaphore=1,
trained_at=None if un_train else datetime.now()) == 1:
self.reload()
else:
raise TrainingLockFreeError
else:
raise TrainingLockAcquireError
@contextmanager
def loading_lock(self):
if Snapshot.objects(
id=self.id,
semaphore__gte=0).update_one(inc__semaphore=1) == 1:
try:
yield self
finally:
if Snapshot.objects(
id=self.id,
semaphore__gt=0).update_one(dec__semaphore=1) == 1:
self.reload()
else:
raise LoadingLockFreeError
else:
raise LoadingLockAcquireError
class Lecture(mongo.Document):
syjc = mongo.BooleanField()
subject = mongo.StringField()
division = mongo.StringField()
lecturer = mongo.StringField()
date = mongo.DateTimeField()
start_time = mongo.IntField()
end_time = mongo.IntField()
num_students = mongo.IntField()
present = mongo.MapField(mongo.BooleanField())
meta = {'collection': 'lectures'}
class Editor(mongoengine.Document):
"""
An Editor of a publication.
"""
meta = {'collection': 'test_editor'}
id = mongoengine.StringField(primary_key=True)
first_name = mongoengine.StringField(required=True, help_text="Editor's first name.", db_field='fname')
last_name = mongoengine.StringField(required=True, help_text="Editor's last name.")
metadata = mongoengine.MapField(field=mongoengine.StringField(), help_text="Arbitrary metadata.")
company = mongoengine.LazyReferenceField(Publisher)
avatar = mongoengine.FileField()
seq = mongoengine.SequenceField()
class UserIndex(mongoengine.EmbeddedDocument):
"""
用户索引内嵌文档模型:
typeid: 类型id 同一种索引方式相同
value: 用户索引值
description: 用户索引描述
extension: 扩展描述字典
"""
typeid = mongoengine.StringField(
unique=not settings.AllowMultiAccountBinding, required=True)
value = mongoengine.StringField(unique=True, required=True)
description = mongoengine.StringField(default=str)
extension = mongoengine.MapField(field=mongoengine.StringField())
class Job(me.DynamicDocument):
jobid = me.UUIDField(binary=False, required=True)
name = me.StringField(max_length=128, default="Unknown")
failed = me.BooleanField(default=False)
reason = me.StringField(max_length=512)
version = me.StringField(max_length=10, default=baleen.get_version)
started = me.DateTimeField(default=datetime.now, required=True)
finished = me.DateTimeField(default=None)
updated = me.DateTimeField(default=datetime.now, required=True)
errors = me.MapField(field=me.IntField())
counts = me.MapField(field=me.IntField())
totals = me.MapField(field=me.IntField())
@classmethod
def pre_save(cls, sender, document, **kwargs):
document.updated = datetime.now()
meta = {
'collection': 'jobs',
}
def duration(self, humanize=False):
"""
Returns the timedelta of the duration.
"""
finished = self.finished or datetime.now()
delta = finished - self.started
if humanize:
return humanizedelta(days=delta.days,
seconds=delta.seconds,
microseconds=delta.microseconds)
return delta
def __unicode__(self):
return "{} Job {}".format(self.name, self.jobid)
class Group(mongoengine.Document):
"""
用户组模型:
name: 用户组的名称
description: 用户组的描述信息
permission: 用户组的权限值
users: 用户组中包含的用户
extensions: 扩展信息存储
"""
name = mongoengine.StringField(required=True, default=str)
description = mongoengine.StringField(default=str)
permission = mongoengine.IntField(required=True, default=int)
users = mongoengine.ListField(mongoengine.ObjectIdField())
extensions = mongoengine.MapField(field=mongoengine.StringField())
class Job(me.DynamicDocument):
jobid = me.UUIDField(binary=False, required=True)
name = me.StringField(max_length=128, default="Unknown")
failed = me.BooleanField(default=False)
reason = me.StringField(max_length=512)
version = me.StringField(max_length=10, default=baleen.get_version)
started = me.DateTimeField(default=datetime.now, required=True)
finished = me.DateTimeField(default=None)
updated = me.DateTimeField(default=datetime.now, required=True)
errors = me.MapField(field=me.IntField())
counts = me.MapField(field=me.IntField())
totals = me.MapField(field=me.IntField())
@classmethod
def pre_save(cls, sender, document, **kwargs):
document.updated = datetime.now()
meta = {
'collection': 'jobs',
}
def __unicode__(self):
return "{} Job {}".format(self.name, self.jobid)
class HistoryStep(me.EmbeddedDocument):
log_message = me.StringField()
public_view = me.DictField()
public_view_delta = me.DictField()
player_views = me.MapField(me.DictField())
player_view_deltas = me.MapField(me.DictField())
class ExampleCardGameModel(TurnBasedModel):
hands = me.MapField(me.ListField(me.IntField))
deck = me.ListField(me.IntField())
stack = me.ListField(me.IntField())
viewing = me.ListField(me.IntField(), required=False)
@property
def current_total(self):
return sum(self.stack)
def setup(self):
cards_to_deal = 4 if self.config.use_special_cards else 5
cards = list(range(1, 6)) * 5
random.shuffle(cards)
self.hands = {}
self.hands[self.config.players[0].id] = cards[0:cards_to_deal]
self.hands[
self.config.players[1].id] = cards[cards_to_deal:cards_to_deal * 2]
self.deck = cards[cards_to_deal * 2:]
if self.config.use_special_cards:
for p in self.config.players:
self.hands[p.id].append(
self.config.player_opts[p.id].special_card)
self.stack = []
self.viewing = None
super(ExampleCardGameModel, self).setup()
def update_legal_actions(self):
if self.result is not None:
self._cached_actions = {
p.id: [['restart']]
for p in self.config.players
}
else:
super(ExampleCardGameModel, self).update_legal_actions()
def get_actions_for_active_player(self):
if self.viewing:
yield from [['pick', n] for n in sorted(set(self.viewing))]
else:
yield from [['play', n]
for n in sorted(set(self.hands[self.active_user.id]))]
c = Counter(self.hands[self.active_user.id])
for v in c:
if c[v] > 1:
yield ['discard_double', v]
def apply_action(self, user, action, log_callback):
if not self.is_legal_action(user, action):
return False
if action == ['restart']:
self.result = None
self.setup()
log_callback(
f'New game starting. {self.active_user.nickname} plays first.')
else:
self.apply_action_for_active_player(action, log_callback)
self.update_legal_actions()
def apply_action_for_active_player(self, action, log_callback):
move_type = action[0]
if move_type == 'play':
self.stack.append(int(action[1]))
log_callback(
f'{self.active_user.nickname} plays a {action[1]}, making the total {self.current_total}.'
)
self.hands[self.active_user.id].remove(int(action[1]))
if self.current_total == 21:
self.result = {'winner': self.active_user.id}
log_callback(
f'{self.active_user.nickname} wins by reaching 21.')
elif self.current_total > 21:
self.result = {
'winner':
(self.turn_order[1 - self.active_player_index]).id
}
winner_nick = [
p.nickname for p in self.config.players
if p.id == self.result['winner']
][0]
log_callback(
f'{self.active_user.nickname} went over 21. {winner_nick} wins.'
)
else:
self.hands[self.active_user.id].append(self.deck.pop())
self.advance_turn()
elif move_type == 'discard_double':
log_callback(
f'{self.active_user.nickname} discards a pair of {action[1]}s.'
)
self.hands[self.active_user.id].remove(int(action[1]))
self.hands[self.active_user.id].remove(int(action[1]))
self.viewing = self.deck[:5]
self.deck = self.deck[5:]
elif move_type == 'pick':
log_callback(
f'{self.active_user.nickname} chooses a card from the top five of the deck.'
)
self.hands[self.active_user.id].append(int(action[1]))
self.viewing.remove(int(action[1]))
random.shuffle(self.viewing)
self.deck.extend(self.viewing)
log_callback(
f'{len(self.viewing)} cards are shuffled and placed on the bottom.'
)
self.viewing = None
self.advance_turn()
def get_public_view(self):
result = super(ExampleCardGameModel, self).get_public_view()
result['current_total'] = self.current_total
result['deck_count'] = len(self.deck)
result['stack'] = self.stack.copy()
result['hand_counts'] = {
p.id: len(self.hands[p.id])
for p in self.config.players
}
return result
def get_player_view(self, user):
result = super(ExampleCardGameModel, self).get_player_view(user)
result['my_hand'] = self.hands[user.id].copy()
if self.viewing and user == self.active_user:
result['viewing'] = self.viewing.copy()
return result
class Fight(warcraftlogs_base.EmbeddedDocument):
fight_id = me.IntField(primary_key=True)
start_time: arrow.Arrow = mongoengine_arrow.ArrowDateTimeField()
# fight duration in milliseconds
duration: int = me.IntField(default=0)
# deprecated in favor of "duration".
end_time_old: arrow.Arrow = mongoengine_arrow.ArrowDateTimeField(
db_field="end_time")
boss_id = me.IntField()
players: typing.Dict[str, Player] = me.MapField(
me.EmbeddedDocumentField(Player))
boss: Boss = me.EmbeddedDocumentField(Boss)
composition = me.DictField()
deaths = me.IntField(default=0)
ilvl = me.FloatField(default=0)
damage_taken = me.IntField(default=0)
# boss percentage at the end. (its 0.01 for kills)
percent = me.FloatField(default=0)
kill = me.BooleanField(default=True)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.report = None
if self.boss:
self.boss.fight = self
for player in self.players.values():
player.fight = self
def __str__(self):
return f"{self.__class__.__name__}(id={self.fight_id}, players={len(self.players)})"
def summary(self):
raid_boss_name = self.boss and self.boss.raid_boss and self.boss.raid_boss.full_name_slug
return {
"report_id": self.report.report_id,
"fight_id": self.fight_id,
"percent": self.percent,
"kill": self.kill,
"duration": self.duration,
"time": self.start_time.timestamp(),
"boss": {
"name": raid_boss_name
},
}
def as_dict(self, player_ids: typing.List[int] = None) -> dict:
# Get players
players = list(self.players.values())
if player_ids:
players = [
player for player in players if player.source_id in player_ids
]
players = sorted(players,
key=lambda player:
(player.spec.role, player.spec, player.name))
# Return
return {
**self.summary(),
"players": [player.as_dict() for player in players],
"boss": self.boss.as_dict() if self.boss else {},
}
##########################
# Attributes
@property
def end_time(self) -> arrow.Arrow:
return self.start_time.shift(seconds=self.duration / 1000)
@property
def start_time_rel(self) -> int:
"""Fight start time, relative the parent report (in milliseconds)."""
return 1000 * int(self.start_time.timestamp() -
self.report.start_time.timestamp())
@property
def end_time_rel(self) -> int:
"""fight end time, relative to the report (in milliseconds)."""
return 1000 * int(self.end_time.timestamp() -
self.report.start_time.timestamp())
@property
def raid_boss(self) -> RaidBoss:
return RaidBoss.get(id=self.boss_id)
#################################
# Methods
#
def get_player(self, **kwargs) -> Player:
"""Returns a single Player based on the kwargs."""
return utils.get(self.players.values(), **kwargs)
def get_players(self, source_ids: typing.List[int] = None):
"""Gets multiple players based on source id."""
players = list(self.players.values())
if source_ids:
players = [
player for player in players if player.source_id in source_ids
]
return [player for player in players if player]
def add_boss(self, boss_id) -> RaidBoss:
self.boss_id = boss_id
self.boss = Boss(boss_id=boss_id)
self.boss.fight = self
return self.boss
############################################################################
# Query
#
@property
def table_query_args(self) -> str:
return f"fightIDs: {self.fight_id}, startTime: {self.start_time_rel}, endTime: {self.end_time_rel}"
############################################################################
# Summary
#
def get_summary_query(self):
"""Get the Query to load the fights summary."""
if self.players:
return ""
return textwrap.dedent(f"""\
reportData
{{
report(code: "{self.report.report_id}")
{{
summary: table({self.table_query_args}, dataType: Summary)
}}
}}
""")
def process_players(self, players_data):
if not players_data:
logger.warning("players_data is empty")
return
total_damage = players_data.get("damageDone", [])
total_healing = players_data.get("healingDone", [])
for composition_data in players_data.get("composition", []):
spec_data = composition_data.get("specs", [])
if not spec_data:
logger.warning("Player has no spec: %s",
composition_data.get("name"))
continue
spec_data = spec_data[0]
spec_name = spec_data.get("spec")
class_name = composition_data.get("type")
spec = WowSpec.get(name_slug_cap=spec_name,
wow_class__name_slug_cap=class_name)
if not spec:
logger.warning("Unknown Spec: %s", spec_name)
continue
# Get Total Damage or Healing
spec_role = spec_data.get("role")
total_data = total_healing if spec_role == "healer" else total_damage
for data in total_data:
if data.get("id", -1) == composition_data.get("id"):
total = data.get("total", 0) / (self.duration / 1000)
break
else:
total = 0
# create and return yield player object
player = Player()
player.fight = self
player.spec_slug = spec.full_name_slug
player.source_id = composition_data.get("id")
player.name = composition_data.get("name")
player.total = int(total)
player.process_death_events(players_data.get("deathEvents", []))
self.players[str(player.source_id)] = player
# call this before filtering to always get the full comp
self.composition = get_composition(self.players.values())
def process_overview(self, data):
"""Process the data retured from an Overview-Query."""
data = data.get("reportData") or data
summary_data = utils.get_nested_value(data, "report", "summary",
"data") or {}
self.duration = self.duration or summary_data.get("totalTime", 0)
self.process_players(summary_data)
async def load_summary(self, force=False):
"""Load this fights Summary.
Args:
force(boolean, optional): load even if its already loaded
"""
if force:
self.players = {}
if self.players:
return ""
query = self.get_summary_query()
result = await self.client.query(query)
self.process_overview(result)
# alias to set the default "load"-behaviour
process_query_result = process_overview
get_query = get_summary_query
############################################################################
# Load Player:
#
async def load_players(self, player_ids: typing.List[int] = None):
if not self.players:
await self.load_summary()
# Get Players to load
players_to_load = self.get_players(player_ids)
players_to_load = [
player for player in players_to_load if not player.casts
]
# see if we need to load the boss
boss = [] if (self.boss and self.boss.casts) else [self.boss]
if not (players_to_load or boss):
return
# load
await self.load_many(players_to_load + boss)
# re-add them to the dict, as otherwise we get some mongodb issues
for actor in players_to_load:
self.players[str(actor.source_id)] = actor
class Rule(me.Document):
"""The base Rule mongoengine model.
The Rule class defines the base schema of all rule types. All documents of
any Rule subclass will be stored in the same mongo collection.
All Rule subclasses MUST define a `_controller_cls` class attribute and a
backend plugin. Controllers are used to perform actions on instances of
Rule, such as adding or updating. Backend plugins are used to transform a
Rule into the corresponding query to be executed against a certain data
storage. Different types of rules, such as a rule on monitoring metrics or
a rule on logging data, should also define and utilize their respective
backend plugins. For instance, a rule on monitoring data, which is stored
in a TSDB like Graphite, will have to utilize a different plugin than a
rule on logging data, stored in Elasticsearch, in order to successfully
query the database.
The Rule class is mainly divided into two categories:
1. Arbitrary rules - defined entirely by the user. This type of rules gives
users the freedom to execute arbitrary queries on arbitrary data. The query
may include (nested) expressions and aggregations on arbitrary fields whose
result will be evaluated against a threshold based on a comparison operator
(=, <, etc).
2. Resource rules - defined by using Mist.io UUIDs and tags. This type of
rules can be used to easily setup alerts on resources given their tags or
UUIDs. In this case, users have to explicitly specify the target metric's
name, aggregation function, and resources either by their UUIDs or tags.
This type of rules allows for easier alert configuration on known resources
in the expense of less elastic query expressions.
The Rule base class can be used to query the database and fetch documents
created by any Rule subclass. However, in order to add new rules one must
use one of the Rule subclasses, which represent different rule type, each
associated with the corresponding backend plugin.
"""
id = me.StringField(primary_key=True, default=lambda: uuid.uuid4().hex)
title = me.StringField(required=True)
owner_id = me.StringField(required=True)
# Specifies a list of queries to be evaluated. Results will be logically
# ANDed together in order to decide whether an alert should be raised.
queries = me.EmbeddedDocumentListField(QueryCondition, required=True)
# Defines the time window and frequency of each search.
window = me.EmbeddedDocumentField(Window, required=True)
frequency = me.EmbeddedDocumentField(Frequency, required=True)
# Associates a reminder offset, which will cause an alert to be fired if
# and only if the threshold is exceeded for a number of trigger_after
# intervals.
trigger_after = me.EmbeddedDocumentField(
TriggerOffset, default=lambda: TriggerOffset(period='minutes'))
# Defines a list of actions to be executed once the rule is triggered.
# Defaults to just notifying the users.
actions = me.EmbeddedDocumentListField(
BaseAlertAction, required=True, default=lambda: [NotificationAction()])
# Disable the rule organization-wide.
disabled = me.BooleanField(default=False)
# Fields passed to celerybeat as optional arguments.
queue = me.StringField()
exchange = me.StringField()
routing_key = me.StringField()
soft_time_limit = me.IntField()
# Fields updated by the scheduler.
last_run_at = me.DateTimeField()
run_immediately = me.BooleanField()
total_run_count = me.IntField(min_value=0, default=0)
total_check_count = me.IntField(min_value=0, default=0)
# Field updated by celery workers. This is where celery workers keep state.
states = me.MapField(field=me.EmbeddedDocumentField(RuleState))
meta = {
'strict':
False,
'collection':
'rules',
'allow_inheritance':
True,
'indexes': [{
'fields': ['owner_id', 'title'],
'sparse': False,
'unique': True,
'cls': False,
}]
}
_controller_cls = None
_backend_plugin = None
_data_type_str = None
def __init__(self, *args, **kwargs):
super(Rule, self).__init__(*args, **kwargs)
if self._controller_cls is None:
raise TypeError(
"Cannot instantiate self. %s is a base class and cannot be "
"used to insert or update alert rules and actions. Use a "
"subclass of self that defines a `_controller_cls` class "
"attribute derived from `mist.api.rules.base:BaseController`, "
"instead." % self.__class__.__name__)
if self._backend_plugin is None:
raise NotImplementedError(
"Cannot instantiate self. %s does not define a backend_plugin "
"in order to evaluate rules against the corresponding backend "
"storage." % self.__class__.__name__)
if self._data_type_str not in (
'metrics',
'logs',
):
raise TypeError(
"Cannot instantiate self. %s is a base class and cannot be "
"used to insert or update rules. Use a subclass of self that "
"defines a `_backend_plugin` class attribute, as well as the "
"requested data's type via the `_data_type_str` attribute, "
"instead." % self.__class__.__name__)
self.ctl = self._controller_cls(self)
@classmethod
def add(cls, auth_context, title=None, **kwargs):
"""Add a new Rule.
New rules should be added by invoking this class method on a Rule
subclass.
Arguments:
owner: instance of mist.api.users.models.Organization
title: the name of the rule. This must be unique per Organization
kwargs: additional keyword arguments that will be passed to the
corresponding controller in order to setup the self
"""
try:
cls.objects.get(owner_id=auth_context.owner.id, title=title)
except cls.DoesNotExist:
rule = cls(owner_id=auth_context.owner.id, title=title)
rule.ctl.set_auth_context(auth_context)
rule.ctl.add(**kwargs)
else:
raise BadRequestError('Title "%s" is already in use' % title)
return rule
@property
def owner(self):
"""Return the Organization (instance) owning self.
We refrain from storing the owner as a me.ReferenceField in order to
avoid automatic/unwanted dereferencing.
"""
return Organization.objects.get(id=self.owner_id)
@property
def plugin(self):
"""Return the instance of a backend plugin.
Subclasses MUST define the plugin to be used, instantiated with `self`.
"""
return self._backend_plugin(self)
# NOTE The following properties are required by the scheduler.
@property
def name(self):
"""Return the name of the celery task.
This must be globally unique, since celerybeat-mongo uses schedule
names as keys of the dictionary of schedules to run.
"""
return 'Org(%s):Rule(%s)' % (self.owner_id, self.id)
@property
def task(self):
"""Return the celery task to run.
This is the most basic celery task that should be used for most rule
evaluations. However, subclasses may provide their own property or
class attribute based on their needs.
"""
return 'mist.api.rules.tasks.evaluate'
@property
def args(self):
"""Return the args of the celery task."""
return (self.id, )
@property
def kwargs(self):
"""Return the kwargs of the celery task."""
return {}
@property
def expires(self):
"""Return None to denote that self is not meant to expire."""
return None
@property
def enabled(self):
"""Return True if the celery task is currently enabled.
Subclasses MAY override or extend this property.
"""
return not self.disabled
@property
def schedule(self):
"""Return a celery schedule instance.
Used internally by the scheduler. Subclasses MUST NOT override this.
"""
return celery.schedules.schedule(self.frequency.timedelta)
def is_arbitrary(self):
"""Return True if self is arbitrary.
Arbitrary rules lack a list of `selectors` that refer to resources
either by their UUIDs or by tags. Such a list makes it easy to setup
rules referencing specific resources without the need to provide the
raw query expression.
"""
return 'selectors' not in type(self)._fields
def clean(self):
# FIXME This is needed in order to ensure rule name convention remains
# backwards compatible with the old monitoring stack. However, it will
# have to change in the future due to uniqueness constrains.
if not self.title:
self.title = 'rule%d' % self.owner.rule_counter
def as_dict(self):
return {
'id': self.id,
'title': self.title,
'queries': [query.as_dict() for query in self.queries],
'window': self.window.as_dict(),
'frequency': self.frequency.as_dict(),
'trigger_after': self.trigger_after.as_dict(),
'actions': [action.as_dict() for action in self.actions],
'disabled': self.disabled,
'data_type': self._data_type_str,
}
def __str__(self):
return '%s %s of %s' % (self.__class__.__name__, self.title,
self.owner)
class Host(common.BaseDocument):
meta = {
'ordering': ['-updated'],
'queryset_class': common.SerializableQuerySet,
'indexes': ['created', 'updated', 'hostname']
}
hostname = mongoengine.StringField(required=True)
pid = mongoengine.IntField(required=True)
mac_address = mongoengine.StringField()
job_slots = mongoengine.MapField(field=mongoengine.IntField(), default={})
job_imports = mongoengine.ListField(field=mongoengine.StringField(),
default=[])
platform = mongoengine.DictField()
boot_time = mongoengine.DateTimeField()
python_version = mongoengine.StringField()
python_packages = mongoengine.ListField(field=mongoengine.StringField())
def history(self, offset=0, limit=30, step=0):
step_filter = {}
if step and step > 1:
step_filter = {'index__mod': (step, 0)}
statuses = HostStatus.objects(
host=self, **step_filter).order_by('-created')[offset:limit]
return [s.to_safe_dict(with_host=False) for s in statuses]
def alive(self):
recent_count = HostStatus.objects(host=self,
created__gte=datetime.utcnow() -
timedelta(minutes=0.5)).count()
return recent_count > 0
def last_seen_alive(self):
last_status = HostStatus.objects(
host=self).order_by('-created').only('created').first()
if not last_status or not last_status.created:
return None
return last_status.created
def to_safe_dict(self,
alive=False,
with_history=False,
offset=0,
limit=30,
step=0):
r = super(Host, self).to_safe_dict()
if alive:
r['alive'] = self.alive()
r['last_seen_alive'] = self.last_seen_alive()
if with_history:
r['history'] = self.history(offset=offset, limit=limit, step=step)
return r
def update_status(self):
partitions = []
try:
for f in psutil.disk_partitions():
usage = psutil.disk_usage(path=f.mountpoint)
p = {
'type': f.fstype,
'device': f.device,
'mountpoint': f.mountpoint,
'total': usage.total,
'used': usage.used,
'percent': usage.percent,
}
partitions.append(p)
except Exception as e:
pass
self_process = psutil.Process(os.getpid())
processes = [{
'ppid': self_process.ppid(),
'pid': self_process.pid,
'cmd': ' '.join(self_process.cmdline())
}]
for c in self_process.children():
try:
processes.append({
'ppid': c.ppid(),
'pid': c.pid,
'cmd': ' '.join(c.cmdline())
})
except psutil.Error:
pass
self.host_status_index += 1
status = HostStatus()
status.index = self.host_status_index
status.host = self
status.current_jobs = [{
'uuid': j.uuid,
'type': j._cls
} for j in self.client_service.current_jobs]
virtual_memory = psutil.virtual_memory()
swap_memory = psutil.swap_memory()
status.system_status = {
'processes': processes,
'cpu': {
'percent': psutil.cpu_percent(),
'percents': psutil.cpu_percent(percpu=True)
},
'memory': {
'virtual': {
'total': virtual_memory.total,
'used': virtual_memory.used,
'percent': virtual_memory.percent,
},
'swap': {
'total': swap_memory.total,
'used': swap_memory.used,
'percent': swap_memory.percent,
},
},
'disk': partitions,
#'disk_io': safe_dict(psutil.disk_io_counters, perdisk=False)
}
status.save()
@classmethod
def localhost(cls):
hostname = socket.gethostname()
hosts = Host.objects(hostname=hostname)
if not hosts:
logging.info('Host unknown. Initializing it in the database...')
host = Host()
host.hostname = hostname
host.mac_address = tbx.network.get_mac_address()
host.platform = tbx.code.safe_dict(platform.uname)
host.host_status_index = 1
logging.info(
"Now, configure Host '%s' through API or Web UI to be able to use it."
% hostname)
else:
host = hosts[0]
last_status = HostStatus.objects(
host=host).order_by('-created').first()
if not last_status:
host.host_status_index = 1
else:
host.host_status_index = last_status.index
logging.info("Host '%s' already found in database." % hostname)
host.boot_time = datetime.fromtimestamp(psutil.boot_time())
host.pid = os.getpid()
host.python_version = sys.version.split(' ')[0]
host.python_packages = sorted([
"%s (%s)" % (i.key, i.version) for i in pkg_resources.working_set
])
host.save()
logging.info("Host '%s' config updated in database." % hostname)
return host
def update_slots(self, job_slots=None):
from jobmanager.common.job import Job, JobTask
job_classes = tbx.code.get_subclasses(Job)
job_tasks = tbx.code.get_subclasses(JobTask)
if not job_slots:
logging.info(
'Job Slots not set in env or command line args. Setting to default job defined amount.'
)
job_slots = {
k.__name__: k.default_slot_amount()
for k in job_classes
}
available_class_names = {c.__name__ for c in job_classes}
previous_class_names = set(self.job_slots.keys())
all_class_names = previous_class_names | available_class_names
for class_name in all_class_names:
if class_name not in job_slots.keys():
self.job_slots[class_name] = 0
else:
self.job_slots[class_name] = job_slots[class_name]
logging.info(" - Job type found : %s (%d slots)" %
(class_name, self.job_slots[class_name]))
logging.info("Also found following job tasks : %s" %
', '.join([k.__name__ for k in job_tasks]))
return self.save()
def check_capacity(self):
if self.job_slots:
#logging.info("Jobs types allowed : %s" % (', '.join(self.job_slots.keys())))
logging.info("Jobs capacity :")
total_capacity = 0
for c in self.job_slots:
logging.info(" - %s\t: %d" % (c, self.job_slots[c]))
total_capacity += self.job_slots[c]
if total_capacity == 0:
logging.error(
"No job capacities setup. Configure host to add slots to some job types."
)
raise common.ConfigurationException(
"No job capacities setup. Configure host to add slots to some job types (see --add-slot option)."
)
else:
logging.error(
"No job class found to be run. Configure host to import packages that contain job subclasses."
)
raise common.ConfigurationException(
"No Job sub-class found in imports. Please configure host (see --import option)."
)
def do_import(self, imports):
return common.safely_import_from_name(imports)
@classmethod
def get_all_alive(cls):
raise NotImplementedError()
class Job(me.DynamicDocument):
jobid = me.UUIDField(binary=False, required=True)
name = me.StringField(max_length=128, default="Unknown")
failed = me.BooleanField(default=False)
reason = me.StringField(max_length=512)
version = me.StringField(max_length=10, default=baleen.get_version)
started = me.DateTimeField(default=datetime.now, required=True)
finished = me.DateTimeField(default=None)
updated = me.DateTimeField(default=datetime.now, required=True)
errors = me.MapField(field=me.IntField())
counts = me.MapField(field=me.IntField())
totals = me.MapField(field=me.IntField())
@classmethod
def pre_save(cls, sender, document, **kwargs):
document.updated = datetime.now()
meta = {
'collection': 'jobs',
}
def duration(self, humanize=False):
"""
Returns the timedelta of the duration.
"""
finished = self.finished or datetime.now()
delta = finished - self.started
if humanize:
return humanizedelta(days=delta.days,
seconds=delta.seconds,
microseconds=delta.microseconds)
return delta
@property
def bootstrap_class(self):
"""
Uses the duration to determine the colorization of the job.
"""
if self.finished and self.failed:
return "danger"
if self.finished and not self.failed:
if self.duration() > timedelta(minutes=30):
return "warning"
return "success"
if not self.finished:
if self.duration() < timedelta(minutes=30):
return "success"
elif timedelta(minutes=30) < self.duration() < timedelta(hours=2):
return "warning"
else:
return "danger"
return ""
def __unicode__(self):
return "{} Job {}".format(self.name, self.jobid)
class Doc(me.Document):
id = me.IntField(primary_key=True, default=1)
map = me.MapField(me.EmbeddedDocumentField(MappedDoc))
str = me.MapField(me.StringField())
class Report(warcraftlogs_base.EmbeddedDocument):
"""Defines a Report read from WarcraftLogs.com and stores in our DB."""
# 16 digit unique id/code as used on warcraftlogs
report_id: str = me.StringField(primary_key=True)
# time the report (!) has started. The first fight might start later
start_time: arrow.Arrow = mongoengine_arrow.ArrowDateTimeField(default=lambda: arrow.get(0))
# title of the report
title: str = me.StringField()
zone_id: int = me.IntField(default=0)
# The guild that the report belongs to. None if it was a logged as a personal report
guild: str = me.StringField(default="")
# The user that uploaded the report.
owner: str = me.StringField(default="")
# fights in this report keyed by fight_id. (they may or may not be loaded)
fights: typing.Dict[str, Fight] = me.MapField(me.EmbeddedDocumentField(Fight), default={})
# players in this report.
# Note: not every player might participate in every fight.
players: typing.Dict[str, Player] = me.MapField(me.EmbeddedDocumentField(Player), default={})
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# convert list to dict for old DB entries
# if isinstance(self.fights, list):
# self.fights = {fight.fight_id: fight for fight in self.fights}
for fight in self.fights.values():
fight.report = self
def __str__(self):
return f"<BaseReport({self.report_id}, num_fights={len(self.fights)})>"
##########################
# Attributes
#
def as_dict(self):
"""Return a Summary/Overview about this report."""
info = {
"title": self.title,
"report_id": self.report_id,
"date": int(self.start_time.timestamp()),
"zone_id": self.zone_id,
"guild": self.guild,
"owner": self.owner,
}
# for players and fights we only include essential data
info["fights"] = {fight.fight_id: fight.summary() for fight in self.fights.values()}
info["players"] = {player.source_id: player.summary() for player in self.players.values()}
return info
##########################
# Methods
#
def add_fight(self, **fight_data):
"""Add a new Fight to this Report."""
# skip trash fights
boss_id = fight_data.get("encounterID")
if not boss_id:
return
fight = Fight()
fight.fight_id = fight_data.get("id", "0")
fight.report = self
fight.percent = fight_data.get("fightPercentage")
fight.kill = fight_data.get("kill", True)
# Fight: Time/Duration
start_time = fight_data.get("startTime", 0) / 1000
end_time = fight_data.get("endTime", 0) / 1000
fight.start_time = self.start_time.shift(seconds=start_time)
fight.duration = (end_time - start_time) * 1000
# Fight: Boss
fight.boss = Boss(boss_id=boss_id)
if fight.boss: # could be a boss unknown to Lorrgs
fight.boss.fight = fight
# store and return
self.fights[str(fight.fight_id)] = fight
return fight
def get_fight(self, fight_id: int):
"""Get a single fight from this Report."""
return self.fights.get(str(fight_id))
def get_fights(self, *fight_ids: int):
"""Get a multiple fights based of their fight ids."""
fights = [self.get_fight(fight_id) for fight_id in fight_ids]
return [f for f in fights if f] # filter out nones
def add_player(self, **actor_data):
# pets
if actor_data.get("subType") == "Unknown":
return
# guess spec from the icon
# WCL gives us an icon matching the spec, IF a player
# played the same spec in all fights inside a report.
# Otherwise it only includes a class-name.
icon_name = actor_data.get("icon", "")
spec_slug = icon_name.lower() if "-" in icon_name else ""
# create the new player
player = warcraftlogs_actor.Player()
player.source_id = actor_data.get("id")
player.name = actor_data.get("name")
player.class_slug = actor_data.get("subType", "").lower()
player.spec_slug = spec_slug
# add to to the report
self.players[str(player.source_id)] = player
############################################################################
# Query
#
def get_query(self):
"""Get the Query to load this Reports Overview."""
return textwrap.dedent(f"""
reportData
{{
report(code: "{self.report_id}")
{{
title
zone {{name id}}
startTime
owner {{ name }}
guild {{
name
server {{ name }}
}}
masterData
{{
actors(type: "Player")
{{
name
id
subType
icon # the icon includes the spec name
}}
}}
fights
{{
id
encounterID
startTime
endTime
fightPercentage
kill
}}
}}
}}
""")
def process_master_data(self, master_data: typing.Dict[str, typing.Any]):
"""Create the Players from the passed Report-MasterData"""
if not master_data:
return
# clear out any old instances
self.players = {}
for actor_data in master_data.get("actors", []):
self.add_player(**actor_data)
def process_report_fights(self, fights_data: typing.List[typing.Dict]):
"""Update the Fights in this report."""
# clear out any old data
self.fights = {}
fights_data = fights_data or []
for fight_data in fights_data:
self.add_fight(**fight_data)
def process_query_result(self, query_result: dict):
report_data = query_result.get("report", {})
# Update the Report itself
self.title = report_data.get("title", "")
self.start_time = arrow.get(report_data.get("startTime", 0))
zone = report_data.get("zone") or {} # might contain a "None" in the results
self.zone_id = zone.get("id") or -1
self.owner = report_data.get("owner", {}).get("name", "")
guild_info: typing.Dict = report_data.get("guild") or {}
self.guild = guild_info.get("name") or ""
self.process_master_data(report_data.get("masterData"))
self.process_report_fights(report_data.get("fights"))
async def load_summary(self):
await self.load()
async def load_fight(self, fight_id: int, player_ids=typing.List[int]):
"""Load a single Fight from this Report."""
fight = self.fights[str(fight_id)]
if not fight:
raise ValueError("invalid fight id")
await fight.load_players(player_ids=player_ids)
async def load_fights(self, fight_ids: typing.List[int], player_ids: typing.List[int]):
await self.client.ensure_auth()
if not self.fights:
await self.load_summary()
for ids in utils.chunks(fight_ids, 10):
logger.info(f"loading fights: {ids}")
tasks = [self.load_fight(fight_id=fight_id, player_ids=player_ids) for fight_id in ids]
await asyncio.gather(*tasks)
