class Stores(me.Document): created_at = me.DateTimeField(default=datetime.now()) name = me.StringField(required=True) phone = me.StringField(require=True) type_ = me.StringField(required=True) sub_type = me.StringField(required=True) is_active = me.BooleanField(default=True) location = me.MapField(me.EmbeddedDocumentListField(Locations)) owner_id = me.StringField() products = me.EmbeddedDocumentListField(Product) web_info = me.MapField(me.EmbeddedDocumentListField(Web)) schedule = me.StringField() meta = {'db_alias': 'core', 'collection': 'stores'}
class User(document.Document): meta = {'allow_inheritance': True} _hidden = ['password', 'completion_token'] type = orm.StringField() name = orm.StringField(required=True) image = orm.URLField() email = orm.EmailField(required=True, unique=True) password = orm.StringField() gender = orm.StringField() phone = orm.StringField() complete = orm.BooleanField(default=True) completion_token = orm.StringField() # Different based on context of user - whether events attended, mentored, or organized checkins = orm.ListField(orm.ReferenceField(events.Event)) events = orm.ListField(orm.ReferenceField(events.Event)) shirt_type = orm.StringField() shirt_size = orm.StringField() dietary = orm.StringField() notes = orm.MapField(orm.StringField()) stripe_id = orm.StringField()
class BaseConfig(me.EmbeddedDocument): meta = {'allow_inheritance': True} player_opts_class = BasePlayerOpts game_type = me.StringField(required=True, choices=[]) gameid = me.StringField(required=True) # TODO get rid of this players = me.ListField(me.ReferenceField('User')) player_opts = me.MapField(me.EmbeddedDocumentField(BasePlayerOpts))
class Owner(me.Document): id = me.StringField(primary_key=True, default=lambda: uuid4().hex) activation_date = me.FloatField() # TODO: Use datetime object # this exists for preventing conflicting rule id's rule_counter = me.IntField(default=0) total_machine_count = me.IntField() rules = me.MapField(field=me.ReferenceField(Rule)) alerts_email = me.ListField(me.StringField(), default=[]) avatar = me.StringField(default='') last_active = me.DateTimeField() meta = { 'allow_inheritance': True, 'ordering': ['-activation_date'], 'strict': False, } def count_mon_machines(self): from mist.api.clouds.models import Cloud from mist.api.machines.models import Machine clouds = Cloud.objects(owner=self, deleted=None) return Machine.objects(cloud__in=clouds, monitoring__hasmonitoring=True).count() def get_id(self): # TODO: This must be deprecated if not self.id: self.id = lambda: uuid.uuid4().hex return self.id def get_external_id(self, service): import mist.api.helpers return mist.api.helpers.encrypt(self.id, key_salt=service, no_iv=True) def as_dict(self): # FIXME: Now, this is just silly return json.loads(self.to_json()) def clean(self): # TODO: check if these are valid email addresses, # to avoid possible spam if self.alerts_email: if isinstance(self.alerts_email, basestring): emails = [] for email in self.alerts_email.split(','): if re.match("[^@]+@[^@]+\.[^@]+", email): emails.append(email.replace(' ', '')) self.emails = emails super(Owner, self).clean()
class Snapshot(me.Document): id = me.SequenceField(primary_key=True) created_at = me.DateTimeField(default=datetime.now(), required=True) trained_at = me.DateTimeField() types = me.MapField(me.EmbeddedDocumentField(Type), required=True) semaphore = me.IntField(default=0) meta = {'indexes': ['semaphore']} @staticmethod def from_string(version_string): if 'CURRENT' in version_string: return Snapshot.current() return Snapshot.objects.get(id=int(version_string[1:])) @staticmethod def current(): return Snapshot.objects.get(id=CURRENT_ID) def __str__(self): return f'v{"CURRENT" if self.id == CURRENT_ID else self.id}' @contextmanager def training_lock(self, un_train: bool = False): if Snapshot.objects(id=self.id, semaphore=0).update_one(dec__semaphore=1) == 1: try: yield self finally: if Snapshot.objects(id=self.id, semaphore=-1).update_one( inc__semaphore=1, trained_at=None if un_train else datetime.now()) == 1: self.reload() else: raise TrainingLockFreeError else: raise TrainingLockAcquireError @contextmanager def loading_lock(self): if Snapshot.objects( id=self.id, semaphore__gte=0).update_one(inc__semaphore=1) == 1: try: yield self finally: if Snapshot.objects( id=self.id, semaphore__gt=0).update_one(dec__semaphore=1) == 1: self.reload() else: raise LoadingLockFreeError else: raise LoadingLockAcquireError
class Lecture(mongo.Document): syjc = mongo.BooleanField() subject = mongo.StringField() division = mongo.StringField() lecturer = mongo.StringField() date = mongo.DateTimeField() start_time = mongo.IntField() end_time = mongo.IntField() num_students = mongo.IntField() present = mongo.MapField(mongo.BooleanField()) meta = {'collection': 'lectures'}
class Editor(mongoengine.Document): """ An Editor of a publication. """ meta = {'collection': 'test_editor'} id = mongoengine.StringField(primary_key=True) first_name = mongoengine.StringField(required=True, help_text="Editor's first name.", db_field='fname') last_name = mongoengine.StringField(required=True, help_text="Editor's last name.") metadata = mongoengine.MapField(field=mongoengine.StringField(), help_text="Arbitrary metadata.") company = mongoengine.LazyReferenceField(Publisher) avatar = mongoengine.FileField() seq = mongoengine.SequenceField()
class UserIndex(mongoengine.EmbeddedDocument): """ 用户索引内嵌文档模型: typeid: 类型id 同一种索引方式相同 value: 用户索引值 description: 用户索引描述 extension: 扩展描述字典 """ typeid = mongoengine.StringField( unique=not settings.AllowMultiAccountBinding, required=True) value = mongoengine.StringField(unique=True, required=True) description = mongoengine.StringField(default=str) extension = mongoengine.MapField(field=mongoengine.StringField())
class Job(me.DynamicDocument): jobid = me.UUIDField(binary=False, required=True) name = me.StringField(max_length=128, default="Unknown") failed = me.BooleanField(default=False) reason = me.StringField(max_length=512) version = me.StringField(max_length=10, default=baleen.get_version) started = me.DateTimeField(default=datetime.now, required=True) finished = me.DateTimeField(default=None) updated = me.DateTimeField(default=datetime.now, required=True) errors = me.MapField(field=me.IntField()) counts = me.MapField(field=me.IntField()) totals = me.MapField(field=me.IntField()) @classmethod def pre_save(cls, sender, document, **kwargs): document.updated = datetime.now() meta = { 'collection': 'jobs', } def duration(self, humanize=False): """ Returns the timedelta of the duration. """ finished = self.finished or datetime.now() delta = finished - self.started if humanize: return humanizedelta(days=delta.days, seconds=delta.seconds, microseconds=delta.microseconds) return delta def __unicode__(self): return "{} Job {}".format(self.name, self.jobid)
class Group(mongoengine.Document): """ 用户组模型: name: 用户组的名称 description: 用户组的描述信息 permission: 用户组的权限值 users: 用户组中包含的用户 extensions: 扩展信息存储 """ name = mongoengine.StringField(required=True, default=str) description = mongoengine.StringField(default=str) permission = mongoengine.IntField(required=True, default=int) users = mongoengine.ListField(mongoengine.ObjectIdField()) extensions = mongoengine.MapField(field=mongoengine.StringField())
class Job(me.DynamicDocument): jobid = me.UUIDField(binary=False, required=True) name = me.StringField(max_length=128, default="Unknown") failed = me.BooleanField(default=False) reason = me.StringField(max_length=512) version = me.StringField(max_length=10, default=baleen.get_version) started = me.DateTimeField(default=datetime.now, required=True) finished = me.DateTimeField(default=None) updated = me.DateTimeField(default=datetime.now, required=True) errors = me.MapField(field=me.IntField()) counts = me.MapField(field=me.IntField()) totals = me.MapField(field=me.IntField()) @classmethod def pre_save(cls, sender, document, **kwargs): document.updated = datetime.now() meta = { 'collection': 'jobs', } def __unicode__(self): return "{} Job {}".format(self.name, self.jobid)
class HistoryStep(me.EmbeddedDocument): log_message = me.StringField() public_view = me.DictField() public_view_delta = me.DictField() player_views = me.MapField(me.DictField()) player_view_deltas = me.MapField(me.DictField())
class ExampleCardGameModel(TurnBasedModel): hands = me.MapField(me.ListField(me.IntField)) deck = me.ListField(me.IntField()) stack = me.ListField(me.IntField()) viewing = me.ListField(me.IntField(), required=False) @property def current_total(self): return sum(self.stack) def setup(self): cards_to_deal = 4 if self.config.use_special_cards else 5 cards = list(range(1, 6)) * 5 random.shuffle(cards) self.hands = {} self.hands[self.config.players[0].id] = cards[0:cards_to_deal] self.hands[ self.config.players[1].id] = cards[cards_to_deal:cards_to_deal * 2] self.deck = cards[cards_to_deal * 2:] if self.config.use_special_cards: for p in self.config.players: self.hands[p.id].append( self.config.player_opts[p.id].special_card) self.stack = [] self.viewing = None super(ExampleCardGameModel, self).setup() def update_legal_actions(self): if self.result is not None: self._cached_actions = { p.id: [['restart']] for p in self.config.players } else: super(ExampleCardGameModel, self).update_legal_actions() def get_actions_for_active_player(self): if self.viewing: yield from [['pick', n] for n in sorted(set(self.viewing))] else: yield from [['play', n] for n in sorted(set(self.hands[self.active_user.id]))] c = Counter(self.hands[self.active_user.id]) for v in c: if c[v] > 1: yield ['discard_double', v] def apply_action(self, user, action, log_callback): if not self.is_legal_action(user, action): return False if action == ['restart']: self.result = None self.setup() log_callback( f'New game starting. {self.active_user.nickname} plays first.') else: self.apply_action_for_active_player(action, log_callback) self.update_legal_actions() def apply_action_for_active_player(self, action, log_callback): move_type = action[0] if move_type == 'play': self.stack.append(int(action[1])) log_callback( f'{self.active_user.nickname} plays a {action[1]}, making the total {self.current_total}.' ) self.hands[self.active_user.id].remove(int(action[1])) if self.current_total == 21: self.result = {'winner': self.active_user.id} log_callback( f'{self.active_user.nickname} wins by reaching 21.') elif self.current_total > 21: self.result = { 'winner': (self.turn_order[1 - self.active_player_index]).id } winner_nick = [ p.nickname for p in self.config.players if p.id == self.result['winner'] ][0] log_callback( f'{self.active_user.nickname} went over 21. {winner_nick} wins.' ) else: self.hands[self.active_user.id].append(self.deck.pop()) self.advance_turn() elif move_type == 'discard_double': log_callback( f'{self.active_user.nickname} discards a pair of {action[1]}s.' ) self.hands[self.active_user.id].remove(int(action[1])) self.hands[self.active_user.id].remove(int(action[1])) self.viewing = self.deck[:5] self.deck = self.deck[5:] elif move_type == 'pick': log_callback( f'{self.active_user.nickname} chooses a card from the top five of the deck.' ) self.hands[self.active_user.id].append(int(action[1])) self.viewing.remove(int(action[1])) random.shuffle(self.viewing) self.deck.extend(self.viewing) log_callback( f'{len(self.viewing)} cards are shuffled and placed on the bottom.' ) self.viewing = None self.advance_turn() def get_public_view(self): result = super(ExampleCardGameModel, self).get_public_view() result['current_total'] = self.current_total result['deck_count'] = len(self.deck) result['stack'] = self.stack.copy() result['hand_counts'] = { p.id: len(self.hands[p.id]) for p in self.config.players } return result def get_player_view(self, user): result = super(ExampleCardGameModel, self).get_player_view(user) result['my_hand'] = self.hands[user.id].copy() if self.viewing and user == self.active_user: result['viewing'] = self.viewing.copy() return result
class Fight(warcraftlogs_base.EmbeddedDocument): fight_id = me.IntField(primary_key=True) start_time: arrow.Arrow = mongoengine_arrow.ArrowDateTimeField() # fight duration in milliseconds duration: int = me.IntField(default=0) # deprecated in favor of "duration". end_time_old: arrow.Arrow = mongoengine_arrow.ArrowDateTimeField( db_field="end_time") boss_id = me.IntField() players: typing.Dict[str, Player] = me.MapField( me.EmbeddedDocumentField(Player)) boss: Boss = me.EmbeddedDocumentField(Boss) composition = me.DictField() deaths = me.IntField(default=0) ilvl = me.FloatField(default=0) damage_taken = me.IntField(default=0) # boss percentage at the end. (its 0.01 for kills) percent = me.FloatField(default=0) kill = me.BooleanField(default=True) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.report = None if self.boss: self.boss.fight = self for player in self.players.values(): player.fight = self def __str__(self): return f"{self.__class__.__name__}(id={self.fight_id}, players={len(self.players)})" def summary(self): raid_boss_name = self.boss and self.boss.raid_boss and self.boss.raid_boss.full_name_slug return { "report_id": self.report.report_id, "fight_id": self.fight_id, "percent": self.percent, "kill": self.kill, "duration": self.duration, "time": self.start_time.timestamp(), "boss": { "name": raid_boss_name }, } def as_dict(self, player_ids: typing.List[int] = None) -> dict: # Get players players = list(self.players.values()) if player_ids: players = [ player for player in players if player.source_id in player_ids ] players = sorted(players, key=lambda player: (player.spec.role, player.spec, player.name)) # Return return { **self.summary(), "players": [player.as_dict() for player in players], "boss": self.boss.as_dict() if self.boss else {}, } ########################## # Attributes @property def end_time(self) -> arrow.Arrow: return self.start_time.shift(seconds=self.duration / 1000) @property def start_time_rel(self) -> int: """Fight start time, relative the parent report (in milliseconds).""" return 1000 * int(self.start_time.timestamp() - self.report.start_time.timestamp()) @property def end_time_rel(self) -> int: """fight end time, relative to the report (in milliseconds).""" return 1000 * int(self.end_time.timestamp() - self.report.start_time.timestamp()) @property def raid_boss(self) -> RaidBoss: return RaidBoss.get(id=self.boss_id) ################################# # Methods # def get_player(self, **kwargs) -> Player: """Returns a single Player based on the kwargs.""" return utils.get(self.players.values(), **kwargs) def get_players(self, source_ids: typing.List[int] = None): """Gets multiple players based on source id.""" players = list(self.players.values()) if source_ids: players = [ player for player in players if player.source_id in source_ids ] return [player for player in players if player] def add_boss(self, boss_id) -> RaidBoss: self.boss_id = boss_id self.boss = Boss(boss_id=boss_id) self.boss.fight = self return self.boss ############################################################################ # Query # @property def table_query_args(self) -> str: return f"fightIDs: {self.fight_id}, startTime: {self.start_time_rel}, endTime: {self.end_time_rel}" ############################################################################ # Summary # def get_summary_query(self): """Get the Query to load the fights summary.""" if self.players: return "" return textwrap.dedent(f"""\ reportData {{ report(code: "{self.report.report_id}") {{ summary: table({self.table_query_args}, dataType: Summary) }} }} """) def process_players(self, players_data): if not players_data: logger.warning("players_data is empty") return total_damage = players_data.get("damageDone", []) total_healing = players_data.get("healingDone", []) for composition_data in players_data.get("composition", []): spec_data = composition_data.get("specs", []) if not spec_data: logger.warning("Player has no spec: %s", composition_data.get("name")) continue spec_data = spec_data[0] spec_name = spec_data.get("spec") class_name = composition_data.get("type") spec = WowSpec.get(name_slug_cap=spec_name, wow_class__name_slug_cap=class_name) if not spec: logger.warning("Unknown Spec: %s", spec_name) continue # Get Total Damage or Healing spec_role = spec_data.get("role") total_data = total_healing if spec_role == "healer" else total_damage for data in total_data: if data.get("id", -1) == composition_data.get("id"): total = data.get("total", 0) / (self.duration / 1000) break else: total = 0 # create and return yield player object player = Player() player.fight = self player.spec_slug = spec.full_name_slug player.source_id = composition_data.get("id") player.name = composition_data.get("name") player.total = int(total) player.process_death_events(players_data.get("deathEvents", [])) self.players[str(player.source_id)] = player # call this before filtering to always get the full comp self.composition = get_composition(self.players.values()) def process_overview(self, data): """Process the data retured from an Overview-Query.""" data = data.get("reportData") or data summary_data = utils.get_nested_value(data, "report", "summary", "data") or {} self.duration = self.duration or summary_data.get("totalTime", 0) self.process_players(summary_data) async def load_summary(self, force=False): """Load this fights Summary. Args: force(boolean, optional): load even if its already loaded """ if force: self.players = {} if self.players: return "" query = self.get_summary_query() result = await self.client.query(query) self.process_overview(result) # alias to set the default "load"-behaviour process_query_result = process_overview get_query = get_summary_query ############################################################################ # Load Player: # async def load_players(self, player_ids: typing.List[int] = None): if not self.players: await self.load_summary() # Get Players to load players_to_load = self.get_players(player_ids) players_to_load = [ player for player in players_to_load if not player.casts ] # see if we need to load the boss boss = [] if (self.boss and self.boss.casts) else [self.boss] if not (players_to_load or boss): return # load await self.load_many(players_to_load + boss) # re-add them to the dict, as otherwise we get some mongodb issues for actor in players_to_load: self.players[str(actor.source_id)] = actor
class Rule(me.Document): """The base Rule mongoengine model. The Rule class defines the base schema of all rule types. All documents of any Rule subclass will be stored in the same mongo collection. All Rule subclasses MUST define a `_controller_cls` class attribute and a backend plugin. Controllers are used to perform actions on instances of Rule, such as adding or updating. Backend plugins are used to transform a Rule into the corresponding query to be executed against a certain data storage. Different types of rules, such as a rule on monitoring metrics or a rule on logging data, should also define and utilize their respective backend plugins. For instance, a rule on monitoring data, which is stored in a TSDB like Graphite, will have to utilize a different plugin than a rule on logging data, stored in Elasticsearch, in order to successfully query the database. The Rule class is mainly divided into two categories: 1. Arbitrary rules - defined entirely by the user. This type of rules gives users the freedom to execute arbitrary queries on arbitrary data. The query may include (nested) expressions and aggregations on arbitrary fields whose result will be evaluated against a threshold based on a comparison operator (=, <, etc). 2. Resource rules - defined by using Mist.io UUIDs and tags. This type of rules can be used to easily setup alerts on resources given their tags or UUIDs. In this case, users have to explicitly specify the target metric's name, aggregation function, and resources either by their UUIDs or tags. This type of rules allows for easier alert configuration on known resources in the expense of less elastic query expressions. The Rule base class can be used to query the database and fetch documents created by any Rule subclass. However, in order to add new rules one must use one of the Rule subclasses, which represent different rule type, each associated with the corresponding backend plugin. """ id = me.StringField(primary_key=True, default=lambda: uuid.uuid4().hex) title = me.StringField(required=True) owner_id = me.StringField(required=True) # Specifies a list of queries to be evaluated. Results will be logically # ANDed together in order to decide whether an alert should be raised. queries = me.EmbeddedDocumentListField(QueryCondition, required=True) # Defines the time window and frequency of each search. window = me.EmbeddedDocumentField(Window, required=True) frequency = me.EmbeddedDocumentField(Frequency, required=True) # Associates a reminder offset, which will cause an alert to be fired if # and only if the threshold is exceeded for a number of trigger_after # intervals. trigger_after = me.EmbeddedDocumentField( TriggerOffset, default=lambda: TriggerOffset(period='minutes')) # Defines a list of actions to be executed once the rule is triggered. # Defaults to just notifying the users. actions = me.EmbeddedDocumentListField( BaseAlertAction, required=True, default=lambda: [NotificationAction()]) # Disable the rule organization-wide. disabled = me.BooleanField(default=False) # Fields passed to celerybeat as optional arguments. queue = me.StringField() exchange = me.StringField() routing_key = me.StringField() soft_time_limit = me.IntField() # Fields updated by the scheduler. last_run_at = me.DateTimeField() run_immediately = me.BooleanField() total_run_count = me.IntField(min_value=0, default=0) total_check_count = me.IntField(min_value=0, default=0) # Field updated by celery workers. This is where celery workers keep state. states = me.MapField(field=me.EmbeddedDocumentField(RuleState)) meta = { 'strict': False, 'collection': 'rules', 'allow_inheritance': True, 'indexes': [{ 'fields': ['owner_id', 'title'], 'sparse': False, 'unique': True, 'cls': False, }] } _controller_cls = None _backend_plugin = None _data_type_str = None def __init__(self, *args, **kwargs): super(Rule, self).__init__(*args, **kwargs) if self._controller_cls is None: raise TypeError( "Cannot instantiate self. %s is a base class and cannot be " "used to insert or update alert rules and actions. Use a " "subclass of self that defines a `_controller_cls` class " "attribute derived from `mist.api.rules.base:BaseController`, " "instead." % self.__class__.__name__) if self._backend_plugin is None: raise NotImplementedError( "Cannot instantiate self. %s does not define a backend_plugin " "in order to evaluate rules against the corresponding backend " "storage." % self.__class__.__name__) if self._data_type_str not in ( 'metrics', 'logs', ): raise TypeError( "Cannot instantiate self. %s is a base class and cannot be " "used to insert or update rules. Use a subclass of self that " "defines a `_backend_plugin` class attribute, as well as the " "requested data's type via the `_data_type_str` attribute, " "instead." % self.__class__.__name__) self.ctl = self._controller_cls(self) @classmethod def add(cls, auth_context, title=None, **kwargs): """Add a new Rule. New rules should be added by invoking this class method on a Rule subclass. Arguments: owner: instance of mist.api.users.models.Organization title: the name of the rule. This must be unique per Organization kwargs: additional keyword arguments that will be passed to the corresponding controller in order to setup the self """ try: cls.objects.get(owner_id=auth_context.owner.id, title=title) except cls.DoesNotExist: rule = cls(owner_id=auth_context.owner.id, title=title) rule.ctl.set_auth_context(auth_context) rule.ctl.add(**kwargs) else: raise BadRequestError('Title "%s" is already in use' % title) return rule @property def owner(self): """Return the Organization (instance) owning self. We refrain from storing the owner as a me.ReferenceField in order to avoid automatic/unwanted dereferencing. """ return Organization.objects.get(id=self.owner_id) @property def plugin(self): """Return the instance of a backend plugin. Subclasses MUST define the plugin to be used, instantiated with `self`. """ return self._backend_plugin(self) # NOTE The following properties are required by the scheduler. @property def name(self): """Return the name of the celery task. This must be globally unique, since celerybeat-mongo uses schedule names as keys of the dictionary of schedules to run. """ return 'Org(%s):Rule(%s)' % (self.owner_id, self.id) @property def task(self): """Return the celery task to run. This is the most basic celery task that should be used for most rule evaluations. However, subclasses may provide their own property or class attribute based on their needs. """ return 'mist.api.rules.tasks.evaluate' @property def args(self): """Return the args of the celery task.""" return (self.id, ) @property def kwargs(self): """Return the kwargs of the celery task.""" return {} @property def expires(self): """Return None to denote that self is not meant to expire.""" return None @property def enabled(self): """Return True if the celery task is currently enabled. Subclasses MAY override or extend this property. """ return not self.disabled @property def schedule(self): """Return a celery schedule instance. Used internally by the scheduler. Subclasses MUST NOT override this. """ return celery.schedules.schedule(self.frequency.timedelta) def is_arbitrary(self): """Return True if self is arbitrary. Arbitrary rules lack a list of `selectors` that refer to resources either by their UUIDs or by tags. Such a list makes it easy to setup rules referencing specific resources without the need to provide the raw query expression. """ return 'selectors' not in type(self)._fields def clean(self): # FIXME This is needed in order to ensure rule name convention remains # backwards compatible with the old monitoring stack. However, it will # have to change in the future due to uniqueness constrains. if not self.title: self.title = 'rule%d' % self.owner.rule_counter def as_dict(self): return { 'id': self.id, 'title': self.title, 'queries': [query.as_dict() for query in self.queries], 'window': self.window.as_dict(), 'frequency': self.frequency.as_dict(), 'trigger_after': self.trigger_after.as_dict(), 'actions': [action.as_dict() for action in self.actions], 'disabled': self.disabled, 'data_type': self._data_type_str, } def __str__(self): return '%s %s of %s' % (self.__class__.__name__, self.title, self.owner)
class Host(common.BaseDocument): meta = { 'ordering': ['-updated'], 'queryset_class': common.SerializableQuerySet, 'indexes': ['created', 'updated', 'hostname'] } hostname = mongoengine.StringField(required=True) pid = mongoengine.IntField(required=True) mac_address = mongoengine.StringField() job_slots = mongoengine.MapField(field=mongoengine.IntField(), default={}) job_imports = mongoengine.ListField(field=mongoengine.StringField(), default=[]) platform = mongoengine.DictField() boot_time = mongoengine.DateTimeField() python_version = mongoengine.StringField() python_packages = mongoengine.ListField(field=mongoengine.StringField()) def history(self, offset=0, limit=30, step=0): step_filter = {} if step and step > 1: step_filter = {'index__mod': (step, 0)} statuses = HostStatus.objects( host=self, **step_filter).order_by('-created')[offset:limit] return [s.to_safe_dict(with_host=False) for s in statuses] def alive(self): recent_count = HostStatus.objects(host=self, created__gte=datetime.utcnow() - timedelta(minutes=0.5)).count() return recent_count > 0 def last_seen_alive(self): last_status = HostStatus.objects( host=self).order_by('-created').only('created').first() if not last_status or not last_status.created: return None return last_status.created def to_safe_dict(self, alive=False, with_history=False, offset=0, limit=30, step=0): r = super(Host, self).to_safe_dict() if alive: r['alive'] = self.alive() r['last_seen_alive'] = self.last_seen_alive() if with_history: r['history'] = self.history(offset=offset, limit=limit, step=step) return r def update_status(self): partitions = [] try: for f in psutil.disk_partitions(): usage = psutil.disk_usage(path=f.mountpoint) p = { 'type': f.fstype, 'device': f.device, 'mountpoint': f.mountpoint, 'total': usage.total, 'used': usage.used, 'percent': usage.percent, } partitions.append(p) except Exception as e: pass self_process = psutil.Process(os.getpid()) processes = [{ 'ppid': self_process.ppid(), 'pid': self_process.pid, 'cmd': ' '.join(self_process.cmdline()) }] for c in self_process.children(): try: processes.append({ 'ppid': c.ppid(), 'pid': c.pid, 'cmd': ' '.join(c.cmdline()) }) except psutil.Error: pass self.host_status_index += 1 status = HostStatus() status.index = self.host_status_index status.host = self status.current_jobs = [{ 'uuid': j.uuid, 'type': j._cls } for j in self.client_service.current_jobs] virtual_memory = psutil.virtual_memory() swap_memory = psutil.swap_memory() status.system_status = { 'processes': processes, 'cpu': { 'percent': psutil.cpu_percent(), 'percents': psutil.cpu_percent(percpu=True) }, 'memory': { 'virtual': { 'total': virtual_memory.total, 'used': virtual_memory.used, 'percent': virtual_memory.percent, }, 'swap': { 'total': swap_memory.total, 'used': swap_memory.used, 'percent': swap_memory.percent, }, }, 'disk': partitions, #'disk_io': safe_dict(psutil.disk_io_counters, perdisk=False) } status.save() @classmethod def localhost(cls): hostname = socket.gethostname() hosts = Host.objects(hostname=hostname) if not hosts: logging.info('Host unknown. Initializing it in the database...') host = Host() host.hostname = hostname host.mac_address = tbx.network.get_mac_address() host.platform = tbx.code.safe_dict(platform.uname) host.host_status_index = 1 logging.info( "Now, configure Host '%s' through API or Web UI to be able to use it." % hostname) else: host = hosts[0] last_status = HostStatus.objects( host=host).order_by('-created').first() if not last_status: host.host_status_index = 1 else: host.host_status_index = last_status.index logging.info("Host '%s' already found in database." % hostname) host.boot_time = datetime.fromtimestamp(psutil.boot_time()) host.pid = os.getpid() host.python_version = sys.version.split(' ')[0] host.python_packages = sorted([ "%s (%s)" % (i.key, i.version) for i in pkg_resources.working_set ]) host.save() logging.info("Host '%s' config updated in database." % hostname) return host def update_slots(self, job_slots=None): from jobmanager.common.job import Job, JobTask job_classes = tbx.code.get_subclasses(Job) job_tasks = tbx.code.get_subclasses(JobTask) if not job_slots: logging.info( 'Job Slots not set in env or command line args. Setting to default job defined amount.' ) job_slots = { k.__name__: k.default_slot_amount() for k in job_classes } available_class_names = {c.__name__ for c in job_classes} previous_class_names = set(self.job_slots.keys()) all_class_names = previous_class_names | available_class_names for class_name in all_class_names: if class_name not in job_slots.keys(): self.job_slots[class_name] = 0 else: self.job_slots[class_name] = job_slots[class_name] logging.info(" - Job type found : %s (%d slots)" % (class_name, self.job_slots[class_name])) logging.info("Also found following job tasks : %s" % ', '.join([k.__name__ for k in job_tasks])) return self.save() def check_capacity(self): if self.job_slots: #logging.info("Jobs types allowed : %s" % (', '.join(self.job_slots.keys()))) logging.info("Jobs capacity :") total_capacity = 0 for c in self.job_slots: logging.info(" - %s\t: %d" % (c, self.job_slots[c])) total_capacity += self.job_slots[c] if total_capacity == 0: logging.error( "No job capacities setup. Configure host to add slots to some job types." ) raise common.ConfigurationException( "No job capacities setup. Configure host to add slots to some job types (see --add-slot option)." ) else: logging.error( "No job class found to be run. Configure host to import packages that contain job subclasses." ) raise common.ConfigurationException( "No Job sub-class found in imports. Please configure host (see --import option)." ) def do_import(self, imports): return common.safely_import_from_name(imports) @classmethod def get_all_alive(cls): raise NotImplementedError()
class Job(me.DynamicDocument): jobid = me.UUIDField(binary=False, required=True) name = me.StringField(max_length=128, default="Unknown") failed = me.BooleanField(default=False) reason = me.StringField(max_length=512) version = me.StringField(max_length=10, default=baleen.get_version) started = me.DateTimeField(default=datetime.now, required=True) finished = me.DateTimeField(default=None) updated = me.DateTimeField(default=datetime.now, required=True) errors = me.MapField(field=me.IntField()) counts = me.MapField(field=me.IntField()) totals = me.MapField(field=me.IntField()) @classmethod def pre_save(cls, sender, document, **kwargs): document.updated = datetime.now() meta = { 'collection': 'jobs', } def duration(self, humanize=False): """ Returns the timedelta of the duration. """ finished = self.finished or datetime.now() delta = finished - self.started if humanize: return humanizedelta(days=delta.days, seconds=delta.seconds, microseconds=delta.microseconds) return delta @property def bootstrap_class(self): """ Uses the duration to determine the colorization of the job. """ if self.finished and self.failed: return "danger" if self.finished and not self.failed: if self.duration() > timedelta(minutes=30): return "warning" return "success" if not self.finished: if self.duration() < timedelta(minutes=30): return "success" elif timedelta(minutes=30) < self.duration() < timedelta(hours=2): return "warning" else: return "danger" return "" def __unicode__(self): return "{} Job {}".format(self.name, self.jobid)
class Doc(me.Document): id = me.IntField(primary_key=True, default=1) map = me.MapField(me.EmbeddedDocumentField(MappedDoc)) str = me.MapField(me.StringField())
class Report(warcraftlogs_base.EmbeddedDocument): """Defines a Report read from WarcraftLogs.com and stores in our DB.""" # 16 digit unique id/code as used on warcraftlogs report_id: str = me.StringField(primary_key=True) # time the report (!) has started. The first fight might start later start_time: arrow.Arrow = mongoengine_arrow.ArrowDateTimeField(default=lambda: arrow.get(0)) # title of the report title: str = me.StringField() zone_id: int = me.IntField(default=0) # The guild that the report belongs to. None if it was a logged as a personal report guild: str = me.StringField(default="") # The user that uploaded the report. owner: str = me.StringField(default="") # fights in this report keyed by fight_id. (they may or may not be loaded) fights: typing.Dict[str, Fight] = me.MapField(me.EmbeddedDocumentField(Fight), default={}) # players in this report. # Note: not every player might participate in every fight. players: typing.Dict[str, Player] = me.MapField(me.EmbeddedDocumentField(Player), default={}) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # convert list to dict for old DB entries # if isinstance(self.fights, list): # self.fights = {fight.fight_id: fight for fight in self.fights} for fight in self.fights.values(): fight.report = self def __str__(self): return f"<BaseReport({self.report_id}, num_fights={len(self.fights)})>" ########################## # Attributes # def as_dict(self): """Return a Summary/Overview about this report.""" info = { "title": self.title, "report_id": self.report_id, "date": int(self.start_time.timestamp()), "zone_id": self.zone_id, "guild": self.guild, "owner": self.owner, } # for players and fights we only include essential data info["fights"] = {fight.fight_id: fight.summary() for fight in self.fights.values()} info["players"] = {player.source_id: player.summary() for player in self.players.values()} return info ########################## # Methods # def add_fight(self, **fight_data): """Add a new Fight to this Report.""" # skip trash fights boss_id = fight_data.get("encounterID") if not boss_id: return fight = Fight() fight.fight_id = fight_data.get("id", "0") fight.report = self fight.percent = fight_data.get("fightPercentage") fight.kill = fight_data.get("kill", True) # Fight: Time/Duration start_time = fight_data.get("startTime", 0) / 1000 end_time = fight_data.get("endTime", 0) / 1000 fight.start_time = self.start_time.shift(seconds=start_time) fight.duration = (end_time - start_time) * 1000 # Fight: Boss fight.boss = Boss(boss_id=boss_id) if fight.boss: # could be a boss unknown to Lorrgs fight.boss.fight = fight # store and return self.fights[str(fight.fight_id)] = fight return fight def get_fight(self, fight_id: int): """Get a single fight from this Report.""" return self.fights.get(str(fight_id)) def get_fights(self, *fight_ids: int): """Get a multiple fights based of their fight ids.""" fights = [self.get_fight(fight_id) for fight_id in fight_ids] return [f for f in fights if f] # filter out nones def add_player(self, **actor_data): # pets if actor_data.get("subType") == "Unknown": return # guess spec from the icon # WCL gives us an icon matching the spec, IF a player # played the same spec in all fights inside a report. # Otherwise it only includes a class-name. icon_name = actor_data.get("icon", "") spec_slug = icon_name.lower() if "-" in icon_name else "" # create the new player player = warcraftlogs_actor.Player() player.source_id = actor_data.get("id") player.name = actor_data.get("name") player.class_slug = actor_data.get("subType", "").lower() player.spec_slug = spec_slug # add to to the report self.players[str(player.source_id)] = player ############################################################################ # Query # def get_query(self): """Get the Query to load this Reports Overview.""" return textwrap.dedent(f""" reportData {{ report(code: "{self.report_id}") {{ title zone {{name id}} startTime owner {{ name }} guild {{ name server {{ name }} }} masterData {{ actors(type: "Player") {{ name id subType icon # the icon includes the spec name }} }} fights {{ id encounterID startTime endTime fightPercentage kill }} }} }} """) def process_master_data(self, master_data: typing.Dict[str, typing.Any]): """Create the Players from the passed Report-MasterData""" if not master_data: return # clear out any old instances self.players = {} for actor_data in master_data.get("actors", []): self.add_player(**actor_data) def process_report_fights(self, fights_data: typing.List[typing.Dict]): """Update the Fights in this report.""" # clear out any old data self.fights = {} fights_data = fights_data or [] for fight_data in fights_data: self.add_fight(**fight_data) def process_query_result(self, query_result: dict): report_data = query_result.get("report", {}) # Update the Report itself self.title = report_data.get("title", "") self.start_time = arrow.get(report_data.get("startTime", 0)) zone = report_data.get("zone") or {} # might contain a "None" in the results self.zone_id = zone.get("id") or -1 self.owner = report_data.get("owner", {}).get("name", "") guild_info: typing.Dict = report_data.get("guild") or {} self.guild = guild_info.get("name") or "" self.process_master_data(report_data.get("masterData")) self.process_report_fights(report_data.get("fights")) async def load_summary(self): await self.load() async def load_fight(self, fight_id: int, player_ids=typing.List[int]): """Load a single Fight from this Report.""" fight = self.fights[str(fight_id)] if not fight: raise ValueError("invalid fight id") await fight.load_players(player_ids=player_ids) async def load_fights(self, fight_ids: typing.List[int], player_ids: typing.List[int]): await self.client.ensure_auth() if not self.fights: await self.load_summary() for ids in utils.chunks(fight_ids, 10): logger.info(f"loading fights: {ids}") tasks = [self.load_fight(fight_id=fight_id, player_ids=player_ids) for fight_id in ids] await asyncio.gather(*tasks)