class Car(me.Document): id = me.IntField(primary_key=True) model = me.StringField() color = me.StringField() year = me.IntField()
class MongoDocument(mongoengine.Document): meta = {'collection': COLLECTION} number = mongoengine.IntField()
class Count(mongoengine.EmbeddedDocument): total_count = mongoengine.IntField()
class Machine(OwnershipMixin, me.Document): """The basic machine model""" id = me.StringField(primary_key=True, default=lambda: uuid.uuid4().hex) cloud = me.ReferenceField('Cloud', required=True) owner = me.ReferenceField('Organization', required=True) location = me.ReferenceField('CloudLocation', required=False) size = me.ReferenceField('CloudSize', required=False) network = me.ReferenceField('Network', required=False) subnet = me.ReferenceField('Subnet', required=False) name = me.StringField() # Info gathered mostly by libcloud (or in some cases user input). # Be more specific about what this is. # We should perhaps come up with a better name. machine_id = me.StringField(required=True) hostname = me.StringField() public_ips = me.ListField() private_ips = me.ListField() ssh_port = me.IntField(default=22) OS_TYPES = ('windows', 'coreos', 'freebsd', 'linux', 'unix') os_type = me.StringField(default='unix', choices=OS_TYPES) rdp_port = me.IntField(default=3389) actions = me.EmbeddedDocumentField(Actions, default=lambda: Actions()) extra = me.DictField() cost = me.EmbeddedDocumentField(Cost, default=lambda: Cost()) image_id = me.StringField() # libcloud.compute.types.NodeState state = me.StringField(default='unknown', choices=('running', 'starting', 'rebooting', 'terminated', 'pending', 'unknown', 'stopping', 'stopped', 'suspended', 'error', 'paused', 'reconfiguring')) machine_type = me.StringField(default='machine', choices=('machine', 'vm', 'container', 'hypervisor', 'container-host')) parent = me.ReferenceField('Machine', required=False) # We should think this through a bit. key_associations = me.EmbeddedDocumentListField(KeyAssociation) last_seen = me.DateTimeField() missing_since = me.DateTimeField() unreachable_since = me.DateTimeField() created = me.DateTimeField() monitoring = me.EmbeddedDocumentField(Monitoring, default=lambda: Monitoring()) ssh_probe = me.EmbeddedDocumentField(SSHProbe, required=False) ping_probe = me.EmbeddedDocumentField(PingProbe, required=False) # Number of vCPUs gathered from various sources. This field is meant to # be updated ONLY by the mist.api.metering.tasks:find_machine_cores task. cores = me.IntField() meta = { 'collection': 'machines', 'indexes': [{ 'fields': ['cloud', 'machine_id'], 'sparse': False, 'unique': True, 'cls': False, }, { 'fields': ['monitoring.installation_status.activated_at'], 'sparse': True, 'unique': False }], 'strict': False, } def __init__(self, *args, **kwargs): super(Machine, self).__init__(*args, **kwargs) self.ctl = MachineController(self) def clean(self): # Remove any KeyAssociation, whose `keypair` has been deleted. Do NOT # perform an atomic update on self, but rather remove items from the # self.key_associations list by iterating over it and popping matched # embedded documents in order to ensure that the most recent list is # always processed and saved. for ka in reversed(range(len(self.key_associations))): if self.key_associations[ka].keypair.deleted: self.key_associations.pop(ka) # Populate owner field based on self.cloud.owner if not self.owner: self.owner = self.cloud.owner self.clean_os_type() if self.monitoring.method not in config.MONITORING_METHODS: self.monitoring.method = config.DEFAULT_MONITORING_METHOD def clean_os_type(self): """Clean self.os_type""" if self.os_type not in self.OS_TYPES: for os_type in self.OS_TYPES: if self.os_type.lower() == os_type: self.os_type = os_type break else: self.os_type = 'unix' def delete(self): super(Machine, self).delete() mist.api.tag.models.Tag.objects(resource=self).delete() try: self.owner.mapper.remove(self) except (AttributeError, me.DoesNotExist) as exc: log.error(exc) try: if self.owned_by: self.owned_by.get_ownership_mapper(self.owner).remove(self) except (AttributeError, me.DoesNotExist) as exc: log.error(exc) def as_dict(self): # Return a dict as it will be returned to the API # tags as a list return for the ui tags = { tag.key: tag.value for tag in mist.api.tag.models.Tag.objects( resource=self).only('key', 'value') } # Optimize tags data structure for js... if isinstance(tags, dict): tags = [{ 'key': key, 'value': value } for key, value in tags.iteritems()] return { 'id': self.id, 'hostname': self.hostname, 'public_ips': self.public_ips, 'private_ips': self.private_ips, 'name': self.name, 'ssh_port': self.ssh_port, 'os_type': self.os_type, 'rdp_port': self.rdp_port, 'machine_id': self.machine_id, 'actions': {action: self.actions[action] for action in self.actions}, 'extra': self.extra, 'cost': self.cost.as_dict(), 'image_id': self.image_id, 'state': self.state, 'tags': tags, 'monitoring': self.monitoring.as_dict() if self.monitoring and self.monitoring.hasmonitoring else '', 'key_associations': [ka.as_dict() for ka in self.key_associations], 'cloud': self.cloud.id, 'location': self.location.id if self.location else '', 'size': self.size.name if self.size else '', 'cloud_title': self.cloud.title, 'last_seen': str(self.last_seen.replace(tzinfo=None) if self.last_seen else ''), 'missing_since': str( self.missing_since.replace( tzinfo=None) if self.missing_since else ''), 'unreachable_since': str( self.unreachable_since.replace( tzinfo=None) if self.unreachable_since else ''), 'created': str(self.created.replace(tzinfo=None) if self.created else ''), 'machine_type': self.machine_type, 'parent_id': self.parent.id if self.parent is not None else '', 'probe': { 'ping': (self.ping_probe.as_dict() if self.ping_probe is not None else PingProbe().as_dict()), 'ssh': (self.ssh_probe.as_dict() if self.ssh_probe is not None else SSHProbe().as_dict()), }, 'cores': self.cores, 'network': self.network.id if self.network else '', 'subnet': self.subnet.id if self.subnet else '', 'owned_by': self.owned_by.id if self.owned_by else '', 'created_by': self.created_by.id if self.created_by else '', } def __str__(self): return 'Machine %s (%s) in %s' % (self.name, self.id, self.cloud)
class DiskUsage(me.EmbeddedDocument): used = me.IntField(default=0) free = me.IntField(default=0) percent = me.FloatField(default=0) # show in percent total = me.IntField(required=True, default=0)
class ReferenceDoc(me.Document): field = me.IntField(primary_key=True, default=42)
class BonusCardTrx(mge.EmbeddedDocument): trx_id = mge.StringField(required=True) # номер транзакции trx_value = mge.IntField(required=True) # Сколько начислено бонусных единиц(миль) departure_airport = mge.StringField(required=True) # откуда arrival_airport = mge.StringField(required=True) # куда flight_date = mge.DateTimeField(required=True) # дата полёта
class Image(mongoengine.EmbeddedDocument): path = mongoengine.StringField(required=True) width = mongoengine.IntField(required=True) height = mongoengine.IntField(required=True) content_type = mongoengine.StringField(required=True)
class UserProfile(me.Document): """ User profile model which has other social data and related user info """ first_name = me.StringField(max_length=30) last_name = me.StringField(max_length=30) google = me.DictField() github = me.DictField() avatar = me.URLField() facebook_handle = me.StringField(max_length=100) twitter_handle = me.StringField(max_length=100) linkedin_handle = me.StringField(max_length=100) description = me.StringField(max_length=500) company = me.StringField(max_length=100) designation = me.StringField(max_length=100) mobile = me.IntField() created_at = me.DateTimeField(default=datetime.utcnow) updated_at = me.DateTimeField(default=datetime.utcnow) created_by = me.ReferenceField('User', required=False) updated_by = me.ReferenceField('User', required=False) is_active = me.BooleanField(default=True) def __str__(self): return ' '.join([self.first_name, self.last_name]) def __init__(self, *args, **kwargs): super(UserProfile, self).__init__(*args, **kwargs) @classmethod def pre_save(cls, sender, document, **kwargs): if document.google: if UserProfile.objects.filter( google__email=document.google['email']).count() > 0: raise ValidationError('This account is already registered.') if document.github: if UserProfile.objects.filter( github__email=document.github['email']).count() > 0: raise ValidationError('This account is already registered.') if document.description and len(document.description) > 500: raise ValidationError('Description exceeds 500 characters') if document.facebook_handle and len(document.facebook_handle) > 100: raise ValidationError('Facebook id exceeds 100 characters') if document.twitter_handle and len(document.twitter_handle) > 100: raise ValidationError('Twitter id exceeds 100 characters') if document.linkedin_handle and len(document.linkedin_handle) > 100: raise ValidationError('Linked-in id exceeds 100 characters') if document.company and len(document.company) > 100: raise ValidationError('Company exceeds 100 characters') if document.designation and len(document.designation) > 100: raise ValidationError('Designation exceeds 100 characters') if document.first_name and len(document.first_name) > 30: raise ValidationError('First name exceeds 30 characters') if document.last_name and len(document.last_name) > 30: raise ValidationError('Last name exceeds 30 characters') if document.mobile and len(str(document.mobile)) > 13: raise ValidationError('Mobile number exceeds 13 characters') @classmethod def post_save(cls, sender, document, **kwargs): if not document.first_name and not document.last_name: first_name = last_name = '' if document.google: if 'first_name' in document.google.keys(): first_name = document.google['first_name'] if 'last_name' in document.google.keys(): last_name = document.google['last_name'] if document.github: try: first_name, last_name = document.github['name'].split(' ') except ValueError: first_name, last_name = document.github['name'], '' except KeyError: pass document.update(set__first_name=first_name, set__last_name=last_name) def save(self, *args, **kwargs): super(UserProfile, self).save(*args, **kwargs) self.reload() def update(self, *args, **kwargs): super(UserProfile, self).update(*args, **kwargs) self.reload() def to_json(self, fields=None, exclude=None): return json_dumper(self, fields, exclude)
class Schedule(me.Document, ConditionalClassMixin): """Abstract base class for every schedule attr mongoengine model. This model is based on celery periodic task and creates defines the fields common to all schedules of all types. For each different schedule type, a subclass should be created adding any schedule specific fields and methods. Documents of all Schedule subclasses will be stored on the same mongo collection. One can perform a query directly on Schedule to fetch all cloud types, like this: Schedule.objects(owner=owner).count() """ condition_resource_cls = Machine meta = { 'collection': 'schedules', 'allow_inheritance': True, 'indexes': [ { 'fields': ['owner', 'name', 'deleted'], 'sparse': False, 'unique': True, 'cls': False, }, ], } id = me.StringField(primary_key=True, default=lambda: uuid4().hex) name = me.StringField(required=True) description = me.StringField() deleted = me.DateTimeField() owner = me.ReferenceField(Organization, required=True) # celery periodic task specific fields queue = me.StringField() exchange = me.StringField() routing_key = me.StringField() soft_time_limit = me.IntField() # mist specific fields schedule_type = me.EmbeddedDocumentField(BaseScheduleType, required=True) task_type = me.EmbeddedDocumentField(BaseTaskType, required=True) # celerybeat-mongo specific fields expires = me.DateTimeField() start_after = me.DateTimeField() task_enabled = me.BooleanField(default=False) run_immediately = me.BooleanField() last_run_at = me.DateTimeField() total_run_count = me.IntField(min_value=0, default=0) max_run_count = me.IntField(min_value=0, default=0) no_changes = False def __init__(self, *args, **kwargs): # FIXME import mist.api.schedules.base super(Schedule, self).__init__(*args, **kwargs) self.ctl = mist.api.schedules.base.BaseController(self) def owner_query(self): return me.Q(cloud__in=Cloud.objects(owner=self.owner).only('id')) @classmethod def add(cls, auth_context, name, **kwargs): """Add schedule This is a class method, meaning that it is meant to be called on the class itself and not on an instance of the class. You're not meant to be calling this directly, but on a schedule class instead like this: schedule = Schedule.add(owner=owner, **kwargs) """ owner = auth_context.owner if not name: raise RequiredParameterMissingError('name') if not owner or not isinstance(owner, Organization): raise BadRequestError('owner') if Schedule.objects(owner=owner, name=name, deleted=None): raise ScheduleNameExistsError() schedule = cls(owner=owner, name=name) schedule.ctl.set_auth_context(auth_context) schedule.ctl.add(**kwargs) return schedule @property def schedule(self): if self.schedule_type: return self.schedule_type.schedule else: raise Exception("must define interval, crontab, one_off schedule") @property def args(self): mids = [ machine.id for machine in self.get_resources() if machine.state != 'terminated' ] fire_up = self.task_type.args return [self.owner.id, fire_up, self.name, mids] @property def kwargs(self): return {} @property def task(self): return self.task_type.task @property def enabled(self): if self.deleted: return False if not self.get_resources().count(): return False if self.expires and self.expires < datetime.datetime.now(): return False # if self.start_after and self.start_after < datetime.datetime.now(): # return False if self.max_run_count and ( (self.total_run_count or 0) >= self.max_run_count): return False else: return self.task_enabled def __unicode__(self): fmt = '{0.name}: {{no schedule}}' if self.schedule_type: fmt = 'name: {0.name} type: {0.schedule_type._cls}' else: raise Exception("must define interval or crontab schedule") return fmt.format(self) def validate(self, clean=True): """ Override mongoengine validate. We should validate crontab entry. Use crontab_parser for crontab expressions. The parser is a general purpose one, useful for parsing hours, minutes and day_of_week expressions. example for minutes: minutes = crontab_parser(60).parse('*/15') [0, 15, 30, 45] """ if isinstance(self.schedule_type, Crontab): cronj_entry = self.schedule_type.as_dict() try: for k, v in cronj_entry.items(): if k == 'minute': celery.schedules.crontab_parser(60).parse(v) elif k == 'hour': celery.schedules.crontab_parser(24).parse(v) elif k == 'day_of_week': celery.schedules.crontab_parser(7).parse(v) elif k == 'day_of_month': celery.schedules.crontab_parser(31, 1).parse(v) elif k == 'month_of_year': celery.schedules.crontab_parser(12, 1).parse(v) else: raise me.ValidationError( 'You should provide valid period of time') except celery.schedules.ParseException: raise me.ValidationError('Crontab entry is not valid') except Exception as exc: raise me.ValidationError('Crontab entry is not valid:%s' % exc.message) super(Schedule, self).validate(clean=True) def delete(self): super(Schedule, self).delete() Tag.objects(resource=self).delete() self.owner.mapper.remove(self) def as_dict(self): # Return a dict as it will be returned to the API last_run = '' if self.total_run_count == 0 else str(self.last_run_at) conditions = [condition.as_dict() for condition in self.conditions] sdict = { 'id': self.id, 'name': self.name, 'description': self.description or '', 'schedule': unicode(self.schedule_type), 'schedule_type': self.schedule_type.type, 'schedule_entry': self.schedule_type.as_dict(), 'task_type': str(self.task_type), 'expires': str(self.expires or ''), 'start_after': str(self.start_after or ''), 'task_enabled': self.task_enabled, 'active': self.enabled, 'run_immediately': self.run_immediately or '', 'last_run_at': last_run, 'total_run_count': self.total_run_count, 'max_run_count': self.max_run_count, 'conditions': conditions, } return sdict
class PollingSchedule(ShardedScheduleMixin, me.Document): meta = { 'allow_inheritance': True, 'strict': False, 'indexes': ['shard_id'] } # We use a unique name for easy identification and to avoid running the # same schedule twice. The name is autopopulated during the invocation of # the `clean` method. name = me.StringField(unique=True) # The following fields are defined in celerybeatmongo.models.PeriodicTask. # Here, we define no fields in the base class, and expect subclasses to # either define their fields, or simply use properties. # task = me.StringField(required=True) # args = me.ListField() # kwargs = me.DictField() # Scheduling information. Don't edit them directly, just use the model # methods. default_interval = me.EmbeddedDocumentField( PollingInterval, required=True, default=PollingInterval(every=0)) override_intervals = me.EmbeddedDocumentListField(PollingInterval) # Optional arguments. queue = me.StringField() exchange = me.StringField() routing_key = me.StringField() soft_time_limit = me.IntField() # Used internally by the scheduler. last_run_at = me.DateTimeField() total_run_count = me.IntField(min_value=0) run_immediately = me.BooleanField() def get_name(self): """Construct name based on self.task""" try: return self.task.split('.')[-1] except NotImplementedError: return '%s: No task specified.' % self.__class__.__name__ def clean(self): """Automatically set value of name""" self.name = self.get_name() @property def task(self): """Return task name for this schedule Subclasses should define an attribute, property or field to do this. """ raise NotImplementedError() @property def args(self): """Return task args for this schedule""" return [str(self.id)] @property def kwargs(self): """Return task kwargs for this schedule""" return {} @property def enabled(self): """Whether this task is currently enabled or not""" return bool(self.interval.timedelta) @property def interval(self): """Merge multiple intervals into one Returns a dynamic PollingInterval, with the highest frequency of any override schedule or the default schedule. """ interval = self.default_interval for i in self.override_intervals: if not i.expired(): if not interval.timedelta or i.timedelta < interval.timedelta: interval = i return interval @property def schedule(self): """Return a celery schedule instance This is used internally by celerybeatmongo scheduler """ return celery.schedules.schedule(self.interval.timedelta) @property def expires(self): return None def add_interval(self, interval, ttl=300, name=''): """Add an override schedule to the scheduled task Override schedules must define an interval in seconds, as well as a TTL (time to live), also in seconds. Override schedules cannot be removed, so short TTL's should be used. You can however add a new override schedule again, thus practically extending the time where an override is in effect. Override schedules can only increase, not decrease frequency of the schedule, in relation to that define in the `default_interval`. """ assert isinstance(interval, int) and interval > 0 assert isinstance(ttl, int) and 0 < ttl < 3600 expires = datetime.datetime.now() + datetime.timedelta(seconds=ttl) self.override_intervals.append( PollingInterval(name=name, expires=expires, every=interval)) def cleanup_expired_intervals(self): """Remove override schedules that have expired""" self.override_intervals = [ override for override in self.override_intervals if not override.expired() ] def set_default_interval(self, interval): """Set default interval This is the interval used for this schedule, if there is no active override schedule with a smaller interval. The default interval never expires. To disable a task, simply set `enabled` equal to False. """ self.default_interval = PollingInterval(name='default', every=interval) def __unicode__(self): return "%s %s" % (self.get_name(), self.interval or '(no interval)')
class College(mgng.EmbeddedDocument): name = mgng.StringField() status = mgng.StringField() rank = mgng.IntField() essays = mgng.ListField(mgng.EmbeddedDocumentField(Essay))
class Mark(me.EmbeddedDocument): mark = me.IntField(min_value=1, max_value=12) id_item = me.ReferenceField(Items) name_item = me.StringField(min_length=1, max_length=255, required=True)
class Result(mongoengine.Document): task_id = mongoengine.StringField(required=True, unique=True) result = mongoengine.StringField() high_count = mongoengine.IntField() medium_count = mongoengine.IntField() low_count = mongoengine.IntField()
class EmbeddedB(me.EmbeddedDocument): field_b = me.IntField(default=42)
class PeriodicTaskInfo(me.Document): class Lock(me.EmbeddedDocument): id = me.StringField(default=lambda: uuid.uuid4().hex) created_at = me.DateTimeField(default=datetime.datetime.now()) # Unique task identifier. key = me.StringField(primary_key=True) # Track successes/failures for autodisabling. last_success = me.DateTimeField() last_failure = me.DateTimeField() failures_count = me.IntField(default=0) # Lock to prevent concurrent running of the same task. lock = me.EmbeddedDocumentField(Lock) # Class attributes (NOT FIELDS). This define constants on the class. # Subclasses may override by setting attributes, dynamic properties or # fields. # Task won't be autodisabled if it has failed less times. min_failures_count = 50 # Task won't be autodisabled if it has succeeded in this period. min_failures_period = datetime.timedelta(hours=1) # Task will be autodisabled if it has failed more times. max_failures_count = 100 # Task will be autodisabled if it hasn't succeeded in this period. max_failures_period = datetime.timedelta(days=2) # Lock will be broken if it was last acquired more than this time ago. break_lock_after = datetime.timedelta(seconds=300) # Abort task if previous attempt was in less than this time before. min_interval = datetime.timedelta(seconds=5) @classmethod def get_or_add(cls, key): try: task = cls.objects.get(key=key) except cls.DoesNotExist: log.info("PeriodicTaskInfo for '%s' missing, will create.", key) task = cls(key=key) try: task.save() except me.NotUniqueError: # Work around race condition where document was created since # we checked. log.warning( "PeriodicTaskInfo for '%s' creation race " "condition, will reload.", key) task = cls.objects.get(key=key) log.debug("Loaded PeriodicTaskInfo for '%s'.", key) return task def get_last_run(self): if self.last_success and self.last_failure: return max(self.last_success, self.last_failure) return self.last_success or self.last_failure def check_failures_threshold_exceeded(self): """Raise PeriodicTaskThresholdExceeed if task should be autodisabled""" now = datetime.datetime.now() # If it hasn't run recently, then allow it to run. last_run = self.get_last_run() if not last_run or now - last_run > datetime.timedelta(days=1): return # Not exceeded if it hasn't failed enough times. if self.min_failures_count is not None: if self.failures_count < self.min_failures_count: return # Not exceeded if it hasn't failed for long enough. if self.min_failures_period is not None: if now - self.last_failure < self.min_failures_period: return # Exceeded if it has failed too many times. if self.max_failures_count is not None: if self.failures_count > self.max_failures_count: raise PeriodicTaskThresholdExceeded() # Exceed if it has been failing for too long. if self.max_failures_period is not None: if now - self.last_failure > self.max_failures_period: raise PeriodicTaskThresholdExceeded() # None of the conditions matched, so threshold hasn't been exceeded. return def check_too_soon(self): """Raise error if task has been run too recently""" now = datetime.datetime.now() # Find last run. If too recent, abort. if self.min_interval: last_run = self.get_last_run() if last_run: if now - last_run < self.min_interval: raise PeriodicTaskTooRecentLastRun() def acquire_lock(self, attempts=1, retry_sleep=1): """Acquire run lock""" # Is another same task running? for i in range(attempts): if not self.lock: break if self.break_lock_after: now = datetime.datetime.now() if now - self.lock.created_at > self.break_lock_after: # Has been running for too long or has died. Ignore. log.error( "Other task '%s' seems to have started, but " "it's been quite a while, will ignore and run.", self.key) break if i < attempts - 1: time.sleep(retry_sleep) self.reload() else: log.warning("Lock for task '%s' is taken.", self.key) raise PeriodicTaskLockTakenError() self.lock = self.Lock() self.save() def release_lock(self): lock_id = self.lock.id self.reload() if not self.lock or lock_id != self.lock.id: log.error( "Someone broke our lock for task '%s' since we " "acquired it!", self.key) return self.lock = None self.save() @contextlib.contextmanager def task_runner(self, persist=False): """Context manager to run periodic tasks that update model state This is a context manager, so it meant be used in a `with` statement, like this: with task_runner('unique-task-key'): do_something() What this does: 1. Takes care of using locks to prevent concurrent runs of the same task. 2. Tracks last success, last failure, and failure count of this task. """ if not persist: self.check_failures_threshold_exceeded() self.check_too_soon() self.acquire_lock(attempts=60 if persist else 1) try: yield except Exception: self.last_failure = datetime.datetime.now() self.failures_count += 1 raise else: self.last_success = datetime.datetime.now() self.failures_count = 0 finally: self.last_attempt_started = None self.save() self.release_lock() def __str__(self): return '%s: %s' % (self.__class__.__name__, self.id)
class Doc(me.Document): id = me.IntField(primary_key=True, default=1) map = me.MapField(me.EmbeddedDocumentField(MappedDoc)) str = me.MapField(me.StringField())
class Products(medb.Document): description = medb.StringField(required=True,max_length=500) price = medb.IntField(required=True,max_value=15)
class BonusCard(mge.Document): bonus_card_number = mge.StringField(min_length=3, max_length=10, unique=True, required=True) balance = mge.IntField(min_value=0, default=0) card_history = mge.ListField(mge.EmbeddedDocumentField(BonusCardTrx)) meta = {'allow_inheritance': True}
class Task(mongoengine.Document): """Tasks are the smallest unit of work, which contain an exact target that needs to be processed.""" QUEUED = "QUEUED" PROCESSING = "PROCESSING" FAILED = "FAILED" FINISHED = "FINISHED" STATUSES = ( (QUEUED, "Queued"), (PROCESSING, "Processing"), (FAILED, "Failed"), (FINISHED, "Finished"), ) meta = {'indexes': ['md5']} job_id = mongoengine.StringField(required=True, max_length=MD5_LENGTH, min_length=MD5_LENGTH, help_text="ID of parent Job.") target = mongoengine.StringField(required=True, help_text="Target URL or filesystem path") md5 = mongoengine.StringField(max_length=MD5_LENGTH, min_length=MD5_LENGTH, help_text="MD5 hexdigest of target.") title = mongoengine.StringField(help_text="Processed page title.") size = mongoengine.IntField(min_value=0, help_text="Size of page in bytes.") images = mongoengine.IntField(min_value=0, help_text="Number of images on the page.") timestamp = mongoengine.DateTimeField(help_text="End time of task.") status = mongoengine.StringField(choices=STATUSES, default=QUEUED, help_text="Job status.") notes = mongoengine.StringField(help_text="Arbitary information") @classmethod def create(cls, job_id, target): """Create a new task for the passed in target. :param job_id: The string ID of the parent job instance :param target: The target URL or filesystem path :returns: ``Task`` instance """ task = cls(job_id=job_id, target=target) task.save() return task @property def is_queued(self): return self.status == self.QUEUED @property def is_processing(self): return self.status == self.PROCESSING @property def is_finished(self): return self.status == self.FINISHED @property def is_failed(self): return self.status == self.FAILED @property def download_link(self): url_template = settings.BOTTLE_CONFIG['artexin.zipball_url_template'] return url_template.format(self.md5) @property def zipball_path(self): zipball_root = settings.BOTTLE_CONFIG['artexin.out_dir'] filename = '{0}.zip'.format(self.md5) return os.path.join(zipball_root, filename) def mark_queued(self): self.status = self.QUEUED self.save() def mark_processing(self): self.status = self.PROCESSING self.save() def mark_failed(self, reason): self.status = self.FAILED self.notes = reason self.save() def mark_finished(self): self.status = self.FINISHED self.notes = '' self.save()
class SSHProbe(me.EmbeddedDocument): uptime = me.FloatField() # seconds loadavg = me.ListField(me.FloatField()) cores = me.IntField() users = me.IntField() pub_ips = me.ListField(me.StringField()) priv_ips = me.ListField(me.StringField()) macs = me.ListField(me.StringField()) df = me.StringField() kernel = me.StringField() os = me.StringField() os_version = me.StringField() distro = me.StringField() dirty_cow = me.BooleanField() updated_at = me.DateTimeField() unreachable_since = me.DateTimeField() meta = {'strict': False} def update_from_dict(self, data): uptime = data.get('uptime') try: self.uptime = float(uptime) except (ValueError, TypeError): log.error("Invalid uptime value: %s", uptime) self.uptime = 0 loadavg = data.get('loadavg') try: assert isinstance(loadavg, list) assert len(loadavg) == 3 for i in range(3): loadavg[i] = float(loadavg[i]) self.loadavg = loadavg except Exception as exc: log.error("Invalid loadavg '%s': %r", loadavg, exc) self.loadavg = [] for int_attr in ('cores', 'users'): val = data.get(int_attr) try: setattr(self, int_attr, int(val)) except Exception as exc: log.error("Invalid %s '%s': %r", int_attr, val, exc) setattr(self, int_attr, 0) for strarr_attr in ('pub_ips', 'priv_ips', 'macs'): val = data.get(strarr_attr) try: assert isinstance(val, list) assert all(isinstance(item, basestring) for item in val) setattr(self, strarr_attr, val) except Exception as exc: log.error("Invalid %s '%s': %r", strarr_attr, val, exc) setattr(self, strarr_attr, []) for str_attr in ('df', 'kernel', 'os', 'os_version', 'distro'): setattr(self, str_attr, str(data.get(str_attr, ''))) self.dirty_cow = bool(data.get('dirty_cow')) self.unreachable_since = None self.updated_at = datetime.datetime.now() def as_dict(self): data = { key: getattr(self, key) for key in ( 'uptime', 'loadavg', 'cores', 'users', 'pub_ips', 'priv_ips', 'df', 'macs', 'kernel', 'os', 'os_version', 'distro', 'dirty_cow', 'updated_at', 'unreachable_since', ) } # Handle datetime objects for key in ('updated_at', 'unreachable_since'): if data[key]: data[key] = str(data[key].replace(tzinfo=None)) return data
class BigdataDeplay(mongoengine.Document): _id = mongoengine.ObjectIdField() file_total = mongoengine.IntField() hdfs_total = mongoengine.StringField() t_time = mongoengine.StringField() hdfs_remaining = mongoengine.StringField() miss_block = mongoengine.IntField() active_name = mongoengine.StringField() version = mongoengine.StringField() live_node = mongoengine.IntField() blocks_total = mongoengine.IntField() dead_node = mongoengine.IntField() cluster_id = mongoengine.StringField() hdfs_used = mongoengine.StringField() cluster_status = mongoengine.StringField() node_size_total = mongoengine.IntField() node_size_remaining = mongoengine.IntField() node_ip = mongoengine.StringField() non_dfs_used = mongoengine.IntField() node_size_used = mongoengine.IntField() node_name = mongoengine.StringField() node_state = mongoengine.StringField() block_num = mongoengine.IntField() total_cpu = mongoengine.IntField() cluster_state = mongoengine.StringField() used_cpu = mongoengine.IntField() appsPending = mongoengine.IntField() appsRuning = mongoengine.IntField() total_mem = mongoengine.IntField() total_live = mongoengine.IntField() used_mem = mongoengine.IntField() unhealthyNodes = mongoengine.IntField() table_name = mongoengine.StringField() meta = {"collection": 'Bigdata_deplay'} def __str__(self): return self._id
class MemoryUsage(me.EmbeddedDocument): used = me.IntField(default=0) free = me.IntField(default=0) total = me.IntField(required=True, default=0)
class Imdb(mongoengine.EmbeddedDocument): id = mongoengine.StringField() rating = mongoengine.DecimalField() votes = mongoengine.IntField()
class SystemLoad(me.EmbeddedDocument): cpu = me.FloatField(default=0) memory = me.IntField(default=0)
class Awards(mongoengine.EmbeddedDocument): wins = mongoengine.IntField() nominations = mongoengine.IntField() text = mongoengine.StringField()
class User(mongo.Document): meta = {'queryset_class': CustomUserQuerySet, 'allow_inheritance': True} full_name = mongo.StringField(verbose_name=_('full name'), max_length=30) email = mongo.EmailField(verbose_name=_('email address'), required=True, unique=True) password = mongo.StringField(max_length=128, verbose_name=_('password')) last_login = mongo.DateTimeField(verbose_name=_('last login'), default=timezone.now) is_staff = mongo.BooleanField(default=False) is_active = mongo.BooleanField(default=True) is_superuser = mongo.BooleanField(default=False) site_id = mongo.IntField(required=True, default=settings.SITE_ID) current_role = mongo.StringField(required=False) image = mongo.StringField(required=False) user_types = mongo.ListField(mongo.ReferenceField(UserTypes), required=True, verbose_name=_('User Types')) primary_user_type = mongo.ReferenceField( UserTypes, required=False, verbose_name=_('Primary User Type')) organization = mongo.ReferenceField(Organization, required=False, verbose_name=_('Organization')) custom_field_form = mongo.ReferenceField("FormSchema", required=False) activation_key = mongo.StringField(required=False) key_expires = mongo.DateTimeField(required=False) ban = mongo.BooleanField(default=False) ban_reason = mongo.StringField(max_length=255, required=False) address = mongo.StringField(max_length=255, required=False) city = mongo.StringField(max_length=255, required=False) state = mongo.StringField(max_length=255, required=False) zip = mongo.StringField(max_length=50, required=False) phone = mongo.StringField(max_length=20, required=False) description = mongo.StringField(required=False) point = mongo.PointField(required=False) #used for dashboard resource reference dashboard_resource_id = mongo.StringField(max_length=24, required=False) # TODO disabled for now, either needs to become a custom form, or hard coded like the address above # billing_phone = mongo.StringField(max_length=20, required=False, verbose_name=_('Billing Phone')) # shipping_phone = mongo.StringField(max_length=20, required=False, verbose_name=_('Shipping Phone')) # same_address = mongo.BooleanField(default=False, verbose_name=_('Is shipping adddress same as billing address')) # address_verified = mongo.BooleanField(default=False, verbose_name=_('Address verified by TaxCloud')) created = mongo.StringField(required=True, default=str(datetime.now())) updated = mongo.StringField(required=True, default=str(datetime.now())) USERNAME_FIELD = 'email' REQUIRED_FIELDS = ['full_name'] @mongo.queryset.queryset_manager def objects(doc_cls, queryset): # This may actually also be done by defining a default ordering for # the document, but this illustrates the use of manager methods return queryset.filter(site_id=settings.SITE_ID) @mongo.queryset.queryset_manager def all(doc_cls, queryset): # This may actually also be done by defining a default ordering for # the document, but this illustrates the use of manager methods return queryset.all() def save(self, *args, **kwargs): self.updated = datetime.now().strftime("%Y-%m-%d %H:%M:%S") if self.address and self.zip and self.city and self.state: address = (self.address + ", " + self.zip, self.city, self.state) geocoded_addr = verify_address(address) if geocoded_addr != 'Error': self.point = geocoded_addr super(User, self).save(*args, **kwargs) if not self.primary_user_type: self.set_primary_user_type() def set_current_role(self, role): self.current_role = role self.save() def set_primary_user_type(self, typ=None): admin = UserTypes.objects.get(name='Admin') if typ: typ = UserTypes.objects.get(name=typ) elif admin in self.user_types: typ = admin else: typ = self.user_types[0] self.primary_user_type = typ if self.site_id == 1: self.current_role = 'maps-admin' else: self.current_role = typ.name self.save() # def is_provider(self): # return len(self.user_types.filter(type='provider')) > 0 def get_username(self): "Return the identifying username for this User" return getattr(self, self.USERNAME_FIELD) def __unicode__(self): return self.get_username() def natural_key(self): return (self.get_username(), ) def is_anonymous(self): """ Always returns False. This is a way of comparing User objects to anonymous users. """ return False def is_authenticated(self): """ Always return True. This is a way to tell if the user has been authenticated in templates. """ return True def set_password(self, raw_password): self.password = make_password(raw_password) def check_password(self, raw_password): """ Returns a boolean of whether the raw_password was correct. Handles hashing formats behind the scenes. """ def setter(raw_password): self.set_password(raw_password) self.save(update_fields=["password"]) return check_password(raw_password, self.password, setter) def set_unusable_password(self): # Sets a value that will never be a valid hash self.password = make_password(None) def has_usable_password(self): return is_password_usable(self.password) def get_session_auth_hash(self): """ Returns an HMAC of the password field. """ key_salt = "maps.users.models.AbstractBaseUser.get_session_auth_hash" return salted_hmac(key_salt, self.password).hexdigest() def email_user(self, subject, message, from_email=None, **kwargs): """ Sends an email to this User. """ send_mail(subject, message, from_email, [self.email], **kwargs) @property def location_count(self): try: return Location.objects(created_by=str(self.id)).count() except mongo.ValidationError: return 0 @property def image_url(self): url = None if self.image: url = settings.MEDIA_URL + self.image return url def custom_form(self): output = [] try: if self.custom_field_form: serializer = FormSchemaIdSerializer(self.custom_field_form) output.append(serializer.data) except: pass return output def has_permission(self, obj_name, action): for ut in self.user_types: if ut.has_permission(obj_name, action): return True return False def has_edit_other_permission(self, obj_name): return self.has_permission(obj_name, '4') def has_delete_other_permission(self, obj_name): return self.has_permission(obj_name, '5') def has_edit_own_permission(self, obj_name): return self.has_permission(obj_name, '1') def has_delete_own_permission(self, obj_name): return self.has_permission(obj_name, '2') def can_delete(self, obj, perm_type=None): if not perm_type: perm_type = obj.permission_type if obj.created_by == self.id: if self.has_delete_own_permission(perm_type): return True else: if self.has_delete_other_permission(perm_type): return True return False def can_edit(self, obj, perm_type=None): if not perm_type: perm_type = obj.permission_type if obj.created_by == self.id: if self.has_edit_own_permission(perm_type): return True else: if self.has_edit_other_permission(perm_type): return True return False @property def permission_type(self): return "users"
class SubDocB(me.Document): field_b = me.IntField(primary_key=True, default=42)
class Friend(mongoengine.EmbeddedDocument): id = mongoengine.IntField(name='fb_id') name = mongoengine.StringField()
class PowResult(ModelMixin, mg.Document): meta = {"collection": "zil_pow_results", "strict": False} header = mg.StringField(max_length=128, required=True) seed = mg.StringField(max_length=128, required=True) boundary = mg.StringField(max_length=128, required=True) pub_key = mg.StringField(max_length=128, required=True) mix_digest = mg.StringField(max_length=128, required=True) nonce = mg.StringField(max_length=128, required=True) hash_result = mg.StringField(max_length=128, required=True) block_num = mg.IntField(default=0) pow_fee = mg.FloatField(default=0.0) finished_date = mg.DateField() finished_time = mg.DateTimeField() verified_time = mg.DateTimeField() verified = mg.BooleanField(default=False) miner_wallet = mg.StringField(max_length=128) worker_name = mg.StringField(max_length=64, default="") def __str__(self): return f"[PowResult: {self.pub_key}, {self.header}]" @classmethod def avg_pow_fee(cls, block_num): return cls.query(block_num=block_num).average("pow_fee") @classmethod def get_pow_result(cls, header, boundary, pub_key=None, order="-finished_time"): query = Q(header=header) & Q(boundary=boundary) if pub_key is not None: query = query & Q(pub_key=pub_key) cursor = cls.objects(query).order_by(order) # default to get latest one return cursor.first() @classmethod def epoch_rewards(cls, block_num=None, miner_wallet=None, worker_name=None): match = {} if block_num is not None: if isinstance(block_num, int): match = { "block_num": { "$eq": block_num, } } else: start, end = block_num match = { "block_num": { "$gte": start, "$lte": end, } } if miner_wallet is not None: match.update({ "miner_wallet": { "$eq": miner_wallet, } }) if worker_name is not None: match.update({ "worker_name": { "$eq": worker_name, } }) group = { "_id": None, "rewards": {"$sum": "$pow_fee"}, "count": {"$sum": 1}, "verified": {"$sum": {"$cond": ["$verified", 1, 0]}}, "first_work_at": {"$min": "$finished_time"}, "last_work_at": {"$max": "$finished_time"} } pipeline = [ {"$match": match}, {"$group": group}, ] res = list(cls.objects.aggregate(*pipeline)) if res: rewards = res[0] rewards.pop("_id", None) return rewards return {"rewards": None, "count": 0, "verified": 0, "first_work_at": None, "last_work_at": None} @classmethod def rewards_by_miners(cls, block_num): if block_num is None: block_num = PowWork.get_latest_block_num() match = { "block_num": { "$eq": block_num, } } group = { "_id": "$miner_wallet", "block_num": {"$first": "$block_num"}, "date": {"$first": "$finished_date"}, "date_time": {"$first": "$finished_time"}, "rewards": {"$sum": "$pow_fee"}, "finished": {"$sum": 1}, "verified": {"$sum": {"$cond": ["$verified", 1, 0]}}, } project = { "_id": 0, "miner_wallet": "$_id", "block_num": 1, "date": 1, "date_time": 1, "rewards": 1, "finished": 1, "verified": 1, } pipeline = [ {"$match": match}, {"$group": group}, {"$project": project} ] return list(cls.objects.aggregate(*pipeline)) def get_worker(self): return miner.Worker.get_or_create(self.miner_wallet, self.worker_name)