class Car(me.Document):
id = me.IntField(primary_key=True)
model = me.StringField()
color = me.StringField()
year = me.IntField()
class MongoDocument(mongoengine.Document):
meta = {'collection': COLLECTION}
number = mongoengine.IntField()
class Count(mongoengine.EmbeddedDocument):
total_count = mongoengine.IntField()
class Machine(OwnershipMixin, me.Document):
"""The basic machine model"""
id = me.StringField(primary_key=True, default=lambda: uuid.uuid4().hex)
cloud = me.ReferenceField('Cloud', required=True)
owner = me.ReferenceField('Organization', required=True)
location = me.ReferenceField('CloudLocation', required=False)
size = me.ReferenceField('CloudSize', required=False)
network = me.ReferenceField('Network', required=False)
subnet = me.ReferenceField('Subnet', required=False)
name = me.StringField()
# Info gathered mostly by libcloud (or in some cases user input).
# Be more specific about what this is.
# We should perhaps come up with a better name.
machine_id = me.StringField(required=True)
hostname = me.StringField()
public_ips = me.ListField()
private_ips = me.ListField()
ssh_port = me.IntField(default=22)
OS_TYPES = ('windows', 'coreos', 'freebsd', 'linux', 'unix')
os_type = me.StringField(default='unix', choices=OS_TYPES)
rdp_port = me.IntField(default=3389)
actions = me.EmbeddedDocumentField(Actions, default=lambda: Actions())
extra = me.DictField()
cost = me.EmbeddedDocumentField(Cost, default=lambda: Cost())
image_id = me.StringField()
# libcloud.compute.types.NodeState
state = me.StringField(default='unknown',
choices=('running', 'starting', 'rebooting',
'terminated', 'pending', 'unknown',
'stopping', 'stopped', 'suspended',
'error', 'paused', 'reconfiguring'))
machine_type = me.StringField(default='machine',
choices=('machine', 'vm', 'container',
'hypervisor', 'container-host'))
parent = me.ReferenceField('Machine', required=False)
# We should think this through a bit.
key_associations = me.EmbeddedDocumentListField(KeyAssociation)
last_seen = me.DateTimeField()
missing_since = me.DateTimeField()
unreachable_since = me.DateTimeField()
created = me.DateTimeField()
monitoring = me.EmbeddedDocumentField(Monitoring,
default=lambda: Monitoring())
ssh_probe = me.EmbeddedDocumentField(SSHProbe, required=False)
ping_probe = me.EmbeddedDocumentField(PingProbe, required=False)
# Number of vCPUs gathered from various sources. This field is meant to
# be updated ONLY by the mist.api.metering.tasks:find_machine_cores task.
cores = me.IntField()
meta = {
'collection':
'machines',
'indexes': [{
'fields': ['cloud', 'machine_id'],
'sparse': False,
'unique': True,
'cls': False,
}, {
'fields': ['monitoring.installation_status.activated_at'],
'sparse': True,
'unique': False
}],
'strict':
False,
}
def __init__(self, *args, **kwargs):
super(Machine, self).__init__(*args, **kwargs)
self.ctl = MachineController(self)
def clean(self):
# Remove any KeyAssociation, whose `keypair` has been deleted. Do NOT
# perform an atomic update on self, but rather remove items from the
# self.key_associations list by iterating over it and popping matched
# embedded documents in order to ensure that the most recent list is
# always processed and saved.
for ka in reversed(range(len(self.key_associations))):
if self.key_associations[ka].keypair.deleted:
self.key_associations.pop(ka)
# Populate owner field based on self.cloud.owner
if not self.owner:
self.owner = self.cloud.owner
self.clean_os_type()
if self.monitoring.method not in config.MONITORING_METHODS:
self.monitoring.method = config.DEFAULT_MONITORING_METHOD
def clean_os_type(self):
"""Clean self.os_type"""
if self.os_type not in self.OS_TYPES:
for os_type in self.OS_TYPES:
if self.os_type.lower() == os_type:
self.os_type = os_type
break
else:
self.os_type = 'unix'
def delete(self):
super(Machine, self).delete()
mist.api.tag.models.Tag.objects(resource=self).delete()
try:
self.owner.mapper.remove(self)
except (AttributeError, me.DoesNotExist) as exc:
log.error(exc)
try:
if self.owned_by:
self.owned_by.get_ownership_mapper(self.owner).remove(self)
except (AttributeError, me.DoesNotExist) as exc:
log.error(exc)
def as_dict(self):
# Return a dict as it will be returned to the API
# tags as a list return for the ui
tags = {
tag.key: tag.value
for tag in mist.api.tag.models.Tag.objects(
resource=self).only('key', 'value')
}
# Optimize tags data structure for js...
if isinstance(tags, dict):
tags = [{
'key': key,
'value': value
} for key, value in tags.iteritems()]
return {
'id':
self.id,
'hostname':
self.hostname,
'public_ips':
self.public_ips,
'private_ips':
self.private_ips,
'name':
self.name,
'ssh_port':
self.ssh_port,
'os_type':
self.os_type,
'rdp_port':
self.rdp_port,
'machine_id':
self.machine_id,
'actions':
{action: self.actions[action]
for action in self.actions},
'extra':
self.extra,
'cost':
self.cost.as_dict(),
'image_id':
self.image_id,
'state':
self.state,
'tags':
tags,
'monitoring':
self.monitoring.as_dict()
if self.monitoring and self.monitoring.hasmonitoring else '',
'key_associations': [ka.as_dict() for ka in self.key_associations],
'cloud':
self.cloud.id,
'location':
self.location.id if self.location else '',
'size':
self.size.name if self.size else '',
'cloud_title':
self.cloud.title,
'last_seen':
str(self.last_seen.replace(tzinfo=None) if self.last_seen else ''),
'missing_since':
str(
self.missing_since.replace(
tzinfo=None) if self.missing_since else ''),
'unreachable_since':
str(
self.unreachable_since.replace(
tzinfo=None) if self.unreachable_since else ''),
'created':
str(self.created.replace(tzinfo=None) if self.created else ''),
'machine_type':
self.machine_type,
'parent_id':
self.parent.id if self.parent is not None else '',
'probe': {
'ping': (self.ping_probe.as_dict() if self.ping_probe
is not None else PingProbe().as_dict()),
'ssh': (self.ssh_probe.as_dict() if self.ssh_probe is not None
else SSHProbe().as_dict()),
},
'cores':
self.cores,
'network':
self.network.id if self.network else '',
'subnet':
self.subnet.id if self.subnet else '',
'owned_by':
self.owned_by.id if self.owned_by else '',
'created_by':
self.created_by.id if self.created_by else '',
}
def __str__(self):
return 'Machine %s (%s) in %s' % (self.name, self.id, self.cloud)
class DiskUsage(me.EmbeddedDocument):
used = me.IntField(default=0)
free = me.IntField(default=0)
percent = me.FloatField(default=0) # show in percent
total = me.IntField(required=True, default=0)
class ReferenceDoc(me.Document):
field = me.IntField(primary_key=True, default=42)
class BonusCardTrx(mge.EmbeddedDocument):
trx_id = mge.StringField(required=True) # номер транзакции
trx_value = mge.IntField(required=True) # Сколько начислено бонусных единиц(миль)
departure_airport = mge.StringField(required=True) # откуда
arrival_airport = mge.StringField(required=True) # куда
flight_date = mge.DateTimeField(required=True) # дата полёта
class Image(mongoengine.EmbeddedDocument):
path = mongoengine.StringField(required=True)
width = mongoengine.IntField(required=True)
height = mongoengine.IntField(required=True)
content_type = mongoengine.StringField(required=True)
class UserProfile(me.Document):
"""
User profile model which has other social data and related
user info
"""
first_name = me.StringField(max_length=30)
last_name = me.StringField(max_length=30)
google = me.DictField()
github = me.DictField()
avatar = me.URLField()
facebook_handle = me.StringField(max_length=100)
twitter_handle = me.StringField(max_length=100)
linkedin_handle = me.StringField(max_length=100)
description = me.StringField(max_length=500)
company = me.StringField(max_length=100)
designation = me.StringField(max_length=100)
mobile = me.IntField()
created_at = me.DateTimeField(default=datetime.utcnow)
updated_at = me.DateTimeField(default=datetime.utcnow)
created_by = me.ReferenceField('User', required=False)
updated_by = me.ReferenceField('User', required=False)
is_active = me.BooleanField(default=True)
def __str__(self):
return ' '.join([self.first_name, self.last_name])
def __init__(self, *args, **kwargs):
super(UserProfile, self).__init__(*args, **kwargs)
@classmethod
def pre_save(cls, sender, document, **kwargs):
if document.google:
if UserProfile.objects.filter(
google__email=document.google['email']).count() > 0:
raise ValidationError('This account is already registered.')
if document.github:
if UserProfile.objects.filter(
github__email=document.github['email']).count() > 0:
raise ValidationError('This account is already registered.')
if document.description and len(document.description) > 500:
raise ValidationError('Description exceeds 500 characters')
if document.facebook_handle and len(document.facebook_handle) > 100:
raise ValidationError('Facebook id exceeds 100 characters')
if document.twitter_handle and len(document.twitter_handle) > 100:
raise ValidationError('Twitter id exceeds 100 characters')
if document.linkedin_handle and len(document.linkedin_handle) > 100:
raise ValidationError('Linked-in id exceeds 100 characters')
if document.company and len(document.company) > 100:
raise ValidationError('Company exceeds 100 characters')
if document.designation and len(document.designation) > 100:
raise ValidationError('Designation exceeds 100 characters')
if document.first_name and len(document.first_name) > 30:
raise ValidationError('First name exceeds 30 characters')
if document.last_name and len(document.last_name) > 30:
raise ValidationError('Last name exceeds 30 characters')
if document.mobile and len(str(document.mobile)) > 13:
raise ValidationError('Mobile number exceeds 13 characters')
@classmethod
def post_save(cls, sender, document, **kwargs):
if not document.first_name and not document.last_name:
first_name = last_name = ''
if document.google:
if 'first_name' in document.google.keys():
first_name = document.google['first_name']
if 'last_name' in document.google.keys():
last_name = document.google['last_name']
if document.github:
try:
first_name, last_name = document.github['name'].split(' ')
except ValueError:
first_name, last_name = document.github['name'], ''
except KeyError:
pass
document.update(set__first_name=first_name,
set__last_name=last_name)
def save(self, *args, **kwargs):
super(UserProfile, self).save(*args, **kwargs)
self.reload()
def update(self, *args, **kwargs):
super(UserProfile, self).update(*args, **kwargs)
self.reload()
def to_json(self, fields=None, exclude=None):
return json_dumper(self, fields, exclude)
class Schedule(me.Document, ConditionalClassMixin):
"""Abstract base class for every schedule attr mongoengine model.
This model is based on celery periodic task and creates defines the fields
common to all schedules of all types. For each different schedule type, a
subclass should be created adding any schedule specific fields and methods.
Documents of all Schedule subclasses will be stored on the same mongo
collection.
One can perform a query directly on Schedule to fetch all cloud types, like
this:
Schedule.objects(owner=owner).count()
"""
condition_resource_cls = Machine
meta = {
'collection':
'schedules',
'allow_inheritance':
True,
'indexes': [
{
'fields': ['owner', 'name', 'deleted'],
'sparse': False,
'unique': True,
'cls': False,
},
],
}
id = me.StringField(primary_key=True, default=lambda: uuid4().hex)
name = me.StringField(required=True)
description = me.StringField()
deleted = me.DateTimeField()
owner = me.ReferenceField(Organization, required=True)
# celery periodic task specific fields
queue = me.StringField()
exchange = me.StringField()
routing_key = me.StringField()
soft_time_limit = me.IntField()
# mist specific fields
schedule_type = me.EmbeddedDocumentField(BaseScheduleType, required=True)
task_type = me.EmbeddedDocumentField(BaseTaskType, required=True)
# celerybeat-mongo specific fields
expires = me.DateTimeField()
start_after = me.DateTimeField()
task_enabled = me.BooleanField(default=False)
run_immediately = me.BooleanField()
last_run_at = me.DateTimeField()
total_run_count = me.IntField(min_value=0, default=0)
max_run_count = me.IntField(min_value=0, default=0)
no_changes = False
def __init__(self, *args, **kwargs):
# FIXME
import mist.api.schedules.base
super(Schedule, self).__init__(*args, **kwargs)
self.ctl = mist.api.schedules.base.BaseController(self)
def owner_query(self):
return me.Q(cloud__in=Cloud.objects(owner=self.owner).only('id'))
@classmethod
def add(cls, auth_context, name, **kwargs):
"""Add schedule
This is a class method, meaning that it is meant to be called on the
class itself and not on an instance of the class.
You're not meant to be calling this directly, but on a schedule class
instead like this:
schedule = Schedule.add(owner=owner, **kwargs)
"""
owner = auth_context.owner
if not name:
raise RequiredParameterMissingError('name')
if not owner or not isinstance(owner, Organization):
raise BadRequestError('owner')
if Schedule.objects(owner=owner, name=name, deleted=None):
raise ScheduleNameExistsError()
schedule = cls(owner=owner, name=name)
schedule.ctl.set_auth_context(auth_context)
schedule.ctl.add(**kwargs)
return schedule
@property
def schedule(self):
if self.schedule_type:
return self.schedule_type.schedule
else:
raise Exception("must define interval, crontab, one_off schedule")
@property
def args(self):
mids = [
machine.id for machine in self.get_resources()
if machine.state != 'terminated'
]
fire_up = self.task_type.args
return [self.owner.id, fire_up, self.name, mids]
@property
def kwargs(self):
return {}
@property
def task(self):
return self.task_type.task
@property
def enabled(self):
if self.deleted:
return False
if not self.get_resources().count():
return False
if self.expires and self.expires < datetime.datetime.now():
return False
# if self.start_after and self.start_after < datetime.datetime.now():
# return False
if self.max_run_count and (
(self.total_run_count or 0) >= self.max_run_count):
return False
else:
return self.task_enabled
def __unicode__(self):
fmt = '{0.name}: {{no schedule}}'
if self.schedule_type:
fmt = 'name: {0.name} type: {0.schedule_type._cls}'
else:
raise Exception("must define interval or crontab schedule")
return fmt.format(self)
def validate(self, clean=True):
"""
Override mongoengine validate. We should validate crontab entry.
Use crontab_parser for crontab expressions.
The parser is a general purpose one, useful for parsing hours,
minutes and day_of_week expressions.
example for minutes:
minutes = crontab_parser(60).parse('*/15')
[0, 15, 30, 45]
"""
if isinstance(self.schedule_type, Crontab):
cronj_entry = self.schedule_type.as_dict()
try:
for k, v in cronj_entry.items():
if k == 'minute':
celery.schedules.crontab_parser(60).parse(v)
elif k == 'hour':
celery.schedules.crontab_parser(24).parse(v)
elif k == 'day_of_week':
celery.schedules.crontab_parser(7).parse(v)
elif k == 'day_of_month':
celery.schedules.crontab_parser(31, 1).parse(v)
elif k == 'month_of_year':
celery.schedules.crontab_parser(12, 1).parse(v)
else:
raise me.ValidationError(
'You should provide valid period of time')
except celery.schedules.ParseException:
raise me.ValidationError('Crontab entry is not valid')
except Exception as exc:
raise me.ValidationError('Crontab entry is not valid:%s' %
exc.message)
super(Schedule, self).validate(clean=True)
def delete(self):
super(Schedule, self).delete()
Tag.objects(resource=self).delete()
self.owner.mapper.remove(self)
def as_dict(self):
# Return a dict as it will be returned to the API
last_run = '' if self.total_run_count == 0 else str(self.last_run_at)
conditions = [condition.as_dict() for condition in self.conditions]
sdict = {
'id': self.id,
'name': self.name,
'description': self.description or '',
'schedule': unicode(self.schedule_type),
'schedule_type': self.schedule_type.type,
'schedule_entry': self.schedule_type.as_dict(),
'task_type': str(self.task_type),
'expires': str(self.expires or ''),
'start_after': str(self.start_after or ''),
'task_enabled': self.task_enabled,
'active': self.enabled,
'run_immediately': self.run_immediately or '',
'last_run_at': last_run,
'total_run_count': self.total_run_count,
'max_run_count': self.max_run_count,
'conditions': conditions,
}
return sdict
class PollingSchedule(ShardedScheduleMixin, me.Document):
meta = {
'allow_inheritance': True,
'strict': False,
'indexes': ['shard_id']
}
# We use a unique name for easy identification and to avoid running the
# same schedule twice. The name is autopopulated during the invocation of
# the `clean` method.
name = me.StringField(unique=True)
# The following fields are defined in celerybeatmongo.models.PeriodicTask.
# Here, we define no fields in the base class, and expect subclasses to
# either define their fields, or simply use properties.
# task = me.StringField(required=True)
# args = me.ListField()
# kwargs = me.DictField()
# Scheduling information. Don't edit them directly, just use the model
# methods.
default_interval = me.EmbeddedDocumentField(
PollingInterval, required=True, default=PollingInterval(every=0))
override_intervals = me.EmbeddedDocumentListField(PollingInterval)
# Optional arguments.
queue = me.StringField()
exchange = me.StringField()
routing_key = me.StringField()
soft_time_limit = me.IntField()
# Used internally by the scheduler.
last_run_at = me.DateTimeField()
total_run_count = me.IntField(min_value=0)
run_immediately = me.BooleanField()
def get_name(self):
"""Construct name based on self.task"""
try:
return self.task.split('.')[-1]
except NotImplementedError:
return '%s: No task specified.' % self.__class__.__name__
def clean(self):
"""Automatically set value of name"""
self.name = self.get_name()
@property
def task(self):
"""Return task name for this schedule
Subclasses should define an attribute, property or field to do this.
"""
raise NotImplementedError()
@property
def args(self):
"""Return task args for this schedule"""
return [str(self.id)]
@property
def kwargs(self):
"""Return task kwargs for this schedule"""
return {}
@property
def enabled(self):
"""Whether this task is currently enabled or not"""
return bool(self.interval.timedelta)
@property
def interval(self):
"""Merge multiple intervals into one
Returns a dynamic PollingInterval, with the highest frequency of any
override schedule or the default schedule.
"""
interval = self.default_interval
for i in self.override_intervals:
if not i.expired():
if not interval.timedelta or i.timedelta < interval.timedelta:
interval = i
return interval
@property
def schedule(self):
"""Return a celery schedule instance
This is used internally by celerybeatmongo scheduler
"""
return celery.schedules.schedule(self.interval.timedelta)
@property
def expires(self):
return None
def add_interval(self, interval, ttl=300, name=''):
"""Add an override schedule to the scheduled task
Override schedules must define an interval in seconds, as well as a
TTL (time to live), also in seconds. Override schedules cannot be
removed, so short TTL's should be used. You can however add a new
override schedule again, thus practically extending the time where an
override is in effect.
Override schedules can only increase, not decrease frequency of the
schedule, in relation to that define in the `default_interval`.
"""
assert isinstance(interval, int) and interval > 0
assert isinstance(ttl, int) and 0 < ttl < 3600
expires = datetime.datetime.now() + datetime.timedelta(seconds=ttl)
self.override_intervals.append(
PollingInterval(name=name, expires=expires, every=interval))
def cleanup_expired_intervals(self):
"""Remove override schedules that have expired"""
self.override_intervals = [
override for override in self.override_intervals
if not override.expired()
]
def set_default_interval(self, interval):
"""Set default interval
This is the interval used for this schedule, if there is no active
override schedule with a smaller interval. The default interval never
expires. To disable a task, simply set `enabled` equal to False.
"""
self.default_interval = PollingInterval(name='default', every=interval)
def __unicode__(self):
return "%s %s" % (self.get_name(), self.interval or '(no interval)')
class College(mgng.EmbeddedDocument):
name = mgng.StringField()
status = mgng.StringField()
rank = mgng.IntField()
essays = mgng.ListField(mgng.EmbeddedDocumentField(Essay))
class Mark(me.EmbeddedDocument):
mark = me.IntField(min_value=1, max_value=12)
id_item = me.ReferenceField(Items)
name_item = me.StringField(min_length=1, max_length=255, required=True)
class Result(mongoengine.Document):
task_id = mongoengine.StringField(required=True, unique=True)
result = mongoengine.StringField()
high_count = mongoengine.IntField()
medium_count = mongoengine.IntField()
low_count = mongoengine.IntField()
class EmbeddedB(me.EmbeddedDocument):
field_b = me.IntField(default=42)
class PeriodicTaskInfo(me.Document):
class Lock(me.EmbeddedDocument):
id = me.StringField(default=lambda: uuid.uuid4().hex)
created_at = me.DateTimeField(default=datetime.datetime.now())
# Unique task identifier.
key = me.StringField(primary_key=True)
# Track successes/failures for autodisabling.
last_success = me.DateTimeField()
last_failure = me.DateTimeField()
failures_count = me.IntField(default=0)
# Lock to prevent concurrent running of the same task.
lock = me.EmbeddedDocumentField(Lock)
# Class attributes (NOT FIELDS). This define constants on the class.
# Subclasses may override by setting attributes, dynamic properties or
# fields.
# Task won't be autodisabled if it has failed less times.
min_failures_count = 50
# Task won't be autodisabled if it has succeeded in this period.
min_failures_period = datetime.timedelta(hours=1)
# Task will be autodisabled if it has failed more times.
max_failures_count = 100
# Task will be autodisabled if it hasn't succeeded in this period.
max_failures_period = datetime.timedelta(days=2)
# Lock will be broken if it was last acquired more than this time ago.
break_lock_after = datetime.timedelta(seconds=300)
# Abort task if previous attempt was in less than this time before.
min_interval = datetime.timedelta(seconds=5)
@classmethod
def get_or_add(cls, key):
try:
task = cls.objects.get(key=key)
except cls.DoesNotExist:
log.info("PeriodicTaskInfo for '%s' missing, will create.", key)
task = cls(key=key)
try:
task.save()
except me.NotUniqueError:
# Work around race condition where document was created since
# we checked.
log.warning(
"PeriodicTaskInfo for '%s' creation race "
"condition, will reload.", key)
task = cls.objects.get(key=key)
log.debug("Loaded PeriodicTaskInfo for '%s'.", key)
return task
def get_last_run(self):
if self.last_success and self.last_failure:
return max(self.last_success, self.last_failure)
return self.last_success or self.last_failure
def check_failures_threshold_exceeded(self):
"""Raise PeriodicTaskThresholdExceeed if task should be autodisabled"""
now = datetime.datetime.now()
# If it hasn't run recently, then allow it to run.
last_run = self.get_last_run()
if not last_run or now - last_run > datetime.timedelta(days=1):
return
# Not exceeded if it hasn't failed enough times.
if self.min_failures_count is not None:
if self.failures_count < self.min_failures_count:
return
# Not exceeded if it hasn't failed for long enough.
if self.min_failures_period is not None:
if now - self.last_failure < self.min_failures_period:
return
# Exceeded if it has failed too many times.
if self.max_failures_count is not None:
if self.failures_count > self.max_failures_count:
raise PeriodicTaskThresholdExceeded()
# Exceed if it has been failing for too long.
if self.max_failures_period is not None:
if now - self.last_failure > self.max_failures_period:
raise PeriodicTaskThresholdExceeded()
# None of the conditions matched, so threshold hasn't been exceeded.
return
def check_too_soon(self):
"""Raise error if task has been run too recently"""
now = datetime.datetime.now()
# Find last run. If too recent, abort.
if self.min_interval:
last_run = self.get_last_run()
if last_run:
if now - last_run < self.min_interval:
raise PeriodicTaskTooRecentLastRun()
def acquire_lock(self, attempts=1, retry_sleep=1):
"""Acquire run lock"""
# Is another same task running?
for i in range(attempts):
if not self.lock:
break
if self.break_lock_after:
now = datetime.datetime.now()
if now - self.lock.created_at > self.break_lock_after:
# Has been running for too long or has died. Ignore.
log.error(
"Other task '%s' seems to have started, but "
"it's been quite a while, will ignore and run.",
self.key)
break
if i < attempts - 1:
time.sleep(retry_sleep)
self.reload()
else:
log.warning("Lock for task '%s' is taken.", self.key)
raise PeriodicTaskLockTakenError()
self.lock = self.Lock()
self.save()
def release_lock(self):
lock_id = self.lock.id
self.reload()
if not self.lock or lock_id != self.lock.id:
log.error(
"Someone broke our lock for task '%s' since we "
"acquired it!", self.key)
return
self.lock = None
self.save()
@contextlib.contextmanager
def task_runner(self, persist=False):
"""Context manager to run periodic tasks that update model state
This is a context manager, so it meant be used in a `with` statement,
like this:
with task_runner('unique-task-key'):
do_something()
What this does:
1. Takes care of using locks to prevent concurrent runs of the same
task.
2. Tracks last success, last failure, and failure count of this task.
"""
if not persist:
self.check_failures_threshold_exceeded()
self.check_too_soon()
self.acquire_lock(attempts=60 if persist else 1)
try:
yield
except Exception:
self.last_failure = datetime.datetime.now()
self.failures_count += 1
raise
else:
self.last_success = datetime.datetime.now()
self.failures_count = 0
finally:
self.last_attempt_started = None
self.save()
self.release_lock()
def __str__(self):
return '%s: %s' % (self.__class__.__name__, self.id)
class Doc(me.Document):
id = me.IntField(primary_key=True, default=1)
map = me.MapField(me.EmbeddedDocumentField(MappedDoc))
str = me.MapField(me.StringField())
class Products(medb.Document):
description = medb.StringField(required=True,max_length=500)
price = medb.IntField(required=True,max_value=15)
class BonusCard(mge.Document):
bonus_card_number = mge.StringField(min_length=3, max_length=10, unique=True, required=True)
balance = mge.IntField(min_value=0, default=0)
card_history = mge.ListField(mge.EmbeddedDocumentField(BonusCardTrx))
meta = {'allow_inheritance': True}
class Task(mongoengine.Document):
"""Tasks are the smallest unit of work, which contain an exact target that
needs to be processed."""
QUEUED = "QUEUED"
PROCESSING = "PROCESSING"
FAILED = "FAILED"
FINISHED = "FINISHED"
STATUSES = (
(QUEUED, "Queued"),
(PROCESSING, "Processing"),
(FAILED, "Failed"),
(FINISHED, "Finished"),
)
meta = {'indexes': ['md5']}
job_id = mongoengine.StringField(required=True,
max_length=MD5_LENGTH,
min_length=MD5_LENGTH,
help_text="ID of parent Job.")
target = mongoengine.StringField(required=True,
help_text="Target URL or filesystem path")
md5 = mongoengine.StringField(max_length=MD5_LENGTH,
min_length=MD5_LENGTH,
help_text="MD5 hexdigest of target.")
title = mongoengine.StringField(help_text="Processed page title.")
size = mongoengine.IntField(min_value=0,
help_text="Size of page in bytes.")
images = mongoengine.IntField(min_value=0,
help_text="Number of images on the page.")
timestamp = mongoengine.DateTimeField(help_text="End time of task.")
status = mongoengine.StringField(choices=STATUSES,
default=QUEUED,
help_text="Job status.")
notes = mongoengine.StringField(help_text="Arbitary information")
@classmethod
def create(cls, job_id, target):
"""Create a new task for the passed in target.
:param job_id: The string ID of the parent job instance
:param target: The target URL or filesystem path
:returns: ``Task`` instance
"""
task = cls(job_id=job_id, target=target)
task.save()
return task
@property
def is_queued(self):
return self.status == self.QUEUED
@property
def is_processing(self):
return self.status == self.PROCESSING
@property
def is_finished(self):
return self.status == self.FINISHED
@property
def is_failed(self):
return self.status == self.FAILED
@property
def download_link(self):
url_template = settings.BOTTLE_CONFIG['artexin.zipball_url_template']
return url_template.format(self.md5)
@property
def zipball_path(self):
zipball_root = settings.BOTTLE_CONFIG['artexin.out_dir']
filename = '{0}.zip'.format(self.md5)
return os.path.join(zipball_root, filename)
def mark_queued(self):
self.status = self.QUEUED
self.save()
def mark_processing(self):
self.status = self.PROCESSING
self.save()
def mark_failed(self, reason):
self.status = self.FAILED
self.notes = reason
self.save()
def mark_finished(self):
self.status = self.FINISHED
self.notes = ''
self.save()
class SSHProbe(me.EmbeddedDocument):
uptime = me.FloatField() # seconds
loadavg = me.ListField(me.FloatField())
cores = me.IntField()
users = me.IntField()
pub_ips = me.ListField(me.StringField())
priv_ips = me.ListField(me.StringField())
macs = me.ListField(me.StringField())
df = me.StringField()
kernel = me.StringField()
os = me.StringField()
os_version = me.StringField()
distro = me.StringField()
dirty_cow = me.BooleanField()
updated_at = me.DateTimeField()
unreachable_since = me.DateTimeField()
meta = {'strict': False}
def update_from_dict(self, data):
uptime = data.get('uptime')
try:
self.uptime = float(uptime)
except (ValueError, TypeError):
log.error("Invalid uptime value: %s", uptime)
self.uptime = 0
loadavg = data.get('loadavg')
try:
assert isinstance(loadavg, list)
assert len(loadavg) == 3
for i in range(3):
loadavg[i] = float(loadavg[i])
self.loadavg = loadavg
except Exception as exc:
log.error("Invalid loadavg '%s': %r", loadavg, exc)
self.loadavg = []
for int_attr in ('cores', 'users'):
val = data.get(int_attr)
try:
setattr(self, int_attr, int(val))
except Exception as exc:
log.error("Invalid %s '%s': %r", int_attr, val, exc)
setattr(self, int_attr, 0)
for strarr_attr in ('pub_ips', 'priv_ips', 'macs'):
val = data.get(strarr_attr)
try:
assert isinstance(val, list)
assert all(isinstance(item, basestring) for item in val)
setattr(self, strarr_attr, val)
except Exception as exc:
log.error("Invalid %s '%s': %r", strarr_attr, val, exc)
setattr(self, strarr_attr, [])
for str_attr in ('df', 'kernel', 'os', 'os_version', 'distro'):
setattr(self, str_attr, str(data.get(str_attr, '')))
self.dirty_cow = bool(data.get('dirty_cow'))
self.unreachable_since = None
self.updated_at = datetime.datetime.now()
def as_dict(self):
data = {
key: getattr(self, key)
for key in (
'uptime',
'loadavg',
'cores',
'users',
'pub_ips',
'priv_ips',
'df',
'macs',
'kernel',
'os',
'os_version',
'distro',
'dirty_cow',
'updated_at',
'unreachable_since',
)
}
# Handle datetime objects
for key in ('updated_at', 'unreachable_since'):
if data[key]:
data[key] = str(data[key].replace(tzinfo=None))
return data
class BigdataDeplay(mongoengine.Document):
_id = mongoengine.ObjectIdField()
file_total = mongoengine.IntField()
hdfs_total = mongoengine.StringField()
t_time = mongoengine.StringField()
hdfs_remaining = mongoengine.StringField()
miss_block = mongoengine.IntField()
active_name = mongoengine.StringField()
version = mongoengine.StringField()
live_node = mongoengine.IntField()
blocks_total = mongoengine.IntField()
dead_node = mongoengine.IntField()
cluster_id = mongoengine.StringField()
hdfs_used = mongoengine.StringField()
cluster_status = mongoengine.StringField()
node_size_total = mongoengine.IntField()
node_size_remaining = mongoengine.IntField()
node_ip = mongoengine.StringField()
non_dfs_used = mongoengine.IntField()
node_size_used = mongoengine.IntField()
node_name = mongoengine.StringField()
node_state = mongoengine.StringField()
block_num = mongoengine.IntField()
total_cpu = mongoengine.IntField()
cluster_state = mongoengine.StringField()
used_cpu = mongoengine.IntField()
appsPending = mongoengine.IntField()
appsRuning = mongoengine.IntField()
total_mem = mongoengine.IntField()
total_live = mongoengine.IntField()
used_mem = mongoengine.IntField()
unhealthyNodes = mongoengine.IntField()
table_name = mongoengine.StringField()
meta = {"collection": 'Bigdata_deplay'}
def __str__(self):
return self._id
class MemoryUsage(me.EmbeddedDocument):
used = me.IntField(default=0)
free = me.IntField(default=0)
total = me.IntField(required=True, default=0)
class Imdb(mongoengine.EmbeddedDocument):
id = mongoengine.StringField()
rating = mongoengine.DecimalField()
votes = mongoengine.IntField()
class SystemLoad(me.EmbeddedDocument):
cpu = me.FloatField(default=0)
memory = me.IntField(default=0)
class Awards(mongoengine.EmbeddedDocument):
wins = mongoengine.IntField()
nominations = mongoengine.IntField()
text = mongoengine.StringField()
class User(mongo.Document):
meta = {'queryset_class': CustomUserQuerySet, 'allow_inheritance': True}
full_name = mongo.StringField(verbose_name=_('full name'), max_length=30)
email = mongo.EmailField(verbose_name=_('email address'),
required=True,
unique=True)
password = mongo.StringField(max_length=128, verbose_name=_('password'))
last_login = mongo.DateTimeField(verbose_name=_('last login'),
default=timezone.now)
is_staff = mongo.BooleanField(default=False)
is_active = mongo.BooleanField(default=True)
is_superuser = mongo.BooleanField(default=False)
site_id = mongo.IntField(required=True, default=settings.SITE_ID)
current_role = mongo.StringField(required=False)
image = mongo.StringField(required=False)
user_types = mongo.ListField(mongo.ReferenceField(UserTypes),
required=True,
verbose_name=_('User Types'))
primary_user_type = mongo.ReferenceField(
UserTypes, required=False, verbose_name=_('Primary User Type'))
organization = mongo.ReferenceField(Organization,
required=False,
verbose_name=_('Organization'))
custom_field_form = mongo.ReferenceField("FormSchema", required=False)
activation_key = mongo.StringField(required=False)
key_expires = mongo.DateTimeField(required=False)
ban = mongo.BooleanField(default=False)
ban_reason = mongo.StringField(max_length=255, required=False)
address = mongo.StringField(max_length=255, required=False)
city = mongo.StringField(max_length=255, required=False)
state = mongo.StringField(max_length=255, required=False)
zip = mongo.StringField(max_length=50, required=False)
phone = mongo.StringField(max_length=20, required=False)
description = mongo.StringField(required=False)
point = mongo.PointField(required=False)
#used for dashboard resource reference
dashboard_resource_id = mongo.StringField(max_length=24, required=False)
# TODO disabled for now, either needs to become a custom form, or hard coded like the address above
# billing_phone = mongo.StringField(max_length=20, required=False, verbose_name=_('Billing Phone'))
# shipping_phone = mongo.StringField(max_length=20, required=False, verbose_name=_('Shipping Phone'))
# same_address = mongo.BooleanField(default=False, verbose_name=_('Is shipping adddress same as billing address'))
# address_verified = mongo.BooleanField(default=False, verbose_name=_('Address verified by TaxCloud'))
created = mongo.StringField(required=True, default=str(datetime.now()))
updated = mongo.StringField(required=True, default=str(datetime.now()))
USERNAME_FIELD = 'email'
REQUIRED_FIELDS = ['full_name']
@mongo.queryset.queryset_manager
def objects(doc_cls, queryset):
# This may actually also be done by defining a default ordering for
# the document, but this illustrates the use of manager methods
return queryset.filter(site_id=settings.SITE_ID)
@mongo.queryset.queryset_manager
def all(doc_cls, queryset):
# This may actually also be done by defining a default ordering for
# the document, but this illustrates the use of manager methods
return queryset.all()
def save(self, *args, **kwargs):
self.updated = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if self.address and self.zip and self.city and self.state:
address = (self.address + ", " + self.zip, self.city, self.state)
geocoded_addr = verify_address(address)
if geocoded_addr != 'Error':
self.point = geocoded_addr
super(User, self).save(*args, **kwargs)
if not self.primary_user_type:
self.set_primary_user_type()
def set_current_role(self, role):
self.current_role = role
self.save()
def set_primary_user_type(self, typ=None):
admin = UserTypes.objects.get(name='Admin')
if typ:
typ = UserTypes.objects.get(name=typ)
elif admin in self.user_types:
typ = admin
else:
typ = self.user_types[0]
self.primary_user_type = typ
if self.site_id == 1:
self.current_role = 'maps-admin'
else:
self.current_role = typ.name
self.save()
# def is_provider(self):
# return len(self.user_types.filter(type='provider')) > 0
def get_username(self):
"Return the identifying username for this User"
return getattr(self, self.USERNAME_FIELD)
def __unicode__(self):
return self.get_username()
def natural_key(self):
return (self.get_username(), )
def is_anonymous(self):
"""
Always returns False. This is a way of comparing User objects to
anonymous users.
"""
return False
def is_authenticated(self):
"""
Always return True. This is a way to tell if the user has been
authenticated in templates.
"""
return True
def set_password(self, raw_password):
self.password = make_password(raw_password)
def check_password(self, raw_password):
"""
Returns a boolean of whether the raw_password was correct. Handles
hashing formats behind the scenes.
"""
def setter(raw_password):
self.set_password(raw_password)
self.save(update_fields=["password"])
return check_password(raw_password, self.password, setter)
def set_unusable_password(self):
# Sets a value that will never be a valid hash
self.password = make_password(None)
def has_usable_password(self):
return is_password_usable(self.password)
def get_session_auth_hash(self):
"""
Returns an HMAC of the password field.
"""
key_salt = "maps.users.models.AbstractBaseUser.get_session_auth_hash"
return salted_hmac(key_salt, self.password).hexdigest()
def email_user(self, subject, message, from_email=None, **kwargs):
"""
Sends an email to this User.
"""
send_mail(subject, message, from_email, [self.email], **kwargs)
@property
def location_count(self):
try:
return Location.objects(created_by=str(self.id)).count()
except mongo.ValidationError:
return 0
@property
def image_url(self):
url = None
if self.image:
url = settings.MEDIA_URL + self.image
return url
def custom_form(self):
output = []
try:
if self.custom_field_form:
serializer = FormSchemaIdSerializer(self.custom_field_form)
output.append(serializer.data)
except:
pass
return output
def has_permission(self, obj_name, action):
for ut in self.user_types:
if ut.has_permission(obj_name, action):
return True
return False
def has_edit_other_permission(self, obj_name):
return self.has_permission(obj_name, '4')
def has_delete_other_permission(self, obj_name):
return self.has_permission(obj_name, '5')
def has_edit_own_permission(self, obj_name):
return self.has_permission(obj_name, '1')
def has_delete_own_permission(self, obj_name):
return self.has_permission(obj_name, '2')
def can_delete(self, obj, perm_type=None):
if not perm_type:
perm_type = obj.permission_type
if obj.created_by == self.id:
if self.has_delete_own_permission(perm_type):
return True
else:
if self.has_delete_other_permission(perm_type):
return True
return False
def can_edit(self, obj, perm_type=None):
if not perm_type:
perm_type = obj.permission_type
if obj.created_by == self.id:
if self.has_edit_own_permission(perm_type):
return True
else:
if self.has_edit_other_permission(perm_type):
return True
return False
@property
def permission_type(self):
return "users"
class SubDocB(me.Document):
field_b = me.IntField(primary_key=True, default=42)
class Friend(mongoengine.EmbeddedDocument):
id = mongoengine.IntField(name='fb_id')
name = mongoengine.StringField()
class PowResult(ModelMixin, mg.Document):
meta = {"collection": "zil_pow_results", "strict": False}
header = mg.StringField(max_length=128, required=True)
seed = mg.StringField(max_length=128, required=True)
boundary = mg.StringField(max_length=128, required=True)
pub_key = mg.StringField(max_length=128, required=True)
mix_digest = mg.StringField(max_length=128, required=True)
nonce = mg.StringField(max_length=128, required=True)
hash_result = mg.StringField(max_length=128, required=True)
block_num = mg.IntField(default=0)
pow_fee = mg.FloatField(default=0.0)
finished_date = mg.DateField()
finished_time = mg.DateTimeField()
verified_time = mg.DateTimeField()
verified = mg.BooleanField(default=False)
miner_wallet = mg.StringField(max_length=128)
worker_name = mg.StringField(max_length=64, default="")
def __str__(self):
return f"[PowResult: {self.pub_key}, {self.header}]"
@classmethod
def avg_pow_fee(cls, block_num):
return cls.query(block_num=block_num).average("pow_fee")
@classmethod
def get_pow_result(cls, header, boundary, pub_key=None, order="-finished_time"):
query = Q(header=header) & Q(boundary=boundary)
if pub_key is not None:
query = query & Q(pub_key=pub_key)
cursor = cls.objects(query).order_by(order) # default to get latest one
return cursor.first()
@classmethod
def epoch_rewards(cls, block_num=None, miner_wallet=None, worker_name=None):
match = {}
if block_num is not None:
if isinstance(block_num, int):
match = {
"block_num": {
"$eq": block_num,
}
}
else:
start, end = block_num
match = {
"block_num": {
"$gte": start,
"$lte": end,
}
}
if miner_wallet is not None:
match.update({
"miner_wallet": {
"$eq": miner_wallet,
}
})
if worker_name is not None:
match.update({
"worker_name": {
"$eq": worker_name,
}
})
group = {
"_id": None,
"rewards": {"$sum": "$pow_fee"},
"count": {"$sum": 1},
"verified": {"$sum": {"$cond": ["$verified", 1, 0]}},
"first_work_at": {"$min": "$finished_time"},
"last_work_at": {"$max": "$finished_time"}
}
pipeline = [
{"$match": match},
{"$group": group},
]
res = list(cls.objects.aggregate(*pipeline))
if res:
rewards = res[0]
rewards.pop("_id", None)
return rewards
return {"rewards": None, "count": 0, "verified": 0,
"first_work_at": None, "last_work_at": None}
@classmethod
def rewards_by_miners(cls, block_num):
if block_num is None:
block_num = PowWork.get_latest_block_num()
match = {
"block_num": {
"$eq": block_num,
}
}
group = {
"_id": "$miner_wallet",
"block_num": {"$first": "$block_num"},
"date": {"$first": "$finished_date"},
"date_time": {"$first": "$finished_time"},
"rewards": {"$sum": "$pow_fee"},
"finished": {"$sum": 1},
"verified": {"$sum": {"$cond": ["$verified", 1, 0]}},
}
project = {
"_id": 0,
"miner_wallet": "$_id",
"block_num": 1,
"date": 1,
"date_time": 1,
"rewards": 1,
"finished": 1,
"verified": 1,
}
pipeline = [
{"$match": match},
{"$group": group},
{"$project": project}
]
return list(cls.objects.aggregate(*pipeline))
def get_worker(self):
return miner.Worker.get_or_create(self.miner_wallet, self.worker_name)
