def test_fancy(self):
si = S.SchemaItem.make(dict(
a=S.Int(required=True),
b=S.Int(if_missing=5)))
self.assertRaises(S.Invalid, si.validate, dict(b=10))
self.assertEqual(si.validate(dict(a=10)),
dict(a=10, b=5))
class TPremiumSize(MappedClass, EnhancingClass):
class __mongometa__:
session = session
name = 'testpremiumssizes'
_id = FieldProperty(schema.ObjectId)
name = FieldProperty(schema.String(required=True))
min = FieldProperty(schema.Int(required=True))
max = FieldProperty(schema.Int(required=True))
description = FieldProperty(schema.String(if_missing = ''))
class Login(MappedClass):
class __mongometa__:
session = MingSession
name = "logins"
_id = s.ObjectId()
id = s.Int()
device_id = s.Int()
datetime = s.DateTime()
def __str__(self):
return f"<Login(mongo) device_id={self.device_id} datetime={self.datetime}>"
class NestedModel(SproxTestClass):
class __mongometa__:
name = 'nested_model'
_id = FieldProperty(S.ObjectId)
group_name = FieldProperty(S.String)
display_name = FieldProperty(S.String)
number = FieldProperty(S.Int)
author = FieldProperty({
'name':
S.String,
'surname':
S.String,
'age':
S.Int(required=False, if_missing=21),
'extra': {
'key': S.String,
'val': S.String
},
'interests': [S.String],
'other': [{
'key': S.String(required=False, if_missing=''),
'val': S.String,
'meta': [S.String]
}]
})
contributors = FieldProperty([{
'name': S.String,
'surname': S.String,
'age': S.Int
}])
groups = FieldProperty([S.String])
class TaskState(Document):
class __mongometa__:
name = 'chapman.task'
session = doc_session
indexes = [
[('parent_id', 1), ('data.composite_position', 1)],
]
_id = Field(int, if_missing=lambda: getrandbits(63))
type = Field(str)
parent_id = Field(int, if_missing=None)
status = Field(str, if_missing='pending')
_result = Field('result', S.Binary)
data = Field({str: None})
options = Field(
dict(
queue=S.String(if_missing='chapman'),
priority=S.Int(if_missing=10),
immutable=S.Bool(if_missing=False),
ignore_result=S.Bool(if_missing=False),
semaphores=[str],
))
on_complete = Field(int, if_missing=None)
mq = Field([int])
result = pickle_property('_result')
@classmethod
def set_result(cls, id, result):
cls.m.update_partial(
{'_id': id},
{'$set': {
'result': dumps(result),
'status': result.status
}})
def setUp(self):
self.MockSession = mock.Mock()
self.MockSession.db = mock.MagicMock()
self.TestDoc = collection(
'test_doc', self.MockSession,
Field('a', int, if_missing=None, index=True),
Field('b', S.Object, if_missing=dict(a=S.Int(if_missing=None))))
self.TestDocNoSchema = collection('test_doc', self.MockSession)
class TestDoc(Document):
class __mongometa__:
name = 'test_doc'
session = self.MockSession
indexes = [('a', )]
a = Field(S.Int, if_missing=None)
b = Field(S.Object(dict(a=S.Int(if_missing=None))))
class AirBus(Bus):
class __mongometa__:
polymorphic_identity = 'airbus'
_type = FieldProperty(str, if_missing='airbus')
wings_count = FieldProperty(schema.Int(if_missing=2))
def move(self):
return 'flying from {} to {}'.format(self.origin, self.destination)
class Bus(Transport):
class __mongometa__:
polymorphic_identity = 'bus'
_type = FieldProperty(str, if_missing='bus')
passengers_count = FieldProperty(schema.Int(if_missing=0))
def move(self):
return 'driving from {} to {}'.format(self.origin, self.destination)
class TestDoc(Document):
class __mongometa__:
name='test_doc'
session = self.session
indexes = [ ('b','c') ]
unique_indexes = [ ('cc'), ]
_id=Field(S.ObjectId, if_missing=None)
a=Field(S.Int, if_missing=None)
b=Field(S.Object(dict(a=S.Int(if_missing=None))))
cc=Field(dict(dd=int, ee=int))
class TPremium(MappedClass, EnhancingClass):
class __mongometa__:
session = session
name = 'testpremiums'
extensions = [ MyExtension ]
_id = FieldProperty(schema.ObjectId)
premium_id = ForeignIdProperty(TPremiumSize)
worker_id = ForeignIdProperty(TWorker)
earning_date = FieldProperty(schema.DateTime(required=True))
premium_size = FieldProperty(schema.Int(required=True))
note = FieldProperty(schema.String(if_missing = ''))
worker = RelationProperty(TWorker)
premium = RelationProperty(TPremiumSize)
class TPremium(MappedClass, EnhancingClass):
class __mongometa__:
session = session
name = 'testpremiums'
extensions = [ MyExtension ]
#primary key
_id = FieldProperty(schema.ObjectId)
#foreign keys
premium_id = FieldProperty(schema.String(required = True))
worker_id = FieldProperty(schema.String(required = True))
#properties
earning_date = FieldProperty(schema.DateTime(required=True))
premium_size = FieldProperty(schema.Int(required=True))
note = FieldProperty(schema.String(if_missing = ''))
class TaskState(Document):
class __mongometa__:
name = 'chapman.task'
session = doc_session
indexes = [
[('parent_id', 1), ('data.composite_position', 1)],
]
_id = Field(int, if_missing=lambda: getrandbits(63))
type = Field(str)
parent_id = Field(int, if_missing=None)
status = Field(str, if_missing='pending')
_result = Field('result', S.Binary)
data = Field({str: None})
options = Field(
dict(
queue=S.String(if_missing='chapman'),
priority=S.Int(if_missing=10),
immutable=S.Bool(if_missing=False),
ignore_result=S.Bool(if_missing=False),
path=S.String(if_missing=None),
semaphores=[str],
))
on_complete = Field(int, if_missing=None)
active = Field([int]) # just one message active
queued = Field([int]) # any number queued
result = pickle_property('_result')
@classmethod
def set_result(cls, id, result):
cls.m.update_partial(
{'_id': id},
{'$set': {
'result': dumps(result),
'status': result.status
}})
def __repr__(self):
parts = [self.type, self._id]
if self.options['path']:
parts.append(self.options['path'])
return '<{}>'.format(' '.join(map(str, parts)))
def test_dont_allow_none(self):
si = S.Int(allow_none=False)
self.assertRaises(S.Invalid, si.validate, None)
class HTTPMessage(Document):
missing_client = '-' * 50
channel = ChannelProxy('chapman.event')
class __mongometa__:
name = 'chapman.http_message'
session = doc_session
indexes = [[('s.status', 1), ('s.pri', -1), ('s.ts_enqueue', 1),
('s.q', 1)],
[('s.q', 1), ('s.status', 1), ('s.pri', -1),
('s.ts_enqueue', 1)], [('tags', 1)]]
_id = Field(int, if_missing=lambda: getrandbits(63))
_data = Field('data', S.Binary(if_missing=bson.Binary('{}')))
tags = Field('tags', [str])
schedule = Field(
's',
dict(status=S.String(if_missing='ready'),
ts_enqueue=S.DateTime(if_missing=datetime.utcnow),
ts_reserve=S.DateTime(if_missing=None),
ts_timeout=S.DateTime(if_missing=None),
timeout=S.Int(if_missing=300),
after=S.DateTime(if_missing=datetime.fromtimestamp(0)),
q=S.String(if_missing='chapman.http'),
pri=S.Int(if_missing=10),
cli=S.String(if_missing=missing_client)))
def __json__(self, request):
return {self.url(request): self.data}
@property
def data(self):
return json.loads(self._data)
@data.setter
def data(self, value):
self._data = bson.Binary(json.dumps(value, default=util.default_json))
def url(self, request):
scheme = request.registry.settings.get('chapman.message_url_scheme',
None)
args = dict(message_id=self._id)
if scheme is not None:
args['_scheme'] = scheme
return request.route_url('chapman.1_0.message', **args)
@classmethod
def new(cls,
data,
timeout=300,
after=None,
q='chapman.http',
pri=10,
tags=None):
if tags is None:
tags = []
_data = bson.Binary(json.dumps(data, default=util.default_json))
if after is None:
after = datetime.utcnow()
self = cls.make(
dict(data=_data,
tags=tags,
s=dict(timeout=timeout, after=after, q=q, pri=pri)))
self.m.insert()
self.channel.pub('enqueue', self._id)
return self
@classmethod
def reserve(cls, cli, queues):
return cls._reserve(cli, {'$in': queues})
@classmethod
def _reserve(cls, cli, qspec):
now = datetime.utcnow()
self = cls.m.find_and_modify(
{
's.status': 'ready',
's.q': qspec,
's.after': {
'$lte': now
}
},
sort=[('s.pri', -1), ('s.ts_enqueue', 1)],
update={
'$set': {
's.cli': cli,
's.status': 'reserved',
's.ts_reserve': now
}
},
new=True)
if self is not None and self.s.timeout:
self.m.set(
{'s.ts_timeout': now + timedelta(seconds=self.s.timeout)})
return self
class Message(Document):
missing_worker = '-' * 10
channel = ChannelProxy('chapman.event')
class __mongometa__:
name = 'chapman.message'
session = doc_session
indexes = [
[('s.status', 1), ('s.pri', -1), ('s.ts', 1), ('s.q', 1)],
[('s.q', 1), ('s.status', 1), ('s.sub_status', -1), ('s.pri', -1), ('s.ts', 1)],
[('task_id', 1)],
]
_id = Field(int, if_missing=lambda: getrandbits(63))
task_id = Field(int, if_missing=None)
task_repr = Field(str, if_missing=None)
slot = Field(str)
_args = Field('args', S.Binary)
_kwargs = Field('kwargs', S.Binary)
_send_args = Field('send_args', S.Binary)
_send_kwargs = Field('send_kwargs', S.Binary)
schedule = Field('s', dict(
status=S.String(if_missing='pending'),
sub_status=S.Int(if_missing=0),
ts=S.DateTime(if_missing=datetime.utcnow),
after=S.DateTime(if_missing=datetime.utcnow),
q=S.String(if_missing='chapman'),
pri=S.Int(if_missing=10),
w=S.String(if_missing=missing_worker),
semaphores=[str]))
def __repr__(self):
return '<msg (%s:%s) %s to %s %s on %s>' % (
self.schedule.status, self.schedule.sub_status,
self._id, self.slot, self.task_repr,
self.schedule.w)
@classmethod
def n(cls, task, slot, *args, **kwargs):
'''Convenience method for Message.new'''
return cls.new(task, slot, args, kwargs)
@classmethod
def new(cls, task, slot, args, kwargs, after=None):
if args is None:
args = ()
if kwargs is None:
kwargs = {}
self = cls.make(dict(
task_id=task.id,
task_repr=repr(task),
slot=slot,
s=task.schedule_options()))
if after is not None:
self.s.after = after
self.args = args
self.kwargs = kwargs
self.m.insert()
return self
@classmethod
def reserve(cls, worker, queues):
'''Reserve a message & try to lock the task state.
- If no message could be reserved, return (None, None)
- If a message was reserved, but the resources could not be acquired,
return (msg, None)
- If a message was reserved, and the resources were acquired, return
(msg, task)
'''
qspec = {'$in': queues}
return cls._reserve(worker, qspec)
@classmethod
def reserve_qspec(cls, worker, qspec):
'''Reserve according to a queue specification to allow for
more interesting queue topologies
'''
return cls._reserve(worker, qspec)
def retire(self):
'''Retire the message.'''
self._release_resources()
self.m.delete()
def unlock(self):
'''Make a message ready for processing'''
self._release_resources()
# Re-dispatch this message
self.__class__.m.update_partial(
{'_id': self._id},
{'$set': {
's.status': 'ready',
's.sub_status': 0,
's.w': self.missing_worker}})
self.channel.pub('send', self._id)
def send(self, *args, **kwargs):
self.m.set(
{'s.status': 'ready',
's.ts': datetime.utcnow(),
'send_args': dumps(args),
'send_kwargs': dumps(kwargs)})
self.channel.pub('send', self._id)
@property
def args(self):
result = []
if self._send_args is not None:
result += loads(self._send_args)
if self._args is not None:
result += loads(self._args)
return tuple(result)
@args.setter
def args(self, value):
self._args = dumps(value)
@property
def kwargs(self):
result = {}
if self._kwargs is not None:
result.update(loads(self._kwargs))
if self._send_kwargs is not None:
result.update(loads(self._send_kwargs))
return result
@kwargs.setter
def kwargs(self, value):
self._kwargs = dumps(value)
@property
def resources(self):
for sem in self.s.semaphores:
yield SemaphoreResource(sem)
yield TaskStateResource(self.task_id)
@classmethod
def _reserve(cls, worker, qspec):
'''Reserves a message.'''
now = datetime.utcnow()
# Begin acquisition of resources
self = cls.m.find_and_modify(
{'s.q': qspec, 's.status': 'ready', 's.after': {'$lte': now}},
sort=[('s.sub_status', -1), ('s.pri', -1), ('s.ts', 1)],
update={'$set': {'s.w': worker, 's.status': 'acquire'}},
new=True)
if self is None:
return None, None
# Acquire any resources necessary
for i, res in enumerate(self.resources):
if i < self.s.sub_status: # already acquired
continue
if not res.acquire(self._id):
self.m.set({'s.status': 'queued'})
return self, None
else:
res = cls.m.update_partial(
{'_id': self._id},
{'$set': {'s.sub_status': i + 1}})
self.m.set({'s.status': 'busy'})
return self, TaskState.m.get(_id=self.task_id)
def _release_resources(self):
msg_id = None
for res in reversed(list(self.resources)):
to_release = res.release(self._id)
res = Message.m.update_partial(
{'_id': {'$in': to_release}, 's.status': 'queued'},
{'$set': {'s.status': 'ready'}},
multi=True)
if to_release and res['updatedExisting']:
msg_id = to_release[0]
if msg_id is not None:
self.channel.pub('send', msg_id)
