def test_fancy(self): si = S.SchemaItem.make(dict( a=S.Int(required=True), b=S.Int(if_missing=5))) self.assertRaises(S.Invalid, si.validate, dict(b=10)) self.assertEqual(si.validate(dict(a=10)), dict(a=10, b=5))
class TPremiumSize(MappedClass, EnhancingClass): class __mongometa__: session = session name = 'testpremiumssizes' _id = FieldProperty(schema.ObjectId) name = FieldProperty(schema.String(required=True)) min = FieldProperty(schema.Int(required=True)) max = FieldProperty(schema.Int(required=True)) description = FieldProperty(schema.String(if_missing = ''))
class Login(MappedClass): class __mongometa__: session = MingSession name = "logins" _id = s.ObjectId() id = s.Int() device_id = s.Int() datetime = s.DateTime() def __str__(self): return f"<Login(mongo) device_id={self.device_id} datetime={self.datetime}>"
class NestedModel(SproxTestClass): class __mongometa__: name = 'nested_model' _id = FieldProperty(S.ObjectId) group_name = FieldProperty(S.String) display_name = FieldProperty(S.String) number = FieldProperty(S.Int) author = FieldProperty({ 'name': S.String, 'surname': S.String, 'age': S.Int(required=False, if_missing=21), 'extra': { 'key': S.String, 'val': S.String }, 'interests': [S.String], 'other': [{ 'key': S.String(required=False, if_missing=''), 'val': S.String, 'meta': [S.String] }] }) contributors = FieldProperty([{ 'name': S.String, 'surname': S.String, 'age': S.Int }]) groups = FieldProperty([S.String])
class TaskState(Document): class __mongometa__: name = 'chapman.task' session = doc_session indexes = [ [('parent_id', 1), ('data.composite_position', 1)], ] _id = Field(int, if_missing=lambda: getrandbits(63)) type = Field(str) parent_id = Field(int, if_missing=None) status = Field(str, if_missing='pending') _result = Field('result', S.Binary) data = Field({str: None}) options = Field( dict( queue=S.String(if_missing='chapman'), priority=S.Int(if_missing=10), immutable=S.Bool(if_missing=False), ignore_result=S.Bool(if_missing=False), semaphores=[str], )) on_complete = Field(int, if_missing=None) mq = Field([int]) result = pickle_property('_result') @classmethod def set_result(cls, id, result): cls.m.update_partial( {'_id': id}, {'$set': { 'result': dumps(result), 'status': result.status }})
def setUp(self): self.MockSession = mock.Mock() self.MockSession.db = mock.MagicMock() self.TestDoc = collection( 'test_doc', self.MockSession, Field('a', int, if_missing=None, index=True), Field('b', S.Object, if_missing=dict(a=S.Int(if_missing=None)))) self.TestDocNoSchema = collection('test_doc', self.MockSession)
class TestDoc(Document): class __mongometa__: name = 'test_doc' session = self.MockSession indexes = [('a', )] a = Field(S.Int, if_missing=None) b = Field(S.Object(dict(a=S.Int(if_missing=None))))
class AirBus(Bus): class __mongometa__: polymorphic_identity = 'airbus' _type = FieldProperty(str, if_missing='airbus') wings_count = FieldProperty(schema.Int(if_missing=2)) def move(self): return 'flying from {} to {}'.format(self.origin, self.destination)
class Bus(Transport): class __mongometa__: polymorphic_identity = 'bus' _type = FieldProperty(str, if_missing='bus') passengers_count = FieldProperty(schema.Int(if_missing=0)) def move(self): return 'driving from {} to {}'.format(self.origin, self.destination)
class TestDoc(Document): class __mongometa__: name='test_doc' session = self.session indexes = [ ('b','c') ] unique_indexes = [ ('cc'), ] _id=Field(S.ObjectId, if_missing=None) a=Field(S.Int, if_missing=None) b=Field(S.Object(dict(a=S.Int(if_missing=None)))) cc=Field(dict(dd=int, ee=int))
class TPremium(MappedClass, EnhancingClass): class __mongometa__: session = session name = 'testpremiums' extensions = [ MyExtension ] _id = FieldProperty(schema.ObjectId) premium_id = ForeignIdProperty(TPremiumSize) worker_id = ForeignIdProperty(TWorker) earning_date = FieldProperty(schema.DateTime(required=True)) premium_size = FieldProperty(schema.Int(required=True)) note = FieldProperty(schema.String(if_missing = '')) worker = RelationProperty(TWorker) premium = RelationProperty(TPremiumSize)
class TPremium(MappedClass, EnhancingClass): class __mongometa__: session = session name = 'testpremiums' extensions = [ MyExtension ] #primary key _id = FieldProperty(schema.ObjectId) #foreign keys premium_id = FieldProperty(schema.String(required = True)) worker_id = FieldProperty(schema.String(required = True)) #properties earning_date = FieldProperty(schema.DateTime(required=True)) premium_size = FieldProperty(schema.Int(required=True)) note = FieldProperty(schema.String(if_missing = ''))
class TaskState(Document): class __mongometa__: name = 'chapman.task' session = doc_session indexes = [ [('parent_id', 1), ('data.composite_position', 1)], ] _id = Field(int, if_missing=lambda: getrandbits(63)) type = Field(str) parent_id = Field(int, if_missing=None) status = Field(str, if_missing='pending') _result = Field('result', S.Binary) data = Field({str: None}) options = Field( dict( queue=S.String(if_missing='chapman'), priority=S.Int(if_missing=10), immutable=S.Bool(if_missing=False), ignore_result=S.Bool(if_missing=False), path=S.String(if_missing=None), semaphores=[str], )) on_complete = Field(int, if_missing=None) active = Field([int]) # just one message active queued = Field([int]) # any number queued result = pickle_property('_result') @classmethod def set_result(cls, id, result): cls.m.update_partial( {'_id': id}, {'$set': { 'result': dumps(result), 'status': result.status }}) def __repr__(self): parts = [self.type, self._id] if self.options['path']: parts.append(self.options['path']) return '<{}>'.format(' '.join(map(str, parts)))
def test_dont_allow_none(self): si = S.Int(allow_none=False) self.assertRaises(S.Invalid, si.validate, None)
class HTTPMessage(Document): missing_client = '-' * 50 channel = ChannelProxy('chapman.event') class __mongometa__: name = 'chapman.http_message' session = doc_session indexes = [[('s.status', 1), ('s.pri', -1), ('s.ts_enqueue', 1), ('s.q', 1)], [('s.q', 1), ('s.status', 1), ('s.pri', -1), ('s.ts_enqueue', 1)], [('tags', 1)]] _id = Field(int, if_missing=lambda: getrandbits(63)) _data = Field('data', S.Binary(if_missing=bson.Binary('{}'))) tags = Field('tags', [str]) schedule = Field( 's', dict(status=S.String(if_missing='ready'), ts_enqueue=S.DateTime(if_missing=datetime.utcnow), ts_reserve=S.DateTime(if_missing=None), ts_timeout=S.DateTime(if_missing=None), timeout=S.Int(if_missing=300), after=S.DateTime(if_missing=datetime.fromtimestamp(0)), q=S.String(if_missing='chapman.http'), pri=S.Int(if_missing=10), cli=S.String(if_missing=missing_client))) def __json__(self, request): return {self.url(request): self.data} @property def data(self): return json.loads(self._data) @data.setter def data(self, value): self._data = bson.Binary(json.dumps(value, default=util.default_json)) def url(self, request): scheme = request.registry.settings.get('chapman.message_url_scheme', None) args = dict(message_id=self._id) if scheme is not None: args['_scheme'] = scheme return request.route_url('chapman.1_0.message', **args) @classmethod def new(cls, data, timeout=300, after=None, q='chapman.http', pri=10, tags=None): if tags is None: tags = [] _data = bson.Binary(json.dumps(data, default=util.default_json)) if after is None: after = datetime.utcnow() self = cls.make( dict(data=_data, tags=tags, s=dict(timeout=timeout, after=after, q=q, pri=pri))) self.m.insert() self.channel.pub('enqueue', self._id) return self @classmethod def reserve(cls, cli, queues): return cls._reserve(cli, {'$in': queues}) @classmethod def _reserve(cls, cli, qspec): now = datetime.utcnow() self = cls.m.find_and_modify( { 's.status': 'ready', 's.q': qspec, 's.after': { '$lte': now } }, sort=[('s.pri', -1), ('s.ts_enqueue', 1)], update={ '$set': { 's.cli': cli, 's.status': 'reserved', 's.ts_reserve': now } }, new=True) if self is not None and self.s.timeout: self.m.set( {'s.ts_timeout': now + timedelta(seconds=self.s.timeout)}) return self
class Message(Document): missing_worker = '-' * 10 channel = ChannelProxy('chapman.event') class __mongometa__: name = 'chapman.message' session = doc_session indexes = [ [('s.status', 1), ('s.pri', -1), ('s.ts', 1), ('s.q', 1)], [('s.q', 1), ('s.status', 1), ('s.sub_status', -1), ('s.pri', -1), ('s.ts', 1)], [('task_id', 1)], ] _id = Field(int, if_missing=lambda: getrandbits(63)) task_id = Field(int, if_missing=None) task_repr = Field(str, if_missing=None) slot = Field(str) _args = Field('args', S.Binary) _kwargs = Field('kwargs', S.Binary) _send_args = Field('send_args', S.Binary) _send_kwargs = Field('send_kwargs', S.Binary) schedule = Field('s', dict( status=S.String(if_missing='pending'), sub_status=S.Int(if_missing=0), ts=S.DateTime(if_missing=datetime.utcnow), after=S.DateTime(if_missing=datetime.utcnow), q=S.String(if_missing='chapman'), pri=S.Int(if_missing=10), w=S.String(if_missing=missing_worker), semaphores=[str])) def __repr__(self): return '<msg (%s:%s) %s to %s %s on %s>' % ( self.schedule.status, self.schedule.sub_status, self._id, self.slot, self.task_repr, self.schedule.w) @classmethod def n(cls, task, slot, *args, **kwargs): '''Convenience method for Message.new''' return cls.new(task, slot, args, kwargs) @classmethod def new(cls, task, slot, args, kwargs, after=None): if args is None: args = () if kwargs is None: kwargs = {} self = cls.make(dict( task_id=task.id, task_repr=repr(task), slot=slot, s=task.schedule_options())) if after is not None: self.s.after = after self.args = args self.kwargs = kwargs self.m.insert() return self @classmethod def reserve(cls, worker, queues): '''Reserve a message & try to lock the task state. - If no message could be reserved, return (None, None) - If a message was reserved, but the resources could not be acquired, return (msg, None) - If a message was reserved, and the resources were acquired, return (msg, task) ''' qspec = {'$in': queues} return cls._reserve(worker, qspec) @classmethod def reserve_qspec(cls, worker, qspec): '''Reserve according to a queue specification to allow for more interesting queue topologies ''' return cls._reserve(worker, qspec) def retire(self): '''Retire the message.''' self._release_resources() self.m.delete() def unlock(self): '''Make a message ready for processing''' self._release_resources() # Re-dispatch this message self.__class__.m.update_partial( {'_id': self._id}, {'$set': { 's.status': 'ready', 's.sub_status': 0, 's.w': self.missing_worker}}) self.channel.pub('send', self._id) def send(self, *args, **kwargs): self.m.set( {'s.status': 'ready', 's.ts': datetime.utcnow(), 'send_args': dumps(args), 'send_kwargs': dumps(kwargs)}) self.channel.pub('send', self._id) @property def args(self): result = [] if self._send_args is not None: result += loads(self._send_args) if self._args is not None: result += loads(self._args) return tuple(result) @args.setter def args(self, value): self._args = dumps(value) @property def kwargs(self): result = {} if self._kwargs is not None: result.update(loads(self._kwargs)) if self._send_kwargs is not None: result.update(loads(self._send_kwargs)) return result @kwargs.setter def kwargs(self, value): self._kwargs = dumps(value) @property def resources(self): for sem in self.s.semaphores: yield SemaphoreResource(sem) yield TaskStateResource(self.task_id) @classmethod def _reserve(cls, worker, qspec): '''Reserves a message.''' now = datetime.utcnow() # Begin acquisition of resources self = cls.m.find_and_modify( {'s.q': qspec, 's.status': 'ready', 's.after': {'$lte': now}}, sort=[('s.sub_status', -1), ('s.pri', -1), ('s.ts', 1)], update={'$set': {'s.w': worker, 's.status': 'acquire'}}, new=True) if self is None: return None, None # Acquire any resources necessary for i, res in enumerate(self.resources): if i < self.s.sub_status: # already acquired continue if not res.acquire(self._id): self.m.set({'s.status': 'queued'}) return self, None else: res = cls.m.update_partial( {'_id': self._id}, {'$set': {'s.sub_status': i + 1}}) self.m.set({'s.status': 'busy'}) return self, TaskState.m.get(_id=self.task_id) def _release_resources(self): msg_id = None for res in reversed(list(self.resources)): to_release = res.release(self._id) res = Message.m.update_partial( {'_id': {'$in': to_release}, 's.status': 'queued'}, {'$set': {'s.status': 'ready'}}, multi=True) if to_release and res['updatedExisting']: msg_id = to_release[0] if msg_id is not None: self.channel.pub('send', msg_id)