class TestEntify(unittest.TestCase): def setUp(self): self.db = setup_test() self.db.autocommit(1) self.people = EntityStore(self.db, Person) self.joe_id = self.people.put(Person(name='Joe', age=50)) self.sam_id = self.people.put(Person(name='Sam', age=25)) self.people.put(Person(name='Ann', age=30)) self.id_name = '_id' def tearDown(self): self.people.zap() self.db.close() def test_attribute(self, value='text'): joe = self.people.first(name='Joe') assert joe joe.entify_test_attribute = value joe.save() joe = self.people.first(name='Joe') self.assertEqual(joe.entify_test_attribute, value) def test_text(self): self.test_attribute('text value') def test_int(self): self.test_attribute(10) def test_float(self): self.test_attribute(1.2) def test_decimal(self): self.test_attribute(Decimal("1.20")) def test_date(self): self.test_attribute(date(2017, 12, 1)) def test_datetime(self): self.test_attribute(datetime(2017, 12, 1, 1, 25, 3)) def test_bool(self): self.test_attribute(True) def test_bool_false(self): self.test_attribute(False) def test_none(self): self.test_attribute(None) def test_list(self): self.test_attribute([1, 2, 3, 4]) def test_tuple(self): self.test_attribute((1, 2, 3, 4)) def test_bytes(self): self.test_attribute(b'this is binary')
class Topic(object): """ message topic """ def __init__(self, name, newest=None, db=None): self.name = name self.db = db self.messages = EntityStore(db, Message) self.newest = newest is not None and newest or self.last() or -1 def last(self): """get row_id of the last (newest) message in the topic""" if self.name: cmd = """ select max(row_id) n from attributes where kind=%s and attribute="topic" and value=%s """ rec = self.db(cmd, self.messages.kind, self.name) else: cmd = 'select max(row_id) n from attributes where kind=%s' rec = self.db(cmd, self.messages.kind) if type(rec) == int: return 0 row_id = rec.first()[0] return row_id or 0 def put(self, message): """put a message in the topic""" return self.messages.put( Message( topic=self.name, timestamp=now(), node=platform.node(), body=json.dumps(message), )) def clear(self): """clear the topic >>> messages = setup_test() >>> t = messages.get('test_topic') >>> t.send('hey!', 'you!') [1, 2] >>> len(t) 2 >>> t.clear() >>> len(t) 0 """ if self.name: self.messages.delete(topic=self.name) else: self.messages.zap() def send(self, *messages): """send list of messages >>> messages = setup_test() >>> t = messages.get('test_topic') >>> t.send('hey!', 'you!') [1, 2] >>> t.peek() 'hey!' >>> t.peek() 'hey!' """ return [self.put(message) for message in messages] def _peek(self, newest=None): def decoded(value): if type(value) is bytes: return value.decode('utf8') return value top_one = newest is not None and newest or self.newest or 0 db = self.db if self.last() > self.newest: if self.name: cmd = """ select min(row_id) as row_id from attributes where kind=%s and attribute="topic" and value=%s and row_id > %s """ rec = db(cmd, self.messages.kind, self.name, top_one) else: cmd = """ select min(row_id) as row_id from attributes where kind=%s and row_id > %s """ rec = db(cmd, self.messages.kind, top_one) if type(rec) == int: row_id = 0 else: row_id = rec.first()[0] if row_id: message = self.messages.get(row_id) if message and message.body is not None and message.topic is not None: # Checking for message body because finding an id does not guarantee # that a message is ready to go depending on isolation level of # database. Rather than require a specific isolation level we # just check to see if the message body and topic are present and if # not, we ignore the message. return row_id, decoded(message.topic), json.loads( decoded(message.body)) raise EmptyException def peek(self, newest=None): """ return the next message but don't remove it >>> messages = setup_test() >>> t = messages.get('test_topic') >>> t.peek() >>> t.put('hey!') 1 >>> t.put('you!') 2 >>> t.peek() 'hey!' >>> t.peek() 'hey!' """ try: return self._peek(newest)[2] except EmptyException: return None def _poll(self, newest=None): r = self._peek(newest) self.newest = r[0] return r def poll(self, newest=None): """ peek at the next message and increment internal pointer >>> messages = setup_test() >>> t = messages.get('test_topic') >>> t.put('hey!') 1 >>> t.put('you!') 2 >>> t.newest -1 >>> t.poll() 'hey!' >>> t.newest 1 >>> t.poll() 'you!' >>> raised = False >>> try: ... t.poll() ... except EmptyException: ... raised = True >>> raised True >>> t.newest = -1 >>> t.poll() 'hey!' """ return self._poll(newest)[2] def _pop(self): r = self._peek() row_id = r[0] self.messages.delete(row_id) if self.messages.db.rowcount > 0: self.newest = row_id return r else: # If we were unable to delete it then soneone else # has already deleted it between the time that # we saw it and the time we attempted to delete it. raise EmptyException def pop(self): """ read next message and remove it from the topic >>> messages = setup_test() >>> t = messages.get('test_topic') >>> t.put('hey!') 1 >>> t.put('you!') 2 >>> t.len() 2 >>> t._peek() (1, 'test_topic', 'hey!') >>> t.pop() 'hey!' >>> t.len() 1 >>> t.pop() 'you!' >>> t.len() 0 >>> t.pop() >>> t.newest = -1 >>> raised = False >>> try: ... t._pop() ... except EmptyException: ... raised = True >>> raised True """ try: return self._pop()[2] except EmptyException: return None def len(self, newest=None): """ return the number of messages in the topic >>> messages = setup_test() >>> t = messages.get('test_topic') >>> t.put('hey!') 1 >>> t.put('you!') 2 >>> t.len() 2 """ if self.last() > self.newest: if self.name: cmd = """ select count(row_id) as n from attributes where kind=%s and attribute="topic" and value=%s and row_id>%s """ t = self.db(cmd, self.messages.kind, self.name, self.newest) else: cmd = """select count(row_id) as n from attributes where kind=%s and row_id>%s """ t = self.db(cmd, self.messages.kind, self.newest) n = t.first()[0] or 0 return n return 0 def __len__(self): """ return the number of messages in a topic as an int (note: for large number of messages use t.len() >>> messages = setup_test() >>> t = messages.get('test_topic') >>> t.put('hey!') 1 >>> t.put('you!') 2 >>> len(t) 2 """ return self.len() def __iter__(self): """ iterate through a topic >>> messages = setup_test() >>> t = messages.get('test_topic') >>> t.put('hey!') 1 >>> t.put('you!') 2 >>> for m in t: print(m) hey! you! """ return TopicIterator(self, self.newest) def wait(self, delay=DEFAULT_DELAY, timeout=DEFAULT_TIMEOUT): """ wait for a message to arrive and return it >>> messages = setup_test() >>> t = messages.get('test_topic') >>> t.put('hey!') 1 >>> t.put('you!') 2 >>> t.wait() 'hey!' >>> t.wait() 'you!' """ deadline = time.time() + timeout while True: msg = self.pop() if msg: return msg time.sleep(delay) if time.time() > deadline: raise WaitException def listen(self, f, delay=DEFAULT_DELAY, meta=False): """ observe but don't consume messages >>> messages = setup_test() >>> t = messages.get('test_topic') >>> t.put('hey!') 1 >>> t.put('you!') 2 >>> def echo(m): ... print(m) ... return m == 'you!' >>> t.listen(echo) hey! you! 2 >>> t1 = messages.topic('test_topic1') >>> t2 = messages.topic('test_topic2') >>> t3 = messages.topic(None) >>> t1.put('hey!') 3 >>> t2.put('you!') 4 >>> def echo(m): ... print(m) ... return m == 'you!' >>> t3.listen(echo) hey! you! 2 """ n = 0 done = False while not done: try: more_to_do = True while more_to_do: try: p = self._poll() except EmptyException: more_to_do = False else: if meta: done = f(p) else: done = f(p[2]) n += 1 except StopListening: return n else: time.sleep(delay) return n def join(self, jobs, delay=DEFAULT_DELAY, timeout=DEFAULT_TIMEOUT): """wait for responses for consumers NOTE: the assumption at this point is that any provided delay or timeout applies to all jobs. If jobs need varying arguments then mutliple calls to join should be considered. """ return [ Topic( response_topic_name(self.name, job), newest=job, db=self.db, ).wait(delay=delay, timeout=timeout) for job in jobs ] def call(self, *messages, delay=DEFAULT_DELAY, timeout=DEFAULT_TIMEOUT): """send messages and wait for responses""" return self.join(self.send(*messages), delay=delay, timeout=timeout) def handle(self, f, timeout=0, delay=DEFAULT_DELAY, one_pass=False): """respond to and consume messages >>> messages = setup_test() >>> t = messages.get('test_topic') >>> def echo(m): ... if m == 'quit': raise StopHandling ... print('got', repr(m)) >>> t.put('hey!') 1 >>> t.put('you!') 2 >>> t.put('quit') 3 >>> t.handle(echo) got 'hey!' got 'you!' 2 """ deadline = timeout and time.time() + timeout done = False n = 0 while not done: try: try: more_to_do = True while more_to_do: try: row, topic, message = self._pop() result = f(message) t = Topic(response_topic_name(topic, row), None, self.db) t.send(result) deadline = timeout and time.time() + timeout n += 1 except EmptyException: more_to_do = False time.sleep(0) except StopHandling: done = True else: time.sleep(delay) except KeyboardInterrupt: done = True if timeout and time.time() > deadline: done = True return n def process(self, f): """respond to and consume current messages >>> messages = setup_test() >>> t = messages.get('test_topic') >>> def echo(m): ... if m == 'quit': raise StopProcessing ... print('got', repr(m)) >>> t.put('hey!') 1 >>> t.put('you!') 2 >>> t.put('quit') 3 >>> t.process(echo) got 'hey!' got 'you!' 2 >>> t.process(echo) 0 """ n = 0 more_to_do = True while more_to_do: try: row, topic, message = self._pop() if message is None: result = f() else: result = f(message) response_topic = response_topic_name(topic, row) t = Topic(response_topic, None, self.db) t.put(result) n += 1 except StopProcessing: more_to_do = False except EmptyException: more_to_do = False time.sleep(0) return n def perform(self, task, *args, **kwargs): """consume a single message and perform task with it >>> messages = setup_test() >>> t = messages.get('test_topic') >>> def echo(m): ... print('got', repr(m)) >>> t.put('hey!') 1 >>> t.put('you!') 2 >>> t.perform(echo) got 'hey!' True >>> t.perform(echo) got 'you!' True >>> t.perform(echo) False """ try: row, topic, message = self._pop() if message is None: result = task(*args, **kwargs) else: result = task(message, *args, **kwargs) response_topic = response_topic_name(topic, row) response_queue = Topic(response_topic, None, self.db) response_queue.put(result) except EmptyException: return False else: return True
class TestStore(unittest.TestCase): def setUp(self): self.db = setup_test() self.db.autocommit(1) self.people = EntityStore(self.db, Person) self.joe_id = self.people.put(Person(name='Joe', age=50)) self.sam_id = self.people.put(Person(name='Sam', age=25)) self.people.put(Person(name='Ann', age=30)) self.id_name = '_id' def tearDown(self): self.people.zap() self.db.close() def test_put(self): jane_id = self.people.put(Person(name='Jane', age=25)) person = self.people.get(jane_id) del person['__store'] self.assertEqual(dict(person), dict(_id=jane_id, name='Jane', age=25)) def test_get(self): joe = self.people.get(self.joe_id) del joe['__store'] self.assertEqual(dict(joe), dict(_id=self.joe_id, name='Joe', age=50)) def test_get_missing(self): joe = Person(name='Joe', age=50) joe_id = self.people.put(joe) person = self.people.get(joe_id) self.assertEqual(None, self.people.get(joe_id + 1)) def test_get_multiple(self): def sort_order(item): return keys.index(item['_id']) keys = [self.sam_id, self.joe_id] r = self.people.get(keys) sam = self.people.get(self.sam_id) joe = self.people.get(self.joe_id) self.assertEqual(sorted(r, key=sort_order), [sam, joe]) def test_get_put_get(self): sam = self.people.get(self.sam_id) self.assertEqual(sam.age, 25) self.assertEqual(len(self.people), 3) sam.age += 1 self.people.put(sam) self.assertEqual(len(self.people), 3) person = self.people.get(self.sam_id) self.assertEqual(person.age, 26) def test_get_save(self): sam = self.people.get(self.sam_id) self.assertEqual(sam.age, 25) self.assertEqual(len(self.people), 3) sam.age += 1 sam.save() self.assertEqual(len(self.people), 3) person = self.people.get(self.sam_id) self.assertEqual(person.age, 26) def test_resave(self): jane = Person(name='Jane', age=25) jane_id = self.people.put(jane) self.assertEqual(jane[self.id_name], 4) jane['age'] += 1 jane.age += 1 new_id = jane.save() self.assertEqual(new_id, 4) person = self.people.get(jane_id) self.assertEqual(person.age, 27) def test_delete_by_entity(self): sam = self.people.get(self.sam_id) self.assert_(sam) self.people.delete(sam) sam = self.people.get(self.sam_id) self.assertEqual(None, sam) def test_delete_by_id(self): sam = self.people.get(self.sam_id) self.assert_(sam) self.people.delete(self.sam_id) sam = self.people.get(self.sam_id) self.assertEqual(None, sam) def test_none(self): al_id = self.people.put(Person(name='Al', age=None)) al = self.people.get(al_id) self.assertEqual(al.age, None) def test_bool(self): al_id = self.people.put(Person(name='Al', done=False)) al = self.people.get(al_id) self.assertEqual(al.done, False) al.done = True self.people.put(al) person = self.people.get(al_id) self.assertEqual(person.done, True) def test_kind(self): self.assertEqual(self.people.kind, 'person') self.assertEqual(EntityStore(self.db, TestPerson).kind, 'test_person') def test_len(self): self.assertEqual(3, len(self.people)) def test_zap(self): self.assertEqual(3, len(self.people)) self.people.zap() self.assertEqual(0, len(self.people))
class TestStore(unittest.TestCase): def setUp(self): params = dict( host='database', user='******', passwd='password', db='test', ) self.db = Database(MySQLdb.Connect, **params) self.db.autocommit(1) self.people = EntityStore(self.db, Person) self.joe_id = self.people.put(Person(name='Joe', age=50)) self.sam_id = self.people.put(Person(name='Sam', age=25)) self.people.put(Person(name='Ann', age=30)) def tearDown(self): self.people.zap() self.db.close() def test_put(self): jane_id = self.people.put(Person(name='Jane', age=25)) person = self.people.get(jane_id) self.assertEqual(dict(person), dict(_id=jane_id, name='Jane', age=25)) def test_get(self): joe = self.people.get(self.joe_id) self.assertEqual(dict(joe), dict(_id=self.joe_id, name='Joe', age=50)) def test_get_missing(self): joe = Person(name='Joe', age=50) joe_id = self.people.put(joe) person = self.people.get(joe_id) self.assertEqual(None, self.people.get(joe_id + 1)) def test_get_multiple(self): def sort_order(item): return keys.index(item['_id']) keys = [self.sam_id, self.joe_id] r = self.people.get(keys) sam = self.people.get(self.sam_id) joe = self.people.get(self.joe_id) self.assertEqual(sorted(r, key=sort_order), [sam, joe]) def test_get_put_get(self): sam = self.people.get(self.sam_id) self.assertEqual(sam.age, 25) self.assertEqual(len(self.people), 3) sam.age += 1 self.people.put(sam) self.assertEqual(len(self.people), 3) person = self.people.get(self.sam_id) self.assertEqual(person.age, 26) def test_delete_by_entity(self): sam = self.people.get(self.sam_id) self.assert_(sam) self.people.delete(sam) sam = self.people.get(self.sam_id) self.assertEqual(None, sam) def test_delete_by_id(self): sam = self.people.get(self.sam_id) self.assert_(sam) self.people.delete(self.sam_id) sam = self.people.get(self.sam_id) self.assertEqual(None, sam) def test_none(self): al_id = self.people.put(Person(name='Al', age=None)) al = self.people.get(al_id) self.assertEqual(al.age, None) def test_bool(self): al_id = self.people.put(Person(name='Al', done=False)) al = self.people.get(al_id) self.assertEqual(al.done, False) al.done = True self.people.put(al) person = self.people.get(al_id) self.assertEqual(person.done, True) def test_kind(self): self.assertEqual(self.people.kind, 'person') self.assertEqual(EntityStore(self.db, TestPerson).kind, 'test_person') def test_len(self): self.assertEqual(3, len(self.people)) def test_zap(self): self.assertEqual(3, len(self.people)) self.people.zap() self.assertEqual(0, len(self.people))
class TestStore(unittest.TestCase): def setUp(self): self.db = setup_test() self.db.autocommit(1) self.people = EntityStore(self.db, Person) self.joe_id = self.people.put(Person(name='Joe', age=50)) self.sam_id = self.people.put(Person(name='Sam', age=25)) self.people.put(Person(name='Ann', age=30)) self.id_name = '_id' def tearDown(self): self.people.zap() self.db.close() def test_put(self): jane_id = self.people.put(Person(name='Jane', age=25)) person = self.people.get(jane_id) del person['__store'] self.assertEqual(dict(person), dict(_id=jane_id, name='Jane', age=25)) def test_get(self): joe = self.people.get(self.joe_id) del joe['__store'] self.assertEqual(dict(joe), dict(_id=self.joe_id, name='Joe', age=50)) def test_get_missing(self): joe = Person(name='Joe', age=50) joe_id = self.people.put(joe) person = self.people.get(joe_id) self.assertEqual(None, self.people.get(joe_id + 1)) def test_get_multiple(self): def sort_order(item): return keys.index(item['_id']) keys = [self.sam_id, self.joe_id] r = self.people.get(keys) sam = self.people.get(self.sam_id) joe = self.people.get(self.joe_id) self.assertEqual(sorted(r, key=sort_order), [sam, joe]) def test_get_put_get(self): sam = self.people.get(self.sam_id) self.assertEqual(sam.age, 25) self.assertEqual(len(self.people), 3) sam.age += 1 self.people.put(sam) self.assertEqual(len(self.people), 3) person = self.people.get(self.sam_id) self.assertEqual(person.age, 26) def test_get_save(self): sam = self.people.get(self.sam_id) self.assertEqual(sam.age, 25) self.assertEqual(len(self.people), 3) sam.age += 1 sam.save() self.assertEqual(len(self.people), 3) person = self.people.get(self.sam_id) self.assertEqual(person.age, 26) def test_resave(self): jane = Person(name='Jane', age=25) jane_id = self.people.put(jane) self.assertEqual(jane[self.id_name], 4) jane['age'] += 1 jane.age += 1 new_id = jane.save() self.assertEqual(new_id, 4) person = self.people.get(jane_id) self.assertEqual(person.age, 27) def test_delete_by_entity(self): sam = self.people.get(self.sam_id) self.assert_(sam) self.people.delete(sam) sam = self.people.get(self.sam_id) self.assertEqual(None, sam) def test_delete_by_id(self): sam = self.people.get(self.sam_id) self.assert_(sam) self.people.delete(self.sam_id) sam = self.people.get(self.sam_id) self.assertEqual(None, sam) def test_none(self): al_id = self.people.put(Person(name='Al', age=None)) al = self.people.get(al_id) self.assertEqual(al.age, None) def test_bool(self): al_id = self.people.put(Person(name='Al', done=False)) al = self.people.get(al_id) self.assertEqual(al.done, False) al.done = True self.people.put(al) person = self.people.get(al_id) self.assertEqual(person.done, True) def test_kind(self): self.assertEqual(self.people.kind, 'person') self.assertEqual(EntityStore(self.db, TestPerson).kind, 'test_person') def test_len(self): self.assertEqual(3, len(self.people)) def test_zap(self): self.assertEqual(3, len(self.people)) self.people.zap() self.assertEqual(0, len(self.people)) def test_empty(self): empty_store = EntityStore(self.db, 'none_of_these') self.assertEqual(type(empty_store.all()), EntityList) self.assertEqual(str(empty_store), 'Empty list') def test_make_store_select(self): data = [ dict(name='Alice', size=100, amount=2200), dict(name='Janice', size=510, amount=200), dict(name='Jimmy', size=200, amount=2100), dict(name='Ozzy', size=401, amount=1900), dict(name='Angus', size=510, amount=200), ] people = zoom.store_of(Person, self.db) for row in data: people.put(row) low = set([person.name for person in people.find(amount=200)]) self.assertEqual(low, set(['Janice', 'Angus'])) cmd = make_store_select('person', amount=less_than(2000), size=gt(401)) rows = entify(self.db(cmd)) by_id = lambda a: a['_id'] self.assertEqual( sorted(rows, key=by_id), sorted([ {'_id': 5, 'name': 'Janice', 'size': 510, 'amount': 200}, {'_id': 8, 'name': 'Angus', 'size': 510, 'amount': 200}, ], key=by_id) )