def make_primary_key(self, message_box: MessageBox,
payload: 'Payload') -> ModuleMessage:
timestamp = ModuleMessage.make_timestamp()
with self.counter_lock:
counter = self.counters[message_box.uuid]
if counter >= 32767:
counter = 1
else:
counter += 1
self.counters[message_box.uuid] = counter
return ModuleMessage(message_box.uuid, timestamp, counter, payload)
async def test_post_date(mysql, mock_circlecore):
mbox_id = uuid.uuid4()
with MetaDataSession.begin():
schema = Schema.create(display_name='Schema',
properties='date:datetime,id1:float')
module = Module.create(display_name='Module')
mbox = MessageBox(uuid=mbox_id,
schema_uuid=schema.uuid,
module_uuid=module.uuid)
MetaDataSession.add(schema)
MetaDataSession.add(module)
MetaDataSession.add(mbox)
envdir = mock_circlecore[1]
database = Database(mysql.url, time_db_dir=envdir, log_dir=envdir)
journal_writer = database.make_writer()
queued_writer = journal_writer.child_writer
run_loop = asyncio.ensure_future(journal_writer.run())
# post
message = ModuleMessage(mbox.uuid, 123456.789, 0, {
'date': '2017-09-01T24:00:00Z',
'id1': 3.14
})
await queued_writer.store(mbox, message)
await queued_writer.flush()
# close database
await journal_writer.close()
await run_loop
with database._engine.begin() as connection:
table = database.find_table_for_message_box(mbox)
rows = connection.execute(select([table])).fetchall()
assert len(rows) == 0
def on_new_message(self, topic, jsonobj):
"""新しいメッセージを受けとった"""
message = ModuleMessage.from_json(jsonobj)
logger.debug('on_new_message: %s', message)
if message.box_id not in self.sync_states:
# not target
return
sync_state = self.sync_states[message.box_id]
try:
if not sync_state.is_synced():
# boxがまだsync中であればnew_messageは無視する。syncの方で同期されるはずなので
logger.debug('skip message %s, because not synced', message)
logger.debug(' master %s, slave %s',
self.sync_states[message.box_id].master_head,
self.sync_states[message.box_id].slave_head)
else:
sync_state.slave_head = message.primary_key
# pass to slave
logger.debug('pass message %s', message)
self._send_command(MasterCommand.NEW_MESSAGE,
message=message.to_json())
finally:
sync_state.master_head = message.primary_key
async def _store_message(self, data):
logger.info('!?data', repr(data))
message = ModuleMessage.from_json(data)
logger.info('!?message', repr(message))
if message.box_id not in self.target_boxes:
raise DataConfilictedError('invalid box id')
message_box = MessageBox.query.filter_by(uuid=message.box_id).one()
await self.writer.store(message_box, message)
async def test_store_blob(mysql, mock_circlecore):
envdir = mock_circlecore[1]
with MetaDataSession.begin():
ccinfo = CcInfo.query.filter_by(myself=True).one()
module = Module.create()
schema = Schema.create(display_name='Schema',
properties='x:int,y:float,data:blob')
box = MessageBox(uuid=generate_uuid(model=MessageBox),
schema=schema,
module=module,
display_name='Box')
MetaDataSession.add(schema)
MetaDataSession.add(module)
MetaDataSession.add(box)
database = Database(mysql.url, time_db_dir=envdir, log_dir=envdir)
connection = database._engine.connect()
with database._engine.begin() as connection:
database.store_message(
box,
ModuleMessage(
box.uuid,
'1545895047.000',
0,
{
'x':
0,
'y':
3.14,
'data':
BlobMetadata(ccinfo.uuid, 'text/plain',
'deadbeafdeadbeafdeadbeafdeadbeaf')
},
),
connection=connection)
with database._engine.begin() as connection:
table = database.find_table_for_message_box(box)
rows = connection.execute(select([table.c.data])).fetchall()
print(rows)
assert len(rows) == 1
expected = '{{"$data": "deadbeafdeadbeafdeadbeafdeadbeaf", "$source": "{source}", "$type": "text/plain"}}' \
.format(
source=str(ccinfo.uuid)
)
assert rows[0][0] == expected
async def test_rotate_journal(tmpdir, dummy_mbox, caplog):
schema, module, mbox = dummy_mbox
import logging
caplog.set_level(logging.DEBUG)
# prepare test data
pos_file_path = os.path.join(tmpdir, 'journal.pos')
for log_index in range(10):
log_file_path = os.path.join(tmpdir,
'journal.{:03d}'.format(log_index))
with open(log_file_path, 'wt') as fp:
for x in range(5):
fp.write('{{"x": {}}}\n'.format(x))
with open(pos_file_path, 'wt') as fp:
log_file_path = os.path.join(tmpdir,
'journal.{:03d}'.format(log_index))
fp.write('{}\n{}'.format(log_index, os.stat(log_file_path).st_size))
before_files = os.listdir(tmpdir)
assert len(before_files) == 11
# open writer
child_writer_mock = MagicMock()
child_writer_mock.store.side_effect = asyncio.coroutine(
lambda *args, **kwargs: True)
writer = JournalDBWriter(child_writer_mock, tmpdir, max_log_file_size=20)
message = ModuleMessage(mbox.uuid, 123456.789, 0, {'x': 1, 'y': 2})
await writer.store(mbox, message)
await asyncio.sleep(0.1)
#
after_files = os.listdir(tmpdir)
print(before_files)
print(after_files)
after_files.sort()
assert after_files == ['journal.011', 'journal.pos']
await writer.close()
del writer
async def test():
counterloop = asyncio.ensure_future(countup())
setup_db()
mbox_id = uuid.UUID('49EB92A3-AAE8-43A1-BC43-B7933DE96C6A')
with MetaDataSession.begin():
schema = Schema.create(display_name='Schema',
properties='x:int,y:float')
module = Module.create(display_name='Module')
mbox = MessageBox(uuid=mbox_id,
schema_uuid=schema.uuid,
module_uuid=module.uuid)
MetaDataSession.merge(schema)
MetaDataSession.merge(module)
MetaDataSession.merge(mbox)
database = Database('mysql+pymysql://root:hogehoge@localhost/crcr_test',
'./tmp/',
'./tmp/',
connect_args=dict(write_timeout=3, read_timeout=3))
database.drop_message_box(mbox)
queued_writer = QueuedDBWriter(database, './tmp/')
class Delegate(QueuedDBWriterDelegate):
async def on_reconnect(self) -> None:
print('on_reconnect')
writer.touch()
queued_writer = QueuedDBWriter(database, './tmp/', delegate=Delegate())
writer = JournalDBWriter(queued_writer, './tmp/')
print('connect')
# input('> ')
database.connect()
print('store')
# input('> ')
message = ModuleMessage(mbox.uuid, 123456.789, 0, {'x': 1, 'y': 2})
assert (await writer.store(mbox, message)) is True
print('store2 , please shutdown mysql')
await async_input('> ')
message = ModuleMessage(mbox.uuid, 123457.0, 1, {'x': 3, 'y': 4})
assert (await writer.store(mbox, message)) is True
assert (await writer.store(
mbox, ModuleMessage(mbox.uuid, 123458.0, 2, {
'x': 4,
'y': 4
}))) is True
assert (await writer.store(
mbox, ModuleMessage(mbox.uuid, 123459.0, 3, {
'x': 5,
'y': 4
}))) is True
assert (await writer.store(
mbox, ModuleMessage(mbox.uuid, 123460.0, 4, {
'x': 6,
'y': 4
}))) is True
assert (await writer.store(
mbox, ModuleMessage(mbox.uuid, 123461.0, 5, {
'x': 7,
'y': 4
}))) is True
print('store3 , please wakeup mysql')
await async_input('> ')
logger.info('DONE!')
global done
done = True
await counterloop
def test_message_jsonize(payload, expected):
message = ModuleMessage(uuid.UUID('539CE356-A7CB-4BFC-853E-C1A8147F021F'),
'1545895047.000', 0, payload)
serialized = serialize(message)
assert serialized == expected
async def test_journal_db_writer(tmpdir, dummy_mbox, caplog):
schema, module, mbox = dummy_mbox
import logging
caplog.set_level(logging.DEBUG)
# check dir is empty
assert not os.listdir(tmpdir)
# open writer
child_writer_mock = MagicMock()
writer = JournalDBWriter(child_writer_mock, tmpdir)
files = os.listdir(tmpdir)
assert 'journal.pos' in files
assert open(os.path.join(tmpdir, 'journal.pos'), 'rt').read() == ''
# store message
message = ModuleMessage(mbox.uuid, 123456.789, 0, {'x': 1, 'y': 2})
child_writer_mock.store.side_effect = asyncio.coroutine(
lambda *args, **kwargs: True)
await writer.store(mbox, message)
await asyncio.sleep(0.1)
await writer.close()
del writer
files = os.listdir(tmpdir)
assert 'journal.000' in files
assert 'journal.pos' in files
ln = open(os.path.join(tmpdir, 'journal.000')).read()
assert ln.endswith('\n')
assert ln and json.loads(ln) == {
'boxId': mbox.uuid.hex,
'timestamp': '123456.7890000000',
'counter': 0,
'payload': {
'x': 1,
'y': 2
}
}
child_writer_mock.store.assert_called_once()
checkpoint = '0\n{}'.format(len(ln))
assert open(os.path.join(tmpdir, 'journal.pos')).read() == checkpoint
# re-start journal
child_writer_mock.store.side_effect = asyncio.coroutine(
lambda *args, **kwargs: False)
writer = JournalDBWriter(child_writer_mock, tmpdir)
message = ModuleMessage(mbox.uuid, 123500.789, 1, {'x': 3, 'y': 4})
await writer.store(mbox, message)
await asyncio.sleep(0.1)
await writer.close()
del writer
with open(os.path.join(tmpdir, 'journal.000')) as fp:
ln = fp.readline()
assert ln and json.loads(ln) == {
'boxId': mbox.uuid.hex,
'timestamp': '123456.7890000000',
'counter': 0,
'payload': {
'x': 1,
'y': 2
}
}
ln = fp.readline()
assert ln and json.loads(ln) == {
'boxId': mbox.uuid.hex,
'timestamp': '123500.7890000000',
'counter': 1,
'payload': {
'x': 3,
'y': 4
}
}
print(child_writer_mock.store.call_args)
print(child_writer_mock.store.call_args[0])
assert child_writer_mock.store.call_count == 2
assert child_writer_mock.store.call_args[0][
1].timestamp == decimal.Decimal('123500.7890000000')
assert open(os.path.join(tmpdir, 'journal.pos')).read() == checkpoint