async def test_datareceiver_store_blob(mock_circlecore, mysql, monkeypatch):
metadata_db_engine, tmp_dir = mock_circlecore
writer_mock = MagicMock()
make_writer_mock = Mock(name='make_writer', return_value=writer_mock)
async def dummy_store(*args, **kwargs):
return DEFAULT
writer_mock.store.side_effect = dummy_store
monkeypatch.setattr(Database, 'make_writer', make_writer_mock)
# make test schema/mbox
with MetaDataSession.begin():
schema = Schema.create(display_name='Schema', properties='x:int,y:float,data:blob')
module = Module.create(display_name='Module')
mbox = MessageBox(uuid=generate_uuid(model=MessageBox), schema_uuid=schema.uuid, module_uuid=module.uuid)
MetaDataSession.add(schema)
MetaDataSession.add(module)
MetaDataSession.add(mbox)
core_mock = MagicMock()
worker = DataReceiverWorker(
core_mock,
'worker_key',
db_url=mysql.url,
time_db_dir=tmp_dir,
log_dir=tmp_dir,
cycle_time=10,
cycle_count=10,
)
datahash = (
'2b7e36b16f8a849ef312f9ef5ff9b3f4281a8681d0657150899f1113a0eecfdb'
'b4491da763159055b55e122e85281415b11897d268e124f9ef2b40457a63a465'
)
blobobj = StoredBlob(None, 'text/plain', datahash)
await worker.receive_new_message(mbox.uuid, {'x': 1, 'y': 2.0, 'data': blobobj})
publish_mock = core_mock.hub.publish
publish_mock.assert_called_once()
message = publish_mock.call_args[0][1]
assert message.payload['data'].content_type == 'text/plain'
assert message.payload['data'].datahash == datahash
def _post_modules() -> 'Response':
"""Moduleを作成する.
:return: 作成したModuleの情報
:rtype: Response
"""
try:
with MetaDataSession.begin():
module = Module.create()
module.update_from_json(request.json, with_boxes=True)
MetaDataSession.add(module)
except KeyError:
raise
return respond_failure('key error', _status=400)
return respond_success(module=module.to_json(with_boxes=True, with_schema=True))
def api_user_renew_token(user_uuid):
"""UserのTokenを再生する
admin ... 全員
user ... 自分だけ
"""
user = User.query.get(user_uuid)
if user is None:
return respond_failure('User not found.', _status=404)
if not request.oauth.user.is_admin():
if user.uuid != request.oauth.user.uuid:
return respond_failure('Permission denied.')
with MetaDataSession.begin():
user.renew_token()
return respond_success(user=user.to_json(True))
def dummy_mbox():
print(" setup before function")
with mock_circlecore_context():
mbox_id = uuid.uuid4()
with MetaDataSession.begin():
schema = Schema.create(display_name='Schema',
properties='x:int,y:float')
module = Module.create(display_name='Module')
mbox = MessageBox(uuid=mbox_id,
schema_uuid=schema.uuid,
module_uuid=module.uuid)
MetaDataSession.add(schema)
MetaDataSession.add(module)
MetaDataSession.add(mbox)
yield (schema, module, mbox)
def test_websocket_auth_failed(self):
"""Websocketにも認証がいる"""
# make dummy environ
with MetaDataSession.begin():
schema = Schema.create(display_name='Schema', properties='x:int,y:float')
module = Module.create(display_name='Module')
mbox = MessageBox(
uuid='4ffab839-cf56-478a-8614-6003a5980855', schema_uuid=schema.uuid, module_uuid=module.uuid
)
MetaDataSession.add(schema)
MetaDataSession.add(module)
MetaDataSession.add(mbox)
with self.assertRaises(httpclient.HTTPClientError):
dummy_module = yield websocket_connect(self.get_url('/modules/{}/{}'.format(module.uuid, mbox.uuid)))
dummy_module.write_message(json.dumps({'x': 1, 'y': 2}))
yield sleep(1)
self.datareceiver.receive_new_message.assert_not_called()
def test_cc_info(self, _input, expected, mock_circlecore):
other_cc_info = CcInfo(**_input)
with MetaDataSession.begin():
MetaDataSession.add(other_cc_info)
own_cc_info = CcInfo.query.filter_by(myself=True).all()
assert len(own_cc_info) == 1
other_cc_info = CcInfo.query.get(_input['uuid'])
assert isinstance(other_cc_info, CcInfo)
assert other_cc_info.display_name == expected['display_name']
assert str(other_cc_info.uuid).lower() == expected['uuid'].lower()
assert other_cc_info.myself == expected['myself']
assert other_cc_info.work == expected['work']
jsonobj = other_cc_info.to_json()
assert str(other_cc_info.uuid) == jsonobj['uuid']
assert other_cc_info.display_name == jsonobj['displayName']
assert other_cc_info.work == jsonobj['work']
assert other_cc_info.myself == jsonobj['myself']
def test_rest_not_found(self):
"""登録されていないModuleからのPOSTは404"""
with MetaDataSession.begin():
user = User.create(account='tester', password='******')
user.renew_token()
MetaDataSession.add(user)
response = self.fetch(
self.get_url('/modules/4ffab839-cf56-478a-8614-6003a5980855/00000000-0000-0000-0000-000000000000'),
method='POST',
body=json.dumps({
'x': 1,
'y': 2
}),
headers={
'Content-Type': 'application/json',
'Authorization': 'Bearer {token}'.format(token=user.encoded_token),
}
)
self.assertEqual(response.code, 404)
def _delete_schema(schema_uuid):
"""Schemaを削除する.
:param str schema_uuid: 削除するSchemaのUUID
:return: Schemaの情報
:rtype: Response
"""
schema = Schema.query.get(schema_uuid)
if not schema:
return respond_failure('not found', _status=404)
if len(schema.message_boxes) > 0:
reason = 'message box {uuids} {verb} attached'.format(
uuids=', '.join([str(box.uuid) for box in schema.message_boxes]),
verb='is' if len(schema.message_boxes) == 1 else 'are')
return respond_failure(reason, _status=400)
with MetaDataSession.begin():
MetaDataSession.delete(schema)
return respond_success(schema={'uuid': schema_uuid})
def test_schema(self, _input, expected, mock_circlecore):
schema = Schema.create()
schema.update_from_json(_input)
module = Module.create(display_name='Module')
box = MessageBox(uuid=generate_uuid(model=MessageBox),
schema_uuid=schema.uuid,
module_uuid=module.uuid,
display_name='Box')
with MetaDataSession.begin():
MetaDataSession.add(schema)
MetaDataSession.add(module)
MetaDataSession.add(box)
schema = Schema.query.get(schema.uuid)
assert isinstance(schema, Schema)
assert schema.display_name == expected['displayName']
assert schema.memo == expected['memo']
assert isinstance(schema.properties, SchemaProperties)
assert len(schema.properties) == len(expected['properties'])
for prop, exp_prop in zip(schema.properties, expected['properties']):
assert isinstance(prop, SchemaProperty)
assert prop.name == exp_prop[0]
assert prop.type == exp_prop[1]
assert isinstance(hash(schema), int)
jsonobj = schema.to_json(with_modules=True)
assert str(schema.uuid) == jsonobj['uuid']
assert str(schema.cc_uuid) == jsonobj['ccUuid']
assert schema.display_name == jsonobj['displayName']
for prop, json_prop in zip(schema.properties, jsonobj['properties']):
assert prop.name == json_prop['name']
assert prop.type == json_prop['type']
assert schema.memo == jsonobj['memo']
assert len(jsonobj['modules']) == 1
assert module.display_name == jsonobj['modules'][0]['displayName']
def invitation_endpoint(link_uuid: 'uuid.UUID'):
"""User招待リンク.
Args:
link_uuid: User招待リンクのUUID
"""
invitation = Invitation.query.get(link_uuid)
if not invitation or not invitation.can_invite():
raise abort(404)
error = None
user = None
is_completed = False
if request.method == 'POST':
form = request.form
try:
with MetaDataSession.begin():
user = User.create(
account=form['account'],
work=form['work'],
telephone=form['telephone'],
mail_address=form['mailAddress'],
)
user.set_password(form['password'])
invitation.inc_invites()
MetaDataSession.add(user)
MetaDataSession.add(invitation)
except sqlalchemy.exc.IntegrityError:
error = 'このアカウントはすでに使われています。'
except ValueError as exc:
error = str(exc)
else:
is_completed = True
return render_template(
'invitation.html', error=error, user=user.to_json() if user else None, is_completed=is_completed
)
def mock_circlecore_context():
with tempfile.TemporaryDirectory('cc_') as tmp_dir:
assert os.path.exists(tmp_dir)
assert not os.path.exists(os.path.join(tmp_dir, 'metadata.sqlite3'))
metadata_db_engine = sqlalchemy.create_engine(
'sqlite:///' + os.path.join(tmp_dir, 'metadata.sqlite3'),
poolclass=NullPool)
MetaDataSession.configure(bind=metadata_db_engine)
# empty
MetaDataBase.metadata.create_all(metadata_db_engine)
# make my CcInfo
with MetaDataSession.begin():
my_cc_info = CcInfo(display_name='My CircleCore',
myself=True,
work='')
my_cc_info.uuid = generate_uuid(model=CcInfo)
MetaDataSession.add(my_cc_info)
yield metadata_db_engine, tmp_dir
MetaDataBase.metadata.drop_all(metadata_db_engine)
MetaDataSession.remove()
def test_websocket_pass_to_nanomsg(self):
"""WebSocketで受け取ったModuleからのMessageに適切なtimestamp/countを付与してnanomsgに流せているかどうか."""
# make dummy environ
with MetaDataSession.begin():
user = User.create(account='tester', password='******')
user.renew_token()
schema = Schema.create(display_name='Schema', properties='x:int,y:float')
module = Module.create(display_name='Module')
mbox = MessageBox(
uuid='4ffab839-cf56-478a-8614-6003a5980855', schema_uuid=schema.uuid, module_uuid=module.uuid
)
MetaDataSession.add(user)
MetaDataSession.add(schema)
MetaDataSession.add(module)
MetaDataSession.add(mbox)
dummy_module = yield websocket_connect(
httpclient.HTTPRequest(
self.get_url('/modules/{}/{}'.format(module.uuid, mbox.uuid)),
headers={
'Authorization': 'Bearer {token}'.format(token=user.encoded_token),
}
)
)
dummy_module.write_message(json.dumps({'x': 1, 'y': 2}))
# 素直にrecvするとIOLoopがブロックされてModuleHandlerが何も返せなくなるのでModuleHandlerをまず動かす
yield sleep(1)
self.datareceiver.receive_new_message.assert_called_once_with(
'4ffab839-cf56-478a-8614-6003a5980855', {
'x': 1,
'y': 2
}
)
async def test_post_date(mysql, mock_circlecore):
mbox_id = uuid.uuid4()
with MetaDataSession.begin():
schema = Schema.create(display_name='Schema',
properties='date:date,id1:float')
module = Module.create(display_name='Module')
mbox = MessageBox(uuid=mbox_id,
schema_uuid=schema.uuid,
module_uuid=module.uuid)
MetaDataSession.add(schema)
MetaDataSession.add(module)
MetaDataSession.add(mbox)
envdir = mock_circlecore[1]
database = Database(mysql.url, time_db_dir=envdir, log_dir=envdir)
journal_writer = database.make_writer()
run_loop = asyncio.ensure_future(journal_writer.run())
# post
message = ModuleMessage(mbox.uuid, 123456.789, 0, {
'date': '2017-09-01T00:00:00Z',
'id1': 3.14
})
await journal_writer.store(mbox, message)
await journal_writer.flush()
# close database
await journal_writer.close()
await run_loop
with database._engine.begin() as connection:
table = database.find_table_for_message_box(mbox)
rows = connection.execute(select([table])).fetchall()
assert len(rows) == 1
assert rows[0][2] == datetime.date(2017, 9, 1)
assert rows[0][3] == 3.14
async def test():
counterloop = asyncio.ensure_future(countup())
setup_db()
mbox_id = uuid.UUID('49EB92A3-AAE8-43A1-BC43-B7933DE96C6A')
with MetaDataSession.begin():
schema = Schema.create(display_name='Schema',
properties='x:int,y:float')
module = Module.create(display_name='Module')
mbox = MessageBox(uuid=mbox_id,
schema_uuid=schema.uuid,
module_uuid=module.uuid)
MetaDataSession.merge(schema)
MetaDataSession.merge(module)
MetaDataSession.merge(mbox)
database = Database('mysql+pymysql://root:hogehoge@localhost/crcr_test',
'./tmp/',
'./tmp/',
connect_args=dict(write_timeout=3, read_timeout=3))
database.drop_message_box(mbox)
queued_writer = QueuedDBWriter(database, './tmp/')
class Delegate(QueuedDBWriterDelegate):
async def on_reconnect(self) -> None:
print('on_reconnect')
writer.touch()
queued_writer = QueuedDBWriter(database, './tmp/', delegate=Delegate())
writer = JournalDBWriter(queued_writer, './tmp/')
print('connect')
# input('> ')
database.connect()
print('store')
# input('> ')
message = ModuleMessage(mbox.uuid, 123456.789, 0, {'x': 1, 'y': 2})
assert (await writer.store(mbox, message)) is True
print('store2 , please shutdown mysql')
await async_input('> ')
message = ModuleMessage(mbox.uuid, 123457.0, 1, {'x': 3, 'y': 4})
assert (await writer.store(mbox, message)) is True
assert (await writer.store(
mbox, ModuleMessage(mbox.uuid, 123458.0, 2, {
'x': 4,
'y': 4
}))) is True
assert (await writer.store(
mbox, ModuleMessage(mbox.uuid, 123459.0, 3, {
'x': 5,
'y': 4
}))) is True
assert (await writer.store(
mbox, ModuleMessage(mbox.uuid, 123460.0, 4, {
'x': 6,
'y': 4
}))) is True
assert (await writer.store(
mbox, ModuleMessage(mbox.uuid, 123461.0, 5, {
'x': 7,
'y': 4
}))) is True
print('store3 , please wakeup mysql')
await async_input('> ')
logger.info('DONE!')
global done
done = True
await counterloop
async def test_replicate_blob(mysql, mock_circlecore):
"""BLOBをReplicateできるかテスト
このテストは見ている範囲が広すぎるにしてはみたいところがみれてないのでは"""
mock_cc = MagicMock()
envdir = mock_circlecore[1]
database = Database(mysql.url, time_db_dir=envdir, log_dir=envdir)
with MetaDataSession.begin():
master_uuid = generate_uuid(model=CcInfo)
# ccinfo = CcInfo.query.filter_by(myself=True).one()
master_cc_info = CcInfo(uuid=master_uuid,
display_name='test master',
myself=False,
work='')
replication_master = ReplicationMaster(endpoint_url='',
info=master_cc_info)
module = Module(uuid=generate_uuid(model=Module),
cc_info=master_cc_info)
schema = Schema.create(cc_uuid=master_cc_info.uuid,
display_name='Schema',
properties='x:int,y:float,data:blob')
box = MessageBox(uuid=generate_uuid(model=MessageBox),
schema=schema,
module=module,
display_name='Box')
MetaDataSession.add(master_cc_info)
MetaDataSession.add(replication_master)
MetaDataSession.add(module)
MetaDataSession.add(schema)
MetaDataSession.add(box)
async def dummy_store(*args):
return DEFAULT
slave_dirver = SlaveDriverWorker(mock_cc, '', False)
slave_dirver.initialize()
assert len(slave_dirver.replicators) == 1
assert replication_master.id in slave_dirver.replicators
replicator = slave_dirver.replicators[replication_master.id]
replicator.state = ReplicationState.SYNCING
replicator.target_boxes = {box.uuid: box}
replicator.ws = MagicMock()
writer = QueuedDBWriter(database, envdir)
replicator.writer = writer
datahash = (
'bf3408132f944568dd02827441f8c69b1f3e5a36bd7693a7aeeffd490037b56d'
'9ad9406892ecd70cb9c0d31a746e7e790e731ae491dc0176a49369a2a4044581')
replicator.ws.read_message = Mock(side_effect=make_dummy_read_message('''\
{{
"command": "new_message",
"message": {{
"boxId": "{box_id}",
"timestamp": "1546006083.213117",
"counter": 0,
"payload": {{
"x": 32768,
"y": 3.14,
"data": {{"$data": "{datahash}", "$source": "{source}", "$type": "text/plain"}}
}}
}}
}}
'''.format(box_id=str(box.uuid), datahash=datahash,
source=master_cc_info.uuid)))
await replicator.wait_command((MasterCommand.NEW_MESSAGE, ))
replicator.ws.read_message.assert_called_once()
with database._engine.begin() as connection:
table = database.find_table_for_message_box(box)
rows = connection.execute(select([table.c.data])).fetchall()
assert len(rows) == 1
expected = '{{"$data": "{datahash}", "$source": "{source}", "$type": "text/plain"}}' \
.format(
datahash=datahash,
source=str(master_cc_info.uuid)
)
assert rows[0][0] == expected
def test_rest_with_data(self):
"""登録されているModuleからのPOSTは404"""
# make dummy environ
with MetaDataSession.begin():
user = User.create(account='tester', password='******')
user.renew_token()
schema = Schema.create(display_name='Schema', properties='x:int,y:float,data:blob')
module = Module.create(display_name='Module')
mbox = MessageBox(
uuid='4ffab839-cf56-478a-8614-6003a5980857', schema_uuid=schema.uuid, module_uuid=module.uuid
)
MetaDataSession.add(user)
MetaDataSession.add(schema)
MetaDataSession.add(module)
MetaDataSession.add(mbox)
async def _async_side_effect():
print('_async_side_effect')
return True
# data encodingはOK
response = self.fetch(
self.get_url('/modules/{}/{}'.format(module.uuid, mbox.uuid)),
method='POST',
body=json.dumps({
'x': 10.,
'y': 20.5,
'data': encode_to_data(*load_file('test.jpg'))
}),
headers={
'Content-Type': 'application/json',
'Authorization': 'Bearer {token}'.format(token=user.encoded_token),
}
)
self.assertEqual(response.code, 200)
self.datareceiver.receive_new_message.assert_called_once()
args, kwargs = self.datareceiver.receive_new_message.call_args
assert args[0] == str(mbox.uuid)
assert args[1]['x'] == 10.
assert args[1]['y'] == 20.5
assert args[1]['data'].startswith('data:image/jpeg;')
self.reset_mock()
# そうじゃないのはNG
response = self.fetch(
self.get_url('/modules/{}/{}'.format(module.uuid, mbox.uuid)),
method='POST',
body=json.dumps({
'x': 10.,
'y': 20.5,
'data': 'hogehoge'
}),
headers={
'Content-Type': 'application/json',
'Authorization': 'Bearer {token}'.format(token=user.encoded_token),
}
)
self.assertEqual(response.code, 400)
self.datareceiver.receive_new_message.assert_not_called()
self.reset_mock()
# multipartもOK
body, headers = make_multipart_request(
'application/json', json.dumps({
'x': 10.,
'y': 20.5,
'data': 'file:///test.jpg'
}), 'test.jpg'
)
headers['Authorization'] = 'Bearer {token}'.format(token=user.encoded_token)
response = self.fetch(
self.get_url('/modules/{}/{}'.format(module.uuid, mbox.uuid)),
method='POST',
headers=headers,
body=body,
)
self.assertEqual(response.code, 200)
args, kwargs = self.datareceiver.receive_new_message.call_args
assert args[0] == str(mbox.uuid)
assert args[1]['x'] == 10.
assert args[1]['y'] == 20.5
assert 'data' in args[1]