def make_own_cc_info(self, config_uuid: 'UUIDLike') -> CcInfo: """自身のCircleCore Infoを作成する. Args: config_uuid (str): uuid, autoの場合は生成する Returns: circle_core.models.CcInfo: 自身のCircleCore Info """ with MetaDataSession.begin(): try: my_cc_info = CcInfo.query.filter_by(myself=True).one() except NoResultFound: logger.info('My CCInfo not found. Create new one') my_cc_info = CcInfo(display_name='My CircleCore', myself=True, work='') if config_uuid == 'auto': my_cc_info.uuid = generate_uuid(model=CcInfo) else: my_cc_info.uuid = config_uuid MetaDataSession.add(my_cc_info) return my_cc_info
async def on_slave_hello(self, json_msg): """hello コマンドが送られてきた""" assert self.state == ReplicationState.HANDSHAKING self.state = ReplicationState.MIGRATING # slave's cc info slave_info = json_msg['ccInfo'] slave_uuid = uuid.UUID(slave_info['uuid']) with MetaDataSession.begin(): # store slave's information if slave_uuid not in [ slave.slave_uuid for slave in self.replication_link.slaves ]: self.replication_link.slaves.append( ReplicationSlave(link_uuid=self.replication_link.uuid, slave_uuid=slave_uuid)) cc_info = CcInfo.query.get(slave_uuid) if not cc_info: cc_info = CcInfo(uuid=slave_uuid, myself=False) cc_info.update_from_json(slave_info) MetaDataSession.add(cc_info) await self.send_migrate()
def test_user(self, _input, expected, mock_circlecore): user = User.create(**_input) with MetaDataSession.begin(): MetaDataSession.add(user) user = User.query.get(user.uuid) assert isinstance(user, User) assert user.account == expected['account'] assert user.is_password_matched(expected['password']) is True assert user.is_password_matched(expected['password_no_match']) is False assert len(user.permissions) == len(expected['permissions']) for permission, exp_permission in zip(user.permissions, expected['permissions']): assert isinstance(permission, str) assert permission == exp_permission assert user.work == expected['work'] assert user.mail_address == expected['mail_address'] assert user.telephone == expected['telephone'] assert user.is_admin() == ('admin' in expected['permissions']) jsonobj = user.to_json(full=True) assert str(user.uuid) == jsonobj['uuid'] assert user.account == jsonobj['account'] assert user.work == jsonobj['work'] assert user.mail_address == jsonobj['mailAddress'] assert user.telephone == jsonobj['telephone'] assert len(user.permissions) == len(jsonobj['permissions']) for permission, exp_permission in zip(user.permissions, jsonobj['permissions']): assert permission == exp_permission
def test_check_match(self, _input, data, expected, mock_circlecore): jsonobj = dict(displayName='Schema', memo='memo', properties=_input) schema = Schema.create() schema.update_from_json(jsonobj) with MetaDataSession.begin(): MetaDataSession.add(schema) schema = Schema.query.get(schema.uuid) ok, msg = schema.check_match(data) assert ok is expected
def _put_core(cc_info): """CircleCoreを更新する. :param CcInfo cc_info: 更新するCircleCore :return: CircleCoreの情報 :rtype: Response """ with MetaDataSession.begin(): cc_info.update_from_json(request.json) MetaDataSession.add(cc_info) return respond_success(ccInfo=cc_info.to_json())
def check_login(): """ログイン確認.""" user = get_user_from_request() if not user: raise abort(403) # update user's last access from circle_core.models import MetaDataSession with MetaDataSession.begin(): user.last_access_at = datetime.utcnow() MetaDataSession.add(user)
def _post_schemas(): """Schemaを作成する. :return: 作成したSchemaの情報 :rtype: Response """ with MetaDataSession.begin(): schema = Schema.create() schema.update_from_json(request.json) MetaDataSession.add(schema) return respond_success(schema=schema.to_json(with_modules=True))
def test_replication_link(self, _input, expected, mock_circlecore): schema = Schema.create(display_name='Schema', properties='x:int,y:float') module = Module.create(display_name='Module') box = MessageBox(uuid=generate_uuid(model=MessageBox), schema_uuid=schema.uuid, module_uuid=module.uuid, display_name='Box') with MetaDataSession.begin(): MetaDataSession.add(schema) MetaDataSession.add(module) MetaDataSession.add(box) replication_link = ReplicationLink.create(message_box_uuids=[box.uuid], **_input) with MetaDataSession.begin(): MetaDataSession.add(replication_link) replication_link = ReplicationLink.query.get(replication_link.uuid) assert isinstance(replication_link, ReplicationLink) assert replication_link.display_name == expected['display_name'] assert replication_link.memo == expected['memo']
def test_moge(self): """API認証""" with MetaDataSession.begin(): user = User.create(account='tester', password='******') user.renew_token() schema = Schema.create(display_name='Schema', properties='x:int,y:float,data:blob') module = Module.create(display_name='Module') mbox = MessageBox(uuid='4ffab839-cf56-478a-8614-6003a5980857', schema_uuid=schema.uuid, module_uuid=module.uuid) MetaDataSession.add(user) MetaDataSession.add(schema) MetaDataSession.add(module) MetaDataSession.add(mbox) data_api_endpoint = self.get_url('/api/modules/{}/{}/data'.format( module.uuid, mbox.uuid)) # 認証がいるよ response = self.fetch(data_api_endpoint) assert response.code == 403 # 認証がいるよ response = self.fetch( data_api_endpoint, headers={'Authorization': 'Bearer {}'.format(user.encoded_token)}) assert response.code == 200
def _post_invitation(): """Invitationを作成する. :return: 作成したInvitationの情報 :rtype: Response """ # maxInvites項目しか許可しない with MetaDataSession.begin(): obj = Invitation(uuid=generate_uuid(model=Invitation), max_invites=request.json['maxInvites'], created_at=datetime.datetime.utcnow()) MetaDataSession.add(obj) return respond_success(invitation=obj.to_json())
def test_rest(self): """登録されているModuleからのPOSTは404""" # make dummy environ with MetaDataSession.begin(): user = User.create(account='tester', password='******') user.renew_token() schema = Schema.create(display_name='Schema', properties='x:int,y:float') module = Module.create(display_name='Module') mbox = MessageBox( uuid='4ffab839-cf56-478a-8614-6003a5980856', schema_uuid=schema.uuid, module_uuid=module.uuid ) MetaDataSession.add(user) MetaDataSession.add(schema) MetaDataSession.add(module) MetaDataSession.add(mbox) response = self.fetch( self.get_url('/modules/{}/{}'.format(module.uuid, mbox.uuid)), method='POST', body=json.dumps({ 'x': 1, 'y': 2.5 }), headers={ 'Content-Type': 'application/json', 'Authorization': 'Bearer {token}'.format(token=user.encoded_token), } ) self.assertEqual(response.code, 200) self.datareceiver.receive_new_message.assert_called_once_with(str(mbox.uuid), {'x': 1, 'y': 2.5})
async def on_master_migrate(self, message): assert self.state == ReplicationState.MIGRATING with MetaDataSession.begin(): # save master info data = message['masterInfo'] try: obj = CcInfo.query.filter_by(uuid=data['uuid']).one() except NoResultFound: obj = CcInfo(uuid=data['uuid'], myself=False) obj.update_from_json(data) obj.replication_master_id = self.master.id MetaDataSession.add(obj) master_info = obj self.master.master_uuid = master_info.uuid # migrate schemas for schema_uuid, data in message['schemas'].items(): try: obj = Schema.query.filter_by(uuid=data['uuid']).one() except NoResultFound: obj = Schema(uuid=data['uuid']) obj.update_from_json(data) obj.cc_uuid = master_info.uuid MetaDataSession.add(obj) # migrate modules for module_uuid, data in message['modules'].items(): try: obj = Module.query.filter_by(uuid=data['uuid']).one() except NoResultFound: obj = Module(uuid=data['uuid']) obj.update_from_json(data) obj.cc_uuid = master_info.uuid obj.replication_master_id = self.master.id MetaDataSession.add(obj) # migrate boxes self.target_boxes = {} for box_uuid, data in message['messageBoxes'].items(): try: obj = MessageBox.query.filter_by(uuid=data['uuid']).one() if obj.schema_uuid != uuid.UUID(data['schemaUuid']): raise DataConfilictedError('schemaUuid not match') if obj.module_uuid != uuid.UUID(data['moduleUuid']): raise DataConfilictedError('moduleUuid not match') except NoResultFound: obj = MessageBox( uuid=data['uuid'], schema_uuid=data['schemaUuid'], module_uuid=data['moduleUuid'], ) obj.update_from_json(data) MetaDataSession.add(obj) self.target_boxes[obj.uuid] = obj
def _post_replicas(): """ReplicationLinkを作成する. :return: 作成したReplicationLinkの情報 :rtype: Response """ data = request.json with MetaDataSession.begin(): replication_link = ReplicationLink.create( data['displayName'], data['memo'], data['messageBoxes'], ) MetaDataSession.add(replication_link) return respond_success(replicationLink=replication_link.to_json())
def test_invitation(self, _input, expected, mock_circlecore): invitation = Invitation(uuid=generate_uuid(model=Invitation), **_input) with MetaDataSession.begin(): MetaDataSession.add(invitation) invitation = Invitation.query.get(invitation.uuid) assert isinstance(invitation, Invitation) assert invitation.max_invites == expected['max_invites'] assert invitation.current_invites == expected['current_invites'] for i in range(_input['max_invites']): assert invitation.can_invite() is True invitation.inc_invites() else: assert invitation.can_invite() is False
def _post_replication_masters(): """ReplicationMasterを作成する. :return: 作成したReplicationMasterの情報 :rtype: Response """ data = request.json try: with MetaDataSession.begin(): replication_master = ReplicationMaster( endpoint_url=data['endpointUrl'], ) MetaDataSession.add(replication_master) except sqlalchemy.exc.IntegrityError: return respond_failure('このURLは既に登録されています') return respond_success(replicationMaster=replication_master.to_json())
def _put_module(module): """Moduleを更新する. :param Module module: 更新するModule :return: Moduleの情報 :rtype: Response """ try: with MetaDataSession.begin(): module.update_from_json(request.json, with_boxes=True) MetaDataSession.add(module) except KeyError: return respond_failure('key error') except ValueError as e: return respond_failure(str(e)) return respond_success(module=module.to_json(with_boxes=True, with_schema=True))
def _post_modules() -> 'Response': """Moduleを作成する. :return: 作成したModuleの情報 :rtype: Response """ try: with MetaDataSession.begin(): module = Module.create() module.update_from_json(request.json, with_boxes=True) MetaDataSession.add(module) except KeyError: raise return respond_failure('key error', _status=400) return respond_success(module=module.to_json(with_boxes=True, with_schema=True))
def test_cc_info(self, _input, expected, mock_circlecore): other_cc_info = CcInfo(**_input) with MetaDataSession.begin(): MetaDataSession.add(other_cc_info) own_cc_info = CcInfo.query.filter_by(myself=True).all() assert len(own_cc_info) == 1 other_cc_info = CcInfo.query.get(_input['uuid']) assert isinstance(other_cc_info, CcInfo) assert other_cc_info.display_name == expected['display_name'] assert str(other_cc_info.uuid).lower() == expected['uuid'].lower() assert other_cc_info.myself == expected['myself'] assert other_cc_info.work == expected['work'] jsonobj = other_cc_info.to_json() assert str(other_cc_info.uuid) == jsonobj['uuid'] assert other_cc_info.display_name == jsonobj['displayName'] assert other_cc_info.work == jsonobj['work'] assert other_cc_info.myself == jsonobj['myself']
def test_rest_not_found(self): """登録されていないModuleからのPOSTは404""" with MetaDataSession.begin(): user = User.create(account='tester', password='******') user.renew_token() MetaDataSession.add(user) response = self.fetch( self.get_url('/modules/4ffab839-cf56-478a-8614-6003a5980855/00000000-0000-0000-0000-000000000000'), method='POST', body=json.dumps({ 'x': 1, 'y': 2 }), headers={ 'Content-Type': 'application/json', 'Authorization': 'Bearer {token}'.format(token=user.encoded_token), } ) self.assertEqual(response.code, 404)
def test_message_box(self, _input, expected, mock_circlecore): schema = Schema.create(display_name='Schema', properties='x:int,y:float') module = Module.create(display_name='Module') box = MessageBox(uuid=generate_uuid(model=MessageBox), schema_uuid=schema.uuid, module_uuid=module.uuid, **_input) with MetaDataSession.begin(): MetaDataSession.add(schema) MetaDataSession.add(module) MetaDataSession.add(box) box = MessageBox.query.get(box.uuid) assert isinstance(box, MessageBox) assert box.display_name == expected['display_name'] assert box.memo == expected['memo'] assert isinstance(hash(box), int) assert isinstance(box.cc_uuid, uuid.UUID) jsonobj = box.to_json(with_schema=True, with_module=True) assert str(box.uuid) == jsonobj['uuid'] assert box.display_name == jsonobj['displayName'] assert box.memo == jsonobj['memo'] assert str(box.module_uuid) == jsonobj['moduleUuid'] assert str(box.schema_uuid) == jsonobj['schemaUuid'] assert schema.display_name == jsonobj['schema']['displayName'] assert module.display_name == jsonobj['module']['displayName']
async def test_post_date(mysql, mock_circlecore): mbox_id = uuid.uuid4() with MetaDataSession.begin(): schema = Schema.create(display_name='Schema', properties='date:datetime,id1:float') module = Module.create(display_name='Module') mbox = MessageBox(uuid=mbox_id, schema_uuid=schema.uuid, module_uuid=module.uuid) MetaDataSession.add(schema) MetaDataSession.add(module) MetaDataSession.add(mbox) envdir = mock_circlecore[1] database = Database(mysql.url, time_db_dir=envdir, log_dir=envdir) journal_writer = database.make_writer() queued_writer = journal_writer.child_writer run_loop = asyncio.ensure_future(journal_writer.run()) # post message = ModuleMessage(mbox.uuid, 123456.789, 0, { 'date': '2017-09-01T24:00:00Z', 'id1': 3.14 }) await queued_writer.store(mbox, message) await queued_writer.flush() # close database await journal_writer.close() await run_loop with database._engine.begin() as connection: table = database.find_table_for_message_box(mbox) rows = connection.execute(select([table])).fetchall() assert len(rows) == 0
def test_update_from_json(self, old, new, expected, mock_circlecore): user = User.create(**old) with MetaDataSession.begin(): MetaDataSession.add(user) user = User.query.get(user.uuid) assert user.account != expected['account'] assert user.is_password_matched(expected['password']) is False assert user.work != expected['work'] assert user.mail_address != expected['mail_address'] assert user.telephone != expected['telephone'] user.update_from_json(new) with MetaDataSession.begin(): MetaDataSession.add(user) user = User.query.get(user.uuid) assert user.account == expected['account'] assert user.is_password_matched(expected['password']) is True assert user.work == expected['work'] assert user.mail_address == expected['mail_address'] assert user.telephone == expected['telephone']
def invitation_endpoint(link_uuid: 'uuid.UUID'): """User招待リンク. Args: link_uuid: User招待リンクのUUID """ invitation = Invitation.query.get(link_uuid) if not invitation or not invitation.can_invite(): raise abort(404) error = None user = None is_completed = False if request.method == 'POST': form = request.form try: with MetaDataSession.begin(): user = User.create( account=form['account'], work=form['work'], telephone=form['telephone'], mail_address=form['mailAddress'], ) user.set_password(form['password']) invitation.inc_invites() MetaDataSession.add(user) MetaDataSession.add(invitation) except sqlalchemy.exc.IntegrityError: error = 'このアカウントはすでに使われています。' except ValueError as exc: error = str(exc) else: is_completed = True return render_template( 'invitation.html', error=error, user=user.to_json() if user else None, is_completed=is_completed )
def mock_circlecore_context(): with tempfile.TemporaryDirectory('cc_') as tmp_dir: assert os.path.exists(tmp_dir) assert not os.path.exists(os.path.join(tmp_dir, 'metadata.sqlite3')) metadata_db_engine = sqlalchemy.create_engine( 'sqlite:///' + os.path.join(tmp_dir, 'metadata.sqlite3'), poolclass=NullPool) MetaDataSession.configure(bind=metadata_db_engine) # empty MetaDataBase.metadata.create_all(metadata_db_engine) # make my CcInfo with MetaDataSession.begin(): my_cc_info = CcInfo(display_name='My CircleCore', myself=True, work='') my_cc_info.uuid = generate_uuid(model=CcInfo) MetaDataSession.add(my_cc_info) yield metadata_db_engine, tmp_dir MetaDataBase.metadata.drop_all(metadata_db_engine) MetaDataSession.remove()
def test_user_token(self, mock_circlecore): user = User.create(account='testuser', password='******') with MetaDataSession.begin(): MetaDataSession.add(user) user = User.query.get(user.uuid) # 初期状態ではuser tokenはNone assert user.token is None assert user.to_json(True)['token'] is None # tokenを生成する with MetaDataSession.begin(): user.renew_token() MetaDataSession.add(user) user = User.query.get(user.uuid) assert user.token is not None assert len(user.token) == 128 assert isinstance(user.to_json(True)['token'], str) assert base64.b64decode(user.to_json(True)['token']) == user.token
async def test_store_blob(mysql, mock_circlecore): envdir = mock_circlecore[1] with MetaDataSession.begin(): ccinfo = CcInfo.query.filter_by(myself=True).one() module = Module.create() schema = Schema.create(display_name='Schema', properties='x:int,y:float,data:blob') box = MessageBox(uuid=generate_uuid(model=MessageBox), schema=schema, module=module, display_name='Box') MetaDataSession.add(schema) MetaDataSession.add(module) MetaDataSession.add(box) database = Database(mysql.url, time_db_dir=envdir, log_dir=envdir) connection = database._engine.connect() with database._engine.begin() as connection: database.store_message( box, ModuleMessage( box.uuid, '1545895047.000', 0, { 'x': 0, 'y': 3.14, 'data': BlobMetadata(ccinfo.uuid, 'text/plain', 'deadbeafdeadbeafdeadbeafdeadbeaf') }, ), connection=connection) with database._engine.begin() as connection: table = database.find_table_for_message_box(box) rows = connection.execute(select([table.c.data])).fetchall() print(rows) assert len(rows) == 1 expected = '{{"$data": "deadbeafdeadbeafdeadbeafdeadbeaf", "$source": "{source}", "$type": "text/plain"}}' \ .format( source=str(ccinfo.uuid) ) assert rows[0][0] == expected
def test_module(self, _input, expected, mock_circlecore): module = Module.create() schema = Schema.create(display_name='Schema', properties='x:int,y:float') box = MessageBox( uuid=generate_uuid(model=MessageBox), schema_uuid=schema.uuid, module_uuid=module.uuid, display_name='Box' ) _input['messageBoxes'] = [dict(schema=schema.uuid, displayName='Box2', memo='memo')] module.update_from_json(_input, with_boxes=True) with MetaDataSession.begin(): MetaDataSession.add(schema) MetaDataSession.add(module) MetaDataSession.add(box) module = Module.query.get(module.uuid) assert isinstance(module, Module) assert module.display_name == expected['displayName'] assert isinstance(module.tags, list) assert len(module.tags) == len(expected['tags']) for tag, exp_tag in zip(module.tags, expected['tags']): assert isinstance(tag, str) assert tag == exp_tag assert isinstance(module.attributes, ModuleAttributes) assert len(module.attributes) == len(expected['attributes']) for attr, exp_attr in zip(module.attributes, expected['attributes']): assert isinstance(attr, ModuleAttribute) assert attr.name == exp_attr[0] assert attr.value == exp_attr[1] assert module.memo == expected['memo'] assert isinstance(hash(module), int) jsonobj = module.to_json(with_boxes=True, with_schema=True, with_cc_info=True) assert str(module.uuid) == jsonobj['uuid'] assert str(module.cc_uuid) == jsonobj['ccUuid'] assert module.display_name == jsonobj['displayName'] for tag, json_tag in zip(module.tags, jsonobj['tags']): assert tag == json_tag for attr, json_attr in zip(module.attributes, jsonobj['attributes']): assert attr.name == json_attr['name'] assert attr.value == json_attr['value'] assert module.memo == jsonobj['memo'] assert len(jsonobj['messageBoxes']) == 2 assert jsonobj['ccInfo'] assert jsonobj['isReplication'] is False
def test_update_from_json(self, _input, expected, mock_circlecore): schema = Schema.create(display_name='Schema', properties='x:int,y:float') module = Module.create(display_name='Module') box = MessageBox(uuid=generate_uuid(model=MessageBox), schema_uuid=schema.uuid, module_uuid=module.uuid, display_name='MessageBoxOldName') with MetaDataSession.begin(): MetaDataSession.add(schema) MetaDataSession.add(module) MetaDataSession.add(box) box = MessageBox.query.get(box.uuid) box.update_from_json(_input) with MetaDataSession.begin(): MetaDataSession.add(box) box = MessageBox.query.get(box.uuid) assert isinstance(box, MessageBox) assert box.display_name == expected['display_name'] assert box.memo == expected['memo']
def test_websocket_pass_to_nanomsg(self): """WebSocketで受け取ったModuleからのMessageに適切なtimestamp/countを付与してnanomsgに流せているかどうか.""" # make dummy environ with MetaDataSession.begin(): user = User.create(account='tester', password='******') user.renew_token() schema = Schema.create(display_name='Schema', properties='x:int,y:float') module = Module.create(display_name='Module') mbox = MessageBox( uuid='4ffab839-cf56-478a-8614-6003a5980855', schema_uuid=schema.uuid, module_uuid=module.uuid ) MetaDataSession.add(user) MetaDataSession.add(schema) MetaDataSession.add(module) MetaDataSession.add(mbox) dummy_module = yield websocket_connect( httpclient.HTTPRequest( self.get_url('/modules/{}/{}'.format(module.uuid, mbox.uuid)), headers={ 'Authorization': 'Bearer {token}'.format(token=user.encoded_token), } ) ) dummy_module.write_message(json.dumps({'x': 1, 'y': 2})) # 素直にrecvするとIOLoopがブロックされてModuleHandlerが何も返せなくなるのでModuleHandlerをまず動かす yield sleep(1) self.datareceiver.receive_new_message.assert_called_once_with( '4ffab839-cf56-478a-8614-6003a5980855', { 'x': 1, 'y': 2 } )
async def test_datareceiver_store_blob(mock_circlecore, mysql, monkeypatch): metadata_db_engine, tmp_dir = mock_circlecore writer_mock = MagicMock() make_writer_mock = Mock(name='make_writer', return_value=writer_mock) async def dummy_store(*args, **kwargs): return DEFAULT writer_mock.store.side_effect = dummy_store monkeypatch.setattr(Database, 'make_writer', make_writer_mock) # make test schema/mbox with MetaDataSession.begin(): schema = Schema.create(display_name='Schema', properties='x:int,y:float,data:blob') module = Module.create(display_name='Module') mbox = MessageBox(uuid=generate_uuid(model=MessageBox), schema_uuid=schema.uuid, module_uuid=module.uuid) MetaDataSession.add(schema) MetaDataSession.add(module) MetaDataSession.add(mbox) core_mock = MagicMock() worker = DataReceiverWorker( core_mock, 'worker_key', db_url=mysql.url, time_db_dir=tmp_dir, log_dir=tmp_dir, cycle_time=10, cycle_count=10, ) datahash = ( '2b7e36b16f8a849ef312f9ef5ff9b3f4281a8681d0657150899f1113a0eecfdb' 'b4491da763159055b55e122e85281415b11897d268e124f9ef2b40457a63a465' ) blobobj = StoredBlob(None, 'text/plain', datahash) await worker.receive_new_message(mbox.uuid, {'x': 1, 'y': 2.0, 'data': blobobj}) publish_mock = core_mock.hub.publish publish_mock.assert_called_once() message = publish_mock.call_args[0][1] assert message.payload['data'].content_type == 'text/plain' assert message.payload['data'].datahash == datahash