class ServiceEventConsumer(EventConsumer):
_connection = Repository(NETWORK_ID, NETWORKS=NETWORKS)
_service_repository = ServiceRepository(_connection)
def __init__(self, ws_provider, ipfs_url, ipfs_port):
self._blockchain_util = BlockChainUtil("WS_PROVIDER", ws_provider)
self._s3_util = S3Util(S3_BUCKET_ACCESS_KEY, S3_BUCKET_SECRET_KEY)
self._ipfs_util = IPFSUtil(ipfs_url, ipfs_port)
def on_event(self, event):
pass
def _fetch_tags(self, registry_contract, org_id_hex, service_id_hex):
tags_data = registry_contract.functions.getServiceRegistrationById(
org_id_hex, service_id_hex).call()
return tags_data
def _get_org_id_from_event(self, event):
event_data = event['data']
service_data = eval(event_data['json_str'])
org_id_bytes = service_data['orgId']
org_id = Web3.toText(org_id_bytes).rstrip("\x00")
return org_id
def _get_service_id_from_event(self, event):
event_data = event['data']
service_data = eval(event_data['json_str'])
service_id_bytes = service_data['serviceId']
service_id = Web3.toText(service_id_bytes).rstrip("\x00")
return service_id
def _get_metadata_uri_from_event(self, event):
event_data = event['data']
service_data = eval(event_data['json_str'])
metadata_uri = Web3.toText(
service_data['metadataURI'])[7:].rstrip("\u0000")
return metadata_uri
def _get_registry_contract(self):
net_id = NETWORK_ID
base_contract_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..', 'node_modules',
'singularitynet-platform-contracts'))
registry_contract = self._blockchain_util.get_contract_instance(
base_contract_path, "REGISTRY", net_id)
return registry_contract
def _get_service_details_from_blockchain(self, event):
logger.info(f"processing service event {event}")
registry_contract = self._get_registry_contract()
org_id = self._get_org_id_from_event(event)
service_id = self._get_service_id_from_event(event)
tags_data = self._fetch_tags(registry_contract=registry_contract,
org_id_hex=org_id.encode("utf-8"),
service_id_hex=service_id.encode("utf-8"))
return org_id, service_id, tags_data
def curate_service(self, org_id, service_id, curated):
service_repo = ServiceRepository(self.repo)
if str(curated).lower() == "true":
service_repo.curate_service(org_id, service_id, 1)
elif str(curated).lower() == "false":
service_repo.curate_service(org_id, service_id, 0)
else:
Exception("Invalid curation flag")
def test_on_service_created_event(self, mock_fetch_tags, nock_read_bytesio_from_ipfs, mock_ipfs_read, mock_s3_push):
event = {"data": {'row_id': 202, 'block_no': 6325625, 'event': 'ServiceCreated',
'json_str': "{'orgId': b'snet\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00', 'serviceId': b'gene-annotation-service\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00', 'metadataURI': b'ipfs://QmdGjaVYPMSGpC1qT3LDALSNCCu7JPf7j51H1GQirvQJYf\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00'}",
'processed': b'\x00',
'transactionHash': 'b"\\xa7P*\\xaf\\xfd\\xd5.E\\x8c\\x0bKAF\'\\x15\\x03\\xef\\xdaO\'\\x86/<\\xfb\\xc4\\xf0@\\xf0\\xc1P\\x8c\\xc7"',
'logIndex': '0', 'error_code': 1, 'error_msg': '',
'row_updated': datetime(2019, 10, 21, 9, 59, 37),
'row_created': datetime(2019, 10, 21, 9, 59, 37)}, "name": "ServiceCreated"}
connection = Repository(NETWORK_ID, NETWORKS=NETWORKS)
service_repository = ServiceRepository(connection)
service_repository.delete_service(org_id='snet', service_id='gene-annotation-service')
service_repository.delete_service_dependents(org_id='snet', service_id='gene-annotation-service')
nock_read_bytesio_from_ipfs.return_value = "some_value to_be_pushed_to_s3_whic_is_mocked"
mock_ipfs_read.return_value = {
"version": 1,
"display_name": "Annotation Service",
"encoding": "proto",
"service_type": "grpc",
"model_ipfs_hash": "QmXqonxB9EvNBe11J8oCYXMQAtPKAb2x8CyFLmQpkvVaLf",
"mpe_address": "0x8FB1dC8df86b388C7e00689d1eCb533A160B4D0C",
"groups": [
{
"group_name": "default_group",
"pricing": [
{
"price_model": "fixed_price",
"price_in_cogs": 1,
"default": True
}
],
"endpoints": [
"https://mozi.ai:8000"
],
"group_id": "m5FKWq4hW0foGW5qSbzGSjgZRuKs7A1ZwbIrJ9e96rc="
}
],
"assets": {
"hero_image": "QmVcE6fEDP764ibadXTjZHk251Lmt5xAxdc4P9mPA4kksk/hero_gene-annotation-2b.png"
},
"service_description": {
"url": "https://mozi-ai.github.io/annotation-service/",
"description": "Use this service to annotate a humane genome with uniform terms, Reactome pathway memberships, and BioGrid protein interactions.",
"short_description": "short description"
},
"contributors": [
{
"name": "dummy dummy",
"email_id": "*****@*****.**"
}
]
}
mock_fetch_tags.return_value = ["test", "", "", [b'\x61\x74\x6F\x6D\x65\x73\x65',
b'\x62\x69\x6F\x69\x6E\x66\x6F\x72\x6D\x61\x74\x69\x63\x73']]
mock_s3_push.return_value = "https://test-s3-push"
org_event_consumer = ServiceCreatedEventConsumer("wss://ropsten.infura.io/ws", "http://ipfs.singularitynet.io",
80)
org_event_consumer.on_event(event=event)
service = service_repository.get_service(org_id='snet', service_id='gene-annotation-service')
service_metadata = service_repository.get_service_metadata(org_id='snet', service_id='gene-annotation-service')
service_endpoints = service_repository.get_service_endpoints(org_id='snet',
service_id='gene-annotation-service')
service_tags = service_repository.get_service_tags(org_id='snet', service_id='gene-annotation-service')
assert service == {'org_id': 'snet', 'service_id': 'gene-annotation-service', 'service_path': None,
'ipfs_hash': 'QmdGjaVYPMSGpC1qT3LDALSNCCu7JPf7j51H1GQirvQJYf', 'is_curated': 0}
assert service_metadata == {'org_id': 'snet', 'service_id': 'gene-annotation-service',
'display_name': 'Annotation Service',
'description': 'Use this service to annotate a humane genome with uniform terms, Reactome pathway memberships, and BioGrid protein interactions.',
'short_description': 'short description',
'url': 'https://mozi-ai.github.io/annotation-service/', 'json': '',
'model_ipfs_hash': 'QmXqonxB9EvNBe11J8oCYXMQAtPKAb2x8CyFLmQpkvVaLf',
'encoding': 'proto', 'type': 'grpc',
'mpe_address': '0x8FB1dC8df86b388C7e00689d1eCb533A160B4D0C',
'assets_url': '{"hero_image": "https://test-s3-push"}',
'assets_hash': '{"hero_image": "QmVcE6fEDP764ibadXTjZHk251Lmt5xAxdc4P9mPA4kksk/hero_gene-annotation-2b.png"}',
'service_rating': '{"rating": 0.0, "total_users_rated": 0}', 'ranking': 1,
'contributors': '[{"name": "dummy dummy", "email_id": "*****@*****.**"}]'}
assert service_endpoints == [{'org_id': 'snet', 'service_id': 'gene-annotation-service',
'group_id': 'm5FKWq4hW0foGW5qSbzGSjgZRuKs7A1ZwbIrJ9e96rc=',
'endpoint': 'https://mozi.ai:8000'}]
assert service_tags == [{'org_id': 'snet', 'service_id': 'gene-annotation-service', 'tag_name': 'atomese'},
{'org_id': 'snet', 'service_id': 'gene-annotation-service',
'tag_name': 'bioinformatics'}]
class OrganizationEventConsumer(EventConsumer):
_connection = Repository(NETWORK_ID, NETWORKS=NETWORKS)
_organization_repository = OrganizationRepository(_connection)
_service_repository = ServiceRepository(_connection)
def __init__(self, ws_provider, ipfs_url, ipfs_port):
self._ipfs_util = IPFSUtil(ipfs_url, ipfs_port)
self._blockchain_util = BlockChainUtil("WS_PROVIDER", ws_provider)
self._s3_util = S3Util(S3_BUCKET_ACCESS_KEY, S3_BUCKET_SECRET_KEY)
def on_event(self, event):
pass
def _push_asset_to_s3_using_hash(self, hash, org_id, service_id):
io_bytes = self._ipfs_util.read_bytesio_from_ipfs(
hash.lstrip("ipfs://"))
filename = hash.split("/")[1]
if service_id:
s3_filename = ASSETS_PREFIX + "/" + org_id + "/" + service_id + "/" + filename
else:
s3_filename = ASSETS_PREFIX + "/" + org_id + "/" + filename
new_url = self._s3_util.push_io_bytes_to_s3(s3_filename,
ASSETS_BUCKET_NAME,
io_bytes)
return new_url
def _get_new_assets_url(self, org_id, new_ipfs_data):
new_assets_hash = new_ipfs_data.get('assets', {})
existing_assets_hash = {}
existing_assets_url = {}
existing_organization = self._organization_repository.get_organization(
org_id)
if existing_organization:
existing_assets_hash = json.loads(
existing_organization["assets_hash"])
existing_assets_url = json.loads(
existing_organization["org_assets_url"])
new_assets_url_mapping = self._comapre_assets_and_push_to_s3(
existing_assets_hash, new_assets_hash, existing_assets_url, org_id,
"")
return new_assets_url_mapping
def _get_org_id_from_event(self, event):
event_org_data = eval(event['data']['json_str'])
org_id_bytes = event_org_data['orgId']
org_id = Web3.toText(org_id_bytes).rstrip("\x00")
return org_id
def _get_registry_contract(self):
net_id = NETWORK_ID
base_contract_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..', 'node_modules',
'singularitynet-platform-contracts'))
registry_contract = self._blockchain_util.get_contract_instance(
base_contract_path, "REGISTRY", net_id)
return registry_contract
def _get_org_details_from_blockchain(self, event):
logger.info(f"processing org event {event}")
registry_contract = self._get_registry_contract()
org_id = self._get_org_id_from_event(event)
blockchain_org_data = registry_contract.functions.getOrganizationById(
org_id.encode('utf-8')).call()
org_metadata_uri = Web3.toText(
blockchain_org_data[2]).rstrip("\x00").lstrip("ipfs://")
ipfs_org_metadata = self._ipfs_util.read_file_from_ipfs(
org_metadata_uri)
return org_id, blockchain_org_data, ipfs_org_metadata, org_metadata_uri
def test_on_service_created_event_with_media(self, nock_read_bytesio_from_ipfs, mock_ipfs_read,
mock_s3_push):
event = {"data": {'row_id': 202, 'block_no': 6325625, 'event': 'ServiceCreated',
'json_str': "{'orgId': b'snet\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00', 'serviceId': b'gene-annotation-service\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00', 'metadataURI': b'ipfs://QmdGjaVYPMSGpC1qT3LDALSNCCu7JPf7j51H1GQirvQJYf\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00'}",
'processed': b'\x00',
'transactionHash': 'b"\\xa7P*\\xaf\\xfd\\xd5.E\\x8c\\x0bKAF\'\\x15\\x03\\xef\\xdaO\'\\x86/<\\xfb\\xc4\\xf0@\\xf0\\xc1P\\x8c\\xc7"',
'logIndex': '0', 'error_code': 1, 'error_msg': '',
'row_updated': datetime(2019, 10, 21, 9, 59, 37),
'row_created': datetime(2019, 10, 21, 9, 59, 37)}, "name": "ServiceCreated"}
connection = Repository(NETWORK_ID, NETWORKS=NETWORKS)
service_repository = ServiceRepository(connection)
service_repository.delete_service(org_id='snet', service_id='gene-annotation-service')
service_repository.delete_service_dependents(org_id='snet', service_id='gene-annotation-service')
nock_read_bytesio_from_ipfs.return_value = "some_value to_be_pushed_to_s3_whic_is_mocked"
mock_ipfs_read.return_value = {
"version": 1,
"display_name": "Annotation Service",
"encoding": "proto",
"service_type": "grpc",
"model_ipfs_hash": "QmXqonxB9EvNBe11J8oCYXMQAtPKAb2x8CyFLmQpkvVaLf",
"mpe_address": "0x8FB1dC8df86b388C7e00689d1eCb533A160B4D0C",
"groups": [
{
"free_calls": 12,
"free_call_signer_address": "0x7DF35C98f41F3Af0df1dc4c7F7D4C19a71Dd059F",
"group_name": "default_group",
"pricing": [
{
"price_model": "fixed_price",
"price_in_cogs": 1,
"default": True
}
],
"endpoints": [
"https://mozi.ai:8000"
],
"group_id": "m5FKWq4hW0foGW5qSbzGSjgZRuKs7A1ZwbIrJ9e96rc="
}
],
"assets": {
"hero_image": "QmVcE6fEDP764ibadXTjZHk251Lmt5xAxdc4P9mPA4kksk/hero_gene-annotation-2b.png"
},
"media":[
{
"file_type": "video",
"url": "https://youtu.be/7mj-p1Os6QA",
"asset_type": "image updated",
"order": 5,
"alt_text": "alternate text sample updated",
},
{
"file_type": "text",
"url": "Qmbb7tmKZX2TSxDKsK6DEAbp3tPgNUYP11CC93Cft7EkFb/hero_fbprophet_forecast1",
"order": 2,
"alt_text": "text sample updated",
}],
"service_description": {
"url": "https://mozi-ai.github.io/annotation-service/",
"description": "Use this service to annotate a humane genome with uniform terms, Reactome pathway memberships, and BioGrid protein interactions.",
"short_description": "short description"
},
"contributors": [
{
"name": "dummy dummy",
"email_id": "*****@*****.**"
}
],
"tags": ["1234", "3241"]
}
mock_s3_push.return_value = "https://test-s3-push"
org_event_consumer = ServiceCreatedEventConsumer("wss://ropsten.infura.io/ws", "http://ipfs.singularitynet.io",
80)
org_event_consumer.on_event(event=event)
service = service_repository.get_service(org_id='snet', service_id='gene-annotation-service')
service_metadata = service_repository.get_service_metadata(org_id='snet', service_id='gene-annotation-service')
service_endpoints = service_repository.get_service_endpoints(org_id='snet',
service_id='gene-annotation-service')
service_tags = service_repository.get_service_tags(org_id='snet', service_id='gene-annotation-service')
service_groups=service_repository.get_service_group(org_id='snet',service_id='gene-annotation-service')
service_media = service_repository.get_service_media(org_id='snet',service_id='gene-annotation-service')
assert service == {'org_id': 'snet', 'service_id': 'gene-annotation-service', 'service_path': None,
'ipfs_hash': 'QmdGjaVYPMSGpC1qT3LDALSNCCu7JPf7j51H1GQirvQJYf', 'is_curated': 0}
assert service_metadata == {'org_id': 'snet', 'service_id': 'gene-annotation-service',
'display_name': 'Annotation Service',
'description': 'Use this service to annotate a humane genome with uniform terms, Reactome pathway memberships, and BioGrid protein interactions.',
'short_description': 'short description',
'url': 'https://mozi-ai.github.io/annotation-service/', 'json': '',
'model_ipfs_hash': 'QmXqonxB9EvNBe11J8oCYXMQAtPKAb2x8CyFLmQpkvVaLf',
'encoding': 'proto', 'type': 'grpc',
'mpe_address': '0x8FB1dC8df86b388C7e00689d1eCb533A160B4D0C',
'assets_url': '{"hero_image": "https://test-s3-push"}',
'assets_hash': '{"hero_image": "QmVcE6fEDP764ibadXTjZHk251Lmt5xAxdc4P9mPA4kksk/hero_gene-annotation-2b.png"}',
'service_rating': '{"rating": 0.0, "total_users_rated": 0}', 'ranking': 1,
'contributors': '[{"name": "dummy dummy", "email_id": "*****@*****.**"}]'}
assert service_endpoints == [{'org_id': 'snet', 'service_id': 'gene-annotation-service',
'group_id': 'm5FKWq4hW0foGW5qSbzGSjgZRuKs7A1ZwbIrJ9e96rc=',
'endpoint': 'https://mozi.ai:8000'}]
assert service_tags == [{'org_id': 'snet', 'service_id': 'gene-annotation-service', 'tag_name': '1234'},
{'org_id': 'snet', 'service_id': 'gene-annotation-service', 'tag_name': '3241'}]
assert service_groups == {'org_id': 'snet', 'service_id': 'gene-annotation-service', 'group_id': 'm5FKWq4hW0foGW5qSbzGSjgZRuKs7A1ZwbIrJ9e96rc=', 'group_name': 'default_group', 'pricing': '[{"default": true, "price_model": "fixed_price", "price_in_cogs": 1}]', 'free_call_signer_address': '0x7DF35C98f41F3Af0df1dc4c7F7D4C19a71Dd059F', 'free_calls': 12}
for media_item in service_media:
media_item.pop('row_id')
assert service_media == [
{'org_id': 'snet', 'service_id': 'gene-annotation-service', 'url': 'https://test-s3-push', 'order': 2, 'file_type': 'text', 'asset_type': '', 'alt_text': 'text sample updated', 'ipfs_url': 'Qmbb7tmKZX2TSxDKsK6DEAbp3tPgNUYP11CC93Cft7EkFb/hero_fbprophet_forecast1'},
{'org_id': 'snet', 'service_id': 'gene-annotation-service', 'url': 'https://youtu.be/7mj-p1Os6QA', 'order': 5, 'file_type': 'video', 'asset_type': 'image updated', 'alt_text': 'alternate text sample updated', 'ipfs_url': ''},
{'org_id': 'snet', 'service_id': 'gene-annotation-service', 'url': 'https://test-s3-push', 'order': 6, 'file_type': 'asset', 'asset_type': 'hero_image', 'alt_text': '', 'ipfs_url': 'QmVcE6fEDP764ibadXTjZHk251Lmt5xAxdc4P9mPA4kksk/hero_gene-annotation-2b.png'}
]
