def __init__(self, eos_client, uapp_contract_acc, requesting_app, users):
"""
:param users: A list of users we want to obtain data for. If an empty
list is provided, then data for all permissible users are provided.
"""
self.__uapp_contract_acc = uapp_contract_acc
self.__requesting_app = requesting_app
self.__haiku_conf = UnificationConfig()
self.__my_mother = UnificationMother(eos_client, uapp_contract_acc,
get_cleos(), get_ipfs_client())
self.__my_uapp_sc = UnificationUapp(eos_client, uapp_contract_acc)
self.__my_lookup = UnificationLookup(default_lookup_db())
self.__users = None if len(users) == 0 else users
self.__my_db_schemas = self.__my_uapp_sc.get_all_db_schemas()
self.__db_schema_maps = {}
self.__granted = []
self.__granted_field_lookup = {}
self.__revoked = []
self.__raw_data = None
self.__native_user_meta = self.__my_lookup.get_native_user_meta()
for pkey, db_schema in self.__my_db_schemas.items():
schema_map = self.__my_lookup.get_schema_map(pkey)
self.__db_schema_maps[pkey] = schema_map
self.__generate_data()
def run_test_mother(self, app, demo_apps):
print("Contacting MOTHER FOR: ", app)
eos_client = Client(nodes=[self.cleos.get_nodeos_url()])
um = UnificationMother(eos_client, app, get_cleos(), get_ipfs_client())
print("Valid app: ", um.valid_app())
assert um.valid_app() is True
print("UApp SC Hash in MOTHER: ", um.get_hash_in_mother())
print("Deployed UApp SC hash: ", um.get_deployed_contract_hash())
assert um.get_hash_in_mother() == um.get_deployed_contract_hash()
print("Valid Code: ", um.valid_code())
assert um.valid_code() is True
print("Signed by MOTHER: ", um.signed_by_mother())
assert um.signed_by_mother() is True
print("RPC IP: ", um.get_haiku_rpc_ip())
assert um.get_haiku_rpc_ip() == demo_apps[app]['rpc_server']
print("RPC Port: ", um.get_haiku_rpc_port())
assert int(um.get_haiku_rpc_port()) == int(
demo_apps[app]['rpc_server_port'])
print("RPC Server: ", um.get_haiku_rpc_server())
print("-----------------------------------")
def systest_auth(requesting_app, providing_app, user):
"""
Ensuring that an incorrectly signed request is rejected.
"""
def broken(d, field):
d[field] = 'unlucky' + d[field][7:]
return d
log.info(f'{requesting_app} is requesting data from {providing_app}')
body = {'users': [user], 'data_id': 'request_hash'}
app_config = demo_config['demo_apps'][providing_app]
port = app_config['rpc_server_port']
eos_client = get_eos_rpc_client()
mother = UnificationMother(eos_client, providing_app, get_cleos(),
get_ipfs_client())
provider_obj = Provider(providing_app, 'https', mother)
encoded_password = demo_config['system'][requesting_app]['password']
ks = UnificationKeystore(encoded_password,
app_name=requesting_app,
keystore_path=Path('data/keys'))
payload = bundle(ks, requesting_app, provider_obj.name, body, 'Success')
payload = broken(payload, 'signature')
r = provider_obj.post('data_request', payload)
assert r.status_code == 401
def view(provider, request_hash):
"""
Read data stored locally from an Data Provider for a particular user.
\b
:param provider: The app name of the data provider.
:param request_hash: The particular piece of data in concern.
:return:
"""
requesting_app = os.environ['app_name']
password = os.environ['keystore']
eos_client = get_eos_rpc_client()
mother = UnificationMother(eos_client, provider, get_cleos(),
get_ipfs_client())
provider_obj = Provider(provider, 'https', mother)
req_hash = f'request-{request_hash}'
click.echo(f'App {requesting_app} is reading ingested data from '
f'{provider_obj.name}')
encoded_password = str.encode(password)
keystore = UnificationKeystore(encoded_password)
client = HaikuDataClient(keystore)
data = client.read_data_from_store(provider_obj, req_hash)
click.echo(json.loads(data))
def get_proof():
conf = app.unification_config
d = flask.request.get_json()
user = d['user']
consumer = d['consumer']
ipfs_hash = d['ipfs_hash']
schema_id = d['schema_id']
provider_uapp = UnificationUapp(get_eos_rpc_client(),
conf['uapp_contract'])
permission_db = PermissionBatchDatabase(pb_default_db())
permissions = UnifPermissions(get_ipfs_client(), provider_uapp,
permission_db)
if ipfs_hash is not None:
permissions.load_perms_from_ipfs(ipfs_hash)
else:
permissions.load_consumer_perms(consumer)
proof = permissions.get_proof(user, schema_id=schema_id)
# ToDo: send as JWT
return_d = {'proof': proof}
return flask.jsonify(return_d), 200
def __build_permissions(self):
granted = []
revoked = []
granted_field_lookup = {}
pbdb = PermissionBatchDatabase(default_pb_db())
permissions = UnifPermissions(get_ipfs_client(), self.__my_uapp_sc,
pbdb)
permissions.load_consumer_perms(self.__requesting_app)
permission_obj = permissions.get_all_perms()
# schema_id = self.__my_db_schemas[0]['pkey'] # tmp - only 1 db schema
for user, perms in permission_obj['permissions'].items():
log.debug(perms)
schema_perms = perms['0'] # tmp - only 1 db schema
if schema_perms['perms'] == '':
revoked.append(eosio_account.string_to_name(user))
else:
granted.append(eosio_account.string_to_name(user))
native_id = self.__my_lookup.get_native_user_id(user)
granted_field_lookup[native_id] = schema_perms['perms'].split(
',')
# required
granted_field_lookup[native_id].append('account_name')
return granted, revoked, granted_field_lookup
def systest_merkle_proof_permissions():
ipfs = get_ipfs_client()
users, consumers, providers = compile_actors()
for provider in providers:
log.debug(f'run systest_merkle_proof_'
f'permissions for Provider {provider}')
provider_uapp = UnificationUapp(get_eos_rpc_client(), provider)
permission_db = PermissionBatchDatabase(pb_default_db())
permissions = UnifPermissions(ipfs, provider_uapp, permission_db)
for consumer in consumers:
if consumer != provider:
log.debug(f'Provider {provider}: load '
f'permissions for Consumer {consumer}')
permissions.load_consumer_perms(consumer)
permissions_obj = permissions.get_all_perms()
tree = MerkleTree()
for user, perm in permissions_obj['permissions'].items():
for schema_id, schema_perm in perm.items():
tree.add_leaf(json.dumps(schema_perm))
tree.grow_tree()
log.debug(f"Generated merkle root: {tree.get_root_str()}")
log.debug(f"Recorded merkle root: "
f"{permissions_obj['merkle_root']}")
for user, perm in permissions_obj['permissions'].items():
for schema_id, schema_perm in perm.items():
requested_leaf = json.dumps(schema_perm)
proof_chain = tree.get_proof(requested_leaf,
is_hashed=False)
log.debug(f'Permission leaf for {user}:'
f' {requested_leaf}')
log.debug(f'Proof chain for {user} - '
f'Schema {schema_id} '
f'permission leaf: '
f'{json.dumps(proof_chain)}')
# simulate only having access to leaf,
# root and proof chain for leaf
verify_tree = MerkleTree()
is_good = verify_tree.verify_leaf(
requested_leaf,
permissions_obj['merkle_root'],
proof_chain,
is_hashed=False)
log.debug(f'Leaf is valid: {is_good}')
assert is_good
def get_public_key(app_name):
eos_client = get_eos_rpc_client()
uapp_sc = UnificationUapp(eos_client, app_name)
public_key_hash = uapp_sc.get_public_key_hash(app_name)
store = get_ipfs_client()
public_key = store.cat_file(public_key_hash)
return serialization.load_pem_public_key(public_key,
backend=default_backend())
def __check_app_is_valid(self):
"""
Call the MOTHER Smart Contract, and check if the requesting_app is both
a verified app, and that it's smart contract code is valid (by checking
the code's hash).
"""
um = UnificationMother(self.__eosClient, self.__app_to_validate,
get_cleos(), get_ipfs_client())
self.__is_valid_app = um.valid_app()
self.__is_valid_code = um.valid_code()
self.__signed_by_mother = um.signed_by_mother()
def systest_check_permission_requests():
ipfs = get_ipfs_client()
users, consumers, providers = compile_actors()
for provider in providers:
log.debug(f'run systest_check_permission_requests'
f' for Provider {provider}')
provider_uapp = UnificationUapp(get_eos_rpc_client(), provider)
permission_db = PermissionBatchDatabase(pb_default_db())
permissions = UnifPermissions(ipfs, provider_uapp, permission_db)
for consumer in consumers:
if consumer != provider:
log.debug(f'Provider {provider}: load permissions '
f'for Consumer {consumer}')
permissions.load_consumer_perms(consumer)
for user in users:
user_permissions = permissions.get_user_perms_for_all_schemas(
user)
for schema_id, user_perms in user_permissions.items():
log.debug(f'User {user}, '
f'Schema {schema_id}: {user_perms}')
is_valid = permissions.verify_permission(user_perms)
log.debug(f'Perm sig valid: {is_valid}')
assert is_valid
demo_conf_check = demo_config['demo_permissions'][
user][consumer][provider]
demo_conf_fields = demo_conf_check['fields']
demo_conf_granted = demo_conf_check['granted']
demo_conf_schema_id = demo_conf_check['schema_id']
assert int(demo_conf_schema_id) == int(schema_id)
if demo_conf_granted:
log.debug("Permission granted")
log.debug(f"Demo fields: {demo_conf_fields}, "
f"recorded fields: "
f"{user_perms['perms']}")
assert demo_conf_fields == user_perms['perms']
else:
log.debug("Permission not granted. Recorded "
"perms should be empty")
log.debug(f"Recorded fields: "
f"{user_perms['perms']}")
assert user_perms['perms'] == ''
def data_request():
try:
d = flask.request.get_json()
# Validate requesting app against smart contracts
# config is this Haiku Node's config fle, containing its UApp
# Smart Contract account/address and the EOS RPC server/port used for
# communicating with the blockchain.
conf = app.unification_config
sender = d['eos_account_name']
recipient = conf['uapp_contract']
if sender == recipient:
return error_request_self()
bundle_d = unbundle(app.keystore, sender, d)
eos_client = get_eos_rpc_client()
# Init the validation class for THIS Haiku, and validate the
# REQUESTING APP.
v = UnificationAppScValidation(eos_client, d['eos_account_name'])
# If the REQUESTING APP is valid according to MOTHER, then we can
# generate the data. If not, return an invalid_app response
if v.valid():
users = bundle_d.get('users')
request_id = bundle_d.get('request_id')
# before processing data, check for any stashed permissions
ipfs = get_ipfs_client()
provider_uapp = UnificationUapp(eos_client, conf['uapp_contract'])
permission_db = PermissionBatchDatabase(pb_default_db())
permissions = UnifPermissions(ipfs, provider_uapp, permission_db)
permissions.check_and_process_stashed(sender)
return obtain_data(app.keystore, sender, eos_client,
conf['uapp_contract'], users, request_id)
else:
return invalid_app()
except InvalidSignature:
return invalid_response()
except Exception as e:
logger.exception(e)
return generic_error()
def fetch(provider, request_hash, user):
"""
Fetch data from an App to this App in an Enterprise/DSP/B2B environment,
where data request is agreed external to the UApp Store.
\b
:param provider: The app name of the data provider.
:param request_hash: The particular piece of data in concern.
:param user: Obtain data for a specific user EOS user account.
:return:
"""
requesting_app = os.environ['app_name']
password = os.environ['keystore']
# Write the data request to the Consumer's UApp smart contract
eos_client = get_eos_rpc_client()
mother = UnificationMother(eos_client, provider, get_cleos(),
get_ipfs_client())
provider = Provider(provider, 'https', mother)
req_hash = f'request-{request_hash}'
suffix = 'for all users' if user is None else f'for {user}'
click.echo(f'App {requesting_app} is requesting data from {provider}'
f' {suffix}')
encoded_password = str.encode(password)
keystore = UnificationKeystore(encoded_password)
# tmp - get the price for the transfer from Schema[0] in provider's UApp SC
# This will possibly be determined externally as part of the B2B agreement
provider_uapp_sc = UnificationUapp(eos_client, provider.name)
db_schema = provider_uapp_sc.get_db_schema_by_pkey(
0) # tmp - only 1 schema
sched_price = db_schema['price_sched']
# initiate request in Consumer's UApp SC
consumer_uapp_sc = UnificationUapp(eos_client, requesting_app)
latest_req_id = consumer_uapp_sc.init_data_request(provider.name, "0", "0",
sched_price)
client = HaikuDataClient(keystore)
data_path = client.make_data_request(requesting_app, provider, user,
req_hash, latest_req_id)
click.echo(f'Data written to: {data_path}')
click.echo(f'View using: haiku view {provider.name} {request_hash}')
def systest_smart_contract_mother():
log.info('Running systest smart contract MOTHER')
d_conf = json.loads(Path('data/demo_config.json').read_text())
appnames = ['app1', 'app2', 'app3']
d_apps = d_conf['demo_apps']
conf = UnificationConfig()
eos_client = Client(
nodes=[f"http://{conf['eos_rpc_ip']}:{conf['eos_rpc_port']}"])
for appname in appnames:
log.info("------------------------------------------")
app_data = d_apps[appname]
log.info(f"Contacting MOTHER for {app_data['eos_sc_account']}")
mother = UnificationMother(eos_client, app_data['eos_sc_account'],
get_cleos(), get_ipfs_client())
log.info("App is Valid")
log.info("Expecting: True")
log.info(f"Actual - MOTHER: {mother.valid_app()}")
assert mother.valid_app() is True
log.info("App Code is Valid")
log.info("Expecting: True")
log.info(f"Actual - MOTHER: {mother.valid_code()}")
assert mother.valid_app() is True
log.info("Code Hash")
log.info(
f"Expecting - config.json: {mother.get_deployed_contract_hash()}")
log.info(f"Actual - MOTHER: {mother.get_hash_in_mother()}")
assert (mother.get_deployed_contract_hash()
== mother.get_hash_in_mother()) is True
log.info("RPC IP")
log.info(f"Expecting - config.json: {app_data['rpc_server']}")
log.info(f"Actual - MOTHER: {mother.get_haiku_rpc_ip()}")
assert (app_data['rpc_server'] == mother.get_haiku_rpc_ip()) is True
log.info("RPC Port")
log.info(f"Expecting - config.json: {app_data['rpc_server_port']}")
log.info(f"Actual - MOTHER: {mother.get_haiku_rpc_port()}")
assert (int(app_data['rpc_server_port']) == int(
mother.get_haiku_rpc_port())) is True
log.info("------------------------------------------")
def add_to_mother(self, app_config, appname, unif_mother_private_key):
contract_hash = self.get_code_hash(appname)
ipfs_client = get_ipfs_client()
app_conf = app_config[appname]
schema_vers = ""
for i in app_conf['db_schemas']:
# hard-code version num to 1 for demo
schema_vers = schema_vers + i['schema_name'] + ":1,"
schema_vers = schema_vers.rstrip(",")
uapp_data = {
'uapp_contract_acc': appname,
'schema_vers': schema_vers,
'uapp_contract_hash': contract_hash,
'rpc_server_ip': app_conf['rpc_server'],
'rpc_server_port': app_conf['rpc_server_port'],
'name': app_conf['uapp_name'],
'description': app_conf['uapp_desc'],
'website': app_conf['uapp_website'],
'nonce': generate_nonce(16),
'time_added': int(time.time()),
'time_updated': int(time.time())
}
eosk = UnifEosKey(unif_mother_private_key)
digest_sha = sha256(json.dumps(uapp_data).encode('utf-8'))
mother_sig = eosk.sign(digest_sha)
mother_data = {'data': uapp_data, 'sig': mother_sig}
mother_json = json.dumps(mother_data)
ipfs_hash = ipfs_client.add_json(mother_json)
d = {'uapp_contract_acc': appname, 'ipfs_hash': ipfs_hash}
ret = self.cleos.run([
'push', 'action', 'unif.mother', 'addnew',
json.dumps(d), '-p', 'unif.mother'
])
print(ret.stdout)
def __request_from_uapp_store(data_request):
"""
Receives a data request from the UApp Store, and
processes the request
\b
:param data_request: Dict containing request parameters
"""
requesting_app = os.environ['app_name']
password = os.environ['keystore']
click.echo("Processing request from UApp Store:")
click.echo(data_request)
eos_client = get_eos_rpc_client()
# Write the data request to the Consumer's smart contract
uapp_sc = UnificationUapp(eos_client, requesting_app)
latest_req_id = uapp_sc.init_data_request(data_request['provider'],
data_request['schema_pkey'], "0",
data_request['price'])
request_hash = f"{data_request['provider']}-{data_request['schema_pkey']}" \
f"-{latest_req_id}.dat"
provider_name = data_request['provider']
mother = UnificationMother(eos_client, provider_name, get_cleos(),
get_ipfs_client())
provider_obj = Provider(provider_name, 'https', mother)
req_hash = f'request-{request_hash}'
click.echo(f'App {requesting_app} is requesting data from '
f'{provider_obj.name}')
encoded_password = str.encode(password)
keystore = UnificationKeystore(encoded_password)
client = HaikuDataClient(keystore)
data_path = client.make_data_request(requesting_app, provider_obj, None,
req_hash, latest_req_id)
click.echo(f'Data written to: {data_path}')
click.echo(f'View using: haiku view {provider_obj.name} {request_hash}')
def permissions(user):
"""
Display user permissions.
\b
:param user: The EOS user account name to query.
"""
eos_rpc_client = get_eos_rpc_client()
ipfs_client = get_ipfs_client()
apps = []
valid_apps = eos_rpc_client.get_table_rows(
"unif.mother", "unif.mother", "validapps", True, 0, -1,
-1)
for va in valid_apps['rows']:
apps.append(eosio_account.name_to_string(int(va['uapp_contract_acc'])))
click.echo(f"{bold(user)} Permissions overview:")
for provider in apps:
click.echo(f'Provider: {bold(provider)}')
for consumer in apps:
if consumer != provider:
click.echo(f' Consumer: {bold(consumer)}')
uapp_sc = UnificationUapp(eos_rpc_client, provider)
ipfs_hash, merkle_root = uapp_sc.get_ipfs_perms_for_req_app(
consumer)
if ipfs_hash is not None and ipfs_hash != ZERO_MASK:
permissions_str = ipfs_client.get_json(ipfs_hash)
permissions_json = json.loads(permissions_str)
user_perms = permissions_json[user]
for schema_id, perms in user_perms.items():
click.echo(f' Schema ID: {schema_id}')
if perms['perms'] == '':
click.echo(' Granted: False')
else:
click.echo(' Granted: True')
click.echo(f" Fields: {perms['perms']}")
else:
click.echo('Nothing set')
def systest_ingest(requesting_app, providing_app, user, balances):
log.info(f'Testing Fetch ingestion: {requesting_app} '
f'is requesting data from {providing_app}')
request_hash = f'data-request-{providing_app}-{requesting_app}'
app_config = demo_config['demo_apps'][providing_app]
port = app_config['rpc_server_port']
eos_client = get_eos_rpc_client()
mother = UnificationMother(eos_client, providing_app, get_cleos(),
get_ipfs_client())
provider_obj = Provider(providing_app, 'https', mother)
password = demo_config['system'][requesting_app]['password']
encoded_password = str.encode(password)
keystore = UnificationKeystore(encoded_password,
app_name=requesting_app,
keystore_path=Path('data/keys'))
conf = UnificationConfig()
eos_client = Client(
nodes=[f"http://{conf['eos_rpc_ip']}:{conf['eos_rpc_port']}"])
consumer_uapp_sc = UnificationUapp(eos_client, requesting_app)
price_sched = demo_config['demo_apps'][providing_app]['db_schemas'][0][
'price_sched']
latest_req_id = consumer_uapp_sc.init_data_request(provider_obj.name, "0",
"0", price_sched)
client = HaikuDataClient(keystore)
client.make_data_request(requesting_app, provider_obj, user, request_hash,
latest_req_id)
client.read_data_from_store(provider_obj, request_hash)
# Update the system test record of the balances
balances[requesting_app] = balances[requesting_app] - price_sched
und_rewards = UndRewards(providing_app, price_sched)
balances[providing_app] = (balances[providing_app] +
und_rewards.calculate_reward(is_user=False))
return balances
def systest_process_permission_batches():
appnames = ['app1', 'app2', 'app3']
for app_name in appnames:
log.debug(f'run systest_process_permission_batches for {app_name}')
mother = UnificationMother(get_eos_rpc_client(), app_name, get_cleos(),
get_ipfs_client())
provider_obj = Provider(app_name, 'https', mother)
password = demo_config['system'][app_name]['password']
encoded_password = str.encode(password)
keystore = UnificationKeystore(encoded_password,
app_name=app_name,
keystore_path=Path('data/keys'))
client = HaikuDataClient(keystore)
try:
client.process_permissions_batch(provider_obj)
except Exception as e:
log.error(f'systest_process_permission_batches failed: {e}')
def request_permission_change(self, user, app_permission_list,
private_key):
log.info(f"Process {user} permission change requests")
for consumer, providers in app_permission_list.items():
for provider, permissions in providers.items():
granted = permissions['granted']
if granted:
fields = permissions['fields']
else:
fields = ''
schema_id = int(permissions['schema_id'])
log.debug(f'request_permission_change '
f'{user} requesting {provider} '
f'update perms for {consumer} '
f'in schema {schema_id}: {granted} {fields}')
payload, p_nonce, p_sig = generate_payload(
user, private_key, provider, consumer, fields, 'active',
schema_id)
log.debug(f'request_permission_change payload: '
f'{json.dumps(payload)}')
mother = UnificationMother(get_eos_rpc_client(), provider,
get_cleos(), get_ipfs_client())
provider_obj = Provider(provider, 'https', mother)
r = provider_obj.post('modify_permission', payload)
d = r.json()
if r.status_code != 200:
raise Exception(d['message'])
proc_id = d['proc_id']
ret_app = d['app']
log.debug(f"request_permission_change success: "
f"{ret_app}: Process ID {proc_id}")
def invalidapps():
"""
Display invalid apps.
\b
"""
click.echo(bold("Invalid apps according to MOTHER:"))
eos_client = get_eos_rpc_client()
ipfs_client = get_ipfs_client()
valid_apps = eos_client.get_table_rows(
"unif.mother", "unif.mother", "validapps", True, 0, -1,
-1)
for va in valid_apps['rows']:
if int(va['is_valid']) == 0:
ipfs_hash = va['ipfs_hash']
uapp_json_str = ipfs_client.get_json(ipfs_hash)
uapp_json = json.loads(uapp_json_str)
uapp_data = uapp_json['data']
mother_sig = uapp_json['sig']
acc_name = eosio_account.name_to_string(
int(va['uapp_contract_acc']))
click.echo(f"{bold(acc_name)}: {bold(uapp_data['name'])}")
click.echo(f"{bold(uapp_data['name'])}")
click.echo(f"{uapp_data['description']}")
click.echo(f"{uapp_data['website']}")
click.echo(f"Contract Hash: {uapp_data['uapp_contract_hash']}")
click.echo(f"RPC Server: "
f"http://{uapp_data['rpc_server_ip']}:"
f"{uapp_data['rpc_server_port']}")
click.echo(f"UApp Registered: {added}")
click.echo(f"MOTHER Signature: {mother_sig}")
click.echo("is valid: false")
click.echo("------------------------")
def __init__(self, permissions_db: Path):
self.db = PermissionBatchDatabase(permissions_db)
self.ipfs = get_ipfs_client()
