def get_download_url(url, config_file):
try:
logger.info('Connecting through Osmosis to generate the signed url.')
osm = Osmosis(url, config_file)
download_url = osm.data_plugin.generate_url(url)
logger.debug(f'Osmosis generated the url: {download_url}')
return download_url
except Exception as e:
logger.error(f'Error generating url (using Osmosis): {str(e)}')
raise
def test_compute_on_cloud(client):
# disable compute until it is supported in next release.
return
osm = Osmosis(file_path='config_local.ini')
config = parse_config(file_path='config_local.ini')
elements_before_compute = len(
osm.data_plugin.list(
get_env_property(config, 'AZURE_SHARE_OUTPUT',
'azure.share.output'), False,
get_env_property(config, 'AZURE_ACCOUNT_NAME',
'azure.account.name')))
post = client.post(BaseURLs.BASE_BRIZO_URL + '/services/compute',
data=json.dumps(json_brizo),
content_type='application/json')
assert len(
osm.data_plugin.list(
get_env_property(config, 'AZURE_SHARE_OUTPUT',
'azure.share.output'), False,
get_env_property(
config, 'AZURE_ACCOUNT_NAME',
'azure.account.name'))) == elements_before_compute + 1
osm.data_plugin.delete(
'https://testocnfiles.file.core.windows.net/output/' +
post.data.decode('utf-8'))
def consume():
"""Allows download of asset data file.
---
tags:
- services
consumes:
- application/json
parameters:
- name: consumerAddress
in: query
description: The consumer address.
required: true
type: string
- name: serviceAgreementId
in: query
description: The ID of the service agreement.
required: true
type: string
- name: url
in: query
description: This URL is only valid if Brizo acts as a proxy.
Consumer can't download using the URL if it's not through Brizo.
required: true
type: string
responses:
200:
description: Redirect to valid asset url.
400:
description: One of the required attributes is missing.
401:
description: Invalid asset data.
500:
description: Error
"""
data = request.args
required_attributes = ['serviceAgreementId', 'consumerAddress']
msg, status = check_required_attributes(required_attributes, data,
'consume')
if msg:
return msg, status
if not (data.get('url') or (data.get('signature') and data.get('index'))):
return f'Either `url` or `signature and index` are required in the call to "consume".', 400
try:
provider_account = get_provider_account(ocn)
agreement_id = data.get('serviceAgreementId')
consumer_address = data.get('consumerAddress')
asset_id = ocn.agreements.get(agreement_id).did
did = id_to_did(asset_id)
if ocn.agreements.is_access_granted(agreement_id, did,
consumer_address):
logger.info('Connecting through Osmosis to generate the sign url.')
url = data.get('url')
try:
if not url:
signature = data.get('signature')
index = int(data.get('index'))
address = Keeper.get_instance().ec_recover(
agreement_id, signature)
if address.lower() != consumer_address.lower():
msg = f'Invalid signature {signature} for ' \
f'consumerAddress {consumer_address} and documentId {did}.'
raise ValueError(msg)
asset = ocn.assets.resolve(did)
urls_str = ocn.secret_store.decrypt(
asset_id, asset.encrypted_files, provider_account)
urls = json.loads(urls_str)
if index >= len(urls):
raise ValueError(f'url index "{index}"" is invalid.')
url = urls[index]['url']
osm = Osmosis(url, config_file)
download_url = osm.data_plugin.generate_url(url)
logger.debug("Osmosis generate the url: %s", download_url)
try:
if request.range:
headers = {"Range": request.headers.get('range')}
response = requests_session.get(download_url,
headers=headers,
stream=True)
else:
headers = {
"Content-Disposition":
f'attachment;filename={url.split("/")[-1]}'
}
response = requests_session.get(download_url,
headers=headers,
stream=True)
return Response(io.BytesIO(response.content).read(),
response.status_code,
headers=headers)
except Exception as e:
logger.error(e)
return "Error getting the url content: %s" % e, 401
except Exception as e:
logger.error(e)
return "Error generating url: %s" % e, 401
else:
msg = "Invalid consumer address and/or service agreement id, " \
"or consumer address does not have permission to consume this asset."
logger.warning(msg)
return msg, 401
except Exception as e:
logger.error("Error- " + str(e))
return "Error : " + str(e), 500
from osmosis_driver_interface.osmosis import Osmosis
from osmosis_aws_driver.data_plugin import Plugin
from filecmp import cmp
import os
import time
# os.environ['AWS_PROFILE'] = 'ocean'
aws = Osmosis('./tests/aws.ini').data_plugin()
def test_plugin_type():
assert aws.type == 'AWS'
def test_complete():
# Create folder, upload file, list files, download file, delete file
# TODO: Add finally to clean s3 bucket
dpl = Plugin()
bucket_name = f'ocean-test-osmosis-data-plugin-{int(time.time())}'
print(f'Test bucket: {bucket_name}')
dpl.create_directory(f's3://{bucket_name}/test')
dpl.upload('./LICENSE', f's3://{bucket_name}/test/LICENSE')
files = dpl.list(f's3://{bucket_name}/test/')
assert len(files) == 2 # /test and /test/LICENSE
assert files[0]['Key'] == 'test/'
assert files[1]['Key'] == 'test/LICENSE'
dpl.download(f's3://{bucket_name}/test/LICENSE',
'/tmp/test_osmosis_data_plugin_license')
assert cmp('./LICENSE', '/tmp/test_osmosis_data_plugin_license')
dpl.delete(f's3://{bucket_name}/test/LICENSE')
files = dpl.list(f's3://{bucket_name}/test/')
def test_parse_url_ipfs():
assert Osmosis.parse_url(
'ipfs://ZmOfotxMWnLTXKKW0GPV9NgtEugghgD8HgzSF4gSrp7mL9') == 'ipfs'
def test_parse_url_azure():
assert Osmosis.parse_url(
'https://testocnfiles.blob.core.windows.net/testfiles/testzkp.pdf'
) == 'azure'
def test_parse_url_on_premise():
assert Osmosis.parse_url('http://www.example.com') == 'on_premise'
def test_parse_url_aws():
assert Osmosis.parse_url('s3://my_bucket') == 'aws'
def compute():
"""Allows to execute an algorithm inside a Docker instance in the cloud. Requires the publisher of the assets to provide this service in the service agreement related with the requested `asset_did`.
---
tags:
- services
consumes:
- application/json
parameters:
- in: body
name: body
required: true
description: Asset metadata.
schema:
type: object
required:
- asset_did
- algorithm_did
- consumer_wallet
properties:
asset_did:
description: Identifier of the asset registered in Ocean
type: string
example: '0x0234242345'
algorithm_did:
description: Identifier of the algorithm to execute
type: string
example: '0x0234242345'
consumer_wallet:
description: Address of the wallet of the asset consumer. Ex. data-science...
type: string
example: '0x0234242345'
docker_image:
description: Docker image where the algorithm is going to be executed. Docker image must include all the libraries needed to run it.
type: string
example: python:3.6-alpine
memory:
description: Ammout of memory in GB to run the algorithm
type: number
example: 1.5
cpu:
description: Number of CPUs to execute the algorithm.
type: integer
example: 1
"""
required_attributes = ['asset_did', 'algorithm_did', 'consumer_wallet']
data = request.json
msg, status = check_required_attributes(required_attributes, data, 'compute')
if msg:
return msg, status
osm = Osmosis(config_file)
# TODO use this two asigment in the exec_container to use directly did instead of the name
# asset_url = _parse_url(get_metadata(ocn.assets.get_ddo(data.get('asset_did')))['base']['contentUrls'][0]).file
# algorithm_url = _parse_url(
# get_metadata(ocn.assets.get_ddo(data.get('algorithm_url')))['base']['contentUrls'][0]).file
# share_name_input = _parse_url(
# get_metadata(ocn.assets.get_ddo(data.get('asset_did')))['base']['contentUrls'][0]).file_share
return osm.computing_plugin.exec_container(asset_url=data.get('asset_did'),
algorithm_url=data.get('algorithm_did'),
resource_group_name=get_env_property(
'AZURE_RESOURCE_GROUP', 'azure.resource_group'),
account_name=get_env_property('AZURE_ACCOUNT_NAME',
'azure.account.name'),
account_key=get_env_property('AZURE_ACCOUNT_KEY',
'azure.account.key'),
share_name_input=get_env_property(
'AZURE_SHARE_INPUT', 'azure.share.input'),
share_name_output=get_env_property(
'AZURE_SHARE_OUTPUT', 'azure.share.output'),
location=get_env_property('AZURE_LOCATION',
'azure.location'),
# input_mount_point=data.get('input_mount_point'),
# output_mount_point=data.get('output_mount_point'),
docker_image=data.get('docker_image'),
memory=data.get('memory'),
cpu=data.get('cpu')), 200
def consume():
"""Allows download of asset data file.
---
tags:
- services
consumes:
- application/json
parameters:
- name: consumerAddress
in: query
description: The consumer address.
required: true
type: string
- name: serviceAgreementId
in: query
description: The ID of the service agreement.
required: true
type: string
- name: url
in: query
description: This URL is only valid if Brizo acts as a proxy. Consumer can't download using the URL if it's not through Brizo.
required: true
type: string
responses:
302:
description: Redirect to valid asset url.
400:
description: One of the required attributes is missing.
401:
description: Invalid asset data.
500:
description: Error
"""
data = request.args
required_attributes = ['serviceAgreementId', 'consumerAddress', 'url']
msg, status = check_required_attributes(required_attributes, data, 'consume')
if msg:
return msg, status
try:
if ocn.is_access_granted(
data.get('serviceAgreementId'),
cache.get(data.get('serviceAgreementId')),
data.get('consumerAddress')):
# Delete cached serviceAgreementId.
cache.delete(data.get('serviceAgreementId'))
logging.info('Connecting through Osmosis to generate the sign url.')
try:
osm = Osmosis(config_file)
result = osm.data_plugin.generate_url(data.get('url'))
except Exception as e:
logging.error(e)
return "Error generating url: %s" % e, 401
logging.debug("Osmosis generate the url: %s", result)
return redirect(result, code=302)
else:
msg = "Invalid consumer address and/or service agreement id, " \
"or consumer address does not have permission to consume this asset."
logger.warning(msg)
return msg, 401
except Exception as e:
logger.error("Error- " + str(e))
return "Error : " + str(e), 500
def test_osmosis_instances():
osm = Osmosis('http://www.example.com')
assert osm.data_plugin.type() == 'On premise'