def _retrieve(self, pk):
validate_pk(pk)
data = get_object_from_ledger(pk, self.ledger_query_call)
if not data or not data.get('traintuple'):
raise Exception('Invalid model: missing traintuple field')
if data['traintuple'].get('status') != "done":
raise Exception("Invalid model: traintuple must be at status done")
# Try to get it from local db, else create it in local db
try:
instance = self.get_object()
except Http404:
instance = None
if not instance or not instance.file:
instance = self.create_or_update_model(
data['traintuple'], data['traintuple']['outModel']['hash'])
# For security reason, do not give access to local file address
# Restrain data to some fields
# TODO: do we need to send creation date and/or last modified date ?
serializer = self.get_serializer(instance,
fields=('owner', 'pkhash'))
data.update(serializer.data)
return data
def _retrieve(self, request, pk):
validate_pk(pk)
data = get_object_from_ledger(pk, self.ledger_query_call)
# do not cache if node has not process permission
if node_has_process_permission(data):
# try to get it from local db to check if description exists
try:
instance = self.get_object()
except Http404:
instance = None
finally:
# check if instance has description
if not instance or not instance.description:
instance = self.create_or_update_algo(data, pk)
# For security reason, do not give access to local file address
# Restrain data to some fields
# TODO: do we need to send creation date and/or last modified date ?
serializer = self.get_serializer(instance, fields=('owner', 'pkhash'))
data.update(serializer.data)
replace_storage_addresses(request, data)
return data
def get_objective(tuple_):
from substrapp.models import Objective
objective_hash = tuple_['objective']['hash']
try:
objective = Objective.objects.get(pk=objective_hash)
except ObjectDoesNotExist:
objective = None
# get objective from ledger as it is not available in local db and store it in local db
if objective is None or not objective.metrics:
objective_metadata = get_object_from_ledger(objective_hash,
'queryObjective')
content = get_asset_content(
objective_metadata['metrics']['storageAddress'],
objective_metadata['owner'],
objective_metadata['metrics']['hash'],
)
objective, _ = Objective.objects.update_or_create(
pkhash=objective_hash, validated=True)
tmp_file = tempfile.TemporaryFile()
tmp_file.write(content)
objective.metrics.save('metrics.archive', tmp_file)
return objective.metrics.read()
def test_utils_get_object_from_ledger(self):
with mock.patch(
'substrapp.ledger_utils.query_ledger') as mquery_ledger:
mquery_ledger.return_value = objective
data = get_object_from_ledger('', 'queryObjective')
self.assertEqual(data, objective)
def prepare_testtuple_input_models(directory, tuple_):
"""Get testtuple input models content."""
traintuple_type = tuple_['traintupleType']
traintuple_key = tuple_['traintupleKey']
# TODO we should use the find method to be consistent with the traintuple
if traintuple_type == TRAINTUPLE_TYPE:
metadata = get_object_from_ledger(traintuple_key, 'queryTraintuple')
model = get_model_content(traintuple_type, traintuple_key, metadata,
metadata['outModel'])
model_hash = metadata['outModel']['hash']
_put_model(directory, model, model_hash, traintuple_key)
elif traintuple_type == COMPOSITE_TRAINTUPLE_TYPE:
metadata = get_object_from_ledger(traintuple_key,
'queryCompositeTraintuple')
head_content = get_model_content(
traintuple_type,
traintuple_key,
metadata,
metadata['outHeadModel']['outModel'],
)
trunk_content = get_model_content(
traintuple_type,
traintuple_key,
metadata,
metadata['outTrunkModel']['outModel'],
)
_put_model(directory,
head_content,
metadata['outHeadModel']['outModel']['hash'],
traintuple_key,
filename_prefix=PREFIX_HEAD_FILENAME)
_put_model(directory,
trunk_content,
metadata['outTrunkModel']['outModel']['hash'],
traintuple_key,
filename_prefix=PREFIX_TRUNK_FILENAME)
else:
raise TasksError(
f"Testtuple from type '{traintuple_type}' not supported")
def details(self, request, *args, **kwargs):
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
pk = self.kwargs[lookup_url_kwarg]
try:
data = get_object_from_ledger(pk, self.ledger_query_call)
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
return Response(data, status=status.HTTP_200_OK)
def retrieve(self, request, *args, **kwargs):
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
pk = self.kwargs[lookup_url_kwarg]
try:
validate_pk(pk)
data = get_object_from_ledger(pk, 'queryComputePlan')
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
except Exception as e:
return Response({'message': str(e)},
status=status.HTTP_400_BAD_REQUEST)
else:
return Response(data, status=status.HTTP_200_OK)
def find_training_step_tuple_from_key(tuple_key):
"""Get tuple type and tuple metadata from tuple key.
Applies to traintuple, composite traintuple and aggregatetuple.
"""
metadata = get_object_from_ledger(tuple_key, 'queryModelDetails')
if metadata.get('aggregatetuple'):
return AGGREGATETUPLE_TYPE, metadata['aggregatetuple']
if metadata.get('compositeTraintuple'):
return COMPOSITE_TRAINTUPLE_TYPE, metadata['compositeTraintuple']
if metadata.get('traintuple'):
return TRAINTUPLE_TYPE, metadata['traintuple']
raise TasksError(
f'Key {tuple_key}: no tuple found for training step: model: {metadata}'
)
def test_get_object_from_ledger(self):
with patch('substrapp.ledger_utils.query_ledger') as mquery_ledger:
mquery_ledger.side_effect = LedgerNotFound('Not Found')
self.assertRaises(LedgerNotFound, get_object_from_ledger, 'pk',
'fake_query')
with patch('substrapp.ledger_utils.query_ledger') as mquery_ledger:
mquery_ledger.side_effect = LedgerBadResponse('Bad Response')
self.assertRaises(LedgerBadResponse, get_object_from_ledger, 'pk',
'fake_query')
with patch('substrapp.ledger_utils.query_ledger') as mquery_ledger:
mquery_ledger.return_value = {'key': 'pk'}
data = get_object_from_ledger('pk', 'good_query')
self.assertEqual(data['key'], 'pk')
def download_file(self, request, django_field, ledger_field=None):
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
pk = self.kwargs[lookup_url_kwarg]
try:
asset = get_object_from_ledger(pk, self.ledger_query_call)
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
if not self._has_access(request.user, asset):
return Response({'message': 'Unauthorized'},
status=status.HTTP_403_FORBIDDEN)
if get_owner() == asset['owner']:
obj = self.get_object()
data = getattr(obj, django_field)
response = CustomFileResponse(
open(data.path, 'rb'),
as_attachment=True,
filename=os.path.basename(data.path)
)
else:
node_id = asset['owner']
auth = authenticate_outgoing_request(node_id)
if not ledger_field:
ledger_field = django_field
r = get_remote_file(asset[ledger_field]['storageAddress'], auth, stream=True)
if not r.ok:
return Response({
'message': f'Cannot proxify asset from node {asset["owner"]}: {str(r.text)}'
}, status=r.status_code)
response = CustomFileResponse(
streaming_content=(chunk for chunk in r.iter_content(512 * 1024)),
status=r.status_code)
for header in r.headers:
# We don't use hop_by_hop headers since they are incompatible
# with WSGI
if not is_hop_by_hop(header):
response[header] = r.headers.get(header)
return response
def get_algo(tuple_type, tuple_):
"""Get algo from ledger."""
query_method_names_mapper = {
TRAINTUPLE_TYPE: 'queryAlgo',
COMPOSITE_TRAINTUPLE_TYPE: 'queryCompositeAlgo',
AGGREGATETUPLE_TYPE: 'queryAggregateAlgo',
}
if tuple_type not in query_method_names_mapper:
raise TasksError(
f'Cannot find algo from tuple type {tuple_type}: {tuple_}')
method_name = query_method_names_mapper[tuple_type]
key = tuple_['algo']['hash']
metadata = get_object_from_ledger(key, method_name)
content = get_asset_content(
metadata['content']['storageAddress'],
metadata['owner'],
metadata['content']['hash'],
)
return content
def _retrieve(self, request, pk):
validate_pk(pk)
# get instance from remote node
data = get_object_from_ledger(pk, 'queryDataset')
# do not cache if node has not process permission
if node_has_process_permission(data):
# try to get it from local db to check if description exists
try:
instance = self.get_object()
except Http404:
instance = None
finally:
# check if instance has description or data_opener
if not instance or not instance.description or not instance.data_opener:
instance = self.create_or_update_datamanager(instance, data, pk)
# do not give access to local files address
serializer = self.get_serializer(instance, fields=('owner', 'pkhash', 'creation_date', 'last_modified'))
data.update(serializer.data)
replace_storage_addresses(request, data)
return data
def _retrieve(self, pk):
validate_pk(pk)
return get_object_from_ledger(pk, self.ledger_query_call)