def _retrieve(self, pk): validate_pk(pk) data = get_object_from_ledger(pk, self.ledger_query_call) if not data or not data.get('traintuple'): raise Exception('Invalid model: missing traintuple field') if data['traintuple'].get('status') != "done": raise Exception("Invalid model: traintuple must be at status done") # Try to get it from local db, else create it in local db try: instance = self.get_object() except Http404: instance = None if not instance or not instance.file: instance = self.create_or_update_model( data['traintuple'], data['traintuple']['outModel']['hash']) # For security reason, do not give access to local file address # Restrain data to some fields # TODO: do we need to send creation date and/or last modified date ? serializer = self.get_serializer(instance, fields=('owner', 'pkhash')) data.update(serializer.data) return data
def _retrieve(self, request, pk): validate_pk(pk) data = get_object_from_ledger(pk, self.ledger_query_call) # do not cache if node has not process permission if node_has_process_permission(data): # try to get it from local db to check if description exists try: instance = self.get_object() except Http404: instance = None finally: # check if instance has description if not instance or not instance.description: instance = self.create_or_update_algo(data, pk) # For security reason, do not give access to local file address # Restrain data to some fields # TODO: do we need to send creation date and/or last modified date ? serializer = self.get_serializer(instance, fields=('owner', 'pkhash')) data.update(serializer.data) replace_storage_addresses(request, data) return data
def get_objective(tuple_): from substrapp.models import Objective objective_hash = tuple_['objective']['hash'] try: objective = Objective.objects.get(pk=objective_hash) except ObjectDoesNotExist: objective = None # get objective from ledger as it is not available in local db and store it in local db if objective is None or not objective.metrics: objective_metadata = get_object_from_ledger(objective_hash, 'queryObjective') content = get_asset_content( objective_metadata['metrics']['storageAddress'], objective_metadata['owner'], objective_metadata['metrics']['hash'], ) objective, _ = Objective.objects.update_or_create( pkhash=objective_hash, validated=True) tmp_file = tempfile.TemporaryFile() tmp_file.write(content) objective.metrics.save('metrics.archive', tmp_file) return objective.metrics.read()
def test_utils_get_object_from_ledger(self): with mock.patch( 'substrapp.ledger_utils.query_ledger') as mquery_ledger: mquery_ledger.return_value = objective data = get_object_from_ledger('', 'queryObjective') self.assertEqual(data, objective)
def prepare_testtuple_input_models(directory, tuple_): """Get testtuple input models content.""" traintuple_type = tuple_['traintupleType'] traintuple_key = tuple_['traintupleKey'] # TODO we should use the find method to be consistent with the traintuple if traintuple_type == TRAINTUPLE_TYPE: metadata = get_object_from_ledger(traintuple_key, 'queryTraintuple') model = get_model_content(traintuple_type, traintuple_key, metadata, metadata['outModel']) model_hash = metadata['outModel']['hash'] _put_model(directory, model, model_hash, traintuple_key) elif traintuple_type == COMPOSITE_TRAINTUPLE_TYPE: metadata = get_object_from_ledger(traintuple_key, 'queryCompositeTraintuple') head_content = get_model_content( traintuple_type, traintuple_key, metadata, metadata['outHeadModel']['outModel'], ) trunk_content = get_model_content( traintuple_type, traintuple_key, metadata, metadata['outTrunkModel']['outModel'], ) _put_model(directory, head_content, metadata['outHeadModel']['outModel']['hash'], traintuple_key, filename_prefix=PREFIX_HEAD_FILENAME) _put_model(directory, trunk_content, metadata['outTrunkModel']['outModel']['hash'], traintuple_key, filename_prefix=PREFIX_TRUNK_FILENAME) else: raise TasksError( f"Testtuple from type '{traintuple_type}' not supported")
def details(self, request, *args, **kwargs): lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field pk = self.kwargs[lookup_url_kwarg] try: data = get_object_from_ledger(pk, self.ledger_query_call) except LedgerError as e: return Response({'message': str(e.msg)}, status=e.status) return Response(data, status=status.HTTP_200_OK)
def retrieve(self, request, *args, **kwargs): lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field pk = self.kwargs[lookup_url_kwarg] try: validate_pk(pk) data = get_object_from_ledger(pk, 'queryComputePlan') except LedgerError as e: return Response({'message': str(e.msg)}, status=e.status) except Exception as e: return Response({'message': str(e)}, status=status.HTTP_400_BAD_REQUEST) else: return Response(data, status=status.HTTP_200_OK)
def find_training_step_tuple_from_key(tuple_key): """Get tuple type and tuple metadata from tuple key. Applies to traintuple, composite traintuple and aggregatetuple. """ metadata = get_object_from_ledger(tuple_key, 'queryModelDetails') if metadata.get('aggregatetuple'): return AGGREGATETUPLE_TYPE, metadata['aggregatetuple'] if metadata.get('compositeTraintuple'): return COMPOSITE_TRAINTUPLE_TYPE, metadata['compositeTraintuple'] if metadata.get('traintuple'): return TRAINTUPLE_TYPE, metadata['traintuple'] raise TasksError( f'Key {tuple_key}: no tuple found for training step: model: {metadata}' )
def test_get_object_from_ledger(self): with patch('substrapp.ledger_utils.query_ledger') as mquery_ledger: mquery_ledger.side_effect = LedgerNotFound('Not Found') self.assertRaises(LedgerNotFound, get_object_from_ledger, 'pk', 'fake_query') with patch('substrapp.ledger_utils.query_ledger') as mquery_ledger: mquery_ledger.side_effect = LedgerBadResponse('Bad Response') self.assertRaises(LedgerBadResponse, get_object_from_ledger, 'pk', 'fake_query') with patch('substrapp.ledger_utils.query_ledger') as mquery_ledger: mquery_ledger.return_value = {'key': 'pk'} data = get_object_from_ledger('pk', 'good_query') self.assertEqual(data['key'], 'pk')
def download_file(self, request, django_field, ledger_field=None): lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field pk = self.kwargs[lookup_url_kwarg] try: asset = get_object_from_ledger(pk, self.ledger_query_call) except LedgerError as e: return Response({'message': str(e.msg)}, status=e.status) if not self._has_access(request.user, asset): return Response({'message': 'Unauthorized'}, status=status.HTTP_403_FORBIDDEN) if get_owner() == asset['owner']: obj = self.get_object() data = getattr(obj, django_field) response = CustomFileResponse( open(data.path, 'rb'), as_attachment=True, filename=os.path.basename(data.path) ) else: node_id = asset['owner'] auth = authenticate_outgoing_request(node_id) if not ledger_field: ledger_field = django_field r = get_remote_file(asset[ledger_field]['storageAddress'], auth, stream=True) if not r.ok: return Response({ 'message': f'Cannot proxify asset from node {asset["owner"]}: {str(r.text)}' }, status=r.status_code) response = CustomFileResponse( streaming_content=(chunk for chunk in r.iter_content(512 * 1024)), status=r.status_code) for header in r.headers: # We don't use hop_by_hop headers since they are incompatible # with WSGI if not is_hop_by_hop(header): response[header] = r.headers.get(header) return response
def get_algo(tuple_type, tuple_): """Get algo from ledger.""" query_method_names_mapper = { TRAINTUPLE_TYPE: 'queryAlgo', COMPOSITE_TRAINTUPLE_TYPE: 'queryCompositeAlgo', AGGREGATETUPLE_TYPE: 'queryAggregateAlgo', } if tuple_type not in query_method_names_mapper: raise TasksError( f'Cannot find algo from tuple type {tuple_type}: {tuple_}') method_name = query_method_names_mapper[tuple_type] key = tuple_['algo']['hash'] metadata = get_object_from_ledger(key, method_name) content = get_asset_content( metadata['content']['storageAddress'], metadata['owner'], metadata['content']['hash'], ) return content
def _retrieve(self, request, pk): validate_pk(pk) # get instance from remote node data = get_object_from_ledger(pk, 'queryDataset') # do not cache if node has not process permission if node_has_process_permission(data): # try to get it from local db to check if description exists try: instance = self.get_object() except Http404: instance = None finally: # check if instance has description or data_opener if not instance or not instance.description or not instance.data_opener: instance = self.create_or_update_datamanager(instance, data, pk) # do not give access to local files address serializer = self.get_serializer(instance, fields=('owner', 'pkhash', 'creation_date', 'last_modified')) data.update(serializer.data) replace_storage_addresses(request, data) return data
def _retrieve(self, pk): validate_pk(pk) return get_object_from_ledger(pk, self.ledger_query_call)