def _retrieve(self, request, key): validate_key(key) # get instance from remote node data = get_object_from_ledger(get_channel_name(request), key, 'queryDataset') # do not cache if node has not process permission if node_has_process_permission(data): # try to get it from local db to check if description exists try: instance = self.get_object() except Http404: instance = None finally: # check if instance has description or data_opener if not instance or not instance.description or not instance.data_opener: instance = self.create_or_update_datamanager( get_channel_name(request), instance, data, key) # do not give access to local files address serializer = self.get_serializer(instance, fields=('owner')) data.update(serializer.data) replace_storage_addresses(request, data) return data
def list(self, request, *args, **kwargs): try: data = query_ledger(get_channel_name(request), fcn='queryDataManagers', args=[]) except LedgerError as e: return Response({'message': str(e.msg)}, status=e.status) # parse filters query_params = request.query_params.get('search') if query_params is not None: try: data = filter_list(channel_name=get_channel_name(request), object_type='dataset', data=data, query_params=query_params) except LedgerError as e: return Response({'message': str(e.msg)}, status=e.status) for data_manager in data: replace_storage_addresses(request, data_manager) return Response(data, status=status.HTTP_200_OK)
def _retrieve(self, request, key): validate_key(key) data = get_object_from_ledger(get_channel_name(request), key, self.ledger_query_call) # do not cache if node has not process permission if node_has_process_permission(data): # try to get it from local db to check if description exists try: instance = self.get_object() except Http404: instance = None finally: # check if instance has description if not instance or not instance.description: instance = self.create_or_update_algo( get_channel_name(request), data, key) # For security reason, do not give access to local file address # Restrain data to some fields # TODO: do we need to send creation date and/or last modified date ? serializer = self.get_serializer(instance, fields=('owner')) data.update(serializer.data) replace_storage_addresses(request, data) return data
def _create(self, request): key = uuid.uuid4() data = { 'key': key, 'algo_key': request.data.get('algo_key'), 'data_manager_key': request.data.get('data_manager_key'), 'rank': request.data.get('rank'), 'compute_plan_key': request.data.get('compute_plan_key'), 'in_head_model_key': request.data.get('in_head_model_key'), 'in_trunk_model_key': request.data.get('in_trunk_model_key'), 'out_trunk_model_permissions': request.data.get('out_trunk_model_permissions'), 'train_data_sample_keys': request.data.get('train_data_sample_keys'), 'tag': request.data.get('tag', ''), 'metadata': request.data.get('metadata') } serializer = self.get_serializer(data=data) serializer.is_valid(raise_exception=True) return self.commit(serializer, get_channel_name(request))
def update_ledger(self, request, *args, **kwargs): lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field key = self.kwargs[lookup_url_kwarg] validate_key(key) objective_key = request.data.get('objective_key') args = { 'data_manager_key': key, 'objective_key': objective_key, } if settings.LEDGER_SYNC_ENABLED: st = status.HTTP_200_OK else: st = status.HTTP_202_ACCEPTED try: data = ledger.assets.update_datamanager(get_channel_name(request), args) except LedgerError as e: return Response({'message': str(e.msg)}, status=e.status) return Response(data, status=st)
def list(self, request, *args, **kwargs): try: data = query_ledger(get_channel_name(request), fcn='queryModels', args=[]) except LedgerError as e: return Response({'message': str(e.msg)}, status=e.status) query_params = request.query_params.get('search') if query_params is not None: try: data = filter_list( channel_name=get_channel_name(request), object_type='model', data=data, query_params=query_params) except LedgerError as e: return Response({'message': str(e.msg)}, status=e.status) return Response(data, status=status.HTTP_200_OK)
def retrieve(self, request, *args, **kwargs): lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field key = self.kwargs[lookup_url_kwarg] try: data = self._retrieve(get_channel_name(request), key) except LedgerError as e: return Response({'message': str(e.msg)}, status=e.status) else: return Response(data, status=status.HTTP_200_OK)
def get(self, request, *args, **kwargs): channel_name = get_channel_name(request) channel = settings.LEDGER_CHANNELS[channel_name] return Response({ 'host': settings.DEFAULT_DOMAIN, 'channel': channel_name, 'config': { 'model_export_enabled': channel['model_export_enabled'], } })
def create(self, request, *args, **kwargs): try: data = create_compute_plan(get_channel_name(request), dict(request.data)) except LedgerError as e: error = {'message': str(e.msg), 'key': data['key']} return Response(error, status=e.status) # send successful response headers = self.get_success_headers(data) status = get_success_create_code() return Response(data, status=status, headers=headers)
def list(self, request, *args, **kwargs): try: nodes = query_ledger(get_channel_name(request), fcn=self.ledger_query_call) except LedgerError as e: return Response({'message': str(e.msg)}, status=e.status) current_node_id = get_owner() for node in nodes: node.update({ 'is_current': node['id'] == current_node_id, }) return Response(nodes, status=status.HTTP_200_OK)
def retrieve(self, request, *args, **kwargs): lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field key = self.kwargs[lookup_url_kwarg] validate_key(key) try: data = get_object_from_ledger(get_channel_name(request), key, 'queryComputePlan') except LedgerError as e: return Response({'message': str(e.msg)}, status=e.status) else: return Response(data, status=status.HTTP_200_OK)
def cancel(self, request, *args, **kwargs): lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field key = self.kwargs[lookup_url_kwarg] validate_key(key) try: compute_plan = invoke_ledger(get_channel_name(request), fcn='cancelComputePlan', args={'key': key}, only_key=False) except LedgerError as e: return Response({'message': str(e.msg)}, status=e.status) return Response(compute_plan, status=status.HTTP_200_OK)
def commit(self, serializer, request): # create on local db try: instance = self.perform_create(serializer) except Exception as e: raise Exception(e.args) # init ledger serializer ledger_data = { 'test_data_sample_keys': request.data.get('test_data_sample_keys') or [], 'test_data_manager_key': request.data.get('test_data_manager_key'), 'name': request.data.get('name'), 'permissions': request.data.get('permissions'), 'metrics_name': request.data.get('metrics_name'), 'metadata': request.data.get('metadata') } ledger_data.update({'instance': instance}) ledger_serializer = LedgerObjectiveSerializer( data=ledger_data, context={'request': request}) if not ledger_serializer.is_valid(): # delete instance instance.delete() raise ValidationError(ledger_serializer.errors) # create on ledger try: data = ledger_serializer.create(get_channel_name(request), ledger_serializer.validated_data) except LedgerTimeout as e: raise LedgerException('timeout', e.status) except LedgerConflict as e: raise ValidationException(e.msg, e.key, e.status) except LedgerError as e: instance.delete() raise LedgerException(str(e.msg), e.status) except Exception: instance.delete() raise d = dict(serializer.data) d.update(data) return d
def _create(self, request): key = uuid.uuid4() data = { 'key': key, 'objective_key': request.data.get('objective_key'), 'traintuple_key': request.data.get('traintuple_key'), 'data_manager_key': request.data.get('data_manager_key'), 'test_data_sample_keys': request.data.get('test_data_sample_keys'), 'tag': request.data.get('tag', ''), 'metadata': request.data.get('metadata') } serializer = self.get_serializer(data=data) serializer.is_valid(raise_exception=True) return self.commit(serializer, get_channel_name(request))
def _create(self, request): key = uuid.uuid4() data = { 'key': key, 'algo_key': request.data.get('algo_key'), 'rank': request.data.get('rank'), 'compute_plan_key': request.data.get('compute_plan_key'), 'in_models_keys': request.data.get('in_models_keys'), 'worker': request.data.get('worker'), 'tag': request.data.get('tag', ''), 'metadata': request.data.get('metadata') } serializer = self.get_serializer(data=data) serializer.is_valid(raise_exception=True) return self.commit(serializer, get_channel_name(request))
def _create(self, request): key = uuid.uuid4() data = { 'key': key, 'algo_key': request.data.get('algo_key'), 'data_manager_key': request.data.get('data_manager_key'), 'rank': request.data.get('rank'), 'compute_plan_key': request.data.get('compute_plan_key'), 'in_models_keys': request.data.get('in_models_keys'), # list of train data keys (which are stored in the train worker node) 'train_data_sample_keys': request.data.get('train_data_sample_keys'), 'tag': request.data.get('tag', ''), 'metadata': request.data.get('metadata') } serializer = self.get_serializer(data=data) serializer.is_valid(raise_exception=True) return self.commit(serializer, get_channel_name(request))
def commit(self, serializer, request): # create on db instance = self.perform_create(serializer) ledger_data = { 'name': request.data.get('name'), 'permissions': request.data.get('permissions'), 'metadata': request.data.get('metadata') } # init ledger serializer ledger_data.update({'instance': instance}) ledger_serializer = LedgerAlgoSerializer(data=ledger_data, context={'request': request}) if not ledger_serializer.is_valid(): # delete instance instance.delete() raise ValidationError(ledger_serializer.errors) # create on ledger try: data = ledger_serializer.create(get_channel_name(request), ledger_serializer.validated_data) except LedgerTimeout as e: if isinstance(serializer.data, list): key = [x['key'] for x in serializer.data] else: key = [serializer.data['key']] data = {'key': key, 'validated': False} raise LedgerException(data, e.status) except LedgerConflict as e: raise ValidationException(e.msg, e.key, e.status) except LedgerError as e: instance.delete() raise LedgerException(str(e.msg), e.status) except Exception: instance.delete() raise d = dict(serializer.data) d.update(data) return d
def update_ledger(self, request, *args, **kwargs): lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field key = self.kwargs[lookup_url_kwarg] validate_key(key) serializer = self.get_serializer(data=dict(request.data)) serializer.is_valid(raise_exception=True) # update compute plan in ledger try: data = serializer.update(get_channel_name(request), key, serializer.validated_data) except LedgerError as e: error = {'message': str(e.msg), 'key': key} return Response(error, status=e.status) # send successful response headers = self.get_success_headers(data) status = get_success_create_code() return Response(data, status=status, headers=headers)
def leaderboard(self, request, *args, **kwargs): lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field key = self.kwargs[lookup_url_kwarg] validate_key(key) sort = request.query_params.get('sort', 'desc') try: validate_sort(sort) except Exception as e: return Response({'message': str(e)}, status=status.HTTP_400_BAD_REQUEST) try: leaderboard = query_ledger(get_channel_name(request), fcn='queryObjectiveLeaderboard', args={ 'objective_key': key, 'ascendingOrder': sort == 'asc', }) except LedgerError as e: return Response({'message': str(e.msg)}, status=e.status) return Response(leaderboard, status=status.HTTP_200_OK)