def _retrieve(self, request, key):
validate_key(key)
# get instance from remote node
data = get_object_from_ledger(get_channel_name(request), key,
'queryDataset')
# do not cache if node has not process permission
if node_has_process_permission(data):
# try to get it from local db to check if description exists
try:
instance = self.get_object()
except Http404:
instance = None
finally:
# check if instance has description or data_opener
if not instance or not instance.description or not instance.data_opener:
instance = self.create_or_update_datamanager(
get_channel_name(request), instance, data, key)
# do not give access to local files address
serializer = self.get_serializer(instance, fields=('owner'))
data.update(serializer.data)
replace_storage_addresses(request, data)
return data
def list(self, request, *args, **kwargs):
try:
data = query_ledger(get_channel_name(request),
fcn='queryDataManagers',
args=[])
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
# parse filters
query_params = request.query_params.get('search')
if query_params is not None:
try:
data = filter_list(channel_name=get_channel_name(request),
object_type='dataset',
data=data,
query_params=query_params)
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
for data_manager in data:
replace_storage_addresses(request, data_manager)
return Response(data, status=status.HTTP_200_OK)
def _retrieve(self, request, key):
validate_key(key)
data = get_object_from_ledger(get_channel_name(request), key,
self.ledger_query_call)
# do not cache if node has not process permission
if node_has_process_permission(data):
# try to get it from local db to check if description exists
try:
instance = self.get_object()
except Http404:
instance = None
finally:
# check if instance has description
if not instance or not instance.description:
instance = self.create_or_update_algo(
get_channel_name(request), data, key)
# For security reason, do not give access to local file address
# Restrain data to some fields
# TODO: do we need to send creation date and/or last modified date ?
serializer = self.get_serializer(instance, fields=('owner'))
data.update(serializer.data)
replace_storage_addresses(request, data)
return data
def _create(self, request):
key = uuid.uuid4()
data = {
'key':
key,
'algo_key':
request.data.get('algo_key'),
'data_manager_key':
request.data.get('data_manager_key'),
'rank':
request.data.get('rank'),
'compute_plan_key':
request.data.get('compute_plan_key'),
'in_head_model_key':
request.data.get('in_head_model_key'),
'in_trunk_model_key':
request.data.get('in_trunk_model_key'),
'out_trunk_model_permissions':
request.data.get('out_trunk_model_permissions'),
'train_data_sample_keys':
request.data.get('train_data_sample_keys'),
'tag':
request.data.get('tag', ''),
'metadata':
request.data.get('metadata')
}
serializer = self.get_serializer(data=data)
serializer.is_valid(raise_exception=True)
return self.commit(serializer, get_channel_name(request))
def update_ledger(self, request, *args, **kwargs):
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
key = self.kwargs[lookup_url_kwarg]
validate_key(key)
objective_key = request.data.get('objective_key')
args = {
'data_manager_key': key,
'objective_key': objective_key,
}
if settings.LEDGER_SYNC_ENABLED:
st = status.HTTP_200_OK
else:
st = status.HTTP_202_ACCEPTED
try:
data = ledger.assets.update_datamanager(get_channel_name(request),
args)
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
return Response(data, status=st)
def list(self, request, *args, **kwargs):
try:
data = query_ledger(get_channel_name(request), fcn='queryModels', args=[])
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
query_params = request.query_params.get('search')
if query_params is not None:
try:
data = filter_list(
channel_name=get_channel_name(request),
object_type='model',
data=data,
query_params=query_params)
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
return Response(data, status=status.HTTP_200_OK)
def retrieve(self, request, *args, **kwargs):
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
key = self.kwargs[lookup_url_kwarg]
try:
data = self._retrieve(get_channel_name(request), key)
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
else:
return Response(data, status=status.HTTP_200_OK)
def get(self, request, *args, **kwargs):
channel_name = get_channel_name(request)
channel = settings.LEDGER_CHANNELS[channel_name]
return Response({
'host': settings.DEFAULT_DOMAIN,
'channel': channel_name,
'config': {
'model_export_enabled': channel['model_export_enabled'],
}
})
def create(self, request, *args, **kwargs):
try:
data = create_compute_plan(get_channel_name(request),
dict(request.data))
except LedgerError as e:
error = {'message': str(e.msg), 'key': data['key']}
return Response(error, status=e.status)
# send successful response
headers = self.get_success_headers(data)
status = get_success_create_code()
return Response(data, status=status, headers=headers)
def list(self, request, *args, **kwargs):
try:
nodes = query_ledger(get_channel_name(request), fcn=self.ledger_query_call)
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
current_node_id = get_owner()
for node in nodes:
node.update({
'is_current': node['id'] == current_node_id,
})
return Response(nodes, status=status.HTTP_200_OK)
def retrieve(self, request, *args, **kwargs):
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
key = self.kwargs[lookup_url_kwarg]
validate_key(key)
try:
data = get_object_from_ledger(get_channel_name(request), key,
'queryComputePlan')
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
else:
return Response(data, status=status.HTTP_200_OK)
def cancel(self, request, *args, **kwargs):
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
key = self.kwargs[lookup_url_kwarg]
validate_key(key)
try:
compute_plan = invoke_ledger(get_channel_name(request),
fcn='cancelComputePlan',
args={'key': key},
only_key=False)
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
return Response(compute_plan, status=status.HTTP_200_OK)
def commit(self, serializer, request):
# create on local db
try:
instance = self.perform_create(serializer)
except Exception as e:
raise Exception(e.args)
# init ledger serializer
ledger_data = {
'test_data_sample_keys':
request.data.get('test_data_sample_keys') or [],
'test_data_manager_key':
request.data.get('test_data_manager_key'),
'name':
request.data.get('name'),
'permissions':
request.data.get('permissions'),
'metrics_name':
request.data.get('metrics_name'),
'metadata':
request.data.get('metadata')
}
ledger_data.update({'instance': instance})
ledger_serializer = LedgerObjectiveSerializer(
data=ledger_data, context={'request': request})
if not ledger_serializer.is_valid():
# delete instance
instance.delete()
raise ValidationError(ledger_serializer.errors)
# create on ledger
try:
data = ledger_serializer.create(get_channel_name(request),
ledger_serializer.validated_data)
except LedgerTimeout as e:
raise LedgerException('timeout', e.status)
except LedgerConflict as e:
raise ValidationException(e.msg, e.key, e.status)
except LedgerError as e:
instance.delete()
raise LedgerException(str(e.msg), e.status)
except Exception:
instance.delete()
raise
d = dict(serializer.data)
d.update(data)
return d
def _create(self, request):
key = uuid.uuid4()
data = {
'key': key,
'objective_key': request.data.get('objective_key'),
'traintuple_key': request.data.get('traintuple_key'),
'data_manager_key': request.data.get('data_manager_key'),
'test_data_sample_keys': request.data.get('test_data_sample_keys'),
'tag': request.data.get('tag', ''),
'metadata': request.data.get('metadata')
}
serializer = self.get_serializer(data=data)
serializer.is_valid(raise_exception=True)
return self.commit(serializer, get_channel_name(request))
def _create(self, request):
key = uuid.uuid4()
data = {
'key': key,
'algo_key': request.data.get('algo_key'),
'rank': request.data.get('rank'),
'compute_plan_key': request.data.get('compute_plan_key'),
'in_models_keys': request.data.get('in_models_keys'),
'worker': request.data.get('worker'),
'tag': request.data.get('tag', ''),
'metadata': request.data.get('metadata')
}
serializer = self.get_serializer(data=data)
serializer.is_valid(raise_exception=True)
return self.commit(serializer, get_channel_name(request))
def _create(self, request):
key = uuid.uuid4()
data = {
'key': key,
'algo_key': request.data.get('algo_key'),
'data_manager_key': request.data.get('data_manager_key'),
'rank': request.data.get('rank'),
'compute_plan_key': request.data.get('compute_plan_key'),
'in_models_keys': request.data.get('in_models_keys'),
# list of train data keys (which are stored in the train worker node)
'train_data_sample_keys': request.data.get('train_data_sample_keys'),
'tag': request.data.get('tag', ''),
'metadata': request.data.get('metadata')
}
serializer = self.get_serializer(data=data)
serializer.is_valid(raise_exception=True)
return self.commit(serializer, get_channel_name(request))
def commit(self, serializer, request):
# create on db
instance = self.perform_create(serializer)
ledger_data = {
'name': request.data.get('name'),
'permissions': request.data.get('permissions'),
'metadata': request.data.get('metadata')
}
# init ledger serializer
ledger_data.update({'instance': instance})
ledger_serializer = LedgerAlgoSerializer(data=ledger_data,
context={'request': request})
if not ledger_serializer.is_valid():
# delete instance
instance.delete()
raise ValidationError(ledger_serializer.errors)
# create on ledger
try:
data = ledger_serializer.create(get_channel_name(request),
ledger_serializer.validated_data)
except LedgerTimeout as e:
if isinstance(serializer.data, list):
key = [x['key'] for x in serializer.data]
else:
key = [serializer.data['key']]
data = {'key': key, 'validated': False}
raise LedgerException(data, e.status)
except LedgerConflict as e:
raise ValidationException(e.msg, e.key, e.status)
except LedgerError as e:
instance.delete()
raise LedgerException(str(e.msg), e.status)
except Exception:
instance.delete()
raise
d = dict(serializer.data)
d.update(data)
return d
def update_ledger(self, request, *args, **kwargs):
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
key = self.kwargs[lookup_url_kwarg]
validate_key(key)
serializer = self.get_serializer(data=dict(request.data))
serializer.is_valid(raise_exception=True)
# update compute plan in ledger
try:
data = serializer.update(get_channel_name(request), key,
serializer.validated_data)
except LedgerError as e:
error = {'message': str(e.msg), 'key': key}
return Response(error, status=e.status)
# send successful response
headers = self.get_success_headers(data)
status = get_success_create_code()
return Response(data, status=status, headers=headers)
def leaderboard(self, request, *args, **kwargs):
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
key = self.kwargs[lookup_url_kwarg]
validate_key(key)
sort = request.query_params.get('sort', 'desc')
try:
validate_sort(sort)
except Exception as e:
return Response({'message': str(e)},
status=status.HTTP_400_BAD_REQUEST)
try:
leaderboard = query_ledger(get_channel_name(request),
fcn='queryObjectiveLeaderboard',
args={
'objective_key': key,
'ascendingOrder': sort == 'asc',
})
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
return Response(leaderboard, status=status.HTTP_200_OK)
