def list(self, request, *args, **kwargs):
try:
data = query_ledger(get_channel_name(request),
fcn='queryDataManagers',
args=[])
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
# parse filters
query_params = request.query_params.get('search')
if query_params is not None:
try:
data = filter_list(channel_name=get_channel_name(request),
object_type='dataset',
data=data,
query_params=query_params)
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
for data_manager in data:
replace_storage_addresses(request, data_manager)
return Response(data, status=status.HTTP_200_OK)
def list(self, request, *args, **kwargs):
try:
nodes = query_ledger(get_channel_name(request), fcn=self.ledger_query_call)
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
current_node_id = get_owner()
for node in nodes:
node.update({
'is_current': node['id'] == current_node_id,
})
return Response(nodes, status=status.HTTP_200_OK)
def list(self, request, *args, **kwargs):
try:
data = query_ledger(get_channel_name(request), fcn='queryModels', args=[])
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
query_params = request.query_params.get('search')
if query_params is not None:
try:
data = filter_list(
channel_name=get_channel_name(request),
object_type='model',
data=data,
query_params=query_params)
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
return Response(data, status=status.HTTP_200_OK)
def leaderboard(self, request, *args, **kwargs):
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
key = self.kwargs[lookup_url_kwarg]
validate_key(key)
sort = request.query_params.get('sort', 'desc')
try:
validate_sort(sort)
except Exception as e:
return Response({'message': str(e)},
status=status.HTTP_400_BAD_REQUEST)
try:
leaderboard = query_ledger(get_channel_name(request),
fcn='queryObjectiveLeaderboard',
args={
'objective_key': key,
'ascendingOrder': sort == 'asc',
})
except LedgerError as e:
return Response({'message': str(e.msg)}, status=e.status)
return Response(leaderboard, status=status.HTTP_200_OK)
def filter_list(channel_name, object_type, data, query_params):
try:
filters = get_filters(query_params)
except Exception:
# TODO add better filters parsing to avoid this catch all
message = f'Malformed search filters: invalid syntax: {query_params}'
logger.exception(message)
raise exceptions.BadRequestError(message)
object_list = []
for user_filter in filters:
for filter_key, subfilters in user_filter.items():
if filter_key not in AUTHORIZED_FILTERS[object_type]:
raise exceptions.BadRequestError(
f'Malformed search filters: not authorized filter key {filter_key} for asset {object_type}')
# Will be appended in object_list after been filtered
filtered_list = data
if _same_nature(filter_key, object_type):
# Filter by own asset
if filter_key == 'model':
for attribute, val in subfilters.items():
filtered_list = [x for x in filtered_list if _get_model_tuple(x).get(attribute) in val]
elif filter_key == 'objective':
for attribute, val in subfilters.items():
if attribute == 'metrics': # specific to nested metrics
filtered_list = [x for x in filtered_list if x[attribute]['name'] in val]
else:
filtered_list = [x for x in filtered_list if x[attribute] in val]
else:
for attribute, val in subfilters.items():
filtered_list = [x for x in filtered_list if x.get(attribute) in val]
else:
# Filter by other asset
# Get other asset list
filtering_data = query_ledger(channel_name, fcn=FILTER_QUERIES[filter_key], args=[])
filtering_data = filtering_data if filtering_data else []
if filter_key in ('algo', 'composite_algo', 'aggregate_algo'):
for attribute, val in subfilters.items():
filtering_data = [x for x in filtering_data if x[attribute] in val]
keys = [x['key'] for x in filtering_data]
if object_type == 'model':
filtered_list = [x for x in filtered_list
if _get_model_tuple(x)['algo']['key'] in keys]
elif filter_key == 'model':
for attribute, val in subfilters.items():
filtering_data = [
x for x in filtering_data
if (
(
_get_model_tuple(x).get('out_model') and
_get_model_tuple(x)['out_model'][attribute] in val
) or (
_get_model_tuple(x).get('out_trunk_model') and
_get_model_tuple(x)['out_trunk_model'].get('out_model') and
_get_model_tuple(x)['out_trunk_model']['out_model'][attribute] in val
) or (
_get_model_tuple(x).get('out_head_model') and
_get_model_tuple(x)['out_head_model'].get('out_model') and
_get_model_tuple(x)['out_head_model']['out_model'][attribute] in val
)
)
]
if object_type in ['algo', 'composite_algo', 'aggregate_algo']:
keys = [_get_model_tuple(x)['algo']['key'] for x in filtering_data]
filtered_list = [x for x in filtered_list if x['key'] in keys]
elif object_type == 'dataset':
checksums = []
for x in filtering_data:
try:
checksums.append(x['testtuple']['dataset']['opener_checksum'])
except KeyError:
pass
try:
checksums.append(_get_model_tuple(x)['dataset']['opener_checksum'])
except KeyError:
pass
filtered_list = [
x for x in filtered_list
if x['opener']['checksum'] in checksums
]
elif object_type == 'objective':
keys = [
x['testtuple']['objective']['key'] for x in filtering_data
if x['testtuple'] and x['testtuple']['objective']
]
filtered_list = [x for x in filtered_list if x['key'] in keys]
elif filter_key == 'dataset':
for attribute, val in subfilters.items():
filtering_data = [x for x in filtering_data if x[attribute] in val]
keys = [x['key'] for x in filtering_data]
if object_type == 'model':
filtered_list = [x for x in filtered_list
if _get_model_tuple(x).get('dataset', {}).get('key', '') in keys]
elif object_type == 'objective':
objective_keys = [x['objective_key'] for x in filtering_data]
filtered_list = [x for x in filtered_list
if x['key'] in objective_keys or
(x['test_dataset'] and x['test_dataset']['data_manager_key'] in keys)]
elif filter_key == 'objective':
for attribute, val in subfilters.items():
if attribute == 'metrics': # specific to nested metrics
filtering_data = [x for x in filtering_data if x[attribute]['name'] in val]
else:
filtering_data = [x for x in filtering_data if x[attribute] in val]
keys = [x['key'] for x in filtering_data]
if object_type == 'model':
filtered_list = [
x for x in filtered_list
if (x['testtuple'] and x['testtuple']['objective'] and
x['testtuple']['objective']['key'] in keys)
]
elif object_type == 'dataset':
filtered_list = [x for x in filtered_list
if x['objective_key'] in keys]
object_list.append(filtered_list)
return flatten_without_duplicates(object_list)