def _create(self, request, data_manager_keys, test_only, dryrun):
if not data_manager_keys:
raise Exception("missing or empty field 'data_manager_keys'")
self.check_datamanagers(data_manager_keys) # can raise
computed_data = self.compute_data(request)
serializer = self.get_serializer(data=computed_data, many=True)
try:
serializer.is_valid(raise_exception=True)
except Exception as e:
pkhashes = [x['pkhash'] for x in computed_data]
st = status.HTTP_400_BAD_REQUEST
if find_primary_key_error(e):
st = status.HTTP_409_CONFLICT
raise ValidationException(e.args, pkhashes, st)
else:
if dryrun:
return self.handle_dryrun(computed_data, data_manager_keys)
# create on ledger + db
ledger_data = {'test_only': test_only,
'data_manager_keys': data_manager_keys}
data, st = self.commit(serializer, ledger_data)
return data, st
def create(self, request, *args, **kwargs):
data = request.data
dryrun = data.get('dryrun', False)
data_opener = data.get('data_opener')
pkhash = get_hash(data_opener)
serializer = self.get_serializer(data={
'pkhash': pkhash,
'data_opener': data_opener,
'description': data.get('description'),
'name': data.get('name'),
})
try:
serializer.is_valid(raise_exception=True)
except Exception as e:
st = status.HTTP_400_BAD_REQUEST
if find_primary_key_error(e):
st = status.HTTP_409_CONFLICT
return Response({'message': e.args, 'pkhash': pkhash}, status=st)
else:
if dryrun:
return self.dryrun(data_opener)
# create on db
try:
instance = self.perform_create(serializer)
except Exception as e:
return Response({'message': e.args},
status=status.HTTP_400_BAD_REQUEST)
else:
# init ledger serializer
ledger_serializer = LedgerDataManagerSerializer(data={'name': data.get('name'),
'permissions': data.get('permissions'),
'type': data.get('type'),
'objective_keys': data.getlist('objective_keys'),
'instance': instance},
context={'request': request})
if not ledger_serializer.is_valid():
# delete instance
instance.delete()
raise ValidationError(ledger_serializer.errors)
# create on ledger
data, st = ledger_serializer.create(ledger_serializer.validated_data)
if st not in (status.HTTP_201_CREATED, status.HTTP_202_ACCEPTED, status.HTTP_408_REQUEST_TIMEOUT):
return Response(data, status=st)
headers = self.get_success_headers(serializer.data)
d = dict(serializer.data)
d.update(data)
return Response(d, status=st, headers=headers)
def create(self, request, *args, **kwargs):
data = request.data
file = data.get('file')
pkhash = get_hash(file)
serializer = self.get_serializer(data={
'pkhash': pkhash,
'file': file,
'description': data.get('description')
})
try:
serializer.is_valid(raise_exception=True)
except Exception as e:
st = status.HTTP_400_BAD_REQUEST
if find_primary_key_error(e):
st = status.HTTP_409_CONFLICT
return Response({'message': e.args, 'pkhash': pkhash}, status=st)
else:
# create on db
try:
instance = self.perform_create(serializer)
except Exception as exc:
return Response({'message': exc.args},
status=status.HTTP_400_BAD_REQUEST)
else:
# init ledger serializer
ledger_serializer = LedgerAlgoSerializer(data={'name': data.get('name'),
'permissions': data.get('permissions', 'all'),
'instance': instance},
context={'request': request})
if not ledger_serializer.is_valid():
# delete instance
instance.delete()
raise ValidationError(ledger_serializer.errors)
# create on ledger
data, st = ledger_serializer.create(ledger_serializer.validated_data)
if st not in (status.HTTP_201_CREATED, status.HTTP_202_ACCEPTED, status.HTTP_408_REQUEST_TIMEOUT):
return Response(data, status=st)
headers = self.get_success_headers(serializer.data)
d = dict(serializer.data)
d.update(data)
return Response(d, status=st, headers=headers)
def _create(self, request, file):
pkhash = get_hash(file)
serializer = self.get_serializer(data={
'pkhash': pkhash,
'file': file,
'description': request.data.get('description')
})
try:
serializer.is_valid(raise_exception=True)
except Exception as e:
st = status.HTTP_400_BAD_REQUEST
if find_primary_key_error(e):
st = status.HTTP_409_CONFLICT
raise ValidationException(e.args, pkhash, st)
else:
# create on ledger + db
return self.commit(serializer, request)
def _create(self, request, data_manager_keys, test_only):
# compute_data will uncompress data archives to paths which will be
# hardlinked thanks to datasample pre_save signal.
# In all other cases, we need to remove those references.
if not data_manager_keys:
raise Exception("missing or empty field 'data_manager_keys'")
self.check_datamanagers(data_manager_keys) # can raise
paths_to_remove = []
try:
# will uncompress data archives to paths
computed_data = self.compute_data(request, paths_to_remove)
serializer = self.get_serializer(data=computed_data, many=True)
try:
serializer.is_valid(raise_exception=True)
except Exception as e:
pkhashes = [x['pkhash'] for x in computed_data]
st = status.HTTP_400_BAD_REQUEST
if find_primary_key_error(e):
st = status.HTTP_409_CONFLICT
raise ValidationException(e.args, pkhashes, st)
else:
# create on ledger + db
ledger_data = {
'test_only': test_only,
'data_manager_keys': data_manager_keys
}
data, st = self.commit(serializer,
ledger_data) # pre_save signal executed
return data, st
finally:
for gpath in paths_to_remove:
shutil.rmtree(gpath, ignore_errors=True)
def create(self, request, *args, **kwargs):
"""
Create a new Objective \n
TODO add info about what has to be posted\n
- Example with curl (on localhost): \n
curl -u username:password -H "Content-Type: application/json"\
-X POST\
-d '{"name": "tough objective", "permissions": "all", "metrics_name": 'accuracy', "test_data":
["data_5c1d9cd1c2c1082dde0921b56d11030c81f62fbb51932758b58ac2569dd0b379",
"data_5c1d9cd1c2c1082dde0921b56d11030c81f62fbb51932758b58ac2569dd0b389"],\
"files": {"description.md": '#My tough objective',\
'metrics.py': 'def AUC_score(y_true, y_pred):\n\treturn 1'}}'\
http://127.0.0.1:8000/substrapp/objective/ \n
Use double quotes for the json, simple quotes don't work.\n
- Example with the python package requests (on localhost): \n
requests.post('http://127.0.0.1:8000/objective/',
#auth=('username', 'password'),
data={'name': 'MSI classification', 'permissions': 'all', 'metrics_name': 'accuracy', 'test_data_sample_keys': ['da1bb7c31f62244c0f3a761cc168804227115793d01c270021fe3f7935482dcc']},
files={'description': open('description.md', 'rb'), 'metrics': open('metrics.py', 'rb')},
headers={'Accept': 'application/json;version=0.0'}) \n
---
response_serializer: ObjectiveSerializer
"""
data = request.data
dryrun = data.get('dryrun', False)
description = data.get('description')
test_data_manager_key = request.data.get(
'test_data_manager_key',
request.POST.get('test_data_manager_key', ''))
try:
test_data_sample_keys = request.data.getlist(
'test_data_sample_keys', [])
except:
test_data_sample_keys = request.data.get(
'test_data_sample_keys',
request.POST.getlist('test_data_sample_keys', []))
metrics = data.get('metrics')
pkhash = get_hash(description)
serializer = self.get_serializer(data={
'pkhash': pkhash,
'metrics': metrics,
'description': description
})
try:
serializer.is_valid(raise_exception=True)
except ValidationError as e:
st = status.HTTP_400_BAD_REQUEST
if find_primary_key_error(e):
st = status.HTTP_409_CONFLICT
return Response({'message': e.args, 'pkhash': pkhash}, status=st)
if dryrun:
try:
metrics_path = os.path.join(getattr(settings, 'DRYRUN_ROOT'),
f'metrics_{pkhash}.py')
with open(metrics_path, 'wb') as metrics_file:
metrics_file.write(metrics.open().read())
task = compute_dryrun.apply_async(
(metrics_path, test_data_manager_key, pkhash),
queue=f"{settings.LEDGER['name']}.dryrunner")
except Exception as e:
return Response(
{
'message':
f'Could not launch objective creation with dry-run on this instance: {str(e)}'
},
status=status.HTTP_400_BAD_REQUEST)
current_site = getattr(settings, "DEFAULT_DOMAIN")
task_route = f'{current_site}{reverse("substrapp:task-detail", args=[task.id])}'
msg = f'Your dry-run has been taken in account. You can follow the task execution on {task_route}'
return Response({
'id': task.id,
'message': msg
},
status=status.HTTP_202_ACCEPTED)
# create on db
try:
instance = self.perform_create(serializer)
except IntegrityError as exc:
try:
pkhash = re.search(r'\(pkhash\)=\((\w+)\)',
exc.args[0]).group(1)
except BaseException:
pkhash = ''
finally:
return Response(
{
'message':
'A objective with this description file already exists.',
'pkhash': pkhash
},
status=status.HTTP_409_CONFLICT)
except Exception as exc:
return Response({'message': exc.args},
status=status.HTTP_400_BAD_REQUEST)
# init ledger serializer
ledger_serializer = LedgerObjectiveSerializer(
data={
'test_data_sample_keys': test_data_sample_keys,
'test_data_manager_key': test_data_manager_key,
'name': data.get('name'),
'permissions': data.get('permissions'),
'metrics_name': data.get('metrics_name'),
'instance': instance
},
context={'request': request})
if not ledger_serializer.is_valid():
# delete instance
instance.delete()
raise ValidationError(ledger_serializer.errors)
# create on ledger
data, st = ledger_serializer.create(ledger_serializer.validated_data)
if st not in (status.HTTP_201_CREATED, status.HTTP_202_ACCEPTED,
status.HTTP_408_REQUEST_TIMEOUT):
return Response(data, status=st)
headers = self.get_success_headers(serializer.data)
d = dict(serializer.data)
d.update(data)
return Response(d, status=st, headers=headers)