def test_removal_skips_inputs(self):
run = ContainerRun(id=42, state=ContainerRun.COMPLETE)
dataset = Dataset(id=43)
argument = ContainerArgument(type=ContainerArgument.INPUT)
run.datasets.create(dataset=dataset, argument=argument)
expected_plan = {'ContainerRuns': {run}}
plan = run.build_removal_plan()
self.assertEqual(expected_plan, strip_removal_plan(plan))
def __init__(self, *args, **kwargs):
super(BulkDatasetUpdateForm, self).__init__(*args, **kwargs)
self.dataset = Dataset()
self.status = 0
def datasets_add_bulk(request):
"""
Add datasets in bulk to db. Redirect to /datasets_bulk view so user can examine upload
status of each dataset.
"""
# Redirect to page to allow user to view status of added datasets.
c = {}
if request.method == 'POST':
try:
# Add new datasets.
bulk_add_dataset_form = BulkAddDatasetForm(data=request.POST,
files=request.FILES)
isok = bulk_add_dataset_form.is_valid()
if isok:
CDT_obj, add_results = bulk_add_dataset_form.create_datasets(
request.user)
isok = len(add_results) > 0
if not isok:
# give up and let user try again
t = loader.get_template('librarian/datasets_add_bulk.html')
c = {'bulkAddDatasetForm': bulk_add_dataset_form}
return HttpResponse(t.render(c, request))
# Generate response.
uploaded_files = bulk_add_dataset_form.cleaned_data[
"dataset_files"]
if len(uploaded_files) != len(add_results):
raise RuntimeError("List length mismatch")
t = loader.get_template('librarian/datasets_bulk.html')
bulk_display_results = []
# Fill in default values for the form fields
for add_result, upload_info in zip(add_results, uploaded_files):
display_result = {}
if isinstance(add_result, dict):
# dataset is invalid
display_result["name"] = add_result["name"]
display_result["description"] = ""
display_result["orig_filename"] = add_result["name"]
display_result["filesize"] = add_result["size"]
display_result["md5"] = ""
display_result["id"] = ""
display_result["is_valid"] = False
else:
display_result["name"] = add_result.name
display_result["description"] = add_result.description
# This is the original filename as uploaded by the client, not the filename as stored
# on the file server.
display_result["orig_filename"] = upload_info[1].name
display_result[
"filesize"] = add_result.get_formatted_filesize()
display_result["md5"] = add_result.compute_md5()
display_result["id"] = add_result.id
display_result["is_valid"] = True
bulk_display_results.append(display_result)
BulkDatasetUpdateFormSet = formset_factory(
form=BulkDatasetUpdateForm, max_num=len(bulk_display_results))
bulk_dataset_update_formset = BulkDatasetUpdateFormSet(
initial=bulk_display_results)
# Fill in the attributes that are not fields in the form
# These are not set by the BulkDatasetUpdateFormSet(initial=...) parameter
for dataset_form, display_result, add_result in zip(
bulk_dataset_update_formset, bulk_display_results,
add_results):
if display_result["is_valid"]:
dataset_form.dataset = add_result
dataset_form.status = BulkDatasetDisplay.STATUS_SUCCESS
else:
dataset_form.dataset = Dataset()
dataset_form.non_field_errors = add_result["errstr"]
dataset_form.status = BulkDatasetDisplay.STATUS_FAIL
# finally, add some other pertinent information which the template will display
num_files_added = sum(
[a["is_valid"] for a in bulk_display_results])
c["bulk_dataset_formset"] = bulk_dataset_update_formset
c["num_files_selected"] = len(add_results)
c["num_files_added"] = num_files_added
c["cdt_typestr"] = "Unstructured" if CDT_obj is None else CDT_obj
except ValidationError as e:
LOGGER.exception(e.message)
bulk_add_dataset_form.add_error(None, e)
c.update({'bulkAddDatasetForm': bulk_add_dataset_form})
else: # return an empty form for the user to fill in
t = loader.get_template('librarian/datasets_add_bulk.html')
c.update({'bulkAddDatasetForm': BulkAddDatasetForm()})
return HttpResponse(t.render(c, request))
def datasets_add_archive(request):
"""
Add datasets in bulk to db from an archive file (zip or tarfile).
Redirect to /datasets_bulk view so user can examine upload status of each dataset.
"""
c = {}
# If we got posted to, try to create DB entries
if request.method == 'POST':
try:
archive_add_dataset_form = ArchiveAddDatasetForm(
data=request.POST, files=request.FILES)
# Try to retrieve new datasets. If this fails, we return to our current page
is_ok = archive_add_dataset_form.is_valid()
if is_ok:
CDT_obj, add_results = archive_add_dataset_form.create_datasets(
request.user)
is_ok = len(add_results) > 0
if not is_ok:
# give up and let user try again
t = loader.get_template('librarian/datasets_add_archive.html')
c = {'archiveAddDatasetForm': archive_add_dataset_form}
return HttpResponse(t.render(c, request))
# have some files in the archive, lets display them
# NOTE: at this point, we have a list of files in the archive.
# some files might be legit, others not.
# we have to cobble together information from add_results and the form cleaned data
# for display.
uploaded_files = archive_add_dataset_form.cleaned_data[
"dataset_file"]
if len(uploaded_files) != len(add_results):
raise RuntimeError("List length mismatch")
t = loader.get_template('librarian/datasets_bulk.html')
# Now have add_results, a list of elements e, where e is either
# a dataset if the dataset was successfully created
# or
# a dict if a dataset was not successfully created
# Generate a response
archive_display_results = []
# Fill in default values for the form fields
for add_result, upload_info in zip(add_results, uploaded_files):
display_result = {}
if isinstance(add_result, dict):
# the dataset is invalid
display_result["name"] = add_result["name"]
display_result["description"] = ""
display_result["orig_filename"] = add_result["name"]
display_result["filesize"] = add_result["size"]
display_result["md5"] = ""
display_result["id"] = ""
display_result["is_valid"] = False
else:
display_result["name"] = add_result.name
display_result["description"] = add_result.description
# This is the original filename as uploaded by the client, not the filename as stored
# on the file server.
display_result["orig_filename"] = upload_info[1].name
display_result[
"filesize"] = add_result.get_formatted_filesize()
display_result["md5"] = add_result.compute_md5()
display_result["id"] = add_result.id
display_result["is_valid"] = True
archive_display_results.append(display_result)
# now create forms from the display results.
BulkDatasetUpdateFormSet = formset_factory(
form=BulkDatasetUpdateForm,
max_num=len(archive_display_results))
bulk_dataset_update_formset = BulkDatasetUpdateFormSet(
initial=archive_display_results)
# Fill in the attributes that are not fields in the form
# These are not set by the BulkDatasetUpdateFormSet(initial=...) parameter,
# so we have to tweak the forms after they have been created
for dataset_form, display_result, add_result in zip(
bulk_dataset_update_formset, archive_display_results,
add_results):
if display_result["is_valid"]:
dataset_form.dataset = add_result
dataset_form.status = BulkDatasetDisplay.STATUS_SUCCESS
else:
dataset_form.dataset = Dataset()
dataset_form.non_field_errors = add_result["errstr"]
dataset_form.status = BulkDatasetDisplay.STATUS_FAIL
# finally, add some other pertinent information which the template will display
num_files_added = sum(
[a["is_valid"] for a in archive_display_results])
c["bulk_dataset_formset"] = bulk_dataset_update_formset
c["num_files_selected"] = len(add_results)
c["num_files_added"] = num_files_added
c["cdt_typestr"] = "Unstructured" if CDT_obj is None else CDT_obj
except ValidationError as e:
LOGGER.exception(e.message)
archive_add_dataset_form.add_error(None, e)
t = loader.get_template('librarian/datasets_add_archive.html')
c.update({'archiveAddDatasetForm': archive_add_dataset_form})
else: # return an empty form for the user to fill in
t = loader.get_template('librarian/datasets_add_archive.html')
c['archiveAddDatasetForm'] = ArchiveAddDatasetForm()
return HttpResponse(t.render(c, request))