def check_for_existing_layer(dataverse_info):
"""
Using the dataverse_info information
Do the datafile_id and dataverse_installation_name
in dataverse_info match an existing DataverseLayerMetadata object?
Yes: return first matching DataverseLayerMetadata object
No: return None
"""
assert isinstance(dataverse_info, dict)\
, "dataverse_info must be an instance of a dict. Found type: %s" % type(dataverse_info)
# Validate the data
f = DataverseLayerMetadataValidationForm(dataverse_info)
if not f.is_valid():
LOGGER.error('check_for_existing_layer. failed validation')
LOGGER.error('Errors: %s' % f.errors)
raise forms.ValidationError('Failed to validate dataverse_info data')
# Check for DataverseLayerMetadata objects with
# the same "datafile_id" AND "dataverse_installation_name"
l = DataverseLayerMetadata.objects.filter(\
datafile_id=f.cleaned_data.get('datafile_id'),
dataverse_installation_name=f.cleaned_data.get('dataverse_installation_name'))
# If DataverseLayerMetadata objects match, return the 1st one
# (Note: Not ~yet~ enforcing only 1 DataverseLayerMetadata per datafile_id.
# But will be only making one layer 'in practice'
# This is a late date chance where original discussions included multiple layers per file.
#
if l.count() > 0:
return l[0]
return None
def update_the_layer_metadata(dv_layer_metadata, dataverse_info):
"""Update the DataverseLayerMetadata object with given DataverseInfo"""
assert type(dv_layer_metadata) is DataverseLayerMetadata,\
('dv_layer_metadata must be a DataverseLayerMetadata'
' object. Found type: %s' % type(dv_layer_metadata))
assert type(dataverse_info) is dict,\
"dataverse_info must be type dict. Found type: %s" % type(dataverse_info)
# Validate the data
f = DataverseLayerMetadataValidationForm(dataverse_info)
if not f.is_valid():
raise forms.ValidationError('Failed to validate dataverse_info data')
# Update the metadata, field by field
for k, v in f.cleaned_data.items():
setattr(dv_layer_metadata, k, v)
dv_layer_metadata.save()
# Update the Layer object title and abstract
# Using the 'ShapefileImportDataForm' is a bit redundant,
# but not sure where updates will arise in the future
#
f2 = ShapefileImportDataForm(dataverse_info)
if not f2.is_valid():
raise forms.ValidationError(
'Failed to validate form_shapefile_import data')
dv_layer_metadata.map_layer.abstract = f2.cleaned_data['abstract']
dv_layer_metadata.map_layer.title = f2.cleaned_data['title']
dv_layer_metadata.map_layer.save()
def test_form_validation1(self):
msgt('(1) Test valid data')
validation_form = DataverseLayerMetadataValidationForm(self.test_data)
#print 'valid',validation_form.is_valid()
self.assertEqual(validation_form.is_valid(), True)
def add_dataverse_layer_metadata(saved_layer, dataverse_info):
"""
If a Layer has been created via Dataverse, create a DataverseLayerMetadata object.
fail: return None
success: return DataverseLayerMetadata object
"""
assert type(saved_layer) is Layer,\
"saved_layer must be type Layer. Found: %s" % type(Layer)
assert type(dataverse_info) is dict,\
"dataverse_info must be type dict. Found type: %s" % type(dataverse_info)
(success, create_datetime_obj_or_err_str) =\
DataverseLayerMetadataValidationForm.format_datafile_create_datetime(\
dataverse_info.get('datafile_create_datetime', None))
if success is False:
print('failed to format datetime', create_datetime_obj_or_err_str)
LOGGER.error('Invalid "datafile_create_datetime"\n%s',
create_datetime_obj_or_err_str)
return None
dataverse_info['datafile_create_datetime'] = create_datetime_obj_or_err_str
f = DataverseLayerMetadataValidationForm(dataverse_info)
if not f.is_valid():
print('failed validation')
print(f.errors)
LOGGER.error(('Unexpected form validation error'
' in add_dataverse_layer_metadata. dvn import: %s'),\
f.errors)
return None
# Create the DataverseLayerMetadata object
layer_metadata = f.save(commit=False)
# Add the related Layer object
layer_metadata.map_layer = saved_layer
# Save it!!
layer_metadata.save()
is_linked, err_msg_or_None = link_layer_permissions(layer_metadata)
print(is_linked, err_msg_or_None)
return layer_metadata
def test_form_validation2(self):
msgt('(2) Test invalid data for: [dataset_id, return_to_dataverse_url]')
tdata = self.test_data.copy()
tdata['dataset_id'] = '11z'
tdata['return_to_dataverse_url'] = 'ha'
#msg(tdata)
validation_form = DataverseLayerMetadataValidationForm(tdata)
#msg('valid: %s' % validation_form.is_valid())
self.assertEqual(validation_form.is_valid(), False)
msg('check for attributes in error')
err_keys = validation_form.errors.keys()
msg(err_keys)
self.assertEqual('return_to_dataverse_url' in err_keys, True)
self.assertEqual('dataset_id' in err_keys, True)
msg('check for err messages')
err_msgs = validation_form.errors.values()
msg(err_msgs)
self.assertEqual([u'Enter a valid URL.'] in err_msgs, True)
self.assertEqual([u'Enter a whole number.'] in err_msgs, True)
def test_form_validation3(self):
msgt('(3) Does form validate with the Geoconnect GISDataFile object turned to a dict--it has extra fields')
tdata = self.test_data.copy()
tdata['gis_scratch_work_directory'] = 'geoconnect/test_setup/gis_scratch_work/2014-0911-1622__1'
tdata['map_layer'] = 'Another extra field...'
validation_form = DataverseLayerMetadataValidationForm(tdata)
validation_form.is_valid()
msg(validation_form.errors)
self.assertEqual(validation_form.is_valid(), True)
dvinfo_obj = validation_form.save(commit=False)
self.assertEqual(type(dvinfo_obj), DataverseLayerMetadata)
msg('yes, converts into a DataverseMetadata object (minus the map_layer)')
def view_upload_table_and_join_layer(request):
"""
Upload a tabular file originating from Dataverse/Geoconnect and join it to a layer.
- Check if the Dataverse Metadata is valid
- No, error
- Does a layer already exist for this file?
- If yes, return it
- Check if the Table Join POST data is valid
- No error
- Attempt to Join the Table
- Fail, error
- Create a DataverseInfo object and attach it to the layer
"""
if request.method != 'POST':
return HttpResponse("Invalid Request", mimetype="text/plain", status=500)
post_data_dict = request.POST.dict()
msg('step 1')
LOGGER.info(('Upload tabular file from DV/Geoconnect and join to a layer.'
'\nStep 1: Is the Dataverse Layer Metadata valid?'))
# -------------------------------------------
# Is the Dataverse Layer Metadata valid?
# -------------------------------------------
form_dv_metadata = DataverseLayerMetadataValidationForm(post_data_dict)
if not form_dv_metadata.is_valid():
LOGGER.error(('check_for_existing_layer. failed validation'
'\nErrors: %s'), form_dv_metadata.errors)
#raise forms.ValidationError('Failed to validate dataverse_info data')
json_msg = MessageHelperJSON.get_json_fail_msg(\
'Failed to validate dataverse_info data',
data_dict=form_dv_metadata.errors)
return HttpResponse(json_msg, mimetype="application/json", status=400)
# -------------------------------------------
# Does a layer already exist for this DataverseInfo?
# -------------------------------------------
msg('step 2')
LOGGER.info('Step 2: Does a layer already exist for this DataverseInfo?')
existing_dv_layer_metadata = \
retrieve_dataverse_layer_metadata_by_kwargs_installation_and_file_id(**post_data_dict)
#-----------------------------------------------------------
# A layer was found!
# Update the DataverseLayerMetadata and return the layer.
#-----------------------------------------------------------
if existing_dv_layer_metadata:
msg("Found existing layer!")
LOGGER.info("Found existing layer!")
metadata_dict = get_layer_and_join_metadata(existing_dv_layer_metadata.map_layer)
json_msg = MessageHelperJSON.get_json_msg(success=True,\
msg='A layer already exists for the join.',\
data_dict=metadata_dict)
return HttpResponse(status=200, content=json_msg, content_type="application/json")
# -------------------------------------------
# Is the Upload and join info valid?
# -------------------------------------------
msg('step 3')
LOGGER.info("Step 3: Is the Upload and join info valid?")
form_upload_and_join = TableUploadAndJoinRequestForm(post_data_dict, request.FILES)
if not form_upload_and_join.is_valid():
json_msg = MessageHelperJSON.get_json_fail_msg(\
"Invalid Data for Upload and Join: %s" % form_upload_and_join.errors)
return HttpResponse(status=400, content=json_msg, content_type="application/json")
# ----------------------------------------------------
# Does the DataTable join column need to be char?
# - Check if the existing target join column is char?
# ----------------------------------------------------
(check_worked, force_char_convert) = ColumnHelper.is_char_column_conversion_recommended(\
form_upload_and_join.cleaned_data['layer_name'],\
form_upload_and_join.cleaned_data['layer_attribute'])
if not check_worked:
err_msg = 'Could not check the target column type'
LOGGER.error(err_msg)
json_msg = MessageHelperJSON.get_json_fail_msg(err_msg)
return HttpResponse(json_msg, mimetype="application/json", status=400)
if force_char_convert: # It is, make sure the Datatable col will be char
force_char_column = post_data_dict['table_attribute']
else: # Nope, let the datatable col stand, good or bad
force_char_column = None
# ----------------------------------------------------
# Attempt to upload the table
# ----------------------------------------------------
LOGGER.info("Step 4: Attempt to UPLOAD the table")
(success, data_table_or_error) = attempt_datatable_upload_from_request_params(request,\
request.user,\
is_dataverse_db=True,\
force_char_column=force_char_column)
if not success:
json_msg = MessageHelperJSON.get_json_fail_msg(data_table_or_error)
return HttpResponse(json_msg, mimetype="application/json", status=400)
# ----------------------------------------------------
# Attempt to join the table
# ----------------------------------------------------
LOGGER.info("Step 5: Prepare to JOIN the table")
new_datatable = data_table_or_error
join_props = request.POST.copy()
# Update attributes for the join, including the name of the new DataTable
#
join_props['table_name'] = data_table_or_error.table_name
original_table_attribute = join_props['table_attribute']
sanitized_table_attribute = standardize_column_name(original_table_attribute)
join_props['table_attribute'] = sanitized_table_attribute
# ---------------------------------
# Make the join!
# ---------------------------------
LOGGER.info("Step 6: Make the JOIN to the table")
(success, tablejoin_obj_or_err_msg) = \
attempt_tablejoin_from_request_params(join_props, request.user)
if not success: # FAILED!
new_datatable.delete() # remove the datatable
LOGGER.error('Failed join!: %s', tablejoin_obj_or_err_msg)
json_msg = MessageHelperJSON.get_json_fail_msg(tablejoin_obj_or_err_msg)
return HttpResponse(json_msg, mimetype="application/json", status=400)
# SUCCESS!
#
new_tablejoin = tablejoin_obj_or_err_msg
new_layer = new_tablejoin.join_layer
# ----------------------------------------------------
# Make a new DataverseInfo object and attach it to the Layer
# ----------------------------------------------------
LOGGER.info('Step 7: Make a new DataverseInfo object and attach it to the Layer')
(object_created, err_msg_or_dv_metadata) = create_dataverse_metadata(\
new_layer, post_data_dict)
if object_created is False:
json_msg = MessageHelperJSON.get_json_fail_msg(err_msg_or_dv_metadata)
return HttpResponse(status=400, content=json_msg, content_type="application/json")
# ------------------------------
# We made it! Send back a JSON response!
# ------------------------------
LOGGER.info('Step 8: We made it! Send back a JSON response!')
layer_metadata_obj = LayerMetadata(new_layer)
response_params = layer_metadata_obj.get_metadata_dict()
response_params.update(TableJoinResultForm.get_cleaned_data_from_table_join(new_tablejoin))
# Return the response!
json_msg = MessageHelperJSON.get_json_msg(success=True, msg='worked', data_dict=response_params)
msg('step 8b')
LOGGER.info('Step 8a: json_msg', json_msg)
msg('json_msg: %s' % json_msg)
return HttpResponse(status=200, content=json_msg, content_type="application/json")
def view_upload_lat_lng_table(request):
"""Upload a tabular file with lat/lng columns"""
msg('view_upload_lat_lng_table 0')
# -------------------------------------------
# Is it a POST?
# -------------------------------------------
if not request.method == 'POST':
json_msg = MessageHelperJSON.get_json_fail_msg("Unsupported Method")
return HttpResponse(json_msg, mimetype="application/json", status=500)
post_data_dict = request.POST.dict()
LOGGER.info('Upload lat/lng file from DV/Geoconnect and join to a layer.')
LOGGER.info('Step 1: Is the Dataverse Layer Metadata valid?')
msg('view_upload_lat_lng_table 1')
# -------------------------------------------
# (1) Is the Dataverse Layer Metadata valid?
# -------------------------------------------
f = DataverseLayerMetadataValidationForm(post_data_dict)
if not f.is_valid():
LOGGER.error('check_for_existing_layer. failed validation')
LOGGER.error('Errors: %s', f.errors)
json_msg = MessageHelperJSON.get_json_fail_msg('Failed to validate Dataverse metadata',
data_dict=f.errors)
return HttpResponse(json_msg, mimetype="application/json", status=400)
# -------------------------------------------
# (1b) Does a layer already exist for this DataverseInfo?
# -------------------------------------------
msg('step 2')
LOGGER.info('Step 2: Does a layer already exist for this DataverseInfo?')
existing_dv_layer_metadata =\
retrieve_dataverse_layer_metadata_by_kwargs_installation_and_file_id(**post_data_dict)
#-----------------------------------------------------------
# A layer was found!
# Update the DataverseLayerMetadata and return the layer.
#-----------------------------------------------------------
if existing_dv_layer_metadata:
msg("Found existing layer!")
LOGGER.info("Found existing layer!")
#update_the_layer_metadata(existing_dv_layer_metadata, post_data_dict)
metadata_dict = get_layer_and_join_metadata(existing_dv_layer_metadata.map_layer)
json_msg = MessageHelperJSON.get_json_msg(success=True,\
msg='A layer already exists for the join.',\
data_dict=metadata_dict)
return HttpResponse(status=200, content=json_msg, content_type="application/json")
# -------------------------------------------
# (2) Is the Lat/Lng request data valid? Check with the MapLatLngLayerRequestForm
# -------------------------------------------
LOGGER.info('Step 2: Is Lat/Lng data valid? Check via MapLatLngLayerRequestForm')
f = DataTableUploadFormLatLng(request.POST, request.FILES)
if not f.is_valid():
err_msg = "Invalid data in request: %s" % format_errors_as_text(f)
LOGGER.error("datatable_upload_lat_lon_api. %s", err_msg)
json_msg = MessageHelperJSON.get_json_fail_msg(err_msg, data_dict=f.errors)
return HttpResponse(json_msg, mimetype="application/json", status=400)
# Set the new table/layer owner
#
new_table_owner = request.user
# --------------------------------------
# (3) Datatable Upload
# --------------------------------------
LOGGER.info('Step 3: Datatable Upload')
try:
# Upload Lat/Lng Datatables to the Monthly table--not the Dataverse table
#
resp = datatable_upload_api(request, is_dataverse_db=False)
upload_return_dict = json.loads(resp.content)
if upload_return_dict.get('success', None) is not True:
return HttpResponse(json.dumps(upload_return_dict),
mimetype='application/json',
status=400)
else:
pass # keep going
except:
err_msg = 'Uncaught error ingesting Data Table'
LOGGER.error(err_msg)
json_msg = MessageHelperJSON.get_json_fail_msg(err_msg)
traceback.print_exc(sys.exc_info())
return HttpResponse(json_msg, mimetype='application/json', status=400)
# --------------------------------------
# (4) Create layer using the Lat/Lng columns
# --------------------------------------
LOGGER.info('Step 4: Create layer using the Lat/Lng columns')
try:
success, latlng_record_or_err_msg = create_point_col_from_lat_lon(\
new_table_owner,
upload_return_dict['data']['datatable_name'],
f.cleaned_data['lat_attribute'],
f.cleaned_data['lng_attribute'])
if not success:
LOGGER.error('Failed to (2) Create layer for map lat/lng table: %s',\
latlng_record_or_err_msg)
# FAILED
#
json_msg = MessageHelperJSON.get_json_fail_msg(latlng_record_or_err_msg)
return HttpResponse(json_msg, mimetype="application/json", status=400)
else:
# -------------------------------------------
# Add DataverseLayerMetadata object
# -------------------------------------------
(object_created, err_msg_or_dv_metadata) = create_dataverse_metadata(\
latlng_record_or_err_msg.layer,\
post_data_dict)
# -------------------------------------------
# Failed to create DataverseLayerMetadata
# -------------------------------------------
if object_created is False:
# delete LatLngTableMappingRecord
latlng_record_or_err_msg.delete()
json_msg = MessageHelperJSON.get_json_fail_msg(err_msg_or_dv_metadata)
return HttpResponse(status=400, content=json_msg, content_type="application/json")
# -------------------------------------------
# Success! Send user response
# -------------------------------------------
# Add DV info
response_params = get_layer_metadata_dict(latlng_record_or_err_msg.layer,\
latlng_record_or_err_msg.as_json())
json_msg = MessageHelperJSON.get_json_success_msg(\
msg='New layer created',\
data_dict=response_params)
return HttpResponse(json_msg, mimetype="application/json", status=200)
except:
traceback.print_exc(sys.exc_info())
err_msg = 'Uncaught error ingesting Data Table'
json_msg = MessageHelperJSON.get_json_fail_msg(err_msg)
LOGGER.error(err_msg)
return HttpResponse(json_msg, mimetype="application/json", status=400)