def post(self, request):
"""Validate form data and create a .yaml file
Forms include all forms from the ingestion forms module, validated and dump to YAML.
Returns:
JsonResponse with a status and url if there is no error.
"""
form_data = request.POST
measurements = json.loads(form_data.get('measurements'))
metadata = json.loads(form_data.get('metadata_form'))
#each measurement_form contains a dict of other forms..
measurement_forms = [
forms.IngestionMeasurementForm(measurements[measurement])
for measurement in measurements
]
#just a single form
metadata_form = forms.IngestionMetadataForm(metadata)
storage_form = forms.IngestionStorageForm(metadata)
ingestion_bounds_form = forms.IngestionBoundsForm(metadata)
valid, error = utils.validate_form_groups(metadata_form, storage_form,
ingestion_bounds_form,
*measurement_forms)
if not valid:
return JsonResponse({'status': "ERROR", 'message': error})
#since everything is valid, now create yaml from defs..
ingestion_def = utils.ingestion_definition_from_forms(
metadata_form, storage_form, ingestion_bounds_form,
measurement_forms)
try:
os.makedirs(
os.path.join(settings.RESULTS_DATA_DIR,
'data_cube_manager/ingestion_configurations/'))
except:
pass
def _dict_representer(dumper, data):
return dumper.represent_mapping(
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items())
yaml.SafeDumper.add_representer(OrderedDict, _dict_representer)
yaml_url = os.path.join(settings.RESULTS_DATA_DIR, 'data_cube_manager/ingestion_configurations/') \
+ str(uuid.uuid4()) + '.yaml'
with open(yaml_url, 'w') as yaml_file:
yaml.dump(ingestion_def,
yaml_file,
Dumper=yaml.SafeDumper,
default_flow_style=False,
indent=4)
return JsonResponse({'status': 'OK', 'url': yaml_url})
def post(self, request):
"""Validate a set of measurements, metadata, storage, and bounds forms to create an ingestion task
Validates all forms, returning any errors. In the case of no errors, populate an ingestion
definition and kick off a celery task that will multiproc ingestion.
"""
if not request.user.is_superuser:
return JsonResponse({
'status':
"ERROR",
'message':
"Only superusers can ingest new data."
})
if models.IngestionRequest.objects.filter(
user=request.user.username).exists(
) and models.IngestionRequest.objects.filter(
user=request.user.username)[0].status != "OK":
return JsonResponse({
'status':
"ERROR",
'message':
"Please wait until your previous ingestion request is complete before submitting."
})
form_data = request.POST
measurements = json.loads(form_data.get('measurements'))
metadata = json.loads(form_data.get('metadata_form'))
#each measurement_form contains a dict of other forms..
measurement_forms = [
forms.IngestionMeasurementForm(measurements[measurement])
for measurement in measurements
]
#just a single form
metadata_form = forms.IngestionMetadataForm(metadata)
storage_form = forms.IngestionStorageForm(metadata)
ingestion_bounds_form = forms.IngestionBoundsForm(metadata)
valid, error = utils.validate_form_groups(metadata_form, storage_form,
ingestion_bounds_form,
*measurement_forms)
if not valid:
return JsonResponse({'status': "ERROR", 'message': error})
#since everything is valid, now create yaml from defs..
ingestion_def = utils.ingestion_definition_from_forms(
metadata_form, storage_form, ingestion_bounds_form,
measurement_forms)
output_id = tasks.run_ingestion(ingestion_def)
return JsonResponse({'status': 'OK', 'dataset_type_ref': output_id})
def post(self, request):
"""Submit a Data Cube subset request using form data
POST Data:
IngestionRequestForm, IngestionStorageForm, IngestionBoundsForm
Returns:
Json response containing a pk to an ingestion request.
"""
form_data = request.POST
measurements = json.loads(form_data.get('measurements'))
metadata = json.loads(form_data.get('metadata_form'))
#each measurement_form contains a dict of other forms..
measurement_forms = [forms.IngestionMeasurementForm(measurements[measurement]) for measurement in measurements]
#just a single form
metadata_form = forms.IngestionRequestForm(metadata)
storage_form = forms.IngestionStorageForm(metadata)
ingestion_bounds_form = forms.IngestionBoundsForm({
'left': metadata.get('longitude_min', None),
'right': metadata.get('longitude_max', None),
'bottom': metadata.get('latitude_min', None),
'top': metadata.get('latitude_max', None)
})
valid, error = utils.validate_form_groups(metadata_form, storage_form, ingestion_bounds_form,
*measurement_forms)
if not valid:
return JsonResponse({'status': "ERROR", 'message': error})
dataset_type = metadata_form.cleaned_data.get('dataset_type_ref')
metadata_form.cleaned_data.update({'dataset_type': dataset_type, 'dataset_type_ref': [dataset_type]})
if not models.Dataset.filter_datasets(metadata_form.cleaned_data).exists():
return JsonResponse({
'status':
"ERROR",
'message':
"There are no datasets for the entered parameter set. Use the Dataset Viewer page to create a request from a list of datasets."
})
metadata_form.cleaned_data.update({
'dataset_type_ref':
metadata_form.cleaned_data['dataset_type_ref'][0],
'output_type':
dataset_type.name + "_sample",
'description':
"Sample subset of {} created for {}".format(dataset_type.name, request.user.username),
'location':
"/datacube/ingested_data/{}".format(request.user.username),
'file_path_template':
"SAMPLE_CUBE_4326_{tile_index[0]}_{tile_index[1]}_{start_time}.nc",
'summary':
"Contains a small subset of {}.".format(dataset_type.name),
'platform':
dataset_type.get_description(),
'instrument':
dataset_type.get_instrument(),
'processing_level':
dataset_type.get_processing_level(),
'title':
"Sample Data Cube created by the CEOS Data Cube UI",
'source':
"CEOS Data Cube UI",
'institution':
"CEOS",
'product_version':
"1.0",
'references':
"https://github.com/ceos-seo/data_cube_ui",
})
#since everything is valid, now create yaml from defs..
ingestion_def = utils.ingestion_definition_from_forms(metadata_form, storage_form, ingestion_bounds_form,
measurement_forms)
existing_requests = models.IngestionRequest.objects.filter(user=request.user.username)
if existing_requests.exists():
tasks.delete_ingestion_request(ingestion_request_id=existing_requests[0].pk)
existing_requests.delete()
ingestion_request = models.IngestionRequest(
user=request.user.username,
dataset_type_ref=dataset_type.pk,
ingestion_definition=ingestion_def,
start_date=metadata_form.cleaned_data['start_date'],
end_date=metadata_form.cleaned_data['end_date'],
latitude_min=metadata_form.cleaned_data['latitude_min'],
latitude_max=metadata_form.cleaned_data['latitude_max'],
longitude_min=metadata_form.cleaned_data['longitude_min'],
longitude_max=metadata_form.cleaned_data['longitude_max'])
ingestion_request.save()
tasks.ingestion_on_demand.delay(ingestion_request_id=ingestion_request.pk)
return JsonResponse({'status': 'OK', 'ingestion_request_id': ingestion_request.pk})
