Пример #1
0
    def get(self, request):
        """Return a rendered html page that contains the metadata form and ability to
        add or delete measurements.

        This can be built upon an existing dataset type (dataset_type_id) or a blank form.

        Args:
            dataset_type_id: optional id to an existing dataset

        Returns:
            Rendered HTML for a page that will allow users to customize a dataset

        """
        context = {
            'metadata_form':
            forms.IngestionMetadataForm(),
            'measurement_form':
            forms.IngestionMeasurementForm(),
            'storage_form':
            forms.IngestionStorageForm(),
            'ingestion_bounds_form':
            forms.IngestionBoundsForm(),
            'product_details':
            models.DatasetType.objects.using('agdc').filter(~Q(definition__has_keys=['managed']) & Q(
                definition__has_keys=['measurements']))
        }

        return render(request, 'data_cube_manager/ingestion.html', context)
Пример #2
0
    def post(self, request):
        """Validate form data and create a .yaml file

        Forms include all forms from the ingestion forms module, validated and dump to YAML.

        Returns:
            JsonResponse with a status and url if there is no error.
        """

        form_data = request.POST
        measurements = json.loads(form_data.get('measurements'))
        metadata = json.loads(form_data.get('metadata_form'))
        #each measurement_form contains a dict of other forms..
        measurement_forms = [
            forms.IngestionMeasurementForm(measurements[measurement])
            for measurement in measurements
        ]
        #just a single form
        metadata_form = forms.IngestionMetadataForm(metadata)
        storage_form = forms.IngestionStorageForm(metadata)
        ingestion_bounds_form = forms.IngestionBoundsForm(metadata)

        valid, error = utils.validate_form_groups(metadata_form, storage_form,
                                                  ingestion_bounds_form,
                                                  *measurement_forms)
        if not valid:
            return JsonResponse({'status': "ERROR", 'message': error})

        #since everything is valid, now create yaml from defs..
        ingestion_def = utils.ingestion_definition_from_forms(
            metadata_form, storage_form, ingestion_bounds_form,
            measurement_forms)
        try:
            os.makedirs(
                os.path.join(settings.RESULTS_DATA_DIR,
                             'data_cube_manager/ingestion_configurations/'))
        except:
            pass

        def _dict_representer(dumper, data):
            return dumper.represent_mapping(
                yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items())

        yaml.SafeDumper.add_representer(OrderedDict, _dict_representer)

        yaml_url = os.path.join(settings.RESULTS_DATA_DIR, 'data_cube_manager/ingestion_configurations/') \
            + str(uuid.uuid4()) + '.yaml'
        with open(yaml_url, 'w') as yaml_file:
            yaml.dump(ingestion_def,
                      yaml_file,
                      Dumper=yaml.SafeDumper,
                      default_flow_style=False,
                      indent=4)
        return JsonResponse({'status': 'OK', 'url': yaml_url})
Пример #3
0
    def post(self, request):
        """Validate a set of measurements, metadata, storage, and bounds forms to create an ingestion task

        Validates all forms, returning any errors. In the case of no errors, populate an ingestion
        definition and kick off a celery task that will multiproc ingestion.

        """

        if not request.user.is_superuser:
            return JsonResponse({
                'status':
                "ERROR",
                'message':
                "Only superusers can ingest new data."
            })
        if models.IngestionRequest.objects.filter(
                user=request.user.username).exists(
                ) and models.IngestionRequest.objects.filter(
                    user=request.user.username)[0].status != "OK":
            return JsonResponse({
                'status':
                "ERROR",
                'message':
                "Please wait until your previous ingestion request is complete before submitting."
            })

        form_data = request.POST
        measurements = json.loads(form_data.get('measurements'))
        metadata = json.loads(form_data.get('metadata_form'))
        #each measurement_form contains a dict of other forms..
        measurement_forms = [
            forms.IngestionMeasurementForm(measurements[measurement])
            for measurement in measurements
        ]
        #just a single form
        metadata_form = forms.IngestionMetadataForm(metadata)
        storage_form = forms.IngestionStorageForm(metadata)
        ingestion_bounds_form = forms.IngestionBoundsForm(metadata)

        valid, error = utils.validate_form_groups(metadata_form, storage_form,
                                                  ingestion_bounds_form,
                                                  *measurement_forms)
        if not valid:
            return JsonResponse({'status': "ERROR", 'message': error})

        #since everything is valid, now create yaml from defs..
        ingestion_def = utils.ingestion_definition_from_forms(
            metadata_form, storage_form, ingestion_bounds_form,
            measurement_forms)

        output_id = tasks.run_ingestion(ingestion_def)

        return JsonResponse({'status': 'OK', 'dataset_type_ref': output_id})