示例#1
0
def ingest_subset(ingestion_request_id=None):
    """Run the ingestion process on the new database

    Open a connection to the new database and run ingestion based on the
    ingestion configuration found on the IngestionRequest model.

    """

    ingestion_request = IngestionRequest.objects.get(pk=ingestion_request_id)

    config = get_config(ingestion_request.get_database_name())
    index = index_connect(local_config=config, validate_connection=True)

    # Thisis done because of something that the agdc guys do in ingest: https://github.com/opendatacube/datacube-core/blob/develop/datacube/scripts/ingest.py#L168
    ingestion_request.ingestion_definition[
        'filename'] = "ceos_data_cube_sample.yaml"
    try:
        # source_type, output_type = ingest.make_output_type(index, ingestion_request.ingestion_definition)

        source_type = index.products.get_by_name(
            ingestion_request.ingestion_definition['source_type'])
        output_type = index.products.add(ingest.morph_dataset_type(
            source_type, ingestion_request.ingestion_definition),
                                         allow_table_lock=True)

        tasks = list(
            ingest.create_task_list(index, output_type, None, source_type,
                                    ingestion_request.ingestion_definition))

        ingestion_request.total_storage_units = len(tasks)
        ingestion_request.update_status("WAIT",
                                        "Starting the ingestion process...")

        executor = SerialExecutor()
        successful, failed = ingest.process_tasks(
            index, ingestion_request.ingestion_definition, source_type,
            output_type, tasks, 3200, executor)
    except:
        index.close()
        raise

    index.close()
示例#2
0
def ingestion_work(output_type, source_type, ingestion_definition):
    """Run the ingestion process for a user defined configuration

    Args:
        output_type, source_type: types produced by ingest.make_output_type
        ingestion_definition: dict representing a Data Cube ingestion def produced using the utils func.
    """
    conf_path = os.environ.get('DATACUBE_CONFIG_PATH')
    index = index_connect(local_config=LocalConfig.find([conf_path]))

    tasks = ingest.create_task_list(index, output_type, None, source_type, ingestion_definition)

    # this is a dry run
    # paths = [ingest.get_filename(ingestion_definition, task['tile_index'], task['tile'].sources) for task in tasks]
    # ingest.check_existing_files(paths)

    # this actually ingests stuff
    successful, failed = ingest.process_tasks(index, ingestion_definition, source_type, output_type, tasks, 3200,
                                              get_executor(None, None))

    index.close()
    return 0