def ingest_subset(ingestion_request_id=None):
"""Run the ingestion process on the new database
Open a connection to the new database and run ingestion based on the
ingestion configuration found on the IngestionRequest model.
"""
ingestion_request = IngestionRequest.objects.get(pk=ingestion_request_id)
config = get_config(ingestion_request.get_database_name())
index = index_connect(local_config=config, validate_connection=True)
# Thisis done because of something that the agdc guys do in ingest: https://github.com/opendatacube/datacube-core/blob/develop/datacube/scripts/ingest.py#L168
ingestion_request.ingestion_definition[
'filename'] = "ceos_data_cube_sample.yaml"
try:
# source_type, output_type = ingest.make_output_type(index, ingestion_request.ingestion_definition)
source_type = index.products.get_by_name(
ingestion_request.ingestion_definition['source_type'])
output_type = index.products.add(ingest.morph_dataset_type(
source_type, ingestion_request.ingestion_definition),
allow_table_lock=True)
tasks = list(
ingest.create_task_list(index, output_type, None, source_type,
ingestion_request.ingestion_definition))
ingestion_request.total_storage_units = len(tasks)
ingestion_request.update_status("WAIT",
"Starting the ingestion process...")
executor = SerialExecutor()
successful, failed = ingest.process_tasks(
index, ingestion_request.ingestion_definition, source_type,
output_type, tasks, 3200, executor)
except:
index.close()
raise
index.close()
def ingestion_work(output_type, source_type, ingestion_definition):
"""Run the ingestion process for a user defined configuration
Args:
output_type, source_type: types produced by ingest.make_output_type
ingestion_definition: dict representing a Data Cube ingestion def produced using the utils func.
"""
conf_path = os.environ.get('DATACUBE_CONFIG_PATH')
index = index_connect(local_config=LocalConfig.find([conf_path]))
tasks = ingest.create_task_list(index, output_type, None, source_type, ingestion_definition)
# this is a dry run
# paths = [ingest.get_filename(ingestion_definition, task['tile_index'], task['tile'].sources) for task in tasks]
# ingest.check_existing_files(paths)
# this actually ingests stuff
successful, failed = ingest.process_tasks(index, ingestion_definition, source_type, output_type, tasks, 3200,
get_executor(None, None))
index.close()
return 0
