def test_concurrent_executor(): runner = get_executor(None, 2) assert str(runner).find('Multi') >= 0 run_tests_for_runner(runner, 0.3) runner = get_executor(None, 2, use_cloud_pickle=False) assert str(runner).find('Multi') >= 0 run_tests_for_runner(runner, 0.3)
def test_concurrent_executor(): executor = get_executor(None, 2) assert 'Multiproc' in str(executor) run_executor_tests(executor) executor = get_executor(None, 2, use_cloud_pickle=False) assert 'Multiproc' in str(executor) run_executor_tests(executor)
def ingestion_work(output_type, source_type, ingestion_definition): """Run the ingestion process for a user defined configuration Args: output_type, source_type: types produced by ingest.make_output_type ingestion_definition: dict representing a Data Cube ingestion def produced using the utils func. """ conf_path = '/home/' + settings.LOCAL_USER + '/Datacube/data_cube_ui/config/.datacube.conf' index = index_connect(local_config=LocalConfig.find([conf_path])) tasks = ingest.create_task_list(index, output_type, None, source_type, ingestion_definition) # this is a dry run # paths = [ingest.get_filename(ingestion_definition, task['tile_index'], task['tile'].sources) for task in tasks] # ingest.check_existing_files(paths) # this actually ingests stuff successful, failed = ingest.process_tasks(index, ingestion_definition, source_type, output_type, tasks, 3200, get_executor(None, None)) index.close() return 0
def ingest_subset(ingestion_request_id=None): """Run the ingestion process on the new database Open a connection to the new database and run ingestion based on the ingestion configuration found on the IngestionRequest model. """ ingestion_request = IngestionRequest.objects.get(pk=ingestion_request_id) config = get_config(ingestion_request.user) index = index_connect(local_config=config, validate_connection=False) # Thisis done because of something that the agdc guys do in ingest: https://github.com/opendatacube/datacube-core/blob/develop/datacube/scripts/ingest.py#L168 ingestion_request.ingestion_definition[ 'filename'] = "ceos_data_cube_sample.yaml" try: source_type, output_type = ingest.make_output_type( index, ingestion_request.ingestion_definition) tasks = list( ingest.create_task_list(index, output_type, None, source_type, ingestion_request.ingestion_definition)) ingestion_request.total_storage_units = len(tasks) ingestion_request.update_status("WAIT", "Starting the ingestion process...") successful, failed = ingest.process_tasks( index, ingestion_request.ingestion_definition, source_type, output_type, tasks, 3200, get_executor(None, None)) except: index.close() raise index.close()
@pass_index(app_name=app_name, expect_initialised=expect_initialised) def with_datacube(index, *args, **kwargs): return f(Datacube(index=index), *args, **kwargs) return functools.update_wrapper(with_datacube, f) return decorate def parse_endpoint(value): ip, port = tuple(value.split(':')) return ip, int(port) EXECUTOR_TYPES = { 'serial': lambda _: get_executor(None, None), 'multiproc': lambda workers: get_executor(None, int(workers)), 'distributed': lambda addr: get_executor(parse_endpoint(addr), True), 'celery': lambda addr: mk_celery_executor(*parse_endpoint(addr)) } EXECUTOR_TYPES['dask'] = EXECUTOR_TYPES[ 'distributed'] # Add alias "dask" for distributed def _setup_executor(ctx, param, value): try: return EXECUTOR_TYPES[value[0]](value[1]) except ValueError: ctx.fail("Failed to create '%s' executor with '%s'" % value)
def test_fallback_executor(): runner = get_executor(None, None) assert str(runner).find('Serial') >= 0 run_tests_for_runner(runner, 0)
return f(index, *args, **kwargs) except (OperationalError, ProgrammingError) as e: handle_exception('Error Connecting to database: %s', e) return functools.update_wrapper(with_index, f) return decorate def parse_endpoint(value): ip, port = tuple(value.split(':')) return ip, int(port) EXECUTOR_TYPES = { 'serial': lambda _: get_executor(None, None), 'multiproc': lambda workers: get_executor(None, int(workers)), 'distributed': lambda addr: get_executor(parse_endpoint(addr), True) } def _setup_executor(ctx, param, value): try: return EXECUTOR_TYPES[value[0]](value[1]) except ValueError: ctx.fail("Failed to create '%s' executor with '%s'" % value) executor_cli_options = click.option('--executor', type=(click.Choice(EXECUTOR_TYPES.keys()), str), default=('serial', None),
def test_fallback_executor(): executor = get_executor(None, None) assert 'Serial' in str(executor) run_executor_tests(executor, sleep_time=0)