def run_osm2pgsql(options): """ Run osm2pgsql with the given options. """ env = get_pg_env(options['dsn']) cmd = [ options['osm2pgsql'], '--hstore', '--latlon', '--slim', '--with-forward-dependencies', 'false', '--log-progress', 'true', '--number-processes', str(options['threads']), '--cache', str(options['osm2pgsql_cache']), '--output', 'gazetteer', '--style', str(options['osm2pgsql_style']) ] if options['append']: cmd.append('--append') else: cmd.append('--create') if options['flatnode_file']: cmd.extend(('--flat-nodes', options['flatnode_file'])) for key, param in (('slim_data', '--tablespace-slim-data'), ('slim_index', '--tablespace-slim-index'), ('main_data', '--tablespace-main-data'), ('main_index', '--tablespace-main-index')): if options['tablespaces'][key]: cmd.extend((param, options['tablespaces'][key])) if options.get('disable_jit', False): env['PGOPTIONS'] = '-c jit=off -c max_parallel_workers_per_gather=0' cmd.append(str(options['import_file'])) subprocess.run(cmd, cwd=options.get('cwd', '.'), env=env, check=True)
def create_db(dsn, rouser=None): """ Create a new database for the given DSN. Fails when the database already exists or the PostgreSQL version is too old. Uses `createdb` to create the database. If 'rouser' is given, then the function also checks that the user with that given name exists. Requires superuser rights by the caller. """ proc = subprocess.run(['createdb'], env=get_pg_env(dsn), check=False) if proc.returncode != 0: raise UsageError('Creating new database failed.') with connect(dsn) as conn: postgres_version = conn.server_version_tuple() if postgres_version < POSTGRESQL_REQUIRED_VERSION: LOG.fatal('Minimum supported version of Postgresql is %d.%d. ' 'Found version %d.%d.', POSTGRESQL_REQUIRED_VERSION[0], POSTGRESQL_REQUIRED_VERSION[1], postgres_version[0], postgres_version[1]) raise UsageError('PostgreSQL server is too old.') if rouser is not None: with conn.cursor() as cur: cnt = cur.scalar('SELECT count(*) FROM pg_user where usename = %s', (rouser, )) if cnt == 0: LOG.fatal("Web user '%s' does not exists. Create it with:\n" "\n createuser %s", rouser, rouser) raise UsageError('Missing read-only user.')
def setup_database_skeleton(dsn, rouser=None): """ Create a new database for Nominatim and populate it with the essential extensions. The function fails when the database already exists or Postgresql or PostGIS versions are too old. Uses `createdb` to create the database. If 'rouser' is given, then the function also checks that the user with that given name exists. Requires superuser rights by the caller. """ proc = subprocess.run(['createdb'], env=get_pg_env(dsn), check=False) if proc.returncode != 0: raise UsageError('Creating new database failed.') with connect(dsn) as conn: _require_version('PostgreSQL server', conn.server_version_tuple(), POSTGRESQL_REQUIRED_VERSION) if rouser is not None: with conn.cursor() as cur: cnt = cur.scalar('SELECT count(*) FROM pg_user where usename = %s', (rouser, )) if cnt == 0: LOG.fatal("Web user '%s' does not exists. Create it with:\n" "\n createuser %s", rouser, rouser) raise UsageError('Missing read-only user.') # Create extensions. with conn.cursor() as cur: cur.execute('CREATE EXTENSION IF NOT EXISTS hstore') cur.execute('CREATE EXTENSION IF NOT EXISTS postgis') conn.commit() _require_version('PostGIS', conn.postgis_version_tuple(), POSTGIS_REQUIRED_VERSION)
def execute_file(dsn, fname, ignore_errors=False, pre_code=None, post_code=None): """ Read an SQL file and run its contents against the given database using psql. Use `pre_code` and `post_code` to run extra commands before or after executing the file. The commands are run within the same session, so they may be used to wrap the file execution in a transaction. """ cmd = ['psql'] if not ignore_errors: cmd.extend(('-v', 'ON_ERROR_STOP=1')) if not LOG.isEnabledFor(logging.INFO): cmd.append('--quiet') proc = subprocess.Popen(cmd, env=get_pg_env(dsn), stdin=subprocess.PIPE) try: if not LOG.isEnabledFor(logging.INFO): proc.stdin.write( 'set client_min_messages to WARNING;'.encode('utf-8')) if pre_code: proc.stdin.write((pre_code + ';').encode('utf-8')) if fname.suffix == '.gz': with gzip.open(str(fname), 'rb') as fdesc: remain = _pipe_to_proc(proc, fdesc) else: with fname.open('rb') as fdesc: remain = _pipe_to_proc(proc, fdesc) if remain == 0 and post_code: proc.stdin.write((';' + post_code).encode('utf-8')) finally: proc.stdin.close() ret = proc.wait() if ret != 0 or remain > 0: raise UsageError("Failed to execute SQL file.")
def test_get_pg_env_overwrite_variable(monkeypatch): monkeypatch.setenv('PGUSER', 'some default') env = get_pg_env('user=overwriter') assert env['PGUSER'] == 'overwriter'
def test_get_pg_env_add_variable(monkeypatch): monkeypatch.delenv('PGPASSWORD', raising=False) env = get_pg_env('user=fooF') assert env['PGUSER'] == 'fooF' assert 'PGPASSWORD' not in env
def test_get_pg_env_ignore_unknown(): env = get_pg_env('tty=stuff', base_env={}) assert env == {}
def test_get_pg_env_ignore_unknown(): env = get_pg_env('client_encoding=stuff', base_env={}) assert env == {}