def bootstrap_privileged(skip_apt_update=skip_apt_update_default, postgis_install=postgis_install_default, pg_version=pg_version_default):
if not os.path.exists(bin_dir):
os.makedirs(bin_dir)
if check_os.check_ubuntu() or check_os.check_debian() or check_os.check_linuxmint():
pm_utils.install_packages(["maven", "openjdk-7-jdk",
"ant", # for postgis-jdbc
"sudo", # very small probability that it is not installed, but it is a prequisite of the script...
"software-properties-common", # provides add-apt-repository which is used by pm_utils module in osm_postgis_transform_prequisites.install_postgresql
"python-software-properties", # provides add-apt-repository on Ubuntu 12.04.4, is available in Ubuntu 14.04
"xterm",
], package_manager=apt_get, skip_apt_update=skip_apt_update)
elif check_os.check_opensuse():
# install maven
#sp.check_call(["/sbin/OCICLI", "http://software.opensuse.org/ymp/Application:Geo/openSUSE_12.3/maven.ymp?base=openSUSE%3A12.3&query=maven"]) # (not sure whether this installs futher instable software, provided maven version 3.0.4 isn't a hit neither)
maven_bin_dir = os.path.join(external_bin_dir, maven_bin_dir_name)
if not check_dir(maven_bin_dir):
maven_bin_archive = os.path.join(external_bin_dir, maven_bin_archive_name)
if not check_file(maven_bin_archive, maven_bin_archive_md5):
do_wget(maven_bin_archive_url, maven_bin_archive)
sp.check_call([tar, "xf", maven_bin_archive], cwd=external_bin_dir)
shutil.copytree(maven_bin_dir, maven_bin_dir_install_target)
sp.check_call(chown, "-Rc", "%s:%s" %(user,user), dirpath)
# install remaining prequisites
sp.check_call([zypper, "install", "java-1_7_0-openjdk", "java-1_7_0-openjdk-devel", "java-1_7_0-openjdk-src", "java-1_7_0-openjdk-javadoc"])
else:
# better to let the script fail here than to get some less comprehensive error message later
raise RuntimeError("operating system not supported!")
# postgresql
osm_postgis_transform_prequisites.install_postgresql(pg_version=pg_version, skip_apt_update=skip_apt_update)
# install postgis (demote for source installation)
if postgis_install == "pm":
if check_os.check_ubuntu() or check_os.check_debian() or check_os.check_linuxmint():
packages = ["postgis"] # 'postgis-2.1' interpreted as regular expression in apt!
# versions for which conditions exist are not necessarily supported!
if pg_version == (9,2):
packages += ["postgresql-9.2-postgis-2.1", "postgresql-contrib-9.2", "postgresql-9.2-postgis-2.1-scripts"]
elif pg_version == (9,3):
packages += ["postgresql-9.3-postgis-2.1", "postgresql-contrib-9.3", "postgresql-9.3-postgis-2.1-scripts"]
else:
raise ValueError("postgresql version not supported")
sp.call([service, "stop", "postgresql"]) # crashes installation if port 5432 is occupied
pm_utils.install_packages(packages, package_manager=apt_get, skip_apt_update=skip_apt_update)
elif check_os.check_opensuse():
sp.check_call([zypper, "install", "postgis2", "postgis2-devel", "postgis2-utils"])
else:
raise ValueError("operating system not supported")
elif postgis_install == "source":
logger.info("postgis installation from source occurs (unprivileged) in bootstrap.py")
else:
raise ValueError("postgis_install has to be one of %s" % str(postgis_installs))
# prequisites for start_db
pm_utils.install_packages([
"python-pip",
"python-dev", # soft dependency of `pip install subprocess32`, not fulfilled, e.g. on Debian 7.4
], package_manager="apt-get")
sp.check_call([pip, "install", "--upgrade", "setuptools"]) # saves a lot of trouble and hurts much less than it helps
sp.check_call([pip, "install", "subprocess32", "pexpect"]) # pip manages update of available import automatically so that import xxx can be invoked, already installed packages don't cause returncode != 0
sp.check_call([git, "checkout", "master"], cwd=checkout_target) # get on a branch because HEAD might be detached (then the following git pull without remote specification would fail)
sp.check_call([git, "pull", "origin", "master"], cwd=checkout_target) # not 100 % sure why this is necessary
except Exception as ex:
logger.error("directory '%s' exists, but updating python-essentials failed (see the following exception for details), something has to be wrong. If you can't figure out what, remove '%s' and run the script again" % (checkout_target, checkout_target))
raise ex
else:
logger.info("checking out python-essentials as script dependency")
sp.check_call([git, "clone", "https://github.com/krichter722/python-essentials.git"], cwd=base_dir)
sys.path.append(checkout_target)
sys.path.append(os.path.join(checkout_target, "lib"))
import os_utils
import check_os
if os_utils.which("pip") is None:
hint = ""
if check_os.check_ubuntu() or check_os.check_debian() or check_os.check_linuxmint():
hint = " Install with 'sudo apt-get install python-pip' for example."
if check_os.check_opensuse():
hint = " Install with 'zypper install python-pip' for example."
raise RuntimeError("pip isn't available, install it and rerun the script, please!%s" % (hint,))
# python augeas bindings are not availble in pip, nor in easy_install
try:
import augeas # just for testing that pm_utils dependencies are fulfilled
except ImportError:
hint = ""
if check_os.check_ubuntu() or check_os.check_debian() or check_os.check_linuxmint():
hint = " Install with 'sudo apt-get install python-augeas' for example."
if check_os.check_opensuse():
hint = " Install with 'zypper install python-augeas' for example."
raise RuntimeError("augeas python bindings aren't available, install it and rerun the script, please!%s" % (hint,))
def start_db(postgresql_pgalise_version=postgresql_pgalise_version_default,
postgresql_osm_version=postgresql_osm_version_default,
pgalise_data_dir=pgalise_data_dir_default,
osm_data_dir=osm_data_dir_default):
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
postgresql_pgalise_version_string = string.join(
[str(x) for x in postgresql_pgalise_version], ".")
postgresql_osm_version_string = string.join(
[str(x) for x in postgresql_osm_version], ".")
if check_os.check_ubuntu() or check_os.check_debian(
) or check_os.check_linuxmint():
pgalise_psql = "/usr/lib/postgresql/%s/bin/psql" % postgresql_pgalise_version_string
pgalise_initdb = "/usr/lib/postgresql/%s/bin/initdb" % postgresql_pgalise_version_string
pgalise_createdb = "/usr/lib/postgresql/%s/bin/createdb" % postgresql_pgalise_version_string
pgalise_postgres = "/usr/lib/postgresql/%s/bin/postgres" % postgresql_pgalise_version_string
osm_psql = "/usr/lib/postgresql/%s/bin/psql" % postgresql_osm_version_string
osm_initdb = "/usr/lib/postgresql/%s/bin/initdb" % postgresql_osm_version_string
osm_createdb = "/usr/lib/postgresql/%s/bin/createdb" % postgresql_osm_version_string
osm_postgres = "/usr/lib/postgresql/%s/bin/postgres" % postgresql_osm_version_string
elif check_os.check_opensuse():
opensuse_postgresql_pgalise_version_string = string.join(
[str(x) for x in postgresql_pgalise_version], "")
pgalise_psql = "/usr/lib/postgresql%s/bin/psql" % opensuse_postgresql_pgalise_version_string
pgalise_initdb = "/usr/lib/postgresql%s/bin/initdb" % opensuse_postgresql_pgalise_version_string
pgalise_createdb = "/usr/lib/postgresql%s/bin/createdb" % opensuse_postgresql_pgalise_version_string
pgalise_postgres = "/usr/lib/postgresql%s/bin/postgres" % opensuse_postgresql_pgalise_version_string
opensuse_postgresql_osm_version_string = string.join(
[str(x) for x in postgresql_osm_version], "")
osm_psql = "/usr/lib/postgresql%s/bin/psql" % opensuse_postgresql_osm_version_string
osm_initdb = "/usr/lib/postgresql%s/bin/initdb" % opensuse_postgresql_osm_version_string
osm_createdb = "/usr/lib/postgresql%s/bin/createdb" % opensuse_postgresql_osm_version_string
osm_postgres = "/usr/lib/postgresql%s/bin/postgres" % opensuse_postgresql_osm_version_string
else:
# better to let the script fail here than to get some less comprehensive error message later
raise RuntimeError("operating system not supported!")
log_dir = os.path.join(base_dir_path, "log")
if not os.path.exists(log_dir):
os.mkdir(log_dir)
timestamp_suffix = datetime.datetime.today().isoformat()
terminal_cmds = ["xterm", "-hold", "-e"]
postgres_log_file_path = os.path.join(log_dir,
"postgis-%s.log" % timestamp_suffix)
postgres_log_file = None # open(postgres_log_file_path, "w") # None doesn't work correctly on OpenSUSE (no output) (not sure which xterm version)
postgres_proc = sp32.Popen(
terminal_cmds + [
pgalise_postgres, "-D", pgalise_data_dir, "-p", "5201", "-h",
"localhost", "-k", tmp_dir
],
stdout=postgres_log_file,
start_new_session=
True, # necessary in order to prevent SIGINT being sent to postgres process which causes messed up data directries and stale pid files
)
print(
"started pgalise postgresql database process with data directory %s" %
postgres_log_file_path)
if postgres_log_file_path != None:
print("logging output of postgres process to %s" %
(postgres_log_file_path))
osm_postgres_log_file_path = os.path.join(
log_dir, "postgis-osm-%s.log" % timestamp_suffix)
osm_postgres_log_file = None # open(osm_postgres_log_file_path, "w") # None doesn't work correctly on OpenSUSE (no output) (not sure which xterm version)
osm_postgres_proc = sp32.Popen(
terminal_cmds + [
osm_postgres, "-D", osm_data_dir, "-p", "5204", "-h", "localhost",
"-k", tmp_dir
],
stdout=osm_postgres_log_file,
start_new_session=
True, # necessary in order to prevent SIGINT being sent to postgres process which causes messed up data directries and stale pid files
)
print(
"started PostGIS postgresql database process with data directory %s" %
(osm_data_dir))
if osm_postgres_log_file != None:
print("logging output of OSM postgres process to %s" %
osm_postgres_log_file_path)
print(
"waiting for both terminal processes (of pgalise and PostGIS database processes) to terminate (kill them with SIGTERM signal (on Debian systems and other with Strg+C) and close terminal windows)"
)
global shutdown_requested
shutdown_requested = False
def signal_handler(signal, frame):
global shutdown_requested
if not shutdown_requested:
print(
'You pressed Ctrl+C or sent SIGINT otherwise, try to send SIGINT to the database processes first (press Ctrl+C in the child terminals and close the windows) in order to shutdown the database processes gracefully! Press Ctrl+C again to force shutdown'
)
shutdown_requested = True
return
postgres_proc.terminate()
osm_postgres_proc.terminate()
signal.signal(signal.SIGINT, signal_handler)
postgres_proc.communicate(
) # necessary to prevent Popen from waiting for ever
postgres_proc.wait()
osm_postgres_proc.communicate()
osm_postgres_proc.wait()
def bootstrap_privileged(skip_apt_update=skip_apt_update_default,
postgis_install=postgis_install_default,
pg_version=pg_version_default):
if not os.path.exists(bin_dir):
os.makedirs(bin_dir)
if check_os.check_ubuntu() or check_os.check_debian(
) or check_os.check_linuxmint():
pm_utils.install_packages(
[
"maven",
"openjdk-7-jdk",
"ant", # for postgis-jdbc
"sudo", # very small probability that it is not installed, but it is a prequisite of the script...
"software-properties-common", # provides add-apt-repository which is used by pm_utils module in osm_postgis_transform_prequisites.install_postgresql
"python-software-properties", # provides add-apt-repository on Ubuntu 12.04.4, is available in Ubuntu 14.04
"xterm",
],
package_manager=apt_get,
skip_apt_update=skip_apt_update)
elif check_os.check_opensuse():
# install maven
#sp.check_call(["/sbin/OCICLI", "http://software.opensuse.org/ymp/Application:Geo/openSUSE_12.3/maven.ymp?base=openSUSE%3A12.3&query=maven"]) # (not sure whether this installs futher instable software, provided maven version 3.0.4 isn't a hit neither)
maven_bin_dir = os.path.join(external_bin_dir, maven_bin_dir_name)
if not check_dir(maven_bin_dir):
maven_bin_archive = os.path.join(external_bin_dir,
maven_bin_archive_name)
if not check_file(maven_bin_archive, maven_bin_archive_md5):
do_wget(maven_bin_archive_url, maven_bin_archive)
sp.check_call([tar, "xf", maven_bin_archive], cwd=external_bin_dir)
shutil.copytree(maven_bin_dir, maven_bin_dir_install_target)
sp.check_call(chown, "-Rc", "%s:%s" % (user, user), dirpath)
# install remaining prequisites
sp.check_call([
zypper, "install", "java-1_7_0-openjdk",
"java-1_7_0-openjdk-devel", "java-1_7_0-openjdk-src",
"java-1_7_0-openjdk-javadoc"
])
else:
# better to let the script fail here than to get some less comprehensive error message later
raise RuntimeError("operating system not supported!")
# postgresql
osm_postgis_transform_prequisites.install_postgresql(
pg_version=pg_version, skip_apt_update=skip_apt_update)
# install postgis (demote for source installation)
if postgis_install == "pm":
if check_os.check_ubuntu() or check_os.check_debian(
) or check_os.check_linuxmint():
packages = [
"postgis"
] # 'postgis-2.1' interpreted as regular expression in apt!
# versions for which conditions exist are not necessarily supported!
if pg_version == (9, 2):
packages += [
"postgresql-9.2-postgis-2.1", "postgresql-contrib-9.2",
"postgresql-9.2-postgis-2.1-scripts"
]
elif pg_version == (9, 3):
packages += [
"postgresql-9.3-postgis-2.1", "postgresql-contrib-9.3",
"postgresql-9.3-postgis-2.1-scripts"
]
else:
raise ValueError("postgresql version not supported")
sp.call([service, "stop", "postgresql"
]) # crashes installation if port 5432 is occupied
pm_utils.install_packages(packages,
package_manager=apt_get,
skip_apt_update=skip_apt_update)
elif check_os.check_opensuse():
sp.check_call([
zypper, "install", "postgis2", "postgis2-devel",
"postgis2-utils"
])
else:
raise ValueError("operating system not supported")
elif postgis_install == "source":
logger.info(
"postgis installation from source occurs (unprivileged) in bootstrap.py"
)
else:
raise ValueError("postgis_install has to be one of %s" %
str(postgis_installs))
# prequisites for start_db
pm_utils.install_packages(
[
"python-pip",
"python-dev", # soft dependency of `pip install subprocess32`, not fulfilled, e.g. on Debian 7.4
],
package_manager="apt-get")
sp.check_call([
pip, "install", "--upgrade", "setuptools"
]) # saves a lot of trouble and hurts much less than it helps
sp.check_call(
[pip, "install", "subprocess32", "pexpect"]
) # pip manages update of available import automatically so that import xxx can be invoked, already installed packages don't cause returncode != 0
else:
logger.info("checking out python-essentials as script dependency")
sp.check_call([
git, "clone",
"https://github.com/krichter722/python-essentials.git"
],
cwd=base_dir)
sys.path.append(checkout_target)
sys.path.append(os.path.join(checkout_target, "lib"))
import os_utils
import check_os
if os_utils.which("pip") is None:
hint = ""
if check_os.check_ubuntu() or check_os.check_debian(
) or check_os.check_linuxmint():
hint = " Install with 'sudo apt-get install python-pip' for example."
if check_os.check_opensuse():
hint = " Install with 'zypper install python-pip' for example."
raise RuntimeError(
"pip isn't available, install it and rerun the script, please!%s" %
(hint, ))
# python augeas bindings are not availble in pip, nor in easy_install
try:
import augeas # just for testing that pm_utils dependencies are fulfilled
except ImportError:
hint = ""
if check_os.check_ubuntu() or check_os.check_debian(
) or check_os.check_linuxmint():
hint = " Install with 'sudo apt-get install python-augeas' for example."
if check_os.check_opensuse():
def start_db(postgresql_pgalise_version=postgresql_pgalise_version_default, postgresql_osm_version=postgresql_osm_version_default, pgalise_data_dir=pgalise_data_dir_default, osm_data_dir=osm_data_dir_default):
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
postgresql_pgalise_version_string = string.join([str(x) for x in postgresql_pgalise_version], ".")
postgresql_osm_version_string = string.join([str(x) for x in postgresql_osm_version], ".")
if check_os.check_ubuntu() or check_os.check_debian() or check_os.check_linuxmint():
pgalise_psql = "/usr/lib/postgresql/%s/bin/psql" % postgresql_pgalise_version_string
pgalise_initdb = "/usr/lib/postgresql/%s/bin/initdb" % postgresql_pgalise_version_string
pgalise_createdb = "/usr/lib/postgresql/%s/bin/createdb" % postgresql_pgalise_version_string
pgalise_postgres = "/usr/lib/postgresql/%s/bin/postgres" % postgresql_pgalise_version_string
osm_psql = "/usr/lib/postgresql/%s/bin/psql" % postgresql_osm_version_string
osm_initdb = "/usr/lib/postgresql/%s/bin/initdb" % postgresql_osm_version_string
osm_createdb = "/usr/lib/postgresql/%s/bin/createdb" % postgresql_osm_version_string
osm_postgres = "/usr/lib/postgresql/%s/bin/postgres" % postgresql_osm_version_string
elif check_os.check_opensuse():
opensuse_postgresql_pgalise_version_string = string.join([str(x) for x in postgresql_pgalise_version], "")
pgalise_psql = "/usr/lib/postgresql%s/bin/psql" % opensuse_postgresql_pgalise_version_string
pgalise_initdb = "/usr/lib/postgresql%s/bin/initdb" % opensuse_postgresql_pgalise_version_string
pgalise_createdb = "/usr/lib/postgresql%s/bin/createdb" % opensuse_postgresql_pgalise_version_string
pgalise_postgres = "/usr/lib/postgresql%s/bin/postgres" % opensuse_postgresql_pgalise_version_string
opensuse_postgresql_osm_version_string = string.join([str(x) for x in postgresql_osm_version], "")
osm_psql = "/usr/lib/postgresql%s/bin/psql" % opensuse_postgresql_osm_version_string
osm_initdb = "/usr/lib/postgresql%s/bin/initdb" % opensuse_postgresql_osm_version_string
osm_createdb = "/usr/lib/postgresql%s/bin/createdb" % opensuse_postgresql_osm_version_string
osm_postgres = "/usr/lib/postgresql%s/bin/postgres" % opensuse_postgresql_osm_version_string
else:
# better to let the script fail here than to get some less comprehensive error message later
raise RuntimeError("operating system not supported!")
log_dir = os.path.join(base_dir_path, "log")
if not os.path.exists(log_dir):
os.mkdir(log_dir)
timestamp_suffix = datetime.datetime.today().isoformat()
terminal_cmds = ["xterm", "-hold", "-e"]
postgres_log_file_path = os.path.join(log_dir, "postgis-%s.log" % timestamp_suffix)
postgres_log_file = None # open(postgres_log_file_path, "w") # None doesn't work correctly on OpenSUSE (no output) (not sure which xterm version)
postgres_proc = sp32.Popen(
terminal_cmds+[pgalise_postgres, "-D", pgalise_data_dir, "-p", "5201", "-h", "localhost", "-k", tmp_dir],
stdout=postgres_log_file,
start_new_session=True, # necessary in order to prevent SIGINT being sent to postgres process which causes messed up data directries and stale pid files
)
print("started pgalise postgresql database process with data directory %s" % postgres_log_file_path)
if postgres_log_file_path != None:
print("logging output of postgres process to %s" % (postgres_log_file_path))
osm_postgres_log_file_path = os.path.join(log_dir, "postgis-osm-%s.log" % timestamp_suffix)
osm_postgres_log_file = None # open(osm_postgres_log_file_path, "w") # None doesn't work correctly on OpenSUSE (no output) (not sure which xterm version)
osm_postgres_proc = sp32.Popen(
terminal_cmds+[osm_postgres, "-D", osm_data_dir, "-p", "5204", "-h", "localhost", "-k", tmp_dir],
stdout=osm_postgres_log_file,
start_new_session=True, # necessary in order to prevent SIGINT being sent to postgres process which causes messed up data directries and stale pid files
)
print("started PostGIS postgresql database process with data directory %s" % (osm_data_dir))
if osm_postgres_log_file != None:
print("logging output of OSM postgres process to %s" % osm_postgres_log_file_path)
print("waiting for both terminal processes (of pgalise and PostGIS database processes) to terminate (kill them with SIGTERM signal (on Debian systems and other with Strg+C) and close terminal windows)")
global shutdown_requested
shutdown_requested = False
def signal_handler(signal, frame):
global shutdown_requested
if not shutdown_requested:
print('You pressed Ctrl+C or sent SIGINT otherwise, try to send SIGINT to the database processes first (press Ctrl+C in the child terminals and close the windows) in order to shutdown the database processes gracefully! Press Ctrl+C again to force shutdown')
shutdown_requested = True
return
postgres_proc.terminate()
osm_postgres_proc.terminate()
signal.signal(signal.SIGINT, signal_handler)
postgres_proc.communicate() # necessary to prevent Popen from waiting for ever
postgres_proc.wait()
osm_postgres_proc.communicate()
osm_postgres_proc.wait()
