def bootstrap(
ignore_root_warning,
force_overwrite_postgres_datadir,
skip_tests,
bootstrap_dir=bootstrap_dir_default,
skip_build=False,
psql=psql,
initdb=initdb,
createdb=createdb,
postgres=postgres,
postgres_datadir_path=postgres_datadir_path_default,
postgresql_jdbc_url=postgresql_jdbc_url_default,
postgresql_deb_url=postgresql_deb_url_default,
geotools_url=geotools_url_default,
jgrapht_url=jgrapht_url_default,
jfuzzy_url=jfuzzy_url_default,
maven_bin_archive_url=maven_bin_archive_url_default,
commons_src_archive_url=commons_src_archive_url_default,
postgis_install="pm",
postgis_url=postgis_url_default,
):
"Bootstrap the PGALISE simulation, including installation of dependencies (those which can't be fetched by maven), binaries (postgresql, postgis, etc.), setup of database in "
# force specification of flag if script is invoked as root
if os.getuid() == 0 and not ignore_root_warning:
raise RuntimeError(
"Script is invoked as root which is strongly discouraged. Specify the -I/--ignore-root-warning in order to overwrite. You have been warned!"
)
# setup directories
script_dir = os.path.join(base_dir, "scripts")
external_dir = os.path.join(bootstrap_dir, "external")
internal_dir = os.path.join(bootstrap_dir, "internal")
# used for postgresql socket
tmp_dir = os.path.join(
script_dir, "tmp") # "/tmp less save than dir only readable by user"
external_bin_dir = os.path.join(external_dir, "bin")
external_src_dir = os.path.join(external_dir, "src")
if not os.path.exists(bin_dir):
os.makedirs(bin_dir)
if not os.path.exists(bootstrap_dir):
os.makedirs(bootstrap_dir)
if not os.path.exists(external_dir):
os.makedirs(external_dir)
if not os.path.exists(external_bin_dir):
os.makedirs(external_bin_dir)
if not os.path.exists(external_src_dir):
os.makedirs(external_src_dir)
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
user = sp.check_output([whoami])
postgres_socket_dir = tmp_dir # is this ok?
# check some prequisites (after installation as they might be installed properly)
java_home = os.getenv("JAVA_HOME") # necessary for mvn
if java_home is None or java_home == "":
raise RuntimeError("JAVA_HOME is not set")
java_binary = os.path.join(java_home, "bin/java")
if not os.path.exists(java_binary):
raise RuntimeError(
"JAVA_HOME %s doesn't seem to point to a valid JDK (java binary '%s' doesn't exist)"
% (java_home, java_binary))
# install postgresql jdbc driver
postgresql_jdbc_file = os.path.join(external_bin_dir, postgresql_jdbc_name)
if not file_utils.check_file(postgresql_jdbc_file, postgresql_jdbc_md5):
do_wget(postgresql_jdbc_url, postgresql_jdbc_file)
if pg_version == (9, 2):
postgresql_jdbc_mvn_version = "9.2-1004.jdbc4"
elif pg_version == (9, 3):
postgresql_jdbc_mvn_version = "9.2-1004.jdbc4" #@TODO: not optimal to have different client and server versions
else:
raise RuntimeError("postgresql version %s not supported" %
string.join([str(x) for x in pg_version], "."))
sp.check_call([mvn, "install:install-file", \
"-Dfile=%s" % postgresql_jdbc_file, "-DartifactId=postgresql", \
"-DgroupId=postgresql", "-Dversion=%s" % postgresql_jdbc_mvn_version, "-Dpackaging=jar"], cwd=external_bin_dir)
# install jgrapht (install net.sf.jgrapht:jgrapht:0.8.3 as org.jgrapht:jgrapht:0.8.3 (could not be found in any repository))
jgrapht_file = os.path.join(external_bin_dir, jgrapht_name)
if not file_utils.check_file(jgrapht_file, jgrapht_md5):
do_wget(jgrapht_url, jgrapht_file)
sp.check_call([mvn, "install:install-file", \
"-Dfile=%s" % jgrapht_file, "-DartifactId=jgrapht",
"-DgroupId=org.jgrapht", "-Dversion=0.8.3", "-Dpackaging=jar"], cwd=external_bin_dir)
# install jfuzzylogic
jfuzzy_file = os.path.join(external_bin_dir, jfuzzy_name)
if not file_utils.check_file(jfuzzy_file, jfuzzy_md5):
do_wget(jfuzzy_url, jfuzzy_file)
sp.check_call([mvn, "install:install-file", \
"-Dfile=%s" % jfuzzy_file, "-DartifactId=jfuzzy",
"-DgroupId=pcingola", "-Dversion=3.0", "-Dpackaging=jar"], cwd=external_bin_dir)
# geronimo not available as binary (but depends on eclipse source stuff...)
#geronimo_file_name = "geronimo-3.0.1"
#geronimo_src_archive_name = "geronimo-3.0.1-source-release.zip"
#geronimo_url = "http://mirrors.ibiblio.org/maven2/org/apache/geronimo/geronimo/3.0.1/geronimo-3.0.1-source-release.zip"
#geronimo_file_path = os.path.join(external_src_dir, geronimo_file_name)
# install geotools
#geotools_src_dir = os.path.join(external_src_dir, geotools_src_dir_name)
#if not file_utils.check_dir(geotools_src_dir):
# if not ch...le(os.path.join(external_src_dir, geotools_src_archive_name)) or file_utils.retrieve_md5sum(os.path.join(external_src_dir, geotools_src_archive_name)) != geotools_src_archive_md5:
# do_wget() ... sp.check_call([wget, geotools_url], cwd=tmp_dir)
# sp.check_call([unzip, os.path.join(tmp_dir, geotools_src_archive_name)], cwd=external_src_dir)
#if not skip_build:
# mvn_settings_file_path = os.path.join(script_dir, "settings.xml")
# sp.check_call([mvn, "-e", "--global-settings", mvn_settings_file_path, "--settings", mvn_settings_file_path, "install", "-Dall", "-DskipTests=true"], cwd=geotools_src_dir)
# install commons-collections 4
#commons_src_dir = os.path.join(external_src_dir, commons_src_dir_name)
#if not file_utils.check_dir(commons_src_dir):
# commons_src_archive_file = os.path.join(external_src_dir, commons_src_archive_name)
# if not file_utils.check_file(commons_src_archive_file, commons_src_archive_md5):
# do_wget(commons_src_archive_url, commons_src_archive_file)
# sp.check_call([tar, "xf", commons_src_archive_file], cwd=external_src_dir)
#if not skip_build:
# mvn_install(commons_src_dir, skip_tests)
# install postgis from source (installation with package manager occurs in
# bootstrap_privileged.py
if postgis_install == "source":
# installation of postgis-jdbc from bin directory should not occur in sources installation as this refers to the version shipped with the tarball -> will install mvn project in subdirectory of tarball root
postgis_src_dir = os.path.join(external_src_dir, postgis_src_dir_name)
if check_os.check_ubuntu() or check_os.check_debian():
if not file_utils.check_dir(postgis_src_dir):
postgis_src_archive_file = os.path.join(
external_src_dir, postgis_src_archive_name)
if not file_utils.check_file(postgis_src_archive_file,
postgis_src_archive_md5):
do_wget(postgis_url, postgis_src_archive_file)
sp.check_call([tar, "xf", postgis_src_archive_file],
cwd=external_src_dir)
if not skip_build:
pm_utils.install_apt_get_build_dep(
["postgis"],
package_manager=apt_get,
skip_apt_update=skip_apt_update
) # might not be sufficient because Ubuntu 13.10's version of postgis is 1.5.x (we're using 2.x)
pm_utils.install_packages(
["libgdal-dev"],
package_manager=apt_get,
skip_apt_update=skip_apt_update
) # not covered by 1.5.x requirements (see above)
sp.check_call([bash, "autogen.sh"], cwd=postgis_src_dir)
sp.check_call([bash, "configure"], cwd=postgis_src_dir)
sp.check_call([make, "-j8"], cwd=postgis_src_dir)
sp.check_call([make, "install"], cwd=postgis_src_dir)
# install postgis-jdbc (project is shipped inside postgis project (as long as a fixed version of postgis is used here, the version of postgis-jdbc will be fixed, too (postgis 2.1.1 -> postgis-jdbc 2.1.0SVN)
postgis_mvn_project_path = os.path.join(postgis_src_dir,
"java/jdbc")
sp.check_call(
[ant], cwd=postgis_mvn_project_path) # generates maven project
mvn_install(postgis_mvn_project_path, skip_tests)
elif check_os.check_opensuse():
raise RuntimeError("PostGIS source installation not supported")
else:
# better to let the script fail here than to get some less comprehensive error message later
raise RuntimeError("Operating system not supported!")
elif postgis_install == "pm":
# install postgis from scripts/bin (as it is not available from somewhere online)
postgis_jdbc_file = os.path.join(bin_dir, postgis_jdbc_name)
if not os.path.exists(postgis_jdbc_file):
raise RuntimeError(
"postgis JDBC jar %s doesn't exist, can't continue, consider fetching it manually"
% (postgis_jdbc_file, ))
sp.check_call([mvn, "install:install-file", \
"-Dfile=%s" % postgis_jdbc_file, "-DartifactId=postgis-jdbc",
"-DgroupId=org.postgis", "-Dversion=2.1.0SVN", "-Dpackaging=jar"], cwd=bin_dir)
else:
raise RuntimeError("postgis_install %s isn't supported" %
(postgis_install, ))
# always install openejb from bin directory (might be made flexible with source installation like postgis (see above) later)
openejb_jar_file_path = os.path.join(bin_dir, openejb_jar_file_name)
if not os.path.exists(openejb_jar_file_path):
raise RuntimeError(
"OpenEJB jar %s doesn't exist, can't continue, consider fetching it manually"
% (openejb_jar_file_path, ))
sp.check_call([mvn, "install:install-file", \
"-Dfile=%s" % (openejb_jar_file_path,), "-DartifactId=openejb-core",
"-DgroupId=org.apache.openejb", "-Dversion=4.7.0-SNAPSHOT", "-Dpackaging=jar"], cwd=bin_dir)
openejb_api_jar_file_path = os.path.join(bin_dir,
openejb_api_jar_file_name)
if not os.path.exists(openejb_api_jar_file_path):
raise RuntimeError(
"OpenEJB API jar %s doesn't exist, can't continue, consider fetching it manually"
% (openejb_api_jar_file_path, ))
sp.check_call([
mvn, "install:install-file",
"-Dfile=%s" % (openejb_api_jar_file_path, ),
"-DartifactId=openejb-api", "-DgroupId=org.apache.openejb",
"-Dversion=4.7.0-SNAPSHOT", "-Dpackaging=jar"
],
cwd=bin_dir)
#openejb_snapshot_ahtutils_workaround(external_src_dir)
# setup postgis datadir and configuration
if not file_utils.check_dir(
postgres_datadir_path) or force_overwrite_postgres_datadir:
# os.makedirs(postgres_datadir_path) # causes error if directory exists and is not necessary
postgis_utils.bootstrap_datadir(postgres_datadir_path,
postgres_user,
password=postgres_pw,
initdb=initdb)
postgis_utils.bootstrap_database(postgres_datadir_path,
postgres_port,
postgres_host,
postgres_user,
pgalise_db_name,
password=postgres_pw,
initdb=initdb,
postgres=postgres,
createdb=createdb,
psql=psql)
postgis_utils.bootstrap_database(postgres_datadir_path,
postgres_port,
postgres_host,
postgres_user,
pgalise_db_test_name,
password=postgres_pw,
initdb=initdb,
postgres=postgres,
createdb=createdb,
psql=psql)
else:
logger.info(
"Postgres datadir %s has not been overwritten. Check output of script invoked with -h/--help on how to do that."
% (postgres_datadir_path, ))
sp.check_call([git, "checkout", "master"], cwd=checkout_target) # get on a branch because HEAD might be detached (then the following git pull without remote specification would fail)
sp.check_call([git, "pull", "origin", "master"], cwd=checkout_target) # not 100 % sure why this is necessary
except Exception as ex:
logger.error("directory '%s' exists, but updating python-essentials failed (see the following exception for details), something has to be wrong. If you can't figure out what, remove '%s' and run the script again" % (checkout_target, checkout_target))
raise ex
else:
logger.info("checking out python-essentials as script dependency")
sp.check_call([git, "clone", "https://github.com/krichter722/python-essentials.git"], cwd=base_dir)
sys.path.append(checkout_target)
sys.path.append(os.path.join(checkout_target, "lib"))
import os_utils
import check_os
if os_utils.which("pip") is None:
hint = ""
if check_os.check_ubuntu() or check_os.check_debian() or check_os.check_linuxmint():
hint = " Install with 'sudo apt-get install python-pip' for example."
if check_os.check_opensuse():
hint = " Install with 'zypper install python-pip' for example."
raise RuntimeError("pip isn't available, install it and rerun the script, please!%s" % (hint,))
# python augeas bindings are not availble in pip, nor in easy_install
try:
import augeas # just for testing that pm_utils dependencies are fulfilled
except ImportError:
hint = ""
if check_os.check_ubuntu() or check_os.check_debian() or check_os.check_linuxmint():
hint = " Install with 'sudo apt-get install python-augeas' for example."
if check_os.check_opensuse():
hint = " Install with 'zypper install python-augeas' for example."
raise RuntimeError("augeas python bindings aren't available, install it and rerun the script, please!%s" % (hint,))
def bootstrap_privileged(skip_apt_update=skip_apt_update_default, postgis_install=postgis_install_default, pg_version=pg_version_default):
if not os.path.exists(bin_dir):
os.makedirs(bin_dir)
if check_os.check_ubuntu() or check_os.check_debian() or check_os.check_linuxmint():
pm_utils.install_packages(["maven", "openjdk-7-jdk",
"ant", # for postgis-jdbc
"sudo", # very small probability that it is not installed, but it is a prequisite of the script...
"software-properties-common", # provides add-apt-repository which is used by pm_utils module in osm_postgis_transform_prequisites.install_postgresql
"python-software-properties", # provides add-apt-repository on Ubuntu 12.04.4, is available in Ubuntu 14.04
"xterm",
], package_manager=apt_get, skip_apt_update=skip_apt_update)
elif check_os.check_opensuse():
# install maven
#sp.check_call(["/sbin/OCICLI", "http://software.opensuse.org/ymp/Application:Geo/openSUSE_12.3/maven.ymp?base=openSUSE%3A12.3&query=maven"]) # (not sure whether this installs futher instable software, provided maven version 3.0.4 isn't a hit neither)
maven_bin_dir = os.path.join(external_bin_dir, maven_bin_dir_name)
if not check_dir(maven_bin_dir):
maven_bin_archive = os.path.join(external_bin_dir, maven_bin_archive_name)
if not check_file(maven_bin_archive, maven_bin_archive_md5):
do_wget(maven_bin_archive_url, maven_bin_archive)
sp.check_call([tar, "xf", maven_bin_archive], cwd=external_bin_dir)
shutil.copytree(maven_bin_dir, maven_bin_dir_install_target)
sp.check_call(chown, "-Rc", "%s:%s" %(user,user), dirpath)
# install remaining prequisites
sp.check_call([zypper, "install", "java-1_7_0-openjdk", "java-1_7_0-openjdk-devel", "java-1_7_0-openjdk-src", "java-1_7_0-openjdk-javadoc"])
else:
# better to let the script fail here than to get some less comprehensive error message later
raise RuntimeError("operating system not supported!")
# postgresql
osm_postgis_transform_prequisites.install_postgresql(pg_version=pg_version, skip_apt_update=skip_apt_update)
# install postgis (demote for source installation)
if postgis_install == "pm":
if check_os.check_ubuntu() or check_os.check_debian() or check_os.check_linuxmint():
packages = ["postgis"] # 'postgis-2.1' interpreted as regular expression in apt!
# versions for which conditions exist are not necessarily supported!
if pg_version == (9,2):
packages += ["postgresql-9.2-postgis-2.1", "postgresql-contrib-9.2", "postgresql-9.2-postgis-2.1-scripts"]
elif pg_version == (9,3):
packages += ["postgresql-9.3-postgis-2.1", "postgresql-contrib-9.3", "postgresql-9.3-postgis-2.1-scripts"]
else:
raise ValueError("postgresql version not supported")
sp.call([service, "stop", "postgresql"]) # crashes installation if port 5432 is occupied
pm_utils.install_packages(packages, package_manager=apt_get, skip_apt_update=skip_apt_update)
elif check_os.check_opensuse():
sp.check_call([zypper, "install", "postgis2", "postgis2-devel", "postgis2-utils"])
else:
raise ValueError("operating system not supported")
elif postgis_install == "source":
logger.info("postgis installation from source occurs (unprivileged) in bootstrap.py")
else:
raise ValueError("postgis_install has to be one of %s" % str(postgis_installs))
# prequisites for start_db
pm_utils.install_packages([
"python-pip",
"python-dev", # soft dependency of `pip install subprocess32`, not fulfilled, e.g. on Debian 7.4
], package_manager="apt-get")
sp.check_call([pip, "install", "--upgrade", "setuptools"]) # saves a lot of trouble and hurts much less than it helps
sp.check_call([pip, "install", "subprocess32", "pexpect"]) # pip manages update of available import automatically so that import xxx can be invoked, already installed packages don't cause returncode != 0
def start_db(postgresql_pgalise_version=postgresql_pgalise_version_default,
postgresql_osm_version=postgresql_osm_version_default,
pgalise_data_dir=pgalise_data_dir_default,
osm_data_dir=osm_data_dir_default):
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
postgresql_pgalise_version_string = string.join(
[str(x) for x in postgresql_pgalise_version], ".")
postgresql_osm_version_string = string.join(
[str(x) for x in postgresql_osm_version], ".")
if check_os.check_ubuntu() or check_os.check_debian(
) or check_os.check_linuxmint():
pgalise_psql = "/usr/lib/postgresql/%s/bin/psql" % postgresql_pgalise_version_string
pgalise_initdb = "/usr/lib/postgresql/%s/bin/initdb" % postgresql_pgalise_version_string
pgalise_createdb = "/usr/lib/postgresql/%s/bin/createdb" % postgresql_pgalise_version_string
pgalise_postgres = "/usr/lib/postgresql/%s/bin/postgres" % postgresql_pgalise_version_string
osm_psql = "/usr/lib/postgresql/%s/bin/psql" % postgresql_osm_version_string
osm_initdb = "/usr/lib/postgresql/%s/bin/initdb" % postgresql_osm_version_string
osm_createdb = "/usr/lib/postgresql/%s/bin/createdb" % postgresql_osm_version_string
osm_postgres = "/usr/lib/postgresql/%s/bin/postgres" % postgresql_osm_version_string
elif check_os.check_opensuse():
opensuse_postgresql_pgalise_version_string = string.join(
[str(x) for x in postgresql_pgalise_version], "")
pgalise_psql = "/usr/lib/postgresql%s/bin/psql" % opensuse_postgresql_pgalise_version_string
pgalise_initdb = "/usr/lib/postgresql%s/bin/initdb" % opensuse_postgresql_pgalise_version_string
pgalise_createdb = "/usr/lib/postgresql%s/bin/createdb" % opensuse_postgresql_pgalise_version_string
pgalise_postgres = "/usr/lib/postgresql%s/bin/postgres" % opensuse_postgresql_pgalise_version_string
opensuse_postgresql_osm_version_string = string.join(
[str(x) for x in postgresql_osm_version], "")
osm_psql = "/usr/lib/postgresql%s/bin/psql" % opensuse_postgresql_osm_version_string
osm_initdb = "/usr/lib/postgresql%s/bin/initdb" % opensuse_postgresql_osm_version_string
osm_createdb = "/usr/lib/postgresql%s/bin/createdb" % opensuse_postgresql_osm_version_string
osm_postgres = "/usr/lib/postgresql%s/bin/postgres" % opensuse_postgresql_osm_version_string
else:
# better to let the script fail here than to get some less comprehensive error message later
raise RuntimeError("operating system not supported!")
log_dir = os.path.join(base_dir_path, "log")
if not os.path.exists(log_dir):
os.mkdir(log_dir)
timestamp_suffix = datetime.datetime.today().isoformat()
terminal_cmds = ["xterm", "-hold", "-e"]
postgres_log_file_path = os.path.join(log_dir,
"postgis-%s.log" % timestamp_suffix)
postgres_log_file = None # open(postgres_log_file_path, "w") # None doesn't work correctly on OpenSUSE (no output) (not sure which xterm version)
postgres_proc = sp32.Popen(
terminal_cmds + [
pgalise_postgres, "-D", pgalise_data_dir, "-p", "5201", "-h",
"localhost", "-k", tmp_dir
],
stdout=postgres_log_file,
start_new_session=
True, # necessary in order to prevent SIGINT being sent to postgres process which causes messed up data directries and stale pid files
)
print(
"started pgalise postgresql database process with data directory %s" %
postgres_log_file_path)
if postgres_log_file_path != None:
print("logging output of postgres process to %s" %
(postgres_log_file_path))
osm_postgres_log_file_path = os.path.join(
log_dir, "postgis-osm-%s.log" % timestamp_suffix)
osm_postgres_log_file = None # open(osm_postgres_log_file_path, "w") # None doesn't work correctly on OpenSUSE (no output) (not sure which xterm version)
osm_postgres_proc = sp32.Popen(
terminal_cmds + [
osm_postgres, "-D", osm_data_dir, "-p", "5204", "-h", "localhost",
"-k", tmp_dir
],
stdout=osm_postgres_log_file,
start_new_session=
True, # necessary in order to prevent SIGINT being sent to postgres process which causes messed up data directries and stale pid files
)
print(
"started PostGIS postgresql database process with data directory %s" %
(osm_data_dir))
if osm_postgres_log_file != None:
print("logging output of OSM postgres process to %s" %
osm_postgres_log_file_path)
print(
"waiting for both terminal processes (of pgalise and PostGIS database processes) to terminate (kill them with SIGTERM signal (on Debian systems and other with Strg+C) and close terminal windows)"
)
global shutdown_requested
shutdown_requested = False
def signal_handler(signal, frame):
global shutdown_requested
if not shutdown_requested:
print(
'You pressed Ctrl+C or sent SIGINT otherwise, try to send SIGINT to the database processes first (press Ctrl+C in the child terminals and close the windows) in order to shutdown the database processes gracefully! Press Ctrl+C again to force shutdown'
)
shutdown_requested = True
return
postgres_proc.terminate()
osm_postgres_proc.terminate()
signal.signal(signal.SIGINT, signal_handler)
postgres_proc.communicate(
) # necessary to prevent Popen from waiting for ever
postgres_proc.wait()
osm_postgres_proc.communicate()
osm_postgres_proc.wait()
def bootstrap_privileged(skip_apt_update=skip_apt_update_default,
postgis_install=postgis_install_default,
pg_version=pg_version_default):
if not os.path.exists(bin_dir):
os.makedirs(bin_dir)
if check_os.check_ubuntu() or check_os.check_debian(
) or check_os.check_linuxmint():
pm_utils.install_packages(
[
"maven",
"openjdk-7-jdk",
"ant", # for postgis-jdbc
"sudo", # very small probability that it is not installed, but it is a prequisite of the script...
"software-properties-common", # provides add-apt-repository which is used by pm_utils module in osm_postgis_transform_prequisites.install_postgresql
"python-software-properties", # provides add-apt-repository on Ubuntu 12.04.4, is available in Ubuntu 14.04
"xterm",
],
package_manager=apt_get,
skip_apt_update=skip_apt_update)
elif check_os.check_opensuse():
# install maven
#sp.check_call(["/sbin/OCICLI", "http://software.opensuse.org/ymp/Application:Geo/openSUSE_12.3/maven.ymp?base=openSUSE%3A12.3&query=maven"]) # (not sure whether this installs futher instable software, provided maven version 3.0.4 isn't a hit neither)
maven_bin_dir = os.path.join(external_bin_dir, maven_bin_dir_name)
if not check_dir(maven_bin_dir):
maven_bin_archive = os.path.join(external_bin_dir,
maven_bin_archive_name)
if not check_file(maven_bin_archive, maven_bin_archive_md5):
do_wget(maven_bin_archive_url, maven_bin_archive)
sp.check_call([tar, "xf", maven_bin_archive], cwd=external_bin_dir)
shutil.copytree(maven_bin_dir, maven_bin_dir_install_target)
sp.check_call(chown, "-Rc", "%s:%s" % (user, user), dirpath)
# install remaining prequisites
sp.check_call([
zypper, "install", "java-1_7_0-openjdk",
"java-1_7_0-openjdk-devel", "java-1_7_0-openjdk-src",
"java-1_7_0-openjdk-javadoc"
])
else:
# better to let the script fail here than to get some less comprehensive error message later
raise RuntimeError("operating system not supported!")
# postgresql
osm_postgis_transform_prequisites.install_postgresql(
pg_version=pg_version, skip_apt_update=skip_apt_update)
# install postgis (demote for source installation)
if postgis_install == "pm":
if check_os.check_ubuntu() or check_os.check_debian(
) or check_os.check_linuxmint():
packages = [
"postgis"
] # 'postgis-2.1' interpreted as regular expression in apt!
# versions for which conditions exist are not necessarily supported!
if pg_version == (9, 2):
packages += [
"postgresql-9.2-postgis-2.1", "postgresql-contrib-9.2",
"postgresql-9.2-postgis-2.1-scripts"
]
elif pg_version == (9, 3):
packages += [
"postgresql-9.3-postgis-2.1", "postgresql-contrib-9.3",
"postgresql-9.3-postgis-2.1-scripts"
]
else:
raise ValueError("postgresql version not supported")
sp.call([service, "stop", "postgresql"
]) # crashes installation if port 5432 is occupied
pm_utils.install_packages(packages,
package_manager=apt_get,
skip_apt_update=skip_apt_update)
elif check_os.check_opensuse():
sp.check_call([
zypper, "install", "postgis2", "postgis2-devel",
"postgis2-utils"
])
else:
raise ValueError("operating system not supported")
elif postgis_install == "source":
logger.info(
"postgis installation from source occurs (unprivileged) in bootstrap.py"
)
else:
raise ValueError("postgis_install has to be one of %s" %
str(postgis_installs))
# prequisites for start_db
pm_utils.install_packages(
[
"python-pip",
"python-dev", # soft dependency of `pip install subprocess32`, not fulfilled, e.g. on Debian 7.4
],
package_manager="apt-get")
sp.check_call([
pip, "install", "--upgrade", "setuptools"
]) # saves a lot of trouble and hurts much less than it helps
sp.check_call(
[pip, "install", "subprocess32", "pexpect"]
) # pip manages update of available import automatically so that import xxx can be invoked, already installed packages don't cause returncode != 0
raise ex
else:
logger.info("checking out python-essentials as script dependency")
sp.check_call([
git, "clone",
"https://github.com/krichter722/python-essentials.git"
],
cwd=base_dir)
sys.path.append(checkout_target)
sys.path.append(os.path.join(checkout_target, "lib"))
import os_utils
import check_os
if os_utils.which("pip") is None:
hint = ""
if check_os.check_ubuntu() or check_os.check_debian(
) or check_os.check_linuxmint():
hint = " Install with 'sudo apt-get install python-pip' for example."
if check_os.check_opensuse():
hint = " Install with 'zypper install python-pip' for example."
raise RuntimeError(
"pip isn't available, install it and rerun the script, please!%s" %
(hint, ))
# python augeas bindings are not availble in pip, nor in easy_install
try:
import augeas # just for testing that pm_utils dependencies are fulfilled
except ImportError:
hint = ""
if check_os.check_ubuntu() or check_os.check_debian(
) or check_os.check_linuxmint():
hint = " Install with 'sudo apt-get install python-augeas' for example."
if check_os.check_opensuse():
def start_db(postgresql_pgalise_version=postgresql_pgalise_version_default, postgresql_osm_version=postgresql_osm_version_default, pgalise_data_dir=pgalise_data_dir_default, osm_data_dir=osm_data_dir_default):
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
postgresql_pgalise_version_string = string.join([str(x) for x in postgresql_pgalise_version], ".")
postgresql_osm_version_string = string.join([str(x) for x in postgresql_osm_version], ".")
if check_os.check_ubuntu() or check_os.check_debian() or check_os.check_linuxmint():
pgalise_psql = "/usr/lib/postgresql/%s/bin/psql" % postgresql_pgalise_version_string
pgalise_initdb = "/usr/lib/postgresql/%s/bin/initdb" % postgresql_pgalise_version_string
pgalise_createdb = "/usr/lib/postgresql/%s/bin/createdb" % postgresql_pgalise_version_string
pgalise_postgres = "/usr/lib/postgresql/%s/bin/postgres" % postgresql_pgalise_version_string
osm_psql = "/usr/lib/postgresql/%s/bin/psql" % postgresql_osm_version_string
osm_initdb = "/usr/lib/postgresql/%s/bin/initdb" % postgresql_osm_version_string
osm_createdb = "/usr/lib/postgresql/%s/bin/createdb" % postgresql_osm_version_string
osm_postgres = "/usr/lib/postgresql/%s/bin/postgres" % postgresql_osm_version_string
elif check_os.check_opensuse():
opensuse_postgresql_pgalise_version_string = string.join([str(x) for x in postgresql_pgalise_version], "")
pgalise_psql = "/usr/lib/postgresql%s/bin/psql" % opensuse_postgresql_pgalise_version_string
pgalise_initdb = "/usr/lib/postgresql%s/bin/initdb" % opensuse_postgresql_pgalise_version_string
pgalise_createdb = "/usr/lib/postgresql%s/bin/createdb" % opensuse_postgresql_pgalise_version_string
pgalise_postgres = "/usr/lib/postgresql%s/bin/postgres" % opensuse_postgresql_pgalise_version_string
opensuse_postgresql_osm_version_string = string.join([str(x) for x in postgresql_osm_version], "")
osm_psql = "/usr/lib/postgresql%s/bin/psql" % opensuse_postgresql_osm_version_string
osm_initdb = "/usr/lib/postgresql%s/bin/initdb" % opensuse_postgresql_osm_version_string
osm_createdb = "/usr/lib/postgresql%s/bin/createdb" % opensuse_postgresql_osm_version_string
osm_postgres = "/usr/lib/postgresql%s/bin/postgres" % opensuse_postgresql_osm_version_string
else:
# better to let the script fail here than to get some less comprehensive error message later
raise RuntimeError("operating system not supported!")
log_dir = os.path.join(base_dir_path, "log")
if not os.path.exists(log_dir):
os.mkdir(log_dir)
timestamp_suffix = datetime.datetime.today().isoformat()
terminal_cmds = ["xterm", "-hold", "-e"]
postgres_log_file_path = os.path.join(log_dir, "postgis-%s.log" % timestamp_suffix)
postgres_log_file = None # open(postgres_log_file_path, "w") # None doesn't work correctly on OpenSUSE (no output) (not sure which xterm version)
postgres_proc = sp32.Popen(
terminal_cmds+[pgalise_postgres, "-D", pgalise_data_dir, "-p", "5201", "-h", "localhost", "-k", tmp_dir],
stdout=postgres_log_file,
start_new_session=True, # necessary in order to prevent SIGINT being sent to postgres process which causes messed up data directries and stale pid files
)
print("started pgalise postgresql database process with data directory %s" % postgres_log_file_path)
if postgres_log_file_path != None:
print("logging output of postgres process to %s" % (postgres_log_file_path))
osm_postgres_log_file_path = os.path.join(log_dir, "postgis-osm-%s.log" % timestamp_suffix)
osm_postgres_log_file = None # open(osm_postgres_log_file_path, "w") # None doesn't work correctly on OpenSUSE (no output) (not sure which xterm version)
osm_postgres_proc = sp32.Popen(
terminal_cmds+[osm_postgres, "-D", osm_data_dir, "-p", "5204", "-h", "localhost", "-k", tmp_dir],
stdout=osm_postgres_log_file,
start_new_session=True, # necessary in order to prevent SIGINT being sent to postgres process which causes messed up data directries and stale pid files
)
print("started PostGIS postgresql database process with data directory %s" % (osm_data_dir))
if osm_postgres_log_file != None:
print("logging output of OSM postgres process to %s" % osm_postgres_log_file_path)
print("waiting for both terminal processes (of pgalise and PostGIS database processes) to terminate (kill them with SIGTERM signal (on Debian systems and other with Strg+C) and close terminal windows)")
global shutdown_requested
shutdown_requested = False
def signal_handler(signal, frame):
global shutdown_requested
if not shutdown_requested:
print('You pressed Ctrl+C or sent SIGINT otherwise, try to send SIGINT to the database processes first (press Ctrl+C in the child terminals and close the windows) in order to shutdown the database processes gracefully! Press Ctrl+C again to force shutdown')
shutdown_requested = True
return
postgres_proc.terminate()
osm_postgres_proc.terminate()
signal.signal(signal.SIGINT, signal_handler)
postgres_proc.communicate() # necessary to prevent Popen from waiting for ever
postgres_proc.wait()
osm_postgres_proc.communicate()
osm_postgres_proc.wait()
def bootstrap(ignore_root_warning, force_overwrite_postgres_datadir, skip_tests, bootstrap_dir=bootstrap_dir_default, skip_build=False, psql=psql, initdb=initdb, createdb=createdb, postgres=postgres, postgres_datadir_path=postgres_datadir_path_default, postgresql_jdbc_url=postgresql_jdbc_url_default, postgresql_deb_url=postgresql_deb_url_default, geotools_url=geotools_url_default, jgrapht_url=jgrapht_url_default, jfuzzy_url=jfuzzy_url_default, maven_bin_archive_url=maven_bin_archive_url_default, commons_src_archive_url=commons_src_archive_url_default, postgis_install="pm", postgis_url=postgis_url_default, ):
"Bootstrap the PGALISE simulation, including installation of dependencies (those which can't be fetched by maven), binaries (postgresql, postgis, etc.), setup of database in "
# force specification of flag if script is invoked as root
if os.getuid() == 0 and not ignore_root_warning:
raise RuntimeError("Script is invoked as root which is strongly discouraged. Specify the -I/--ignore-root-warning in order to overwrite. You have been warned!")
# setup directories
script_dir = os.path.join(base_dir, "scripts")
external_dir = os.path.join(bootstrap_dir, "external")
internal_dir = os.path.join(bootstrap_dir, "internal")
# used for postgresql socket
tmp_dir = os.path.join(script_dir, "tmp") # "/tmp less save than dir only readable by user"
external_bin_dir = os.path.join(external_dir, "bin")
external_src_dir = os.path.join(external_dir, "src")
if not os.path.exists(bin_dir):
os.makedirs(bin_dir)
if not os.path.exists(bootstrap_dir):
os.makedirs(bootstrap_dir)
if not os.path.exists(external_dir):
os.makedirs(external_dir)
if not os.path.exists(external_bin_dir):
os.makedirs(external_bin_dir)
if not os.path.exists(external_src_dir):
os.makedirs(external_src_dir)
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
user = sp.check_output([whoami])
postgres_socket_dir = tmp_dir # is this ok?
# check some prequisites (after installation as they might be installed properly)
java_home = os.getenv("JAVA_HOME") # necessary for mvn
if java_home is None or java_home == "":
raise RuntimeError("JAVA_HOME is not set")
java_binary = os.path.join(java_home, "bin/java")
if not os.path.exists(java_binary):
raise RuntimeError("JAVA_HOME %s doesn't seem to point to a valid JDK (java binary '%s' doesn't exist)" % (java_home, java_binary))
# install postgresql jdbc driver
postgresql_jdbc_file = os.path.join(external_bin_dir, postgresql_jdbc_name)
if not file_utils.check_file(postgresql_jdbc_file, postgresql_jdbc_md5):
do_wget(postgresql_jdbc_url, postgresql_jdbc_file)
if pg_version == (9,2):
postgresql_jdbc_mvn_version = "9.2-1004.jdbc4"
elif pg_version == (9,3):
postgresql_jdbc_mvn_version = "9.2-1004.jdbc4" #@TODO: not optimal to have different client and server versions
else:
raise RuntimeError("postgresql version %s not supported" % string.join([str(x) for x in pg_version],"."))
sp.check_call([mvn, "install:install-file", \
"-Dfile=%s" % postgresql_jdbc_file, "-DartifactId=postgresql", \
"-DgroupId=postgresql", "-Dversion=%s" % postgresql_jdbc_mvn_version, "-Dpackaging=jar"], cwd=external_bin_dir)
# install jgrapht (install net.sf.jgrapht:jgrapht:0.8.3 as org.jgrapht:jgrapht:0.8.3 (could not be found in any repository))
jgrapht_file = os.path.join(external_bin_dir, jgrapht_name)
if not file_utils.check_file(jgrapht_file, jgrapht_md5):
do_wget(jgrapht_url, jgrapht_file)
sp.check_call([mvn, "install:install-file", \
"-Dfile=%s" % jgrapht_file, "-DartifactId=jgrapht",
"-DgroupId=org.jgrapht", "-Dversion=0.8.3", "-Dpackaging=jar"], cwd=external_bin_dir)
# install jfuzzylogic
jfuzzy_file = os.path.join(external_bin_dir, jfuzzy_name)
if not file_utils.check_file(jfuzzy_file, jfuzzy_md5):
do_wget(jfuzzy_url, jfuzzy_file)
sp.check_call([mvn, "install:install-file", \
"-Dfile=%s" % jfuzzy_file, "-DartifactId=jfuzzy",
"-DgroupId=pcingola", "-Dversion=3.0", "-Dpackaging=jar"], cwd=external_bin_dir)
# geronimo not available as binary (but depends on eclipse source stuff...)
#geronimo_file_name = "geronimo-3.0.1"
#geronimo_src_archive_name = "geronimo-3.0.1-source-release.zip"
#geronimo_url = "http://mirrors.ibiblio.org/maven2/org/apache/geronimo/geronimo/3.0.1/geronimo-3.0.1-source-release.zip"
#geronimo_file_path = os.path.join(external_src_dir, geronimo_file_name)
# install geotools
#geotools_src_dir = os.path.join(external_src_dir, geotools_src_dir_name)
#if not file_utils.check_dir(geotools_src_dir):
# if not ch...le(os.path.join(external_src_dir, geotools_src_archive_name)) or file_utils.retrieve_md5sum(os.path.join(external_src_dir, geotools_src_archive_name)) != geotools_src_archive_md5:
# do_wget() ... sp.check_call([wget, geotools_url], cwd=tmp_dir)
# sp.check_call([unzip, os.path.join(tmp_dir, geotools_src_archive_name)], cwd=external_src_dir)
#if not skip_build:
# mvn_settings_file_path = os.path.join(script_dir, "settings.xml")
# sp.check_call([mvn, "-e", "--global-settings", mvn_settings_file_path, "--settings", mvn_settings_file_path, "install", "-Dall", "-DskipTests=true"], cwd=geotools_src_dir)
# install commons-collections 4
#commons_src_dir = os.path.join(external_src_dir, commons_src_dir_name)
#if not file_utils.check_dir(commons_src_dir):
# commons_src_archive_file = os.path.join(external_src_dir, commons_src_archive_name)
# if not file_utils.check_file(commons_src_archive_file, commons_src_archive_md5):
# do_wget(commons_src_archive_url, commons_src_archive_file)
# sp.check_call([tar, "xf", commons_src_archive_file], cwd=external_src_dir)
#if not skip_build:
# mvn_install(commons_src_dir, skip_tests)
# install postgis from source (installation with package manager occurs in
# bootstrap_privileged.py
if postgis_install == "source":
# installation of postgis-jdbc from bin directory should not occur in sources installation as this refers to the version shipped with the tarball -> will install mvn project in subdirectory of tarball root
postgis_src_dir = os.path.join(external_src_dir, postgis_src_dir_name)
if check_os.check_ubuntu() or check_os.check_debian():
if not file_utils.check_dir(postgis_src_dir):
postgis_src_archive_file = os.path.join(external_src_dir, postgis_src_archive_name)
if not file_utils.check_file(postgis_src_archive_file, postgis_src_archive_md5):
do_wget(postgis_url, postgis_src_archive_file)
sp.check_call([tar, "xf", postgis_src_archive_file], cwd=external_src_dir)
if not skip_build:
pm_utils.install_apt_get_build_dep(["postgis"], package_manager=apt_get, skip_apt_update=skip_apt_update) # might not be sufficient because Ubuntu 13.10's version of postgis is 1.5.x (we're using 2.x)
pm_utils.install_packages(["libgdal-dev"], package_manager=apt_get, skip_apt_update=skip_apt_update) # not covered by 1.5.x requirements (see above)
sp.check_call([bash, "autogen.sh"], cwd=postgis_src_dir)
sp.check_call([bash, "configure"], cwd=postgis_src_dir)
sp.check_call([make, "-j8"], cwd=postgis_src_dir)
sp.check_call([make, "install"], cwd=postgis_src_dir)
# install postgis-jdbc (project is shipped inside postgis project (as long as a fixed version of postgis is used here, the version of postgis-jdbc will be fixed, too (postgis 2.1.1 -> postgis-jdbc 2.1.0SVN)
postgis_mvn_project_path = os.path.join(postgis_src_dir, "java/jdbc")
sp.check_call([ant], cwd=postgis_mvn_project_path) # generates maven project
mvn_install(postgis_mvn_project_path, skip_tests)
elif check_os.check_opensuse():
raise RuntimeError("PostGIS source installation not supported")
else:
# better to let the script fail here than to get some less comprehensive error message later
raise RuntimeError("Operating system not supported!")
elif postgis_install == "pm":
# install postgis from scripts/bin (as it is not available from somewhere online)
postgis_jdbc_file = os.path.join(bin_dir, postgis_jdbc_name)
if not os.path.exists(postgis_jdbc_file):
raise RuntimeError("postgis JDBC jar %s doesn't exist, can't continue, consider fetching it manually" % (postgis_jdbc_file,))
sp.check_call([mvn, "install:install-file", \
"-Dfile=%s" % postgis_jdbc_file, "-DartifactId=postgis-jdbc",
"-DgroupId=org.postgis", "-Dversion=2.1.0SVN", "-Dpackaging=jar"], cwd=bin_dir)
else:
raise RuntimeError("postgis_install %s isn't supported" % (postgis_install,))
# always install openejb from bin directory (might be made flexible with source installation like postgis (see above) later)
openejb_jar_file_path = os.path.join(bin_dir, openejb_jar_file_name)
if not os.path.exists(openejb_jar_file_path):
raise RuntimeError("OpenEJB jar %s doesn't exist, can't continue, consider fetching it manually" % (openejb_jar_file_path,))
sp.check_call([mvn, "install:install-file", \
"-Dfile=%s" % (openejb_jar_file_path,), "-DartifactId=openejb-core",
"-DgroupId=org.apache.openejb", "-Dversion=4.7.0-SNAPSHOT", "-Dpackaging=jar"], cwd=bin_dir)
openejb_api_jar_file_path = os.path.join(bin_dir, openejb_api_jar_file_name)
if not os.path.exists(openejb_api_jar_file_path):
raise RuntimeError("OpenEJB API jar %s doesn't exist, can't continue, consider fetching it manually" % (openejb_api_jar_file_path,))
sp.check_call([mvn, "install:install-file",
"-Dfile=%s" % (openejb_api_jar_file_path,), "-DartifactId=openejb-api",
"-DgroupId=org.apache.openejb", "-Dversion=4.7.0-SNAPSHOT", "-Dpackaging=jar"], cwd=bin_dir)
#openejb_snapshot_ahtutils_workaround(external_src_dir)
# setup postgis datadir and configuration
if not file_utils.check_dir(postgres_datadir_path) or force_overwrite_postgres_datadir:
# os.makedirs(postgres_datadir_path) # causes error if directory exists and is not necessary
postgis_utils.bootstrap_datadir(postgres_datadir_path, postgres_user, password=postgres_pw, initdb=initdb)
postgis_utils.bootstrap_database(postgres_datadir_path, postgres_port, postgres_host, postgres_user, pgalise_db_name, password=postgres_pw, initdb=initdb, postgres=postgres, createdb=createdb, psql=psql)
postgis_utils.bootstrap_database(postgres_datadir_path, postgres_port, postgres_host, postgres_user, pgalise_db_test_name, password=postgres_pw, initdb=initdb, postgres=postgres, createdb=createdb, psql=psql)
else:
logger.info("Postgres datadir %s has not been overwritten. Check output of script invoked with -h/--help on how to do that." % (postgres_datadir_path, ))