def bootstrap_privileged(skip_apt_update=skip_apt_update_default, postgis_install=postgis_install_default, pg_version=pg_version_default): if not os.path.exists(bin_dir): os.makedirs(bin_dir) if check_os.check_ubuntu() or check_os.check_debian() or check_os.check_linuxmint(): pm_utils.install_packages(["maven", "openjdk-7-jdk", "ant", # for postgis-jdbc "sudo", # very small probability that it is not installed, but it is a prequisite of the script... "software-properties-common", # provides add-apt-repository which is used by pm_utils module in osm_postgis_transform_prequisites.install_postgresql "python-software-properties", # provides add-apt-repository on Ubuntu 12.04.4, is available in Ubuntu 14.04 "xterm", ], package_manager=apt_get, skip_apt_update=skip_apt_update) elif check_os.check_opensuse(): # install maven #sp.check_call(["/sbin/OCICLI", "http://software.opensuse.org/ymp/Application:Geo/openSUSE_12.3/maven.ymp?base=openSUSE%3A12.3&query=maven"]) # (not sure whether this installs futher instable software, provided maven version 3.0.4 isn't a hit neither) maven_bin_dir = os.path.join(external_bin_dir, maven_bin_dir_name) if not check_dir(maven_bin_dir): maven_bin_archive = os.path.join(external_bin_dir, maven_bin_archive_name) if not check_file(maven_bin_archive, maven_bin_archive_md5): do_wget(maven_bin_archive_url, maven_bin_archive) sp.check_call([tar, "xf", maven_bin_archive], cwd=external_bin_dir) shutil.copytree(maven_bin_dir, maven_bin_dir_install_target) sp.check_call(chown, "-Rc", "%s:%s" %(user,user), dirpath) # install remaining prequisites sp.check_call([zypper, "install", "java-1_7_0-openjdk", "java-1_7_0-openjdk-devel", "java-1_7_0-openjdk-src", "java-1_7_0-openjdk-javadoc"]) else: # better to let the script fail here than to get some less comprehensive error message later raise RuntimeError("operating system not supported!") # postgresql osm_postgis_transform_prequisites.install_postgresql(pg_version=pg_version, skip_apt_update=skip_apt_update) # install postgis (demote for source installation) if postgis_install == "pm": if check_os.check_ubuntu() or check_os.check_debian() or check_os.check_linuxmint(): packages = ["postgis"] # 'postgis-2.1' interpreted as regular expression in apt! # versions for which conditions exist are not necessarily supported! if pg_version == (9,2): packages += ["postgresql-9.2-postgis-2.1", "postgresql-contrib-9.2", "postgresql-9.2-postgis-2.1-scripts"] elif pg_version == (9,3): packages += ["postgresql-9.3-postgis-2.1", "postgresql-contrib-9.3", "postgresql-9.3-postgis-2.1-scripts"] else: raise ValueError("postgresql version not supported") sp.call([service, "stop", "postgresql"]) # crashes installation if port 5432 is occupied pm_utils.install_packages(packages, package_manager=apt_get, skip_apt_update=skip_apt_update) elif check_os.check_opensuse(): sp.check_call([zypper, "install", "postgis2", "postgis2-devel", "postgis2-utils"]) else: raise ValueError("operating system not supported") elif postgis_install == "source": logger.info("postgis installation from source occurs (unprivileged) in bootstrap.py") else: raise ValueError("postgis_install has to be one of %s" % str(postgis_installs)) # prequisites for start_db pm_utils.install_packages([ "python-pip", "python-dev", # soft dependency of `pip install subprocess32`, not fulfilled, e.g. on Debian 7.4 ], package_manager="apt-get") sp.check_call([pip, "install", "--upgrade", "setuptools"]) # saves a lot of trouble and hurts much less than it helps sp.check_call([pip, "install", "subprocess32", "pexpect"]) # pip manages update of available import automatically so that import xxx can be invoked, already installed packages don't cause returncode != 0
sp.check_call([git, "checkout", "master"], cwd=checkout_target) # get on a branch because HEAD might be detached (then the following git pull without remote specification would fail) sp.check_call([git, "pull", "origin", "master"], cwd=checkout_target) # not 100 % sure why this is necessary except Exception as ex: logger.error("directory '%s' exists, but updating python-essentials failed (see the following exception for details), something has to be wrong. If you can't figure out what, remove '%s' and run the script again" % (checkout_target, checkout_target)) raise ex else: logger.info("checking out python-essentials as script dependency") sp.check_call([git, "clone", "https://github.com/krichter722/python-essentials.git"], cwd=base_dir) sys.path.append(checkout_target) sys.path.append(os.path.join(checkout_target, "lib")) import os_utils import check_os if os_utils.which("pip") is None: hint = "" if check_os.check_ubuntu() or check_os.check_debian() or check_os.check_linuxmint(): hint = " Install with 'sudo apt-get install python-pip' for example." if check_os.check_opensuse(): hint = " Install with 'zypper install python-pip' for example." raise RuntimeError("pip isn't available, install it and rerun the script, please!%s" % (hint,)) # python augeas bindings are not availble in pip, nor in easy_install try: import augeas # just for testing that pm_utils dependencies are fulfilled except ImportError: hint = "" if check_os.check_ubuntu() or check_os.check_debian() or check_os.check_linuxmint(): hint = " Install with 'sudo apt-get install python-augeas' for example." if check_os.check_opensuse(): hint = " Install with 'zypper install python-augeas' for example." raise RuntimeError("augeas python bindings aren't available, install it and rerun the script, please!%s" % (hint,))
def start_db(postgresql_pgalise_version=postgresql_pgalise_version_default, postgresql_osm_version=postgresql_osm_version_default, pgalise_data_dir=pgalise_data_dir_default, osm_data_dir=osm_data_dir_default): if not os.path.exists(tmp_dir): os.makedirs(tmp_dir) postgresql_pgalise_version_string = string.join( [str(x) for x in postgresql_pgalise_version], ".") postgresql_osm_version_string = string.join( [str(x) for x in postgresql_osm_version], ".") if check_os.check_ubuntu() or check_os.check_debian( ) or check_os.check_linuxmint(): pgalise_psql = "/usr/lib/postgresql/%s/bin/psql" % postgresql_pgalise_version_string pgalise_initdb = "/usr/lib/postgresql/%s/bin/initdb" % postgresql_pgalise_version_string pgalise_createdb = "/usr/lib/postgresql/%s/bin/createdb" % postgresql_pgalise_version_string pgalise_postgres = "/usr/lib/postgresql/%s/bin/postgres" % postgresql_pgalise_version_string osm_psql = "/usr/lib/postgresql/%s/bin/psql" % postgresql_osm_version_string osm_initdb = "/usr/lib/postgresql/%s/bin/initdb" % postgresql_osm_version_string osm_createdb = "/usr/lib/postgresql/%s/bin/createdb" % postgresql_osm_version_string osm_postgres = "/usr/lib/postgresql/%s/bin/postgres" % postgresql_osm_version_string elif check_os.check_opensuse(): opensuse_postgresql_pgalise_version_string = string.join( [str(x) for x in postgresql_pgalise_version], "") pgalise_psql = "/usr/lib/postgresql%s/bin/psql" % opensuse_postgresql_pgalise_version_string pgalise_initdb = "/usr/lib/postgresql%s/bin/initdb" % opensuse_postgresql_pgalise_version_string pgalise_createdb = "/usr/lib/postgresql%s/bin/createdb" % opensuse_postgresql_pgalise_version_string pgalise_postgres = "/usr/lib/postgresql%s/bin/postgres" % opensuse_postgresql_pgalise_version_string opensuse_postgresql_osm_version_string = string.join( [str(x) for x in postgresql_osm_version], "") osm_psql = "/usr/lib/postgresql%s/bin/psql" % opensuse_postgresql_osm_version_string osm_initdb = "/usr/lib/postgresql%s/bin/initdb" % opensuse_postgresql_osm_version_string osm_createdb = "/usr/lib/postgresql%s/bin/createdb" % opensuse_postgresql_osm_version_string osm_postgres = "/usr/lib/postgresql%s/bin/postgres" % opensuse_postgresql_osm_version_string else: # better to let the script fail here than to get some less comprehensive error message later raise RuntimeError("operating system not supported!") log_dir = os.path.join(base_dir_path, "log") if not os.path.exists(log_dir): os.mkdir(log_dir) timestamp_suffix = datetime.datetime.today().isoformat() terminal_cmds = ["xterm", "-hold", "-e"] postgres_log_file_path = os.path.join(log_dir, "postgis-%s.log" % timestamp_suffix) postgres_log_file = None # open(postgres_log_file_path, "w") # None doesn't work correctly on OpenSUSE (no output) (not sure which xterm version) postgres_proc = sp32.Popen( terminal_cmds + [ pgalise_postgres, "-D", pgalise_data_dir, "-p", "5201", "-h", "localhost", "-k", tmp_dir ], stdout=postgres_log_file, start_new_session= True, # necessary in order to prevent SIGINT being sent to postgres process which causes messed up data directries and stale pid files ) print( "started pgalise postgresql database process with data directory %s" % postgres_log_file_path) if postgres_log_file_path != None: print("logging output of postgres process to %s" % (postgres_log_file_path)) osm_postgres_log_file_path = os.path.join( log_dir, "postgis-osm-%s.log" % timestamp_suffix) osm_postgres_log_file = None # open(osm_postgres_log_file_path, "w") # None doesn't work correctly on OpenSUSE (no output) (not sure which xterm version) osm_postgres_proc = sp32.Popen( terminal_cmds + [ osm_postgres, "-D", osm_data_dir, "-p", "5204", "-h", "localhost", "-k", tmp_dir ], stdout=osm_postgres_log_file, start_new_session= True, # necessary in order to prevent SIGINT being sent to postgres process which causes messed up data directries and stale pid files ) print( "started PostGIS postgresql database process with data directory %s" % (osm_data_dir)) if osm_postgres_log_file != None: print("logging output of OSM postgres process to %s" % osm_postgres_log_file_path) print( "waiting for both terminal processes (of pgalise and PostGIS database processes) to terminate (kill them with SIGTERM signal (on Debian systems and other with Strg+C) and close terminal windows)" ) global shutdown_requested shutdown_requested = False def signal_handler(signal, frame): global shutdown_requested if not shutdown_requested: print( 'You pressed Ctrl+C or sent SIGINT otherwise, try to send SIGINT to the database processes first (press Ctrl+C in the child terminals and close the windows) in order to shutdown the database processes gracefully! Press Ctrl+C again to force shutdown' ) shutdown_requested = True return postgres_proc.terminate() osm_postgres_proc.terminate() signal.signal(signal.SIGINT, signal_handler) postgres_proc.communicate( ) # necessary to prevent Popen from waiting for ever postgres_proc.wait() osm_postgres_proc.communicate() osm_postgres_proc.wait()
def bootstrap_privileged(skip_apt_update=skip_apt_update_default, postgis_install=postgis_install_default, pg_version=pg_version_default): if not os.path.exists(bin_dir): os.makedirs(bin_dir) if check_os.check_ubuntu() or check_os.check_debian( ) or check_os.check_linuxmint(): pm_utils.install_packages( [ "maven", "openjdk-7-jdk", "ant", # for postgis-jdbc "sudo", # very small probability that it is not installed, but it is a prequisite of the script... "software-properties-common", # provides add-apt-repository which is used by pm_utils module in osm_postgis_transform_prequisites.install_postgresql "python-software-properties", # provides add-apt-repository on Ubuntu 12.04.4, is available in Ubuntu 14.04 "xterm", ], package_manager=apt_get, skip_apt_update=skip_apt_update) elif check_os.check_opensuse(): # install maven #sp.check_call(["/sbin/OCICLI", "http://software.opensuse.org/ymp/Application:Geo/openSUSE_12.3/maven.ymp?base=openSUSE%3A12.3&query=maven"]) # (not sure whether this installs futher instable software, provided maven version 3.0.4 isn't a hit neither) maven_bin_dir = os.path.join(external_bin_dir, maven_bin_dir_name) if not check_dir(maven_bin_dir): maven_bin_archive = os.path.join(external_bin_dir, maven_bin_archive_name) if not check_file(maven_bin_archive, maven_bin_archive_md5): do_wget(maven_bin_archive_url, maven_bin_archive) sp.check_call([tar, "xf", maven_bin_archive], cwd=external_bin_dir) shutil.copytree(maven_bin_dir, maven_bin_dir_install_target) sp.check_call(chown, "-Rc", "%s:%s" % (user, user), dirpath) # install remaining prequisites sp.check_call([ zypper, "install", "java-1_7_0-openjdk", "java-1_7_0-openjdk-devel", "java-1_7_0-openjdk-src", "java-1_7_0-openjdk-javadoc" ]) else: # better to let the script fail here than to get some less comprehensive error message later raise RuntimeError("operating system not supported!") # postgresql osm_postgis_transform_prequisites.install_postgresql( pg_version=pg_version, skip_apt_update=skip_apt_update) # install postgis (demote for source installation) if postgis_install == "pm": if check_os.check_ubuntu() or check_os.check_debian( ) or check_os.check_linuxmint(): packages = [ "postgis" ] # 'postgis-2.1' interpreted as regular expression in apt! # versions for which conditions exist are not necessarily supported! if pg_version == (9, 2): packages += [ "postgresql-9.2-postgis-2.1", "postgresql-contrib-9.2", "postgresql-9.2-postgis-2.1-scripts" ] elif pg_version == (9, 3): packages += [ "postgresql-9.3-postgis-2.1", "postgresql-contrib-9.3", "postgresql-9.3-postgis-2.1-scripts" ] else: raise ValueError("postgresql version not supported") sp.call([service, "stop", "postgresql" ]) # crashes installation if port 5432 is occupied pm_utils.install_packages(packages, package_manager=apt_get, skip_apt_update=skip_apt_update) elif check_os.check_opensuse(): sp.check_call([ zypper, "install", "postgis2", "postgis2-devel", "postgis2-utils" ]) else: raise ValueError("operating system not supported") elif postgis_install == "source": logger.info( "postgis installation from source occurs (unprivileged) in bootstrap.py" ) else: raise ValueError("postgis_install has to be one of %s" % str(postgis_installs)) # prequisites for start_db pm_utils.install_packages( [ "python-pip", "python-dev", # soft dependency of `pip install subprocess32`, not fulfilled, e.g. on Debian 7.4 ], package_manager="apt-get") sp.check_call([ pip, "install", "--upgrade", "setuptools" ]) # saves a lot of trouble and hurts much less than it helps sp.check_call( [pip, "install", "subprocess32", "pexpect"] ) # pip manages update of available import automatically so that import xxx can be invoked, already installed packages don't cause returncode != 0
else: logger.info("checking out python-essentials as script dependency") sp.check_call([ git, "clone", "https://github.com/krichter722/python-essentials.git" ], cwd=base_dir) sys.path.append(checkout_target) sys.path.append(os.path.join(checkout_target, "lib")) import os_utils import check_os if os_utils.which("pip") is None: hint = "" if check_os.check_ubuntu() or check_os.check_debian( ) or check_os.check_linuxmint(): hint = " Install with 'sudo apt-get install python-pip' for example." if check_os.check_opensuse(): hint = " Install with 'zypper install python-pip' for example." raise RuntimeError( "pip isn't available, install it and rerun the script, please!%s" % (hint, )) # python augeas bindings are not availble in pip, nor in easy_install try: import augeas # just for testing that pm_utils dependencies are fulfilled except ImportError: hint = "" if check_os.check_ubuntu() or check_os.check_debian( ) or check_os.check_linuxmint(): hint = " Install with 'sudo apt-get install python-augeas' for example." if check_os.check_opensuse():
def start_db(postgresql_pgalise_version=postgresql_pgalise_version_default, postgresql_osm_version=postgresql_osm_version_default, pgalise_data_dir=pgalise_data_dir_default, osm_data_dir=osm_data_dir_default): if not os.path.exists(tmp_dir): os.makedirs(tmp_dir) postgresql_pgalise_version_string = string.join([str(x) for x in postgresql_pgalise_version], ".") postgresql_osm_version_string = string.join([str(x) for x in postgresql_osm_version], ".") if check_os.check_ubuntu() or check_os.check_debian() or check_os.check_linuxmint(): pgalise_psql = "/usr/lib/postgresql/%s/bin/psql" % postgresql_pgalise_version_string pgalise_initdb = "/usr/lib/postgresql/%s/bin/initdb" % postgresql_pgalise_version_string pgalise_createdb = "/usr/lib/postgresql/%s/bin/createdb" % postgresql_pgalise_version_string pgalise_postgres = "/usr/lib/postgresql/%s/bin/postgres" % postgresql_pgalise_version_string osm_psql = "/usr/lib/postgresql/%s/bin/psql" % postgresql_osm_version_string osm_initdb = "/usr/lib/postgresql/%s/bin/initdb" % postgresql_osm_version_string osm_createdb = "/usr/lib/postgresql/%s/bin/createdb" % postgresql_osm_version_string osm_postgres = "/usr/lib/postgresql/%s/bin/postgres" % postgresql_osm_version_string elif check_os.check_opensuse(): opensuse_postgresql_pgalise_version_string = string.join([str(x) for x in postgresql_pgalise_version], "") pgalise_psql = "/usr/lib/postgresql%s/bin/psql" % opensuse_postgresql_pgalise_version_string pgalise_initdb = "/usr/lib/postgresql%s/bin/initdb" % opensuse_postgresql_pgalise_version_string pgalise_createdb = "/usr/lib/postgresql%s/bin/createdb" % opensuse_postgresql_pgalise_version_string pgalise_postgres = "/usr/lib/postgresql%s/bin/postgres" % opensuse_postgresql_pgalise_version_string opensuse_postgresql_osm_version_string = string.join([str(x) for x in postgresql_osm_version], "") osm_psql = "/usr/lib/postgresql%s/bin/psql" % opensuse_postgresql_osm_version_string osm_initdb = "/usr/lib/postgresql%s/bin/initdb" % opensuse_postgresql_osm_version_string osm_createdb = "/usr/lib/postgresql%s/bin/createdb" % opensuse_postgresql_osm_version_string osm_postgres = "/usr/lib/postgresql%s/bin/postgres" % opensuse_postgresql_osm_version_string else: # better to let the script fail here than to get some less comprehensive error message later raise RuntimeError("operating system not supported!") log_dir = os.path.join(base_dir_path, "log") if not os.path.exists(log_dir): os.mkdir(log_dir) timestamp_suffix = datetime.datetime.today().isoformat() terminal_cmds = ["xterm", "-hold", "-e"] postgres_log_file_path = os.path.join(log_dir, "postgis-%s.log" % timestamp_suffix) postgres_log_file = None # open(postgres_log_file_path, "w") # None doesn't work correctly on OpenSUSE (no output) (not sure which xterm version) postgres_proc = sp32.Popen( terminal_cmds+[pgalise_postgres, "-D", pgalise_data_dir, "-p", "5201", "-h", "localhost", "-k", tmp_dir], stdout=postgres_log_file, start_new_session=True, # necessary in order to prevent SIGINT being sent to postgres process which causes messed up data directries and stale pid files ) print("started pgalise postgresql database process with data directory %s" % postgres_log_file_path) if postgres_log_file_path != None: print("logging output of postgres process to %s" % (postgres_log_file_path)) osm_postgres_log_file_path = os.path.join(log_dir, "postgis-osm-%s.log" % timestamp_suffix) osm_postgres_log_file = None # open(osm_postgres_log_file_path, "w") # None doesn't work correctly on OpenSUSE (no output) (not sure which xterm version) osm_postgres_proc = sp32.Popen( terminal_cmds+[osm_postgres, "-D", osm_data_dir, "-p", "5204", "-h", "localhost", "-k", tmp_dir], stdout=osm_postgres_log_file, start_new_session=True, # necessary in order to prevent SIGINT being sent to postgres process which causes messed up data directries and stale pid files ) print("started PostGIS postgresql database process with data directory %s" % (osm_data_dir)) if osm_postgres_log_file != None: print("logging output of OSM postgres process to %s" % osm_postgres_log_file_path) print("waiting for both terminal processes (of pgalise and PostGIS database processes) to terminate (kill them with SIGTERM signal (on Debian systems and other with Strg+C) and close terminal windows)") global shutdown_requested shutdown_requested = False def signal_handler(signal, frame): global shutdown_requested if not shutdown_requested: print('You pressed Ctrl+C or sent SIGINT otherwise, try to send SIGINT to the database processes first (press Ctrl+C in the child terminals and close the windows) in order to shutdown the database processes gracefully! Press Ctrl+C again to force shutdown') shutdown_requested = True return postgres_proc.terminate() osm_postgres_proc.terminate() signal.signal(signal.SIGINT, signal_handler) postgres_proc.communicate() # necessary to prevent Popen from waiting for ever postgres_proc.wait() osm_postgres_proc.communicate() osm_postgres_proc.wait()