def __init__(self, queue, cache, estimated_records=None, merge=False, log=None,
marshaled_data=False):
Process.__init__(self)
self.daemon = True
setproctitle('imposm writer')
self.queue = queue
self.cache = cache
self.merge = merge
self.log = log
self.marshaled_data = marshaled_data
self.estimated_records = estimated_records
def __init__(self, in_queue, db, mapper, osm_cache, dry_run):
Process.__init__(self)
self.daemon = True
setproctitle('imposm %s process' % self.name)
self.in_queue = in_queue
self.mapper = mapper
self.osm_cache = osm_cache
self.db = db
self.dry_run = dry_run
self.db_queue = Queue(256)
def __init__(self,
queue,
cache,
estimated_records=None,
merge=False,
log=None,
marshaled_data=False):
Process.__init__(self)
self.daemon = True
setproctitle('imposm writer')
self.queue = queue
self.cache = cache
self.merge = merge
self.log = log
self.marshaled_data = marshaled_data
self.estimated_records = estimated_records
def main(argv=None):
setproctitle("imposm: main")
usage = "%prog [options] [input]..."
parser = optparse.OptionParser(usage=usage, add_help_option=False, version="%prog " + __version__)
parser.add_option("--help", dest="help", action="store_true", default=False, help="show this help message and exit")
parser.add_option("--debug", action="store_true", default=False, help="show debug information")
parser.add_option("--quiet", action="store_true", default=False, help="only print progress every 60 seconds")
parser.add_option("-m", "--mapping-file", dest="mapping_file", metavar="<file>")
parser.add_option("-h", "--host", dest="host", metavar="<host>")
parser.add_option("-p", "--port", dest="port", metavar="<port>")
parser.add_option("-d", "--database", dest="db", metavar="<dbname>")
parser.add_option("-U", "--user", dest="user", metavar="<user>")
parser.add_option("--proj", dest="proj", metavar="EPSG:900913")
parser.add_option(
"--connection",
dest="connection",
help="connection string like postgis://user:pass@host:port/database,"
" this overwrites the -h/-p/-d/-U options",
)
parser.add_option("-c", "--concurrency", dest="concurrency", metavar="N", type="int", default=n_cpu)
parser.add_option("--merge-cache", dest="merge_cache", default=False, action="store_true")
parser.add_option("--overwrite-cache", dest="overwrite_cache", default=False, action="store_true")
parser.add_option(
"--cache-dir",
dest="cache_dir",
default=".",
help="path where node/ways/relations should be cached [current working dir]",
)
parser.add_option(
"--table-prefix", dest="table_prefix", default=None, metavar="osm_new_", help="prefix for imported tables"
)
parser.add_option(
"--table-prefix-production",
dest="table_prefix_production",
default="osm_",
metavar="osm_",
help="prefix for production tables",
)
parser.add_option(
"--table-prefix-backup",
dest="table_prefix_backup",
default="osm_old_",
metavar="osm_old_",
help="prefix for backup tables",
)
parser.add_option("--read", dest="read", default=False, action="store_true")
parser.add_option("--write", dest="write", default=False, action="store_true")
parser.add_option("--optimize", dest="optimize", default=False, action="store_true")
parser.add_option(
"--deploy-production-tables",
dest="deploy_tables",
default=False,
action="store_true",
help="remove backup tables, move production tables "
"to backup tables and move import tables to production tables",
)
parser.add_option(
"--recover-production-tables",
dest="recover_tables",
default=False,
action="store_true",
help="move production tables to import tables and" "move backup tables to production tables",
)
parser.add_option("--remove-backup-tables", dest="remove_backup_tables", default=False, action="store_true")
parser.add_option("-n", "--dry-run", dest="dry_run", default=False, action="store_true")
parser.add_option(
"--limit-to", dest="limit_to", metavar="file", help="limit imported geometries to (multi)polygons in EPSG:4326"
)
(options, args) = parser.parse_args(argv)
setup_logging(debug=options.debug)
if (argv and len(argv) == 0) or (not argv and len(sys.argv) == 1):
options.help = True
if not any(
[
options.read,
options.write,
options.optimize,
options.deploy_tables,
options.recover_tables,
options.remove_backup_tables,
]
):
options.help = True
if options.help:
parser.print_help()
sys.exit(1)
if options.quiet:
logger = imposm.util.QuietProgressLog
logger_parser = imposm.util.QuietParserProgress
else:
logger = imposm.util.ProgressLog
logger_parser = imposm.util.ParserProgress
if options.proj:
if ":" not in options.proj:
print "ERROR: --proj should be in EPSG:00000 format"
sys.exit(1)
# check proj if meter_to_mapunit needs to do anything
if options.proj.lower() == "epsg:4326":
imposm.mapping.import_srs_is_geographic = True
mapping_file = os.path.join(os.path.dirname(__file__), "defaultmapping.py")
if options.mapping_file:
print "loading %s as mapping" % options.mapping_file
mapping_file = options.mapping_file
polygon = None
if options.limit_to:
logger.message("## reading --limit-to %s" % options.limit_to)
polygon_timer = imposm.util.Timer("reading", logger)
polygon = load_geom(options.limit_to)
polygon_timer.stop()
if polygon is None:
print "ERROR: No valid polygon/multipolygon found"
sys.exit(1)
mappings = {}
execfile(mapping_file, mappings)
tag_mapping = TagMapper(
[m for n, m in mappings.iteritems() if isinstance(m, imposm.mapping.Mapping)], limit_to=polygon
)
if "IMPOSM_MULTIPOLYGON_REPORT" in os.environ:
imposm.config.imposm_multipolygon_report = float(os.environ["IMPOSM_MULTIPOLYGON_REPORT"])
if "IMPOSM_MULTIPOLYGON_MAX_RING" in os.environ:
imposm.config.imposm_multipolygon_max_ring = int(os.environ["IMPOSM_MULTIPOLYGON_MAX_RING"])
if options.table_prefix:
options.table_prefix = options.table_prefix.rstrip("_") + "_"
if options.table_prefix_production:
options.table_prefix_production = options.table_prefix_production.rstrip("_") + "_"
if options.table_prefix_backup:
options.table_prefix_backup = options.table_prefix_backup.rstrip("_") + "_"
if (
options.write
or options.optimize
or options.deploy_tables
or options.remove_backup_tables
or options.recover_tables
):
db_conf = mappings["db_conf"]
if options.table_prefix:
db_conf.prefix = options.table_prefix
else:
options.table_prefix = db_conf.prefix.rstrip("_") + "_"
if options.connection:
from imposm.db.config import db_conf_from_string
db_conf = db_conf_from_string(options.connection, db_conf)
else:
db_conf.host = options.host or db_conf.host
db_conf.port = options.port or getattr(db_conf, "port", None) # backw. compat
if not options.db:
parser.error("-d/--database is required for this mode")
db_conf.db = options.db or db_conf.db
db_conf.user = options.user or db_conf.user
if options.user:
from getpass import getpass
db_conf.password = getpass("password for %(user)s at %(host)s:" % db_conf)
if options.proj:
db_conf.proj = options.proj
imposm_timer = imposm.util.Timer("imposm", logger)
if options.read:
if not options.merge_cache:
cache_files = glob.glob(os.path.join(options.cache_dir, "imposm_*.cache"))
if cache_files:
if not options.overwrite_cache:
print (
"ERROR: found existing cache files in '%s'. "
"Remove --read option to use the existing cache "
"or use --overwrite-cache or --merge-cache to "
"overwrite or merge it." % os.path.abspath(options.cache_dir)
)
sys.exit(2)
for cache_file in cache_files:
os.unlink(cache_file)
cache = OSMCache(options.cache_dir)
if options.read:
read_timer = imposm.util.Timer("reading", logger)
if not args:
print "no file(s) supplied"
sys.exit(2)
if options.write:
err = check_connection(db_conf)
if err:
logger.message("ERROR: unable to connect to database. Check your DB settings.\n{0}".format(err))
sys.exit(2)
reader = ImposmReader(
tag_mapping, cache=cache, merge=options.merge_cache, pool_size=options.concurrency, logger=logger_parser
)
reader.estimated_coords = imposm.util.estimate_records(args)
for arg in args:
logger.message("## reading %s" % arg)
reader.read(arg)
read_timer.stop()
if options.write:
db = DB(db_conf)
write_timer = imposm.util.Timer("writing", logger)
logger.message("## dropping/creating tables")
if not options.dry_run:
db.create_tables(tag_mapping.mappings)
logger.message("## writing data")
# create views so we can access the table during the insert, ignore
# errors for missing tables (i.e. generalized tables)
if not options.dry_run:
db.create_views(mappings, ignore_errors=True)
db.commit()
writer = ImposmWriter(
tag_mapping, db, cache=cache, pool_size=options.concurrency, logger=logger, dry_run=options.dry_run
)
writer.relations()
writer.ways()
writer.nodes()
if not options.dry_run:
db = DB(db_conf)
logger.message("## creating generalized tables")
generalized_timer = imposm.util.Timer("generalizing tables", logger)
db.create_generalized_tables(mappings)
generalized_timer.stop()
logger.message("## creating user generalized tables")
generalized_timer = imposm.util.Timer("generalizing user tables", logger)
db.simplify_tables()
generalized_timer.stop()
logger.message("## creating union views")
view_timer = imposm.util.Timer("creating views", logger)
db.create_views(mappings)
view_timer.stop()
logger.message("## creating geometry indexes")
index_timer = imposm.util.Timer("creating indexes", logger)
db.post_insert(mappings)
index_timer.stop()
logger.message("## post-processing tables")
valid_timer = imposm.util.Timer("post-processing tables", logger)
db.postprocess_tables(mappings)
valid_timer.stop()
db.commit()
write_timer.stop()
if options.optimize:
db = DB(db_conf)
optimize_timer = imposm.util.Timer("optimizing", logger)
logger.message("## optimizing tables")
db.optimize(mappings)
optimize_timer.stop()
if options.recover_tables:
assert not options.deploy_tables, "cannot swap and recover production " "tables at the same time"
options.table_prefix, options.table_prefix_backup = options.table_prefix_backup, options.table_prefix
if options.deploy_tables or options.recover_tables:
db = DB(db_conf)
db.swap_tables(options.table_prefix, options.table_prefix_production, options.table_prefix_backup)
# db.remove_views(options.table_prefix)
# db.db_conf.prefix = options.table_prefix_production
# db.create_views(mappings)
db.commit()
if options.remove_backup_tables:
db = DB(db_conf)
db.remove_tables(options.table_prefix_backup)
db.commit()
imposm_timer.stop()
def main(argv=None):
setproctitle('imposm: main')
usage = '%prog [options] [input]...'
parser = optparse.OptionParser(usage=usage, add_help_option=False,
version="%prog " + __version__)
parser.add_option('--help', dest='help', action='store_true',
default=False, help='show this help message and exit')
parser.add_option('--debug', action='store_true',
default=False, help='show debug information')
parser.add_option('--quiet', action='store_true',
default=False, help='only print progress every 60 seconds')
parser.add_option('-m', '--mapping-file', dest='mapping_file',
metavar='<file>')
parser.add_option('-h', '--host', dest='host', metavar='<host>')
parser.add_option('-p', '--port', dest='port', metavar='<port>')
parser.add_option('-d', '--database', dest='db', metavar='<dbname>')
parser.add_option('-U', '--user', dest='user', metavar='<user>')
parser.add_option('--proj', dest='proj', metavar='EPSG:900913')
parser.add_option('--connection', dest='connection',
help="connection string like postgis://user:pass@host:port/database,"
" this overwrites the -h/-p/-d/-U options")
parser.add_option('-c', '--concurrency', dest='concurrency', metavar='N',
type='int', default=n_cpu)
parser.add_option('--merge-cache', dest='merge_cache', default=False,
action='store_true')
parser.add_option('--overwrite-cache', dest='overwrite_cache', default=False,
action='store_true')
parser.add_option('--cache-dir', dest='cache_dir', default='.',
help="path where node/ways/relations should be cached [current working dir]")
parser.add_option('--table-prefix',
dest='table_prefix', default=None, metavar='osm_new_',
help='prefix for imported tables')
parser.add_option('--table-prefix-production',
dest='table_prefix_production', default='osm_', metavar='osm_',
help='prefix for production tables')
parser.add_option('--table-prefix-backup',
dest='table_prefix_backup', default='osm_old_', metavar='osm_old_',
help='prefix for backup tables')
parser.add_option('--read', dest='read', default=False,
action='store_true')
parser.add_option('--write', dest='write', default=False,
action='store_true')
parser.add_option('--optimize', dest='optimize', default=False,
action='store_true')
parser.add_option('--deploy-production-tables', dest='deploy_tables', default=False,
action='store_true', help='remove backup tables, move production tables '
'to backup tables and move import tables to production tables')
parser.add_option('--recover-production-tables', dest='recover_tables', default=False,
action='store_true', help='move production tables to import tables and'
'move backup tables to production tables')
parser.add_option('--remove-backup-tables', dest='remove_backup_tables', default=False,
action='store_true')
parser.add_option('-n', '--dry-run', dest='dry_run', default=False,
action='store_true')
parser.add_option('--limit-to', dest='limit_to', metavar='WKT file',
help='limit imported geometries to WKT (multi)polygons in EPSG:4326')
(options, args) = parser.parse_args(argv)
setup_logging(debug=options.debug)
if (argv and len(argv) == 0) or (not argv and len(sys.argv) == 1):
options.help = True
if not any([options.read, options.write, options.optimize, options.deploy_tables,
options.recover_tables, options.remove_backup_tables]):
options.help = True
if options.help:
parser.print_help()
sys.exit(1)
if options.quiet:
logger = imposm.util.QuietProgressLog
logger_parser = imposm.util.QuietParserProgress
else:
logger = imposm.util.ProgressLog
logger_parser = imposm.util.ParserProgress
if options.proj:
if ':' not in options.proj:
print 'ERROR: --proj should be in EPSG:00000 format'
sys.exit(1)
# check proj if meter_to_mapunit needs to do anything
if options.proj.lower() == 'epsg:4326':
imposm.mapping.import_srs_is_geographic = True
mapping_file = os.path.join(os.path.dirname(__file__),
'defaultmapping.py')
if options.mapping_file:
print 'loading %s as mapping' % options.mapping_file
mapping_file = options.mapping_file
polygon = None
if options.limit_to:
logger.message('## reading --limit-to %s' % options.limit_to)
polygon_timer = imposm.util.Timer('reading', logger)
polygon = load_geom(options.limit_to)
polygon_timer.stop()
if polygon is None:
print 'ERROR: No valid polygon/multipolygon found'
sys.exit(1)
mappings = {}
execfile(mapping_file, mappings)
tag_mapping = TagMapper([m for n, m in mappings.iteritems()
if isinstance(m, imposm.mapping.Mapping)], limit_to=polygon)
if 'IMPOSM_MULTIPOLYGON_REPORT' in os.environ:
imposm.config.imposm_multipolygon_report = float(os.environ['IMPOSM_MULTIPOLYGON_REPORT'])
if 'IMPOSM_MULTIPOLYGON_MAX_RING' in os.environ:
imposm.config.imposm_multipolygon_max_ring = int(os.environ['IMPOSM_MULTIPOLYGON_MAX_RING'])
if options.table_prefix:
options.table_prefix = options.table_prefix.rstrip('_') + '_'
if options.table_prefix_production:
options.table_prefix_production = options.table_prefix_production.rstrip('_') + '_'
if options.table_prefix_backup:
options.table_prefix_backup = options.table_prefix_backup.rstrip('_') + '_'
if (options.write or options.optimize or options.deploy_tables
or options.remove_backup_tables or options.recover_tables):
db_conf = mappings['db_conf']
if options.table_prefix:
db_conf.prefix = options.table_prefix
else:
options.table_prefix = db_conf.prefix.rstrip('_') + '_'
if options.connection:
from imposm.db.config import db_conf_from_string
db_conf = db_conf_from_string(options.connection, db_conf)
else:
db_conf.host = options.host or db_conf.host
db_conf.port = options.port or getattr(db_conf, 'port', None) #backw. compat
if not options.db:
parser.error('-d/--database is required for this mode')
db_conf.db = options.db or db_conf.db
db_conf.user = options.user or db_conf.user
if options.user:
from getpass import getpass
db_conf.password = getpass('password for %(user)s at %(host)s:' % db_conf)
if options.proj:
db_conf.proj = options.proj
imposm_timer = imposm.util.Timer('imposm', logger)
if options.read:
if not options.merge_cache:
cache_files = glob.glob(os.path.join(options.cache_dir, 'imposm_*.cache'))
if cache_files:
if not options.overwrite_cache:
print (
"ERROR: found existing cache files in '%s'. "
'remove files or use --overwrite-cache or --merge-cache.'
% os.path.abspath(options.cache_dir)
)
sys.exit(2)
for cache_file in cache_files:
os.unlink(cache_file)
cache = OSMCache(options.cache_dir)
if options.read:
read_timer = imposm.util.Timer('reading', logger)
if not args:
print "no file(s) supplied"
sys.exit(2)
reader = ImposmReader(tag_mapping, cache=cache, merge=options.merge_cache,
pool_size=options.concurrency, logger=logger_parser)
reader.estimated_coords = imposm.util.estimate_records(args)
for arg in args:
logger.message('## reading %s' % arg)
reader.read(arg)
read_timer.stop()
if options.write:
db = DB(db_conf)
write_timer = imposm.util.Timer('writing', logger)
logger.message('## dropping/creating tables')
if not options.dry_run:
db.create_tables(tag_mapping.mappings)
logger.message('## writing data')
# create views so we can access the table during the insert, ignore
# errors for missing tables (i.e. generalized tables)
if not options.dry_run:
db.create_views(mappings, ignore_errors=True)
db.commit()
writer = ImposmWriter(tag_mapping, db, cache=cache,
pool_size=options.concurrency, logger=logger,
dry_run=options.dry_run)
writer.relations()
writer.ways()
writer.nodes()
if not options.dry_run:
db = DB(db_conf)
logger.message('## creating generalized tables')
generalized_timer = imposm.util.Timer('generalizing tables', logger)
db.create_generalized_tables(mappings)
generalized_timer.stop()
logger.message('## creating union views')
view_timer = imposm.util.Timer('creating views', logger)
db.create_views(mappings)
view_timer.stop()
logger.message('## creating geometry indexes')
index_timer = imposm.util.Timer('creating indexes', logger)
db.post_insert(mappings);
index_timer.stop()
db.commit()
write_timer.stop()
if options.optimize:
db = DB(db_conf)
optimize_timer = imposm.util.Timer('optimizing', logger)
logger.message('## optimizing tables')
db.optimize(mappings)
optimize_timer.stop()
if options.recover_tables:
assert not options.deploy_tables, ('cannot swap and recover production '
'tables at the same time')
options.table_prefix, options.table_prefix_backup = \
options.table_prefix_backup, options.table_prefix
if options.deploy_tables or options.recover_tables:
db = DB(db_conf)
db.swap_tables(options.table_prefix,
options.table_prefix_production, options.table_prefix_backup)
db.remove_views(options.table_prefix)
db.db_conf.prefix = options.table_prefix_production
db.create_views(mappings)
db.commit()
if options.remove_backup_tables:
db = DB(db_conf)
db.remove_tables(options.table_prefix_backup)
db.commit()
imposm_timer.stop()
def main(argv=None):
setproctitle('imposm: main')
usage = '%prog [options] [input]...'
parser = optparse.OptionParser(usage=usage, add_help_option=False,
version="%prog " + __version__)
parser.add_option('--help', dest='help', action='store_true',
default=False, help='show this help message and exit')
parser.add_option('--debug', action='store_true',
default=False, help='show debug information')
parser.add_option('--quiet', action='store_true',
default=False, help='only print progress every 60 seconds')
parser.add_option('-m', '--mapping-file', dest='mapping_file',
metavar='<file>')
parser.add_option('-h', '--host', dest='host', metavar='<host>')
parser.add_option('-p', '--port', dest='port', metavar='<port>')
parser.add_option('-d', '--database', dest='db', metavar='<dbname>')
parser.add_option('-U', '--user', dest='user', metavar='<user>')
parser.add_option('--proj', dest='proj', metavar='EPSG:900913')
parser.add_option('--connection', dest='connection',
help="connection string like postgis://user:pass@host:port/database,"
" this overwrites the -h/-p/-d/-U options")
parser.add_option('-c', '--concurrency', dest='concurrency', metavar='N',
type='int', default=n_cpu)
parser.add_option('--merge-cache', dest='merge_cache', default=False,
action='store_true')
parser.add_option('--overwrite-cache', dest='overwrite_cache', default=False,
action='store_true')
parser.add_option('--cache-dir', dest='cache_dir', default='.',
help="path where node/ways/relations should be cached [current working dir]")
parser.add_option('--table-prefix',
dest='table_prefix', default=None, metavar='osm_new_',
help='prefix for imported tables')
parser.add_option('--table-prefix-production',
dest='table_prefix_production', default='osm_', metavar='osm_',
help='prefix for production tables')
parser.add_option('--table-prefix-backup',
dest='table_prefix_backup', default='osm_old_', metavar='osm_old_',
help='prefix for backup tables')
parser.add_option('--read', dest='read', default=False,
action='store_true')
parser.add_option('--write', dest='write', default=False,
action='store_true')
parser.add_option('--optimize', dest='optimize', default=False,
action='store_true')
parser.add_option('--deploy-production-tables', dest='deploy_tables', default=False,
action='store_true', help='remove backup tables, move production tables '
'to backup tables and move import tables to production tables')
parser.add_option('--recover-production-tables', dest='recover_tables', default=False,
action='store_true', help='move production tables to import tables and'
'move backup tables to production tables')
parser.add_option('--remove-backup-tables', dest='remove_backup_tables', default=False,
action='store_true')
parser.add_option('-n', '--dry-run', dest='dry_run', default=False,
action='store_true')
parser.add_option('--limit-to', dest='limit_to', metavar='file',
help='limit imported geometries to (multi)polygons in EPSG:4326')
(options, args) = parser.parse_args(argv)
setup_logging(debug=options.debug)
if (argv and len(argv) == 0) or (not argv and len(sys.argv) == 1):
options.help = True
if not any([options.read, options.write, options.optimize, options.deploy_tables,
options.recover_tables, options.remove_backup_tables]):
options.help = True
if options.help:
parser.print_help()
sys.exit(1)
if options.quiet:
logger = imposm.util.QuietProgressLog
logger_parser = imposm.util.QuietParserProgress
else:
logger = imposm.util.ProgressLog
logger_parser = imposm.util.ParserProgress
if options.proj:
if ':' not in options.proj:
print 'ERROR: --proj should be in EPSG:00000 format'
sys.exit(1)
# check proj if meter_to_mapunit needs to do anything
if options.proj.lower() == 'epsg:4326':
imposm.mapping.import_srs_is_geographic = True
mapping_file = os.path.join(os.path.dirname(__file__),
'defaultmapping.py')
if options.mapping_file:
print 'loading %s as mapping' % options.mapping_file
mapping_file = options.mapping_file
polygon = None
if options.limit_to:
logger.message('## reading --limit-to %s' % options.limit_to)
polygon_timer = imposm.util.Timer('reading', logger)
polygon = load_geom(options.limit_to)
polygon_timer.stop()
if polygon is None:
print 'ERROR: No valid polygon/multipolygon found'
sys.exit(1)
mappings = {}
execfile(mapping_file, mappings)
tag_mapping = TagMapper([m for n, m in mappings.iteritems()
if isinstance(m, imposm.mapping.Mapping)], limit_to=polygon)
if 'IMPOSM_MULTIPOLYGON_REPORT' in os.environ:
imposm.config.imposm_multipolygon_report = float(os.environ['IMPOSM_MULTIPOLYGON_REPORT'])
if 'IMPOSM_MULTIPOLYGON_MAX_RING' in os.environ:
imposm.config.imposm_multipolygon_max_ring = int(os.environ['IMPOSM_MULTIPOLYGON_MAX_RING'])
if options.table_prefix:
options.table_prefix = options.table_prefix.rstrip('_') + '_'
if options.table_prefix_production:
options.table_prefix_production = options.table_prefix_production.rstrip('_') + '_'
if options.table_prefix_backup:
options.table_prefix_backup = options.table_prefix_backup.rstrip('_') + '_'
if (options.write or options.optimize or options.deploy_tables
or options.remove_backup_tables or options.recover_tables):
db_conf = mappings['db_conf']
if options.table_prefix:
db_conf.prefix = options.table_prefix
else:
options.table_prefix = db_conf.prefix.rstrip('_') + '_'
if options.connection:
from imposm.db.config import db_conf_from_string
db_conf = db_conf_from_string(options.connection, db_conf)
else:
db_conf.host = options.host or db_conf.host
db_conf.port = options.port or getattr(db_conf, 'port', None) #backw. compat
if not options.db:
parser.error('-d/--database is required for this mode')
db_conf.db = options.db or db_conf.db
db_conf.user = options.user or db_conf.user
if options.user:
from getpass import getpass
db_conf.password = getpass('password for %(user)s at %(host)s:' % db_conf)
if options.proj:
db_conf.proj = options.proj
imposm_timer = imposm.util.Timer('imposm', logger)
if options.read:
if not options.merge_cache:
cache_files = glob.glob(os.path.join(options.cache_dir, 'imposm_*.cache'))
if cache_files:
if not options.overwrite_cache:
print (
"ERROR: found existing cache files in '%s'. "
'Remove --read option to use the existing cache '
'or use --overwrite-cache or --merge-cache to '
'overwrite or merge it.'
% os.path.abspath(options.cache_dir)
)
sys.exit(2)
for cache_file in cache_files:
os.unlink(cache_file)
cache = OSMCache(options.cache_dir)
if options.read:
read_timer = imposm.util.Timer('reading', logger)
if not args:
print "no file(s) supplied"
sys.exit(2)
if options.write:
err = check_connection(db_conf)
if err:
logger.message("ERROR: unable to connect to database. Check your DB settings.\n{0}".format(err))
sys.exit(2)
reader = ImposmReader(tag_mapping, cache=cache, merge=options.merge_cache,
pool_size=options.concurrency, logger=logger_parser)
reader.estimated_coords = imposm.util.estimate_records(args)
for arg in args:
logger.message('## reading %s' % arg)
reader.read(arg)
read_timer.stop()
if options.write:
db = DB(db_conf)
write_timer = imposm.util.Timer('writing', logger)
logger.message('## dropping/creating tables')
if not options.dry_run:
db.create_tables(tag_mapping.mappings)
logger.message('## writing data')
# create views so we can access the table during the insert, ignore
# errors for missing tables (i.e. generalized tables)
if not options.dry_run:
db.create_views(mappings, ignore_errors=True)
db.commit()
writer = ImposmWriter(tag_mapping, db, cache=cache,
pool_size=options.concurrency, logger=logger,
dry_run=options.dry_run)
writer.relations()
writer.ways()
writer.nodes()
if not options.dry_run:
db = DB(db_conf)
logger.message('## creating generalized tables')
generalized_timer = imposm.util.Timer('generalizing tables', logger)
db.create_generalized_tables(mappings)
generalized_timer.stop()
logger.message('## creating union views')
view_timer = imposm.util.Timer('creating views', logger)
db.create_views(mappings)
view_timer.stop()
logger.message('## creating geometry indexes')
index_timer = imposm.util.Timer('creating indexes', logger)
db.post_insert(mappings);
index_timer.stop()
logger.message('## post-processing tables')
valid_timer = imposm.util.Timer('post-processing tables', logger)
db.postprocess_tables(mappings)
valid_timer.stop()
db.commit()
write_timer.stop()
if options.optimize:
db = DB(db_conf)
optimize_timer = imposm.util.Timer('optimizing', logger)
logger.message('## optimizing tables')
db.optimize(mappings)
optimize_timer.stop()
if options.recover_tables:
assert not options.deploy_tables, ('cannot swap and recover production '
'tables at the same time')
options.table_prefix, options.table_prefix_backup = \
options.table_prefix_backup, options.table_prefix
if options.deploy_tables or options.recover_tables:
db = DB(db_conf)
db.swap_tables(options.table_prefix,
options.table_prefix_production, options.table_prefix_backup)
db.remove_views(options.table_prefix)
db.db_conf.prefix = options.table_prefix_production
db.create_views(mappings)
db.commit()
if options.remove_backup_tables:
db = DB(db_conf)
db.remove_tables(options.table_prefix_backup)
db.commit()
imposm_timer.stop()
