def shape_to_pgsql(conn, shape_path, table, mode, srid=-1, log_file=None, batch_size=1000):
dbf_file = shape_path[0:-4] + ".dbf"
prj_file = shape_path[0:-4] + ".prj"
# Try detecting the SRID, by default we set to 4326 and hope the best
srid = get_prj_srid(prj_file)
args = [
"shp2pgsql", # TODO: pass the command dynamically in the config file of the server
"-%s" % mode,
"-W",
"latin1",
"-s",
str(srid),
shape_path,
table,
]
print args
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log_file)
cursor = conn.cursor()
try:
with p.stdout as stdout:
for commands in util.groupsgen(util.read_until(stdout, ";"), batch_size):
command = "".join(commands).strip()
if len(command) > 0:
cursor.execute(command)
conn.commit()
except:
conn.rollback()
raise
finally:
cursor.close()
def shape_to_pgsql(config, conn, shape_path, table, import_mode, srid=-1,
encoding='latin1', log_file=None, batch_size=1000):
command_args = [
config['SHAPE2PGSQL'],
import_mode,
'-W', encoding,
'-s', str(srid) + ':4326',
shape_path,
table
]
p = subprocess.Popen(command_args, stdout=subprocess.PIPE, stderr=log_file)
cursor = conn.cursor()
try:
with p.stdout as stdout:
for commands in util.groupsgen(util.read_until(stdout, ';'),
batch_size):
command = ''.join(commands).strip()
if len(command) > 0:
cursor.execute(command)
conn.commit()
except:
conn.rollback()
raise
finally:
cursor.close()
def shape_to_pgsql(config, conn, shape_path, table, mode, srid=-1, log_file=None, batch_size=1000):
modeflags = {
str(IMPORT_MODE_CREATE): "c",
str(IMPORT_MODE_APPEND): "a",
str(IMPORT_MODE_STRUCTURE): "p",
str(IMPORT_MODE_DATA): "",
str(IMPORT_MODE_SPATIAL_INDEX): ""
}
args = [
config.shp2pgsql,
"-%s" % ''.join([modeflags[f] for f in modeflags.keys() if int(f) & mode]),
"-W", "latin1",
"-s", str(srid),
shape_path,
table]
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log_file)
cursor = conn.cursor()
try:
with p.stdout as stdout:
for commands in util.groupsgen(util.read_until(stdout, ';'), batch_size):
command = ''.join(commands).strip()
if len(command) > 0:
cursor.execute(command)
conn.commit()
except:
conn.rollback()
raise
finally:
cursor.close()
def shape_to_pgsql(config, conn, shape_path, table, mode, srid=-1, log_file=None, batch_size=1000):
modeflags = {
str(IMPORT_MODE_CREATE): "c",
str(IMPORT_MODE_APPEND): "a",
str(IMPORT_MODE_STRUCTURE): "p",
str(IMPORT_MODE_DATA): "",
str(IMPORT_MODE_SPATIAL_INDEX): ""
}
args = [
config.shp2pgsql,
"-%s" % ''.join([modeflags[f] for f in modeflags.keys() if int(f) & mode]),
"-W", "latin1",
"-g", "the_geom",
"-s", str(srid),
shape_path,
table]
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log_file)
cursor = conn.cursor()
try:
with p.stdout as stdout:
for commands in util.groupsgen(util.read_until(stdout, ';'), batch_size):
command = ''.join(commands).strip()
if len(command) > 0:
cursor.execute(command)
conn.commit()
except:
conn.rollback()
raise
finally:
cursor.close()
def shape_to_pgsql(conn,
shape_path,
table,
mode,
srid=-1,
log_file=None,
batch_size=1000):
result = False
modeflags = {
str(IMPORT_MODE_CREATE): "c",
str(IMPORT_MODE_APPEND): "a",
str(IMPORT_MODE_STRUCTURE): "p",
str(IMPORT_MODE_DATA): "",
str(IMPORT_MODE_SPATIAL_INDEX): ""
}
args = [
config.shp2pgsql,
"-%s" %
''.join([modeflags[f] for f in modeflags.keys() if int(f) & mode]),
# para solo importar la data
#"-d",
"-W",
"latin1",
"-g",
"the_geom",
"-s",
str(srid),
shape_path,
table
]
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log_file)
cursor = conn.cursor()
try:
with p.stdout as stdout:
for commands in util.groupsgen(util.read_until(stdout, ';'),
batch_size):
command = ''.join(commands).strip()
if len(command) > 0:
cursor.execute(command)
conn.commit()
result = True
#util.write_outputs('geoimporter: Successful import: ' + shape_path)
print 'geoimporter: Successful import', table
except:
error = 'geoimporter: Error import', table, 'for folder', shape_path, 'SRID', srid
util.write_outputs('geoimporter: Error import ' + table +
' for folder' + shape_path + ' SRID' + srid)
print error
conn.rollback()
result = False
#raise
finally:
cursor.close()
return result
def tab_to_pgsql(config, conn, tab_path, table, mode, srid=-1, log_file=None, batch_size=1000):
# ogr2ogr --config GEOMETRY_NAME the_geom
# --config PG_USE_COPY YES
# --config PGCLIENTENCODING LATIN1
# -lco DROP_TABLE=NO
# -lco GEOMETRY_NAME=the_geom
# -lco SRID=3006
# -lco CREATE_TABLE=OFF
# -f PGDump ~/Documents/geo-data/af_0114.sql af_0114.tab
args = [
config.ogr2ogr,
'-lco', 'GEOMETRY_NAME=the_geom',
# '--config', 'PG_USE_COPY', 'YES',
'--config', 'PGCLIENTENCODING', 'LATIN1',
'-lco', 'SRID=%d' % srid,
'-lco', 'DROP_TABLE=NO',
'-f', 'PGDump',
]
ogr_table_name = os.path.splitext(os.path.split(tab_path)[1])[0]
if not (IMPORT_MODE_CREATE & mode):
args.extend(['-lco', 'CREATE_TABLE=NO'])
args.extend([
'/vsistdout/',
tab_path
])
if not (IMPORT_MODE_DATA & mode):
filter_fn = lambda c: not 'INSERT' in c
else:
filter_fn = lambda c: True
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log_file)
cursor = conn.cursor()
try:
with p.stdout as stdout:
for commands in util.groupsgen(util.read_until(stdout, ';\n'), batch_size):
command = ''.join(filter(filter_fn, commands)).strip()
if len(command) > 0:
command = command.decode('iso-8859-1').replace(ogr_table_name, table)
cursor.execute(command.encode('utf-8'))
except:
conn.rollback()
raise
finally:
cursor.close()
def shape_to_pgsql(shape_path,
table,
mode,
srid=-1,
log_file=None,
batch_size=1000):
#util.write_outputs( 'geoimporter: second time import Successful ' + table)
result = False
modeflags = {
str(IMPORT_MODE_CREATE): "c",
str(IMPORT_MODE_APPEND): "a",
str(IMPORT_MODE_STRUCTURE): "p",
str(IMPORT_MODE_DATA): "",
str(IMPORT_MODE_SPATIAL_INDEX): ""
}
args = [
config.shp2pgsql,
"-%s" %
''.join([modeflags[f] for f in modeflags.keys() if int(f) & mode]),
# para solo importar la data
#"-d",
"-W",
"latin1",
"-g",
"the_geom",
"-s",
str(srid),
shape_path,
table
]
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log_file)
cursor = conn.cursor()
try:
with p.stdout as stdout:
for commands in util.groupsgen(util.read_until(stdout, ';'),
batch_size):
command = ''.join(commands).strip()
if len(command) > 0:
cursor.execute(command)
conn.commit()
result = True
#util.write_outputs('geoimporter: Info Successful import: ' + table)
print 'geoimporter: Successful import', table
except psycopg2.ProgrammingError as e:
conn.rollback()
util.write_outputs('geoimporter: ' + str(e.pgerror) + str(e.pgcode))
if str(e.pgcode) == '42P07':
table = util.redefine_name(table)
cursor.close()
result = shape_to_pgsql(
shape_path, table, IMPORT_MODE_CREATE + IMPORT_MODE_DATA +
IMPORT_MODE_SPATIAL_INDEX, srid)
else:
result = False
except psycopg2.Error as e:
conn.rollback()
error = 'table ' + table + ' for folder ' + shape_path
util.write_outputs('geoimporter: ' + str(e.pgerror) + str(error))
result = False
finally:
cursor.close()
return result
