def shape_to_pgsql(conn, shape_path, table, mode, srid=-1, log_file=None, batch_size=1000): dbf_file = shape_path[0:-4] + ".dbf" prj_file = shape_path[0:-4] + ".prj" # Try detecting the SRID, by default we set to 4326 and hope the best srid = get_prj_srid(prj_file) args = [ "shp2pgsql", # TODO: pass the command dynamically in the config file of the server "-%s" % mode, "-W", "latin1", "-s", str(srid), shape_path, table, ] print args p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log_file) cursor = conn.cursor() try: with p.stdout as stdout: for commands in util.groupsgen(util.read_until(stdout, ";"), batch_size): command = "".join(commands).strip() if len(command) > 0: cursor.execute(command) conn.commit() except: conn.rollback() raise finally: cursor.close()
def shape_to_pgsql(config, conn, shape_path, table, import_mode, srid=-1, encoding='latin1', log_file=None, batch_size=1000): command_args = [ config['SHAPE2PGSQL'], import_mode, '-W', encoding, '-s', str(srid) + ':4326', shape_path, table ] p = subprocess.Popen(command_args, stdout=subprocess.PIPE, stderr=log_file) cursor = conn.cursor() try: with p.stdout as stdout: for commands in util.groupsgen(util.read_until(stdout, ';'), batch_size): command = ''.join(commands).strip() if len(command) > 0: cursor.execute(command) conn.commit() except: conn.rollback() raise finally: cursor.close()
def shape_to_pgsql(config, conn, shape_path, table, mode, srid=-1, log_file=None, batch_size=1000): modeflags = { str(IMPORT_MODE_CREATE): "c", str(IMPORT_MODE_APPEND): "a", str(IMPORT_MODE_STRUCTURE): "p", str(IMPORT_MODE_DATA): "", str(IMPORT_MODE_SPATIAL_INDEX): "" } args = [ config.shp2pgsql, "-%s" % ''.join([modeflags[f] for f in modeflags.keys() if int(f) & mode]), "-W", "latin1", "-s", str(srid), shape_path, table] p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log_file) cursor = conn.cursor() try: with p.stdout as stdout: for commands in util.groupsgen(util.read_until(stdout, ';'), batch_size): command = ''.join(commands).strip() if len(command) > 0: cursor.execute(command) conn.commit() except: conn.rollback() raise finally: cursor.close()
def shape_to_pgsql(config, conn, shape_path, table, mode, srid=-1, log_file=None, batch_size=1000): modeflags = { str(IMPORT_MODE_CREATE): "c", str(IMPORT_MODE_APPEND): "a", str(IMPORT_MODE_STRUCTURE): "p", str(IMPORT_MODE_DATA): "", str(IMPORT_MODE_SPATIAL_INDEX): "" } args = [ config.shp2pgsql, "-%s" % ''.join([modeflags[f] for f in modeflags.keys() if int(f) & mode]), "-W", "latin1", "-g", "the_geom", "-s", str(srid), shape_path, table] p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log_file) cursor = conn.cursor() try: with p.stdout as stdout: for commands in util.groupsgen(util.read_until(stdout, ';'), batch_size): command = ''.join(commands).strip() if len(command) > 0: cursor.execute(command) conn.commit() except: conn.rollback() raise finally: cursor.close()
def shape_to_pgsql(conn, shape_path, table, mode, srid=-1, log_file=None, batch_size=1000): result = False modeflags = { str(IMPORT_MODE_CREATE): "c", str(IMPORT_MODE_APPEND): "a", str(IMPORT_MODE_STRUCTURE): "p", str(IMPORT_MODE_DATA): "", str(IMPORT_MODE_SPATIAL_INDEX): "" } args = [ config.shp2pgsql, "-%s" % ''.join([modeflags[f] for f in modeflags.keys() if int(f) & mode]), # para solo importar la data #"-d", "-W", "latin1", "-g", "the_geom", "-s", str(srid), shape_path, table ] p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log_file) cursor = conn.cursor() try: with p.stdout as stdout: for commands in util.groupsgen(util.read_until(stdout, ';'), batch_size): command = ''.join(commands).strip() if len(command) > 0: cursor.execute(command) conn.commit() result = True #util.write_outputs('geoimporter: Successful import: ' + shape_path) print 'geoimporter: Successful import', table except: error = 'geoimporter: Error import', table, 'for folder', shape_path, 'SRID', srid util.write_outputs('geoimporter: Error import ' + table + ' for folder' + shape_path + ' SRID' + srid) print error conn.rollback() result = False #raise finally: cursor.close() return result
def tab_to_pgsql(config, conn, tab_path, table, mode, srid=-1, log_file=None, batch_size=1000): # ogr2ogr --config GEOMETRY_NAME the_geom # --config PG_USE_COPY YES # --config PGCLIENTENCODING LATIN1 # -lco DROP_TABLE=NO # -lco GEOMETRY_NAME=the_geom # -lco SRID=3006 # -lco CREATE_TABLE=OFF # -f PGDump ~/Documents/geo-data/af_0114.sql af_0114.tab args = [ config.ogr2ogr, '-lco', 'GEOMETRY_NAME=the_geom', # '--config', 'PG_USE_COPY', 'YES', '--config', 'PGCLIENTENCODING', 'LATIN1', '-lco', 'SRID=%d' % srid, '-lco', 'DROP_TABLE=NO', '-f', 'PGDump', ] ogr_table_name = os.path.splitext(os.path.split(tab_path)[1])[0] if not (IMPORT_MODE_CREATE & mode): args.extend(['-lco', 'CREATE_TABLE=NO']) args.extend([ '/vsistdout/', tab_path ]) if not (IMPORT_MODE_DATA & mode): filter_fn = lambda c: not 'INSERT' in c else: filter_fn = lambda c: True p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log_file) cursor = conn.cursor() try: with p.stdout as stdout: for commands in util.groupsgen(util.read_until(stdout, ';\n'), batch_size): command = ''.join(filter(filter_fn, commands)).strip() if len(command) > 0: command = command.decode('iso-8859-1').replace(ogr_table_name, table) cursor.execute(command.encode('utf-8')) except: conn.rollback() raise finally: cursor.close()
def shape_to_pgsql(shape_path, table, mode, srid=-1, log_file=None, batch_size=1000): #util.write_outputs( 'geoimporter: second time import Successful ' + table) result = False modeflags = { str(IMPORT_MODE_CREATE): "c", str(IMPORT_MODE_APPEND): "a", str(IMPORT_MODE_STRUCTURE): "p", str(IMPORT_MODE_DATA): "", str(IMPORT_MODE_SPATIAL_INDEX): "" } args = [ config.shp2pgsql, "-%s" % ''.join([modeflags[f] for f in modeflags.keys() if int(f) & mode]), # para solo importar la data #"-d", "-W", "latin1", "-g", "the_geom", "-s", str(srid), shape_path, table ] p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log_file) cursor = conn.cursor() try: with p.stdout as stdout: for commands in util.groupsgen(util.read_until(stdout, ';'), batch_size): command = ''.join(commands).strip() if len(command) > 0: cursor.execute(command) conn.commit() result = True #util.write_outputs('geoimporter: Info Successful import: ' + table) print 'geoimporter: Successful import', table except psycopg2.ProgrammingError as e: conn.rollback() util.write_outputs('geoimporter: ' + str(e.pgerror) + str(e.pgcode)) if str(e.pgcode) == '42P07': table = util.redefine_name(table) cursor.close() result = shape_to_pgsql( shape_path, table, IMPORT_MODE_CREATE + IMPORT_MODE_DATA + IMPORT_MODE_SPATIAL_INDEX, srid) else: result = False except psycopg2.Error as e: conn.rollback() error = 'table ' + table + ' for folder ' + shape_path util.write_outputs('geoimporter: ' + str(e.pgerror) + str(error)) result = False finally: cursor.close() return result