def print_results():
texto = ('*' * 50 + '\nNumber of files what does not find in folder: ' +
str(no_in_file) + '\nNumber of files stored in BD: ' +
str(total_imported_geo) +
'\nNumber of files does not sotres in BD: ' +
str(total_no_imported_geo) + '\nNumber issues extraction: ' +
str(issue_extraction) + '\n' + '*' * 50)
print texto
util.write_outputs(texto)
def shape_to_pgsql(conn,
shape_path,
table,
mode,
srid=-1,
log_file=None,
batch_size=1000):
result = False
modeflags = {
str(IMPORT_MODE_CREATE): "c",
str(IMPORT_MODE_APPEND): "a",
str(IMPORT_MODE_STRUCTURE): "p",
str(IMPORT_MODE_DATA): "",
str(IMPORT_MODE_SPATIAL_INDEX): ""
}
args = [
config.shp2pgsql,
"-%s" %
''.join([modeflags[f] for f in modeflags.keys() if int(f) & mode]),
# para solo importar la data
#"-d",
"-W",
"latin1",
"-g",
"the_geom",
"-s",
str(srid),
shape_path,
table
]
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log_file)
cursor = conn.cursor()
try:
with p.stdout as stdout:
for commands in util.groupsgen(util.read_until(stdout, ';'),
batch_size):
command = ''.join(commands).strip()
if len(command) > 0:
cursor.execute(command)
conn.commit()
result = True
#util.write_outputs('geoimporter: Successful import: ' + shape_path)
print 'geoimporter: Successful import', table
except:
error = 'geoimporter: Error import', table, 'for folder', shape_path, 'SRID', srid
util.write_outputs('geoimporter: Error import ' + table +
' for folder' + shape_path + ' SRID' + srid)
print error
conn.rollback()
result = False
#raise
finally:
cursor.close()
return result
def extract_file(name_file):
full_path = os.path.join(config.path_geo_files, name_file)
name_folder_shp = str(uuid.uuid4())
path_folder_shp = os.path.join(PATH_TEMP, name_folder_shp)
os.makedirs(path_folder_shp)
try:
patoolib.extract_archive(full_path, outdir=path_folder_shp)
except:
global issue_extraction
issue_extraction += 1
error = 'geoimporter: Error in extration file: ' + str(name_file)
print error
util.write_outputs(error)
return path_folder_shp
def publish_geo(files, esquemas):
for file in files.find({'ARCHIVO': {'$regex': 'geo'}}):
name_file = file['ARCHIVO']
result = is_file_in_folder(name_file)
if not result:
error = 'geoimporter: Warning: Dont found file' + name_file + 'Dont matching table mongodb and file in folder'
print error
util.write_outputs(error)
global no_in_file
no_in_file += 1
else:
#try:
path_folder_shp = extract_file(name_file)
shape_importer(path_folder_shp, file, esquemas)
def run(shape_file, table, crs):
shape_file = shape_file.replace("\\", "/")
if table == '':
table = os.path.splitext(os.path.split(shape_file)[1])[0]
result = shape_to_pgsql(
shape_file, table,
IMPORT_MODE_CREATE + IMPORT_MODE_DATA + IMPORT_MODE_SPATIAL_INDEX, crs)
if result:
try:
vacuum_analyze(table)
except psycopg2.Error as e:
error = 'table ' + table + ' for folder ' + shape_file
util.write_outputs('geoimporter: ' + str(e.pgerror) + str(error))
result = False
return result
def shape_to_pgsql(shape_path,
table,
mode,
srid=-1,
log_file=None,
batch_size=1000):
#util.write_outputs( 'geoimporter: second time import Successful ' + table)
result = False
modeflags = {
str(IMPORT_MODE_CREATE): "c",
str(IMPORT_MODE_APPEND): "a",
str(IMPORT_MODE_STRUCTURE): "p",
str(IMPORT_MODE_DATA): "",
str(IMPORT_MODE_SPATIAL_INDEX): ""
}
args = [
config.shp2pgsql,
"-%s" %
''.join([modeflags[f] for f in modeflags.keys() if int(f) & mode]),
# para solo importar la data
#"-d",
"-W",
"latin1",
"-g",
"the_geom",
"-s",
str(srid),
shape_path,
table
]
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log_file)
cursor = conn.cursor()
try:
with p.stdout as stdout:
for commands in util.groupsgen(util.read_until(stdout, ';'),
batch_size):
command = ''.join(commands).strip()
if len(command) > 0:
cursor.execute(command)
conn.commit()
result = True
#util.write_outputs('geoimporter: Info Successful import: ' + table)
print 'geoimporter: Successful import', table
except psycopg2.ProgrammingError as e:
conn.rollback()
util.write_outputs('geoimporter: ' + str(e.pgerror) + str(e.pgcode))
if str(e.pgcode) == '42P07':
table = util.redefine_name(table)
cursor.close()
result = shape_to_pgsql(
shape_path, table, IMPORT_MODE_CREATE + IMPORT_MODE_DATA +
IMPORT_MODE_SPATIAL_INDEX, srid)
else:
result = False
except psycopg2.Error as e:
conn.rollback()
error = 'table ' + table + ' for folder ' + shape_path
util.write_outputs('geoimporter: ' + str(e.pgerror) + str(error))
result = False
finally:
cursor.close()
return result