def getConnectionString(self, superUser=False, commandLine=False):
if not superUser:
return postgresops.getConnectString(self.dbName, self.userName,
self.password, self.dbHost,
self.dbPort, commandLine)
else:
return postgresops.getConnectString(self.userName, self.userName,
self.password, self.dbHost,
self.dbPort, commandLine)
def getConnectionString(self, superUser = False, commandLine = False):
if not superUser:
return postgresops.getConnectString(self.dbName, self.userName, self.password, self.dbHost, self.dbPort, commandLine)
else:
return postgresops.getConnectString(self.userName, self.userName, self.password, self.dbHost, self.dbPort, commandLine)
def run(inputFolder, numcores, dbname, dbuser, dbpass, dbhost, dbport,
createdb, dbtable, srid):
opts = 0
childrenQueue = multiprocessing.Queue()
ifiles = utils.getFiles(inputFolder)
for i in range(len(ifiles)):
childrenQueue.put([i, ifiles[i]])
for i in range(
int(numcores)
): #we add as many None jobs as numWorkers to tell them to terminate (queue is FIFO)
childrenQueue.put(None)
clineConString = postgresops.getConnectString(dbname, dbuser, dbpass,
dbhost, dbport, True)
psycopgConString = postgresops.getConnectString(dbname, dbuser, dbpass,
dbhost, dbport, False)
if createdb:
os.system('dropdb ' + clineConString)
os.system('createdb ' + clineConString)
connection = psycopg2.connect(psycopgConString)
cursor = connection.cursor()
if createdb:
cursor.execute('CREATE EXTENSION postgis')
connection.commit()
q = """
CREATE TABLE """ + dbtable + """ (
id integer,
filepath text,
num integer,
scalex double precision,
scaley double precision,
scalez double precision,
offsetx double precision,
offsety double precision,
offsetz double precision,
geom public.geometry(Geometry,""" + str(srid) + """)
)"""
logging.info(cursor.mogrify(q))
cursor.execute(q)
connection.commit()
# q = "select addgeometrycolumn('" + dbtable + "','geom',28992,'POLYGON',2)"
# logging.info(cursor.mogrify(q))
# cursor.execute(q)
# connection.commit()
print 'numcores', numcores
children = []
# We start numcores children processes
for i in range(int(numcores)):
children.append(
multiprocessing.Process(target=runChild,
args=(i, childrenQueue, psycopgConString,
dbtable, srid)))
children[-1].start()
# wait for all children to finish their execution
for i in range(int(numcores)):
children[i].join()
q = "create index ON " + dbtable + " using GIST (geom)"
logging.info(cursor.mogrify(q))
cursor.execute(q)
connection.commit()
old_isolation_level = connection.isolation_level
connection.set_isolation_level(0)
q = "VACUUM FULL ANALYZE " + dbtable
logging.info(cursor.mogrify(q))
cursor.execute(q)
connection.commit()
connection.set_isolation_level(old_isolation_level)
cursor.close()
def run(inputFolder, numcores, dbname, dbuser, dbpass, dbhost, dbport, createdb, dbtable, srid):
opts = 0
childrenQueue = multiprocessing.Queue()
ifiles = utils.getFiles(inputFolder)
for i in range(len(ifiles)):
childrenQueue.put([i, ifiles[i]])
for i in range(int(numcores)): #we add as many None jobs as numWorkers to tell them to terminate (queue is FIFO)
childrenQueue.put(None)
clineConString = postgresops.getConnectString(dbname, dbuser, dbpass, dbhost, dbport, True)
psycopgConString = postgresops.getConnectString(dbname, dbuser, dbpass, dbhost, dbport, False)
if createdb:
os.system('dropdb ' + clineConString)
os.system('createdb ' + clineConString)
connection = psycopg2.connect(psycopgConString)
cursor = connection.cursor()
if createdb:
cursor.execute('CREATE EXTENSION postgis')
connection.commit()
q = """
CREATE TABLE """ + dbtable + """ (
id integer,
filepath text,
num integer,
scalex double precision,
scaley double precision,
scalez double precision,
offsetx double precision,
offsety double precision,
offsetz double precision,
geom public.geometry(Geometry,""" + str(srid) + """)
)"""
logging.info(cursor.mogrify(q))
cursor.execute(q)
connection.commit()
# q = "select addgeometrycolumn('" + dbtable + "','geom',28992,'POLYGON',2)"
# logging.info(cursor.mogrify(q))
# cursor.execute(q)
# connection.commit()
print 'numcores',numcores
children = []
# We start numcores children processes
for i in range(int(numcores)):
children.append(multiprocessing.Process(target=runChild,
args=(i, childrenQueue, psycopgConString, dbtable, srid)))
children[-1].start()
# wait for all children to finish their execution
for i in range(int(numcores)):
children[i].join()
q = "create index ON " + dbtable + " using GIST (geom)"
logging.info(cursor.mogrify(q))
cursor.execute(q)
connection.commit()
old_isolation_level = connection.isolation_level
connection.set_isolation_level(0)
q = "VACUUM FULL ANALYZE " + dbtable
logging.info(cursor.mogrify(q))
cursor.execute(q)
connection.commit()
connection.set_isolation_level(old_isolation_level)
cursor.close()
