def getConnectionString(self, superUser=False, commandLine=False): if not superUser: return postgresops.getConnectString(self.dbName, self.userName, self.password, self.dbHost, self.dbPort, commandLine) else: return postgresops.getConnectString(self.userName, self.userName, self.password, self.dbHost, self.dbPort, commandLine)
def getConnectionString(self, superUser = False, commandLine = False): if not superUser: return postgresops.getConnectString(self.dbName, self.userName, self.password, self.dbHost, self.dbPort, commandLine) else: return postgresops.getConnectString(self.userName, self.userName, self.password, self.dbHost, self.dbPort, commandLine)
def run(inputFolder, numcores, dbname, dbuser, dbpass, dbhost, dbport, createdb, dbtable, srid): opts = 0 childrenQueue = multiprocessing.Queue() ifiles = utils.getFiles(inputFolder) for i in range(len(ifiles)): childrenQueue.put([i, ifiles[i]]) for i in range( int(numcores) ): #we add as many None jobs as numWorkers to tell them to terminate (queue is FIFO) childrenQueue.put(None) clineConString = postgresops.getConnectString(dbname, dbuser, dbpass, dbhost, dbport, True) psycopgConString = postgresops.getConnectString(dbname, dbuser, dbpass, dbhost, dbport, False) if createdb: os.system('dropdb ' + clineConString) os.system('createdb ' + clineConString) connection = psycopg2.connect(psycopgConString) cursor = connection.cursor() if createdb: cursor.execute('CREATE EXTENSION postgis') connection.commit() q = """ CREATE TABLE """ + dbtable + """ ( id integer, filepath text, num integer, scalex double precision, scaley double precision, scalez double precision, offsetx double precision, offsety double precision, offsetz double precision, geom public.geometry(Geometry,""" + str(srid) + """) )""" logging.info(cursor.mogrify(q)) cursor.execute(q) connection.commit() # q = "select addgeometrycolumn('" + dbtable + "','geom',28992,'POLYGON',2)" # logging.info(cursor.mogrify(q)) # cursor.execute(q) # connection.commit() print 'numcores', numcores children = [] # We start numcores children processes for i in range(int(numcores)): children.append( multiprocessing.Process(target=runChild, args=(i, childrenQueue, psycopgConString, dbtable, srid))) children[-1].start() # wait for all children to finish their execution for i in range(int(numcores)): children[i].join() q = "create index ON " + dbtable + " using GIST (geom)" logging.info(cursor.mogrify(q)) cursor.execute(q) connection.commit() old_isolation_level = connection.isolation_level connection.set_isolation_level(0) q = "VACUUM FULL ANALYZE " + dbtable logging.info(cursor.mogrify(q)) cursor.execute(q) connection.commit() connection.set_isolation_level(old_isolation_level) cursor.close()
def run(inputFolder, numcores, dbname, dbuser, dbpass, dbhost, dbport, createdb, dbtable, srid): opts = 0 childrenQueue = multiprocessing.Queue() ifiles = utils.getFiles(inputFolder) for i in range(len(ifiles)): childrenQueue.put([i, ifiles[i]]) for i in range(int(numcores)): #we add as many None jobs as numWorkers to tell them to terminate (queue is FIFO) childrenQueue.put(None) clineConString = postgresops.getConnectString(dbname, dbuser, dbpass, dbhost, dbport, True) psycopgConString = postgresops.getConnectString(dbname, dbuser, dbpass, dbhost, dbport, False) if createdb: os.system('dropdb ' + clineConString) os.system('createdb ' + clineConString) connection = psycopg2.connect(psycopgConString) cursor = connection.cursor() if createdb: cursor.execute('CREATE EXTENSION postgis') connection.commit() q = """ CREATE TABLE """ + dbtable + """ ( id integer, filepath text, num integer, scalex double precision, scaley double precision, scalez double precision, offsetx double precision, offsety double precision, offsetz double precision, geom public.geometry(Geometry,""" + str(srid) + """) )""" logging.info(cursor.mogrify(q)) cursor.execute(q) connection.commit() # q = "select addgeometrycolumn('" + dbtable + "','geom',28992,'POLYGON',2)" # logging.info(cursor.mogrify(q)) # cursor.execute(q) # connection.commit() print 'numcores',numcores children = [] # We start numcores children processes for i in range(int(numcores)): children.append(multiprocessing.Process(target=runChild, args=(i, childrenQueue, psycopgConString, dbtable, srid))) children[-1].start() # wait for all children to finish their execution for i in range(int(numcores)): children[i].join() q = "create index ON " + dbtable + " using GIST (geom)" logging.info(cursor.mogrify(q)) cursor.execute(q) connection.commit() old_isolation_level = connection.isolation_level connection.set_isolation_level(0) q = "VACUUM FULL ANALYZE " + dbtable logging.info(cursor.mogrify(q)) cursor.execute(q) connection.commit() connection.set_isolation_level(old_isolation_level) cursor.close()