Poloma wrapper for psycopg2.extensions.connection class.
Go to /polomapy in the github and run python3 setup.py install. If you are using a specific environment run source activate <env> before installing.
-
Set environment variable:
export POLOMA_PASS=<password> -
Import package
from polomapy import PolomaConn- Make connection and query database.
c = PolomaConn()
res = c.query('select * from test.test')Get columns:
col = c.columns- Insert to database.
c = PolomaConn()
nested = [ ['this','is'],['a','test'] ]
c.insert('test.test',nested)- Execute on database.
c = PolomaConn()
c.execute("update test.test set column = 'test'")- Create Async Buffer. The buffer spawns a number of workers that take from the buffer and send each batch to the database in a separate thread.
from polomapy import PolomaBuff
table = 'test.test'
b = PolomaBuff( table
workers=4, # set number of processes
maxconn = 8, # set maximum postgres connections
maxbuff = 50000, # set buffer size to be held in memory
batchsize = 5000) # set batchsize to send to postgres
for i in range(10):
values = ('this','is','a','tuple')
b.append(values) # send rows one at a time
batch = []
for i in range(10):
batch.append( ('this','is','a','tuple') )
b.append(batch,batch=True) # send batch of rows rather than individual row
b.kill() # kill threads once buffer is empty