def main():
s = process.server(args=[],
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
client(script1)
client(script2)
client(script3)
client(script4)
client(script10)
client(script11)
client(script12)
server_stop(s)
s = process.server(args=["--readonly"],
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
client(script1)
client(script3)
client(script6)
client(script5)
client(script7)
client(script8)
client(script9)
server_stop(s)
def main():
s = process.server(args = [],
stdin = process.PIPE,
stdout = process.PIPE,
stderr = process.PIPE)
client(script1)
client(script3)
client(script4)
client(script2)
client(script5)
server_stop(s)
s = process.server(args = ["--readonly"],
stdin = process.PIPE,
stdout = process.PIPE,
stderr = process.PIPE)
client(script8)
client(script4)
client(script2)
client(script6)
client(script7)
client(script9)
client(script10)
client(script11)
client(script12)
client(script3)
client(script13)
client(script14)
server_stop(s)
def server_start():
sys.stderr.write('#mserver\n')
sys.stderr.flush()
srv = process.server(stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
return srv
def create_workers(fn_template, nworkers, cmovies, ratings_table_def_fk):
workers = []
for i in range(nworkers):
workerport = freeport()
workerdbname = 'worker_{}'.format(i)
workerrec = {
'num': i,
'port': workerport,
'dbname': workerdbname,
'dbfarm': os.path.join(TMPDIR, workerdbname),
'mapi': 'mapi:monetdb://localhost:{}/{}/sys/ratings'.format(workerport, workerdbname),
}
os.mkdir(workerrec['dbfarm'])
workerrec['proc'] = process.server(mapiport=workerrec['port'], dbname=workerrec['dbname'], dbfarm=workerrec['dbfarm'], stdin=process.PIPE, stdout=process.PIPE)
workerrec['conn'] = pymonetdb.connect(database=workerrec['dbname'], port=workerport, autocommit=True)
filename = fn_template.format(workerrec['num'])
t = threading.Thread(target=worker_load, args=[filename, workerrec, cmovies, ratings_table_def_fk])
t.start()
workerrec['loadthread'] = t
workers.append(workerrec)
for wrec in workers:
wrec['loadthread'].join()
return workers
def create_workers(fn_template, nworkers, cmovies, ratings_table_def_fk):
workers = []
for i in range(nworkers):
workerport = freeport()
workerdbname = 'worker_{}'.format(i)
workerrec = {
'num': i,
'port': workerport,
'dbname': workerdbname,
'dbfarm': os.path.join(TMPDIR, workerdbname),
'mapi': 'mapi:monetdb://localhost:{}/{}/sys/ratings'.format(workerport, workerdbname),
}
os.mkdir(workerrec['dbfarm'])
workerrec['proc'] = process.server(mapiport=workerrec['port'], dbname=workerrec['dbname'], dbfarm=workerrec['dbfarm'], stdin=process.PIPE, stdout=process.PIPE)
workerrec['conn'] = pymonetdb.connect(database=workerrec['dbname'], port=workerport, autocommit=True)
filename = fn_template.format(workerrec['num'])
t = threading.Thread(target=worker_load, args=[filename, workerrec, cmovies, ratings_table_def_fk])
t.start()
workerrec['loadthread'] = t
workers.append(workerrec)
for wrec in workers:
wrec['loadthread'].join()
return workers
def server_start(args):
sys.stderr.write('#mserver: "%s"\n' % ' '.join(args))
sys.stderr.flush()
srv = process.server(args=args,
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
return srv
def main():
s = process.server(args=["--set", "gdk_nr_threads=2", "--forcemito"],
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
client(script1)
out, err = s.communicate()
sys.stdout.write(out)
sys.stderr.write(err)
def main():
s = process.server(args = ["--set", "gdk_readonly=yes"],
stdin = process.PIPE,
stdout = process.PIPE,
stderr = process.PIPE)
client(script1)
out, err = s.communicate()
sys.stdout.write(out)
sys.stderr.write(err)
def main():
s = process.server(args = ["--set", "gdk_nr_threads=2", "--forcemito"],
stdin = process.PIPE,
stdout = process.PIPE,
stderr = process.PIPE)
client(script1)
out, err = s.communicate()
sys.stdout.write(out)
sys.stderr.write(err)
def main():
s = process.server(stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE)
client(script1, 'monetdb', 'monetdb')
client(script2, 'mydummyuser', 'mydummyuser')
client(script3, 'monetdb', 'monetdb')
client(script4, 'mydummyuser', 'mydummyuser')
client(script5, 'monetdb', 'monetdb')
out, err = s.communicate()
sys.stdout.write(out)
sys.stderr.write(err)
def server_start(x,s):
sys.stdout.write('\nserver %d%d\n' % (x,s))
sys.stderr.write('\nserver %d%d\n' % (x,s))
sys.stderr.flush()
sys.stderr.write('#mserver\n')
sys.stdout.flush()
sys.stderr.flush()
srv = process.server(stdin = process.PIPE,
stdout = process.PIPE, stderr = process.PIPE)
return srv
def server_start(x, s):
sys.stdout.write('\nserver %d%d\n' % (x, s))
sys.stderr.write('\nserver %d%d\n' % (x, s))
sys.stderr.flush()
sys.stderr.write('#mserver\n')
sys.stdout.flush()
sys.stderr.flush()
srv = process.server(stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
return srv
def main():
s = process.server(stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
client(script1, 'monetdb', 'monetdb')
client(script2, 'mydummyuser', 'mydummyuser')
client(script3, 'monetdb', 'monetdb')
client(script4, 'mydummyuser', 'mydummyuser')
client(script5, 'monetdb', 'monetdb')
out, err = s.communicate()
sys.stdout.write(out)
sys.stderr.write(err)
def remote_server_start(x,s):
sys.stdout.write('\nserver %d%d\n' % (x,s))
sys.stderr.write('\nserver %d%d\n' % (x,s))
sys.stderr.flush()
sys.stderr.write('#remote mserver\n')
sys.stdout.flush()
sys.stderr.flush()
port = os.getenv('MAPIPORT', '50000')
srv = process.server(mapiport = int(port) + 1,
dbname = '%s_test1' % os.getenv('TSTDB'),
stdin = process.PIPE,
stdout = process.PIPE, stderr = process.PIPE)
return srv
def remote_server_start(x, s):
sys.stdout.write('\nserver %d%d\n' % (x, s))
sys.stderr.write('\nserver %d%d\n' % (x, s))
sys.stderr.flush()
sys.stderr.write('#remote mserver\n')
sys.stdout.flush()
sys.stderr.flush()
port = os.getenv('MAPIPORT', '50000')
srv = process.server(mapiport=int(port) + 1,
dbname='%s_test1' % os.getenv('TSTDB'),
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
return srv
workerrec['conn'] = pymonetdb.connect(database=workerrec['dbname'], port=workerport, autocommit=True)
filename = fn_template.format(workerrec['num'])
t = threading.Thread(target=worker_load, args=[filename, workerrec, cmovies, ratings_table_def_fk])
t.start()
workerrec['loadthread'] = t
workers.append(workerrec)
for wrec in workers:
wrec['loadthread'].join()
return workers
# Start supervisor database
supervisorport = freeport()
os.mkdir(os.path.join(TMPDIR, "supervisor"))
supervisorproc = process.server(mapiport=supervisorport, dbname="supervisor", dbfarm=os.path.join(TMPDIR, "supervisor"), stdin=process.PIPE, stdout=process.PIPE)
supervisorconn = pymonetdb.connect(database='supervisor', port=supervisorport, autocommit=True)
supervisor_uri = "mapi:monetdb://localhost:{}/supervisor".format(supervisorport)
c = supervisorconn.cursor()
# Create the movies table and load the data
movies_filename=os.getenv("TSTDATAPATH")+"/netflix_data/movies.csv"
movies_create = "CREATE TABLE movies {}".format(MOVIES_TABLE_DEF)
c.execute(movies_create)
load_movies = "COPY INTO movies FROM '{}' USING DELIMITERS ',','\n','\"'".format(movies_filename)
c.execute(load_movies)
# Declare the ratings merge table on supervisor
mtable = "CREATE MERGE TABLE ratings {}".format(RATINGS_TABLE_DEF)
c.execute(mtable)
import sys, os, tempfile, pymonetdb
try:
from MonetDBtesting import process
except ImportError:
import process
with tempfile.TemporaryDirectory() as farm_dir:
os.mkdir(os.path.join(farm_dir, 'node1'))
os.mkdir(os.path.join(farm_dir, 'node2'))
os.mkdir(os.path.join(farm_dir, 'node3'))
with process.server(mapiport='0',
dbname='node1',
dbfarm=os.path.join(farm_dir, 'node1'),
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE) as node1_proc:
with process.server(mapiport='0',
dbname='node2',
dbfarm=os.path.join(farm_dir, 'node2'),
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE) as node2_proc:
with process.server(mapiport='0',
dbname='node3',
dbfarm=os.path.join(farm_dir, 'node3'),
stdin=process.PIPE,
stdout=process.PIPE,
def freeport():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 0))
port = sock.getsockname()[1]
sock.close()
return port
port2 = freeport()
tstdb2 = tstdb + '-clone'
if os.path.exists(os.path.join(dbfarm, tstdb2)):
shutil.rmtree(os.path.join(dbfarm, tstdb2))
# start the first server
s1 = process.server(stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
# load data into the first server's database
c1 = process.client(lang='sql',
server=s1,
args=[os.path.join(tstsrcdir, os.pardir, os.pardir, 'testdb', 'Tests', 'load.sql')],
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
c1out, c1err = c1.communicate()
# start the second server
s2 = process.server(dbname=tstdb2,
mapiport=port2,
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
def client(next_user, next_passwd, input):
c = process.client('sql',
user=next_user,
passwd=next_passwd,
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
out, err = c.communicate(input)
sys.stdout.write(out)
sys.stderr.write(err)
s = process.server(args=[],
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
client(
'monetdb', 'monetdb', '''\
CREATE schema "myschema";\
CREATE TABLE "myschema"."test" ("id" integer, "name" varchar(20));\
INSERT INTO "myschema"."test" ("id", "name") VALUES (1,'Tom'),(2,'Karen');\
CREATE USER myuser WITH UNENCRYPTED PASSWORD 'Test123' NAME 'Hulk' SCHEMA "myschema";\
GRANT SELECT ON "myschema"."test" TO myuser;
''')
client('myuser', 'Test123', '''\
SELECT "id", "name" FROM "myschema"."test";
''')
def query(conn, sql):
print(sql)
cur = conn.cursor()
cur.execute(sql)
r = cur.fetchall()
cur.close()
return r
farm_dir = tempfile.mkdtemp()
os.mkdir(os.path.join(farm_dir, 'node1'))
os.mkdir(os.path.join(farm_dir, 'node2'))
# node1 is the worker
prt1 = freeport()
prc1 = process.server(mapiport=prt1, dbname='node1', dbfarm=os.path.join(farm_dir, 'node1'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE)
conn1 = pymonetdb.connect(database='node1', port=prt1, autocommit=True)
q = "create table s1 (i int)"
print(q); conn1.execute(q)
q = "insert into s1 values (23), (42)"
print(q); conn1.execute(q)
q = "create table t1 (s varchar(10))"
print(q); conn1.execute(q)
q = "insert into t1 values ('abc'), ('efg')"
print(q); conn1.execute(q)
# node2 is the master
prt2 = freeport()
prc2 = process.server(mapiport=prt2, dbname='node2', dbfarm=os.path.join(farm_dir, 'node2'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE)
conn2 = pymonetdb.connect(database='node2', port=prt2, autocommit=True)
return pymonetdb.connect(database = os.getenv('TSTDB'),
hostname = '127.0.0.1',
port = int(os.getenv('MAPIPORT')),
username = '******',
password = '******',
autocommit = autocommit)
def query(conn, sql):
cur = conn.cursor()
cur.execute(sql)
r = cur.fetchall()
cur.close()
return r
# no timeout since we need to kill mserver5, not the inbetween Mtimeout
s = process.server(stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE)
# boring setup and schema creation stuff:
c1 = connect(True)
c1.execute('create table foo (a int)')
c1.execute('create table bar (a int)')
c1.execute('insert into foo values (1),(2),(3)')
# print query(c1, """select * from storage() where "table"='foo'""")
print query(c1, 'select * from foo')
# Run 'delete from foo' with store_nr_active > 1
# This causes MonetDB to allocate a new file for foo rather than
# wiping the existing one
c2 = connect(True)
c2.execute('start transaction')
c1.execute('delete from foo')
workerrec['conn'] = pymonetdb.connect(database=workerrec['dbname'], port=workerport, autocommit=True)
filename = fn_template.format(workerrec['num'])
t = threading.Thread(target=worker_load, args=[filename, workerrec, cmovies, ratings_table_def_fk])
t.start()
workerrec['loadthread'] = t
workers.append(workerrec)
for wrec in workers:
wrec['loadthread'].join()
return workers
# Start supervisor database
supervisorport = freeport()
os.mkdir(os.path.join(TMPDIR, "supervisor"))
supervisorproc = process.server(mapiport=supervisorport, dbname="supervisor", dbfarm=os.path.join(TMPDIR, "supervisor"), stdin=process.PIPE, stdout=process.PIPE)
supervisorconn = pymonetdb.connect(database='supervisor', port=supervisorport, autocommit=True)
supervisor_uri = "mapi:monetdb://localhost:{}/supervisor".format(supervisorport)
c = supervisorconn.cursor()
# Create the movies table and load the data
movies_filename=os.getenv("TSTDATAPATH")+"/netflix_data/movies.csv"
movies_create = "CREATE TABLE movies {}".format(MOVIES_TABLE_DEF)
c.execute(movies_create)
load_movies = "COPY INTO movies FROM '{}' USING DELIMITERS ',','\n','\"'".format(movies_filename)
c.execute(load_movies)
# Declare the ratings merge table on supervisor
mtable = "CREATE MERGE TABLE ratings {}".format(RATINGS_TABLE_DEF)
c.execute(mtable)
import os, tempfile
try:
from MonetDBtesting import process
except ImportError:
import process
from MonetDBtesting.sqltest import SQLTestCase
with tempfile.TemporaryDirectory() as farm_dir:
os.mkdir(os.path.join(farm_dir, 'mydb'))
with process.server(mapiport='0',
dbname='mydb',
dbfarm=os.path.join(farm_dir, 'mydb'),
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE) as s:
with SQLTestCase() as mdb:
mdb.connect(username="******",
password="******",
database='mydb',
port=s.dbport)
mdb.execute("create table test (col int);").assertSucceeded()
mdb.execute("insert into test values (1), (2), (3);"
).assertSucceeded().assertRowCount(3)
mdb.execute("begin transaction;").assertSucceeded()
mdb.execute("create table test_new (col int);").assertSucceeded()
mdb.execute("commit;").assertSucceeded()
mdb.execute("begin transaction;").assertSucceeded()
try:
from MonetDBtesting import process
except ImportError:
import process
import sys
import os
p = os.path.join(os.getenv('GDK_DBFARM'), 'transient_dir')
with process.server(stdin=process.PIPE, stdout=process.PIPE,
stderr=process.PIPE, dbextra=p) as s1:
with process.server(stdin=process.PIPE, stdout=process.PIPE,
stderr=process.PIPE, dbextra=p) as s2:
out, err = s1.communicate()
sys.stdout.write(out)
sys.stderr.write(err)
out, err = s2.communicate()
sys.stdout.write(out)
sys.stderr.write(err)
try:
from MonetDBtesting import process
except ImportError:
import process
import sys
import os
p = os.path.join(os.getenv('GDK_DBFARM'), 'transient_dir')
s1 = process.server(stdin = process.PIPE, stdout = process.PIPE,
stderr = process.PIPE, dbextra=p)
s2 = process.server(stdin = process.PIPE, stdout = process.PIPE,
stderr = process.PIPE, dbextra=p)
out, err = s1.communicate()
sys.stdout.write(out)
sys.stderr.write(err)
out, err = s2.communicate()
sys.stdout.write(out)
sys.stderr.write(err)
import os, tempfile
from MonetDBtesting.sqltest import SQLTestCase
try:
from MonetDBtesting import process
except ImportError:
import process
with tempfile.TemporaryDirectory() as farm_dir:
os.mkdir(os.path.join(farm_dir, 'db1'))
# Start the server without readonly one time to initialize catalog
with process.server(mapiport='0',
dbname='db1',
dbfarm=os.path.join(farm_dir, 'db1'),
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE) as s:
s.communicate()
with process.server(mapiport='0',
dbname='db1',
dbfarm=os.path.join(farm_dir, 'db1'),
args=["--set", "gdk_readonly=yes"],
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE) as s:
with SQLTestCase() as tc:
tc.connect(username="******",
password="******",
port=s.dbport,
database='db1')
tc.execute("create table t2 (a int);").assertFailed(
except ImportError:
import process
dbfarm = os.getenv("GDK_DBFARM")
tstdb = os.getenv("TSTDB")
if not tstdb or not dbfarm:
print "No TSTDB or GDK_DBFARM in environment"
sys.exit(1)
dbname = tstdb + "-bug2875"
# clean up before we start
if os.path.exists(os.path.join(dbfarm, dbname)):
import shutil
shutil.rmtree(os.path.join(dbfarm, dbname))
s = process.server(stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE, dbname=dbname)
out, err = s.communicate()
sys.stdout.write(out)
sys.stderr.write(err)
s = process.server(stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE, dbname=dbname)
c = process.client(lang="sqldump", stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE, dbname=dbname)
out, err = c.communicate()
sys.stdout.write(out)
sys.stderr.write(err)
out, err = s.communicate()
sys.stdout.write(out)
sys.stderr.write(err)
sys.exit(1)
def freeport():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 0))
port = sock.getsockname()[1]
sock.close()
return port
cloneport = freeport()
dbname = tstdb
dbnameclone = tstdb + '-clone'
#master = process.server(dbname = dbname, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE)
slave = process.server(dbname = dbnameclone, mapiport = cloneport, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE)
c = process.client('sql', server = slave, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE)
cout, cerr = c.communicate('''\
select * from tmp;
call replicate('%s');
select * from tmp;
''' %dbname)
sout, serr = slave.communicate()
#mout, merr = master.communicate()
#sys.stdout.write(mout)
sys.stdout.write(sout)
sys.stdout.write(cout)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 0))
port = sock.getsockname()[1]
sock.close()
return port
myport = freeport()
with tempfile.TemporaryDirectory() as farm_dir:
os.mkdir(os.path.join(farm_dir, 'db1'))
with process.server(mapiport=myport,
dbname='db1',
dbfarm=os.path.join(farm_dir, 'db1'),
args=[
"--set", "gdk_nr_threads=2", "--forcemito",
"--loadmodule", "udf"
],
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE) as s:
with SQLTestCase() as tc:
tc.connect(username="******",
password="******",
port=myport,
database='db1')
tc.execute(
"create table tab1 (group_by_col int, index_col int, f float);"
).assertSucceeded()
tc.execute("create table tab2 (index_col int, f float);"
).assertSucceeded()
tc.execute(
# find zipped database
dbfarm = os.environ['GDK_DBFARM']
db = os.path.join(dbfarm, os.environ['TSTDB'])
archive = os.path.join(dbfarm, 'lasthgerelempty.zip')
if not os.path.exists(archive):
print('file "{}" not found'.format(archive), file=sys.stderr)
sys.exit(1)
# unpackage database
with zipfile.ZipFile(archive) as z:
z.extractall(path=db)
# start server and dump database
with process.server(mapiport='0',
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE) as srv:
with process.client(lang='sqldump',
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE,
server=srv) as clt:
cltout, clterr = clt.communicate()
srvout, srverr = srv.communicate()
srvout = [
line for line in srvout.splitlines(keepends=True)
if not line.startswith('#')
]
cltout = [
line for line in cltout.splitlines(keepends=False)
# Find a free network port
def freeport():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 0))
port = sock.getsockname()[1]
sock.close()
return port
farm_dir = tempfile.mkdtemp()
node1_port = freeport()
node1_proc = process.server(mapiport=node1_port,
dbname='node1',
dbfarm=os.path.join(farm_dir, 'node1'),
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
node1_conn = pymonetdb.connect(database='node1',
port=node1_port,
autocommit=True)
node1_cur = node1_conn.cursor()
print("# node1: CREATE TABLE tbl (id INT, name TEXT)")
node1_cur.execute("CREATE TABLE tbl (id INT, name TEXT)")
print("# node1: INSERT INTO tbl VALUES (1, '1')")
node1_cur.execute("INSERT INTO tbl VALUES (1, '1')")
print("# node1: INSERT INTO tbl VALUES (2, '2')")
node1_cur.execute("INSERT INTO tbl VALUES (2, '2')")
print("# node1: INSERT INTO tbl (id) VALUES (3)")
node1_cur.execute("INSERT INTO tbl (id) VALUES (3)")
def server():
return process.server(stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 0))
port = sock.getsockname()[1]
sock.close()
return port
with tempfile.TemporaryDirectory() as farm_dir:
os.mkdir(os.path.join(farm_dir, 'node1'))
os.mkdir(os.path.join(farm_dir, 'node2'))
# node1 is the worker
prt1 = freeport()
with process.server(mapiport=prt1,
dbname='node1',
dbfarm=os.path.join(farm_dir, 'node1'),
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE) as prc1:
conn1 = pymonetdb.connect(database='node1', port=prt1, autocommit=True)
cur1 = conn1.cursor()
cur1.execute("create table s1 (i int)")
if cur1.execute("insert into s1 values (23), (42)") != 2:
sys.stderr.write("2 rows inserted expected")
cur1.execute("create table t1 (s varchar(10))")
if cur1.execute("insert into t1 values ('abc'), ('efg')") != 2:
sys.stderr.write("2 rows inserted expected")
cur1.close()
conn1.close()
# node2 is the master
prt2 = freeport()
def test_snapshot(z_extension, expected_initial_bytes, unpack=True):
mydb = tstdb + '_snap'
mydbdir = os.path.join(dbfarm, mydb)
tarname = os.path.join(dbfarm, mydb + '.tar' + z_extension)
# clean up remainder of earlier run
if os.path.exists(mydbdir):
shutil.rmtree(mydbdir)
if os.path.exists(tarname):
os.remove(tarname)
try:
# figure out a free port number
s = socket.socket()
s.bind(('0.0.0.0', 0))
mapi_port = s.getsockname()[1]
s.close()
s = None
# start the server
with process.server(dbname=mydb,
mapiport=mapi_port,
stdin=process.PIPE) as server:
# connection 1 creates, inserts, commits and inserts uncommitted
conn1 = pymonetdb.connect(database=server.dbname,
hostname='localhost',
port=mapi_port,
username="******",
password="******",
autocommit=False)
cur1 = conn1.cursor()
cur1.execute("create table foo(t varchar(40))")
cur1.execute("insert into foo values ('committed1')")
conn1.commit()
cur1.execute("insert into foo values ('uncommitted1')")
# connection 2 inserts some more uncommitted
conn2 = pymonetdb.connect(database=server.dbname,
hostname='localhost',
port=mapi_port,
username="******",
password="******",
autocommit=False)
cur2 = conn2.cursor()
try:
cur2.execute("insert into foo values ('uncommitted2')")
except:
pass
# then conn1 creates the snapshot
cur1.execute("call sys.hot_snapshot(%(tarname)s)",
dict(tarname=tarname))
# we shut down the server and delete the dbdir
cur1.close()
conn1.close()
cur2.close()
conn2.close()
server.terminate()
server.wait()
shutil.rmtree(mydbdir)
# check if the file is actually compressed using the right algorithm
with open(tarname, "rb") as f:
initial_bytes = f.read(len(expected_initial_bytes))
assert initial_bytes == expected_initial_bytes
if not unpack:
return
# open with decompression
if tarname.endswith('.bz2'):
import bz2
f = bz2.BZ2File(tarname, 'rb')
elif tarname.endswith('.gz'):
import gzip
f = gzip.GzipFile(tarname, 'rb')
elif tarname.endswith('.lz4'):
import lz4.frame
f = lz4.frame.LZ4FrameFile(tarname, 'rb')
elif tarname.endswith('.xz'):
import lzma
f = lzma.LZMAFile(tarname, 'rb')
else:
f = open(tarname, 'rb')
# and extract the tar file
with tarfile.open(fileobj=f) as tar:
tar.extractall(dbfarm)
f.close()
# and restart the server
with process.server(dbname=mydb,
mapiport=mapi_port,
stdin=process.PIPE) as server:
# question is, is our data still there?
conn3 = pymonetdb.connect(database=server.dbname,
hostname='localhost',
port=mapi_port,
username="******",
password="******",
autocommit=False)
cur3 = conn3.cursor()
cur3.execute('select * from foo')
foo = cur3.fetchall()
# uncommitted1 and uncommitted2 should not be present
assert foo == [('committed1', )]
cur3.close()
conn3.close()
server.terminate()
server.wait()
finally:
if os.path.exists(mydbdir):
shutil.rmtree(mydbdir)
if os.path.exists(tarname):
os.remove(tarname)
try:
from MonetDBtesting import process
except ImportError:
import process
import os, sys
dbfarm = os.getenv('GDK_DBFARM')
tstdb = os.getenv('TSTDB')
if not tstdb or not dbfarm:
print('No TSTDB or GDK_DBFARM in environment')
sys.exit(1)
dbname = tstdb
with process.server(dbname=dbname, stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as s, \
process.client('sql', server = s, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) as c:
cout, cerr = c.communicate('''\
insert into tmp values(3,'blah'),(4,'bloh');
select * from tmp;
''')
sout, serr = s.communicate()
sys.stdout.write(sout)
sys.stdout.write(cout)
sys.stderr.write(serr)
sys.stderr.write(cerr)
def freeport():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 0))
port = sock.getsockname()[1]
sock.close()
return port
port2 = freeport()
tstdb2 = tstdb + '-clone'
if os.path.exists(os.path.join(dbfarm, tstdb2)):
shutil.rmtree(os.path.join(dbfarm, tstdb2))
# start the first server
s1 = process.server(stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
# load data into the first server's database
c1 = process.client(lang='sql',
server=s1,
args=[os.path.join(tstsrcdir, os.pardir, os.pardir, 'testdb', 'Tests', 'load.sql')],
stdin=process.PIPE,
stdout=process.DEVNULL,
stderr=process.DEVNULL)
c1out, c1err = c1.communicate()
# start the second server
s2 = process.server(dbname=tstdb2,
mapiport=port2,
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
except ImportError:
import process
import os, sys
dbfarm = os.getenv('GDK_DBFARM')
tstdb = os.getenv('TSTDB')
if not tstdb or not dbfarm:
print('No TSTDB or GDK_DBFARM in environment')
sys.exit(1)
dbname = tstdb
dbnameclone = tstdb + 'clone'
#master = process.server(dbname = dbname, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE)
with process.server(dbname=dbnameclone, mapiport='0', stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as slave, \
process.client('sql', server = slave, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) as c:
# check some errors
cout, cerr = c.communicate('''\
call wlr.master(1);
call wlr.replicate('halo');
''')
sout, serr = slave.communicate()
#mout, merr = master.communicate()
#sys.stdout.write(mout)
sys.stdout.write(sout)
sys.stdout.write(cout)
#sys.stderr.write(merr)
try:
from MonetDBtesting import process
except ImportError:
import process
import os, sys
dbfarm = os.getenv('GDK_DBFARM')
tstdb = os.getenv('TSTDB')
if not tstdb or not dbfarm:
print('No TSTDB or GDK_DBFARM in environment')
sys.exit(1)
dbname = tstdb
s = process.server(dbname = dbname, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE)
c = process.client('sql', server = s, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE)
cout, cerr = c.communicate('''\
update tmp set i = 3 where i = 1;
update tmp set s = 'blah';
select * from tmp;
''')
sout, serr = s.communicate()
sys.stdout.write(sout)
sys.stdout.write(cout)
sys.stderr.write(serr)
sys.stderr.write(cerr)
import process
def freeport():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 0))
port = sock.getsockname()[1]
sock.close()
return port
farm_dir = tempfile.mkdtemp()
os.mkdir(os.path.join(farm_dir, 'db1'))
node1_port = freeport()
node1_proc = process.server(mapiport=node1_port, dbname='db1', dbfarm=os.path.join(farm_dir, 'db1'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE)
node1_conn = pymonetdb.connect(database='db1', port=node1_port, autocommit=True)
node1_cur = node1_conn.cursor()
os.mkdir(os.path.join(farm_dir, 'db2'))
node2_port = freeport()
node2_proc = process.server(mapiport=node2_port, dbname='db2', dbfarm=os.path.join(farm_dir, 'db2'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE)
node2_conn = pymonetdb.connect(database='db2', port=node2_port, autocommit=True)
node2_cur = node2_conn.cursor()
node2_cur.execute('CREATE TABLE "tb2" ("col1" int, "col2" int);')
node2_cur.execute('INSERT INTO "tb2" VALUES (1, 1), (2, 2), (3, 3);')
node1_cur.execute('CREATE MERGE TABLE "tb1" ("col1" int, "col2" int) PARTITION BY RANGE ON ("col1");')
node1_cur.execute('CREATE REMOTE TABLE "tb2" ("col1" int, "col2" int) ON \'mapi:monetdb://localhost:'+str(node2_port)+'/db2\';')
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 0))
port = sock.getsockname()[1]
sock.close()
return port
cloneport = freeport()
dbname = tstdb
dbnameclone = tstdb + 'clone'
#master = process.server(dbname = dbname, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE)
slave = process.server(dbname=dbnameclone,
mapiport=cloneport,
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
c = process.client('sql',
server=slave,
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
# Generate a wrong master record
cout, cerr = c.communicate('''\
call wlr.master('demo');
call wlr.replicate();
select * from tmp;
call wlr.stop();
tabledata = """
INSERT INTO %SHARD% VALUES (42);
"""
def freeport():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 0))
port = sock.getsockname()[1]
sock.close()
return port
tmpdir = tempfile.mkdtemp()
try:
masterport = freeport()
masterproc = process.server(mapiport=masterport, dbname="master", dbfarm=os.path.join(tmpdir, 'master'), stdin = process.PIPE, stdout = process.PIPE)
masterconn = monetdb.sql.connect(database='', port=masterport, autocommit=True)
# load data (in parallel)
def worker_load(workerrec):
c = workerrec['conn'].cursor()
stable = shardtable + workerrec['tpf']
screateq = 'create table ' + stable + ' ' + shardedtabledefslightlydifferent;
c.execute(screateq)
c.execute(tabledata.replace("%SHARD%", stable))
# setup and start workers
workers = []
for i in range(nworkers):
import process
def freeport():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 0))
port = sock.getsockname()[1]
sock.close()
return port
with tempfile.TemporaryDirectory() as farm_dir:
os.mkdir(os.path.join(farm_dir, 'db1'))
myport = freeport()
with process.server(mapiport=myport, dbname='db1',
dbfarm=os.path.join(farm_dir, 'db1'),
stdin=process.PIPE, stdout=process.PIPE,
stderr=process.PIPE) as s:
with SQLTestCase() as tc:
tc.connect(username="******", password="******", port=myport, database='db1')
tc.execute('CREATE MERGE TABLE testme (a int, b varchar(32)) PARTITION BY RANGE ON (a);').assertSucceeded()
tc.execute('CREATE TABLE subtable1 (a int, b varchar(32));').assertSucceeded()
tc.execute('CREATE TABLE subtable2 (a int, b varchar(32));').assertSucceeded()
tc.execute('CREATE TABLE subtable3 (a int, b varchar(32));').assertSucceeded()
tc.execute('CREATE TABLE subtable4 (a int, b varchar(32));').assertSucceeded()
tc.execute('CREATE TABLE subtable5 (a int, b varchar(32));').assertSucceeded()
tc.execute('ALTER TABLE testme ADD TABLE subtable1 AS PARTITION FROM 5 TO 10;').assertSucceeded()
tc.execute('ALTER TABLE testme ADD TABLE subtable5 AS PARTITION FOR NULL VALUES;').assertSucceeded()
tc.execute('CREATE MERGE TABLE anothertest (a int, b varchar(32)) PARTITION BY RANGE USING (a + 1);').assertSucceeded()
tc.execute('ALTER TABLE anothertest ADD TABLE subtable3 AS PARTITION FROM 11 TO 20;').assertSucceeded()
tc.execute('SELECT "minimum", "maximum" FROM range_partitions;').assertSucceeded().assertDataResultMatch([("5","10"),(None,None),("11","20")])
s.communicate()
def freeport():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 0))
port = sock.getsockname()[1]
sock.close()
return port
farm_dir = tempfile.mkdtemp()
os.mkdir(os.path.join(farm_dir, 'renames'))
sport = freeport()
s = process.server(mapiport=sport,
dbname='renames',
dbfarm=os.path.join(farm_dir, 'renames'),
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE)
client1 = pymonetdb.connect(database='renames', port=sport, autocommit=True)
client2 = pymonetdb.connect(database='renames', port=sport, autocommit=True)
cursor1 = client1.cursor()
cursor2 = client2.cursor()
cursor1.execute('CREATE TABLE tab1 (col1 tinyint);')
cursor1.execute('INSERT INTO tab1 VALUES (1);')
cursor2.execute('SELECT col1 FROM tab1;')
cursor1.execute('SELECT col1 FROM tab1;')
cursor1.execute('ALTER TABLE tab1 RENAME TO tab2;')
cursor1.execute('SELECT col1 FROM tab2;')
def freeport():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 0))
port = sock.getsockname()[1]
sock.close()
return port
with tempfile.TemporaryDirectory() as farm_dir:
mypath = os.path.join(farm_dir, '进起都家', 'myserver', 'mynode')
os.makedirs(mypath)
prt = freeport()
with process.server(mapiport=prt,
dbname='mynode',
dbfarm=mypath,
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE) as prc:
conn = pymonetdb.connect(database='mynode', port=prt, autocommit=True)
cur = conn.cursor()
cur.execute('SELECT \'进起都家\';')
if cur.fetchall() != [('进起都家', )]:
sys.stderr.write("'进起都家' expected")
cur.close()
conn.close()
prc.communicate()
def server_start(args = []):
sys.stderr.write('#mserver\n')
sys.stderr.flush()
srv = process.server(args = args, stdin = process.PIPE,
stdout = process.PIPE, stderr = process.PIPE)
return srv
out, err = s.communicate()
sys.stdout.write(out)
sys.stderr.write(err)
def client(input):
c = process.client('sql', stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE)
out, err = c.communicate(input)
sys.stdout.write(out)
sys.stderr.write(err)
script1 = '''\
create table "something" (a int);\
alter table "something" rename to "newname";\
insert into "newname" values (1);\
select "a" from "newname";
'''
script2 = '''\
select "name" from sys.tables where "name" = 'newname';\
insert into "newname" values (2);\
select "a" from "newname";\
drop table "newname";
'''
s = process.server(args=[], stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE)
client(script1)
server_stop(s)
s = process.server(args=[], stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE)
client(script2)
server_stop(s)
return pymonetdb.connect(database = os.getenv('TSTDB'),
hostname = '127.0.0.1',
port = int(os.getenv('MAPIPORT')),
username = '******',
password = '******',
autocommit = autocommit)
def query(conn, sql):
cur = conn.cursor()
cur.execute(sql)
r = cur.fetchall()
cur.close()
return r
# no timeout since we need to kill mserver5, not the inbetween Mtimeout
s = process.server(stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE)
# boring setup and schema creation stuff:
c1 = connect(True)
c1.execute('create table foo (a int)')
c1.execute('create table bar (a int)')
c1.execute('insert into foo values (1),(2),(3)')
# print(query(c1, """select * from storage() where "table"='foo'"""))
print(query(c1, 'select * from foo'))
# Run 'delete from foo' with store_nr_active > 1
# This causes MonetDB to allocate a new file for foo rather than
# wiping the existing one
c2 = connect(True)
c2.execute('start transaction')
c1.execute('delete from foo')
def freeport():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 0))
port = sock.getsockname()[1]
sock.close()
return port
tmpdir = tempfile.mkdtemp()
try:
masterport = freeport()
masterproc = process.server(mapiport=masterport,
dbname="master",
dbfarm=os.path.join(tmpdir, 'master'),
stdin=process.PIPE,
stdout=process.PIPE)
masterconn = monetdb.sql.connect(database='',
port=masterport,
autocommit=True)
# load data (in parallel)
def worker_load(workerrec):
c = workerrec['conn'].cursor()
stable = shardtable + workerrec['tpf']
screateq = 'create table ' + stable + ' ' + shardedtabledefslightlydifferent
c.execute(screateq)
c.execute(tabledata.replace("%SHARD%", stable))
from __future__ import print_function
try:
from MonetDBtesting import process
except ImportError:
import process
import os
dbfarm = os.getenv('GDK_DBFARM')
tstdb = os.getenv('TSTDB')
if not tstdb or not dbfarm:
print('No TSTDB or GDK_DBFARM in environment')
sys.exit(1)
dbname = tstdb + '-2695'
# clean up before we start
if os.path.exists(os.path.join(dbfarm, dbname)):
import shutil
shutil.rmtree(os.path.join(dbfarm, dbname))
s = process.server(args = ['--readonly'], dbname = dbname)
s.wait()
from MonetDBtesting.sqltest import SQLTestCase
try:
from MonetDBtesting import process
except ImportError:
import process
with process.server(args=[],
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE) as s:
with SQLTestCase() as tc:
tc.connect(username="******", password="******")
tc.execute("create table t1 (a int);").assertSucceeded()
tc.execute("insert into t1 (a) values ( 1 );").assertSucceeded(
).assertRowCount(1)
tc.execute(
"select * from t1;").assertSucceeded().assertDataResultMatch([(1, )
])
tc.execute("create table t2 (a int);").assertSucceeded()
tc.execute("drop table t2;").assertSucceeded()
s.communicate()
with process.server(args=["--readonly"],
stdin=process.PIPE,
stdout=process.PIPE,
stderr=process.PIPE) as s:
with SQLTestCase() as tc:
tc.connect(username="******", password="******")
tc.execute("drop table t1;").assertFailed(
err_message=
def server_start(args):
sys.stderr.write('#mserver: "%s"\n' % " ".join(args))
sys.stderr.flush()
srv = process.server(args=args, stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE)
return srv
