def main(): s = process.server(args=[], stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) client(script1) client(script2) client(script3) client(script4) client(script10) client(script11) client(script12) server_stop(s) s = process.server(args=["--readonly"], stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) client(script1) client(script3) client(script6) client(script5) client(script7) client(script8) client(script9) server_stop(s)
def main(): s = process.server(args = [], stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) client(script1) client(script3) client(script4) client(script2) client(script5) server_stop(s) s = process.server(args = ["--readonly"], stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) client(script8) client(script4) client(script2) client(script6) client(script7) client(script9) client(script10) client(script11) client(script12) client(script3) client(script13) client(script14) server_stop(s)
def server_start(): sys.stderr.write('#mserver\n') sys.stderr.flush() srv = process.server(stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) return srv
def create_workers(fn_template, nworkers, cmovies, ratings_table_def_fk): workers = [] for i in range(nworkers): workerport = freeport() workerdbname = 'worker_{}'.format(i) workerrec = { 'num': i, 'port': workerport, 'dbname': workerdbname, 'dbfarm': os.path.join(TMPDIR, workerdbname), 'mapi': 'mapi:monetdb://localhost:{}/{}/sys/ratings'.format(workerport, workerdbname), } os.mkdir(workerrec['dbfarm']) workerrec['proc'] = process.server(mapiport=workerrec['port'], dbname=workerrec['dbname'], dbfarm=workerrec['dbfarm'], stdin=process.PIPE, stdout=process.PIPE) workerrec['conn'] = pymonetdb.connect(database=workerrec['dbname'], port=workerport, autocommit=True) filename = fn_template.format(workerrec['num']) t = threading.Thread(target=worker_load, args=[filename, workerrec, cmovies, ratings_table_def_fk]) t.start() workerrec['loadthread'] = t workers.append(workerrec) for wrec in workers: wrec['loadthread'].join() return workers
def server_start(args): sys.stderr.write('#mserver: "%s"\n' % ' '.join(args)) sys.stderr.flush() srv = process.server(args=args, stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) return srv
def main(): s = process.server(args=["--set", "gdk_nr_threads=2", "--forcemito"], stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) client(script1) out, err = s.communicate() sys.stdout.write(out) sys.stderr.write(err)
def main(): s = process.server(args = ["--set", "gdk_readonly=yes"], stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) client(script1) out, err = s.communicate() sys.stdout.write(out) sys.stderr.write(err)
def main(): s = process.server(args = ["--set", "gdk_nr_threads=2", "--forcemito"], stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) client(script1) out, err = s.communicate() sys.stdout.write(out) sys.stderr.write(err)
def main(): s = process.server(stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) client(script1, 'monetdb', 'monetdb') client(script2, 'mydummyuser', 'mydummyuser') client(script3, 'monetdb', 'monetdb') client(script4, 'mydummyuser', 'mydummyuser') client(script5, 'monetdb', 'monetdb') out, err = s.communicate() sys.stdout.write(out) sys.stderr.write(err)
def server_start(x,s): sys.stdout.write('\nserver %d%d\n' % (x,s)) sys.stderr.write('\nserver %d%d\n' % (x,s)) sys.stderr.flush() sys.stderr.write('#mserver\n') sys.stdout.flush() sys.stderr.flush() srv = process.server(stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) return srv
def server_start(x, s): sys.stdout.write('\nserver %d%d\n' % (x, s)) sys.stderr.write('\nserver %d%d\n' % (x, s)) sys.stderr.flush() sys.stderr.write('#mserver\n') sys.stdout.flush() sys.stderr.flush() srv = process.server(stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) return srv
def remote_server_start(x,s): sys.stdout.write('\nserver %d%d\n' % (x,s)) sys.stderr.write('\nserver %d%d\n' % (x,s)) sys.stderr.flush() sys.stderr.write('#remote mserver\n') sys.stdout.flush() sys.stderr.flush() port = os.getenv('MAPIPORT', '50000') srv = process.server(mapiport = int(port) + 1, dbname = '%s_test1' % os.getenv('TSTDB'), stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) return srv
def remote_server_start(x, s): sys.stdout.write('\nserver %d%d\n' % (x, s)) sys.stderr.write('\nserver %d%d\n' % (x, s)) sys.stderr.flush() sys.stderr.write('#remote mserver\n') sys.stdout.flush() sys.stderr.flush() port = os.getenv('MAPIPORT', '50000') srv = process.server(mapiport=int(port) + 1, dbname='%s_test1' % os.getenv('TSTDB'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) return srv
workerrec['conn'] = pymonetdb.connect(database=workerrec['dbname'], port=workerport, autocommit=True) filename = fn_template.format(workerrec['num']) t = threading.Thread(target=worker_load, args=[filename, workerrec, cmovies, ratings_table_def_fk]) t.start() workerrec['loadthread'] = t workers.append(workerrec) for wrec in workers: wrec['loadthread'].join() return workers # Start supervisor database supervisorport = freeport() os.mkdir(os.path.join(TMPDIR, "supervisor")) supervisorproc = process.server(mapiport=supervisorport, dbname="supervisor", dbfarm=os.path.join(TMPDIR, "supervisor"), stdin=process.PIPE, stdout=process.PIPE) supervisorconn = pymonetdb.connect(database='supervisor', port=supervisorport, autocommit=True) supervisor_uri = "mapi:monetdb://localhost:{}/supervisor".format(supervisorport) c = supervisorconn.cursor() # Create the movies table and load the data movies_filename=os.getenv("TSTDATAPATH")+"/netflix_data/movies.csv" movies_create = "CREATE TABLE movies {}".format(MOVIES_TABLE_DEF) c.execute(movies_create) load_movies = "COPY INTO movies FROM '{}' USING DELIMITERS ',','\n','\"'".format(movies_filename) c.execute(load_movies) # Declare the ratings merge table on supervisor mtable = "CREATE MERGE TABLE ratings {}".format(RATINGS_TABLE_DEF) c.execute(mtable)
import sys, os, tempfile, pymonetdb try: from MonetDBtesting import process except ImportError: import process with tempfile.TemporaryDirectory() as farm_dir: os.mkdir(os.path.join(farm_dir, 'node1')) os.mkdir(os.path.join(farm_dir, 'node2')) os.mkdir(os.path.join(farm_dir, 'node3')) with process.server(mapiport='0', dbname='node1', dbfarm=os.path.join(farm_dir, 'node1'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as node1_proc: with process.server(mapiport='0', dbname='node2', dbfarm=os.path.join(farm_dir, 'node2'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as node2_proc: with process.server(mapiport='0', dbname='node3', dbfarm=os.path.join(farm_dir, 'node3'), stdin=process.PIPE, stdout=process.PIPE,
def freeport(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', 0)) port = sock.getsockname()[1] sock.close() return port port2 = freeport() tstdb2 = tstdb + '-clone' if os.path.exists(os.path.join(dbfarm, tstdb2)): shutil.rmtree(os.path.join(dbfarm, tstdb2)) # start the first server s1 = process.server(stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) # load data into the first server's database c1 = process.client(lang='sql', server=s1, args=[os.path.join(tstsrcdir, os.pardir, os.pardir, 'testdb', 'Tests', 'load.sql')], stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) c1out, c1err = c1.communicate() # start the second server s2 = process.server(dbname=tstdb2, mapiport=port2, stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE)
def client(next_user, next_passwd, input): c = process.client('sql', user=next_user, passwd=next_passwd, stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) out, err = c.communicate(input) sys.stdout.write(out) sys.stderr.write(err) s = process.server(args=[], stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) client( 'monetdb', 'monetdb', '''\ CREATE schema "myschema";\ CREATE TABLE "myschema"."test" ("id" integer, "name" varchar(20));\ INSERT INTO "myschema"."test" ("id", "name") VALUES (1,'Tom'),(2,'Karen');\ CREATE USER myuser WITH UNENCRYPTED PASSWORD 'Test123' NAME 'Hulk' SCHEMA "myschema";\ GRANT SELECT ON "myschema"."test" TO myuser; ''') client('myuser', 'Test123', '''\ SELECT "id", "name" FROM "myschema"."test"; ''')
def query(conn, sql): print(sql) cur = conn.cursor() cur.execute(sql) r = cur.fetchall() cur.close() return r farm_dir = tempfile.mkdtemp() os.mkdir(os.path.join(farm_dir, 'node1')) os.mkdir(os.path.join(farm_dir, 'node2')) # node1 is the worker prt1 = freeport() prc1 = process.server(mapiport=prt1, dbname='node1', dbfarm=os.path.join(farm_dir, 'node1'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) conn1 = pymonetdb.connect(database='node1', port=prt1, autocommit=True) q = "create table s1 (i int)" print(q); conn1.execute(q) q = "insert into s1 values (23), (42)" print(q); conn1.execute(q) q = "create table t1 (s varchar(10))" print(q); conn1.execute(q) q = "insert into t1 values ('abc'), ('efg')" print(q); conn1.execute(q) # node2 is the master prt2 = freeport() prc2 = process.server(mapiport=prt2, dbname='node2', dbfarm=os.path.join(farm_dir, 'node2'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) conn2 = pymonetdb.connect(database='node2', port=prt2, autocommit=True)
return pymonetdb.connect(database = os.getenv('TSTDB'), hostname = '127.0.0.1', port = int(os.getenv('MAPIPORT')), username = '******', password = '******', autocommit = autocommit) def query(conn, sql): cur = conn.cursor() cur.execute(sql) r = cur.fetchall() cur.close() return r # no timeout since we need to kill mserver5, not the inbetween Mtimeout s = process.server(stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) # boring setup and schema creation stuff: c1 = connect(True) c1.execute('create table foo (a int)') c1.execute('create table bar (a int)') c1.execute('insert into foo values (1),(2),(3)') # print query(c1, """select * from storage() where "table"='foo'""") print query(c1, 'select * from foo') # Run 'delete from foo' with store_nr_active > 1 # This causes MonetDB to allocate a new file for foo rather than # wiping the existing one c2 = connect(True) c2.execute('start transaction') c1.execute('delete from foo')
import os, tempfile try: from MonetDBtesting import process except ImportError: import process from MonetDBtesting.sqltest import SQLTestCase with tempfile.TemporaryDirectory() as farm_dir: os.mkdir(os.path.join(farm_dir, 'mydb')) with process.server(mapiport='0', dbname='mydb', dbfarm=os.path.join(farm_dir, 'mydb'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as s: with SQLTestCase() as mdb: mdb.connect(username="******", password="******", database='mydb', port=s.dbport) mdb.execute("create table test (col int);").assertSucceeded() mdb.execute("insert into test values (1), (2), (3);" ).assertSucceeded().assertRowCount(3) mdb.execute("begin transaction;").assertSucceeded() mdb.execute("create table test_new (col int);").assertSucceeded() mdb.execute("commit;").assertSucceeded() mdb.execute("begin transaction;").assertSucceeded()
try: from MonetDBtesting import process except ImportError: import process import sys import os p = os.path.join(os.getenv('GDK_DBFARM'), 'transient_dir') with process.server(stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE, dbextra=p) as s1: with process.server(stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE, dbextra=p) as s2: out, err = s1.communicate() sys.stdout.write(out) sys.stderr.write(err) out, err = s2.communicate() sys.stdout.write(out) sys.stderr.write(err)
try: from MonetDBtesting import process except ImportError: import process import sys import os p = os.path.join(os.getenv('GDK_DBFARM'), 'transient_dir') s1 = process.server(stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE, dbextra=p) s2 = process.server(stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE, dbextra=p) out, err = s1.communicate() sys.stdout.write(out) sys.stderr.write(err) out, err = s2.communicate() sys.stdout.write(out) sys.stderr.write(err)
import os, tempfile from MonetDBtesting.sqltest import SQLTestCase try: from MonetDBtesting import process except ImportError: import process with tempfile.TemporaryDirectory() as farm_dir: os.mkdir(os.path.join(farm_dir, 'db1')) # Start the server without readonly one time to initialize catalog with process.server(mapiport='0', dbname='db1', dbfarm=os.path.join(farm_dir, 'db1'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as s: s.communicate() with process.server(mapiport='0', dbname='db1', dbfarm=os.path.join(farm_dir, 'db1'), args=["--set", "gdk_readonly=yes"], stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as s: with SQLTestCase() as tc: tc.connect(username="******", password="******", port=s.dbport, database='db1') tc.execute("create table t2 (a int);").assertFailed(
except ImportError: import process dbfarm = os.getenv("GDK_DBFARM") tstdb = os.getenv("TSTDB") if not tstdb or not dbfarm: print "No TSTDB or GDK_DBFARM in environment" sys.exit(1) dbname = tstdb + "-bug2875" # clean up before we start if os.path.exists(os.path.join(dbfarm, dbname)): import shutil shutil.rmtree(os.path.join(dbfarm, dbname)) s = process.server(stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE, dbname=dbname) out, err = s.communicate() sys.stdout.write(out) sys.stderr.write(err) s = process.server(stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE, dbname=dbname) c = process.client(lang="sqldump", stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE, dbname=dbname) out, err = c.communicate() sys.stdout.write(out) sys.stderr.write(err) out, err = s.communicate() sys.stdout.write(out) sys.stderr.write(err)
sys.exit(1) def freeport(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', 0)) port = sock.getsockname()[1] sock.close() return port cloneport = freeport() dbname = tstdb dbnameclone = tstdb + '-clone' #master = process.server(dbname = dbname, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) slave = process.server(dbname = dbnameclone, mapiport = cloneport, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) c = process.client('sql', server = slave, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) cout, cerr = c.communicate('''\ select * from tmp; call replicate('%s'); select * from tmp; ''' %dbname) sout, serr = slave.communicate() #mout, merr = master.communicate() #sys.stdout.write(mout) sys.stdout.write(sout) sys.stdout.write(cout)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', 0)) port = sock.getsockname()[1] sock.close() return port myport = freeport() with tempfile.TemporaryDirectory() as farm_dir: os.mkdir(os.path.join(farm_dir, 'db1')) with process.server(mapiport=myport, dbname='db1', dbfarm=os.path.join(farm_dir, 'db1'), args=[ "--set", "gdk_nr_threads=2", "--forcemito", "--loadmodule", "udf" ], stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as s: with SQLTestCase() as tc: tc.connect(username="******", password="******", port=myport, database='db1') tc.execute( "create table tab1 (group_by_col int, index_col int, f float);" ).assertSucceeded() tc.execute("create table tab2 (index_col int, f float);" ).assertSucceeded() tc.execute(
# find zipped database dbfarm = os.environ['GDK_DBFARM'] db = os.path.join(dbfarm, os.environ['TSTDB']) archive = os.path.join(dbfarm, 'lasthgerelempty.zip') if not os.path.exists(archive): print('file "{}" not found'.format(archive), file=sys.stderr) sys.exit(1) # unpackage database with zipfile.ZipFile(archive) as z: z.extractall(path=db) # start server and dump database with process.server(mapiport='0', stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as srv: with process.client(lang='sqldump', stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE, server=srv) as clt: cltout, clterr = clt.communicate() srvout, srverr = srv.communicate() srvout = [ line for line in srvout.splitlines(keepends=True) if not line.startswith('#') ] cltout = [ line for line in cltout.splitlines(keepends=False)
# Find a free network port def freeport(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', 0)) port = sock.getsockname()[1] sock.close() return port farm_dir = tempfile.mkdtemp() node1_port = freeport() node1_proc = process.server(mapiport=node1_port, dbname='node1', dbfarm=os.path.join(farm_dir, 'node1'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) node1_conn = pymonetdb.connect(database='node1', port=node1_port, autocommit=True) node1_cur = node1_conn.cursor() print("# node1: CREATE TABLE tbl (id INT, name TEXT)") node1_cur.execute("CREATE TABLE tbl (id INT, name TEXT)") print("# node1: INSERT INTO tbl VALUES (1, '1')") node1_cur.execute("INSERT INTO tbl VALUES (1, '1')") print("# node1: INSERT INTO tbl VALUES (2, '2')") node1_cur.execute("INSERT INTO tbl VALUES (2, '2')") print("# node1: INSERT INTO tbl (id) VALUES (3)") node1_cur.execute("INSERT INTO tbl (id) VALUES (3)")
def server(): return process.server(stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', 0)) port = sock.getsockname()[1] sock.close() return port with tempfile.TemporaryDirectory() as farm_dir: os.mkdir(os.path.join(farm_dir, 'node1')) os.mkdir(os.path.join(farm_dir, 'node2')) # node1 is the worker prt1 = freeport() with process.server(mapiport=prt1, dbname='node1', dbfarm=os.path.join(farm_dir, 'node1'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as prc1: conn1 = pymonetdb.connect(database='node1', port=prt1, autocommit=True) cur1 = conn1.cursor() cur1.execute("create table s1 (i int)") if cur1.execute("insert into s1 values (23), (42)") != 2: sys.stderr.write("2 rows inserted expected") cur1.execute("create table t1 (s varchar(10))") if cur1.execute("insert into t1 values ('abc'), ('efg')") != 2: sys.stderr.write("2 rows inserted expected") cur1.close() conn1.close() # node2 is the master prt2 = freeport()
def test_snapshot(z_extension, expected_initial_bytes, unpack=True): mydb = tstdb + '_snap' mydbdir = os.path.join(dbfarm, mydb) tarname = os.path.join(dbfarm, mydb + '.tar' + z_extension) # clean up remainder of earlier run if os.path.exists(mydbdir): shutil.rmtree(mydbdir) if os.path.exists(tarname): os.remove(tarname) try: # figure out a free port number s = socket.socket() s.bind(('0.0.0.0', 0)) mapi_port = s.getsockname()[1] s.close() s = None # start the server with process.server(dbname=mydb, mapiport=mapi_port, stdin=process.PIPE) as server: # connection 1 creates, inserts, commits and inserts uncommitted conn1 = pymonetdb.connect(database=server.dbname, hostname='localhost', port=mapi_port, username="******", password="******", autocommit=False) cur1 = conn1.cursor() cur1.execute("create table foo(t varchar(40))") cur1.execute("insert into foo values ('committed1')") conn1.commit() cur1.execute("insert into foo values ('uncommitted1')") # connection 2 inserts some more uncommitted conn2 = pymonetdb.connect(database=server.dbname, hostname='localhost', port=mapi_port, username="******", password="******", autocommit=False) cur2 = conn2.cursor() try: cur2.execute("insert into foo values ('uncommitted2')") except: pass # then conn1 creates the snapshot cur1.execute("call sys.hot_snapshot(%(tarname)s)", dict(tarname=tarname)) # we shut down the server and delete the dbdir cur1.close() conn1.close() cur2.close() conn2.close() server.terminate() server.wait() shutil.rmtree(mydbdir) # check if the file is actually compressed using the right algorithm with open(tarname, "rb") as f: initial_bytes = f.read(len(expected_initial_bytes)) assert initial_bytes == expected_initial_bytes if not unpack: return # open with decompression if tarname.endswith('.bz2'): import bz2 f = bz2.BZ2File(tarname, 'rb') elif tarname.endswith('.gz'): import gzip f = gzip.GzipFile(tarname, 'rb') elif tarname.endswith('.lz4'): import lz4.frame f = lz4.frame.LZ4FrameFile(tarname, 'rb') elif tarname.endswith('.xz'): import lzma f = lzma.LZMAFile(tarname, 'rb') else: f = open(tarname, 'rb') # and extract the tar file with tarfile.open(fileobj=f) as tar: tar.extractall(dbfarm) f.close() # and restart the server with process.server(dbname=mydb, mapiport=mapi_port, stdin=process.PIPE) as server: # question is, is our data still there? conn3 = pymonetdb.connect(database=server.dbname, hostname='localhost', port=mapi_port, username="******", password="******", autocommit=False) cur3 = conn3.cursor() cur3.execute('select * from foo') foo = cur3.fetchall() # uncommitted1 and uncommitted2 should not be present assert foo == [('committed1', )] cur3.close() conn3.close() server.terminate() server.wait() finally: if os.path.exists(mydbdir): shutil.rmtree(mydbdir) if os.path.exists(tarname): os.remove(tarname)
try: from MonetDBtesting import process except ImportError: import process import os, sys dbfarm = os.getenv('GDK_DBFARM') tstdb = os.getenv('TSTDB') if not tstdb or not dbfarm: print('No TSTDB or GDK_DBFARM in environment') sys.exit(1) dbname = tstdb with process.server(dbname=dbname, stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as s, \ process.client('sql', server = s, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) as c: cout, cerr = c.communicate('''\ insert into tmp values(3,'blah'),(4,'bloh'); select * from tmp; ''') sout, serr = s.communicate() sys.stdout.write(sout) sys.stdout.write(cout) sys.stderr.write(serr) sys.stderr.write(cerr)
def freeport(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', 0)) port = sock.getsockname()[1] sock.close() return port port2 = freeport() tstdb2 = tstdb + '-clone' if os.path.exists(os.path.join(dbfarm, tstdb2)): shutil.rmtree(os.path.join(dbfarm, tstdb2)) # start the first server s1 = process.server(stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) # load data into the first server's database c1 = process.client(lang='sql', server=s1, args=[os.path.join(tstsrcdir, os.pardir, os.pardir, 'testdb', 'Tests', 'load.sql')], stdin=process.PIPE, stdout=process.DEVNULL, stderr=process.DEVNULL) c1out, c1err = c1.communicate() # start the second server s2 = process.server(dbname=tstdb2, mapiport=port2, stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE)
except ImportError: import process import os, sys dbfarm = os.getenv('GDK_DBFARM') tstdb = os.getenv('TSTDB') if not tstdb or not dbfarm: print('No TSTDB or GDK_DBFARM in environment') sys.exit(1) dbname = tstdb dbnameclone = tstdb + 'clone' #master = process.server(dbname = dbname, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) with process.server(dbname=dbnameclone, mapiport='0', stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as slave, \ process.client('sql', server = slave, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) as c: # check some errors cout, cerr = c.communicate('''\ call wlr.master(1); call wlr.replicate('halo'); ''') sout, serr = slave.communicate() #mout, merr = master.communicate() #sys.stdout.write(mout) sys.stdout.write(sout) sys.stdout.write(cout) #sys.stderr.write(merr)
try: from MonetDBtesting import process except ImportError: import process import os, sys dbfarm = os.getenv('GDK_DBFARM') tstdb = os.getenv('TSTDB') if not tstdb or not dbfarm: print('No TSTDB or GDK_DBFARM in environment') sys.exit(1) dbname = tstdb s = process.server(dbname = dbname, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) c = process.client('sql', server = s, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) cout, cerr = c.communicate('''\ update tmp set i = 3 where i = 1; update tmp set s = 'blah'; select * from tmp; ''') sout, serr = s.communicate() sys.stdout.write(sout) sys.stdout.write(cout) sys.stderr.write(serr) sys.stderr.write(cerr)
import process def freeport(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', 0)) port = sock.getsockname()[1] sock.close() return port farm_dir = tempfile.mkdtemp() os.mkdir(os.path.join(farm_dir, 'db1')) node1_port = freeport() node1_proc = process.server(mapiport=node1_port, dbname='db1', dbfarm=os.path.join(farm_dir, 'db1'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) node1_conn = pymonetdb.connect(database='db1', port=node1_port, autocommit=True) node1_cur = node1_conn.cursor() os.mkdir(os.path.join(farm_dir, 'db2')) node2_port = freeport() node2_proc = process.server(mapiport=node2_port, dbname='db2', dbfarm=os.path.join(farm_dir, 'db2'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) node2_conn = pymonetdb.connect(database='db2', port=node2_port, autocommit=True) node2_cur = node2_conn.cursor() node2_cur.execute('CREATE TABLE "tb2" ("col1" int, "col2" int);') node2_cur.execute('INSERT INTO "tb2" VALUES (1, 1), (2, 2), (3, 3);') node1_cur.execute('CREATE MERGE TABLE "tb1" ("col1" int, "col2" int) PARTITION BY RANGE ON ("col1");') node1_cur.execute('CREATE REMOTE TABLE "tb2" ("col1" int, "col2" int) ON \'mapi:monetdb://localhost:'+str(node2_port)+'/db2\';') try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', 0)) port = sock.getsockname()[1] sock.close() return port cloneport = freeport() dbname = tstdb dbnameclone = tstdb + 'clone' #master = process.server(dbname = dbname, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) slave = process.server(dbname=dbnameclone, mapiport=cloneport, stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) c = process.client('sql', server=slave, stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) # Generate a wrong master record cout, cerr = c.communicate('''\ call wlr.master('demo'); call wlr.replicate(); select * from tmp; call wlr.stop();
tabledata = """ INSERT INTO %SHARD% VALUES (42); """ def freeport(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', 0)) port = sock.getsockname()[1] sock.close() return port tmpdir = tempfile.mkdtemp() try: masterport = freeport() masterproc = process.server(mapiport=masterport, dbname="master", dbfarm=os.path.join(tmpdir, 'master'), stdin = process.PIPE, stdout = process.PIPE) masterconn = monetdb.sql.connect(database='', port=masterport, autocommit=True) # load data (in parallel) def worker_load(workerrec): c = workerrec['conn'].cursor() stable = shardtable + workerrec['tpf'] screateq = 'create table ' + stable + ' ' + shardedtabledefslightlydifferent; c.execute(screateq) c.execute(tabledata.replace("%SHARD%", stable)) # setup and start workers workers = [] for i in range(nworkers):
import process def freeport(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', 0)) port = sock.getsockname()[1] sock.close() return port with tempfile.TemporaryDirectory() as farm_dir: os.mkdir(os.path.join(farm_dir, 'db1')) myport = freeport() with process.server(mapiport=myport, dbname='db1', dbfarm=os.path.join(farm_dir, 'db1'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as s: with SQLTestCase() as tc: tc.connect(username="******", password="******", port=myport, database='db1') tc.execute('CREATE MERGE TABLE testme (a int, b varchar(32)) PARTITION BY RANGE ON (a);').assertSucceeded() tc.execute('CREATE TABLE subtable1 (a int, b varchar(32));').assertSucceeded() tc.execute('CREATE TABLE subtable2 (a int, b varchar(32));').assertSucceeded() tc.execute('CREATE TABLE subtable3 (a int, b varchar(32));').assertSucceeded() tc.execute('CREATE TABLE subtable4 (a int, b varchar(32));').assertSucceeded() tc.execute('CREATE TABLE subtable5 (a int, b varchar(32));').assertSucceeded() tc.execute('ALTER TABLE testme ADD TABLE subtable1 AS PARTITION FROM 5 TO 10;').assertSucceeded() tc.execute('ALTER TABLE testme ADD TABLE subtable5 AS PARTITION FOR NULL VALUES;').assertSucceeded() tc.execute('CREATE MERGE TABLE anothertest (a int, b varchar(32)) PARTITION BY RANGE USING (a + 1);').assertSucceeded() tc.execute('ALTER TABLE anothertest ADD TABLE subtable3 AS PARTITION FROM 11 TO 20;').assertSucceeded() tc.execute('SELECT "minimum", "maximum" FROM range_partitions;').assertSucceeded().assertDataResultMatch([("5","10"),(None,None),("11","20")]) s.communicate()
def freeport(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', 0)) port = sock.getsockname()[1] sock.close() return port farm_dir = tempfile.mkdtemp() os.mkdir(os.path.join(farm_dir, 'renames')) sport = freeport() s = process.server(mapiport=sport, dbname='renames', dbfarm=os.path.join(farm_dir, 'renames'), stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) client1 = pymonetdb.connect(database='renames', port=sport, autocommit=True) client2 = pymonetdb.connect(database='renames', port=sport, autocommit=True) cursor1 = client1.cursor() cursor2 = client2.cursor() cursor1.execute('CREATE TABLE tab1 (col1 tinyint);') cursor1.execute('INSERT INTO tab1 VALUES (1);') cursor2.execute('SELECT col1 FROM tab1;') cursor1.execute('SELECT col1 FROM tab1;') cursor1.execute('ALTER TABLE tab1 RENAME TO tab2;') cursor1.execute('SELECT col1 FROM tab2;')
def freeport(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', 0)) port = sock.getsockname()[1] sock.close() return port with tempfile.TemporaryDirectory() as farm_dir: mypath = os.path.join(farm_dir, '进起都家', 'myserver', 'mynode') os.makedirs(mypath) prt = freeport() with process.server(mapiport=prt, dbname='mynode', dbfarm=mypath, stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as prc: conn = pymonetdb.connect(database='mynode', port=prt, autocommit=True) cur = conn.cursor() cur.execute('SELECT \'进起都家\';') if cur.fetchall() != [('进起都家', )]: sys.stderr.write("'进起都家' expected") cur.close() conn.close() prc.communicate()
def server_start(args = []): sys.stderr.write('#mserver\n') sys.stderr.flush() srv = process.server(args = args, stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) return srv
out, err = s.communicate() sys.stdout.write(out) sys.stderr.write(err) def client(input): c = process.client('sql', stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) out, err = c.communicate(input) sys.stdout.write(out) sys.stderr.write(err) script1 = '''\ create table "something" (a int);\ alter table "something" rename to "newname";\ insert into "newname" values (1);\ select "a" from "newname"; ''' script2 = '''\ select "name" from sys.tables where "name" = 'newname';\ insert into "newname" values (2);\ select "a" from "newname";\ drop table "newname"; ''' s = process.server(args=[], stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) client(script1) server_stop(s) s = process.server(args=[], stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) client(script2) server_stop(s)
return pymonetdb.connect(database = os.getenv('TSTDB'), hostname = '127.0.0.1', port = int(os.getenv('MAPIPORT')), username = '******', password = '******', autocommit = autocommit) def query(conn, sql): cur = conn.cursor() cur.execute(sql) r = cur.fetchall() cur.close() return r # no timeout since we need to kill mserver5, not the inbetween Mtimeout s = process.server(stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) # boring setup and schema creation stuff: c1 = connect(True) c1.execute('create table foo (a int)') c1.execute('create table bar (a int)') c1.execute('insert into foo values (1),(2),(3)') # print(query(c1, """select * from storage() where "table"='foo'""")) print(query(c1, 'select * from foo')) # Run 'delete from foo' with store_nr_active > 1 # This causes MonetDB to allocate a new file for foo rather than # wiping the existing one c2 = connect(True) c2.execute('start transaction') c1.execute('delete from foo')
def freeport(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', 0)) port = sock.getsockname()[1] sock.close() return port tmpdir = tempfile.mkdtemp() try: masterport = freeport() masterproc = process.server(mapiport=masterport, dbname="master", dbfarm=os.path.join(tmpdir, 'master'), stdin=process.PIPE, stdout=process.PIPE) masterconn = monetdb.sql.connect(database='', port=masterport, autocommit=True) # load data (in parallel) def worker_load(workerrec): c = workerrec['conn'].cursor() stable = shardtable + workerrec['tpf'] screateq = 'create table ' + stable + ' ' + shardedtabledefslightlydifferent c.execute(screateq) c.execute(tabledata.replace("%SHARD%", stable))
from __future__ import print_function try: from MonetDBtesting import process except ImportError: import process import os dbfarm = os.getenv('GDK_DBFARM') tstdb = os.getenv('TSTDB') if not tstdb or not dbfarm: print('No TSTDB or GDK_DBFARM in environment') sys.exit(1) dbname = tstdb + '-2695' # clean up before we start if os.path.exists(os.path.join(dbfarm, dbname)): import shutil shutil.rmtree(os.path.join(dbfarm, dbname)) s = process.server(args = ['--readonly'], dbname = dbname) s.wait()
from MonetDBtesting.sqltest import SQLTestCase try: from MonetDBtesting import process except ImportError: import process with process.server(args=[], stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as s: with SQLTestCase() as tc: tc.connect(username="******", password="******") tc.execute("create table t1 (a int);").assertSucceeded() tc.execute("insert into t1 (a) values ( 1 );").assertSucceeded( ).assertRowCount(1) tc.execute( "select * from t1;").assertSucceeded().assertDataResultMatch([(1, ) ]) tc.execute("create table t2 (a int);").assertSucceeded() tc.execute("drop table t2;").assertSucceeded() s.communicate() with process.server(args=["--readonly"], stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) as s: with SQLTestCase() as tc: tc.connect(username="******", password="******") tc.execute("drop table t1;").assertFailed( err_message=
def server_start(args): sys.stderr.write('#mserver: "%s"\n' % " ".join(args)) sys.stderr.flush() srv = process.server(args=args, stdin=process.PIPE, stdout=process.PIPE, stderr=process.PIPE) return srv