def check_connections(self):
for name, conn in self.connections.items():
try:
r.db_list().run(conn)
except r.ReqlDriverError as e:
self.connections[name] = r.connect(
**self.connection_config[name])
def test_setup_db(self):
""" Test creation of a db and tables """
# test that the 'TEST' database doesn't exist
with rethinkdb.connect(host='localhost', port=28015) as conn:
db_list = rethinkdb.db_list().run(conn)
self.assertTrue('TEST' not in db_list)
creations = self.run_setup_db()
# confirm the correct tables were created
self.assertSetEqual(creations,
set(template.test_dataset.keys()+template.test_tables))
with rethinkdb.connect(host='localhost', port=28015) as conn:
# test that the 'TEST' database was created
db_list = rethinkdb.db_list().run(conn)
self.assertTrue('TEST' in db_list)
conn.use('TEST')
# test that the 'test' table was created
table_list = rethinkdb.table_list().run(conn)
self.assertEqual(len(table_list),
len(template.test_dataset.keys()+template.test_tables))
self.assertTrue(template.test_dataset.keys()[0] in table_list)
# test that the data is correct by checking columns
data = [row for row in rethinkdb.table(
template.test_dataset.keys()[0]).run(conn)]
with open(template.test_json) as f:
self.assertSetEqual(
set(data[0].keys())-set([u'id']),
set(json.loads(f.read())[0].keys()))
self.run_clear_test_db()
def test_drop_programmatically_drops_the_database_when_assume_yes_is_true(monkeypatch):
conn = utils.get_conn()
dbname = bigchaindb.config['database']['name']
# The db is set up by fixtures
assert r.db_list().contains(dbname).run(conn) is True
utils.drop(assume_yes=True)
assert r.db_list().contains(dbname).run(conn) is False
def test_drop_interactively_drops_the_database_when_user_says_yes(monkeypatch):
conn = utils.get_conn()
dbname = bigchaindb.config['database']['name']
# The db is set up by fixtures
assert r.db_list().contains(dbname).run(conn) is True
monkeypatch.setattr(builtins, 'input', lambda x: 'y')
utils.drop()
assert r.db_list().contains(dbname).run(conn) is False
def test_drop_interactively_drops_the_database_when_user_says_yes(monkeypatch):
conn = utils.get_conn()
dbname = bigchaindb.config['database']['name']
# The db is set up by fixtures
assert r.db_list().contains(dbname).run(conn) is True
monkeypatch.setattr(builtins, 'input', lambda x: 'y')
utils.drop()
assert r.db_list().contains(dbname).run(conn) is False
def test_drop_programmatically_drops_the_database_when_assume_yes_is_true(
monkeypatch):
conn = utils.get_conn()
dbname = bigchaindb.config['database']['name']
# The db is set up by fixtures
assert r.db_list().contains(dbname).run(conn) is True
utils.drop(assume_yes=True)
assert r.db_list().contains(dbname).run(conn) is False
async def test_run_query(db_conn, aiorethink_db_session):
cn = await db_conn
q1 = r.db_list()
q2 = r.db_list().run(cn)
for q in [q1, q2]:
res = await ar.db._run_query(q)
assert "testing" in res
with pytest.raises(TypeError):
await ar.db._run_query("Hello")
async def test_run_query(db_conn, aiorethink_db_session):
cn = await db_conn
q1 = r.db_list()
q2 = r.db_list().run(cn)
for q in [q1, q2]:
res = await ar.db._run_query(q)
assert "testing" in res
with pytest.raises(TypeError):
await ar.db._run_query("Hello")
def _rethinkdb(request):
if request.config.getoption("--only-client"):
yield None
return
import rethinkdb as r
conn = r.connect("localhost", 28015)
# drop/create test database
if "test" in r.db_list().run(conn):
r.db_drop("test").run(conn)
r.db_create("test").run(conn)
# reconnect with db=test and repl
r.connect(db="test").repl()
r.table_create("ebooks", primary_key="ebook_id").run()
r.table_create("versions", primary_key="version_id").run()
r.table_create("formats", primary_key="file_hash").run()
r.table_create("sync_events").run()
def create_index(table, name, index=None):
if name not in r.table(table).index_list().run():
if index is None:
r.db("test").table(table).index_create(name).run()
else:
r.db("test").table(table).index_create(name, index).run()
r.db("test").table(table).index_wait(name).run()
# create FK indexes
create_index("ebooks", "authortitle", index=[r.row["author"].downcase(), r.row["title"].downcase()])
create_index("ebooks", "asin", index=r.row["meta"]["asin"])
create_index("ebooks", "isbn", index=r.row["meta"]["isbn"])
create_index("versions", "ebook_id")
create_index("versions", "original_filehash")
create_index("versions", "ebook_username", index=[r.row["ebook_id"], r.row["username"]])
create_index("formats", "version_id")
create_index("formats", "uploaded")
create_index("formats", "uploaded_by")
create_index("formats", "uploadedby_dedrm", index=[r.row["uploaded_by"], r.row["dedrm"]])
create_index("sync_events", "username")
create_index("sync_events", "timestamp")
create_index("sync_events", "user_new_books_count", index=[r.row["username"], r.row["new_books_count"]])
yield r
# remove test database
if "test" in r.db_list().run(conn):
r.db_drop("test").run(conn)
conn.close()
def initialSetup():
print "Setting up database..."
dbs = rethinkdb.db_list().run()
if not con.general.databases["rethink"]["db"] in dbs:
print "Creating database in rethink"
rethinkdb.db_create(con.general.databases["rethink"]["db"]).run()
dbt = list(rethinkdb.table_list().run())
for db in c.general.flush["rethink"]:
if c.general.flush["rethink"][db]:
print "Flushing rethink "+db+" table..."
if db in dbt:
rethinkdb.table_drop(db).run()
dbt.pop(dbt.index(db))
print "Creating new rethink tables..."
for table in c.general.tables:
if not table in dbt:
print "Creating table {}".format(table)
rethinkdb.table_create(table).run()
for key in c.general.flush["redis"]:
if c.general.flush["redis"][key]:
print "Flushing redis "+key+" keys..."
keys = con.redis.keys(key+":*")
for key in keys: con.redis.delete(key)
def table_config_created_and_populated():
try:
r_conn_test = r.connect(RETHINK_HOST, RETHINK_PORT)
if not r.db_list().contains(RETHINK_DB).run(r_conn_test):
print(
f'rethink host {RETHINK_HOST} and port {RETHINK_PORT} has connected but rethink database {RETHINK_DB} is not created'
)
return False
else:
r_conn = new_rethink_connection()
out = False
if r.table_list().contains('config').run(r_conn):
rtable = r.table('config')
out = rtable.get(1).run(r_conn)
close_rethink_connection(r_conn)
if out is not False:
if out is not None:
#print('table config populated in database')
return True
else:
print('table config not populated in database')
return False
else:
return False
except Exception as e:
print(
f'rethink db connectin failed with hostname {RETHINK_HOST} and port {RETHINK_PORT}'
)
print(e)
print('Traceback: \n .{}'.format(traceback.format_exc()))
return False
def table_config_created_and_populated():
try:
r_conn_test = r.connect(RETHINK_HOST, RETHINK_PORT)
if not r.db_list().contains(RETHINK_DB).run(r_conn_test):
print(f'rethink host {RETHINK_HOST} and port {RETHINK_PORT} has connected but rethink database {RETHINK_DB} is not created')
return False
else:
r_conn = new_rethink_connection()
out = False
if r.table_list().contains('config').run(r_conn):
rtable = r.table('config')
out = rtable.get(1).run(r_conn)
close_rethink_connection(r_conn)
if out is not False:
if out is not None:
#print('table config populated in database')
return True
else:
print('table config not populated in database')
return False
else:
return False
except Exception as e:
print(f'rethink db connectin failed with hostname {RETHINK_HOST} and port {RETHINK_PORT}')
print(e)
print('Traceback: \n .{}'.format(traceback.format_exc()))
return False
def table_check(progress, conn, db, table, create_args, force):
pkey = None
if db == "rethinkdb":
raise RuntimeError("Error: Cannot import a table into the system database: 'rethinkdb'")
if db not in r.db_list().run(conn):
r.db_create(db).run(conn)
if table in r.db(db).table_list().run(conn):
if not force:
raise RuntimeError("Error: Table already exists, run with --force if you want to import into the existing table")
if 'primary_key' in create_args:
pkey = r.db(db).table(table).info()["primary_key"].run(conn)
if create_args["primary_key"] != pkey:
raise RuntimeError("Error: Table already exists with a different primary key")
else:
if 'primary_key' in create_args:
pkey = create_args["primary_key"]
else:
if not options["quiet"]:
print("no primary key specified, using default primary key when creating table")
r.db(db).table_create(table, **create_args).run(conn)
return pkey
def makeDB(host):
conn = r.connect(host, 28015)
dbs = r.db_list().run(conn)
if AC in dbs:
return 'already there'
r.db_create(AC).run(conn)
r.db(AC).table_create(WORDS, primary_key=LINE).run(conn)
r.db(AC).table_create(PREFS, primary_key=PREF).run(conn)
ra = {LINE: None, WORD: None, FREQ: None}
f = open(os.path.join(SCRIPT_DIR, "wordsCSV.txt"), 'r')
for line in f:
line = line.strip()
linesplit = line.split(',')
ra[LINE] = int(linesplit[0])
ra[WORD] = linesplit[1]
ra[FREQ] = int(linesplit[2])
if int(linesplit[0]) % 5000 == 0:
print linesplit[0]
r.db(AC).table(WORDS).insert(ra).run(conn)
f.close()
print "========================"
g = open(os.path.join(SCRIPT_DIR, "rangesCSV.txt"), 'r')
ra = {PREF: None, LOWER: None, UPPER: None}
for line in g:
line = line.strip()
linesplit = line.split(',')
ra[PREF] = linesplit[0]
ra[LOWER] = int(linesplit[1])
ra[UPPER] = int(linesplit[2])
if len(linesplit[0]) == 1:
print linesplit[0]
r.db(AC).table(PREFS).insert(ra).run(conn)
g.close()
return 'initialized'
def __init__(self, mountpoint):
self.root = mountpoint
self.connection = r.connect("localhost", 28015)
if META_DB_NAME not in r.db_list().run(self.connection):
r.db_create(META_DB_NAME).run(self.connection)
self._create_tables()
def __init__(self):
# ~ self.rele = 9
self.conf=load_config()
self.conn=False
self.cfg=False
try:
self.conn = r.connect( self.conf['RETHINKDB_HOST'],self.conf['RETHINKDB_PORT'],self.conf['RETHINKDB_DB']).repl()
except Exception as e:
log.error(e)
self.conn=False
if self.conn is not False and r.db_list().contains(self.conf['RETHINKDB_DB']).run():
if r.table_list().contains('config').run():
ready=False
while not ready:
try:
self.cfg = r.table('config').get(1).run()
ready = True
except Exception as e:
log.info('Waiting for database to be ready...')
time.sleep(1)
log.info('Your actual database version is: '+str(self.cfg['version']))
if release_version > self.cfg['version']:
log.warning('Database upgrade needed! You have version '+str(self.cfg['version'])+ ' and source code is for version '+str(release_version)+'!!')
else:
log.info('No database upgrade needed.')
self.upgrade_if_needed()
def get_tables(progress, conn, tables):
dbs = r.db_list().filter(r.row.ne('rethinkdb')).run(conn)
res = []
if len(tables) == 0:
tables = [(db, None) for db in dbs]
for db_table in tables:
if db_table[0] == 'rethinkdb':
raise RuntimeError(
"Error: Cannot export tables from the system database: 'rethinkdb'"
)
if db_table[0] not in dbs:
raise RuntimeError("Error: Database '%s' not found" % db_table[0])
if db_table[1] is None: # This is just a db name
res.extend([(db_table[0], table)
for table in r.db(db_table[0]).table_list().run(conn)])
else: # This is db and table name
if db_table[1] not in r.db(db_table[0]).table_list().run(conn):
raise RuntimeError("Error: Table not found: '%s.%s'" %
tuple(db_table))
res.append(tuple(db_table))
# Remove duplicates by making results a set
return set(res)
def client(request):
"""
Client fixture.
"""
config = ConfigShim()
# check if rethinkdb temp database already exists
db_name = config.get("database", "database_name")
with rethinkdb.connect(host=config.get("database", "ip"),
port=config.get("database", "port"),
db=config.get("database", "database_name")) as conn:
if db_name in rethinkdb.db_list().run(conn):
rethinkdb.db_drop(db_name).run(conn)
log.warning("Dropped temporary test database {0}".format(db_name))
web = WebApp(config)
test_client = web.flask_app.test_client()
def teardown():
with web.db.connect() as conn:
rethinkdb.db_drop(db_name).run(conn)
log.info("Dropped temporary test database {0}".format(db_name))
request.addfinalizer(teardown)
return test_client
def create_database():
r.connect("localhost", 28015).repl()
if (r.db_list().contains(database_name).run()):
pass
else:
r.db_create(database_name).run()
create_tables()
def __init__(self, server, port, database, table):
self.server = server
self.port = port
self.database = database
self.table = table
self.data = []
self.conn = r.connect(self.server, self.port)
if self.database in r.db_list().run(self.conn):
print('Database ' + self.database + ' exists! Using it.')
else:
resp = r.db_create(self.database).run(self.conn)
if resp["dbs_created"] == 1:
print('Database ' + self.database +
' was successfully created.')
else:
print('Error while creating database ' + self.database + '.')
self.conn.use(self.database)
if self.table in r.table_list().run(self.conn):
print('Table ' + self.table + ' exists! Using it.')
else:
resp = r.table_create(self.table).run(self.conn)
if resp["tables_created"] == 1:
print('Table ' + self.table + ' was successfully created.')
else:
print('Error while creating table ' + self.table + '.')
def table_check(progress, conn, db, table, pkey, force):
if db == "rethinkdb":
raise RuntimeError(
"Error: Cannot import a table into the system database: 'rethinkdb'"
)
if db not in r.db_list().run(conn):
r.db_create(db).run(conn)
if table in r.db(db).table_list().run(conn):
if not force:
raise RuntimeError(
"Error: Table already exists, run with --force if you want to import into the existing table"
)
extant_pkey = r.db(db).table(table).info().run(conn)["primary_key"]
if pkey is not None and pkey != extant_pkey:
raise RuntimeError(
"Error: Table already exists with a different primary key")
pkey = extant_pkey
else:
if pkey is None:
print(
"no primary key specified, using default primary key when creating table"
)
r.db(db).table_create(table).run(conn)
else:
r.db(db).table_create(table, primary_key=pkey).run(conn)
return pkey
def get_tables(host, port, auth_key, tables):
try:
conn = r.connect(host, port, auth_key=auth_key)
except r.RqlDriverError as ex:
raise RuntimeError(ex.message)
dbs = r.db_list().run(conn)
res = []
if len(tables) == 0:
tables = [[db] for db in dbs]
for db_table in tables:
if db_table[0] not in dbs:
raise RuntimeError("Error: Database '%s' not found" % db_table[0])
if len(db_table) == 1: # This is just a db name
res.extend([(db_table[0], table) for table in r.db(db_table[0]).table_list().run(conn)])
else: # This is db and table name
if db_table[1] not in r.db(db_table[0]).table_list().run(conn):
raise RuntimeError("Error: Table not found: '%s.%s'" % tuple(db_table))
res.append(tuple(db_table))
# Remove duplicates by making results a set
return set(res)
def create(self):
conn = self.connect()
db_list = r.db_list().run(conn)
db_created = False
table_created = False
if not self.db_name in db_list:
r.db_create(self.db_name).run(conn)
db_created = True
table_list = r.db(self.db_name).table_list().run(conn)
if not self.config_table_name in table_list:
r.db(self.db_name).table_create(
self.config_table_name, primary_key=self.primary_key
).run(conn)
r.db(self.db_name).table(self.config_table_name)\
.index_create(self.secondary_index).run(conn)
table_created = True
return {"db": db_created, "table": table_created}
def init_db():
'Set up the database'
if 'ld33' not in r.db_list().run(rdb.conn):
r.db_create('ld33').run(rdb.conn)
if 'games' not in app.db.table_list().run(rdb.conn):
app.db.table_create('games',
primary_key='container_id').run(rdb.conn)
def rethink(self):
print('Attempting to connect to RethinkDB')
tables = ['settings', 'numbers']
dbc = self.config.get('db')
try:
self.conn = r.connect(host=dbc['host'],
port=dbc['port'],
db=dbc['db'],
user=dbc['username'],
password=dbc['password'])
dbl = r.db_list().run(self.conn)
if dbc['db'] not in dbl:
print('Creating DB...')
r.db_create(dbc['db']).run(self.conn)
tab = r.table_list().run(self.conn)
for i in tables:
if i not in tab:
print(f'Table {i} not found. Now creating...')
r.table_create(i).run(self.conn)
except Exception as e:
print(
f'DB connection failed! Exiting...\nError details:\n{type(e).__name__}: {e}'
)
sys.exit(1)
print('Connected successfully.')
def knn_setup(max_tries=-1):
isSetup = False
if max_tries > 0:
n = 0
else:
n = max_tries - 1
while (not isSetup and n < max_tries):
connection = False
try:
connection = rdb.connect(RDB_HOST, RDB_PORT)
if not rdb.db_list().contains(RDB_DB).run(connection):
rdb.db_create(RDB_DB).run(connection)
if not rdb.db(RDB_DB).table_list().contains(
RDB_JOB_OPTIMIZED_TABLE).run(connection):
rdb.db(RDB_DB).table_create(RDB_JOB_OPTIMIZED_TABLE).run(
connection)
isSetup = update_knn(connection)
except Exception as e:
print 'DB error:', e
if connection:
connection.close()
time.sleep(5)
if max_tries > 0:
n = n + 1
if not isSetup:
# if still not setup: use data file
print "# No DB connection: Using backup file"
isSetup = update_knn(None)
return isSetup
def setUp(self):
self.servers = test_util.RethinkDBTestServers(4, server_build_dir=server_build_dir)
self.servers.__enter__()
self.port = self.servers.driver_port()
conn = r.connect(port=self.port)
if 'test' not in r.db_list().run(conn):
r.db_create('test').run(conn)
def _initialise():
log.debug('initializing rethinkdb for highstatestack data')
try:
conn = _connect()
# Check if DB exists, if not create it
db_exists = r.db_list().contains(DB).run(conn)
if not db_exists:
r.db_create(DB).run(conn)
# Check if events table exist, if not create it
table_exists = r.db(DB).table_list().contains(TABLE).run(conn)
if not table_exists:
result = r.db(DB).table_create(TABLE).run(conn)
# Check if index exists if not add it
rtable = r.db(DB).table(TABLE)
current_indexes = rtable.index_list().run(conn)
for index in INDEXES:
if index not in current_indexes:
log.debug('adding index {0}'.format(index))
rtable.index_create(index).run(conn)
except:
log.error('could not connect to rehinkdb server on {0}:{1}'.format(DB, PORT))
log.error(traceback.format_exc())
def test_drop_non_existent_db_raises_an_error(dummy_db):
from bigchaindb.common import exceptions
conn = backend.connect()
assert conn.run(r.db_list().contains(dummy_db)) is True
schema.drop_database(conn, dummy_db)
with pytest.raises(exceptions.DatabaseDoesNotExist):
schema.drop_database(conn, dummy_db)
def __init__(self,
database='apscheduler',
table='jobs',
client=None,
pickle_protocol=pickle.HIGHEST_PROTOCOL,
**connect_args):
super(RethinkDBJobStore, self).__init__()
self.pickle_protocol = pickle_protocol
if not database:
raise ValueError('The "database" parameter must not be empty')
if not table:
raise ValueError('The "table" parameter must not be empty')
if client:
self.conn = maybe_ref(client)
else:
self.conn = r.connect(db=database, **connect_args)
if database not in r.db_list().run(self.conn):
r.db_create(database).run(self.conn)
if table not in r.table_list().run(self.conn):
r.table_create(table).run(self.conn)
if 'next_run_time' not in r.table(table).index_list().run(self.conn):
r.table(table).index_create('next_run_time').run(self.conn)
self.table = r.db(database).table(table)
def __init__(self):
# ~ self.rele = 9
self.conf = load_config()
self.conn = False
self.cfg = False
try:
self.conn = r.connect(self.conf['RETHINKDB_HOST'],
self.conf['RETHINKDB_PORT'],
self.conf['RETHINKDB_DB']).repl()
except Exception as e:
log.error(e)
self.conn = False
if self.conn is not False and r.db_list().contains(
self.conf['RETHINKDB_DB']).run():
if r.table_list().contains('config').run():
self.cfg = r.table('config').get(1).run()
log.info('Your actual database version is: ' +
str(self.cfg['version']))
if release_version > self.cfg['version']:
log.warning('Database upgrade needed! You have version ' +
str(self.cfg['version']) +
' and source code is for version ' +
str(release_version) + '!!')
else:
log.info('No database upgrade needed.')
self.upgrade_if_needed()
def create_db(self):
db_list = r.db_list().run()
if self.db in db_list:
pass
else:
r.db_create(self.db).run()
def create_database(connection, dbname):
if connection.run(r.db_list().contains(dbname)):
raise exceptions.DatabaseAlreadyExists(
'Database `{}` already exists'.format(dbname))
logger.info('Create database `%s`.', dbname)
connection.run(r.db_create(dbname))
def init_rethinkdb(self):
print('Now initialising RethinkDB...')
dbc = self.config['RETHINKDB']
try:
self.conn = r.connect(host=dbc['HOST'],
port=dbc['PORT'],
db=dbc['DB'],
user=dbc['USERNAME'],
password=dbc['PASSWORD'])
dbs = r.db_list().run(self.conn)
if self.rdb not in dbs:
print('Database not present. Creating...')
r.db_create(self.rdb).run(self.conn)
tables = r.db(self.rdb).table_list().run(self.conn)
for i in self.rtables:
if i not in tables:
print(f'Table {i} not found. Creating...')
r.table_create(i).run(self.conn)
except Exception as e:
print('RethinkDB init error!\n{}: {}'.format(type(e).__name__, e))
sys.exit(1)
print('RethinkDB initialisation successful.')
def table_config_created_and_populated():
try:
r_conn_test = r.connect(RETHINK_HOST, RETHINK_PORT)
if not r.db_list().contains(RETHINK_DB).run(r_conn_test):
return False
else:
r_conn = new_rethink_connection()
out = False
if r.table_list().contains('config').run(r_conn):
rtable = r.table('config')
out = rtable.get(1).run(r_conn)
close_rethink_connection(r_conn)
if out is not False:
if out is not None:
return True
else:
return False
else:
return False
except Exception as e:
log.info('rethink db connectin failed')
log.info(e)
return False
def check_db():
r.connect(properties.get('RETHINK_HOST'), properties.get('RETHINK_PORT')).repl()
if 'relayr' in r.db_list().run():
return True
return False
def __init__(self, database='apscheduler', table='jobs', client=None,
pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
super(RethinkDBJobStore, self).__init__()
self.pickle_protocol = pickle_protocol
if not database:
raise ValueError('The "database" parameter must not be empty')
if not table:
raise ValueError('The "table" parameter must not be empty')
if client:
self.conn = maybe_ref(client)
else:
self.conn = r.connect(db=database, **connect_args)
if database not in r.db_list().run(self.conn):
r.db_create(database).run(self.conn)
if table not in r.table_list().run(self.conn):
r.table_create(table).run(self.conn)
if 'next_run_time' not in r.table(table).index_list().run(self.conn):
r.table(table).index_create('next_run_time').run(self.conn)
self.table = r.db(database).table(table)
def get_default_connection(request, url=None, **rethink_options):
conn = getattr(request.registry, '_r_conn', None)
if conn is not None:
return conn
if url is not None:
rethink_options.pop('password', None)
rethink_options.pop('user', None)
rethink_options.pop('host', None)
rethink_options.pop('port', None)
rethink_options.pop('db', None)
rethink_options.update(parse_url(url))
LOG.debug(rethink_options)
conn = r.connect(**rethink_options)
if rethink_options.get('db', R_DB) not in r.db_list().run(conn):
r.db_create(R_DB).run(conn)
if R_TABLE not in r.table_list().run(conn):
r.table_create(R_TABLE).run(conn)
setattr(request.registry, '_r_conn', conn)
return conn
def get_tables(host, port, auth_key, tables):
try:
conn = r.connect(host, port, auth_key=auth_key)
except (r.RqlError, r.RqlDriverError) as ex:
raise RuntimeError(ex.message)
dbs = r.db_list().run(conn)
res = []
if len(tables) == 0:
tables = [[db] for db in dbs]
for db_table in tables:
if db_table[0] not in dbs:
raise RuntimeError("Error: Database '%s' not found" % db_table[0])
if len(db_table) == 1: # This is just a db name
res.extend([(db_table[0], table)
for table in r.db(db_table[0]).table_list().run(conn)])
else: # This is db and table name
if db_table[1] not in r.db(db_table[0]).table_list().run(conn):
raise RuntimeError("Error: Table not found: '%s.%s'" %
tuple(db_table))
res.append(tuple(db_table))
# Remove duplicates by making results a set
return set(res)
def init_db():
'Set up the database'
if 'ld33' not in r.db_list().run(rdb.conn):
r.db_create('ld33').run(rdb.conn)
if 'games' not in app.db.table_list().run(rdb.conn):
app.db.table_create('games',
primary_key='container_id').run(rdb.conn)
def tables_check(progress, conn, files_info, force):
# Ensure that all needed databases exist and tables don't
db_list = r.db_list().run(conn)
for db in set([file_info["db"] for file_info in files_info]):
if db == "rethinkdb":
raise RuntimeError(
"Error: Cannot import tables into the system database: 'rethinkdb'"
)
if db not in db_list:
r.db_create(db).run(conn)
# Ensure that all tables do not exist (unless --forced)
already_exist = []
for file_info in files_info:
table = file_info["table"]
db = file_info["db"]
if table in r.db(db).table_list().run(conn):
if not force:
already_exist.append("%s.%s" % (db, table))
extant_pkey = r.db(db).table(table).info().run(conn)["primary_key"]
if file_info["info"]["primary_key"] != extant_pkey:
raise RuntimeError(
"Error: Table '%s.%s' already exists with a different primary key"
% (db, table))
return already_exist
def table_check(progress, conn, db, table, create_args, force):
pkey = None
if db == "rethinkdb":
raise RuntimeError(
"Error: Cannot import a table into the system database: 'rethinkdb'"
)
if db not in r.db_list().run(conn):
r.db_create(db).run(conn)
if table in r.db(db).table_list().run(conn):
if not force:
raise RuntimeError(
"Error: Table already exists, run with --force if you want to import into the existing table"
)
if 'primary_key' in create_args:
pkey = r.db(db).table(table).info()["primary_key"].run(conn)
if create_args["primary_key"] != pkey:
raise RuntimeError(
"Error: Table already exists with a different primary key")
else:
if 'primary_key' in create_args:
pkey = create_args["primary_key"]
else:
print(
"no primary key specified, using default primary key when creating table"
)
r.db(db).table_create(table, **create_args).run(conn)
return pkey
def create_database(self):
"""
Database creation.
"""
db_name = self.db_name
with self.connect() as conn:
if db_name not in r.db_list().run(conn):
r.db_create(db_name).run(conn)
def query_databases(self):
result = []
with self.connect() as rethink:
for x in r.db_list().run(rethink.client):
d = Database()
d.name = x
result.append(d)
return result
def connect_and_create():
r.connect('rtdb.goodes.net').repl()
if 'fb4s' not in r.db_list().run():
r.db_create('fb4s').run()
db = r.db('fb4s')
if 'students' not in db.table_list().run():
db.table_create('students').run()
return db.table('students')
def setUp(self):
self.servers = test_util.RethinkDBTestServers(
4, server_build_dir=server_build_dir)
self.servers.__enter__()
self.port = self.servers.driver_port()
conn = r.connect(port=self.port)
if 'test' not in r.db_list().run(conn):
r.db_create('test').run(conn)
def test_init_fails_if_db_exists():
conn = utils.get_conn()
dbname = bigchaindb.config['database']['name']
# The db is set up by fixtures
assert r.db_list().contains(dbname).run(conn) is True
with pytest.raises(bigchaindb.exceptions.DatabaseAlreadyExists):
utils.init()
def tearDownClass(self):
""" Drops the test database after the classes' tests are finished """
with rethinkdb.connect(host='localhost', port=28015) as conn:
if 'TEST' in rethinkdb.db_list().run(conn):
rethinkdb.db_drop('TEST').run(conn)
try:
self.rdb.close()
except AttributeError:
pass
def setup_db():
"""
Set up the database.
Include a sequence to make sure databases and tables exist where they
need to be.
"""
db_conn = r.connect(config['rdb_host'], config['rdb_port'])
# add all setup needed here:
if config['rdb_db'] not in r.db_list().run(db_conn):
r.db_create(config['rdb_db']).run(db_conn)
from models.user import User
from models.notice import Notice
from models.topic import Topic
from models.post import Post
from models.proposal import Proposal
from models.vote import Vote
from models.card import Card
from models.unit import Unit
from models.set import Set
from models.card_parameters import CardParameters
from models.unit_parameters import UnitParameters
from models.set_parameters import SetParameters
from models.follow import Follow
from models.user_sets import UserSets
from models.response import Response
models = (User, Notice, Topic, Post, Proposal, Vote,
Card, Unit, Set,
CardParameters, UnitParameters, SetParameters,
Follow, UserSets, Response)
tables = r.db(config['rdb_db']).table_list().run(db_conn)
for model_cls in models:
tablename = getattr(model_cls, 'tablename', None)
if tablename and tablename not in tables:
(r.db(config['rdb_db'])
.table_create(tablename)
.run(db_conn))
tables.append(tablename)
existant_indexes = (r.db(config['rdb_db'])
.table(tablename)
.index_list()
.run(db_conn))
indexes = getattr(model_cls, 'indexes', [])
for index in indexes:
if index[0] not in existant_indexes:
(r.db(config['rdb_db'])
.index_create(*index)
.run(db_conn))
db_conn.close()
def test_drop_non_existent_db_raises_an_error():
conn = utils.get_conn()
dbname = bigchaindb.config['database']['name']
# The db is set up by fixtures
assert r.db_list().contains(dbname).run(conn) is True
utils.drop(assume_yes=True)
with pytest.raises(bigchaindb.exceptions.DatabaseDoesNotExist):
utils.drop(assume_yes=True)
def provision_db():
c = get_connection()
database_name = dmr.config.db.DATABASE
databases = rethinkdb.db_list().run(c)
if database_name not in databases:
_LOGGER.info("Creating DB: [{0}]".format(database_name))
rethinkdb.db_create(database_name).run(c)
def test_configure(self): # r.table_drop(self.table_name).run(self.connection) r.db_drop(self.db_name).run(self.connection) self.db.configure() db_exists = r.db_list().contains(self.db_name).run(self.connection) self.assertTrue( db_exists ) table_exists = r.db(self.db_name).table_list().contains(self.table_name).run(self.connection) self.assertTrue( table_exists )
def __init__(self, host, port):
self.conn = r.connect(host, port).repl()
if 'resmio' not in r.db_list().run(self.conn):
r.db_create('resmio').run(self.conn)
self.conn.use('resmio')
r.table_create('binary').run(self.conn)
r.table_create('redirects').run(self.conn)
r.table_create('errors').run(self.conn)
r.table('binary').index_create('key').run(self.conn)
self.conn.use('resmio')
def db_create(self):
c = self.__db_connect_retry(config.DB_RETRIES)
res = None
if c :
if self.db_name not in r.db_list().run(c):
res = r.db_create(self.db_name).run(c)
else:
res = False
c.close()
return res
def save(request):
"""Saves Slack bot messages to RethinkDB.
Args:
request (object): the Flask request object, including the the form-
encoded message fields which Slack POSTs
Returns:
bool: result object if successful, False otherwise.
"""
if config.log: print('listening...')
# Grab every key/value from the POST and stuff it into a dict
message = {}
for key, value in request.form.items():
message[key] = value
# Set defaults
if 'channel_name' in message:
channel_name = message['channel_name']
channel_name = ''.join(e for e in channel_name if e.isalnum())
else:
'unknown'
if 'team_domain' in message:
server_name = message['team_domain']
server_name = ''.join(e for e in server_name if e.isalnum())
else:
'unknown'
# Setup logging variables
db_name = server_name
table_name = channel_name
# Connect to RethinkDB
r.connect('localhost', 28015).repl()
# Create RethinkDB database if it doesn't exist
if db_name not in r.db_list().run():
if config.log: print('database {} does not exist'.format(db_name))
r.db_create(db_name).run()
# Create RethinkDB table if it doesn't exist
if table_name not in r.db(db_name).table_list().run():
if config.log: print('table {} does not exist'.format(table_name))
r.db(db_name).table_create(table_name).run()
r.db(db_name).table(table_name).index_create('timestamp').run()
r.db(db_name).table(table_name).index_create('channel_name').run()
# Insert message into table <name>
if config.log: print('Inserting...')
response = r.db(db_name).table(table_name).insert(message).run()
return True
def init():
""" Initialise database """
conn = r.connect()
if not DB in r.db_list().run(conn):
create(conn)
return r.connect(
host = HOST,
port = PORT,
db = DB
)
def database(self):
"""Get database from RethinkDB connection."""
self._get_connection()
db = r.db(self.database_name)
# Ensure database exists
if not self.database_name in r.db_list().run(self.conn):
r.db_create(self.database_name).run(self.conn)
return db
def real_stock_data_load(data, connection):
for db in list(r.db_list().run(connection)):
if db == u"rethinkdb":
# This db is special and can't be deleted.
continue
r.db_drop(db).run(connection)
for db_name, db_data in iteritems(data['dbs']):
r.db_create(db_name).run(connection)
for table_name, table_data in iteritems(db_data['tables']):
r.db(db_name).table_create(table_name).run(connection)
r.db(db_name).table(table_name).insert(table_data).run(connection)
def run_massive_cf():
dbs = r.db_list().run(conn)
tables = r.db(dbs[0]).table_list().run(conn)
initial = r.db(dbs[0]).table(tables[0]).changes().merge({'table':tables[0], 'database':dbs[0]})
for database in dbs:
tables = r.db(database).table_list().run(conn)
for table in tables:
initial = initial.union(r.db(database).table(table).changes().merge({'table':table, 'database': database}))
for change in initial.run(conn):
print change
if change['database'] == 'rethinkdb' and change['table'] == 'table_config':
break
