def setUp(self):
self.db_name = 'radiowcs_test'
assert self.db_name != 'radiowcs'
self.table_name = 'test'
self.db = database.Database()
self.db.database_name = self.db_name
self.db.table_name = self.table_name
self.db.connect()
self.connection = r.connect(
host='localhost',
port=28015,
db=self.db_name,
auth_key='',
timeout=30
)
try:
r.db_create(self.db_name).run(self.connection)
r.table_create(self.table_name).run(self.connection)
except r.RqlRuntimeError:
print 'unittest setup: Drop table'
r.table_drop(self.table_name).run(self.connection)
r.table_create(self.table_name).run(self.connection)
r.db(self.db_name).table(self.table_name).index_create( 'title').run(self.connection)
r.db(self.db_name).table(self.table_name).index_create('artist').run(self.connection)
r.db(self.db_name).table(self.table_name).index_create( 'date').run(self.connection)
# 'out of order' insertions
r.db(self.db_name).table(self.table_name).insert({'title':'foobar', 'artist': 'Selena', 'date': '1430183323'}).run(self.connection)
r.db(self.db_name).table(self.table_name).insert({'title':'hello world', 'artist': 'John', 'date': '1430082566'}).run(self.connection)
r.db(self.db_name).table(self.table_name).insert({'title':'zombie apoc', 'artist': 'xxJANExx', 'date': '1430385845'}).run(self.connection)
r.db(self.db_name).table(self.table_name).insert({'title':'Black', 'artist': 'Kettle', 'date': '1430284300'}).run(self.connection)
def post_entityConfig(self, entityConfig):
LOG.debug("Posting entity config")
schemaTable = self.config['rethink_schema_table']
if schemaTable not in rethinkdb.table_list().run(
self.controller.rethink):
LOG.debug("Creating table for schema: {0}".format(
entityConfig.type))
rethinkdb.table_create(schemaTable, primary_key='type').run(
self.controller.rethink)
entitySchema = self.controller.entityConfigManager.schema[
entityConfig.type]
cacheSchema = dict([(field, s) for field, s in entitySchema.items()
if field in entityConfig['fields']])
if LOG.getEffectiveLevel() < 10:
LOG.debug("Cache Schema:\n{0}".format(
utils.prettyJson(cacheSchema)))
config = {}
config['type'] = entityConfig.type
config['schema'] = cacheSchema
config['created_at'] = datetime.datetime.utcnow().isoformat()
result = rethinkdb.table(schemaTable).insert(
config, conflict="replace").run(self.controller.rethink)
if result['errors']:
raise IOError(result['first_error'])
def create_tables(tables): """ """ existing_tables = r.table_list().run(conn) for table in tables: if table not in existing_tables: r.table_create(table).run(conn)
def get_default_connection(request, url=None, **rethink_options):
conn = getattr(request.registry, '_r_conn', None)
if conn is not None:
return conn
if url is not None:
rethink_options.pop('password', None)
rethink_options.pop('user', None)
rethink_options.pop('host', None)
rethink_options.pop('port', None)
rethink_options.pop('db', None)
rethink_options.update(parse_url(url))
LOG.debug(rethink_options)
conn = r.connect(**rethink_options)
if rethink_options.get('db', R_DB) not in r.db_list().run(conn):
r.db_create(R_DB).run(conn)
if R_TABLE not in r.table_list().run(conn):
r.table_create(R_TABLE).run(conn)
setattr(request.registry, '_r_conn', conn)
return conn
def rethinkdb():
"""Prepare database and table in RethinkDB"""
from rethinkdb.errors import ReqlOpFailedError, ReqlRuntimeError
conn = r.connect(host=conf.RethinkDBConf.HOST)
# Create database
try:
r.db_create(conf.RethinkDBConf.DB).run(conn)
click.secho('Created database {}'.format(conf.RethinkDBConf.DB),
fg='yellow')
except ReqlOpFailedError:
click.secho('Database {} already exists'.format(conf.RethinkDBConf.DB),
fg='green')
# Create table 'domains'
conn = r.connect(host=conf.RethinkDBConf.HOST,
db=conf.RethinkDBConf.DB)
try:
r.table_create('domains', durability=conf.RethinkDBConf.DURABILITY).\
run(conn)
click.secho('Created table domains', fg='yellow')
except ReqlOpFailedError:
click.secho('Table domains already exists', fg='green')
# Create index on domains.name
try:
r.table('domains').index_create('name').run(conn)
click.secho('Created index domains.name', fg='yellow')
except ReqlRuntimeError:
click.secho('Index domains.name already exists', fg='green')
def init_rethinkdb(self):
print('Now initialising RethinkDB...')
dbc = self.config['RETHINKDB']
try:
self.conn = r.connect(host=dbc['HOST'],
port=dbc['PORT'],
db=dbc['DB'],
user=dbc['USERNAME'],
password=dbc['PASSWORD'])
dbs = r.db_list().run(self.conn)
if self.rdb not in dbs:
print('Database not present. Creating...')
r.db_create(self.rdb).run(self.conn)
tables = r.db(self.rdb).table_list().run(self.conn)
for i in self.rtables:
if i not in tables:
print(f'Table {i} not found. Creating...')
r.table_create(i).run(self.conn)
except Exception as e:
print('RethinkDB init error!\n{}: {}'.format(type(e).__name__, e))
sys.exit(1)
print('RethinkDB initialisation successful.')
def __init__(self, ip: str, port: int, table: str, clean_delay: int = 300):
""" Initializes the RethinkDB pipe.
:param ip: The IP of the RethinkDB instance
:type ip: str
:param port: The port of the RethinkDB instance
:type port: int
:param table: The table name of the table to place records into
:type table: str
:param clean_delay: Seconds between cleaning dead records
:type clean_delay: int
"""
(self._ip, self._port) = (ip, port)
self._table_name = table
(self._last_cleaned, self._clean_delay) = (0, clean_delay)
if self._table_name not in rethinkdb.table_list()\
.run(self.connection):
rethinkdb.table_create(self._table_name).run(self.connection)
self.table = rethinkdb.table(self._table_name)
self._cleaning_thread = threading.Thread(
target=self._cleaning_scheduler, args=(self._clean_delay, ))
self._cleaning_thread.daemon = True
const.log.info(
('starting `{self}` cleaning thread as daemon '
'on `{self._clean_delay}` second delay ...').format(self=self))
self._cleaning_thread.start()
def __init__(self, database='apscheduler', table='jobs', client=None,
pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
super(RethinkDBJobStore, self).__init__()
self.pickle_protocol = pickle_protocol
if not database:
raise ValueError('The "database" parameter must not be empty')
if not table:
raise ValueError('The "table" parameter must not be empty')
if client:
self.conn = maybe_ref(client)
else:
self.conn = r.connect(db=database, **connect_args)
if database not in r.db_list().run(self.conn):
r.db_create(database).run(self.conn)
if table not in r.table_list().run(self.conn):
r.table_create(table).run(self.conn)
if 'next_run_time' not in r.table(table).index_list().run(self.conn):
r.table(table).index_create('next_run_time').run(self.conn)
self.table = r.db(database).table(table)
def rethink(self):
print('Attempting to connect to RethinkDB')
tables = ['settings', 'numbers']
dbc = self.config.get('db')
try:
self.conn = r.connect(host=dbc['host'],
port=dbc['port'],
db=dbc['db'],
user=dbc['username'],
password=dbc['password'])
dbl = r.db_list().run(self.conn)
if dbc['db'] not in dbl:
print('Creating DB...')
r.db_create(dbc['db']).run(self.conn)
tab = r.table_list().run(self.conn)
for i in tables:
if i not in tab:
print(f'Table {i} not found. Now creating...')
r.table_create(i).run(self.conn)
except Exception as e:
print(
f'DB connection failed! Exiting...\nError details:\n{type(e).__name__}: {e}'
)
sys.exit(1)
print('Connected successfully.')
def table_create(connection):
for table in TABLE_NAMES:
try:
r.table_create(table).run(connection)
print ('Table ' + table + ' created')
except RqlRuntimeError:
print ('Table ' + table + ' already exists')
def _create(self):
try:
rdb.table_create(self.Meta.table_name).run()
except RqlRuntimeError:
return False
else:
return True
def __init__(self,
database='apscheduler',
table='jobs',
client=None,
pickle_protocol=pickle.HIGHEST_PROTOCOL,
**connect_args):
super(RethinkDBJobStore, self).__init__()
self.pickle_protocol = pickle_protocol
if not database:
raise ValueError('The "database" parameter must not be empty')
if not table:
raise ValueError('The "table" parameter must not be empty')
if client:
self.conn = maybe_ref(client)
else:
self.conn = r.connect(db=database, **connect_args)
if database not in r.db_list().run(self.conn):
r.db_create(database).run(self.conn)
if table not in r.table_list().run(self.conn):
r.table_create(table).run(self.conn)
if 'next_run_time' not in r.table(table).index_list().run(self.conn):
r.table(table).index_create('next_run_time').run(self.conn)
self.table = r.db(database).table(table)
def init_database():
try:
connection = r.connect(host=RDB_HOST, port=RDB_PORT, db=RDB_DB)
except RqlDriverError:
debug("Could not connect to the database %s:%d, exiting..." % (RDB_HOST, RDB_PORT), True)
exit(1)
try:
r.db_create("tomatoesfridge").run( connection )
except r.errors.RqlRuntimeError as e:
# We could parse the error... later...
debug("Database `tomatoesfridge` not created. Reason:")
debug(str(e))
else:
debug("Database `tomatoesfridge` created")
try:
r.table_create("movie").run( connection )
except r.errors.RqlRuntimeError as e:
# We could parse the error... later...
debug("Table `movie` in `tomatoesfridge` not created. Reason")
debug(str(e))
else:
debug("Table `movie` in `tomatoesfridge` created")
try:
connection.close()
except AttributeError:
debug("Could not close a connection", True)
pass
def create_tables(tables):
"""
"""
existing_tables = r.table_list().run(conn)
for table in tables:
if table not in existing_tables:
r.table_create(table).run(conn)
def initialSetup():
print "Setting up database..."
dbs = rethinkdb.db_list().run()
if not con.general.databases["rethink"]["db"] in dbs:
print "Creating database in rethink"
rethinkdb.db_create(con.general.databases["rethink"]["db"]).run()
dbt = list(rethinkdb.table_list().run())
for db in c.general.flush["rethink"]:
if c.general.flush["rethink"][db]:
print "Flushing rethink "+db+" table..."
if db in dbt:
rethinkdb.table_drop(db).run()
dbt.pop(dbt.index(db))
print "Creating new rethink tables..."
for table in c.general.tables:
if not table in dbt:
print "Creating table {}".format(table)
rethinkdb.table_create(table).run()
for key in c.general.flush["redis"]:
if c.general.flush["redis"][key]:
print "Flushing redis "+key+" keys..."
keys = con.redis.keys(key+":*")
for key in keys: con.redis.delete(key)
def ensure_table_exists(table_name, *args, **kwargs):
"""Creates a table if it doesn't exist."""
try:
r.table_create(table_name, *args, **kwargs).run(r_conn())
print 'Successfully created table ' + table_name
except r.RqlRuntimeError:
print 'Table ' + table_name + ' has already been created.'
pass # Ignore table already created
def create_db():
tables = ['datasets', 'visualizations']
for table in tables:
try:
r.table_create(table).run(db.conn)
except RqlRuntimeError:
print 'Table `%s` already exists' % table
print 'Tables have been created.'
def _create_table(table_name):
logger.info("Creating {} table...".format(table_name))
with get_connection() as conn:
try:
r.table_create(table_name).run(conn)
logger.info("Created table {} successfully.".format(table_name))
except:
logger.info("Failed to create table {}!".format(table_name))
def new_table(self, name):
try:
r.table_create(name, primary_key='name').run(self.connection)
self.table = name
except RqlRuntimeError:
print 'Error: Table with name \"' + name + '\" already exists'
return False
return True
def generate_canary():
""" Generates new canary string """
canary = ''.join(random.SystemRandom().choice('abcdef' + string.digits) for _ in range(64))
if 'canary' not in r.table_list().run(flask.g.rdb_conn):
r.table_create('canary').run(flask.g.rdb_conn)
r.table('canary').insert({'date': r.now(), 'canary': canary}).run(flask.g.rdb_conn)
return flask.jsonify(canary=canary)
def make_table(self):
try:
if self.async:
yield r.table_create(self.table).run(self.conn)
else:
r.table_create(self.table).run(self.conn)
log.info("Table %s created successfully." % self.table)
except r.RqlRuntimeError:
log.info("Table %s already exists... skipping." % self.table)
def create_table(conn):
try:
r.table_create("wallpapers").run(conn)
created = True
except r.errors.RqlRuntimeError:
#Already exists
created = False
pass
return created
def create_table(conn):
try:
r.table_create("wallpapers").run(conn)
created = True
except r.errors.RqlRuntimeError:
#Already exists
created = False
pass
return created
def create_table_if_not_exists(tname):
if tname in rdb.table_list().run(dbconn):
print('Table '
'{0}'
' already exists in RethinkDB instance. Skipping creation ...'.
format(tname))
else:
rdb.table_create(tname).run(dbconn)
print('Created {0} table in RethinkDB instance'.format(tname))
def post_stat(self, statDict):
if not self.config['enable_stats']:
return
LOG.debug("Posting stat: {0}".format(statDict['type']))
statTable = self.config['rethink_stat_table_template'].format(type=statDict['type'])
if statTable not in rethinkdb.table_list().run(self.rethink):
rethinkdb.table_create(statTable).run(self.rethink)
rethinkdb.table(statTable).insert(statDict).run(self.rethink)
def configure(self):
"""Creates required database and tables/collections"""
self.connect()
try:
r.db_create(self.connection.db).run(self.connection)
r.table_create(self.table_name).run(self.connection)
r.table(self.table_name).index_create('date')
print 'Database setup completed.'
except r.RqlRuntimeError:
print 'App database already exists..'
def init_table(name, **kwargs):
"""
Initialize a table in the database.
"""
with connect() as con:
try:
r.table_drop(name).run(con)
except r.ReqlOpFailedError:
pass
r.table_create(name, **kwargs).run(con)
def configure(self):
"""Creates required database and tables/collections"""
self.connect()
try:
r.db_create(self.connection.db).run(self.connection)
r.table_create(self.table_name).run(self.connection)
r.table(self.table_name).index_create('date')
print 'Database setup completed.'
except r.RqlRuntimeError:
print 'App database already exists..'
def create_rethink_tables():
current_tables = rethinkdb.table_list().coerce_to("array").run()
logger.debug("Current tables in rethink: {}".format(current_tables))
tables_to_create = list(set(rethink_bags.tables.keys()).difference(set(current_tables)))
logger.info("Creating these new tables in rethink: {}".format(tables_to_create))
for table in tables_to_create:
rethinkdb.table_create(table).run()
logger.debug("Table {} created in rethink".format(table))
def init(self):
current_version = self.current_version()
if current_version is not None:
raise AlreadyInitializedException()
rethinkdb.table_create(MARKER_TABLE_NAME).run(self.conn)
rethinkdb.table(MARKER_TABLE_NAME).insert({
'version': 0,
'date_updated': utc_now(),
}).run(self.conn)
def init(self):
current_version = self.current_version()
if current_version is not None:
raise AlreadyInitializedException()
rethinkdb.table_create(MARKER_TABLE_NAME).run(self.conn)
rethinkdb.table(MARKER_TABLE_NAME).insert({
'version': 0,
'date_updated': utc_now(),
}).run(self.conn)
def post_stat(self, statDict):
if not self.config['enable_stats']:
return
LOG.debug("Posting stat: {0}".format(statDict['type']))
statTable = self.config['rethink_stat_table_template'].format(
type=statDict['type'])
if statTable not in rethinkdb.table_list().run(self.rethink):
rethinkdb.table_create(statTable).run(self.rethink)
rethinkdb.table(statTable).insert(statDict).run(self.rethink)
def create_or_delete_table(table, delete=False):
try:
r.table_create(table).run(connection)
print('Successfully created table <%s>.' % table)
except RqlRuntimeError:
if delete:
r.table(table).delete().run(connection)
print(
'Table <%s> already exists. Deleted all existing documents instead.'
% table)
else:
print('Table <%s> already exists.' % table)
def dbcreate(options):
"""Create pasahero database and tables"""
conn = r.connect(options.dbhost, options.dbport)
if options.dbname in r.db_list().run(conn):
return
r.db_create(options.dbname).run(conn)
conn.use(options.dbname)
r.table_create('users').run(conn)
r.table_create('migrations', primary_key='name').run(conn)
def _ensure_provisioned(conn):
def ignore_exc(fn, *args, **kwargs):
try:
return fn(*args, **kwargs)
except Exception:
LOG.debug('Ignored exception', exc_info=True)
ignore_exc(rethinkdb.db_create(conn.db).run, conn)
ignore_exc(rethinkdb.table_create('accounts').run, conn)
ignore_exc(rethinkdb.table_create('transactions').run, conn)
rethinkdb.table('accounts').delete().run(conn)
rethinkdb.table('transactions').delete().run(conn)
def dbSetup():
connection = r.connect("localhost", 28015)
try:
# r.db_create("test").run(connection)
r.db("test").table_create("rubrik").run(connection)
r.table_create("stargazers").run(connection)
print 'Database setup completed.'
except RqlRuntimeError:
print 'App database already exists.'
finally:
connection.close()
def dbSetup():
with pool.get_resource() as res:
_ensure_db(res.conn)
for table in current_app.config['RDB_TABLES']:
try:
r.table_create(table).run(res.conn)
print('Table: "{}" creation completed'.format(table))
except Exception as e:
error = '%s' % e
print(error.split("in:")[0])
pass
print("DB setup done")
def setup_database(db):
db_name = current_app.config['DATABASE_NAME']
if db_name not in r.db_list().run(db):
r.db_create(db_name).run(db)
tables = r.table_list().run(db)
for table_name, options in _tables.items():
if table_name in tables:
continue
r.table_create(table_name, **options).run(db)
def _initialize(self):
"""
Initialize DB and table
:return:
"""
if self._db not in r.db_list().run(self._connection):
r.db_create(self._db).run(self._connection)
self._connection.use(self._db)
if self._table not in r.table_list().run(self._connection):
r.table_create(self._table)
def test_connection_with_database(self):
app = Flask(__name__)
db = RethinkDB(app, db='test')
with app.test_request_context():
try:
# Make sure RethinkDB is turned on!
r.table_create('table').run(db.conn)
except (RqlDriverError, RqlRuntimeError) as e:
self.fail(e)
else:
# Do some cleanup
r.table_drop('table').run(db.conn)
def setup():
"""
Sets up the connection to RethinkDB which we'll use in the rest of the
tests, and puts it into .repl() mode so we don't have to pass the model
object a connection. After that, we create a new database called `model`
and within that a table called `stargate` and sets the database to use
`model`.
"""
conn = r.connect("localhost", 28015)
r.db_create("model").run(conn)
conn.use("model")
conn.repl()
r.table_create("stargate").run()
def start(self):
print "start"
# Default starting proceedure...
import rethinkdb as r
# output to log...
from datetime import datetime
current_date = datetime.now()
name_of_logfile = "db_log-" + str(current_date.year)+"-"+str(current_date.month)+"-"+str(current_date.day)+"-"+str(current_date.second)
log = open(name_of_logfile,'w')
if True:
print "FIRST NODE"
# then create the database..
import os,sys
import signal
from subprocess import Popen
# The os.setsid() is passed in the argument preexec_fn so
# it's run after the fork() and before exec() to run the shell.
self.db_process = Popen(['rethinkdb', '--http-port', '6999', '--bind', 'all'], stderr=log, preexec_fn=os.setsid)
time.sleep(2)
# then connect to the client driver
self.conn = r.connect('localhost', self.client_driver_port)
# setup auth_key
#r.db('rethinkdb').table('cluster_config').get('auth').update({'auth_key': self.db_auth_key}).run(self.conn)
# create table
try:
result = r.table('files').run(self.conn)
except:
r.table_create('files', primary_key='uuid').run(self.conn)
else:
print "NOT FIRST NODE"
# use the existing database...
import os
import signal
from subprocess import Popen
# The os.setsid() is passed in the argument preexec_fn so
# it's run after the fork() and before exec() to run the shell.
self.db_process = Popen(['rethinkdb','--port-offset', '1', '--directory', 'rethinkdb_data2','--join','127.0.0.1:29015', '--bind', 'all'], stderr=log, preexec_fn=os.setsid)
log.flush()
def setUp(self):
self.db_name = 'radiowcs_test'
assert self.db_name != 'radiowcs'
self.table_name = 'test'
self.db = database.Database()
self.db.database_name = self.db_name
self.db.table_name = self.table_name
self.db.connect()
self.connection = r.connect(host='localhost',
port=28015,
db=self.db_name,
auth_key='',
timeout=30)
try:
r.db_create(self.db_name).run(self.connection)
r.table_create(self.table_name).run(self.connection)
except r.RqlRuntimeError:
print 'unittest setup: Drop table'
r.table_drop(self.table_name).run(self.connection)
r.table_create(self.table_name).run(self.connection)
r.db(self.db_name).table(self.table_name).index_create('title').run(
self.connection)
r.db(self.db_name).table(self.table_name).index_create('artist').run(
self.connection)
r.db(self.db_name).table(self.table_name).index_create('date').run(
self.connection)
# 'out of order' insertions
r.db(self.db_name).table(self.table_name).insert({
'title': 'foobar',
'artist': 'Selena',
'date': '1430183323'
}).run(self.connection)
r.db(self.db_name).table(self.table_name).insert({
'title': 'hello world',
'artist': 'John',
'date': '1430082566'
}).run(self.connection)
r.db(self.db_name).table(self.table_name).insert({
'title': 'zombie apoc',
'artist': 'xxJANExx',
'date': '1430385845'
}).run(self.connection)
r.db(self.db_name).table(self.table_name).insert({
'title': 'Black',
'artist': 'Kettle',
'date': '1430284300'
}).run(self.connection)
def create_table(self, table_name, index_names=None):
if index_names is None:
index_names = []
conn = self.client.conn
try:
r.table_create(table_name).run(conn)
except r.errors.ReqlOpFailedError as e:
pass
for idx_name in index_names:
try:
r.table(table_name).index_create(idx_name).run(conn)
except r.errors.ReqlOpFailedError as e:
pass
def dbSetup():
logging.info("Attempting db connection to %s:%s:%s...",DB_HOST,DB_PORT,DB_NAME)
logging.info("Checking if db %s exists",DB_NAME)
if DB_NAME not in list(r.db_list().run(conn)):
logging.info("db does not exist, creating...")
r.db_create(DB_NAME).run(conn)
logging.info("db exists")
logging.info("Checking to see if table %s exists",'observations')
if 'observations' not in list(r.table_list().run(conn)):
logging.info("table does not exist, creating...")
r.table_create("observations").run(conn)
conn.close()
logging.info("table exists")
def dbSetup():
#logging.info("Attempting db connection to %s:%s:%s...",DB_HOST,DB_PORT,DB_NAME)
logging.info("======================================")
logging.info("Checking if db %s exists",DB_NAME)
if DB_NAME not in list(r.db_list().run(conn)):
logging.info("db does not exist, creating...")
r.db_create(DB_NAME).run(conn)
logging.info("db exists")
logging.info("Checking to see if table %s exists",'observations')
if 'observations' not in list(r.table_list().run(conn)):
logging.info("table does not exist, creating...")
r.table_create("observations").run(conn)
conn.close()
logging.info("table exists")
def flush_rethink_tables():
tables_to_flush = [ table for table, flush in rethink_bags.flush.iteritems() if flush ]
logger.debug("Tables that should be flushed: {}".format(tables_to_flush))
current_tables = rethinkdb.table_list().coerce_to("array").run()
logger.debug("Current tables in rethink: {}".format(current_tables))
flushing_tables = list(set(current_tables).intersection(tables_to_flush))
logger.info("Flushing tables in rethink: {}".format(flushing_tables))
for table in flushing_tables:
rethinkdb.table_drop(table).run()
rethinkdb.table_create(table).run()
logger.debug("Table {} flushed in rethink".format(table))
def reseed():
db_list = r.db_list().run(conn)
if db_name not in db_list:
r.db_create(db_name).run(conn)
table_list = r.table_list().run(conn)
if products_table in table_list:
r.table_drop(products_table).run(conn)
r.table_create(products_table).run(conn)
r.table(products_table).insert(test_products).run(conn)
def setup():
"""
Sets up the connection to RethinkDB which we'll use in the rest of the
tests, and puts it into .repl() mode so we don't have to pass the model
object a connection. After that, we create a new database called `model`
and within that a table called `stargate` and sets the database to use
`model`.
"""
conn = r.connect('localhost', 28015)
r.db_create("model").run(conn)
conn.use("model")
conn.repl()
r.table_create("stargates").run()
r.table_create("episodes").run()
def setup_logging():
try:
r.db_create(app.config['RETHINKDB_DB']).run(db.conn)
except ReqlOpFailedError:
pass
try:
r.table_create(RDB_TABLE).run(db.conn)
except ReqlOpFailedError:
pass
if not app.debug:
app.logger.addHandler(logging.StreamHandler())
app.logger.setLevel(logging.INFO)
def main():
options = {
'server': config['JIRA']
}
jira = JIRA(options, basic_auth=(config['USERNAME'], config['PASSWORD']))
months = [
('2015-03', '2015-04'),
('2015-04', '2015-05'),
('2015-05', '2015-06'),
('2015-06', '2015-07'),
('2015-07', '2015-08'),
('2015-08', '2015-09'),
('2015-09', '2015-10'),
('2015-10', '2015-11'),
('2015-11', '2015-12'),
('2015-12', '2016-01'),
('2016-01', '2016-02'),
('2016-02', '2016-03'),
('2016-03', '2016-04')
]
total_issues = 0
bulk_add = []
for month in months:
print("Downloading issues for interval %s/%s" % month)
jql = "created >= '%s-01' AND created < '%s-01'" % month
issues_in_month = jira.search_issues(jql, maxResults=1000, json_result=True)
issues = issues_in_month['issues']
filtered_issues = filter_issues(issues)
issues_count = len(issues)
filtered_count = len(filtered_issues)
assert filtered_count == issues_count
total_issues = total_issues + issues_count
bulk_add.extend(filtered_issues)
print("Successfully downloaded %d issues" % total_issues)
print("Loading %d issues into RethinkDB" % len(bulk_add))
r.connect(config['RETHINKDB'], 28015, db='jira').repl()
r.table_drop('issues').run()
r.table_create('issues').run()
r.table('issues').insert(bulk_add).run()
print("OK! Bye")
def main():
conn = r.connect(
host='localhost',
port=28015,
db='rwrapper'
).repl()
try:
r.db_drop('rwrapper').run()
except:
pass
r.db_create('rwrapper').run()
r.table_create('test_1').run()
# C
create = MyTable(
field1='something',
field2='something else',
field3={
'test': 'created'
},
field4=[{
'record': 1
}, {
'record': 2
}]
)
create.save()
print "Inserted new row: %s" % create.id
# R
read = MyTable(id=create.id).get()
print "Reading data for %s: %s" % ( create.id, read)
# U
update = MyTable(id=create.id).get(MyTable)
update.field1 = 'something new'
update.save()
print "Updating data for: %s" % create.id
read = MyTable(id=create.id).get()
print "New data for %s: %s" % ( create.id, read)
# D
delete = MyTable(id=create.id).delete()
print "Deleted %s" % create.id
def test_connection_with_inexisting_database(self):
app = Flask(__name__)
db = RethinkDB(app, db='doesnotexist')
with app.test_request_context():
try:
# Make sure RethinkDB is turned on!
# Specifying an inexisting database should raise an exception
r.table_create('table').run(db.conn)
except (RqlDriverError, RqlRuntimeError):
pass
else:
# Do some cleanup
r.table_drop('table').run(db.conn)
self.fail("Should have raised a RqlDriverError")
def setup_db(rdb_host, rdb_port, rdb_name,
datasets=data_paths, # datasets to be loaded
app_tables=application_tables): # tables to be created
"""
Sets up the database from scratch
Will allow the user to execute whether or not the database has already
been instantiated.
@return: a set of tables that were added to the specified database
"""
creations = set() # set of tables created
try:
with rethinkdb.connect(host=rdb_host, port=rdb_port) as conn:
db_list = rethinkdb.db_list().run(conn)
if rdb_name not in db_list:
rethinkdb.db_create(rdb_name).run(conn) # create the db
print 'Database added'
conn.use(rdb_name) # make default db connection
# find tables that still need to be added then add & populate them
tbl_list = [x for x in
rethinkdb.db(rdb_name).table_list().run(conn)]
tbl_to_add = [(k, v) for k, v
in datasets.items() if k not in tbl_list]
print "TBL_TO_ADD"
print tbl_to_add
for (name, path) in tbl_to_add:
rethinkdb.table_create(name).run(conn)
try:
with open(path, 'r') as f:
(rethinkdb
.table(name)
.insert(json.loads(f.read()))
.run(conn))
except IOError:
raise Exception('Invalid Dataset')
print "'{}' table created and populated.".format(name)
creations.add(name)
tbl_to_add = (n for n in app_tables if n not in tbl_list)
for name in tbl_to_add:
rethinkdb.table_create(name).run(conn)
print "'{}' table created and populated.".format(name)
creations.add(name)
except RqlRuntimeError, e:
print e
exit(1)
def __init__(self, server, port, database, table):
self.server = server
self.port = port
self.database = database
self.table = table
self.data = []
self.conn = r.connect(self.server, self.port)
if self.database in r.db_list().run(self.conn):
print('Database ' + self.database + ' exists! Using it.')
else:
resp = r.db_create(self.database).run(self.conn)
if resp["dbs_created"] == 1:
print('Database ' + self.database +
' was successfully created.')
else:
print('Error while creating database ' + self.database + '.')
self.conn.use(self.database)
if self.table in r.table_list().run(self.conn):
print('Table ' + self.table + ' exists! Using it.')
else:
resp = r.table_create(self.table).run(self.conn)
if resp["tables_created"] == 1:
print('Table ' + self.table + ' was successfully created.')
else:
print('Error while creating table ' + self.table + '.')
def init(self):
self.logger.info("Initializing connection to DB")
connection = r.connect(
host=CONFIG.get('rethink_host'),
port=int(CONFIG.get('rethink_port')),
db=self.dbname
)
conn = yield connection
try:
yield r.db_create(self.dbname).run(conn)
except r.ReqlRuntimeError:
self.logger.debug("Database already exists")
finally:
conn.use(self.dbname)
for table in self.tables:
try:
yield r.table_create(table).run(conn)
except r.ReqlOpFailedError:
self.logger.debug("Table already exists: " + table)
if table in self.secondary_indicies:
idxs = self.secondary_indicies[table]
for idx in idxs:
try:
yield r.table(table).index_create(idx).run(conn)
except r.ReqlRuntimeError:
self.logger.debug("Table index exists: "
"{}: {}".format(table, idx))
yield r.db(self.dbname).wait().run(conn)
self.logger.info("Done initializing DB")
self._connection = connection
return connection
def table_create(cls, if_not_exists=True):
if (if_not_exists
and (cls.Meta.table_name in r.table_list().run(get_conn()))):
return
return r.table_create(cls.Meta.table_name,
primary_key=cls.Meta.primary_key_field).run(
get_conn())
def initDB(self):
conn = r.connect(DB_HOST, DB_PORT, DB_NAME)
logging.info(MODULE_NAME + ": Successful DB connection")
logging.info(MODULE_NAME + ": Checking if db %s exists", DB_NAME)
if DB_NAME not in list(r.db_list().run(conn)):
logging.info(MODULE_NAME + ": db does not exist, creating...")
r.db_create(DB_NAME).run(conn)
logging.info(MODULE_NAME + ": db exists")
logging.info(MODULE_NAME + ": Checking to see if table %s exists",
TABLE_NAME)
if TABLE_NAME not in list(r.table_list().run(conn)):
logging.info(MODULE_NAME + ": table does not exist, creating...")
r.table_create(TABLE_NAME).run(conn)
logging.info(MODULE_NAME + ": table exists")
conn.close()
def _reset_data(self, table):
if table in rdb.table_list().run(self.session):
result = rdb.table_drop(table).run(self.session)
assert result['dropped'] == 1
result = rdb.table_create(table).run(self.session)
result = rdb.table(table).index_create('date').run(self.session)
return result.get('created', 0) == 1
def start(self, scheduler, alias):
super(RethinkDBJobStore, self).start(scheduler, alias)
if self.client:
self.conn = maybe_ref(self.client)
else:
self.conn = r.connect(db=self.database, **self.connect_args)
if self.database not in r.db_list().run(self.conn):
r.db_create(self.database).run(self.conn)
if self.table not in r.table_list().run(self.conn):
r.table_create(self.table).run(self.conn)
if 'next_run_time' not in r.table(self.table).index_list().run(self.conn):
r.table(self.table).index_create('next_run_time').run(self.conn)
self.table = r.db(self.database).table(self.table)
