def __init__(self, ip: str, port: int, table: str, clean_delay: int = 300):
""" Initializes the RethinkDB pipe.
:param ip: The IP of the RethinkDB instance
:type ip: str
:param port: The port of the RethinkDB instance
:type port: int
:param table: The table name of the table to place records into
:type table: str
:param clean_delay: Seconds between cleaning dead records
:type clean_delay: int
"""
(self._ip, self._port) = (ip, port)
self._table_name = table
(self._last_cleaned, self._clean_delay) = (0, clean_delay)
if self._table_name not in rethinkdb.table_list()\
.run(self.connection):
rethinkdb.table_create(self._table_name).run(self.connection)
self.table = rethinkdb.table(self._table_name)
self._cleaning_thread = threading.Thread(
target=self._cleaning_scheduler, args=(self._clean_delay, ))
self._cleaning_thread.daemon = True
const.log.info(
('starting `{self}` cleaning thread as daemon '
'on `{self._clean_delay}` second delay ...').format(self=self))
self._cleaning_thread.start()
def post_entityConfig(self, entityConfig):
LOG.debug("Posting entity config")
schemaTable = self.config['rethink_schema_table']
if schemaTable not in rethinkdb.table_list().run(
self.controller.rethink):
LOG.debug("Creating table for schema: {0}".format(
entityConfig.type))
rethinkdb.table_create(schemaTable, primary_key='type').run(
self.controller.rethink)
entitySchema = self.controller.entityConfigManager.schema[
entityConfig.type]
cacheSchema = dict([(field, s) for field, s in entitySchema.items()
if field in entityConfig['fields']])
if LOG.getEffectiveLevel() < 10:
LOG.debug("Cache Schema:\n{0}".format(
utils.prettyJson(cacheSchema)))
config = {}
config['type'] = entityConfig.type
config['schema'] = cacheSchema
config['created_at'] = datetime.datetime.utcnow().isoformat()
result = rethinkdb.table(schemaTable).insert(
config, conflict="replace").run(self.controller.rethink)
if result['errors']:
raise IOError(result['first_error'])
def __init__(self):
# ~ self.rele = 9
self.conf=load_config()
self.conn=False
self.cfg=False
try:
self.conn = r.connect( self.conf['RETHINKDB_HOST'],self.conf['RETHINKDB_PORT'],self.conf['RETHINKDB_DB']).repl()
except Exception as e:
log.error(e)
self.conn=False
if self.conn is not False and r.db_list().contains(self.conf['RETHINKDB_DB']).run():
if r.table_list().contains('config').run():
ready=False
while not ready:
try:
self.cfg = r.table('config').get(1).run()
ready = True
except Exception as e:
log.info('Waiting for database to be ready...')
time.sleep(1)
log.info('Your actual database version is: '+str(self.cfg['version']))
if release_version > self.cfg['version']:
log.warning('Database upgrade needed! You have version '+str(self.cfg['version'])+ ' and source code is for version '+str(release_version)+'!!')
else:
log.info('No database upgrade needed.')
self.upgrade_if_needed()
def table_config_created_and_populated():
try:
r_conn_test = r.connect(RETHINK_HOST, RETHINK_PORT)
if not r.db_list().contains(RETHINK_DB).run(r_conn_test):
print(
f'rethink host {RETHINK_HOST} and port {RETHINK_PORT} has connected but rethink database {RETHINK_DB} is not created'
)
return False
else:
r_conn = new_rethink_connection()
out = False
if r.table_list().contains('config').run(r_conn):
rtable = r.table('config')
out = rtable.get(1).run(r_conn)
close_rethink_connection(r_conn)
if out is not False:
if out is not None:
#print('table config populated in database')
return True
else:
print('table config not populated in database')
return False
else:
return False
except Exception as e:
print(
f'rethink db connectin failed with hostname {RETHINK_HOST} and port {RETHINK_PORT}'
)
print(e)
print('Traceback: \n .{}'.format(traceback.format_exc()))
return False
def create_tables(tables): """ """ existing_tables = r.table_list().run(conn) for table in tables: if table not in existing_tables: r.table_create(table).run(conn)
def deleteUntrackedFromCache(self, configs):
"""
Delete data from cache for entities that are no longer cached
"""
if not self.config['delete_cache_for_untracked_entities']:
return
# Get the list of cached entity types
tableTemplate = self.config['rethink_entity_table_template']
existingTables = rethinkdb.table_list().run(self.rethink)
existingCacheTables = []
tablePattern = tableTemplate.format(type="*")
for table in existingTables:
if fnmatch.fnmatch(table, tablePattern):
existingCacheTables.append(table)
usedCacheTables = [c['table'] for c in configs]
unusedCacheTables = [
t for t in existingCacheTables if t not in usedCacheTables
]
LOG.debug("Unusesd cache tables: {0}".format(unusedCacheTables))
LOG.info("Deleting {0} cache tables".format(len(unusedCacheTables)))
for table in unusedCacheTables:
LOG.info("Deleting table: {0}".format(table))
rethinkdb.table_drop(table).run(self.rethink)
def table_config_created_and_populated():
try:
r_conn_test = r.connect(RETHINK_HOST, RETHINK_PORT)
if not r.db_list().contains(RETHINK_DB).run(r_conn_test):
print(f'rethink host {RETHINK_HOST} and port {RETHINK_PORT} has connected but rethink database {RETHINK_DB} is not created')
return False
else:
r_conn = new_rethink_connection()
out = False
if r.table_list().contains('config').run(r_conn):
rtable = r.table('config')
out = rtable.get(1).run(r_conn)
close_rethink_connection(r_conn)
if out is not False:
if out is not None:
#print('table config populated in database')
return True
else:
print('table config not populated in database')
return False
else:
return False
except Exception as e:
print(f'rethink db connectin failed with hostname {RETHINK_HOST} and port {RETHINK_PORT}')
print(e)
print('Traceback: \n .{}'.format(traceback.format_exc()))
return False
def rethink(self):
print('Attempting to connect to RethinkDB')
tables = ['settings', 'numbers']
dbc = self.config.get('db')
try:
self.conn = r.connect(host=dbc['host'],
port=dbc['port'],
db=dbc['db'],
user=dbc['username'],
password=dbc['password'])
dbl = r.db_list().run(self.conn)
if dbc['db'] not in dbl:
print('Creating DB...')
r.db_create(dbc['db']).run(self.conn)
tab = r.table_list().run(self.conn)
for i in tables:
if i not in tab:
print(f'Table {i} not found. Now creating...')
r.table_create(i).run(self.conn)
except Exception as e:
print(
f'DB connection failed! Exiting...\nError details:\n{type(e).__name__}: {e}'
)
sys.exit(1)
print('Connected successfully.')
def table_config_created_and_populated():
try:
r_conn_test = r.connect(RETHINK_HOST, RETHINK_PORT)
if not r.db_list().contains(RETHINK_DB).run(r_conn_test):
return False
else:
r_conn = new_rethink_connection()
out = False
if r.table_list().contains('config').run(r_conn):
rtable = r.table('config')
out = rtable.get(1).run(r_conn)
close_rethink_connection(r_conn)
if out is not False:
if out is not None:
return True
else:
return False
else:
return False
except Exception as e:
log.info('rethink db connectin failed')
log.info(e)
return False
def __init__(self, database='apscheduler', table='jobs', client=None,
pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
super(RethinkDBJobStore, self).__init__()
self.pickle_protocol = pickle_protocol
if not database:
raise ValueError('The "database" parameter must not be empty')
if not table:
raise ValueError('The "table" parameter must not be empty')
if client:
self.conn = maybe_ref(client)
else:
self.conn = r.connect(db=database, **connect_args)
if database not in r.db_list().run(self.conn):
r.db_create(database).run(self.conn)
if table not in r.table_list().run(self.conn):
r.table_create(table).run(self.conn)
if 'next_run_time' not in r.table(table).index_list().run(self.conn):
r.table(table).index_create('next_run_time').run(self.conn)
self.table = r.db(database).table(table)
def test_setup_db(self):
""" Test creation of a db and tables """
# test that the 'TEST' database doesn't exist
with rethinkdb.connect(host='localhost', port=28015) as conn:
db_list = rethinkdb.db_list().run(conn)
self.assertTrue('TEST' not in db_list)
creations = self.run_setup_db()
# confirm the correct tables were created
self.assertSetEqual(creations,
set(template.test_dataset.keys()+template.test_tables))
with rethinkdb.connect(host='localhost', port=28015) as conn:
# test that the 'TEST' database was created
db_list = rethinkdb.db_list().run(conn)
self.assertTrue('TEST' in db_list)
conn.use('TEST')
# test that the 'test' table was created
table_list = rethinkdb.table_list().run(conn)
self.assertEqual(len(table_list),
len(template.test_dataset.keys()+template.test_tables))
self.assertTrue(template.test_dataset.keys()[0] in table_list)
# test that the data is correct by checking columns
data = [row for row in rethinkdb.table(
template.test_dataset.keys()[0]).run(conn)]
with open(template.test_json) as f:
self.assertSetEqual(
set(data[0].keys())-set([u'id']),
set(json.loads(f.read())[0].keys()))
self.run_clear_test_db()
def __init__(self,
database='apscheduler',
table='jobs',
client=None,
pickle_protocol=pickle.HIGHEST_PROTOCOL,
**connect_args):
super(RethinkDBJobStore, self).__init__()
self.pickle_protocol = pickle_protocol
if not database:
raise ValueError('The "database" parameter must not be empty')
if not table:
raise ValueError('The "table" parameter must not be empty')
if client:
self.conn = maybe_ref(client)
else:
self.conn = r.connect(db=database, **connect_args)
if database not in r.db_list().run(self.conn):
r.db_create(database).run(self.conn)
if table not in r.table_list().run(self.conn):
r.table_create(table).run(self.conn)
if 'next_run_time' not in r.table(table).index_list().run(self.conn):
r.table(table).index_create('next_run_time').run(self.conn)
self.table = r.db(database).table(table)
def create_tables(tables):
"""
"""
existing_tables = r.table_list().run(conn)
for table in tables:
if table not in existing_tables:
r.table_create(table).run(conn)
def deleteUntrackedFromCache(self, configs):
"""
Delete data from cache for entities that are no longer cached
"""
if not self.config['delete_cache_for_untracked_entities']:
return
# Get the list of cached entity types
tableTemplate = self.config['rethink_entity_table_template']
existingTables = rethinkdb.table_list().run(self.rethink)
existingCacheTables = []
tablePattern = tableTemplate.format(type="*")
for table in existingTables:
if fnmatch.fnmatch(table, tablePattern):
existingCacheTables.append(table)
usedCacheTables = [c['table'] for c in configs]
unusedCacheTables = [t for t in existingCacheTables if t not in usedCacheTables]
LOG.debug("Unusesd cache tables: {0}".format(unusedCacheTables))
LOG.info("Deleting {0} cache tables".format(len(unusedCacheTables)))
for table in unusedCacheTables:
LOG.info("Deleting table: {0}".format(table))
rethinkdb.table_drop(table).run(self.rethink)
def __init__(self, server, port, database, table):
self.server = server
self.port = port
self.database = database
self.table = table
self.data = []
self.conn = r.connect(self.server, self.port)
if self.database in r.db_list().run(self.conn):
print('Database ' + self.database + ' exists! Using it.')
else:
resp = r.db_create(self.database).run(self.conn)
if resp["dbs_created"] == 1:
print('Database ' + self.database +
' was successfully created.')
else:
print('Error while creating database ' + self.database + '.')
self.conn.use(self.database)
if self.table in r.table_list().run(self.conn):
print('Table ' + self.table + ' exists! Using it.')
else:
resp = r.table_create(self.table).run(self.conn)
if resp["tables_created"] == 1:
print('Table ' + self.table + ' was successfully created.')
else:
print('Error while creating table ' + self.table + '.')
def __init__(self):
# ~ self.rele = 9
self.conf = load_config()
self.conn = False
self.cfg = False
try:
self.conn = r.connect(self.conf['RETHINKDB_HOST'],
self.conf['RETHINKDB_PORT'],
self.conf['RETHINKDB_DB']).repl()
except Exception as e:
log.error(e)
self.conn = False
if self.conn is not False and r.db_list().contains(
self.conf['RETHINKDB_DB']).run():
if r.table_list().contains('config').run():
self.cfg = r.table('config').get(1).run()
log.info('Your actual database version is: ' +
str(self.cfg['version']))
if release_version > self.cfg['version']:
log.warning('Database upgrade needed! You have version ' +
str(self.cfg['version']) +
' and source code is for version ' +
str(release_version) + '!!')
else:
log.info('No database upgrade needed.')
self.upgrade_if_needed()
def drop_fork(conn, block_num):
"""Deletes all resources from a particular block_num
"""
block_results = (
r.table("blocks")
.filter(lambda rsc: rsc["block_num"].ge(block_num))
.delete()
.run(conn)
)
resource_results = (
r.table_list()
.for_each(
lambda table_name: r.branch(
r.eq(table_name, "blocks"),
[],
r.eq(table_name, "auth"),
[],
r.table(table_name)
.filter(lambda rsc: rsc["start_block_num"].ge(block_num))
.delete(),
)
)
.run(conn)
)
return {k: v + resource_results[k] for k, v in block_results.items()}
def DropBlock(client, blockinfo):
"""
Drop a block and all associated data, this can happen when a block
is removed from the committed chain by a fork.
Args:
client -- sawtooth.client.SawtoothClient for accessing the ledger
blockinfo -- block data
"""
logger.info('drop block %s', blockinfo['id'])
try:
rethinkdb.table('block_list').get(blockinfo['id']).delete().run()
except:
logger.warn('failed to remove block %s from block list table',
blockinfo['id'])
try:
blockstatetable = 'blk' + blockinfo['id']
if blockstatetable in rethinkdb.table_list().run():
rethinkdb.table(blockstatetable).tableDrop().run()
except:
logger.warn('failed to drop state table for block %s', blockinfo['id'])
for txnid in blockinfo['TransactionIDs']:
try:
rethinkdb.table('txn_list').get(txnid).delete().run()
except:
logger.warn('failed to drop transaction %s for block %s', txnid,
blockinfo['id'])
def get_default_connection(request, url=None, **rethink_options):
conn = getattr(request.registry, '_r_conn', None)
if conn is not None:
return conn
if url is not None:
rethink_options.pop('password', None)
rethink_options.pop('user', None)
rethink_options.pop('host', None)
rethink_options.pop('port', None)
rethink_options.pop('db', None)
rethink_options.update(parse_url(url))
LOG.debug(rethink_options)
conn = r.connect(**rethink_options)
if rethink_options.get('db', R_DB) not in r.db_list().run(conn):
r.db_create(R_DB).run(conn)
if R_TABLE not in r.table_list().run(conn):
r.table_create(R_TABLE).run(conn)
setattr(request.registry, '_r_conn', conn)
return conn
def initialSetup():
print "Setting up database..."
dbs = rethinkdb.db_list().run()
if not con.general.databases["rethink"]["db"] in dbs:
print "Creating database in rethink"
rethinkdb.db_create(con.general.databases["rethink"]["db"]).run()
dbt = list(rethinkdb.table_list().run())
for db in c.general.flush["rethink"]:
if c.general.flush["rethink"][db]:
print "Flushing rethink "+db+" table..."
if db in dbt:
rethinkdb.table_drop(db).run()
dbt.pop(dbt.index(db))
print "Creating new rethink tables..."
for table in c.general.tables:
if not table in dbt:
print "Creating table {}".format(table)
rethinkdb.table_create(table).run()
for key in c.general.flush["redis"]:
if c.general.flush["redis"][key]:
print "Flushing redis "+key+" keys..."
keys = con.redis.keys(key+":*")
for key in keys: con.redis.delete(key)
def table_create(cls, if_not_exists=True):
if (if_not_exists
and (cls.Meta.table_name in r.table_list().run(get_conn()))):
return
return r.table_create(cls.Meta.table_name,
primary_key=cls.Meta.primary_key_field).run(
get_conn())
def _reset_data(self, table):
if table in rdb.table_list().run(self.session):
result = rdb.table_drop(table).run(self.session)
assert result['dropped'] == 1
result = rdb.table_create(table).run(self.session)
result = rdb.table(table).index_create('date').run(self.session)
return result.get('created', 0) == 1
def _reset_data(self, table):
if table in rdb.table_list().run(self.session):
result = rdb.table_drop(table).run(self.session)
assert result['dropped'] == 1
result = rdb.table_create(table).run(self.session)
result = rdb.table(table).index_create('date').run(self.session)
return result.get('created', 0) == 1
def table_exists(self, table_name):
"""Checks if a table exists in the database
@param table_name: table name
@returns Whether the table was found in the db or not.
"""
table_list = self.execute(r.table_list())
return table_name in table_list
def do_fix(db, collection=None):
if collection is None:
bad_meta, bad_tables = find_spurious_meta_and_tables(r.table('__METADATA__').run(db), r.table_list().run(db))
if len(bad_meta) == 0 and len(bad_tables) == 0:
return 0, 0
r.table('__METADATA__').get_all(*bad_meta).delete().run(db)
for table in bad_tables:
r.table_drop(table).run(db)
return len(bad_meta), len(bad_tables)
#else
check_collection_name(collection)
meta = r.table('__METADATA__').get(collection).run(db)
if meta is None:
raise BadCollection('collection {} does not exist.'.format(collection))
doing_init = meta.get('doing_init')
appending_filenames = meta.get('appending_filenames')
if not collection in r.table_list().run(db):
raise BadCollection("this is a spurious collection.")
if doing_init:
do_delete(db, collection)
return 'doing_init'
if appending_filenames:
bad_samples = [k for k in meta['samples'] if meta['samples'][k] in appending_filenames]
result = r.table(collection) \
.filter(r.row['IDs'].keys().set_intersection(appending_filenames) != [])\
.replace(lambda x: r.branch(x['IDs'].keys().set_difference(appending_filenames) == [],
None, # delete record
x.merge({
'IDs': r.literal(x['IDs'].without(appending_filenames)),
'QUALs': r.literal(x['QUALs'].without(appending_filenames)),
'FILTERs': r.literal(x['FILTERs'].without(appending_filenames)),
'INFOs': r.literal(x['INFOs'].without(appending_filenames)),
'samples': r.literal(x['samples'].without(bad_samples)),
}))).run(db)
r.table('__METADATA__').get(collection)\
.replace(lambda x: x.merge({
'vcfs': r.literal(x['vcfs'].without(appending_filenames)),
'samples': r.literal(x['samples'].without(bad_samples))
}).without('appending_filenames')).run(db)
return appending_filenames, bad_samples, result['deleted'], result['replaced']
return None
def generate_canary():
""" Generates new canary string """
canary = ''.join(random.SystemRandom().choice('abcdef' + string.digits) for _ in range(64))
if 'canary' not in r.table_list().run(flask.g.rdb_conn):
r.table_create('canary').run(flask.g.rdb_conn)
r.table('canary').insert({'date': r.now(), 'canary': canary}).run(flask.g.rdb_conn)
return flask.jsonify(canary=canary)
def create_table_if_not_exists(tname):
if tname in rdb.table_list().run(dbconn):
print('Table '
'{0}'
' already exists in RethinkDB instance. Skipping creation ...'.
format(tname))
else:
rdb.table_create(tname).run(dbconn)
print('Created {0} table in RethinkDB instance'.format(tname))
def do_delete(db, collection):
check_collection_name(collection)
if not collection in r.table_list().run(db):
return None
r.table_drop(collection).run(db)
r.table('__METADATA__').get(collection).delete().run(db)
return True
def post_stat(self, statDict):
if not self.config['enable_stats']:
return
LOG.debug("Posting stat: {0}".format(statDict['type']))
statTable = self.config['rethink_stat_table_template'].format(type=statDict['type'])
if statTable not in rethinkdb.table_list().run(self.rethink):
rethinkdb.table_create(statTable).run(self.rethink)
rethinkdb.table(statTable).insert(statDict).run(self.rethink)
def dump_all():
"""
Dump all tables contents to stdout.
"""
with connect() as con:
for table in r.table_list().run(con):
print('Table: ' + table)
print('#' * 50)
rows = list(r.table(table).run(con))
print(json.dumps(rows, sort_keys=True, indent=2))
def create_tables():
from .registry import model_registry
created_tables = r.table_list().run()
for model_cls in model_registry.all().values():
if model_cls._table not in created_tables:
result = r.table_create(model_cls._table).run()
if result['tables_created'] != 1:
raise RuntimeError('Could not create table %s for model %s' % (
model_cls._table, model_cls.__name__))
def do_list(db, collection=None):
if collection is None:
return list(t for t in r.table_list().run(db) if not t.startswith('__'))
#else
check_collection_name(collection)
metadata = r.table('__METADATA__').get(collection).run(db)
if metadata is None:
raise BadCollection('collection {} has no metadata.'. format(collection))
return metadata
def create_tables():
from .registry import model_registry
created_tables = r.table_list().run()
for model_cls in model_registry.all().values():
if model_cls._table not in created_tables:
result = r.table_create(model_cls._table).run()
if result['tables_created'] != 1:
raise RuntimeError('Could not create table %s for model %s' %
(model_cls._table, model_cls.__name__))
def create_rethink_tables():
current_tables = rethinkdb.table_list().coerce_to("array").run()
logger.debug("Current tables in rethink: {}".format(current_tables))
tables_to_create = list(set(rethink_bags.tables.keys()).difference(set(current_tables)))
logger.info("Creating these new tables in rethink: {}".format(tables_to_create))
for table in tables_to_create:
rethinkdb.table_create(table).run()
logger.debug("Table {} created in rethink".format(table))
def post_stat(self, statDict):
if not self.config['enable_stats']:
return
LOG.debug("Posting stat: {0}".format(statDict['type']))
statTable = self.config['rethink_stat_table_template'].format(
type=statDict['type'])
if statTable not in rethinkdb.table_list().run(self.rethink):
rethinkdb.table_create(statTable).run(self.rethink)
rethinkdb.table(statTable).insert(statDict).run(self.rethink)
def table_create(cls, if_not_exists=True):
if (
if_not_exists and
(cls.__table_name__ in r.table_list().run(get_conn()))
):
return
return r.table_create(
cls.__table_name__,
primary_key=cls.__primary_key__
).run(get_conn())
def table_create(cls, if_not_exists=True):
if (
if_not_exists and
(cls.Meta.table_name in r.table_list().run(get_conn()))
):
return
return r.table_create(
cls.Meta.table_name,
primary_key=cls.Meta.primary_key_field
).run(get_conn())
def get_main_id(connection, args):
args = check_args()
if args.thread_id:
thread_id = validate_thread_id(connection, args.thread_id)
else:
thread_id = get_latest_thread(connection, args.year, args.month)
if str(thread_id) not in rethinkdb.table_list().run(connection):
x = rethinkdb.table_create(str(thread_id)).run(connection)
if x.get('tables_created', 0) != 1:
raise Exception('Unable to create table {}'.format(thread_id))
return thread_id
def start(self, scheduler, alias):
super(RethinkDBJobStore, self).start(scheduler, alias)
if self.client:
self.conn = maybe_ref(self.client)
else:
self.conn = r.connect(db=self.database, **self.connect_args)
if self.database not in r.db_list().run(self.conn):
r.db_create(self.database).run(self.conn)
if self.table not in r.table_list().run(self.conn):
r.table_create(self.table).run(self.conn)
if self.job_submission_table not in r.table_list().run(self.conn):
r.table_create(self.job_submission_table).run(self.conn)
if 'next_run_time' not in r.table(self.table).index_list().run(self.conn):
r.table(self.table).index_create('next_run_time').run(self.conn)
self.table = r.db(self.database).table(self.table)
self.job_submission_table = r.db(self.database).table(self.job_submission_table)
def _initialize(self):
"""
Initialize DB and table
:return:
"""
if self._db not in r.db_list().run(self._connection):
r.db_create(self._db).run(self._connection)
self._connection.use(self._db)
if self._table not in r.table_list().run(self._connection):
r.table_create(self._table)
def setup_database(db):
db_name = current_app.config['DATABASE_NAME']
if db_name not in r.db_list().run(db):
r.db_create(db_name).run(db)
tables = r.table_list().run(db)
for table_name, options in _tables.items():
if table_name in tables:
continue
r.table_create(table_name, **options).run(db)
def do_check(db):
metadata = list(r.table('__METADATA__').run(db))
bad_meta, bad_tables = find_spurious_meta_and_tables(metadata, r.table_list().run(db))
inconsistent_collections = []
for m in metadata:
if str(m['id']) not in bad_meta:
if m.get('doing_init'):
inconsistent_collections.append((m, 'doing init'))
elif m.get('appending_filenames'):
inconsistent_collections.append((m, 'appending [{}]'.format(', '.join(m['appending_filenames']))))
return bad_meta, bad_tables, inconsistent_collections
def reseed():
db_list = r.db_list().run(conn)
if db_name not in db_list:
r.db_create(db_name).run(conn)
table_list = r.table_list().run(conn)
if products_table in table_list:
r.table_drop(products_table).run(conn)
r.table_create(products_table).run(conn)
r.table(products_table).insert(test_products).run(conn)
def flush_rethink_tables():
tables_to_flush = [ table for table, flush in rethink_bags.flush.iteritems() if flush ]
logger.debug("Tables that should be flushed: {}".format(tables_to_flush))
current_tables = rethinkdb.table_list().coerce_to("array").run()
logger.debug("Current tables in rethink: {}".format(current_tables))
flushing_tables = list(set(current_tables).intersection(tables_to_flush))
logger.info("Flushing tables in rethink: {}".format(flushing_tables))
for table in flushing_tables:
rethinkdb.table_drop(table).run()
rethinkdb.table_create(table).run()
logger.debug("Table {} flushed in rethink".format(table))
def dbSetup():
logging.info("Attempting db connection to %s:%s:%s...",DB_HOST,DB_PORT,DB_NAME)
logging.info("Checking if db %s exists",DB_NAME)
if DB_NAME not in list(r.db_list().run(conn)):
logging.info("db does not exist, creating...")
r.db_create(DB_NAME).run(conn)
logging.info("db exists")
logging.info("Checking to see if table %s exists",'observations')
if 'observations' not in list(r.table_list().run(conn)):
logging.info("table does not exist, creating...")
r.table_create("observations").run(conn)
conn.close()
logging.info("table exists")
def init_db():
connection = rethinkdb.connect(host=DB_HOST, port=DB_PORT)
if DB_DATABASE not in rethinkdb.db_list().run(connection):
x = rethinkdb.db_create(DB_DATABASE).run(connection)
if x.get('dbs_created', 0) != 1:
raise Exception('Unable to create database')
c = rethinkdb.connect(host=DB_HOST, port=DB_PORT, db=DB_DATABASE)
if DB_THREAD_IDS not in rethinkdb.table_list().run(connection):
x = rethinkdb.table_create(DB_THREAD_IDS).run(connection)
if x.get('tables_created', 0) != 1:
raise Exception('Unable to create table {}'.format(DB_THREAD_IDS))
return c
def dbSetup():
#logging.info("Attempting db connection to %s:%s:%s...",DB_HOST,DB_PORT,DB_NAME)
logging.info("======================================")
logging.info("Checking if db %s exists",DB_NAME)
if DB_NAME not in list(r.db_list().run(conn)):
logging.info("db does not exist, creating...")
r.db_create(DB_NAME).run(conn)
logging.info("db exists")
logging.info("Checking to see if table %s exists",'observations')
if 'observations' not in list(r.table_list().run(conn)):
logging.info("table does not exist, creating...")
r.table_create("observations").run(conn)
conn.close()
logging.info("table exists")
def create_all(cls):
"""
creates database and tables
on the r server if they
don't already exist
"""
cls._check_engine()
log.info('engine looks good')
if cls.engine._db not in cls.engine.run(r.db_list()):
cls.engine.run(r.db_create(cls.engine._db))
for tc in cls.__subclasses__():
if not hasattr(tc, '_tablename'):
tc._tablename = tc.__name__
if tc._tablename not in cls.engine.run(r.table_list()):
cls.engine.run(r.table_create(tc._tablename, **tc._table_args))
def initDB(self):
conn = r.connect(DB_HOST, DB_PORT, DB_NAME)
logging.info(MODULE_NAME+": Successful DB connection")
logging.info(MODULE_NAME+": Checking if db %s exists",DB_NAME)
if DB_NAME not in list(r.db_list().run(conn)):
logging.info(MODULE_NAME+": db does not exist, creating...")
r.db_create(DB_NAME).run(conn)
logging.info(MODULE_NAME+": db exists")
logging.info(MODULE_NAME+": Checking to see if table %s exists",TABLE_NAME)
if TABLE_NAME not in list(r.table_list().run(conn)):
logging.info(MODULE_NAME+": table does not exist, creating...")
r.table_create(TABLE_NAME).run(conn)
logging.info(MODULE_NAME+": table exists")
conn.close()
def initDB(self):
conn = r.connect(DB_HOST, DB_PORT, DB_NAME)
logging.info(MODULE_NAME + ": Successful DB connection")
logging.info(MODULE_NAME + ": Checking if db %s exists", DB_NAME)
if DB_NAME not in list(r.db_list().run(conn)):
logging.info(MODULE_NAME + ": db does not exist, creating...")
r.db_create(DB_NAME).run(conn)
logging.info(MODULE_NAME + ": db exists")
logging.info(MODULE_NAME + ": Checking to see if table %s exists",
TABLE_NAME)
if TABLE_NAME not in list(r.table_list().run(conn)):
logging.info(MODULE_NAME + ": table does not exist, creating...")
r.table_create(TABLE_NAME).run(conn)
logging.info(MODULE_NAME + ": table exists")
conn.close()
def CleanupOldState(client, blocklist):
"""
Remove the tables for state that are no longer necessary
:param SawtoothClient client: sawtooth.client.SawtoothClient for
accessing the ledger
:param list blocklist: list of block identifiers
"""
statenames = map(lambda b: 'blk' + b, blocklist)
tablelist = rethinkdb.table_list().run()
for table in tablelist:
if table.startswith('blk') and table not in statenames:
try:
logger.info('drop old state table %s', table)
rethinkdb.table_drop(table).run()
except:
logger.exception('failed to drop state table %s', table)
def start(self, scheduler, alias):
super(RethinkDBJobStore, self).start(scheduler, alias)
if self.client:
self.conn = maybe_ref(self.client)
else:
self.conn = r.connect(db=self.database, **self.connect_args)
if self.database not in r.db_list().run(self.conn):
r.db_create(self.database).run(self.conn)
if self.table not in r.table_list().run(self.conn):
r.table_create(self.table).run(self.conn)
if 'next_run_time' not in r.table(self.table).index_list().run(self.conn):
r.table(self.table).index_create('next_run_time').run(self.conn)
self.table = r.db(self.database).table(self.table)
def do_copy(db, source, dest):
check_collection_name(source)
check_collection_name(dest)
table_list = r.table_list().run(db)
source_meta = r.table('__METADATA__').get(source).run(db)
if not (source in table_list and source_meta is not None):
raise BadCollection("source collection does not exist.")
if not (dest not in table_list and r.table('__METADATA__').get(dest).run(db) is None):
raise BadCollection("destination collection already exists.")
r.table_create(dest).run(db)
result = r.table(dest).insert(r.table(source)).run(db)
source_meta['id'] = dest
r.table('__METADATA__').insert(source_meta).run(db)
return result['inserted']
def create_tables():
for table in ('pokemon', 'pokestops', 'gyms'):
if not r.table_list().contains(table).run():
r.table_create(table).run()
log.info('Created table "{}"'.format(table))
for index, table in [('disappear_time', 'pokemon')]:
if index not in r.table(table).index_list().run():
# Create index for some fields, this _should_ improve performance
# when filtering and sorting by this field.
r.table(table).index_create(index).run()
log.info('Created secondary index "{}" on table "{}"'.format(index, table))
if 'location' not in r.table('pokemon').index_list().run():
r.table('pokemon').index_create('location', geo=True).run()
log.info('Created geo index on table {}.'.format('pokemon'))
r.table('pokemon').update(lambda p: {'location': r.point(p['longitude'], p['latitude'])}).run()
r.table('pokemon').replace(r.row.without('longitude').without('latitude')).run()
log.info('Updated existing pokemon locations.')
def flush(conn, storage_period):
client = Client("", "")
time_res = client.get_server_time()
try:
if time_res['serverTime'] > 2:
print("Deleting old records")
cutoff_time = (time_res['serverTime'] / 1000) - storage_period
tables = r.table_list().run(conn)
if db_const.FEATURES_TABLE in tables:
r.table(db_const.FEATURES_TABLE)\
.filter(
r.row[fields.EPOCH_EVENT_TIME] < r.epoch_time(cutoff_time)
)\
.delete()\
.run(conn)
if db_const.TRADES_TABLE in tables:
r.table(db_const.TRADES_TABLE)\
.filter(
r.row[fields.EPOCH_EVENT_TIME] < r.epoch_time(cutoff_time)
)\
.delete()\
.run(conn)
if db_const.EXECUTION_TABLE in tables:
r.table(db_const.EXECUTION_TABLE)\
.filter(
r.row[fields.EPOCH_EVENT_TIME] < r.epoch_time(cutoff_time)
)\
.delete()\
.run(conn)
if db_const.BALANCES_TABLE in tables:
r.table(db_const.BALANCES_TABLE)\
.filter(
r.row[fields.EPOCH_EVENT_TIME] < r.epoch_time(cutoff_time)
)\
.delete()\
.run(conn)
except Exception as e:
print(e)
def InitializeDatabase(dbhost, dbport, dbname):
"""
"""
rconn = rethinkdb.connect(dbhost, dbport)
rconn.repl()
if dbname not in rethinkdb.db_list().run():
logger.info('create the sync database %s', dbname)
rethinkdb.db_create(dbname).run()
rconn.use(dbname)
tables = rethinkdb.table_list().run()
for tabname in ['block_list', 'chain_info', 'txn_list']:
if tabname not in tables:
rethinkdb.table_create(tabname).run()
rconn.close()
def post_entities(self, entityConfig, entities):
LOG.debug("Posting entities")
tableName = entityConfig['table']
if tableName not in rethinkdb.table_list().run(self.rethink):
LOG.debug("Creating table for type: {0}".format(entityConfig.type))
rethinkdb.table_create(tableName).run(self.rethink)
for entity in entities:
# Get rid of extra data found in sub-entities
# We don't have a way to reliably keep these up to date except
# for the type and id
entitySchema = self.entityConfigManager.schema[entityConfig.type]
for field, val in entity.items():
if field not in entitySchema:
continue
if field in ['type', 'id']:
continue
fieldDataType = entitySchema[field].get('data_type',
{}).get('value', None)
if fieldDataType == 'multi_entity':
val = [utils.getBaseEntity(e) for e in val]
entity[field] = val
elif fieldDataType == 'entity':
val = utils.getBaseEntity(val)
entity[field] = val
# TODO
# Since we aren't deleting the existing table
# Need to delete any extra entities at the end
result = rethinkdb.table(tableName).insert(entities,
conflict="replace").run(
self.rethink)
if result['errors']:
raise IOError(result['first_error'])
if result['errors']:
# TODO
# Better error descriptions?
raise IOError(
"Errors occurred creating entities: {0}".format(result))
def import_rethinkdb():
options = __opts__['rethinkdb']
host_addr = options['host']
db_name = options['db']
table_name = options['grains']
conn = r.connect(host=host_addr, db=db_name)
if table_name not in r.table_list().run(conn):
r.table_create(table_name).run(conn)
cursor = r.table(table_name).filter({'minion': __opts__['id']}).run(conn)
result = []
for grains in cursor:
result.append(grains)
if len(result) > 0:
return result[0]
return {}
def returner(ret):
options = __salt__['config.option']('rethinkdb')
host_addr = options['host']
db_name = options['db']
table_name = options['table']
conn = r.connect(host=host_addr, db=db_name)
if table_name not in r.table_list().run(conn):
log.warning('do not exist table')
r.table_create(table_name).run(conn)
log.info('create new table')
result = {
'job': ret['jid'],
'minion': ret['id'],
'function': ret['fun'],
'body': json.dumps(ret['return']),
#'body': ret['return'],
'return_code': ret['retcode']
}
result = r.table(table_name).insert(result).run(conn)
def LocalMain(config):
"""
Main processing loop for the synchronization process
"""
# pull database and collection names from the configuration and set up the
# connections that we need
dbhost = config.get('DatabaseHost', 'localhost')
dbport = int(config.get('DatabasePort', 28015))
dbname = config['DatabaseName']
rconn = rethinkdb.connect(dbhost, dbport)
rconn.repl()
rconn.use(dbname)
tablelist = rethinkdb.table_list().run()
for table in tablelist:
try:
logger.info('drop table %s', table)
rethinkdb.table_drop(table).run()
except:
logger.exception('failed to drop table %s', table)
rconn.close()
