def sync(self, alias):
engine = get_engine_from_db_alias(alias)
if engine != 'django_cassandra_engine':
raise CommandError('Database {} is not cassandra!'.format(alias))
connection = connections[alias]
connection.connect()
options = connection.settings_dict.get('OPTIONS', {})
keyspace = connection.settings_dict['NAME']
replication_opts = options.get('replication', {})
strategy_class = replication_opts.pop('strategy_class',
'SimpleStrategy')
replication_factor = replication_opts.pop('replication_factor', 1)
self.stdout.write('Creating keyspace {}..'.format(keyspace))
if strategy_class == 'SimpleStrategy':
create_keyspace_simple(keyspace, replication_factor)
else:
create_keyspace_network_topology(keyspace, replication_opts)
for app_name, app_models \
in connection.introspection.cql_models.items():
for model in app_models:
self.stdout.write('Syncing %s.%s' % (app_name, model.__name__))
sync_table(model)
def init():
global already_loaded
if already_loaded:
return
connection.setup(
["localhost"],
default_keyspace=keyspace,
protocol_version=3,
load_balancing_policy=DCAwareRoundRobinPolicy(local_dc='DC1'),
retry_connect=True)
global _cql
_cql = connection.get_session()
management.create_keyspace_network_topology(keyspace, {'DC1': 1})
management.sync_table(Article, keyspaces=[keyspace])
global _es
_es = Elasticsearch(["localhost"],
scheme="http",
port=9200,
sniff_on_start=False,
sniff_on_connection_fail=True)
if not _es.indices.exists(index=keyspace):
print("PUT ES mapping")
_es.indices.create(keyspace,
json.loads(open('article-mapping.json').read()))
already_loaded = True
def sync(self, alias):
engine = get_engine_from_db_alias(alias)
if engine != 'django_cassandra_engine':
raise CommandError('Database {} is not cassandra!'.format(alias))
connection = connections[alias]
connection.connect()
options = connection.settings_dict.get('OPTIONS', {})
keyspace = connection.settings_dict['NAME']
replication_opts = options.get('replication', {})
strategy_class = replication_opts.pop('strategy_class',
'SimpleStrategy')
replication_factor = replication_opts.pop('replication_factor', 1)
self.stdout.write('Creating keyspace {}..'.format(keyspace))
if strategy_class == 'SimpleStrategy':
create_keyspace_simple(keyspace, replication_factor)
else:
create_keyspace_network_topology(keyspace, replication_opts)
for app_name, app_models \
in connection.introspection.cql_models.items():
for model in app_models:
self.stdout.write('Syncing %s.%s' % (app_name, model.__name__))
sync_table(model)
def create_network_keyspace(self):
cluster.max_schema_agreement_wait = 0
setup_cass(self.seeds, 'system')
self.session = get_session()
set_session(self.session)
dc_map = {'DC1-Data': 3, 'DC1-Analytics': 3}
create_keyspace_network_topology(name=self.keyspace, dc_replication_map=dc_map)
create_keyspace_simple(name=self.keyspace, replication_factor=3)
self.logger.debug("ks network topo created")
def test_create_drop_succeeeds(self):
cluster = get_cluster()
keyspace_ss = 'test_ks_ss'
self.assertNotIn(keyspace_ss, cluster.metadata.keyspaces)
management.create_keyspace_simple(keyspace_ss, 2)
self.assertIn(keyspace_ss, cluster.metadata.keyspaces)
management.drop_keyspace(keyspace_ss)
self.assertNotIn(keyspace_ss, cluster.metadata.keyspaces)
keyspace_nts = 'test_ks_nts'
self.assertNotIn(keyspace_nts, cluster.metadata.keyspaces)
management.create_keyspace_network_topology(keyspace_nts, {'dc1': 1})
self.assertIn(keyspace_nts, cluster.metadata.keyspaces)
management.drop_keyspace(keyspace_nts)
self.assertNotIn(keyspace_nts, cluster.metadata.keyspaces)
def test_create_drop_succeeeds(self):
cluster = get_cluster()
keyspace_ss = 'test_ks_ss'
self.assertNotIn(keyspace_ss, cluster.metadata.keyspaces)
management.create_keyspace_simple(keyspace_ss, 2)
self.assertIn(keyspace_ss, cluster.metadata.keyspaces)
management.drop_keyspace(keyspace_ss)
self.assertNotIn(keyspace_ss, cluster.metadata.keyspaces)
keyspace_nts = 'test_ks_nts'
self.assertNotIn(keyspace_nts, cluster.metadata.keyspaces)
management.create_keyspace_network_topology(keyspace_nts, {'dc1': 1})
self.assertIn(keyspace_nts, cluster.metadata.keyspaces)
management.drop_keyspace(keyspace_nts)
self.assertNotIn(keyspace_nts, cluster.metadata.keyspaces)
def __init__(self, nodes=None, keyspace='results_database', auth_provider=None, create_keyspace=False, replication_map=None):
self.keyspace = keyspace
self._depth = 0
self._connection_id = uuid.uuid4()
self._models = {}
self._nodes = nodes if nodes else ['localhost']
self._auth_provider = auth_provider
self._batch = []
try:
register_connection(name=str(self._connection_id), session=Cluster(self._nodes, auth_provider=self._auth_provider).connect())
does_keyspace_exist = self.keyspace in get_cluster(str(self._connection_id)).metadata.keyspaces
if create_keyspace and not does_keyspace_exist:
if not self.can_modify_schema():
raise Exception('Cannot create keyspace, Schema modification is disabled')
if replication_map is None:
create_keyspace_simple(self.keyspace, replication_factor=1, connections=[str(self._connection_id)])
else:
create_keyspace_network_topology(self.keyspace, dc_replication_map=replication_map, connections=[str(self._connection_id)])
elif not does_keyspace_exist:
raise Exception(f'Keyspace {self.keyspace} does not exist and will not be created')
finally:
unregister_connection(name=str(self._connection_id))
def initialise(keyspace, hosts=('127.0.0.1',), strategy='SimpleStrategy',
repl_factor=1, consistency=ConsistencyLevel.ONE):
"""Initialise Cassandra connection"""
num_retries = 6
retry_timeout = 1
for retry in xrange(num_retries):
try:
logger.info('Connecting to Cassandra keyspace "{2}" '
'on "{0}" with strategy "{1}"'.format(hosts, strategy, keyspace))
connection.setup(hosts, "system", protocol_version=3, consistency=ConsistencyLevel.ONE)
if strategy is 'SimpleStrategy':
create_keyspace_simple(keyspace, repl_factor, True)
else:
create_keyspace_network_topology(keyspace, {}, True)
connection.setup(hosts, keyspace, protocol_version=3, consistency=ConsistencyLevel.ONE)
break
except cassandra.cluster.NoHostAvailable:
logger.warning('Unable to connect to Cassandra. Retrying in {0} seconds...'.format(retry_timeout))
time.sleep(retry_timeout)
retry_timeout *= 2
def setup():
log.info("Setup cassandra connection")
connection.setup(settings.CASSANDRA_ENDPOINT, settings.MAIN_KEYSPACE)
log.info("Create keyspace")
{
"simple":
lambda: create_keyspace_simple(settings.MAIN_KEYSPACE, settings.
REPLICATION_FACTOR),
"network":
lambda: create_keyspace_network_topology(
settings.MAIN_KEYSPACE, {settings.DC: settings.REPLICATION_FACTOR})
}[settings.TOPOLOGY]()
log.info("Sync table")
sync_table(Job)
# ssl_opts["ca_certs"] = cfg.config["certpath"]
logger.info("connecting to database")
session = cassandra.cluster.Cluster(
contact_points=[CONTACT_POINT],
port=int(PORT),
auth_provider=cassandra.auth.PlainTextAuthProvider(
username=USERNAME, password=PASSWORD
),
ssl_options={
# "ca_certs": requests.utils.DEFAULT_CA_BUNDLE_PATH,
"ssl_version": ssl.PROTOCOL_TLSv1_2
},
connect_timeout=10,
control_connection_timeout=None,
).connect()
logger.info("connected to database")
session.row_factory = cassandra.query.dict_factory
try:
session.set_keyspace("presidents")
except cassandra.InvalidRequest: # if keyspace does not exist
# TODO: these settings are currently ignored
# https://docs.microsoft.com/en-us/azure/cosmos-db/cassandra-support#keyspace-and-table-options
management.create_keyspace_network_topology(
name="presidents", dc_replication_map={"datacenter": 1}
)
session.set_keyspace("presidents")
connection.set_session(session)
app = Flask(__name__)
app.config.from_object(config)
with app.app_context():
db_connection_setup([config.DB_HOST], config.DB_KEYSPACE, lazy_connect=True)
parser = argparse.ArgumentParser()
parser.add_argument('--recreate', nargs='?', default=DEFAULT)
parser.add_argument('--elastic', nargs='?', default=DEFAULT)
parser.add_argument('--seed', nargs='?', default=DEFAULT)
# get args and ignore unknown
args, unknown = parser.parse_known_args()
if args.recreate != DEFAULT:
print('recreate start')
drop_keyspace(config.DB_KEYSPACE)
create_keyspace_network_topology(config.DB_KEYSPACE, {'DC1': 3})
# sync_tables
def is_model(model, name):
return not name.startswith('_') and inspect.isclass(model) and issubclass(model, Model) and model != Model and getattr(model, 'disable_sync', None) != True
for name in dir(models):
model = getattr(models, name)
if is_model(model, name):
sync_table(model)
for name in dir(materialized_views):
model = getattr(materialized_views, name)
if is_model(model, name):
create_materialized_view(model)
if hasattr(models, 'after_recreate'):
models.after_recreate()
print('recreate end')
if args.elastic != DEFAULT:
def init():
global already_loaded
if already_loaded:
return
# Authentication configuration
if config_auth is not None:
auth_provider = PlainTextAuthProvider(username=config_auth['username'],
password=config_auth['password'])
http_auth = (config_auth['username'], config_auth['password'])
else:
auth_provider = None
http_auth = None
# Encryption configuration
if config_ssl is not None:
ssl_options = {
'ssl_version': ssl.PROTOCOL_TLSv1_2,
'ca_certs': config_ssl['cacert']
}
scheme = "https"
ca_certs = config_ssl['cacert']
else:
ssl_options = None
scheme = "http"
ca_certs = None
# Connection to cassandra
print("endpoints ", endpoints)
connection.setup(
endpoints,
default_keyspace=keyspace,
protocol_version=3,
auth_provider=auth_provider,
ssl_options=ssl_options,
load_balancing_policy=DCAwareRoundRobinPolicy(local_dc=local_dc),
retry_connect=True)
global _cql
_cql = connection.get_session()
# Synchronize cassandra schema
management.create_keyspace_network_topology(keyspace, replication_map)
management.sync_table(Product, keyspaces=[keyspace])
# Connection to elasticsearch HTTP
global _es
_es = Elasticsearch(
endpoints,
http_auth=http_auth,
scheme=scheme,
port=9200,
ca_certs=ca_certs,
sniff_on_start=False,
# refresh nodes after a node fails to respond
sniff_on_connection_fail=True,
# and also every 60 seconds
# sniffer_timeout=60
)
# Create elasticsearch mapping
# es.indices.delete(index=keyspace)
if not _es.indices.exists(index=keyspace):
print("put es mapping")
mapping_path = os.path.join(RESOURCES_DIR, 'mapping.json')
_es.indices.create(keyspace, json.loads(open(mapping_path).read()))
already_loaded = True