def sync(self, alias): engine = get_engine_from_db_alias(alias) if engine != 'django_cassandra_engine': raise CommandError('Database {} is not cassandra!'.format(alias)) connection = connections[alias] connection.connect() options = connection.settings_dict.get('OPTIONS', {}) keyspace = connection.settings_dict['NAME'] replication_opts = options.get('replication', {}) strategy_class = replication_opts.pop('strategy_class', 'SimpleStrategy') replication_factor = replication_opts.pop('replication_factor', 1) self.stdout.write('Creating keyspace {}..'.format(keyspace)) if strategy_class == 'SimpleStrategy': create_keyspace_simple(keyspace, replication_factor) else: create_keyspace_network_topology(keyspace, replication_opts) for app_name, app_models \ in connection.introspection.cql_models.items(): for model in app_models: self.stdout.write('Syncing %s.%s' % (app_name, model.__name__)) sync_table(model)
def init(): global already_loaded if already_loaded: return connection.setup( ["localhost"], default_keyspace=keyspace, protocol_version=3, load_balancing_policy=DCAwareRoundRobinPolicy(local_dc='DC1'), retry_connect=True) global _cql _cql = connection.get_session() management.create_keyspace_network_topology(keyspace, {'DC1': 1}) management.sync_table(Article, keyspaces=[keyspace]) global _es _es = Elasticsearch(["localhost"], scheme="http", port=9200, sniff_on_start=False, sniff_on_connection_fail=True) if not _es.indices.exists(index=keyspace): print("PUT ES mapping") _es.indices.create(keyspace, json.loads(open('article-mapping.json').read())) already_loaded = True
def sync(self, alias): engine = get_engine_from_db_alias(alias) if engine != 'django_cassandra_engine': raise CommandError('Database {} is not cassandra!'.format(alias)) connection = connections[alias] connection.connect() options = connection.settings_dict.get('OPTIONS', {}) keyspace = connection.settings_dict['NAME'] replication_opts = options.get('replication', {}) strategy_class = replication_opts.pop('strategy_class', 'SimpleStrategy') replication_factor = replication_opts.pop('replication_factor', 1) self.stdout.write('Creating keyspace {}..'.format(keyspace)) if strategy_class == 'SimpleStrategy': create_keyspace_simple(keyspace, replication_factor) else: create_keyspace_network_topology(keyspace, replication_opts) for app_name, app_models \ in connection.introspection.cql_models.items(): for model in app_models: self.stdout.write('Syncing %s.%s' % (app_name, model.__name__)) sync_table(model)
def create_network_keyspace(self): cluster.max_schema_agreement_wait = 0 setup_cass(self.seeds, 'system') self.session = get_session() set_session(self.session) dc_map = {'DC1-Data': 3, 'DC1-Analytics': 3} create_keyspace_network_topology(name=self.keyspace, dc_replication_map=dc_map) create_keyspace_simple(name=self.keyspace, replication_factor=3) self.logger.debug("ks network topo created")
def test_create_drop_succeeeds(self): cluster = get_cluster() keyspace_ss = 'test_ks_ss' self.assertNotIn(keyspace_ss, cluster.metadata.keyspaces) management.create_keyspace_simple(keyspace_ss, 2) self.assertIn(keyspace_ss, cluster.metadata.keyspaces) management.drop_keyspace(keyspace_ss) self.assertNotIn(keyspace_ss, cluster.metadata.keyspaces) keyspace_nts = 'test_ks_nts' self.assertNotIn(keyspace_nts, cluster.metadata.keyspaces) management.create_keyspace_network_topology(keyspace_nts, {'dc1': 1}) self.assertIn(keyspace_nts, cluster.metadata.keyspaces) management.drop_keyspace(keyspace_nts) self.assertNotIn(keyspace_nts, cluster.metadata.keyspaces)
def test_create_drop_succeeeds(self): cluster = get_cluster() keyspace_ss = 'test_ks_ss' self.assertNotIn(keyspace_ss, cluster.metadata.keyspaces) management.create_keyspace_simple(keyspace_ss, 2) self.assertIn(keyspace_ss, cluster.metadata.keyspaces) management.drop_keyspace(keyspace_ss) self.assertNotIn(keyspace_ss, cluster.metadata.keyspaces) keyspace_nts = 'test_ks_nts' self.assertNotIn(keyspace_nts, cluster.metadata.keyspaces) management.create_keyspace_network_topology(keyspace_nts, {'dc1': 1}) self.assertIn(keyspace_nts, cluster.metadata.keyspaces) management.drop_keyspace(keyspace_nts) self.assertNotIn(keyspace_nts, cluster.metadata.keyspaces)
def __init__(self, nodes=None, keyspace='results_database', auth_provider=None, create_keyspace=False, replication_map=None): self.keyspace = keyspace self._depth = 0 self._connection_id = uuid.uuid4() self._models = {} self._nodes = nodes if nodes else ['localhost'] self._auth_provider = auth_provider self._batch = [] try: register_connection(name=str(self._connection_id), session=Cluster(self._nodes, auth_provider=self._auth_provider).connect()) does_keyspace_exist = self.keyspace in get_cluster(str(self._connection_id)).metadata.keyspaces if create_keyspace and not does_keyspace_exist: if not self.can_modify_schema(): raise Exception('Cannot create keyspace, Schema modification is disabled') if replication_map is None: create_keyspace_simple(self.keyspace, replication_factor=1, connections=[str(self._connection_id)]) else: create_keyspace_network_topology(self.keyspace, dc_replication_map=replication_map, connections=[str(self._connection_id)]) elif not does_keyspace_exist: raise Exception(f'Keyspace {self.keyspace} does not exist and will not be created') finally: unregister_connection(name=str(self._connection_id))
def initialise(keyspace, hosts=('127.0.0.1',), strategy='SimpleStrategy', repl_factor=1, consistency=ConsistencyLevel.ONE): """Initialise Cassandra connection""" num_retries = 6 retry_timeout = 1 for retry in xrange(num_retries): try: logger.info('Connecting to Cassandra keyspace "{2}" ' 'on "{0}" with strategy "{1}"'.format(hosts, strategy, keyspace)) connection.setup(hosts, "system", protocol_version=3, consistency=ConsistencyLevel.ONE) if strategy is 'SimpleStrategy': create_keyspace_simple(keyspace, repl_factor, True) else: create_keyspace_network_topology(keyspace, {}, True) connection.setup(hosts, keyspace, protocol_version=3, consistency=ConsistencyLevel.ONE) break except cassandra.cluster.NoHostAvailable: logger.warning('Unable to connect to Cassandra. Retrying in {0} seconds...'.format(retry_timeout)) time.sleep(retry_timeout) retry_timeout *= 2
def setup(): log.info("Setup cassandra connection") connection.setup(settings.CASSANDRA_ENDPOINT, settings.MAIN_KEYSPACE) log.info("Create keyspace") { "simple": lambda: create_keyspace_simple(settings.MAIN_KEYSPACE, settings. REPLICATION_FACTOR), "network": lambda: create_keyspace_network_topology( settings.MAIN_KEYSPACE, {settings.DC: settings.REPLICATION_FACTOR}) }[settings.TOPOLOGY]() log.info("Sync table") sync_table(Job)
# ssl_opts["ca_certs"] = cfg.config["certpath"] logger.info("connecting to database") session = cassandra.cluster.Cluster( contact_points=[CONTACT_POINT], port=int(PORT), auth_provider=cassandra.auth.PlainTextAuthProvider( username=USERNAME, password=PASSWORD ), ssl_options={ # "ca_certs": requests.utils.DEFAULT_CA_BUNDLE_PATH, "ssl_version": ssl.PROTOCOL_TLSv1_2 }, connect_timeout=10, control_connection_timeout=None, ).connect() logger.info("connected to database") session.row_factory = cassandra.query.dict_factory try: session.set_keyspace("presidents") except cassandra.InvalidRequest: # if keyspace does not exist # TODO: these settings are currently ignored # https://docs.microsoft.com/en-us/azure/cosmos-db/cassandra-support#keyspace-and-table-options management.create_keyspace_network_topology( name="presidents", dc_replication_map={"datacenter": 1} ) session.set_keyspace("presidents") connection.set_session(session)
app = Flask(__name__) app.config.from_object(config) with app.app_context(): db_connection_setup([config.DB_HOST], config.DB_KEYSPACE, lazy_connect=True) parser = argparse.ArgumentParser() parser.add_argument('--recreate', nargs='?', default=DEFAULT) parser.add_argument('--elastic', nargs='?', default=DEFAULT) parser.add_argument('--seed', nargs='?', default=DEFAULT) # get args and ignore unknown args, unknown = parser.parse_known_args() if args.recreate != DEFAULT: print('recreate start') drop_keyspace(config.DB_KEYSPACE) create_keyspace_network_topology(config.DB_KEYSPACE, {'DC1': 3}) # sync_tables def is_model(model, name): return not name.startswith('_') and inspect.isclass(model) and issubclass(model, Model) and model != Model and getattr(model, 'disable_sync', None) != True for name in dir(models): model = getattr(models, name) if is_model(model, name): sync_table(model) for name in dir(materialized_views): model = getattr(materialized_views, name) if is_model(model, name): create_materialized_view(model) if hasattr(models, 'after_recreate'): models.after_recreate() print('recreate end') if args.elastic != DEFAULT:
def init(): global already_loaded if already_loaded: return # Authentication configuration if config_auth is not None: auth_provider = PlainTextAuthProvider(username=config_auth['username'], password=config_auth['password']) http_auth = (config_auth['username'], config_auth['password']) else: auth_provider = None http_auth = None # Encryption configuration if config_ssl is not None: ssl_options = { 'ssl_version': ssl.PROTOCOL_TLSv1_2, 'ca_certs': config_ssl['cacert'] } scheme = "https" ca_certs = config_ssl['cacert'] else: ssl_options = None scheme = "http" ca_certs = None # Connection to cassandra print("endpoints ", endpoints) connection.setup( endpoints, default_keyspace=keyspace, protocol_version=3, auth_provider=auth_provider, ssl_options=ssl_options, load_balancing_policy=DCAwareRoundRobinPolicy(local_dc=local_dc), retry_connect=True) global _cql _cql = connection.get_session() # Synchronize cassandra schema management.create_keyspace_network_topology(keyspace, replication_map) management.sync_table(Product, keyspaces=[keyspace]) # Connection to elasticsearch HTTP global _es _es = Elasticsearch( endpoints, http_auth=http_auth, scheme=scheme, port=9200, ca_certs=ca_certs, sniff_on_start=False, # refresh nodes after a node fails to respond sniff_on_connection_fail=True, # and also every 60 seconds # sniffer_timeout=60 ) # Create elasticsearch mapping # es.indices.delete(index=keyspace) if not _es.indices.exists(index=keyspace): print("put es mapping") mapping_path = os.path.join(RESOURCES_DIR, 'mapping.json') _es.indices.create(keyspace, json.loads(open(mapping_path).read())) already_loaded = True