def test_read_timeout(self): policy = RetryPolicy() # if this is the second or greater attempt, rethrow retry, consistency = policy.on_read_timeout( query=None, consistency=ONE, required_responses=1, received_responses=2, data_retrieved=True, retry_num=1) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) # if we didn't get enough responses, rethrow retry, consistency = policy.on_read_timeout( query=None, consistency=ONE, required_responses=2, received_responses=1, data_retrieved=True, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) # if we got enough responses, but also got a data response, rethrow retry, consistency = policy.on_read_timeout( query=None, consistency=ONE, required_responses=2, received_responses=2, data_retrieved=True, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) # we got enough responses but no data response, so retry retry, consistency = policy.on_read_timeout( query=None, consistency=ONE, required_responses=2, received_responses=2, data_retrieved=False, retry_num=0) self.assertEqual(retry, RetryPolicy.RETRY) self.assertEqual(consistency, ONE)
def test_write_timeout(self): policy = RetryPolicy() # if this is the second or greater attempt, rethrow retry, consistency = policy.on_write_timeout( query=None, consistency="ONE", write_type=WriteType.SIMPLE, required_responses=1, received_responses=2, retry_num=1) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) # if it's not a BATCH_LOG write, don't retry it retry, consistency = policy.on_write_timeout( query=None, consistency="ONE", write_type=WriteType.SIMPLE, required_responses=1, received_responses=2, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) # retry BATCH_LOG writes regardless of received responses retry, consistency = policy.on_write_timeout( query=None, consistency="ONE", write_type=WriteType.BATCH_LOG, required_responses=10000, received_responses=1, retry_num=0) self.assertEqual(retry, RetryPolicy.RETRY) self.assertEqual(consistency, 'ONE')
def test_unavailable(self): """ Use the same tests for test_write_timeout, but ensure they only RETHROW """ policy = RetryPolicy() retry, consistency = policy.on_unavailable(query=None, consistency="ONE", required_replicas=1, alive_replicas=2, retry_num=1) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_unavailable(query=None, consistency="ONE", required_replicas=1, alive_replicas=2, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_unavailable(query=None, consistency="ONE", required_replicas=10000, alive_replicas=1, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None)
def _local_connect(self): """ assumes single node Cassandra cluster """ connection.setup(hosts=self.hosts, default_keyspace=self.keyspace, consistency=CL.ONE, port=self.port, cql_version=self.cql_version, lazy_connect=False, retry_connect=True, compression=True, auth_provider=None, load_balancing_policy=RoundRobinPolicy(), protocol_version=4, executor_threads=2, reconnection_policy=ConstantReconnectionPolicy(3.0, 5), default_retry_policy=RetryPolicy(), conviction_policy_factory=None, metrics_enabled=False, connection_class=None, ssl_options=None, sockopts=None, max_schema_agreement_wait=10, control_connection_timeout=2.0, idle_heartbeat_interval=30, schema_event_refresh_window=2, topology_event_refresh_window=10, connect_timeout=self.connect_timeout) return
def test_retry_policy(self): self.assertEqual( insights_registry.serialize(RetryPolicy()), { 'type': 'RetryPolicy', 'namespace': 'cassandra.policies', 'options': {} })
def test_exec_profile_clone(self): cluster = Cluster(execution_profiles={EXEC_PROFILE_DEFAULT: ExecutionProfile(), 'one': ExecutionProfile()}) session = Session(cluster, hosts=[Host("127.0.0.1", SimpleConvictionPolicy)]) profile_attrs = {'request_timeout': 1, 'consistency_level': ConsistencyLevel.ANY, 'serial_consistency_level': ConsistencyLevel.SERIAL, 'row_factory': tuple_factory, 'retry_policy': RetryPolicy(), 'load_balancing_policy': default_lbp_factory()} reference_attributes = ('retry_policy', 'load_balancing_policy') # default and one named for profile in (EXEC_PROFILE_DEFAULT, 'one'): active = session.get_execution_profile(profile) clone = session.execution_profile_clone_update(profile) self.assertIsNot(clone, active) all_updated = session.execution_profile_clone_update(clone, **profile_attrs) self.assertIsNot(all_updated, clone) for attr, value in profile_attrs.items(): self.assertEqual(getattr(clone, attr), getattr(active, attr)) if attr in reference_attributes: self.assertIs(getattr(clone, attr), getattr(active, attr)) self.assertNotEqual(getattr(all_updated, attr), getattr(active, attr)) # cannot clone nonexistent profile self.assertRaises(ValueError, session.execution_profile_clone_update, 'DOES NOT EXIST', **profile_attrs)
def test_retry_policy_on_request_error(self): """ Test to verify that on_request_error is called properly. @since 3.18 @jira_ticket PYTHON-1064 @expected_result the appropriate retry policy is called @test_category connection """ overloaded_error = {"result": "overloaded", "message": "overloaded"} bootstrapping_error = { "result": "is_bootstrapping", "message": "isbootstrapping" } truncate_error = { "result": "truncate_error", "message": "truncate_error" } server_error = {"result": "server_error", "message": "server_error"} # Test the on_request_error call retry_policy = CounterRetryPolicy() self.set_cluster(retry_policy) for e in [ overloaded_error, bootstrapping_error, truncate_error, server_error ]: query_to_prime = "SELECT * from simulacron_keyspace.simulacron_table;" prime_query(query_to_prime, then=e, rows=None, column_types=None) rf = self.session.execute_async(query_to_prime) try: rf.result() except: pass self.assertEqual(len(rf.attempted_hosts), 1) # no retry self.assertEqual(next(retry_policy.request_error), 4) # Test that by default, retry on next host retry_policy = RetryPolicy() self.set_cluster(retry_policy) for e in [ overloaded_error, bootstrapping_error, truncate_error, server_error ]: query_to_prime = "SELECT * from simulacron_keyspace.simulacron_table;" prime_query(query_to_prime, then=e, rows=None, column_types=None) rf = self.session.execute_async(query_to_prime) try: rf.result() except: pass self.assertEqual(len(rf.attempted_hosts), 3) # all 3 nodes failed
def connect(self, nodes): cluster = Cluster(contact_points=nodes, default_retry_policy=RetryPolicy()) metadata = cluster.metadata self.session = cluster.connect('DATA') print '\nConnected to cluster: %s' % metadata.cluster_name for host in metadata.all_hosts(): print 'Datacenter: %s; Host: %s; Rack: %s' % ( host.datacenter, host.address, host.rack)
def test_default_connections(self): """ Ensure errors are not thrown when using non-default policies """ Cluster(load_balancing_policy=RoundRobinPolicy(), reconnection_policy=ExponentialReconnectionPolicy(1.0, 600.0), default_retry_policy=RetryPolicy(), conviction_policy_factory=SimpleConvictionPolicy)
def setup_tables(self): key_space = self.env.config.get(ConfigKeys.KEY_SPACE, domain=ConfigKeys.STORAGE) hosts = self.env.config.get(ConfigKeys.HOST, domain=ConfigKeys.STORAGE) hosts = hosts.split(",") # required to specify execution profiles in future versions profiles = { # override the default so we can set consistency level later EXEC_PROFILE_DEFAULT: ExecutionProfile( load_balancing_policy=TokenAwarePolicy( DCAwareRoundRobinPolicy()), retry_policy=RetryPolicy(), request_timeout=10.0, row_factory=Session._row_factory, # noqa # should probably be changed to QUORUM when having more than 3 nodes in the cluster consistency_level=ConsistencyLevel.LOCAL_ONE, ), # TODO: there doesn't seem to be a way to specify execution profile when # using the library's object mapping approach, only when writing pure # cql queries: # https://docs.datastax.com/en/developer/python-driver/3.24/execution_profiles/ # batch profile has longer timeout since they are run async anyway "batch": ExecutionProfile( load_balancing_policy=TokenAwarePolicy( DCAwareRoundRobinPolicy()), request_timeout=120.0, consistency_level=ConsistencyLevel.LOCAL_ONE, ) } kwargs = { "default_keyspace": key_space, "protocol_version": 3, "retry_connect": True, "execution_profiles": profiles, } username = self._get_from_conf(ConfigKeys.USER, ConfigKeys.STORAGE) password = self._get_from_conf(ConfigKeys.PASSWORD, ConfigKeys.STORAGE) if password is not None: auth_provider = PlainTextAuthProvider( username=username, password=password, ) kwargs["auth_provider"] = auth_provider connection.setup(hosts, **kwargs) sync_table(MessageModel) sync_table(AttachmentModel)
def cassandra_cluster(): from cassandra.cluster import Cluster from cassandra.policies import DCAwareRoundRobinPolicy, TokenAwarePolicy, RetryPolicy cassandra_hosts = ['10.0.0.251', '10.0.0.250', '10.0.0.249'] try: return Cluster(contact_points=cassandra_hosts, load_balancing_policy=TokenAwarePolicy( DCAwareRoundRobinPolicy(local_dc='Cassandra')), default_retry_policy=RetryPolicy()) except Error as er: print er
def test_write_timeout(self): policy = RetryPolicy() # if this is the second or greater attempt, rethrow retry, consistency = policy.on_write_timeout( query=None, consistency="ONE", write_type=WriteType.SIMPLE, required_responses=1, received_responses=2, retry_num=1) self.assertEqual(retry, RetryPolicy.RETHROW) # if it's not a BATCH_LOG write, don't retry it retry, consistency = policy.on_write_timeout( query=None, consistency="ONE", write_type=WriteType.SIMPLE, required_responses=1, received_responses=2, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) # retry BATCH_LOG writes regardless of received responses retry, consistency = policy.on_write_timeout( query=None, consistency="ONE", write_type=WriteType.BATCH_LOG, required_responses=10000, received_responses=1, retry_num=0) self.assertEqual(retry, RetryPolicy.RETRY) self.assertEqual(consistency, "ONE")
def cluster(cfg, keyspace=None): auth = PlainTextAuthProvider(username=cfg['cassandra_user'], password=cfg['cassandra_pass']) cluster = Cluster(cfg['cassandra_host'], load_balancing_policy=RoundRobinPolicy(), default_retry_policy=RetryPolicy(), reconnection_policy=ExponentialReconnectionPolicy( 1.0, 600.0), port=cfg['cassandra_port'], auth_provider=auth) return cluster
def test_read_timeout(self): policy = RetryPolicy() # if this is the second or greater attempt, rethrow retry, consistency = policy.on_read_timeout( query=None, consistency="ONE", required_responses=1, received_responses=2, data_retrieved=True, retry_num=1) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) # if we didn't get enough responses, rethrow retry, consistency = policy.on_read_timeout( query=None, consistency="ONE", required_responses=2, received_responses=1, data_retrieved=True, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) # if we got enough responses, but also got a data response, rethrow retry, consistency = policy.on_read_timeout( query=None, consistency="ONE", required_responses=2, received_responses=2, data_retrieved=True, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) # we got enough responses but no data response, so retry retry, consistency = policy.on_read_timeout( query=None, consistency="ONE", required_responses=2, received_responses=2, data_retrieved=False, retry_num=0) self.assertEqual(retry, RetryPolicy.RETRY) self.assertEqual(consistency, 'ONE')
def connect(self, *args, **kwargs): auth = PlainTextAuthProvider(username=self.user, password=self.password) cluster = Cluster(contact_points=self.host, auth_provider=auth, protocol_version=3, load_balancing_policy=default_lbp_factory(), default_retry_policy=RetryPolicy(), reconnection_policy=ConstantReconnectionPolicy( delay=1, max_attempts=10), *args, **kwargs) return cluster
def _connect_to_cassandra(keyspace): """ Connect to the Cassandra cluster and return the session. """ if 'BACKEND_STORAGE_IP' in os.environ: host = os.environ['BACKEND_STORAGE_IP'] else: host = '127.0.0.1' retry=RetryPolicy() retry.RETRY=10 cluster = Cluster( ['127.0.0.1',host,], reconnection_policy=ConstantReconnectionPolicy(5.0, 100), load_balancing_policy=DCAwareRoundRobinPolicy(local_dc="ASH2"), default_retry_policy=retry ) session = cluster.connect(keyspace) session.default_timeout = 99999 session.default_fetch_size = 1000 return session
def test(self): try: status = 0 cluster = Cluster(contact_points = self.hosts, default_retry_policy = RetryPolicy()) if self.db == "job": session = cluster.connect("mgrast_abundance") elif self.db == "m5nr": session = cluster.connect("m5nr_v1") if session: status = 1 cluster.shutdown() return status except: return 0
def test_unavailable(self): """ Use the same tests for test_write_timeout, but ensure they only RETHROW """ policy = RetryPolicy() retry, consistency = policy.on_unavailable( query=None, consistency=ONE, required_replicas=1, alive_replicas=2, retry_num=1) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_unavailable( query=None, consistency=ONE, required_replicas=1, alive_replicas=2, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_unavailable( query=None, consistency=ONE, required_replicas=10000, alive_replicas=1, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None)
def test_init(self): # just make sure Statement attributes are accepted kwargs = {'query_string': object(), 'retry_policy': RetryPolicy(), 'consistency_level': object(), 'fetch_size': object(), 'keyspace': object(), 'custom_payload': object()} statement = SimpleGraphStatement(**kwargs) for k, v in kwargs.items(): self.assertIs(getattr(statement, k), v) # but not a bogus parameter kwargs['bogus'] = object() self.assertRaises(TypeError, SimpleGraphStatement, **kwargs)
def test_no_legacy_with_profile(self): cluster_init = Cluster(execution_profiles={'name': ExecutionProfile()}) cluster_add = Cluster() cluster_add.add_execution_profile('name', ExecutionProfile()) # for clusters with profiles added either way... for cluster in (cluster_init, cluster_init): # don't allow legacy parameters set for attr, value in (('default_retry_policy', RetryPolicy()), ('load_balancing_policy', default_lbp_factory())): self.assertRaises(ValueError, setattr, cluster, attr, value) session = Session(cluster, hosts=[Host("127.0.0.1", SimpleConvictionPolicy)]) for attr, value in (('default_timeout', 1), ('default_consistency_level', ConsistencyLevel.ANY), ('default_serial_consistency_level', ConsistencyLevel.SERIAL), ('row_factory', tuple_factory)): self.assertRaises(ValueError, setattr, session, attr, value)
def _get_session(self) -> Session: if self._username and self._password: auth_provider = PlainTextAuthProvider(username=self._username, password=self._password) else: auth_provider = None cluster = Cluster( contact_points=self._contact_points, port=self._port, # load_balancing_policy= reconnection_policy=ConstantReconnectionPolicy(delay=5.0), default_retry_policy=RetryPolicy(), auth_provider=auth_provider) session = cluster.connect() session.set_keyspace('nexustiles') return session
def test_setting_retry_policy_to_statement(self): """ Test to verify that the retry policy is called as expected for bound and prepared statements when set to the prepared statement @since 3.13 @jira_ticket PYTHON-861 @expected_result the appropriate retry policy is called @test_category connection """ retry_policy = RetryPolicy() self.set_cluster(retry_policy) then = { "result": "write_timeout", "delay_in_ms": 0, "consistency_level": "LOCAL_QUORUM", "received": 1, "block_for": 2, "write_type": "SIMPLE", "ignore_on_prepare": True } query_to_prime_prepared = "SELECT * from simulacron_keyspace.simulacron_table WHERE key = :key" when = {"params": {"key": "0"}, "param_types": {"key": "ascii"}} prime_query(query_to_prime_prepared, when=when, then=then, rows=None, column_types=None) counter_policy = CounterRetryPolicy() prepared_stmt = self.session.prepare(query_to_prime_prepared) prepared_stmt.retry_policy = counter_policy self.session.execute(prepared_stmt, ("0", )) self.assertEqual(next(counter_policy.write_timeout), 1) counter_policy.reset_counters() bound_stmt = prepared_stmt.bind({"key": "0"}) bound_stmt.retry_policy = counter_policy self.session.execute(bound_stmt) self.assertEqual(next(counter_policy.write_timeout), 1)
def execute(self): # Connect to cassandra server cluster = Cluster( load_balancing_policy=TokenAwarePolicy(DCAwareRoundRobinPolicy()), default_retry_policy=RetryPolicy()) session = cluster.connect('cs4224') session.default_consistency_level = DEFAULT_CONSISTENCY_LEVEL session.execute.im_self.default_timeout = 1000000000000000 # Reading transactions line by line, parsing and execute while True: # Break if interrupted by user try: line = sys.stdin.readline().strip() except KeyboardInterrupt: break # Break on empty line if not line: break # Count transaction self.stats_collector.transactions.count() # Parsing transaction transaction_type = self.parser.get_transaction_type(line) extra_line_cnt = self.parser.get_transaction_extra_line_count(transaction_type, line) extra_lines = [] for i in range(extra_line_cnt): extra_line = sys.stdin.readline().strip() extra_lines.append(extra_line) transaction_params = self.parser.parse(line, transaction_type, extra_lines) # Execute transaction and measure time self.stats_collector.transaction_timer.start() self.execute_transaction(session, transaction_type, transaction_params) self.stats_collector.transaction_timer.finish() self.output()
def setUp(self): # create bogus json self.filename = 'testConfig.json' # get current dir self.cur_dir = dirname(abspath(__file__)) # load config self.config = self.loadConfigFile(self.filename) # create cassandra structure cassandra = self.config['cassandra'] clus = Cluster(cassandra['url'], default_retry_policy=RetryPolicy()) sess = clus.connect() sess.row_factory = dict_factory sess.default_timeout = None # save for later cas = { 'cluster': clus, 'session': sess, 'keyspace': cassandra['keyspace'] } self.cassandra = cas self.data_amt = 10000 self.idList = self._generateIDs(self.data_amt) # connect to elasticsearch and create the data self.elastic = {'session': Elasticsearch()} # reset data self.resetData() sql = ''' CREATE KEYSPACE %s WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }''' \ % cassandra['keyspace'] sess.execute(sql)
def create(hosts): global CASS_CLUSTER if not CASS_CLUSTER: CASS_CLUSTER = Cluster(contact_points = hosts, default_retry_policy = RetryPolicy()) return CASS_CLUSTER
def main(args): parser = OptionParser() parser.add_option("", "--batch", dest="batch", type="int", default=100, help="query batch size, default: 100") parser.add_option("", "--count", dest="count", type="int", default=10, help="number of query iterations, default: 10") parser.add_option("", "--phost", dest="phost", default=None, help="Postgres host") parser.add_option("", "--pname", dest="pname", default=None, help="Postgres name") parser.add_option("", "--puser", dest="puser", default=None, help="Postgres user") parser.add_option("", "--chost", dest="chost", default=None, help="Cassandra host") parser.add_option("", "--cname", dest="cname", default=None, help="Cassandra name") parser.add_option("", "--shost", dest="shost", default=None, help="SOLR host") parser.add_option("", "--sname", dest="sname", default=None, help="SOLR collection") (opts, args) = parser.parse_args() handle = None mode = "" # test PSQL if opts.phost and opts.pname and opts.puser: mode = "sql" handle = psycopg2.connect(host=opts.phost, database=opts.pname, user=opts.puser) # test CASS elif opts.chost and opts.cname: mode = "cql" handle = Cluster(contact_points=[opts.chost], default_retry_policy=RetryPolicy()) # test solr elif opts.shost and opts.sname: mode = "solr" surl = "http://" + opts.shost + "/solr/" + opts.sname + "/select" shead = {'Content-Type': 'application/x-www-form-urlencoded'} sfields = "%2C".join( ['md5_id', 'source', 'md5', 'accession', 'function', 'organism']) else: parser.error("Invalid usage") start = time.time() found = set() for i in range(opts.count): ints = random_array(opts.batch) query = "SELECT * FROM md5_id_annotation WHERE id IN (" + ",".join( map(str, ints)) + ");" if mode == "sql": cursor = handle.cursor() cursor.execute(query) data = cursor.fetchone() while (data): found.add(data[2]) data = cursor.fetchone() elif mode == "cql": session = handle.connect(opts.cname) rows = session.execute(query) for r in rows: found.add(r.md5) elif mode == "solr": query = "md5_id:(" + " OR ".join(map(str, ints)) + ")" sdata = "q=*%3A*&fq=" + query + "&start=0&rows=1000000000&wt=json&fl=" + sfields req = requests.post(surl, headers=shead, data=sdata, allow_redirects=True) rj = req.json() for d in rj['response']['docs']: found.add(d['md5']) end = time.time() print "%d loops of size %d ran in %d seconds" % (opts.count, opts.batch, int(round(end - start))) print "%d ids requested, %d md5s found" % ( (opts.count * opts.batch), len(found))
def get_retry(): return RetryPolicy()
def __init__(self, export_process): RetryPolicy.__init__(self) self.max_attempts = export_process.csv_options['maxattempts'] self.printmsg = lambda txt: export_process.printmsg(txt)
# theState=gpsStationState[2] # yellow theState = 2 # No data is available on selected date for this station, and station is # not within either the 1 day or 30 day window. elif (dataOnDate == False): # theState=gpsStationState[3] # light blue theState = 3 return theState #########SAVE TO CASSANDRA DB########### cluster = Cluster(contact_points=['127.0.0.1'], \ port=9042, \ load_balancing_policy= TokenAwarePolicy(DCAwareRoundRobinPolicy(local_dc='datacenter1')), \ default_retry_policy = RetryPolicy() ) session = cluster.connect('system') rows = cluster.metadata.keyspaces.keys() KEYSPACE = 'GPS_' + dataSet KEYSPACE = KEYSPACE.lower() # Create database keyspace_drop_stmt = session.prepare("DROP KEYSPACE " + KEYSPACE) if KEYSPACE in rows: print("dropping existing keyspace...") session.execute(keyspace_drop_stmt) print("creating keyspace...")