Esempio n. 1
0
    def test_read_timeout(self):
        policy = RetryPolicy()

        # if this is the second or greater attempt, rethrow
        retry, consistency = policy.on_read_timeout(
            query=None, consistency=ONE, required_responses=1, received_responses=2,
            data_retrieved=True, retry_num=1)
        self.assertEqual(retry, RetryPolicy.RETHROW)
        self.assertEqual(consistency, None)

        # if we didn't get enough responses, rethrow
        retry, consistency = policy.on_read_timeout(
            query=None, consistency=ONE, required_responses=2, received_responses=1,
            data_retrieved=True, retry_num=0)
        self.assertEqual(retry, RetryPolicy.RETHROW)
        self.assertEqual(consistency, None)

        # if we got enough responses, but also got a data response, rethrow
        retry, consistency = policy.on_read_timeout(
            query=None, consistency=ONE, required_responses=2, received_responses=2,
            data_retrieved=True, retry_num=0)
        self.assertEqual(retry, RetryPolicy.RETHROW)
        self.assertEqual(consistency, None)

        # we got enough responses but no data response, so retry
        retry, consistency = policy.on_read_timeout(
            query=None, consistency=ONE, required_responses=2, received_responses=2,
            data_retrieved=False, retry_num=0)
        self.assertEqual(retry, RetryPolicy.RETRY)
        self.assertEqual(consistency, ONE)
    def test_write_timeout(self):
        policy = RetryPolicy()

        # if this is the second or greater attempt, rethrow
        retry, consistency = policy.on_write_timeout(
            query=None,
            consistency="ONE",
            write_type=WriteType.SIMPLE,
            required_responses=1,
            received_responses=2,
            retry_num=1)
        self.assertEqual(retry, RetryPolicy.RETHROW)
        self.assertEqual(consistency, None)

        # if it's not a BATCH_LOG write, don't retry it
        retry, consistency = policy.on_write_timeout(
            query=None,
            consistency="ONE",
            write_type=WriteType.SIMPLE,
            required_responses=1,
            received_responses=2,
            retry_num=0)
        self.assertEqual(retry, RetryPolicy.RETHROW)
        self.assertEqual(consistency, None)

        # retry BATCH_LOG writes regardless of received responses
        retry, consistency = policy.on_write_timeout(
            query=None,
            consistency="ONE",
            write_type=WriteType.BATCH_LOG,
            required_responses=10000,
            received_responses=1,
            retry_num=0)
        self.assertEqual(retry, RetryPolicy.RETRY)
        self.assertEqual(consistency, 'ONE')
    def test_unavailable(self):
        """
        Use the same tests for test_write_timeout, but ensure they only RETHROW
        """
        policy = RetryPolicy()

        retry, consistency = policy.on_unavailable(query=None,
                                                   consistency="ONE",
                                                   required_replicas=1,
                                                   alive_replicas=2,
                                                   retry_num=1)
        self.assertEqual(retry, RetryPolicy.RETHROW)
        self.assertEqual(consistency, None)

        retry, consistency = policy.on_unavailable(query=None,
                                                   consistency="ONE",
                                                   required_replicas=1,
                                                   alive_replicas=2,
                                                   retry_num=0)
        self.assertEqual(retry, RetryPolicy.RETHROW)
        self.assertEqual(consistency, None)

        retry, consistency = policy.on_unavailable(query=None,
                                                   consistency="ONE",
                                                   required_replicas=10000,
                                                   alive_replicas=1,
                                                   retry_num=0)
        self.assertEqual(retry, RetryPolicy.RETHROW)
        self.assertEqual(consistency, None)
Esempio n. 4
0
 def _local_connect(self):
     """
     assumes single node Cassandra cluster
     """
     connection.setup(hosts=self.hosts,
                      default_keyspace=self.keyspace,
                      consistency=CL.ONE,
                      port=self.port,
                      cql_version=self.cql_version,
                      lazy_connect=False,
                      retry_connect=True,
                      compression=True,
                      auth_provider=None,
                      load_balancing_policy=RoundRobinPolicy(),
                      protocol_version=4,
                      executor_threads=2,
                      reconnection_policy=ConstantReconnectionPolicy(3.0, 5),
                      default_retry_policy=RetryPolicy(),
                      conviction_policy_factory=None,
                      metrics_enabled=False,
                      connection_class=None,
                      ssl_options=None,
                      sockopts=None,
                      max_schema_agreement_wait=10,
                      control_connection_timeout=2.0,
                      idle_heartbeat_interval=30,
                      schema_event_refresh_window=2,
                      topology_event_refresh_window=10,
                      connect_timeout=self.connect_timeout)
     return
Esempio n. 5
0
 def test_retry_policy(self):
     self.assertEqual(
         insights_registry.serialize(RetryPolicy()), {
             'type': 'RetryPolicy',
             'namespace': 'cassandra.policies',
             'options': {}
         })
Esempio n. 6
0
    def test_exec_profile_clone(self):

        cluster = Cluster(execution_profiles={EXEC_PROFILE_DEFAULT: ExecutionProfile(), 'one': ExecutionProfile()})
        session = Session(cluster, hosts=[Host("127.0.0.1", SimpleConvictionPolicy)])

        profile_attrs = {'request_timeout': 1,
                         'consistency_level': ConsistencyLevel.ANY,
                         'serial_consistency_level': ConsistencyLevel.SERIAL,
                         'row_factory': tuple_factory,
                         'retry_policy': RetryPolicy(),
                         'load_balancing_policy': default_lbp_factory()}
        reference_attributes = ('retry_policy', 'load_balancing_policy')

        # default and one named
        for profile in (EXEC_PROFILE_DEFAULT, 'one'):
            active = session.get_execution_profile(profile)
            clone = session.execution_profile_clone_update(profile)
            self.assertIsNot(clone, active)

            all_updated = session.execution_profile_clone_update(clone, **profile_attrs)
            self.assertIsNot(all_updated, clone)
            for attr, value in profile_attrs.items():
                self.assertEqual(getattr(clone, attr), getattr(active, attr))
                if attr in reference_attributes:
                    self.assertIs(getattr(clone, attr), getattr(active, attr))
                self.assertNotEqual(getattr(all_updated, attr), getattr(active, attr))

        # cannot clone nonexistent profile
        self.assertRaises(ValueError, session.execution_profile_clone_update, 'DOES NOT EXIST', **profile_attrs)
Esempio n. 7
0
    def test_retry_policy_on_request_error(self):
        """
        Test to verify that on_request_error is called properly.

        @since 3.18
        @jira_ticket PYTHON-1064
        @expected_result the appropriate retry policy is called

        @test_category connection
        """
        overloaded_error = {"result": "overloaded", "message": "overloaded"}

        bootstrapping_error = {
            "result": "is_bootstrapping",
            "message": "isbootstrapping"
        }

        truncate_error = {
            "result": "truncate_error",
            "message": "truncate_error"
        }

        server_error = {"result": "server_error", "message": "server_error"}

        # Test the on_request_error call
        retry_policy = CounterRetryPolicy()
        self.set_cluster(retry_policy)

        for e in [
                overloaded_error, bootstrapping_error, truncate_error,
                server_error
        ]:
            query_to_prime = "SELECT * from simulacron_keyspace.simulacron_table;"
            prime_query(query_to_prime, then=e, rows=None, column_types=None)
            rf = self.session.execute_async(query_to_prime)
            try:
                rf.result()
            except:
                pass
            self.assertEqual(len(rf.attempted_hosts), 1)  # no retry

        self.assertEqual(next(retry_policy.request_error), 4)

        # Test that by default, retry on next host
        retry_policy = RetryPolicy()
        self.set_cluster(retry_policy)

        for e in [
                overloaded_error, bootstrapping_error, truncate_error,
                server_error
        ]:
            query_to_prime = "SELECT * from simulacron_keyspace.simulacron_table;"
            prime_query(query_to_prime, then=e, rows=None, column_types=None)
            rf = self.session.execute_async(query_to_prime)
            try:
                rf.result()
            except:
                pass
            self.assertEqual(len(rf.attempted_hosts), 3)  # all 3 nodes failed
Esempio n. 8
0
 def connect(self, nodes):
     cluster = Cluster(contact_points=nodes,
                       default_retry_policy=RetryPolicy())
     metadata = cluster.metadata
     self.session = cluster.connect('DATA')
     print '\nConnected to cluster: %s' % metadata.cluster_name
     for host in metadata.all_hosts():
         print 'Datacenter: %s; Host: %s; Rack: %s' % (
             host.datacenter, host.address, host.rack)
Esempio n. 9
0
    def test_default_connections(self):
        """
        Ensure errors are not thrown when using non-default policies
        """

        Cluster(load_balancing_policy=RoundRobinPolicy(),
                reconnection_policy=ExponentialReconnectionPolicy(1.0, 600.0),
                default_retry_policy=RetryPolicy(),
                conviction_policy_factory=SimpleConvictionPolicy)
Esempio n. 10
0
    def setup_tables(self):
        key_space = self.env.config.get(ConfigKeys.KEY_SPACE,
                                        domain=ConfigKeys.STORAGE)
        hosts = self.env.config.get(ConfigKeys.HOST, domain=ConfigKeys.STORAGE)
        hosts = hosts.split(",")

        # required to specify execution profiles in future versions
        profiles = {
            # override the default so we can set consistency level later
            EXEC_PROFILE_DEFAULT:
            ExecutionProfile(
                load_balancing_policy=TokenAwarePolicy(
                    DCAwareRoundRobinPolicy()),
                retry_policy=RetryPolicy(),
                request_timeout=10.0,
                row_factory=Session._row_factory,  # noqa
                # should probably be changed to QUORUM when having more than 3 nodes in the cluster
                consistency_level=ConsistencyLevel.LOCAL_ONE,
            ),
            # TODO: there doesn't seem to be a way to specify execution profile when
            #  using the library's object mapping approach, only when writing pure
            #  cql queries:
            #  https://docs.datastax.com/en/developer/python-driver/3.24/execution_profiles/
            # batch profile has longer timeout since they are run async anyway
            "batch":
            ExecutionProfile(
                load_balancing_policy=TokenAwarePolicy(
                    DCAwareRoundRobinPolicy()),
                request_timeout=120.0,
                consistency_level=ConsistencyLevel.LOCAL_ONE,
            )
        }

        kwargs = {
            "default_keyspace": key_space,
            "protocol_version": 3,
            "retry_connect": True,
            "execution_profiles": profiles,
        }

        username = self._get_from_conf(ConfigKeys.USER, ConfigKeys.STORAGE)
        password = self._get_from_conf(ConfigKeys.PASSWORD, ConfigKeys.STORAGE)

        if password is not None:
            auth_provider = PlainTextAuthProvider(
                username=username,
                password=password,
            )
            kwargs["auth_provider"] = auth_provider

        connection.setup(hosts, **kwargs)

        sync_table(MessageModel)
        sync_table(AttachmentModel)
Esempio n. 11
0
def cassandra_cluster():
    from cassandra.cluster import Cluster
    from cassandra.policies import DCAwareRoundRobinPolicy, TokenAwarePolicy, RetryPolicy

    cassandra_hosts = ['10.0.0.251', '10.0.0.250', '10.0.0.249']
    try:
        return Cluster(contact_points=cassandra_hosts,
                       load_balancing_policy=TokenAwarePolicy(
                           DCAwareRoundRobinPolicy(local_dc='Cassandra')),
                       default_retry_policy=RetryPolicy())
    except Error as er:
        print er
Esempio n. 12
0
    def test_write_timeout(self):
        policy = RetryPolicy()

        # if this is the second or greater attempt, rethrow
        retry, consistency = policy.on_write_timeout(
            query=None, consistency="ONE", write_type=WriteType.SIMPLE,
            required_responses=1, received_responses=2, retry_num=1)
        self.assertEqual(retry, RetryPolicy.RETHROW)

        # if it's not a BATCH_LOG write, don't retry it
        retry, consistency = policy.on_write_timeout(
            query=None, consistency="ONE", write_type=WriteType.SIMPLE,
            required_responses=1, received_responses=2, retry_num=0)
        self.assertEqual(retry, RetryPolicy.RETHROW)

        # retry BATCH_LOG writes regardless of received responses
        retry, consistency = policy.on_write_timeout(
            query=None, consistency="ONE", write_type=WriteType.BATCH_LOG,
            required_responses=10000, received_responses=1, retry_num=0)
        self.assertEqual(retry, RetryPolicy.RETRY)
        self.assertEqual(consistency, "ONE")
Esempio n. 13
0
def cluster(cfg, keyspace=None):
    auth = PlainTextAuthProvider(username=cfg['cassandra_user'],
                                 password=cfg['cassandra_pass'])

    cluster = Cluster(cfg['cassandra_host'],
                      load_balancing_policy=RoundRobinPolicy(),
                      default_retry_policy=RetryPolicy(),
                      reconnection_policy=ExponentialReconnectionPolicy(
                          1.0, 600.0),
                      port=cfg['cassandra_port'],
                      auth_provider=auth)
    return cluster
    def test_read_timeout(self):
        policy = RetryPolicy()

        # if this is the second or greater attempt, rethrow
        retry, consistency = policy.on_read_timeout(
            query=None, consistency="ONE", required_responses=1, received_responses=2,
            data_retrieved=True, retry_num=1)
        self.assertEqual(retry, RetryPolicy.RETHROW)
        self.assertEqual(consistency, None)

        # if we didn't get enough responses, rethrow
        retry, consistency = policy.on_read_timeout(
            query=None, consistency="ONE", required_responses=2, received_responses=1,
            data_retrieved=True, retry_num=0)
        self.assertEqual(retry, RetryPolicy.RETHROW)
        self.assertEqual(consistency, None)

        # if we got enough responses, but also got a data response, rethrow
        retry, consistency = policy.on_read_timeout(
            query=None, consistency="ONE", required_responses=2, received_responses=2,
            data_retrieved=True, retry_num=0)
        self.assertEqual(retry, RetryPolicy.RETHROW)
        self.assertEqual(consistency, None)

        # we got enough responses but no data response, so retry
        retry, consistency = policy.on_read_timeout(
            query=None, consistency="ONE", required_responses=2, received_responses=2,
            data_retrieved=False, retry_num=0)
        self.assertEqual(retry, RetryPolicy.RETRY)
        self.assertEqual(consistency, 'ONE')
Esempio n. 15
0
 def connect(self, *args, **kwargs):
     auth = PlainTextAuthProvider(username=self.user,
                                  password=self.password)
     cluster = Cluster(contact_points=self.host,
                       auth_provider=auth,
                       protocol_version=3,
                       load_balancing_policy=default_lbp_factory(),
                       default_retry_policy=RetryPolicy(),
                       reconnection_policy=ConstantReconnectionPolicy(
                           delay=1, max_attempts=10),
                       *args,
                       **kwargs)
     return cluster
Esempio n. 16
0
def _connect_to_cassandra(keyspace):
    """
    Connect to the Cassandra cluster and return the session.
    """

    if 'BACKEND_STORAGE_IP' in os.environ:
        host = os.environ['BACKEND_STORAGE_IP']
    else:
        host = '127.0.0.1'
    retry=RetryPolicy()
    retry.RETRY=10
    cluster = Cluster(
        ['127.0.0.1',host,],
        reconnection_policy=ConstantReconnectionPolicy(5.0, 100),
        load_balancing_policy=DCAwareRoundRobinPolicy(local_dc="ASH2"),
        default_retry_policy=retry
    )
    session = cluster.connect(keyspace)
    session.default_timeout = 99999
    session.default_fetch_size = 1000

    return session
Esempio n. 17
0
 def test(self):
     try:
         status = 0
         cluster = Cluster(contact_points = self.hosts, default_retry_policy = RetryPolicy())
         if self.db == "job":
             session = cluster.connect("mgrast_abundance")
         elif self.db == "m5nr":
             session = cluster.connect("m5nr_v1")
         if session:
             status = 1
         cluster.shutdown()
         return status
     except:
         return 0
Esempio n. 18
0
    def test_unavailable(self):
        """
        Use the same tests for test_write_timeout, but ensure they only RETHROW
        """
        policy = RetryPolicy()

        retry, consistency = policy.on_unavailable(
            query=None, consistency=ONE,
            required_replicas=1, alive_replicas=2, retry_num=1)
        self.assertEqual(retry, RetryPolicy.RETHROW)
        self.assertEqual(consistency, None)

        retry, consistency = policy.on_unavailable(
            query=None, consistency=ONE,
            required_replicas=1, alive_replicas=2, retry_num=0)
        self.assertEqual(retry, RetryPolicy.RETHROW)
        self.assertEqual(consistency, None)

        retry, consistency = policy.on_unavailable(
            query=None, consistency=ONE,
            required_replicas=10000, alive_replicas=1, retry_num=0)
        self.assertEqual(retry, RetryPolicy.RETHROW)
        self.assertEqual(consistency, None)
Esempio n. 19
0
    def test_init(self):
        # just make sure Statement attributes are accepted
        kwargs = {'query_string': object(),
                  'retry_policy': RetryPolicy(),
                  'consistency_level': object(),
                  'fetch_size': object(),
                  'keyspace': object(),
                  'custom_payload': object()}
        statement = SimpleGraphStatement(**kwargs)
        for k, v in kwargs.items():
            self.assertIs(getattr(statement, k), v)

        # but not a bogus parameter
        kwargs['bogus'] = object()
        self.assertRaises(TypeError, SimpleGraphStatement, **kwargs)
Esempio n. 20
0
 def test_no_legacy_with_profile(self):
     cluster_init = Cluster(execution_profiles={'name': ExecutionProfile()})
     cluster_add = Cluster()
     cluster_add.add_execution_profile('name', ExecutionProfile())
     # for clusters with profiles added either way...
     for cluster in (cluster_init, cluster_init):
         # don't allow legacy parameters set
         for attr, value in (('default_retry_policy', RetryPolicy()),
                             ('load_balancing_policy', default_lbp_factory())):
             self.assertRaises(ValueError, setattr, cluster, attr, value)
         session = Session(cluster, hosts=[Host("127.0.0.1", SimpleConvictionPolicy)])
         for attr, value in (('default_timeout', 1),
                             ('default_consistency_level', ConsistencyLevel.ANY),
                             ('default_serial_consistency_level', ConsistencyLevel.SERIAL),
                             ('row_factory', tuple_factory)):
             self.assertRaises(ValueError, setattr, session, attr, value)
    def _get_session(self) -> Session:

        if self._username and self._password:
            auth_provider = PlainTextAuthProvider(username=self._username,
                                                  password=self._password)
        else:
            auth_provider = None

        cluster = Cluster(
            contact_points=self._contact_points,
            port=self._port,
            # load_balancing_policy=
            reconnection_policy=ConstantReconnectionPolicy(delay=5.0),
            default_retry_policy=RetryPolicy(),
            auth_provider=auth_provider)
        session = cluster.connect()
        session.set_keyspace('nexustiles')
        return session
Esempio n. 22
0
    def test_setting_retry_policy_to_statement(self):
        """
        Test to verify that the retry policy is called as expected
        for bound and prepared statements when set to the prepared statement

        @since 3.13
        @jira_ticket PYTHON-861
        @expected_result the appropriate retry policy is called

        @test_category connection
        """
        retry_policy = RetryPolicy()
        self.set_cluster(retry_policy)
        then = {
            "result": "write_timeout",
            "delay_in_ms": 0,
            "consistency_level": "LOCAL_QUORUM",
            "received": 1,
            "block_for": 2,
            "write_type": "SIMPLE",
            "ignore_on_prepare": True
        }
        query_to_prime_prepared = "SELECT * from simulacron_keyspace.simulacron_table WHERE key = :key"
        when = {"params": {"key": "0"}, "param_types": {"key": "ascii"}}
        prime_query(query_to_prime_prepared,
                    when=when,
                    then=then,
                    rows=None,
                    column_types=None)

        counter_policy = CounterRetryPolicy()
        prepared_stmt = self.session.prepare(query_to_prime_prepared)
        prepared_stmt.retry_policy = counter_policy
        self.session.execute(prepared_stmt, ("0", ))
        self.assertEqual(next(counter_policy.write_timeout), 1)

        counter_policy.reset_counters()
        bound_stmt = prepared_stmt.bind({"key": "0"})
        bound_stmt.retry_policy = counter_policy
        self.session.execute(bound_stmt)
        self.assertEqual(next(counter_policy.write_timeout), 1)
Esempio n. 23
0
    def execute(self):
        # Connect to cassandra server
        cluster = Cluster(
            load_balancing_policy=TokenAwarePolicy(DCAwareRoundRobinPolicy()),
            default_retry_policy=RetryPolicy())
        session = cluster.connect('cs4224')
        session.default_consistency_level = DEFAULT_CONSISTENCY_LEVEL
        session.execute.im_self.default_timeout = 1000000000000000

        # Reading transactions line by line, parsing and execute
        while True:
            # Break if interrupted by user
            try:
                line = sys.stdin.readline().strip()
            except KeyboardInterrupt:
                break

            # Break on empty line
            if not line:
                break

            # Count transaction
            self.stats_collector.transactions.count()

            # Parsing transaction
            transaction_type = self.parser.get_transaction_type(line)
            extra_line_cnt = self.parser.get_transaction_extra_line_count(transaction_type, line)
            extra_lines = []
            for i in range(extra_line_cnt):
                extra_line = sys.stdin.readline().strip()
                extra_lines.append(extra_line)
            transaction_params = self.parser.parse(line, transaction_type, extra_lines)

            # Execute transaction and measure time
            self.stats_collector.transaction_timer.start()
            self.execute_transaction(session, transaction_type, transaction_params)
            self.stats_collector.transaction_timer.finish()

        self.output()
Esempio n. 24
0
    def setUp(self):
        # create bogus json
        self.filename = 'testConfig.json'

        # get current dir
        self.cur_dir = dirname(abspath(__file__))

        # load config
        self.config = self.loadConfigFile(self.filename)

        # create cassandra structure
        cassandra = self.config['cassandra']
        clus = Cluster(cassandra['url'], default_retry_policy=RetryPolicy())
        sess = clus.connect()
        sess.row_factory = dict_factory
        sess.default_timeout = None

        # save for later
        cas = {
            'cluster': clus,
            'session': sess,
            'keyspace': cassandra['keyspace']
        }
        self.cassandra = cas
        self.data_amt = 10000
        self.idList = self._generateIDs(self.data_amt)

        # connect to elasticsearch and create the data
        self.elastic = {'session': Elasticsearch()}

        # reset data
        self.resetData()
        sql = ''' CREATE KEYSPACE %s
            WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }''' \
              % cassandra['keyspace']
        sess.execute(sql)
Esempio n. 25
0
def create(hosts):
    global CASS_CLUSTER
    if not CASS_CLUSTER:
        CASS_CLUSTER = Cluster(contact_points = hosts, default_retry_policy = RetryPolicy())
    return CASS_CLUSTER
Esempio n. 26
0
def main(args):
    parser = OptionParser()
    parser.add_option("",
                      "--batch",
                      dest="batch",
                      type="int",
                      default=100,
                      help="query batch size, default: 100")
    parser.add_option("",
                      "--count",
                      dest="count",
                      type="int",
                      default=10,
                      help="number of query iterations, default: 10")
    parser.add_option("",
                      "--phost",
                      dest="phost",
                      default=None,
                      help="Postgres host")
    parser.add_option("",
                      "--pname",
                      dest="pname",
                      default=None,
                      help="Postgres name")
    parser.add_option("",
                      "--puser",
                      dest="puser",
                      default=None,
                      help="Postgres user")
    parser.add_option("",
                      "--chost",
                      dest="chost",
                      default=None,
                      help="Cassandra host")
    parser.add_option("",
                      "--cname",
                      dest="cname",
                      default=None,
                      help="Cassandra name")
    parser.add_option("",
                      "--shost",
                      dest="shost",
                      default=None,
                      help="SOLR host")
    parser.add_option("",
                      "--sname",
                      dest="sname",
                      default=None,
                      help="SOLR collection")
    (opts, args) = parser.parse_args()

    handle = None
    mode = ""

    # test PSQL
    if opts.phost and opts.pname and opts.puser:
        mode = "sql"
        handle = psycopg2.connect(host=opts.phost,
                                  database=opts.pname,
                                  user=opts.puser)
    # test CASS
    elif opts.chost and opts.cname:
        mode = "cql"
        handle = Cluster(contact_points=[opts.chost],
                         default_retry_policy=RetryPolicy())
    # test solr
    elif opts.shost and opts.sname:
        mode = "solr"
        surl = "http://" + opts.shost + "/solr/" + opts.sname + "/select"
        shead = {'Content-Type': 'application/x-www-form-urlencoded'}
        sfields = "%2C".join(
            ['md5_id', 'source', 'md5', 'accession', 'function', 'organism'])
    else:
        parser.error("Invalid usage")

    start = time.time()
    found = set()
    for i in range(opts.count):
        ints = random_array(opts.batch)
        query = "SELECT * FROM md5_id_annotation WHERE id IN (" + ",".join(
            map(str, ints)) + ");"
        if mode == "sql":
            cursor = handle.cursor()
            cursor.execute(query)
            data = cursor.fetchone()
            while (data):
                found.add(data[2])
                data = cursor.fetchone()
        elif mode == "cql":
            session = handle.connect(opts.cname)
            rows = session.execute(query)
            for r in rows:
                found.add(r.md5)
        elif mode == "solr":
            query = "md5_id:(" + " OR ".join(map(str, ints)) + ")"
            sdata = "q=*%3A*&fq=" + query + "&start=0&rows=1000000000&wt=json&fl=" + sfields
            req = requests.post(surl,
                                headers=shead,
                                data=sdata,
                                allow_redirects=True)
            rj = req.json()
            for d in rj['response']['docs']:
                found.add(d['md5'])

    end = time.time()
    print "%d loops of size %d ran in %d seconds" % (opts.count, opts.batch,
                                                     int(round(end - start)))
    print "%d ids requested, %d md5s found" % (
        (opts.count * opts.batch), len(found))
Esempio n. 27
0
def get_retry():
    return RetryPolicy()
Esempio n. 28
0
 def __init__(self, export_process):
     RetryPolicy.__init__(self)
     self.max_attempts = export_process.csv_options['maxattempts']
     self.printmsg = lambda txt: export_process.printmsg(txt)
                #                 theState=gpsStationState[2] # yellow
                theState = 2
        #         No data is available on selected date for this station, and station is
        #         not within either the 1 day or 30 day window.
        elif (dataOnDate == False):
            #             theState=gpsStationState[3]  # light blue
            theState = 3

    return theState


#########SAVE TO CASSANDRA DB###########
cluster = Cluster(contact_points=['127.0.0.1'], \
                  port=9042, \
                  load_balancing_policy= TokenAwarePolicy(DCAwareRoundRobinPolicy(local_dc='datacenter1')), \
                  default_retry_policy = RetryPolicy()
                 )
session = cluster.connect('system')
rows = cluster.metadata.keyspaces.keys()

KEYSPACE = 'GPS_' + dataSet
KEYSPACE = KEYSPACE.lower()

# Create database
keyspace_drop_stmt = session.prepare("DROP KEYSPACE " + KEYSPACE)

if KEYSPACE in rows:
    print("dropping existing keyspace...")
    session.execute(keyspace_drop_stmt)

print("creating keyspace...")
Esempio n. 30
0
 def __init__(self, export_process):
     RetryPolicy.__init__(self)
     self.max_attempts = export_process.csv_options['maxattempts']
     self.printmsg = lambda txt: export_process.printmsg(txt)