Beispiel #1
0
    def refresh_connection(self,
                           host=None,
                           port=None,
                           cql_version="2.0.0",
                           num_retries=10):
        """
        establish a connection to the server. retry if needed.
        """
        if host:
            self._host = host
        if port:
            self._port = port
        self._cql_version = cql_version or None

        for try_num in xrange(1 + num_retries):
            try:
                if self._cql_version:
                    con = cql.connect(self._host,
                                      self._port,
                                      keyspace=None,
                                      cql_version=self._cql_version)
                else:
                    con = cql.connect(self._host, self._port, keyspace=None)
                self._cached_cursor = con.cursor()
                if self._is_keyspace_created:
                    cql_str = "USE %s" % self._params['keyspace_name']
                    self.execute_query(cql_str)
            except:
                if try_num < num_retries:
                    time.sleep(1)
                else:
                    raise
 def refresh_connection(self, host=None, port=None, cql_version="2.0.0", num_retries=10):
     """
     establish a connection to the server. retry if needed.
     """
     if host:
         self._host = host
     if port:
         self._port = port
     self._cql_version = cql_version or None
         
     for try_num in xrange(1+num_retries):
         try:
             if self._cql_version:
                 con = cql.connect(self._host, self._port, keyspace=None, cql_version=self._cql_version)
             else:
                 con = cql.connect(self._host, self._port, keyspace=None)
             self._cached_cursor = con.cursor()
             if self._is_keyspace_created:
                 cql_str = "USE %s" % self._params['keyspace_name']
                 self.execute_query(cql_str)
         except:
             if try_num < num_retries:
                 time.sleep(1)
             else:
                 raise
Beispiel #3
0
    def cql_connection(self,
                       node,
                       keyspace=None,
                       version=None,
                       user=None,
                       password=None):
        import cql
        host, port = node.network_interfaces['thrift']
        if not version and self.cluster.version() >= "1.2":
            version = "3.0.0"
        elif not version and self.cluster.version() >= "1.1":
            version = "2.0.0"

        if version:
            con = cql.connect(host,
                              port,
                              keyspace=keyspace,
                              cql_version=version,
                              user=user,
                              password=password)
        else:
            con = cql.connect(host,
                              port,
                              keyspace=keyspace,
                              user=user,
                              password=password)
        self.connections.append(con)
        return con
Beispiel #4
0
 def cql_connection(self, node, keyspace=None, version=None):
     import cql
     host, port = node.network_interfaces['thrift']
     if version:
         con = cql.connect(host, port, keyspace=keyspace, cql_version=version)
     else:
         con = cql.connect(host, port, keyspace=keyspace)
     self.connections.append(con)
     return con
 def test_connecting_with_keyspace(self):
     # this conn is just for creating the keyspace
     conn = cql.connect(TEST_HOST, TEST_NATIVE_PORT, native=True)
     curs = conn.cursor()
     with self.with_keyspace(curs, conn.cql_version) as ksname:
         curs.execute('create table blah1_%s (a int primary key, b int);'
                      % self.randstr)
         conn2 = cql.connect(TEST_HOST, TEST_NATIVE_PORT, keyspace=ksname,
                             native=True, cql_version=conn.cql_version)
         curs2 = conn2.cursor()
         curs2.execute('select * from blah1_%s;' % self.randstr)
         conn2.close()
Beispiel #6
0
    def cql_connection(self, node, keyspace=None, version=None):
        import cql

        host, port = node.network_interfaces["thrift"]
        if not version and self.cluster.version() >= "1.2":
            version = "3.0.0"

        if version:
            con = cql.connect(host, port, keyspace=keyspace, cql_version=version)
        else:
            con = cql.connect(host, port, keyspace=keyspace)
        self.connections.append(con)
        return con
Beispiel #7
0
    def cql_connection(self, node, keyspace=None, version=None, user=None, password=None):
        import cql
        host, port = node.network_interfaces['thrift']
        if not version and self.cluster.version() >= "1.2":
            version = "3.0.0"
        elif not version and self.cluster.version() >= "1.1":
            version = "2.0.0"

        if version:
            con = cql.connect(host, port, keyspace=keyspace, cql_version=version, user=user, password=password)
        else:
            con = cql.connect(host, port, keyspace=keyspace, user=user, password=password)
        self.connections.append(con)
        return con
Beispiel #8
0
def dump_archives(archives):
    name = path.replace('/opt/graphite/storage/whisper/', '', 1)
    name = name.replace('.wsp', '', 1)
    name = name.replace('/', '.')
    con = cql.connect('127.0.0.1', 9160, 'metric', cql_version='3.0.0')
    print("Connected to Cassandra!")
    cursor = con.cursor()
    for i, archive in enumerate(archives):
        print 'Archive %d data:' % i
        offset = archive['offset']
        for point in xrange(archive['points']):
            (timestamp,
             value) = struct.unpack(whisper.pointFormat,
                                    map[offset:offset + whisper.pointSize])
            print '%d: %d, %10.35g' % (point, timestamp, value)
            offset += whisper.pointSize
            period = archive['retention']
            rollup = archive['secondsPerPoint']
            ttl = period
            #CQLString = "UPDATE metric USING TTL ? SET data = data + ? WHERE tenant = '' AND rollup = ? AND period = ? AND path = ? AND time = ?;"
            #cursor.execute(CQLString, [ttl, value, rollup, period, name, timestamp])
            #print CQLString, [ttl, value, rollup, period, name, timestamp]
            if timestamp > 0:
                CQLString = "UPDATE metric USING TTL %d SET data = data + [ %d ] WHERE tenant = '' AND rollup = %d AND period = %d AND path = '%s' AND time = %d;" % (
                    ttl, value, rollup, period, name, timestamp)
                #print CQLString
                cursor.execute(CQLString)
        print
Beispiel #9
0
 def __init__(self, config):
     if not cql:
         raise ImportError("cql is required to use Cassandra as storage.")
     self.name = 'cassandra'
     self.storage = cql.connect(config["host"], config["port"], cql_version='3.0.0')
     cursor = self.storage.cursor()
     cursor.execute("""USE %s""" % config["keyspace"])
Beispiel #10
0
def cql_execute(host, port, filename, force, silent=False):
    try:
        connection = cql.connect(host=host, port=port, cql_version='3.0.0')
    except thrift.transport.TTransport.TTransportException, e:
        if not silent:
            print 'Execution failed: {e}'.format(e=e)
        return False
Beispiel #11
0
    def _create_connection(self):
        """
        Creates a new connection for the connection pool.

        should only return a valid connection that it's actually connected to
        """
        if not self.hosts:
            raise Exception("At least one host required")

        random.shuffle(self.hosts)

        for hoststr in self.hosts:
            host, port = hoststr.split(':')
            try:
                new_conn = cql.connect(host,
                                       int(port),
                                       user=self.username,
                                       password=self.password,
                                       consistency_level=self.consistency)
                new_conn.set_cql_version('3.0.0')
                return new_conn
            except Exception as e:
                logging.exception(
                    "Could not establish connection to {}:{}".format(
                        host, port))

        raise Exception("Could not connect to any server in cluster")
 def __init__(self):
     
     dbconn = cql.connect(config_local.cassandra_host, config_local.cassandra_port ) 
     cursor = dbconn.cursor()
     
     try:
         cursor.execute("USE " + config_global.cassandra_default_keyspace)
         cql_query = """ DROP COLUMNFAMILY :rawJsonDump """
         cursor.execute(cql_query, dict(rawJsonDump = config_global.dbname_rawJson))
     except cql.ProgrammingError as programmingError:
         print cql_query
         print programmingError
     
     try:
         cursor.execute("USE " + config_global.cassandra_default_keyspace)
         cursor.execute(""" CREATE COLUMNFAMILY :rawJsonDump 
         ( incomingTime timestamp PRIMARY KEY, jsonString text ) 
         WITH comment = ' simply the raw input data received from the contest server ' """, 
                        dict(rawJsonDump = config_global.dbname_rawJson))
     except cql.ProgrammingError as programmingError:
         print programmingError
     
     
     """ lets try getting the latest items ordered"""
     cursor.execute("USE " + config_global.cassandra_default_keyspace)
     cursor.execute(""" SELECT FIRST 5 * FROM :rawJsonDump LIMIT 5""", dict(rawJsonDump = config_global.dbname_rawJson))
     
     row = cursor.fetchone()
     while (row):
         print row
         row = cursor.fetchone()
    def __init__(self):
    
        dbconn = cql.connect(config_local.cassandra_host, config_local.cassandra_port) 
        cursor = dbconn.cursor()
        
        try:
            cursor.execute(""" USE :keyspace """, dict(keyspace = config_global.cassandra_default_keyspace))
        except cql.ProgrammingError as programmingError:
            print programmingError
        
        try:    
            cursor.execute(""" DROP COLUMNFAMILY :columnfamily; """, dict(columnfamily = config_global.dbname_dimensionList))
            pass
        except cql.ProgrammingError as programmingError:
            print programmingError
        try:
            cursor.execute("""
            CREATE COLUMNFAMILY :columnfamily 
            (
            dimension_name text PRIMARY KEY 
            )
            WITH comparator = int AND default_validation = int;
        """, 
                dict(columnfamily = config_global.dbname_dimensionList))
        except cql.ProgrammingError as programmingError:
            print programmingError
        
    
        """ lets insert row keys """
        
        for dimension in config_global.dbname_dimensionList_rowKeys:
            
            try: 
                
                for dimension_value in xrange(3):
                    
                    cursor.execute("""INSERT INTO :table ( dimension_name, :dimension_value ) VALUES 
                    ( :dimension_name, :dimension_value) USING TTL 10 """, 
                                        dict(table=config_global.dbname_dimensionList,
                                             dimension_name = dimension,
                                             dimension_value = dimension_value) 
                                       )
                    
            except cql.ProgrammingError as programmingError:
                print programmingError

             
            """ now we have to check if a dimension is already known """
            try:
                for test_id in xrange(1):
                       
                    cursor.execute("""SELECT FIRST 3 REVERSED * FROM :table WHERE dimension_name = :dimension_name""", 
                                           dict(table=config_global.dbname_dimensionList,
                                                dimension_name = dimension
                                                ) 
                                          )
                    print cursor.fetchall()
                    
            except cql.ProgrammingError as programmingError:
                print programmingError
Beispiel #14
0
    def _create_connection(self):
        """
        Creates a new connection for the connection pool.

        should only return a valid connection that it's actually connected to
        """
        if not self.hosts:
            raise Exception("At least one host required")

        random.shuffle(self.hosts)

        for hoststr in self.hosts:
            host, port = hoststr.split(':')
            try:
                new_conn = cql.connect(
                    host,
                    int(port),
                    user=self.username,
                    password=self.password,
                    consistency_level=self.consistency
                )
                new_conn.set_cql_version('3.0.0')
                return new_conn
            except Exception as e:
                logging.exception("Could not establish connection to {}:{}".format(host, port))

        raise Exception("Could not connect to any server in cluster")
 def testSynteticDataGet_filter_b(self):
     print "filter_b"
     dbconn2 = cql.connect(config_local.cassandra_host, config_local.cassandra_port )
 
     cursorb = dbconn2.cursor()
     cursorb.execute("USE plistaContest")
     
     startTime = time.time()
     
     count = 300
     for n in xrange(count):
         filter_b_val = int( random.uniform(1,100) )
         cursorb.execute("""SELECT * FROM itemTableFull WHERE 
                         filter_b = :filter_b_val
                         LIMIT 4""", 
                         dict(filter_b_val = filter_b_val))
         resultset = cursorb.fetchall()
     endTime = time.time() - startTime
     print "endTime: \t" + str(endTime)
     
     print "requests per seconds:\t" + str ( count / endTime)
     
     resultset = cursorb.fetchall()
     #print resultset
     #print len(resultset)
     print "\n\n"
Beispiel #16
0
def cql_execute(host, port, filename, force, silent=False):
    try:
        connection = cql.connect(host=host, port=port, cql_version='3.0.0')
    except thrift.transport.TTransport.TTransportException, e:
        if not silent:
            print 'Execution failed: {e}'.format(e=e)
        return False
Beispiel #17
0
    def _create_connection(self):
        """
        Creates a new connection for the connection pool.

        should only return a valid connection that it's actually connected to
        """
        if not self._hosts:
            raise CQLConnectionError("At least one host required")

        hosts = copy(self._hosts)
        random.shuffle(hosts)

        for host in hosts:
            try:
                new_conn = cql.connect(host.name,
                                       host.port,
                                       user=self._username,
                                       password=self._password,
                                       consistency_level=self._consistency)
                new_conn.set_cql_version('3.0.0')
                return new_conn
            except Exception as e:
                logging.debug("Could not establish connection to {}:{}".format(
                    host.name, host.port))
                pass

        raise CQLConnectionError("Could not connect to any server in cluster")
Beispiel #18
0
def truncateCf():
    con = cql.connect('10.1.2.91', 9160, "apixio", cql_version='3.0.0')
    print("Connected!")
    cursor = con.cursor()
    CQLString = "TRUNCATE store457;"
    #cursor.execute(CQLString)
    pass
Beispiel #19
0
    def _create_connection(self):
        """
        Creates a new connection for the connection pool.

        should only return a valid connection that it's actually connected to
        """
        if not self._hosts:
            raise CQLConnectionError("At least one host required")

        hosts = copy(self._hosts)
        random.shuffle(hosts)

        for host in hosts:
            try:
                transport = self._create_transport(host)
                new_conn = cql.connect(
                    host.name,
                    host.port,
                    user=self._username,
                    password=self._password,
                    consistency_level=self._consistency,
                    transport=transport
                )
                new_conn.set_cql_version('3.0.0')
                return new_conn
            except Exception as e:
                logging.debug("Could not establish connection to {}:{}".format(host.name, host.port))
                pass

        raise CQLConnectionError("Could not connect to any server in cluster")
Beispiel #20
0
def status(name):
    try:
        conn = connect(host=cassandra_server, port=cassandra_port)
        cursor = conn.cursor()
        cql_command = "USE {0}".format(name)
        cursor.execute(cql_command)
    except (ProgrammingError, TTransportException), e:
        return e.message, 500
    def validate_schema_consistent(self, node):
        """ Makes sure that there is only one schema """

        host, port = node.network_interfaces['thrift']
        conn = cql.connect(host, port, keyspace=None)
        schemas = conn.client.describe_schema_versions()
        num_schemas = len(schemas)
        assert num_schemas == 1, "There were multiple schema versions: " + pprint.pformat(schemas)
def setUpModule():
    '''Remove the side effects of previous modules in homer'''
    try:
        global connection
        connection = cql.connect("localhost", 9160).cursor()
        connection.execute("DROP KEYSPACE Test;")
    except Exception as e:
        print e
Beispiel #23
0
 def get_connection(self, host, port=9160):
     """ get connection to specified host """
     conn = cql.connect(host,
                        port,
                        user=self.username,
                        password=self.password,
                        consistency_level='ONE')
     conn.set_cql_version('3.0.0')
     return conn
    def validate_schema_consistent(self, node):
        """ Makes sure that there is only one schema """
        debug("validate_schema_consistent() " + node.name)

        host, port = node.network_interfaces['thrift']
        conn = cql.connect(host, port, keyspace=None)
        schemas = conn.client.describe_schema_versions()
        num_schemas = len([ss for ss in schemas.keys() if ss != 'UNREACHABLE'])
        assert num_schemas == 1, "There were multiple schema versions: " + pprint.pformat(schemas)
 def connect(self):
     """Connect to the two databases
     self.connection is for NGram Corpus on Cassandra
     self.connection2 is for UMLS on SQLServer 2008
     """
     ## This is the cassandra database with the 4-gram Google Web Corpus
     
     self.connection = cql.connect('127.0.0.1', 9160, "fourgm", cql_version = '3.0.0')
     self.cursor = self.connection.cursor()
    def connect(self):
        """Connect to the two databases
        self.connection is for NGram Corpus on Cassandra
        self.connection2 is for UMLS on SQLServer 2008
        """
        self.connection = cql.connect('127.0.0.1', 9160, "fourgm", cql_version = '3.0.0')
        self.cursor = self.connection.cursor()

        self.connection2 = pyodbc.connect('DRIVER={SQL Server};SERVER=XXX.XXX.XXX.XXX;DATABASE=umlsSmall;UID=XXXXXX;PWD=XXXXXXXXX')
        self.cursor2=self.connection2.cursor()
Beispiel #27
0
 def test_execution_fails_after_close(self):
     conn = cql.connect(TEST_HOST, TEST_NATIVE_PORT, native=True)
     curs = conn.cursor()
     with self.with_keyspace(curs, conn.cql_version) as ksname:
         curs.execute('use "%s"' % ksname)
         curs.execute('create table blah (a int primary key, b int);')
         curs.execute('select * from blah;')
     conn.close()
     self.assertRaises(cql.ProgrammingError, curs.execute,
                       'select * from blah;')
Beispiel #28
0
    def validate_schema_consistent(self, node):
        """ Makes sure that there is only one schema """
        debug("validate_schema_consistent() " + node.name)

        host, port = node.network_interfaces['thrift']
        conn = cql.connect(host, port, keyspace=None)
        schemas = conn.client.describe_schema_versions()
        num_schemas = len([ss for ss in schemas.keys() if ss != 'UNREACHABLE'])
        assert num_schemas == 1, "There were multiple schema versions: " + pprint.pformat(
            schemas)
Beispiel #29
0
    def setUp(self):
        try:
            self.dbconn = cql.connect(TEST_HOST, TEST_PORT, cql_version=TEST_CQL_VERSION)
        except cql.thrifteries.TApplicationException:
            # set_cql_version (and thus, cql3) not supported; skip all of these
            self.cursor = None
            return

        self.cursor = self.dbconn.cursor()
        self.keyspace = self.create_schema()
	def __init__(self):
	
		dbconn = cql.connect(config_local.cassandra_host, config_local.cassandra_port) 
		cursor = dbconn.cursor()
		
		try:
			cursor.execute(""" USE :keyspace """, dict(keyspace = config_global.cassandra_default_keyspace))
		except cql.ProgrammingError as programmingError:
			print programmingError
		
		try:	
			cursor.execute(""" DROP COLUMNFAMILY :columnfamily; """, dict(columnfamily = config_global.dbname_recommendationsList))
			pass
		except cql.ProgrammingError as programmingError:
			print programmingError
		try:
			cursor.execute("""
			CREATE COLUMNFAMILY :columnfamily 
			(
			user_id text PRIMARY KEY 
			)
			WITH comparator = int AND default_validation = int;
		""", 
				dict(columnfamily = config_global.dbname_recommendationsList))
		except cql.ProgrammingError as programmingError:
			print programmingError
		
	
		""" lets insert row keys """
		
		for user_id in xrange(5):
			for item_id in xrange(5):
				try: 
					cursor.execute("""INSERT INTO :table ( user_id, :item_id ) VALUES 
						( :user_id, :score ) USING TTL 10 """, 
											dict(table = config_global.dbname_recommendationsList,
												user_id = user_id,
												item_id = item_id,
												score = 5 - item_id ) 
										   )
						
				except cql.ProgrammingError as programmingError:
					print programmingError

			 
		try:

			cursor.execute("""SELECT FIRST 3 REVERSED * FROM :table WHERE user_id = 2""", 
									   dict(table=config_global.dbname_recommendationsList
											) 
									  )
			print cursor.fetchone()
				
		except cql.ProgrammingError as programmingError:
			print programmingError
Beispiel #31
0
def get_or_create():
    connection = get()
    if connection == None:
        _log.info("Connecting to Cassandra on %s", settings.CASS_HOST)
        connection = cql.connect(settings.CASS_HOST,
                                 settings.CASS_PORT,
                                 settings.CASS_KEYSPACE,
                                 cql_version='2.0.0')
        assert connection
        thread_local.connection = connection
    return connection
 def __create_connection(self):
     return cql.connect(
         choice(self.hostname),
         port=self.port,
         keyspace=self.keyspace,
         user=self.user,
         password=self.password,
         cql_version=self.cql_version,
         compression=self.compression,
         consistency_level=self.consistency_level,
         transport=self.transport)
 def connect(self):
     """Connect to the two databases
     self.connection is for NGram Corpus on Cassandra
     self.connection2 is for UMLS on SQLServer 2008
     """
     ##self.connection = pyodbc.connect('DRIVER={SQL Server};Trusted_Connection=yes;SERVER=WIN-20IR78KE8UV\SQLSERVER2012;DATABASE=GoogleCorpus;UID=Administrator')
     ##self.cursor = self.connection.cursor()
     ## This is the cassandra database with the 4-gram Google Web Corpus
     
     self.connection = cql.connect('127.0.0.1', 9160, "fourgm", cql_version = '3.0.0')
     self.cursor = self.connection.cursor()
Beispiel #34
0
def get_or_create():
    connection = get()
    if connection == None:
        _log.info("Connecting to Cassandra on %s", settings.CASS_HOST)
        connection = cql.connect(settings.CASS_HOST,
                                 settings.CASS_PORT,
                                 settings.CASS_KEYSPACE,
                                 cql_version='2.0.0')
        assert connection
        thread_local.connection = connection
    return connection
 def standard_connection(self):
     """Set up a connection to Cassandra.
     """
     logging.debug('connecting to %s', self.option_group.keyspace)
     self.conn = cql.connect(self.option_group.hostname,
                             self.option_group.port,
                             self.option_group.keyspace,
                             user=self.option_group.username,
                             password=self.option_group.password,
                             cql_version=self.option_group.cqlversion)
     return
 def standard_connection(self):
     """Set up a connection to Cassandra.
     """
     logging.debug('connecting to %s', self.option_group.keyspace)
     self.conn = cql.connect(self.option_group.hostname,
                             self.option_group.port,
                             self.option_group.keyspace,
                             user=self.option_group.username,
                             password=self.option_group.password,
                             cql_version=self.option_group.cqlversion)
     return
Beispiel #37
0
 def test_connecting_with_compression(self):
     try:
         import snappy
     except ImportError:
         if hasattr(unittest, 'skipTest'):
             unittest.skipTest('Snappy compression not available')
         else:
             return
     conn = cql.connect(TEST_HOST, TEST_NATIVE_PORT, native=True,
                        compression=True)
     self.assertEqual(conn.compressor, snappy.compress)
     self.try_basic_stuff(conn)
Beispiel #38
0
def connect_cql():
    con = None
    cursor = None
    try:
        con = cql.connect('localhost', 9160, 'fav_test', cql_version='3.0.0')
        print("Connected!")
        cursor = con.cursor()

        yield cursor

    except Exception, e:
        print "Error occurred: " + str(e)
Beispiel #39
0
def my_view(request):
    now = datetime.datetime.now()
    fp = open('/Users/Del/Desktop/mushrooms/mushrooms/media/mushroomsWeb/index.html')
    t = Template(fp.read())
    fp.close()
    con = cql.connect('localhost', 9160,'demo', cql_version='3.0.0')
    cursor = con.cursor()
    #CQLString = "create columnfamily t1 (okey int primary key, observer varchar, image varchar)"
    #x=cursor.execute(CQLString)
    
    html = t.render(Context({'current_date': now}))
    return HttpResponse(html)
Beispiel #40
0
def get_or_create(keyspace):
    connection = thread_local.connections.get(keyspace)

    if not connection:
        _log.info("Connecting to Cassandra on %s", settings.CASS_HOST)
        connection = cql.connect(settings.CASS_HOST,
                                 settings.CASS_PORT,
                                 keyspace,
                                 cql_version='2.0.0')
        assert connection
        thread_local.connections[keyspace] = connection
    return connection
Beispiel #41
0
def connect_cql():
    con = None
    cursor = None
    try:
        con = cql.connect('localhost', 9160, 'fav_test', cql_version='3.0.0')
        print ("Connected!")
        cursor = con.cursor()
        
        yield cursor;
    
    except Exception, e:
        print "Error occurred: " + str(e)
Beispiel #42
0
 def __init__(self, host, port, keyspace):
     try:
         self.host = host
         self.port = port
         self.keyspace = keyspace
         self.transport = tfactory.regular_transport_factory('127.0.0.1', 9160, None, None)
         self.con = cql.connect('127.0.0.1', 9160,  cql_version='3', transport=self.transport) 
         self.cursor = self.con.cursor()
         self.cursor.execute('USE ' + self.keyspace + ' ;');
     except cql.ProgrammingError as e:
         print e.args 
         sys.exit(1)
Beispiel #43
0
def add_instance():
    keyspace = None
    if request.method == 'POST' and 'name' in request.form:
        keyspace = request.form.get('name')
        try:
            conn = connect(host=cassandra_server, port=cassandra_port)
            cursor = conn.cursor()
            cql_command = "CREATE KEYSPACE {0} WITH strategy_class = 'SimpleStrategy' AND strategy_options:replication_factor=3;"
            cursor.execute(cql_command.format(keyspace))
        except (ProgrammingError, TTransportException), e:
            return e.message, 500
        return "", 201
def connect_to_cassandra():
    """
    Connect to Cassandra host
    """
    conn = cql.connect(host=c.host,
                       port=c.port,
                       keyspace=c.keyspace,
                       user=c.user,
                       password=c.password,
                       cql_version=c.cql_version)

    logging.info("cassandra_lib connected")
    return conn
Beispiel #45
0
  def url_parse(self, url, **kwargs):
    location = urlparse(url)
    if location.scheme in ('cassandra','cql'):
      host = location.netloc or "localhost:9160"
      if re.search(":[0-9]+$", host):
        ip,port = host.split(':')
      else:
        ip = host
        port = 9160

      keyspace = location.path[1:] or kwargs.get('database', 'kairos')
      if '?' in keyspace:
        keyspace,params = keyspace.split('?')

      return cql.connect(ip, int(port), keyspace, cql_version='3.0.0', **kwargs)
Beispiel #46
0
    def exequteCQL(self, query):
        # establish the cql connection
        if self.cql_cursor == None:
            self.cql_conn = cql.connect(self.host, self.port, self.keyspace)
            self.cql_cursor = self.cql_conn.cursor()

        topics = self.getAllTopics()

        # rename topics to column names ...
        for topic in topics:
            query = query.replace('"' + topic + '"',
                                  '"' + self.topic2Hash(topic) + '"')
        print query

        #return query
        # execute querie
        self.cql_cursor.execute(query)
        return self.cql_cursor.fetchall()
Beispiel #47
0
def connect(keyspace=None, use_cache=True, user=None, password=None):
    key = (keyspace, user)
    if keyspace is None:
        keyspace = KEYSPACE
    if use_cache and pool.has_key(key):
        return pool[key]
    for i in range(len(NODES)):
        try:
            node = get_node()
            con = cql.connect(node,
                              keyspace=keyspace,
                              cql_version='3.0.0',
                              user=user,
                              password=password)
            print "Connected to %s" % node
            pool[key] = con
            return con
        except Exception, msg:
            print msg  # TODO -- logger
Beispiel #48
0
 def _connection(self):
     '''
 Return a connection from the pool
 '''
     try:
         return self._pool.get(False)
     except Empty:
         args = [self._host, self._port, self._keyspace]
         kwargs = {
             'user': None,
             'password': None,
             'cql_version': self._cql_version,
             'compression': self._compression,
             'consistency_level': self._consistency_level,
             'transport': self._transport,
         }
         if self._credentials:
             kwargs['user'] = self._credentials['user']
             kwargs['password'] = self._credentials['password']
         return cql.connect(*args, **kwargs)
    def create_schema(self):
        """Creates the schema if it doesn't exist using the CQL in self.SCHEMA.
        Each query in there will be formatted with locals(), so if you
        update self.SCHEMA, be sure to update this function, too.
        """
        logging.info('creating schema')
        self.conn = cql.connect(self.option_group.hostname,
                                self.option_group.port,
                                "system",
                                user=self.option_group.username,
                                password=self.option_group.password,
                                cql_version=self.option_group.cqlversion)

        data_center = self.query_or_die(
            self.SELECT_ALL_DATACENTERS,
            "Unable to determine the local data center")
        if not data_center:
            logging.fatal(
                "No peers defined, repairs on a single-node cluster are silly")
            exit(0)

        # Cassandra doesn't support 'SELECT foo, 1 FROM ..." or DISTINCT,
        # so we have do something a little complicated to deduplicate the
        # results and then produce the desired string.
        data_center_replication_map = {}
        for row in data_center:
            data_center_replication_map[row[0]] = None
        data_center_replication_map = ", ".join(
            ["'%s':3" % x for x in data_center_replication_map])

        # This declaration is just so that "keyspace" will appear in locals.
        # pylint: disable=unused-variable
        keyspace = self.option_group.keyspace
        # pylint: enable=unused-variable
        for cql_query in self.SCHEMA:
            self.query(cql_query.format(**locals()))
        return
Beispiel #50
0
def query():
    try:

        con = cql.connect(host='114.215.156.15',
                          keyspace="hw2_1452739",
                          cql_version='3.0.0')
        cursor = con.cursor()
        cursor.execute("select * from test")
        list = cursor.result

        #cluster = Cluster(['114.215.156.15', ])
        #session = cluster.connect()
        #session.execute("use hw2_1452739;")
        #cqlString = "SELECT * from test"
        # #list = session.execute(cqlString)
        answerArray = analysis(list)

        for each in answerArray:
            '''print with out enter'''
            print each,

    except Exception, exce:
        print '!!!!!'
        print exce
    def functional_test(self):
        NUM_SUBCOLS = 100
        NUM_ADDS = 100

        cluster = self.cluster
        cluster.populate(3).start()
        node1 = cluster.nodelist()[0]

        time.sleep(.5)
        cursor = self.patient_cql_connection(node1).cursor()
        self.create_ks(cursor, 'ks', 3)
        time.sleep(1)  # wait for propagation

        # create the columnfamily using thrift
        host, port = node1.network_interfaces['thrift']
        thrift_conn = cql.connect(host, port, keyspace='ks')
        cf_def = CfDef(keyspace='ks',
                       name='cf',
                       column_type='Super',
                       default_validation_class='CounterColumnType')
        thrift_conn.client.system_add_column_family(cf_def)

        # let the sediment settle to to the bottom before drinking...
        time.sleep(2)

        for subcol in xrange(NUM_SUBCOLS):
            for add in xrange(NUM_ADDS):
                column_parent = ColumnParent(column_family='cf',
                                             super_column='subcol_%d' % subcol)
                counter_column = CounterColumn('col_0', 1)
                thrift_conn.client.add('row_0', column_parent, counter_column,
                                       ConsistencyLevel.QUORUM)
        time.sleep(1)

        # flush everything and the problem will be mostly corrected.
        #        for node in cluster.nodelist():
        #            node.flush()

        debug("Stopping cluster")
        cluster.stop()
        time.sleep(5)
        debug("Starting cluster")
        cluster.start()
        time.sleep(5)

        thrift_conn = cql.connect(host, port, keyspace='ks')

        from_db = []

        for i in xrange(NUM_SUBCOLS):
            column_path = ColumnPath(column_family='cf',
                                     column='col_0',
                                     super_column='subcol_%d' % i)
            column_or_super_column = thrift_conn.client.get(
                'row_0', column_path, ConsistencyLevel.QUORUM)
            val = column_or_super_column.counter_column.value
            debug(str(val)),
            from_db.append(val)
        debug("")

        expected = [NUM_ADDS for i in xrange(NUM_SUBCOLS)]

        if from_db != expected:
            raise Exception(
                "Expected a bunch of the same values out of the db. Got this: "
                + str(from_db))
Beispiel #52
0
            h = resp.headers

            h['Access-Control-Allow-Origin'] = origin
            h['Access-Control-Allow-Methods'] = get_methods()
            h['Access-Control-Max-Age'] = str(max_age)
            if headers is not None:
                h['Access-Control-Allow-Headers'] = headers
            return resp

        f.provide_automatic_options = False
        return update_wrapper(wrapped_function, f)
    return decorator

# connect to database
con = cql.connect('ec2-54-187-166-118.us-west-2.compute.amazonaws.com',
                '9160', 'janusz_forex_rt_demo', cql_version='3.1.1')
cursor = con.cursor()

tasks = {
	'AUDUSD':0,
	'EURUSD':0,
	'GBPUSD':0,
	'NZDUSD':0,
	'USDCAD':0,
	'USDCHF':0,
	'USDJPY':0
	}

pairs       = ['AUDUSD', 'EURUSD', 'GBPUSD', 'NZDUSD', 'USDCAD', 'USDCHF', 'USDJPY']

q       = "select issued_at, ask, bid from ticks where pair_day = :key and issued_at > :utc"
Beispiel #53
0
def main(argv):
    usage = "Usage: dbcql {query}"

    args, kwargs = parse(argv)

    if len(args) > 1:
        error(output, "Unexpected argument: '%s'" % args[1], 2)

    if len(args) < 1:
        error(output, "No query argument", 2)

    keyspace = kwargs.get('keyspace', None)
    host = kwargs.get('host', settings.DEFAULT_CASSANDRA_HOST)
    port = kwargs.get('port', settings.DEFAULT_CASSANDRA_PORT)
    # UNDONE: credentials ..

    query = args[0]
    if query is None:
        error(output, "Command requires a single query argument", 2)

    # A query may consist of multiple expressions. We execute each of
    # the expressions in order and output the results from the final
    # expression. The primary scenario is:
    #
    #     "USE {keyspace}; SELECT * FROM .."
    #
    # However, arbitrary query expressions may be combined in this way.
    expressions = query.split(';')

    # Scan the expressions looking for anything we need to disable
    for expression in expressions:
        # UNDONE: The following disables usage of the CLQ DROP command
        # in order to prevent users from inadvertently (or maliciously)
        # dropping of critical keyspace data until we can properly enable
        # the capability by integrating with Splunk's role based access
        # control (where this would be an admin-only capability).
        if re.match("\s*drop ", expression, re.I):
            error(output, "The CQL DROP command has been disabled.", 2)

    connection = None
    cursor = None
    try:
        connection = cql.connect(host, port, keyspace)
        cursor = connection.cursor()

        # Everything looks ok .. so run them.
        for expression in expressions:
            cursor.execute(expression)

        # A 'SELECT *' expression requires special handling because the
        # results may be "ragged", meaning that we don't know what fields
        # will appear for any given row. In order to handle this case we
        # scan an initial `batchsize` set of rows, collecting all fields
        # that we see. We then use those fields for the results header and
        # collect any additional fields that we subsequently see into an
        # "_extra" MV field. Note that a row in Cassandra can contain up
        # to 2B columns, so "SELECT *" may have other issues aside from the
        # incrimental processing impact on this search script.

        if re.search("select\s+\*", expression, re.I):
            ragged(cursor)
        else:
            normal(cursor)

    except:
        error(output, excinfo(), 2)

    finally:
        if cursor is not None:
            cursor.close()
        if connection is not None:
            connection.close()
Beispiel #54
0
 def setUp(self):
   self.client = cql.connect('localhost', 9160, os.environ.get('CASSANDRA_KEYSPACE','kairos'), cql_version='3.0.0')
   super(CassandraSetTest,self).setUp()
Beispiel #55
0
import sys
import os
import csv
from datetime import datetime
import time
import cql


# writefile = open('CSULander_Modified.txt', 'w')

con = cql.connect('localhost', cql_version='3.0.0')
print("Connected!")


with open(sys.argv[1], 'r') as data:
  mycsv = csv.reader(data)
	
  # Writing the Header File to the output file
  # writefile.write('Date,TimeBucket,TimeStamp,FromIP,ToIP,PortNumber,DataIn,DataOut\n')	
	
  for row in mycsv:
    day = 1
    month = 12 
    year = 2014
		
    timeBucket_H = 0
    timeBucket_M = 0
    time = 0
    in_pos = 2
    out_pos = 291
Beispiel #56
0
import json
import cql

conn = cql.connect(host="127.0.0.1", port=9160, keyspace="sensordb", cql_version='5.0.1')

cur=conn.cursor()

#===============================================================
# Functions to push Sensor Data into Database

# Function to save Temperature to DB Table
def DHT11_Temp_Data_Handler(jsonData):
    #Parse Data 
    json_Dict = json.loads(jsonData)
    SensorID = json_Dict['Sensor_ID']
    Data_and_Time = json_Dict['Date']
    Temperature = json_Dict['Temperature']
	
    #Push into DB Table
    cur.execute("insert into temperature (SensorID, Date_n_Time, Temperature) VALUES ('SensorID','Data_and_Time','Temperature')")
    cur.close()
    print "Inserted Temperature Data into Database."
    print ""

# Function to save Humidity to DB Table
def DHT11_Humidity_Data_Handler(jsonData):
    #Parse Data 
    json_Dict = json.loads(jsonData)
    SensorID = json_Dict['Sensor_ID']
    Data_and_Time = json_Dict['Date']
    Humidity = json_Dict['Humidity']