def key_exists(key):
logger("key_exists()", key)
c = Connection("127.0.0.1", 3301)
print("key_exists()")
exists = list(c.select("KVstorage", key))
print(exists != [])
return exists != []
def connect_to_tarantool(self, host, port, user=None, password=None, alias=None):
"""
Connection to Tarantool DB.
*Args:*\n
_host_ - host for db connection;\n
_port_ - port for db connection;\n
_user_ - username for db connection;\n
_password_ - password for db connection;\n
_alias_ - connection alias, used for switching between open connections;\n
*Returns:*\n
Returns ID of the new connection. The connection is set as active.
*Example:*\n
| Connect To Tarantool | 127.0.0.1 | 3301 |
"""
logger.debug('Connecting to the Tarantool DB using \
host={host}, port={port}, user={user}'.format(host=host,
port=port,
user=user))
try:
self._connection = Connection(host=host, port=int(port), user=user, password=password)
return self._cache.register(self._connection, alias)
except Exception as exc:
raise Exception("Logon to Tarantool error:", str(exc))
def get_ids(tnt_connection: Connection, table: str, count: int):
trace(table, "Get ids")
table_size = tnt_connection.call("box.space.{}:len".format(table))
offset = randint(0, table_size[0] - count)
trace(table, "Get ids, table size: {}, offset: {}".format(table_size[0], offset))
result = tnt_connection.select(space_name=table, offset=offset, limit=count)
ids_data = [] * len(result)
for item in result:
ids_data.append(item[0])
return ids_data
def add_kv(key, value):
print("sdfgh")
logger("add_kv()", key + " " + value)
c = Connection("127.0.0.1", 3301)
print("add_key()")
try:
c.insert("KVstorage", (key, value))
print(c.select("KVstorage", key))
return True
except:
return False
def get_connection(alias=DEFAULT_ALIAS, reconnect=False):
"""Return connection by alias.
:param alias:
:param reconnect:
"""
global _connections
if reconnect:
disconnect(alias)
if alias not in _connections:
alias_settings = _connection_settings.get(alias)
if not alias_settings:
raise ValueError(
'Connection with alias {alias} have not defined.'.format(
alias=alias))
try:
conn_settings = alias_settings.copy()
host = conn_settings.pop('host')
port = conn_settings.pop('port')
conn_settings.pop('space', None)
_connections[alias] = Connection(host, port, **conn_settings)
except DatabaseError as exc:
message = 'Connect error for alias "{alias}": "{message}".'.format(
alias=alias, message=exc)
raise ConnectionError, message, sys.exc_info()[2]
return _connections[alias]
def test(header, body):
# Connect and authenticate
c = Connection('localhost', server.sql.port)
c.connect()
print 'query', repr_dict(header), repr_dict(body)
header = msgpack.dumps(header)
body = msgpack.dumps(body)
query = msgpack.dumps(len(header) + len(body)) + header + body
# Send raw request using connectred socket
s = c._socket
try:
s.send(query)
except OSError as e:
print ' => ', 'Failed to send request'
c.close()
sql("ping")
def run_requests(table: str, debug: bool = True):
tnt_connection = Connection("127.0.0.1", 3302)
trace(table, "Run requests")
ids = get_ids(tnt_connection, table, 1000)
start = time.perf_counter_ns()
row = 0
for item in ids:
result = tnt_connection.select(table, [item])
if row % 100 == 0 and debug:
trace(table, "Request completed: {}, last result: {}".format(row, result[0]))
row += 1
end = time.perf_counter_ns()
diff = (end - start) / 1000.0
trace(table, "Completed, request count: {}, time spent: {}"
.format(row, diff))
return diff
class KV:
def __init__(self, *, host, port, user, password):
self.space = "kv"
self.conn = Connection(host, port, user=user, password=password)
def put(self, key, value):
try:
ret = self.conn.insert(self.space, (key, value))
except DatabaseError as err:
if err.args[0] == 3:
logger.warning(f"Key {key!r} already exists")
raise KeyError(f"Key {key!r} already exists")
else:
raise
else:
return ret.data
def get(self, key):
ret = self.conn.select(self.space, key)
if not ret.data:
logger.warning(f"Key {key!r} does not exist")
raise KeyError(f"Key {key!r} does not exist")
return ret.data[0][1]
def update(self, key, updater):
data = self.get(key)
data.update(updater)
# delete null's keys
data = {k: v for k, v in data.items() if v is not None}
ret = self.conn.update(self.space, key, [("=", 1, data)])
return ret.data[0][1]
def delete(self, key):
ret = self.conn.delete(self.space, key)
if not ret.data:
logger.warning(f"Key {key!r} does not exist")
raise KeyError(f"Key {key!r} does not exist")
return ret.data[0][1]
def produce(table: str):
tnt_connection = Connection("127.0.0.1", 3302)
trace(table, "Produce start")
start = time.perf_counter_ns()
for row in range(0, 100000, 1):
next_id = tnt_connection.call("box.sequence.test_seq:next")
tnt_connection.insert(table, (str(uuid.uuid1()), next_id[0], "TestBook_" + str(row), randint(1900, 2000)))
if row % 1000 == 0:
trace(table, "Done insert row: {}".format(row))
end = time.perf_counter_ns()
table_size = tnt_connection.call("box.space.{}:len".format(table))
diff = (end - start) / 1000.0
trace(table, "Completed, table size: {}, time spent: {}"
.format(table_size[0], diff))
return diff
def test(header, body):
# Connect and authenticate
c = Connection('localhost', server.iproto.port)
c.connect()
print 'query', repr_dict(header), repr_dict(body)
header = msgpack.dumps(header)
body = msgpack.dumps(body)
query = msgpack.dumps(len(header) + len(body)) + header + body
# Send raw request using connectred socket
s = c._socket
try:
s.send(query)
except OSError as e:
print ' => ', 'Failed to send request'
c.close()
print iproto.py_con.ping() > 0
print "IPROTO_REPLACE"
test({IPROTO_CODE: REQUEST_TYPE_REPLACE}, {IPROTO_SPACE_ID: 280})
print "\n"
print "IPROTO_CALL"
test({IPROTO_CODE: REQUEST_TYPE_CALL}, {})
test({IPROTO_CODE: REQUEST_TYPE_CALL}, {IPROTO_KEY: ('procname', )})
print "\n"
# gh-434 Tarantool crashes on multiple iproto requests with WAL enabled
admin("box.cfg.wal_mode")
admin("space = box.schema.space.create('test', { id = 567 })")
admin("index = space:create_index('primary', { type = 'hash' })")
admin("box.schema.user.grant('guest', 'read,write,execute', 'space', 'test')")
c = Connection('localhost', server.iproto.port)
c.connect()
request1 = RequestInsert(c, 567, [1, "baobab"])
request2 = RequestInsert(c, 567, [2, "obbaba"])
s = c._socket
try:
s.send(bytes(request1) + bytes(request2))
except OSError as e:
print ' => ', 'Failed to send request'
response1 = Response(c, c._read_response())
response2 = Response(c, c._read_response())
print response1.__str__()
print response2.__str__()
request1 = RequestInsert(c, 567, [3, "occama"])
request2 = RequestSelect(c, 567, 0, [1], 0, 1, 0)
def update_value(key, value):
logger("update_value", key + " " + value)
c = Connection("127.0.0.1", 3301)
print("update_value()")
c.update("KVstorage", str(key), [('=', 1, value)] )
print(c.select("KVstorage", key))
def get_kv(key):
logger("get_kv()", key)
c = Connection("127.0.0.1", 3301)
print("get_kv()")
print(c.select("KVstorage", key))
return list(c.select("KVstorage", key))
def update_value_by_key(key, value):
c = Connection('127.0.0.1', 3301)
c.update("KVStorage", key, [('=', 1, str(value))])
return
def add_value_by_key(key, value):
c = Connection('127.0.0.1', 3301)
c.insert("KVStorage", (key, value))
return
#!/usr/bin/python
import sys
import time
from tarantool import Connection
c = Connection("127.0.0.1", 3301)
filename = sys.argv[1]
with open(filename) as f:
content = f.readlines()
data = [[int(x) for x in line.split()] for line in content]
st = time.time()
for x in data:
c.select('trends_uint', x)
ft = time.time()
print "%s\t-\t-\t%d\t%.4f\t%.6f" % (sys.argv[0], len(data), (ft - st),
(ft - st) / len(data))
def delete_kv(key):
logger("delete_kv()", key)
c = Connection("127.0.0.1", 3301)
print("delete_kv()")
c.delete("KVstorage", key)
return
def __init__(self, *, host, port, user, password):
self.space = "kv"
self.conn = Connection(host, port, user=user, password=password)
print "IPROTO_REPLACE"
test({ IPROTO_CODE : REQUEST_TYPE_REPLACE }, { IPROTO_SPACE_ID: 280 })
print "\n"
print "IPROTO_CALL"
test({ IPROTO_CODE : REQUEST_TYPE_CALL }, {})
test({ IPROTO_CODE : REQUEST_TYPE_CALL }, { IPROTO_KEY: ('procname', )})
print "\n"
# gh-434 Tarantool crashes on multiple iproto requests with WAL enabled
admin("box.cfg.wal_mode")
admin("space = box.schema.space.create('test', { id = 567 })")
admin("index = space:create_index('primary', { type = 'hash' })")
admin("box.schema.user.grant('guest', 'read,write,execute', 'space', 'test')")
c = Connection('localhost', server.sql.port)
c.connect()
request1 = RequestInsert(c, 567, [1, "baobab"])
request2 = RequestInsert(c, 567, [2, "obbaba"])
s = c._socket
try:
s.send(bytes(request1) + bytes(request2))
except OSError as e:
print ' => ', 'Failed to send request'
response1 = Response(c, c._read_response())
response2 = Response(c, c._read_response())
print response1.__str__()
print response2.__str__()
request1 = RequestInsert(c, 567, [3, "occama"])
request2 = RequestSelect(c, 567, 0, [1], 0, 1, 0)
print("IPROTO_REPLACE")
test({IPROTO_CODE: REQUEST_TYPE_REPLACE}, {IPROTO_SPACE_ID: 280})
print("\n")
print("IPROTO_CALL")
test({IPROTO_CODE: REQUEST_TYPE_CALL}, {})
test({IPROTO_CODE: REQUEST_TYPE_CALL}, {IPROTO_KEY: ("procname", )})
print("\n")
# gh-434 Tarantool crashes on multiple iproto requests with WAL enabled
admin("box.cfg.wal_mode")
admin("space = box.schema.space.create('test', { id = 567 })")
admin("index = space:create_index('primary', { type = 'hash' })")
admin("box.schema.user.grant('guest', 'read,write,execute', 'space', 'test')")
c = Connection("localhost", server.iproto.port)
c.connect()
request1 = RequestInsert(c, 567, [1, "baobab"])
request2 = RequestInsert(c, 567, [2, "obbaba"])
s = c._socket
try:
s.send(bytes(request1) + bytes(request2))
except OSError as e:
print(" => ", "Failed to send request")
response1 = Response(c, c._read_response())
response2 = Response(c, c._read_response())
print(response1.__str__())
print(response2.__str__())
request1 = RequestInsert(c, 567, [3, "occama"])
request2 = RequestSelect(c, 567, 0, [1], 0, 1, 0)
from tarantool import Connection
conn = Connection("localhost", 3302)
conn.insert("user", (
None,
'a',
1,
))
conn.insert("user", (
None,
'b',
1,
))
conn.insert("user", (
None,
'a',
0,
))
conn.insert("user", (
None,
'b',
0,
))
conn.insert("channel", (None, '1001373637514', 1, 't.me/ofka'))
conn.insert("channel", (None, '1236565135464', 1, 't.me/root'))
conn.insert("channel", (None, 'icqchannel1', 0, 'icq.com/qwe'))
conn.insert("channel", (None, 'icqchannel2', 0, 'icq.com/zxcvbnm'))
conn.insert("link", (None, 1, 3, 1))
def delete_value_by_key(key):
c = Connection('127.0.0.1', 3301)
c.delete("KVStorage", key)
return
def show_samples(table: str):
tnt_connection = Connection("127.0.0.1", 3302)
trace(table, "Show samples")
result = tnt_connection.select(space_name=table, offset=0, limit=10)
for item in result:
print(item)
#!/usr/bin/python
import sys
import time
from tarantool import Connection
c = Connection("127.0.0.1", 3301)
def conv(line):
return [ int(x) for x in line.split() ]
filename = sys.argv[1]
with open(filename) as f:
content = f.readlines()
data = [ conv(line) for line in content ]
if len(data) == 0:
sys.exit(0)
def check_res(x):
res = c.call('select_from_trends_uint2', [x[1], x[2], x[3]])
rval = len(list(res))
return rval == x[0]
start = time.time()
for x in data:
if not check_res(x):
raise Exception(x)
finish = time.time()
print "%s\t%d\t%.2f\t%.6f" % (filename, len(data), finish - start, (finish - start) / len(data))
#!/usr/bin/python
import sys
import time
from tarantool import Connection
c = Connection("127.0.0.1", 3301)
filename = sys.argv[1]
with open(filename) as f:
content = f.readlines()
data = [ [ int(x) for x in line.split() ] for line in content ]
st = time.time()
for x in data:
c.select('trends_uint', x)
ft = time.time()
print "%s\t-\t-\t%d\t%.4f\t%.6f" % (sys.argv[0], len(data), (ft - st), (ft - st) / len(data))
def get_value_by_key(key):
c = Connection('127.0.0.1', 3301)
value = list(c.select("KVStorage", key))
return value
#!/usr/bin/python
import sys
import time
from tarantool import Connection
c = Connection("127.0.0.1", 3301)
itemid = int(sys.argv[1])
clock = int(sys.argv[2])
iterations = int(sys.argv[3])
st = time.time()
for i in range(iterations):
c.select('trends_uint', [itemid, clock])
ft = time.time()
print "%s\t%d\t%d\t%d\t%.4f\t%.6f" % (sys.argv[0], itemid, clock, iterations, (ft - st), (ft - st) / iterations)
#!/usr/bin/python
import sys
import time
from tarantool import Connection
c = Connection("127.0.0.1", 3301)
itemid = int(sys.argv[1])
clock = int(sys.argv[2])
iterations = int(sys.argv[3])
st = time.time()
for i in range(iterations):
c.select('trends_uint', [itemid, clock])
ft = time.time()
print "%s\t%d\t%d\t%d\t%.4f\t%.6f" % (sys.argv[0], itemid, clock, iterations,
(ft - st), (ft - st) / iterations)
#!/usr/bin/python
import sys
from tarantool import Connection
c = Connection("127.0.0.1", 3301)
for line in sys.stdin:
fields = line.rstrip().split('\t')
num_fields = [int(x) for x in fields]
c.insert("test1", tuple(num_fields))
class TarantoolLibrary(object):
"""
Robot Framework library for working with Tarantool DB.
== Dependencies ==
| tarantool | https://pypi.org/project/tarantool/ | version > 0.5 |
| robot framework | http://robotframework.org |
"""
ROBOT_LIBRARY_SCOPE = 'GLOBAL'
def __init__(self):
"""Library initialization.
Robot Framework ConnectionCache() class is prepared for working with concurrent connections."""
self._connection = None
self._cache = ConnectionCache()
def _modify_key_type(self, key, key_type):
"""
Convert key to the required tarantool data type.
Tarantool data types corresponds to the following Python types:
STR - unicode (str for Python 3.x)
NUM - int
NUM64 - int or long (int for Python 3.x)
*Args:*\n
_key_: key to modify;\n
_key_type_: key type: STR, NUM, NUM64;\n
*Returns:*\n
modified key.
"""
key_type = key_type.upper()
if key_type == "STR":
if isinstance(key, bytes):
return codecs.decode(key)
return str(key)
if key_type == "NUM":
return int(key)
if key_type == "NUM64":
if int(sys.version[0]) == 2:
return long(key) # noqa: F821
else:
return int(key)
raise Exception("Wrong key type for conversation: {}. Allowed ones are STR, NUM and NUM64".format(key_type))
def connect_to_tarantool(self, host, port, user=None, password=None, alias=None):
"""
Connection to Tarantool DB.
*Args:*\n
_host_ - host for db connection;\n
_port_ - port for db connection;\n
_user_ - username for db connection;\n
_password_ - password for db connection;\n
_alias_ - connection alias, used for switching between open connections;\n
*Returns:*\n
Returns ID of the new connection. The connection is set as active.
*Example:*\n
| Connect To Tarantool | 127.0.0.1 | 3301 |
"""
logger.debug('Connecting to the Tarantool DB using \
host={host}, port={port}, user={user}'.format(host=host,
port=port,
user=user))
try:
self._connection = Connection(host=host, port=int(port), user=user, password=password)
return self._cache.register(self._connection, alias)
except Exception as exc:
raise Exception("Logon to Tarantool error:", str(exc))
def close_all_tarantool_connections(self):
"""
Close all Tarantool connections that were opened.
After calling this keyword connection index returned by opening new connections [#Connect To Tarantool |Connect To Tarantool],
starts from 1.
*Example:*\n
| Connect To Tarantool | 192.168.0.1 | 3031 | user | password | alias=trnt_1 |
| Connect To Tarantool | 192.168.0.2 | 3031 | user | password | alias=trnt_2 |
| Switch Tarantool Connection | trnt_1 |
| @{data1}= | Select | space1 | key1 |
| Switch Tarantool Connection | trnt_2 |
| @{data2}= | Select | space2 | key2 |
| Close All Tarantool Connections |
"""
self._cache.close_all()
self._connection = None
def switch_tarantool_connection(self, index_or_alias):
"""
Switch to another existing Tarantool connection using its index or alias.\n
The connection index is obtained on creating connection.
Connection alias is optional and can be set at connecting to DB [#Connect To Tarantool|Connect To Tarantool].
*Args:*\n
_index_or_alias_ - connection index or alias assigned to connection;
*Returns:*\n
Index of the previous connection.
*Example:* (switch by alias)\n
| Connect To Tarantool | 192.168.0.1 | 3031 | user | password | alias=trnt_1 |
| Connect To Tarantool | 192.168.0.2 | 3031 | user | password | alias=trnt_2 |
| Switch Tarantool Connection | trnt_1 |
| @{data1}= | Select | space1 | key1 |
| Switch Tarantool Connection | trnt_2 |
| @{data2}= | Select | space2 | key2 |
| Close All Tarantool Connections |
*Example:* (switch by connection index)\n
| ${trnt_index1}= | Connect To Tarantool | 192.168.0.1 | 3031 | user | password |
| ${trnt_index2}= | Connect To Tarantool | 192.168.0.2 | 3031 | user | password |
| @{data1}= | Select | space1 | key1 |
| ${previous_index}= | Switch Tarantool Connection | ${trnt_index1} |
| @{data2}= | Select | space2 | key2 |
| Switch Tarantool Connection | ${previous_index} |
| @{data3}= | Select | space1 | key1 |
| Close All Tarantool Connections |
"""
logger.debug('Switching to tarantool connection with alias/index {}'.format(index_or_alias))
old_index = self._cache.current_index
self._connection = self._cache.switch(index_or_alias)
return old_index
def select(self, space_name, key, offset=0, limit=0xffffffff, index=0, key_type=None, **kwargs):
"""
Select and retrieve data from the database.
*Args:*\n
_space_name_: space id to insert a record;\n
_key_: values to search over the index;\n
_offset_: offset in the resulting tuple set;\n
_limit_: limits the total number of returned tuples. Deafult is max of unsigned int32;\n
_index_: specifies which index to use. Default is 0 which means that the primary index will be used;\n
_key_type_: type of the key;\n
_kwargs_: additional params;\n
*Returns:*\n
Tarantool server response.
*Example:*\n
| ${data_from_trnt}= | Select | space_name=some_space_name | key=0 | key_type=NUM |
| Set Test Variable | ${key} | ${data_from_trnt[0][0]} |
| Set Test Variable | ${data_from_field} | ${data_from_trnt[0][1]} |
"""
logger.debug('Select data from space {space} by key {key}'.format(
space=space_name,
key=key)
)
if key_type:
key = self._modify_key_type(key=key, key_type=key_type)
return self._connection.select(
space_name=space_name,
key=key,
offset=offset,
limit=limit,
index=index,
**kwargs
)
def insert(self, space_name, values):
"""
Execute insert request.
*Args:*\n
_space_name_: space id to insert a record;\n
_values_: record to be inserted. The tuple must contain only scalar (integer or strings) values;\n
*Returns:*\n
Tarantool server response
*Example:*\n
| ${data_to_insert}= | Create List | 1 | ${data} |
| ${response}= | Insert | space_name=${SPACE_NAME} | values=${data_to_insert} |
| Set Test Variable | ${key} | ${response[0][0]} |
"""
logger.debug('Insert values {values} in space {space}'.format(space=space_name, values=values))
return self._connection.insert(space_name=space_name, values=values)
def create_operation(self, operation, field, arg):
"""
Check and prepare operation tuple.
*Allowed operations:*;\n
'+' for addition (values must be numeric);\n
'-' for subtraction (values must be numeric);\n
'&' for bitwise AND (values must be unsigned numeric);\n
'|' for bitwise OR (values must be unsigned numeric);\n
'^' for bitwise XOR (values must be unsigned numeric);\n
':' for string splice (you must provide 'offset', 'count' and 'value'
for this operation);\n
'!' for insertion (provide any element to insert);\n
'=' for assignment (provide any element to assign);\n
'#' for deletion (provide count of fields to delete);\n
*Args:*\n
_operation_: operation sign;\n
_field_: field number, to apply operation to;\n
_arg_: depending on operation argument or list of arguments;\n
*Returns:*\n
Tarantool server response.
*Example:*\n
| ${list_to_append}= | Create List | ${offset} | ${count} | ${value} |
| ${operation}= | Create Operation | operation=: | field=${1} | arg=${list_to_append} |
"""
if operation not in ('+', '-', '&', '|', '^', ':', '!', '=', '#'):
raise Exception('Unsupported operation: {}'.format(operation))
if isinstance(arg, (list, tuple)):
op_field_list = [operation, field]
op_field_list.extend(arg)
return tuple(op_field_list)
else:
return operation, field, arg
def update(self, space_name, key, op_list, key_type=None, **kwargs):
"""
Execute update request.
Update accepts both operation and list of operations for the argument op_list.
*Args:*\n
_space_name_: space number or name to update a record;\n
_key_: key that identifies a record;\n
_op_list_: operation or list of operations. Each operation is tuple of three (or more) values;\n
_key_type_: type of the key;\n
_kwargs_: additional params;\n
*Returns:*\n
Tarantool server response.
*Example:* (list of operations)\n
| ${operation1}= | Create Operation | operation== | field=${1} | arg=NEW DATA |
| ${operation2}= | Create Operation | operation== | field=${2} | arg=ANOTHER NEW DATA |
| ${op_list}= | Create List | ${operation1} | ${operation2} |
| Update | space_name=${SPACE_NAME} | key=${key} | op_list=${op_list} |
*Example:* (one operation)\n
| ${list_to_append}= | Create List | ${offset} | ${count} | ${value} |
| ${operation}= | Create Operation | operation== | field=${1} | arg=NEW DATA |
| Update | space_name=${SPACE_NAME} | key=${key} | op_list=${operation} |
"""
logger.debug('Update data in space {space} with key {key} with operations {op_list}'.format(
space=space_name,
key=key,
op_list=op_list
))
if key_type:
key = self._modify_key_type(key=key, key_type=key_type)
if isinstance(op_list[0], (list, tuple)):
return self._connection.update(space_name=space_name, key=key, op_list=op_list, **kwargs)
else:
return self._connection.update(space_name=space_name, key=key, op_list=[op_list], **kwargs)
def delete(self, space_name, key, key_type=None, **kwargs):
"""
Execute delete request.
*Args:*\n
_space_name_: space number or name to delete a record;\n
_key_: key that identifies a record;\n
_key_type_: type of the key;\n
_kwargs_: additional params;\n
*Returns:*\n
Tarantool server response.
*Example:*\n
| Delete | space_name=${SPACE_NAME}| key=${key} |
"""
logger.debug('Delete data in space {space} by key {key}'.format(space=space_name, key=key))
if key_type:
key = self._modify_key_type(key=key, key_type=key_type)
return self._connection.delete(space_name=space_name, key=key, **kwargs)
def db_connect(self):
return Connection(self.host, self.port)
from tarantool import Connection
from transfer_data import TransferData
conn = Connection("localhost", 3302)
data = TransferData(conn)
data.transfer()
result_transfer = conn.select("links", )
print(result_transfer)
