def wait_for_vbuckets_ready_state(node,
bucket,
timeout_in_seconds=300,
log_msg='',
admin_user='******',
admin_pass='******'):
start_time = time.time()
end_time = start_time + timeout_in_seconds
ready_vbuckets = {}
log = logger.get("infra")
rest = RestConnection(node)
# servers = rest.get_nodes()
bucket_conn = BucketHelper(node)
bucket_conn.vbucket_map_ready(bucket, 60)
vbucket_count = len(bucket_conn.get_vbuckets(bucket))
vbuckets = bucket_conn.get_vbuckets(bucket)
obj = VBucketAwareMemcached(rest, bucket, info=node)
memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(
rest, bucket)
#Create dictionary with key:"ip:port" and value: a list of vbuckets
server_dict = defaultdict(list)
for everyID in range(0, vbucket_count):
memcached_ip_port = str(vbucket_map[everyID])
server_dict[memcached_ip_port].append(everyID)
while time.time() < end_time and len(ready_vbuckets) < vbucket_count:
for every_ip_port in server_dict:
#Retrieve memcached ip and port
ip, port = every_ip_port.split(":")
client = MemcachedClient(ip, int(port), timeout=30)
client.vbucket_count = len(vbuckets)
bucket_info = bucket_conn.get_bucket(bucket)
versions = rest.get_nodes_versions(logging=False)
pre_spock = False
for version in versions:
if "5" > version:
pre_spock = True
if pre_spock:
log.info("Atleast 1 of the server is on pre-spock "
"version. Using the old ssl auth to connect to "
"bucket.")
client.sasl_auth_plain(
bucket_info.name.encode('ascii'),
bucket_info.saslPassword.encode('ascii'))
else:
client.sasl_auth_plain(admin_user, admin_pass)
bucket = bucket.encode('ascii')
client.bucket_select(bucket)
for i in server_dict[every_ip_port]:
try:
(a, b, c) = client.get_vbucket_state(i)
except mc_bin_client.MemcachedError as e:
ex_msg = str(e)
if "Not my vbucket" in log_msg:
log_msg = log_msg[:log_msg.find("vBucketMap") +
12] + "..."
if e.status == memcacheConstants.ERR_NOT_MY_VBUCKET:
# May receive this while waiting for vbuckets, continue and retry...S
continue
log.error("%s: %s" % (log_msg, ex_msg))
continue
except exceptions.EOFError:
# The client was disconnected for some reason. This can
# happen just after the bucket REST API is returned (before
# the buckets are created in each of the memcached processes.)
# See here for some details: http://review.couchbase.org/#/c/49781/
# Longer term when we don't disconnect clients in this state we
# should probably remove this code.
log.error(
"got disconnected from the server, reconnecting")
continue
if c.find("\x01") > 0 or c.find("\x02") > 0:
ready_vbuckets[i] = True
elif i in ready_vbuckets:
log.warning(
"vbucket state changed from active to {0}".format(
c))
del ready_vbuckets[i]
client.close()
return len(ready_vbuckets) == vbucket_count
def flushctl_stop(servers, username=None, password=None):
for server in servers:
c = MemcachedClient(server.ip, 11210)
if username:
c.sasl_auth_plain(username, password)
c.stop_persistence()
def setUp(self):
self.mc=MemcachedClient()
self.mc.flush()
queue = Queue(maxsize=10000)
server = ServerInfo()
server.ip = "10.17.12.20"
bucket = {'name': 'default', 'port': 11220, 'password': ''}
#vam = VBucketAwareMemcached(RestConnection(server), bucket)
#print vam.memcacheds
#print vam.vBucketMap
payload = MemcachedClientHelper.create_value('*', 10240)
keys = ["key_%d" % (i) for i in range(4000)]
#keys = ["key_%s_%d" % (str(uuid.uuid4()), i) for i in range(4)]
total_size = 0
#mc = MemcachedClientHelper.create_memcached_client("172.16.75.128","default",11210,"default")
mc = MemcachedClient("10.17.12.20", 11210)
#for key in keys:
# vam.memcached(key).set(key, 1, 0, payload)
# total_size += len(key) + len(payload) + 200
#time.sleep(10)
#for i in range(0,1023):
# mc.set_vbucket_state(i, 'active')
new_thread = TapListener(queue, server)
new_thread.start()
i = 0
while i < 4000:
for key in keys:
# vam.memcached(key).get(key)
mc.set(key, 10, 0, payload, vbucket=0)
def flushctl_start(servers, username=None, password=None):
for server in servers:
c = MemcachedClient(server.ip, constants.memcached_port)
if username:
c.sasl_auth_plain(username, password)
c.start_persistence()
help="sleep time before each iteration")
parser.add_option("-i",
"--items",
dest="items",
default="10000",
help="number of items")
options, args = parser.parse_args()
node = options.node
sleep_time = int(options.sleep)
prefix = str(uuid.uuid4())
number_of_items = int(options.items)
mc = MemcachedClient("127.0.0.1", 11211)
keys = ["{0}-{1}".format(prefix, i) for i in range(0, number_of_items)]
info("inserting {0} items".format(number_of_items))
for k in keys:
mc.set(k, 0, 0, str(uuid.uuid4())[0:16])
while True:
info(
"now remove 3 chars from 80% of keys - if < 3 chars delete the key - if key does not exist create it"
)
for i in range(0, 3):
for k in keys:
try:
a, b, value = mc.get(k)
if len(value) < 3:
