def consume(self):
_, password = self.cluster_spec.rest_credentials
for master in self.cluster_spec.yield_masters():
host = master.split(':')[0]
for bucket in self.test_config.buckets:
logger.info(
'Reading data via UPR from {}/{}'.format(host, bucket)
)
upr_client = UprClient(host=host, port=11210)
upr_client.sasl_auth_plain(username=bucket, password=password)
mcd_client = MemcachedClient(host=host, port=11210)
mcd_client.sasl_auth_plain(user=bucket, password=password)
op = upr_client.open_producer("stream")
response = op.next_response()
if response['status'] != SUCCESS:
logger.interrupt('Failed to open producer')
for vb in range(1024):
vb_stats = mcd_client.stats('vbucket-seqno {}'.format(vb))
uuid = long(vb_stats['vb_{}:uuid'.format(vb)])
high_seqno = long(vb_stats['vb_{}:high_seqno'.format(vb)])
op = upr_client.stream_req(vb=vb,
flags=0,
start_seqno=0,
end_seqno=high_seqno,
vb_uuid=uuid,
high_seqno=high_seqno)
while op.has_response():
response = op.next_response()
if response['opcode'] != CMD_STREAM_REQ:
break
upr_client.close_stream(vbucket=vb)
upr_client.shutdown()
def restart_memcached(mem_limit: int = 10000, port: int = 8000):
cmd1 = 'killall -9 memcached'
logger.info('Running: {}'.format(cmd1))
with settings(warn_only=True):
local(cmd1, capture=True)
for counter in range(5):
time.sleep(2)
with settings(warn_only=True):
result = local('pgrep memcached', capture=True)
if result.return_code == 1:
break
else:
logger.info('memcached still running')
else:
raise Exception('memcached was not kill properly')
cmd2 = 'memcached -u root -m {mem_limit} -l localhost -p {port} -d'
cmd2 = cmd2.format(mem_limit=mem_limit, port=port)
logger.info('Running: {}'.format(cmd2))
local(cmd2, capture=True)
for counter in range(5):
try:
time.sleep(2)
mc = MemcachedClient(host='localhost', port=port)
mc.stats()
mc.close()
break
except (EOFError, socket.error, MemcachedError):
logger.info('Can not connect to memcached')
else:
raise Exception('memcached did not start properly')
logger.info('memcached restarted')
def mc_iterator(self):
_, password = self.cluster_spec.rest_credentials
for hostname in self.cluster_spec.yield_hostnames():
for bucket in self.test_config.buckets:
mc = MemcachedClient(host=hostname, port=11210)
try:
mc.sasl_auth_plain(user=bucket, password=password)
yield mc
except MemcachedError:
logger.warn('Auth failure')
def mc_iterator(self):
password = self.test_config.bucket.password
for host_port in self.cluster_spec.yield_servers():
host = host_port.split(':')[0]
memcached_port = self.rest.get_memcached_port(host_port)
for bucket in self.test_config.buckets:
mc = MemcachedClient(host=host, port=memcached_port)
try:
mc.sasl_auth_plain(user=bucket, password=password)
yield mc
except MemcachedError:
logger.warn('Auth failure')
def mc_iterator(self):
password = self.test_config.bucket.password
for host_port in self.cluster_spec.yield_servers():
host = host_port.split(':')[0]
memcached_port = self.rest.get_memcached_port(host_port)
for bucket in self.test_config.buckets:
mc = MemcachedClient(host=host, port=memcached_port)
try:
mc.sasl_auth_plain(user=bucket, password=password)
yield mc
except MemcachedError:
logger.warn('Auth failure')
def get_stats(self, host, port, bucket, stats):
proxyPort = self.test_config.bucket.proxyPort
if proxyPort is None:
mc = MemcachedClient(host=host, port=port)
mc.sasl_auth_plain(user=bucket, password=self.password)
else:
mc = MemcachedClient(host=host, port=proxyPort)
return mc.stats(stats)
def restart_memcached(mem_limit: int = 10000, port: int = 8000):
cmd1 = 'killall -9 memcached'
logger.info('Running: {}'.format(cmd1))
with settings(warn_only=True):
local(cmd1, capture=True)
for counter in range(5):
time.sleep(2)
with settings(warn_only=True):
result = local('pgrep memcached', capture=True)
if result.return_code == 1:
break
else:
logger.info('memcached still running')
else:
raise Exception('memcached was not kill properly')
cmd2 = 'memcached -u root -m {mem_limit} -l localhost -p {port} -d'
cmd2 = cmd2.format(mem_limit=mem_limit, port=port)
logger.info('Running: {}'.format(cmd2))
local(cmd2, capture=True)
for counter in range(5):
try:
time.sleep(2)
mc = MemcachedClient(host='localhost', port=port)
mc.stats()
mc.close()
break
except (EOFError, socket.error, MemcachedError):
logger.info('Can not connect to memcached')
else:
raise Exception('memcached did not start properly')
logger.info('memcached restarted')
def get_stats(self,
host: str,
port: int,
bucket: str,
stats: str = '') -> dict:
retries = 0
while True:
try:
mc = MemcachedClient(host=host, port=port, family=self.family)
mc.enable_xerror()
mc.hello("mc")
mc.sasl_auth_plain(user=bucket, password=self.password)
return mc.stats(stats)
except Exception:
if retries < MAX_RETRY:
retries += 1
time.sleep(SOCKET_RETRY_INTERVAL)
else:
raise
def consume(self):
password = self.test_config.bucket.password
for master in self.cluster_spec.yield_masters():
host = master.split(':')[0]
memcached_port = self.rest.get_memcached_port(master)
for bucket in self.test_config.buckets:
logger.info(
'Reading data via UPR from {}/{}'.format(host, bucket)
)
upr_client = DcpClient(host=host, port=memcached_port)
upr_client.sasl_auth_plain(username=bucket, password=password)
mcd_client = MemcachedClient(host=host, port=memcached_port)
mcd_client.sasl_auth_plain(user=bucket, password=password)
op = upr_client.open_producer('stream')
response = op.next_response()
if response['status'] != SUCCESS:
logger.interrupt('Failed to open producer')
for vb in range(1024):
vb_stats = mcd_client.stats('vbucket-seqno {}'.format(vb))
uuid = long(vb_stats['vb_{}:uuid'.format(vb)])
high_seqno = long(vb_stats['vb_{}:high_seqno'.format(vb)])
op = upr_client.stream_req(vb=vb,
flags=0,
start_seqno=0,
end_seqno=high_seqno,
vb_uuid=uuid,
high_seqno=high_seqno)
while op.has_response():
response = op.next_response()
if response['opcode'] != CMD_STREAM_REQ:
break
upr_client.close_stream(vbucket=vb)
upr_client.shutdown()
def get_stats(self, host, port, bucket, stats=''):
mc = MemcachedClient(host=host, port=port, family=self.family)
mc.sasl_auth_plain(user=bucket, password=self.password)
return mc.stats(stats)
def get_stats(self, host: str, port: int, bucket: str, stats: str = '') -> dict:
mc = MemcachedClient(host=host, port=port, family=self.family)
mc.sasl_auth_plain(user=bucket, password=self.password)
return mc.stats(stats)
def set_flusher_param(self, host, port, bucket, key, value):
logger.info('Changing flusher params: {}={}'.format(key, value))
mc = MemcachedClient(host=host, port=port)
mc.sasl_auth_plain(user=bucket, password=self.password)
mc.set_param(key, value, memcacheConstants.ENGINE_PARAM_FLUSH)
def get_stats(self, host, port, bucket, stats):
mc = MemcachedClient(host=host, port=port)
mc.sasl_auth_plain(user=bucket, password=self.password)
return mc.stats(stats)
def set_flusher_param(self, host, port, bucket, key, value):
logger.info('Changing flusher params: {}={}'.format(key, value))
mc = MemcachedClient(host=host, port=port)
mc.sasl_auth_plain(user=bucket, password=self.password)
mc.set_param(key, value, memcacheConstants.ENGINE_PARAM_FLUSH)
