def execute(self):
shortUsage = self.parser.usage
self.parser.usage += "\n" + self.format_command_list()
opts, args = self.parser.parse_args() # -h handled here
self.parser.usage = shortUsage # set usage back to short
if len(args) < 2:
print >> sys.stderr, self.parser.error("Too few arguments")
sys.exit(2)
hp, self.cmd = args[:2]
if ':' in hp:
host, port = hp.split(':', 1)
try:
port = int(port)
except ValueError:
print >> sys.stderr, self.parser.error(
"invalid host[:dataport]")
sys.exit(2)
else:
host = hp
port = 11210
try:
mc = mc_bin_client.MemcachedClient(host, port)
except socket.gaierror, e:
print 'Connection error: %s' % e
sys.exit(1)
def wait_on_persistence(server):
client = mc_bin_client.MemcachedClient(server.host, server.moxi_port)
client.set_flush_param("min_data_age", '0')
wait_on_state(client, "ep_queue_size", '0')
wait_on_state(client, "ep_flusher_todo", '0')
client.close()
def evict_items(server, vbucket, num_of_items):
client = mc_bin_client.MemcachedClient(server.host, server.moxi_port)
client.vbucketId = vbucket
for i in range(num_of_items):
key = "key_" + ` vbucket ` + "_" + ` i `
client.evict_key(key)
client.close()
def execute(self):
opts, args = self.parser.parse_args()
try:
hp, self.cmd = args[:2]
host, port = hp.split(':')
port = int(port)
except ValueError:
self.usage()
mc = mc_bin_client.MemcachedClient(host, port)
f = self.cmds.get(self.cmd)
if not f:
self.usage()
try:
if callable(f[0]):
f[0](mc, *args[2:], **opts.__dict__)
else:
getattr(mc, f[0])(*args[2:])
except socket.error, e:
# "Broken pipe" is confusing, so print "Connection refused" instead.
if type(e) is tuple and e[0] == 32 or \
isinstance(e, socket.error) and e.errno == 32:
print >> sys.stderr, "Could not connect to %s:%d: " \
"Connection refused" % (host, port)
else:
raise
def create_socket(self, family, type):
if not self.user:
mc_bin_server.MemcachedBinaryChannel.create_socket(
self, family, type)
self.client = mc_bin_client.MemcachedClient(self.server, self.port)
return
self.family_and_type = family, type
self.mc = mc_bin_client.MemcachedClient(self.server, self.port)
self.mc.sasl_auth_plain(self.user, self.pswd or "")
sock = self.mc.s
sock.setblocking(0)
self.set_socket(sock)
self.client = mc_bin_client.MemcachedClient(self.server, self.port)
self.client.sasl_auth_plain(self.user, self.pswd or "")
def test_list_sasl_mechanisms(self):
mc = mc_bin_client.MemcachedClient(self.master.ip, 11210)
mechanisms = mc.sasl_mechanisms()
self.log.info("Supported SASL Mechanisms {0}".format(mechanisms))
self.assertTrue(b"PLAIN" in mechanisms)
found = False
for mech in mechanisms:
if b"SCRAM-SHA" in mech:
found = True
self.assertTrue(found)
def get_tap_conn(self):
"""Return previously connected tap conn."""
if not self.tap_conn:
host = self.source_node['hostname'].split(':')[0]
port = self.source_node['ports']['direct']
version = self.source_node['version'] # Ex: '2.0.0-1944-rel-community' or '1.8.1-937-rel-community'.
logging.debug(" TAPDumpSource connecting mc: " +
host + ":" + str(port))
self.tap_conn = mc_bin_client.MemcachedClient(host, port)
if not self.tap_conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port), None
sasl_user = str(self.source_bucket.get("name",
self.opts.username) or "")
sasl_pswd = str(self.source_bucket.get("saslPassword",
self.opts.password) or "")
if sasl_user:
try:
self.tap_conn.sasl_auth_plain(sasl_user, sasl_pswd)
except EOFError:
return "error: SASL auth error: %s:%s, user: %s" % \
(host, port, sasl_user), None
except mc_bin_client.MemcachedError:
return "error: SASL auth failed: %s:%s, user: %s" % \
(host, port, sasl_user), None
except socket.error:
return "error: SASL auth socket error: %s:%s, user: %s" % \
(host, port, sasl_user), None
# We explicitly do not use TAP_FLAG_REGISTERED_CLIENT,
# as that is for checkpoint/incremental backup only.
#
tap_opts = {memcacheConstants.TAP_FLAG_DUMP: '',
memcacheConstants.TAP_FLAG_SUPPORT_ACK: ''}
self.tap_conn.tap_fix_flag_byteorder = version.split(".") >= ["2", "0", "0"]
if self.tap_conn.tap_fix_flag_byteorder:
tap_opts[memcacheConstants.TAP_FLAG_TAP_FIX_FLAG_BYTEORDER] = ''
if self.vbucket_list:
tap_opts[coucbaseConstants.TAP_FLAG_LIST_VBUCKETS] = ''
ext, val = TAPDumpSource.encode_tap_connect_opts(tap_opts)
self.tap_conn._sendCmd(memcacheConstants.CMD_TAP_CONNECT,
self.tap_name, val, 0, ext)
return 0, self.tap_conn
def execute(self):
shortUsage = self.parser.usage
self.parser.usage += "\n" + self.format_command_list()
opts, args = self.parser.parse_args() # -h handled here
self.parser.usage = shortUsage # set usage back to short
if len(args) < 2:
print >> sys.stderr, self.parser.error("Too few arguments")
sys.exit(2)
hp, self.cmd = args[:2]
if ':' in hp:
host, port = hp.split(':', 1)
try:
port = int(port)
except ValueError:
print >> sys.stderr, self.parser.error(
"invalid host[:dataport]")
sys.exit(2)
else:
host = hp
port = 11210
try:
mc = mc_bin_client.MemcachedClient(host, port)
except socket.error as e:
print 'Connection error: %s' % e
sys.exit(1)
f = self.cmds.get(self.cmd)
if not f:
print self.parser.error("Unknown command")
try:
if callable(f[0]):
f[0](mc, *args[2:], **opts.__dict__)
else:
getattr(mc, f[0])(*args[2:])
except socket.error, e:
# "Broken pipe" is confusing, so print "Connection refused" instead.
if type(e) is tuple and e[0] == 32 or \
isinstance(e, socket.error) and e.errno == 32:
print >> sys.stderr, "Could not connect to %s:%d: " \
"Connection refused" % (host, port)
else:
raise
def connect_mc(host, port, user, pswd):
mc = mc_bin_client.MemcachedClient(host, int(port))
if user:
try:
mc.sasl_auth_plain(str(user), str(pswd))
except EOFError:
return "error: SASL auth error: %s:%s, user: %s" % \
(host, port, user), None
except mc_bin_client.MemcachedError:
return "error: SASL auth failed: %s:%s, user: %s" % \
(host, port, user), None
except socket.error:
return "error: SASL auth exception: %s:%s, user: %s" % \
(host, port, user), None
return 0, mc
def validate_items(server, vbucket, num_of_items):
client = mc_bin_client.MemcachedClient(server.host, server.moxi_port)
client.vbucketId = vbucket
count = 0
for cur_op in range(num_of_items):
key = "key_" + ` vbucket ` + "_" + ` cur_op `
try:
flag, keyx, value = client.get(key)
assert (flag)
hflag = socket.ntohl(flag)
if hflag == ctypes.c_uint32(zlib.adler32(value)).value:
count = count + 1
except (mc_bin_client.MemcachedError):
continue
return count
def sample(hp, db):
cur = db.cursor()
try:
h, p = hp.split(':')
p = int(p)
except:
h = hp
port = 11211
mc = mc_bin_client.MemcachedClient(h, p)
stats = mc.stats()
uptime = int(stats['uptime'])
print "Sampling", h, "at uptime =", uptime
for k, v in stats.iteritems():
cur.execute(INS, (h, uptime, k, v))
def get_mc_stats(self, server, bucketlist, nodes):
#print util.pretty_print(bucketlist)
for bucket in bucketlist:
bucket_name = bucket['name']
stats_buffer.node_stats[bucket_name] = {}
for node in nodes:
(node_server, node_port) = util.hostport(node['hostname'])
self.log.info(" node: %s %s" %
(node_server, node['ports']['direct']))
stats = {}
mc = mc_bin_client.MemcachedClient(node_server,
node['ports']['direct'])
if bucket["name"] != "Default":
mc.sasl_auth_plain(bucket_name.encode("utf8"),
bucket["saslPassword"].encode("utf8"))
self.get_mc_stats_per_node(mc, stats)
stats_buffer.node_stats[bucket_name][node['hostname']] = stats
def execute(self):
shortUsage = self.parser.usage
self.parser.usage += "\n" + self.format_command_list()
opts, args = self.parser.parse_args() # -h handled here
self.parser.usage = shortUsage # set usage back to short
if len(args) < 2:
print(self.parser.error("Too few arguments"), file=sys.stderr)
sys.exit(2)
hp, self.cmd = args[:2]
try:
(host, port, family) = mc_bin_client.parse_address(hp)
except Exception as e:
print(e)
sys.exit(1)
try:
mc = mc_bin_client.MemcachedClient(host, port, family)
except OSError as e:
print("Failed to connect to host {} on port {}: {}".format(
host, port, e))
sys.exit(1)
f = self.cmds.get(self.cmd)
if not f:
print(self.parser.error("Unknown command"))
try:
if callable(f[0]):
f[0](mc, *args[2:], **opts.__dict__)
else:
getattr(mc, f[0])(*args[2:])
except socket.error as e:
# "Broken pipe" is confusing, so print "Connection refused" instead.
if type(e) is tuple and e[0] == 32 or \
isinstance(e, socket.error) and e.errno == 32:
print("Could not connect to %s:%d: "
"Connection refused" % (host, port),
file=sys.stderr)
else:
raise
def set_items(server, vbucket, num_of_items):
client = mc_bin_client.MemcachedClient(server.host, server.moxi_port)
client.vbucketId = vbucket
for i in range(num_of_items):
key = "key_" + ` vbucket ` + "_" + ` i `
payload = generate_payload(key + '\0\r\n\0\0\n\r\0',
random.randint(100, 1024))
flag = socket.htonl(ctypes.c_uint32(zlib.adler32(payload)).value)
backoff_sec = 0
while backoff_sec < 4:
(opaque, cas, data) = client.set(key, 0, flag, payload)
if cas > 0:
break
backoff_sec = backoff_sec + 0.1 + (backoff_sec / 20)
print "set %s failed and retry in %f sec" % (key, backoff_sec)
time.sleep(backoff_sec)
client.close()
def execute(self):
try:
opts, args = self.parser.parse_args()
except SystemExit:
self.usage(True)
try:
hp, self.cmd = args[:2]
host, port = hp.split(':')
port = int(port)
except ValueError:
self.usage()
try:
mc = mc_bin_client.MemcachedClient(host, port)
except socket.gaierror, e:
print 'Connection error: %s' % e
sys.exit(1)
def main():
from optparse import OptionParser
parser = OptionParser()
parser.add_option('-H', '--hostname', dest='hostname')
parser.add_option('-m', '--memcached-host', dest='memcached_host')
parser.add_option('-u', '--memcached-username', dest='memcached_username')
parser.add_option('-p', '--memcached-password', dest='memcached_password')
options, args = parser.parse_args()
mc = mc_bin_client.MemcachedClient(options.memcached_host or '127.0.0.1')
if options.memcached_username:
mc.sasl_auth_plain(options.memcached_username,
options.memcached_password)
hostname = options.hostname or get_hostname_from_collectd_config(
'/etc/collectd/collectd.conf')
assert hostname
c = collectd.Exec()
l = list_buckets(mc)
now = time.time()
totals = None
for bucket in l:
try:
mc.bucket_select(bucket)
values = get_bucket_stats(mc)
if totals is None:
totals = values[:]
else:
for i in xrange(len(totals)):
totals[i] += values[i]
totals[i] &= MASK
put_values(c, hostname, bucket, now, values)
except:
#import traceback
#traceback.print_exc()
pass
put_values(c, hostname, 'ALL', now, totals)
# limitations under the License.
#
import mc_bin_client
import sys
if len(sys.argv) < 7:
msg = (
'Usage: {} <node> <port> <user> <password> <bucket> <vbid> <key> <optional: "scopename.collectionname" or collection-ID>'
.format(sys.argv[0]))
print(msg, file=sys.stderr)
sys.exit(1)
HOST = sys.argv[1]
PORT = sys.argv[2]
client = mc_bin_client.MemcachedClient(host=HOST, port=PORT)
client.sasl_auth_plain(user=sys.argv[3], password=sys.argv[4])
client.bucket_select(sys.argv[5])
collection = None
client.enable_xerror()
if len(sys.argv) == 9:
client.enable_collections()
client.hello("write_key")
if len(sys.argv) == 9:
try:
collection = int(sys.argv[8], 16)
except ValueError:
collection = sys.argv[8]
def __init__(self,
base_filepath,
backup_type,
tapname,
vbid_list,
logger,
host,
port,
txn_size=None):
self.base_filepath = base_filepath
self.backup_type = backup_type
self.logger = logger
self.split_no = 0
self.current_split = None
self.split_backup_files = []
self.full_backup = False
self.backfill_chk_start = False
self.current_checkpoint_id = 0
self.update_count = 0
self.source = tapname
self.complete = False
self.host = host
self.port = port
if txn_size:
self.txn_size = txn_size
else:
self.txn_size = TXN_SIZE
if vbid_list == None or vbid_list == []:
vbid_list = []
vbid_list.append(0)
self.vbucketId = vbid_list[0]
if backup_type == "full":
self.full_backup = True
mc = mc_bin_client.MemcachedClient(host, port)
mc.deregister_tap_client(tapname)
time.sleep(
10
) # handle special case when membase takes a few seconds to deregister tap name
ext, val = encodeTAPConnectOpts(
{
memcacheConstants.TAP_FLAG_CHECKPOINT: (1, 0, 0),
memcacheConstants.TAP_FLAG_SUPPORT_ACK: '',
memcacheConstants.TAP_FLAG_REGISTERED_CLIENT:
0x01, # "value > 0" means "closed checkpoints only"
memcacheConstants.TAP_FLAG_BACKFILL: 0x00000000,
memcacheConstants.TAP_FLAG_CKSUM: '',
memcacheConstants.TAP_FLAG_LIST_VBUCKETS: vbid_list,
memcacheConstants.TAP_FLAG_CKSUM: ''
},
True)
self.logger.log("Type of backup is full !!! ")
mc._sendCmd(memcacheConstants.CMD_TAP_CONNECT, tapname, val, 0,
ext)
cmd, opaque, cas, vbucketId, key, ext, val = readTap(mc)
if cmd != memcacheConstants.CMD_TAP_OPAQUE:
raise Exception("ERROR: Could not register tap: %s" % tapname)
mc.close()
sclient = mc_bin_client.MemcachedClient(self.host, self.port)
stats_str = "vb_" + str(vbid_list[0]) + ":open_checkpoint_id"
self.max_cpoint_id = int(sclient.stats('checkpoint')[stats_str])
sclient.close()
self.mc = mc_bin_client.MemcachedClient(host, port)
ext, val = encodeTAPConnectOpts({
memcacheConstants.TAP_FLAG_CHECKPOINT:
'',
memcacheConstants.TAP_FLAG_SUPPORT_ACK:
'',
memcacheConstants.TAP_FLAG_REGISTERED_CLIENT:
0x01, # "value > 0" means "closed checkpoints only"
memcacheConstants.TAP_FLAG_BACKFILL:
0xffffffff,
memcacheConstants.TAP_FLAG_CKSUM:
'',
memcacheConstants.TAP_FLAG_LIST_VBUCKETS:
vbid_list,
memcacheConstants.TAP_FLAG_CKSUM:
''
})
self.mc._sendCmd(memcacheConstants.CMD_TAP_CONNECT, tapname, val, 0,
ext)
self.sinput = [self.mc.s]
self.op_records = []
self.vbmap = {} # Key is vbucketId, value is [checkpointId, seq].
import mc_bin_client
ENGINE = "/install/lib/memcached/ep.so"
CONFIG = ";ht_size=3079;ht_locks=5;db_shards=4;tap_noop_interval=20;max_txn_size=1000;max_size=1048576000;tap_keepalive=300;vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB;"
INIT_FILE="/install/etc/membase/init.sql;"
if __name__ == '__main__':
host = sys.argv[1]
port = sys.argv[2]
user = sys.argv[3]
passwd = sys.argv[4]
base = sys.argv[5]
data = sys.argv[6]
bucket = sys.argv[7]
mc = mc_bin_client.MemcachedClient(host, int(port))
mc.sasl_auth_plain(user, passwd)
while (True):
try:
mc.bucket_select(bucket)
break
except:
dbfile = data + "/" + bucket + "-data/" + bucket
conf = "initfile=" + base + INIT_FILE + "dbname=" + dbfile \
+ CONFIG
mc.bucket_create(bucket, base + ENGINE, conf)
for i in range(1024):
mc.set_vbucket_state(i, 'active')
def wait_on_warmup(server):
client = mc_bin_client.MemcachedClient(server.host, server.port)
wait_on_state(client, 'ep_warmup_thread', 'complete')
client.close()
def get_stat(server, stat, sub=""):
client = mc_bin_client.MemcachedClient(server.host, server.moxi_port)
stats = client.stats(sub)
client.close()
return stats[stat]
def create_mc_bin_clients_for_ips(self, ips):
clients = {}
for ip in ips:
clients[ip] = mc_bin_client.MemcachedClient(ip, 11211)
print clients
return clients
wait_for_lru(mc)
log(">> Checking if deletes are reflected in LRU")
for key in KEYLIST:
if not key_in_lru(mc, key):
raise exceptions.Exception("Expecting " + key +
" in LRU but not found")
# The following needs to change (we need to check if LRU didn't pick it up)
for key in DELETED:
try:
if key_in_lru(mc, key):
raise exceptions.Exception("Not expecting " + key +
" in LRU but found")
except mc_bin_client.MemcachedError:
pass
if __name__ == '__main__':
if '-v' in sys.argv:
VERBOSE = True
mc = mc_bin_client.MemcachedClient()
mc.set_flush_param("exp_pager_stime", str(LRUINTERVAL))
mc.set_flush_param("max_lru_entries", "500")
check_lru_stat_command(mc)
check_total(mc)
check_delete(mc)
check_keys(mc)
# More cases with max_lru_entries limit
def activate_vbucket(server, vbucket):
client = mc_bin_client.MemcachedClient(server.host, server.moxi_port)
client.set_vbucket_state(vbucket, 'active')
def delete_vbucket(server, vbucket):
client = mc_bin_client.MemcachedClient(server.host, server.moxi_port)
client.set_vbucket_state(vbucket, 'dead')
client.delete_vbucket(vbucket)
#!/usr/bin/env python
import sys
import mc_bin_client
if __name__ == '__main__':
mc = mc_bin_client.MemcachedClient(sys.argv[1])
mc.sasl_auth_plain(sys.argv[2], sys.argv[3])
mc.bucket_create(sys.argv[4], sys.argv[5], sys.argv[6])
def connect_host_port(self, host, port, user, pswd):
self.conn = mc_bin_client.MemcachedClient(host, port)
if user:
self.conn.sasl_auth_plain(user, pswd)
server_map = None
vbucket_map = None
if vbucket_mode:
server_map, vbucket_map = getVbuckets(args)
if server_map is None or vbucket_map is None:
self.Log.error("Not able to fetch the bucket information...so exiting")
sys.exit(0)
if not keyonly:
for server in open(new_server_file,"r"):
host, port = server.split(':', 1)
port = int(port)
support_cksum = mc_bin_client.MemcachedClient(host, port).options_supported()
new_clients.append(mc_bin_client_async.MemcachedClient(host, port, queue_size,
sleep_time, support_cksum))
num_new_servers += 1
client_thread = clientThread()
client_thread.start()
if keyonly:
mainLoop(args, cbKeyOnly, user, pswd, server_map, vbucket_map, k=True)
elif num_old_servers:
mainLoop(args, cbRehashRecheck, user, pswd, server_map, vbucket_map)
else:
mainLoop(args, cbRehash, user, pswd, server_map, vbucket_map)
""" Simple CLI for basic SyncWrite operations."""
import mc_bin_client
import memcacheConstants
import sys
if len(sys.argv) < 7:
print("Usage: {} <host[:port]> <user> <password> <bucket> <op> <key> [value] [args]".format(sys.argv[0]), file = sys.stderr)
sys.exit(1)
(host, port) = sys.argv[1].split(":")
if not port:
port = 11210
client = mc_bin_client.MemcachedClient(host=host, port=port)
client.enable_xerror()
client.enable_mutation_seqno()
client.enable_tracing()
client.hello("set_durable")
client.sasl_auth_plain(user=sys.argv[2], password=sys.argv[3])
client.bucket_select(sys.argv[4])
op = sys.argv[5]
key = sys.argv[6]
if len(sys.argv) > 7:
value = sys.argv[7]
level = memcacheConstants.DURABILITY_LEVEL_MAJORITY
if op == "get":
print (client.get(key))
if not self.filename:
usage("Missing kvstore file")
if __name__ == "__main__":
config = Config(sys.argv[1:])
store = {}
# Read the kvstore
try:
f = open(config.filename, 'r')
except IOError:
print "Error opening kvstore file %s " % config.filename
else:
store = pickle.load(f)
f.close()
# Create client connection
client = mc_bin_client.MemcachedClient(config.server, config.port)
if config.bucket != 'default':
client.sasl_auth_plain(config.bucket, config.password)
for vbucketid, keys in store.items():
print "Loading %s vbucketId with %s keys" % (vbucketid, keys)
for key in keys:
try:
client.add(key, 0, 0, key)
except mc_bin_client.MemcachedError, e:
print e
