def execute(self): shortUsage = self.parser.usage self.parser.usage += "\n" + self.format_command_list() opts, args = self.parser.parse_args() # -h handled here self.parser.usage = shortUsage # set usage back to short if len(args) < 2: print >> sys.stderr, self.parser.error("Too few arguments") sys.exit(2) hp, self.cmd = args[:2] if ':' in hp: host, port = hp.split(':', 1) try: port = int(port) except ValueError: print >> sys.stderr, self.parser.error( "invalid host[:dataport]") sys.exit(2) else: host = hp port = 11210 try: mc = mc_bin_client.MemcachedClient(host, port) except socket.gaierror, e: print 'Connection error: %s' % e sys.exit(1)
def wait_on_persistence(server): client = mc_bin_client.MemcachedClient(server.host, server.moxi_port) client.set_flush_param("min_data_age", '0') wait_on_state(client, "ep_queue_size", '0') wait_on_state(client, "ep_flusher_todo", '0') client.close()
def evict_items(server, vbucket, num_of_items): client = mc_bin_client.MemcachedClient(server.host, server.moxi_port) client.vbucketId = vbucket for i in range(num_of_items): key = "key_" + ` vbucket ` + "_" + ` i ` client.evict_key(key) client.close()
def execute(self): opts, args = self.parser.parse_args() try: hp, self.cmd = args[:2] host, port = hp.split(':') port = int(port) except ValueError: self.usage() mc = mc_bin_client.MemcachedClient(host, port) f = self.cmds.get(self.cmd) if not f: self.usage() try: if callable(f[0]): f[0](mc, *args[2:], **opts.__dict__) else: getattr(mc, f[0])(*args[2:]) except socket.error, e: # "Broken pipe" is confusing, so print "Connection refused" instead. if type(e) is tuple and e[0] == 32 or \ isinstance(e, socket.error) and e.errno == 32: print >> sys.stderr, "Could not connect to %s:%d: " \ "Connection refused" % (host, port) else: raise
def create_socket(self, family, type): if not self.user: mc_bin_server.MemcachedBinaryChannel.create_socket( self, family, type) self.client = mc_bin_client.MemcachedClient(self.server, self.port) return self.family_and_type = family, type self.mc = mc_bin_client.MemcachedClient(self.server, self.port) self.mc.sasl_auth_plain(self.user, self.pswd or "") sock = self.mc.s sock.setblocking(0) self.set_socket(sock) self.client = mc_bin_client.MemcachedClient(self.server, self.port) self.client.sasl_auth_plain(self.user, self.pswd or "")
def test_list_sasl_mechanisms(self): mc = mc_bin_client.MemcachedClient(self.master.ip, 11210) mechanisms = mc.sasl_mechanisms() self.log.info("Supported SASL Mechanisms {0}".format(mechanisms)) self.assertTrue(b"PLAIN" in mechanisms) found = False for mech in mechanisms: if b"SCRAM-SHA" in mech: found = True self.assertTrue(found)
def get_tap_conn(self): """Return previously connected tap conn.""" if not self.tap_conn: host = self.source_node['hostname'].split(':')[0] port = self.source_node['ports']['direct'] version = self.source_node['version'] # Ex: '2.0.0-1944-rel-community' or '1.8.1-937-rel-community'. logging.debug(" TAPDumpSource connecting mc: " + host + ":" + str(port)) self.tap_conn = mc_bin_client.MemcachedClient(host, port) if not self.tap_conn: return "error: could not connect to memcached: " + \ host + ":" + str(port), None sasl_user = str(self.source_bucket.get("name", self.opts.username) or "") sasl_pswd = str(self.source_bucket.get("saslPassword", self.opts.password) or "") if sasl_user: try: self.tap_conn.sasl_auth_plain(sasl_user, sasl_pswd) except EOFError: return "error: SASL auth error: %s:%s, user: %s" % \ (host, port, sasl_user), None except mc_bin_client.MemcachedError: return "error: SASL auth failed: %s:%s, user: %s" % \ (host, port, sasl_user), None except socket.error: return "error: SASL auth socket error: %s:%s, user: %s" % \ (host, port, sasl_user), None # We explicitly do not use TAP_FLAG_REGISTERED_CLIENT, # as that is for checkpoint/incremental backup only. # tap_opts = {memcacheConstants.TAP_FLAG_DUMP: '', memcacheConstants.TAP_FLAG_SUPPORT_ACK: ''} self.tap_conn.tap_fix_flag_byteorder = version.split(".") >= ["2", "0", "0"] if self.tap_conn.tap_fix_flag_byteorder: tap_opts[memcacheConstants.TAP_FLAG_TAP_FIX_FLAG_BYTEORDER] = '' if self.vbucket_list: tap_opts[coucbaseConstants.TAP_FLAG_LIST_VBUCKETS] = '' ext, val = TAPDumpSource.encode_tap_connect_opts(tap_opts) self.tap_conn._sendCmd(memcacheConstants.CMD_TAP_CONNECT, self.tap_name, val, 0, ext) return 0, self.tap_conn
def execute(self): shortUsage = self.parser.usage self.parser.usage += "\n" + self.format_command_list() opts, args = self.parser.parse_args() # -h handled here self.parser.usage = shortUsage # set usage back to short if len(args) < 2: print >> sys.stderr, self.parser.error("Too few arguments") sys.exit(2) hp, self.cmd = args[:2] if ':' in hp: host, port = hp.split(':', 1) try: port = int(port) except ValueError: print >> sys.stderr, self.parser.error( "invalid host[:dataport]") sys.exit(2) else: host = hp port = 11210 try: mc = mc_bin_client.MemcachedClient(host, port) except socket.error as e: print 'Connection error: %s' % e sys.exit(1) f = self.cmds.get(self.cmd) if not f: print self.parser.error("Unknown command") try: if callable(f[0]): f[0](mc, *args[2:], **opts.__dict__) else: getattr(mc, f[0])(*args[2:]) except socket.error, e: # "Broken pipe" is confusing, so print "Connection refused" instead. if type(e) is tuple and e[0] == 32 or \ isinstance(e, socket.error) and e.errno == 32: print >> sys.stderr, "Could not connect to %s:%d: " \ "Connection refused" % (host, port) else: raise
def connect_mc(host, port, user, pswd): mc = mc_bin_client.MemcachedClient(host, int(port)) if user: try: mc.sasl_auth_plain(str(user), str(pswd)) except EOFError: return "error: SASL auth error: %s:%s, user: %s" % \ (host, port, user), None except mc_bin_client.MemcachedError: return "error: SASL auth failed: %s:%s, user: %s" % \ (host, port, user), None except socket.error: return "error: SASL auth exception: %s:%s, user: %s" % \ (host, port, user), None return 0, mc
def validate_items(server, vbucket, num_of_items): client = mc_bin_client.MemcachedClient(server.host, server.moxi_port) client.vbucketId = vbucket count = 0 for cur_op in range(num_of_items): key = "key_" + ` vbucket ` + "_" + ` cur_op ` try: flag, keyx, value = client.get(key) assert (flag) hflag = socket.ntohl(flag) if hflag == ctypes.c_uint32(zlib.adler32(value)).value: count = count + 1 except (mc_bin_client.MemcachedError): continue return count
def sample(hp, db): cur = db.cursor() try: h, p = hp.split(':') p = int(p) except: h = hp port = 11211 mc = mc_bin_client.MemcachedClient(h, p) stats = mc.stats() uptime = int(stats['uptime']) print "Sampling", h, "at uptime =", uptime for k, v in stats.iteritems(): cur.execute(INS, (h, uptime, k, v))
def get_mc_stats(self, server, bucketlist, nodes): #print util.pretty_print(bucketlist) for bucket in bucketlist: bucket_name = bucket['name'] stats_buffer.node_stats[bucket_name] = {} for node in nodes: (node_server, node_port) = util.hostport(node['hostname']) self.log.info(" node: %s %s" % (node_server, node['ports']['direct'])) stats = {} mc = mc_bin_client.MemcachedClient(node_server, node['ports']['direct']) if bucket["name"] != "Default": mc.sasl_auth_plain(bucket_name.encode("utf8"), bucket["saslPassword"].encode("utf8")) self.get_mc_stats_per_node(mc, stats) stats_buffer.node_stats[bucket_name][node['hostname']] = stats
def execute(self): shortUsage = self.parser.usage self.parser.usage += "\n" + self.format_command_list() opts, args = self.parser.parse_args() # -h handled here self.parser.usage = shortUsage # set usage back to short if len(args) < 2: print(self.parser.error("Too few arguments"), file=sys.stderr) sys.exit(2) hp, self.cmd = args[:2] try: (host, port, family) = mc_bin_client.parse_address(hp) except Exception as e: print(e) sys.exit(1) try: mc = mc_bin_client.MemcachedClient(host, port, family) except OSError as e: print("Failed to connect to host {} on port {}: {}".format( host, port, e)) sys.exit(1) f = self.cmds.get(self.cmd) if not f: print(self.parser.error("Unknown command")) try: if callable(f[0]): f[0](mc, *args[2:], **opts.__dict__) else: getattr(mc, f[0])(*args[2:]) except socket.error as e: # "Broken pipe" is confusing, so print "Connection refused" instead. if type(e) is tuple and e[0] == 32 or \ isinstance(e, socket.error) and e.errno == 32: print("Could not connect to %s:%d: " "Connection refused" % (host, port), file=sys.stderr) else: raise
def set_items(server, vbucket, num_of_items): client = mc_bin_client.MemcachedClient(server.host, server.moxi_port) client.vbucketId = vbucket for i in range(num_of_items): key = "key_" + ` vbucket ` + "_" + ` i ` payload = generate_payload(key + '\0\r\n\0\0\n\r\0', random.randint(100, 1024)) flag = socket.htonl(ctypes.c_uint32(zlib.adler32(payload)).value) backoff_sec = 0 while backoff_sec < 4: (opaque, cas, data) = client.set(key, 0, flag, payload) if cas > 0: break backoff_sec = backoff_sec + 0.1 + (backoff_sec / 20) print "set %s failed and retry in %f sec" % (key, backoff_sec) time.sleep(backoff_sec) client.close()
def execute(self): try: opts, args = self.parser.parse_args() except SystemExit: self.usage(True) try: hp, self.cmd = args[:2] host, port = hp.split(':') port = int(port) except ValueError: self.usage() try: mc = mc_bin_client.MemcachedClient(host, port) except socket.gaierror, e: print 'Connection error: %s' % e sys.exit(1)
def main(): from optparse import OptionParser parser = OptionParser() parser.add_option('-H', '--hostname', dest='hostname') parser.add_option('-m', '--memcached-host', dest='memcached_host') parser.add_option('-u', '--memcached-username', dest='memcached_username') parser.add_option('-p', '--memcached-password', dest='memcached_password') options, args = parser.parse_args() mc = mc_bin_client.MemcachedClient(options.memcached_host or '127.0.0.1') if options.memcached_username: mc.sasl_auth_plain(options.memcached_username, options.memcached_password) hostname = options.hostname or get_hostname_from_collectd_config( '/etc/collectd/collectd.conf') assert hostname c = collectd.Exec() l = list_buckets(mc) now = time.time() totals = None for bucket in l: try: mc.bucket_select(bucket) values = get_bucket_stats(mc) if totals is None: totals = values[:] else: for i in xrange(len(totals)): totals[i] += values[i] totals[i] &= MASK put_values(c, hostname, bucket, now, values) except: #import traceback #traceback.print_exc() pass put_values(c, hostname, 'ALL', now, totals)
# limitations under the License. # import mc_bin_client import sys if len(sys.argv) < 7: msg = ( 'Usage: {} <node> <port> <user> <password> <bucket> <vbid> <key> <optional: "scopename.collectionname" or collection-ID>' .format(sys.argv[0])) print(msg, file=sys.stderr) sys.exit(1) HOST = sys.argv[1] PORT = sys.argv[2] client = mc_bin_client.MemcachedClient(host=HOST, port=PORT) client.sasl_auth_plain(user=sys.argv[3], password=sys.argv[4]) client.bucket_select(sys.argv[5]) collection = None client.enable_xerror() if len(sys.argv) == 9: client.enable_collections() client.hello("write_key") if len(sys.argv) == 9: try: collection = int(sys.argv[8], 16) except ValueError: collection = sys.argv[8]
def __init__(self, base_filepath, backup_type, tapname, vbid_list, logger, host, port, txn_size=None): self.base_filepath = base_filepath self.backup_type = backup_type self.logger = logger self.split_no = 0 self.current_split = None self.split_backup_files = [] self.full_backup = False self.backfill_chk_start = False self.current_checkpoint_id = 0 self.update_count = 0 self.source = tapname self.complete = False self.host = host self.port = port if txn_size: self.txn_size = txn_size else: self.txn_size = TXN_SIZE if vbid_list == None or vbid_list == []: vbid_list = [] vbid_list.append(0) self.vbucketId = vbid_list[0] if backup_type == "full": self.full_backup = True mc = mc_bin_client.MemcachedClient(host, port) mc.deregister_tap_client(tapname) time.sleep( 10 ) # handle special case when membase takes a few seconds to deregister tap name ext, val = encodeTAPConnectOpts( { memcacheConstants.TAP_FLAG_CHECKPOINT: (1, 0, 0), memcacheConstants.TAP_FLAG_SUPPORT_ACK: '', memcacheConstants.TAP_FLAG_REGISTERED_CLIENT: 0x01, # "value > 0" means "closed checkpoints only" memcacheConstants.TAP_FLAG_BACKFILL: 0x00000000, memcacheConstants.TAP_FLAG_CKSUM: '', memcacheConstants.TAP_FLAG_LIST_VBUCKETS: vbid_list, memcacheConstants.TAP_FLAG_CKSUM: '' }, True) self.logger.log("Type of backup is full !!! ") mc._sendCmd(memcacheConstants.CMD_TAP_CONNECT, tapname, val, 0, ext) cmd, opaque, cas, vbucketId, key, ext, val = readTap(mc) if cmd != memcacheConstants.CMD_TAP_OPAQUE: raise Exception("ERROR: Could not register tap: %s" % tapname) mc.close() sclient = mc_bin_client.MemcachedClient(self.host, self.port) stats_str = "vb_" + str(vbid_list[0]) + ":open_checkpoint_id" self.max_cpoint_id = int(sclient.stats('checkpoint')[stats_str]) sclient.close() self.mc = mc_bin_client.MemcachedClient(host, port) ext, val = encodeTAPConnectOpts({ memcacheConstants.TAP_FLAG_CHECKPOINT: '', memcacheConstants.TAP_FLAG_SUPPORT_ACK: '', memcacheConstants.TAP_FLAG_REGISTERED_CLIENT: 0x01, # "value > 0" means "closed checkpoints only" memcacheConstants.TAP_FLAG_BACKFILL: 0xffffffff, memcacheConstants.TAP_FLAG_CKSUM: '', memcacheConstants.TAP_FLAG_LIST_VBUCKETS: vbid_list, memcacheConstants.TAP_FLAG_CKSUM: '' }) self.mc._sendCmd(memcacheConstants.CMD_TAP_CONNECT, tapname, val, 0, ext) self.sinput = [self.mc.s] self.op_records = [] self.vbmap = {} # Key is vbucketId, value is [checkpointId, seq].
import mc_bin_client ENGINE = "/install/lib/memcached/ep.so" CONFIG = ";ht_size=3079;ht_locks=5;db_shards=4;tap_noop_interval=20;max_txn_size=1000;max_size=1048576000;tap_keepalive=300;vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB;" INIT_FILE="/install/etc/membase/init.sql;" if __name__ == '__main__': host = sys.argv[1] port = sys.argv[2] user = sys.argv[3] passwd = sys.argv[4] base = sys.argv[5] data = sys.argv[6] bucket = sys.argv[7] mc = mc_bin_client.MemcachedClient(host, int(port)) mc.sasl_auth_plain(user, passwd) while (True): try: mc.bucket_select(bucket) break except: dbfile = data + "/" + bucket + "-data/" + bucket conf = "initfile=" + base + INIT_FILE + "dbname=" + dbfile \ + CONFIG mc.bucket_create(bucket, base + ENGINE, conf) for i in range(1024): mc.set_vbucket_state(i, 'active')
def wait_on_warmup(server): client = mc_bin_client.MemcachedClient(server.host, server.port) wait_on_state(client, 'ep_warmup_thread', 'complete') client.close()
def get_stat(server, stat, sub=""): client = mc_bin_client.MemcachedClient(server.host, server.moxi_port) stats = client.stats(sub) client.close() return stats[stat]
def create_mc_bin_clients_for_ips(self, ips): clients = {} for ip in ips: clients[ip] = mc_bin_client.MemcachedClient(ip, 11211) print clients return clients
wait_for_lru(mc) log(">> Checking if deletes are reflected in LRU") for key in KEYLIST: if not key_in_lru(mc, key): raise exceptions.Exception("Expecting " + key + " in LRU but not found") # The following needs to change (we need to check if LRU didn't pick it up) for key in DELETED: try: if key_in_lru(mc, key): raise exceptions.Exception("Not expecting " + key + " in LRU but found") except mc_bin_client.MemcachedError: pass if __name__ == '__main__': if '-v' in sys.argv: VERBOSE = True mc = mc_bin_client.MemcachedClient() mc.set_flush_param("exp_pager_stime", str(LRUINTERVAL)) mc.set_flush_param("max_lru_entries", "500") check_lru_stat_command(mc) check_total(mc) check_delete(mc) check_keys(mc) # More cases with max_lru_entries limit
def activate_vbucket(server, vbucket): client = mc_bin_client.MemcachedClient(server.host, server.moxi_port) client.set_vbucket_state(vbucket, 'active')
def delete_vbucket(server, vbucket): client = mc_bin_client.MemcachedClient(server.host, server.moxi_port) client.set_vbucket_state(vbucket, 'dead') client.delete_vbucket(vbucket)
#!/usr/bin/env python import sys import mc_bin_client if __name__ == '__main__': mc = mc_bin_client.MemcachedClient(sys.argv[1]) mc.sasl_auth_plain(sys.argv[2], sys.argv[3]) mc.bucket_create(sys.argv[4], sys.argv[5], sys.argv[6])
def connect_host_port(self, host, port, user, pswd): self.conn = mc_bin_client.MemcachedClient(host, port) if user: self.conn.sasl_auth_plain(user, pswd)
server_map = None vbucket_map = None if vbucket_mode: server_map, vbucket_map = getVbuckets(args) if server_map is None or vbucket_map is None: self.Log.error("Not able to fetch the bucket information...so exiting") sys.exit(0) if not keyonly: for server in open(new_server_file,"r"): host, port = server.split(':', 1) port = int(port) support_cksum = mc_bin_client.MemcachedClient(host, port).options_supported() new_clients.append(mc_bin_client_async.MemcachedClient(host, port, queue_size, sleep_time, support_cksum)) num_new_servers += 1 client_thread = clientThread() client_thread.start() if keyonly: mainLoop(args, cbKeyOnly, user, pswd, server_map, vbucket_map, k=True) elif num_old_servers: mainLoop(args, cbRehashRecheck, user, pswd, server_map, vbucket_map) else: mainLoop(args, cbRehash, user, pswd, server_map, vbucket_map)
""" Simple CLI for basic SyncWrite operations.""" import mc_bin_client import memcacheConstants import sys if len(sys.argv) < 7: print("Usage: {} <host[:port]> <user> <password> <bucket> <op> <key> [value] [args]".format(sys.argv[0]), file = sys.stderr) sys.exit(1) (host, port) = sys.argv[1].split(":") if not port: port = 11210 client = mc_bin_client.MemcachedClient(host=host, port=port) client.enable_xerror() client.enable_mutation_seqno() client.enable_tracing() client.hello("set_durable") client.sasl_auth_plain(user=sys.argv[2], password=sys.argv[3]) client.bucket_select(sys.argv[4]) op = sys.argv[5] key = sys.argv[6] if len(sys.argv) > 7: value = sys.argv[7] level = memcacheConstants.DURABILITY_LEVEL_MAJORITY if op == "get": print (client.get(key))
if not self.filename: usage("Missing kvstore file") if __name__ == "__main__": config = Config(sys.argv[1:]) store = {} # Read the kvstore try: f = open(config.filename, 'r') except IOError: print "Error opening kvstore file %s " % config.filename else: store = pickle.load(f) f.close() # Create client connection client = mc_bin_client.MemcachedClient(config.server, config.port) if config.bucket != 'default': client.sasl_auth_plain(config.bucket, config.password) for vbucketid, keys in store.items(): print "Loading %s vbucketId with %s keys" % (vbucketid, keys) for key in keys: try: client.add(key, 0, 0, key) except mc_bin_client.MemcachedError, e: print e