def __init__(self, **options):
cmd.Cmd.__init__(self)
hostname = options.get('hostname', 'localhost')
port = options.get('port', 8123)
self.verbose = options.get('verbose') or 0
self.client = AsyncClient(hostname, port)
self.client.connect()
def __init__(self, **options):
cmd.Cmd.__init__(self)
hostname = options.get('hostname', 'localhost')
port = options.get('port', 8123)
self.verbose = options.get('verbose') or 0
self.client = AsyncClient(hostname, port)
self.client.connect()
def __init__(self, logger, host, port, **kwargs):
self.maxReturned = None # use library default
self.host = self.hostname = host
self.port = port
self.response_timeout = kwargs.pop('response_timeout', 30)
self.logger = logger
self.conn = AsyncClient(host, port, **kwargs)
self.conn.connect()
class Cmd(cmd.Cmd, object):
__metaclass__ = add_help
prompt = 'kinetic> '
intro = """
The Kinetic Protocol Command Line Interface Tools
type help for commands
"""
def __init__(self, **options):
cmd.Cmd.__init__(self)
hostname = options.get('hostname', 'localhost')
port = options.get('port', 8123)
self.verbose = options.get('verbose') or 0
self.client = AsyncClient(hostname, port)
self.client.connect()
def do_verbose(self, line):
"""Set active verbosity level [0-3]"""
try:
# see if the last word in the line is an integer
level = int(line.strip().rsplit(None, 1)[-1])
except (ValueError, IndexError):
if line:
print 'Unknown level: %r' % line
print 'Current level: %s' % self.verbose
return
configure_logging(level)
self.verbose = level
def do_quit(self, line):
"""Quit"""
return StopIteration
do_EOF = do_quit
def postcmd(self, stop, line):
if stop is StopIteration:
print '\nuser quit'
return True
@add_parser(put_parser)
def do_put(self, args):
self.client.put(args.key, args.value)
@add_parser(get_parser)
def do_get(self, args):
entry = self.client.get(args.key)
if not entry:
return 1
print entry.value
@add_parser(delete_parser)
def do_delete(self, args):
if not self.client.delete(args.key):
return 1
def _list(self, args):
if not args.end:
args.end = args.start + '\xff'
keys = self.client.getKeyRange(args.start, args.end)
# getKeyRange will return a maximum of 200 keys. if we have that many
# we need to continue retrieving keys until we get none or less than 200.
if len(keys) == 200:
last_key_retrieved = keys[len(keys)-1]
retrieving_keys = True
start_key_inclusive = False
while retrieving_keys:
partial_keys = self.client.getKeyRange(last_key_retrieved, args.end, start_key_inclusive)
if len(partial_keys) > 0:
keys.extend(partial_keys)
if len(partial_keys) == 200:
last_key_retrieved = partial_keys[len(partial_keys)-1]
else:
retrieving_keys = False
else:
retrieving_keys = False
return keys
@add_parser(list_parser)
def do_list(self, args):
keys = self._list(args)
for key in keys:
print key
@add_parser(next_parser)
def do_next(self, args):
entry = self.client.getNext(args.key)
if not entry:
return 1
if args.verbose:
print >> sys.stderr, 'key:', entry.key
print entry.value
@add_parser(prev_parser)
def do_prev(self, args):
entry = self.client.getPrevious(args.key)
if not entry:
return 1
if args.verbose:
print >> sys.stderr, 'key:', entry.key
print entry.value
@add_parser(getr_parser)
def do_getr(self, args):
# behave like a prefix
if not args.end:
args.end = args.start + '\xff'
for entry in self.client.getRange(args.start, args.end):
if args.verbose:
print >> sys.stderr, 'key:', entry.key
sys.stdout.write(entry.value)
print ''
@add_parser(deleter_parser)
def do_deleter(self, args):
def on_success(m): pass
def on_error(ex): pass
keys = self._list(args)
for k in keys:
self.client.deleteAsync(on_success, on_error, k, force=True)
self.client.wait()
class Cmd(cmd.Cmd, object):
__metaclass__ = add_help
prompt = 'kinetic> '
intro = """
The Kinetic Protocol Command Line Interface Tools
type help for commands
"""
def __init__(self, **options):
cmd.Cmd.__init__(self)
hostname = options.get('hostname', 'localhost')
port = options.get('port', 8123)
self.verbose = options.get('verbose') or 0
self.client = AsyncClient(hostname, port)
self.client.connect()
def do_verbose(self, line):
"""Set active verbosity level [0-3]"""
try:
# see if the last word in the line is an integer
level = int(line.strip().rsplit(None, 1)[-1])
except (ValueError, IndexError):
if line:
print 'Unknown level: %r' % line
print 'Current level: %s' % self.verbose
return
configure_logging(level)
self.verbose = level
def do_quit(self, line):
"""Quit"""
return StopIteration
do_EOF = do_quit
def postcmd(self, stop, line):
if stop is StopIteration:
print '\nuser quit'
return True
@add_parser(put_parser)
def do_put(self, args):
self.client.put(args.key, args.value)
@add_parser(get_parser)
def do_get(self, args):
entry = self.client.get(args.key)
if not entry:
return 1
print entry.value
@add_parser(delete_parser)
def do_delete(self, args):
if not self.client.delete(args.key):
return 1
def _list(self, args):
if not args.end:
args.end = args.start + '\xff'
keys = self.client.getKeyRange(args.start, args.end)
# getKeyRange will return a maximum of 200 keys. if we have that many
# we need to continue retrieving keys until we get none or less than 200.
if len(keys) == 200:
last_key_retrieved = keys[len(keys)-1]
retrieving_keys = True
start_key_inclusive = False
while retrieving_keys:
partial_keys = self.client.getKeyRange(last_key_retrieved, args.end, start_key_inclusive)
if len(partial_keys) > 0:
keys.extend(partial_keys)
if len(partial_keys) == 200:
last_key_retrieved = partial_keys[len(partial_keys)-1]
else:
retrieving_keys = False
else:
retrieving_keys = False
return keys
@add_parser(list_parser)
def do_list(self, args):
keys = self._list(args)
for key in keys:
print key
@add_parser(next_parser)
def do_next(self, args):
entry = self.client.getNext(args.key)
if not entry:
return 1
if args.verbose:
print >> sys.stderr, 'key:', entry.key
print entry.value
@add_parser(prev_parser)
def do_prev(self, args):
entry = self.client.getPrevious(args.key)
if not entry:
return 1
if args.verbose:
print >> sys.stderr, 'key:', entry.key
print entry.value
@add_parser(getr_parser)
def do_getr(self, args):
# behave like a prefix
if not args.end:
args.end = args.start + '\xff'
for entry in self.client.getRange(args.start, args.end):
if args.verbose:
print >> sys.stderr, 'key:', entry.key
sys.stdout.write(entry.value)
print ''
@add_parser(deleter_parser)
def do_deleter(self, args):
def on_success(m): pass
def on_error(ex): pass
keys = self._list(args)
for k in keys:
self.client.deleteAsync(on_success, on_error, k, force=True)
self.client.wait()
class KineticSwiftClient(object):
def __init__(self, logger, host, port, **kwargs):
self.maxReturned = None # use library default
self.host = self.hostname = host
self.port = port
self.response_timeout = kwargs.pop('response_timeout', 30)
self.logger = logger
self.conn = AsyncClient(host, port, **kwargs)
self.conn.connect()
def log_info(self, message):
self.logger.info('%s kinetic %s (%s): %s' % (datetime.datetime.now(),
self.conn.hostname,
self.conn.connection_id,
message))
def close(self):
if not self.conn:
return
real_sock = None
green_sock = getattr(self.conn, '_socket', None)
if hasattr(green_sock, 'fd'):
real_sock = getattr(green_sock.fd, '_sock', None)
if self.conn and not self.conn.closing:
self.conn.close()
if real_sock:
real_sock.close()
self.logger.info('Forced shutdown to %s:%s' % (
self.hostname, self.port))
self.conn = None
@property
def isConnected(self):
return self.conn and self.conn.isConnected
@property
def faulted(self):
if not self.conn:
return True
return self.conn.faulted
def reconnect(self):
self.conn.close()
self.conn.faulted = False
self.conn.connect()
def getPrevious(self, *args, **kwargs):
# self.log_info('getPrevious')
promise = Response(self)
self.conn.getPreviousAsync(promise.setResponse, promise.setError,
*args, **kwargs)
return promise
def put(self, key, data, *args, **kwargs):
# self.log_info('put')
promise = Response(self)
self.conn.putAsync(promise.setResponse, promise.setError, key, data,
*args, **kwargs)
return promise
def getKeyRange(self, *args, **kwargs):
# self.log_info('getKeyRange')
promise = Response(self)
self.conn.getKeyRangeAsync(promise.setResponse, promise.setError,
*args, **kwargs)
return promise
def iterKeyRange(self, start_key, end_key, **kwargs):
if 'maxReturned' not in kwargs and self.maxReturned is not None:
kwargs['maxReturned'] = self.maxReturned
keys = self.getKeyRange(start_key, end_key, **kwargs).wait()
while keys:
for key in keys:
yield key
# see if there's any more values
if kwargs.get('reverse', False):
end_key = key
else:
start_key = key
keys = self.getKeyRange(start_key, end_key,
startKeyInclusive=False,
endKeyInclusive=False,
**kwargs).wait()
def delete(self, key, *args, **kwargs):
# self.log_info('delete')
promise = Response(self)
self.conn.deleteAsync(promise.setResponse, promise.setError, key,
*args, **kwargs)
return promise
def get(self, key, *args, **kwargs):
# self.log_info('get')
promise = Response(self)
self.conn.getAsync(promise.setResponse, promise.setError, key,
*args, **kwargs)
return promise
def raise_err(self, *args, **kwargs):
raise Exception(
'error handling request to Drive %s:%s : %r %r' % (
self.host, self.port, args, kwargs))
def rename(self, key, new_key):
promise = Response(self)
def delete_key(*args, **kwargs):
self.conn.deleteAsync(promise.setResponse, promise.setError, key,
*args, **kwargs)
def write_entry(entry):
if not entry:
delete_key()
else:
self.conn.putAsync(delete_key, self.raise_err,
new_key, entry.value)
self.conn.getAsync(write_entry, self.raise_err, key)
return promise
def copy_keys(self, target, keys, depth=16):
# self.log_info('copy_keys')
host, port = target.split(':')
target = self.__class__(self.logger, host, int(port))
def write_entry(entry):
target.put(entry.key, entry.value, force=True)
with closing(target):
for key in keys:
self.conn.getAsync(write_entry, self.raise_err, key)
self.conn.wait()
target.conn.wait()
def delete_keys(self, keys, depth=16):
# self.log_info('delete_keys')
pending = deque()
for key in keys:
while len(pending) >= depth:
found = pending.popleft().wait()
if not found:
break
pending.append(self.delete(key, force=True))
for resp in pending:
resp.wait()
def push_keys(self, target, keys, batch=16):
# self.log_info('push_keys')
host, port = target.split(':')
port = int(port)
key_batch = []
results = []
for key in keys:
key_batch.append(key)
if len(key_batch) < batch:
continue
# send a batch
results.extend(self.conn.push(key_batch, host, port))
key_batch = []
if key_batch:
results.extend(self.conn.push(key_batch, host, port))
return results