def test_do_request_host_not_exist():
backend = Backend(["127.0.0.1:7011", "127.0.0.1:7012"])
try:
backend.do_request("get_domains")
except MogileFSError:
pass
else:
assert False
def test_do_request_trackers_not_exist(self):
backend = Backend(["127.0.0.1:7011", "127.0.0.1:7012"])
try:
backend.do_request("get_domains")
except MogileFSError:
pass
else:
assert False
class TestBackend(unittest.TestCase):
def setUp(self):
self.backend = Backend(['127.0.0.1:7001'])
def test_do_request_trackers_not_exist(self):
backend = Backend(["127.0.0.1:7011", "127.0.0.1:7012"])
try:
backend.do_request("get_domains")
except MogileFSError:
pass
else:
assert False
def test_do_request_one_tracker_down(self):
backend = Backend(["127.0.0.1:7001", "127.0.0.1:7012"])
try:
backend.do_request("get_domains")
except MogileFSError:
assert False
def test_do_request(self):
res = self.backend.do_request("get_domains")
assert res
def test_do_request_cmd_not_exist(self):
try:
self.backend.do_request("asdfkljweioav")
except MogileFSError:
pass
else:
assert False
def test_do_request_with_no_cmd(self):
try:
self.backend.do_request() # pylint: disable-msg=E1120
except TypeError:
pass
except Exception as e:
assert False, "TypeError expected, actual %r" % e
else:
assert False
class TestBackend(unittest.TestCase):
def setUp(self):
self.backend = Backend(['127.0.0.1:7001'])
def test_do_request_trackers_not_exist(self):
backend = Backend(["127.0.0.1:7011", "127.0.0.1:7012"])
try:
backend.do_request("get_domains")
except MogileFSError:
pass
else:
assert False
def test_do_request_one_tracker_down(self):
backend = Backend(["127.0.0.1:7001", "127.0.0.1:7012"])
try:
backend.do_request("get_domains")
except MogileFSError:
assert False
def test_do_request(self):
res = self.backend.do_request("get_domains")
assert res
def test_do_request_cmd_not_exist(self):
try:
self.backend.do_request("asdfkljweioav")
except MogileFSError:
pass
else:
assert False
def test_do_request_with_no_cmd(self):
try:
self.backend.do_request() # pylint: disable-msg=E1120
except TypeError:
pass
except Exception as e:
assert False, "TypeError expected, actual %r" % e
else:
assert False
class Client(object):
def __init__(self, domain, hosts, readonly=False):
self.readonly = bool(readonly)
self.domain = domain
self.backend = Backend(hosts, timeout=3)
def run_hook(self, hookname, *args):
pass
def add_hook(self, hookname, *args):
pass
@property
def last_tracker(self):
"""
Returns a tuple of (ip, port), representing the last mogilefsd
'tracker' server which was talked to.
"""
return self.backend.get_last_tracker()
def new_file(self, key, cls=None, bytes=0, largefile=False,
create_open_arg=None, create_close_arg=None, opts=None):
"""
Start creating a new filehandle with the given key,
and option given class and options.
Returns a filehandle you should then print to,
and later close to complete the operation.
NOTE: check the return value from close!
If your close didn't succeed, the file didn't get saved!
"""
self.run_hook('new_file_start', key, cls, opts)
create_open_arg = create_open_arg or {}
create_close_arg = create_close_arg or {}
# fid should be specified, or pass 0 meaning to auto-generate one
fid = 0
params = {'domain' : self.domain,
'key' : key,
'fid' : fid,
'multi_dest': 1}
if cls is not None:
params['class'] = cls
res = self.backend.do_request('create_open', params)
if not res:
return None
# [ (devid,path), (devid,path), ... ]
dests = []
# determine old vs. new format to populate destinations
if 'dev_count' not in res:
dests.append((res['devid'], res['path']))
else:
for x in xrange(1, int(res['dev_count']) + 1):
devid_key = 'devid_%d' % x
path_key = 'path_%s' % x
dests.append((res[devid_key], res[path_key]))
main_dest = dests[0]
main_devid, main_path = main_dest
self.run_hook("new_file_end", key, cls, opts)
# TODO
if largefile:
file_class = ClientHttpFile
else:
file_class = NewHttpFile
return file_class(mg=self,
fid=res['fid'],
path=main_path,
devid=main_devid,
backup_dests=dests,
cls=cls,
key=key,
content_length=bytes,
create_close_arg=create_close_arg,
overwrite=1)
def read_file(self, *args, **kwds):
"""
Read the file with the the given key.
Returns a seekable filehandle you can read() from.
Note that you cannot read line by line using <$fh> notation.
Takes the same options as get_paths
(which is called internally to get the URIs to read from).
"""
paths = self.get_paths(*args, **kwds)
path = paths[0]
backup_dests = [(None, p) for p in paths[1:]]
return ClientHttpFile(path=path, backup_dests=backup_dests, readonly=1)
def get_paths(self, key, noverify=1, zone='alt', pathcount=None):
"""
Given a key, returns an array of all the locations (HTTP URLs) that the file
has been replicated to.
"""
self.run_hook('get_paths_start', key)
if not pathcount:
pathcount = 2
params = {'domain' : self.domain,
'key' : key,
'noverify' : noverify and 1 or 0,
'zone' : zone,
'pathcount': pathcount}
try:
res = self.backend.do_request('get_paths', params)
paths = [res["path%d" % x] for x in xrange(1, int(res["paths"]) + 1)]
except (MogileFSTrackerError, MogileFSError):
paths = []
self.run_hook('get_paths_end', key)
return paths
def get_file_data(self, key, timeout=10):
"""
Returns scalarref of file contents in a scalarref.
Don't use for large data, as it all comes back to you in one string.
"""
fp = self.read_file(key, noverify=1)
try:
content = fp.read()
return content
finally:
fp.close()
def rename(self, old_key, new_key):
"""
Rename file (key) in MogileFS from oldkey to newkey.
Returns true on success, failure otherwise
"""
_complain_ifreadonly(self.readonly)
self.backend.do_request('rename', {'domain' : self.domain,
'from_key': old_key,
'to_key' : new_key})
return True
def list_keys(self, prefix=None, after=None, limit=None):
"""
Used to get a list of keys matching a certain prefix.
$prefix specifies what you want to get a list of.
$after is the item specified as a return value from this function last time
you called it.
$limit is optional and defaults to 1000 keys returned.
In list context, returns ($after, $keys).
In scalar context, returns arrayref of keys.
The value $after is to be used as $after when you call this function again.
When there are no more keys in the list,
you will get back undef or an empty list
"""
params = {'domain': self.domain}
if prefix:
params['prefix'] = prefix
if after:
params['after'] = after
if limit:
params['limit'] = limit
res = self.backend.do_request('list_keys', params)
reslist = []
for x in xrange(1, int(res['key_count']) + 1):
reslist.append(res['key_%d' % x])
return reslist
def foreach_key(self, *args, **kwds):
raise NotImplementedError()
def update_class(self, *args, **kwds):
raise NotImplementedError()
def sleep(self, duration):
"""
just makes some sleeping happen. first and only argument is number of
seconds to instruct backend thread to sleep for.
"""
self.backend.do_request("sleep", {'duration': duration})
return True
def set_pref_ip(self, *ips):
"""
Weird option for old, weird network architecture. Sets a mapping
table of preferred alternate IPs, if reachable. For instance, if
trying to connect to 10.0.0.2 in the above example, the module would
instead try to connect to 10.2.0.2 quickly first, then then fall back
to 10.0.0.2 if 10.2.0.2 wasn't reachable.
expects as argument a tuple of ("standard-ip", "preferred-ip")
"""
self.backend.set_pref_ip(*ips)
def store_file(self, key, fp, cls=None, **opts):
"""
Wrapper around new_file, print, and close.
Given a key, class, and a filehandle or filename, stores the file
contents in MogileFS. Returns the number of bytes stored on success,
undef on failure.
"""
_complain_ifreadonly(self.readonly)
self.run_hook('store_file_start', key, cls, opts)
try:
output = self.new_file(key, cls, largefile=1, **opts)
bytes = 0
while 1:
buf = fp.read(1024 * 16)
if not buf:
break
bytes += len(buf)
output.write(buf)
self.run_hook('store_file_end', key, cls, opts)
finally:
# finally
fp.close()
output.close()
return bytes
def store_content(self, key, content, cls=None, **opts):
"""
Wrapper around new_file, print, and close. Given a key, class, and
file contents (scalar or scalarref), stores the file contents in
MogileFS. Returns the number of bytes stored on success, undef on
failure.
"""
_complain_ifreadonly(self.readonly)
self.run_hook('store_content_start', key, cls, opts)
output = self.new_file(key, cls, None, **opts)
try:
output.write(content)
finally:
output.close()
self.run_hook('store_content_end', key, cls, opts)
return len(content)
def delete(self, key):
""" Delete a file from MogileFS """
_complain_ifreadonly(self.readonly)
self.backend.do_request('delete', {'domain': self.domain, 'key': key})
return True
def __init__(self, trackers, readonly=False, timeout=None):
self.readonly = bool(readonly)
self.backend = Backend(trackers, timeout)
class Admin(object):
def __init__(self, trackers, readonly=False, timeout=None):
self.readonly = bool(readonly)
self.backend = Backend(trackers, timeout)
def replicate_now(self):
return self.backend.do_request("replicate_now")
def get_hosts(self, hostid=None):
if hostid:
params = {'hostid': hostid}
else:
params = None
res = self.backend.do_request("get_hosts", params)
print(res)
results = []
fields = [
"hostid", "status", "hostname", "hostip", "http_port",
"http_get_port", "altip altmask"
]
hosts = int(res['hosts']) + 1
for ct in range(1, hosts):
results.append(
dict([(f, res.get('host%d_%s' % (ct, f))) for f in fields]))
return results
def get_devices(self, devid=None):
if devid:
params = {'devid': devid}
else:
params = None
res = self.backend.do_request("get_devices", params)
ret = []
for x in range(1, int(res['devices']) + 1):
device = {}
for k in ('devid', 'hostid', 'status', 'observed_state',
'utilization'):
device[k] = res.get('dev%d_%s' % (x, k))
for k in ('mb_total', 'mb_used', 'weight'):
value = res.get('dev%d_%s' % (x, k))
if value:
device[k] = int(value)
else:
device[k] = None
ret.append(device)
return ret
def list_fids(self, from_fid, to_fid):
"""
get raw information about fids, for enumerating the dataset
( from_fid, to_fid )
returns:
{fid => {dict with keys: domain, class, devcount, length, key}}
"""
res = self.backend.do_request('list_fids', {
'from': from_fid,
'to': to_fid
})
results = {}
for x in range(1, int(res['fid_count']) + 1):
key = 'fid_%d_fid' % x
results[key] = dict([(k, res['fid_%d_%s' % (x, k)]) \
for k in ('key', 'length', 'class', 'domain', 'devcount')])
return results
def clear_cache(self):
return self.backend.do_request('clear_cache')
def get_domains(self):
"""
get a dict of the domains we know about in the format of
{
"domain_name": {
"class_name": mindevcount,
"class_name": mindevcount,
...
},
...
}
"""
res = self.backend.do_request('get_domains')
domain_length = int(res['domains'])
ret = {}
for x in range(1, domain_length + 1):
domain_name = res['domain%d' % x]
ret.setdefault(domain_name, {})
class_length = int(res['domain%dclasses' % x])
for y in range(1, class_length + 1):
k = 'domain%dclass%dname' % (x, y)
v = 'domain%dclass%dmindevcount' % (x, y)
ret[domain_name][res[k]] = int(res[v])
return ret
def create_domain(self, domain):
"""
create a new domain
"""
if self.readonly:
return False
res = self.backend.do_request('create_domain', {'domain': domain})
if res['domain'] == domain:
return True
else:
return False
def delete_domain(self, domain):
"""
delete a domain
"""
if self.readonly:
return False
try:
res = self.backend.do_request('delete_domain', {'domain': domain})
except MogileFSError:
return False
if res['domain'] == domain:
return True
else:
return False
def create_class(self, domain, cls, mindevcount):
"""
create a class within a domain
"""
return self._modify_class('create', domain, cls, mindevcount)
def update_class(self, domain, cls, mindevcount):
"""
update a class's mindevcount within a domain
"""
return self._modify_class('update', domain, cls, mindevcount)
def delete_class(self, domain, cls):
"""
delete a class
"""
if self.readonly:
return False
res = self.backend.do_request("delete_class", {
'domain': domain,
'class': cls
})
if res['class'] == cls:
return True
else:
return False
def create_host(self, host, ip, port, getport=None, status=None):
params = {'host': host, 'ip': ip, 'port': port, 'getport': getport}
if status:
params['status'] = status
return self._modify_host('create', params)
def update_host(self, host, ip=None, port=None, getport=None, status=None):
params = {'host': host}
if ip:
params['ip'] = ip
if port:
params['port'] = port
if getport:
params['getport'] = True
if status:
params['status'] = status
return self._modify_host('update', params)
def delete_host(self, host):
self.backend.do_request("delete_host", {'host': host})
def create_device(self, hostname, devid, hostip=None, state=None):
params = {'hostname': hostname, 'devid': devid}
if hostip:
params['hostip'] = hostip
if state:
params['state'] = state
return self.backend.do_request('create_device', params)
def update_device(self, host, device, status=None, weight=None):
if status:
self.change_device_state(host, device, status)
if weight:
self.change_device_weight(host, device, weight)
return True
def change_device_state(self, host, device, state):
"""
change the state of a device; pass in the hostname of the host the
device is located on, the device id number, and the state you want
the host to be set to.
"""
params = {'host': host, 'device': device, 'state': state}
return self.backend.do_request('set_state', params)
def change_device_weight(self, host, device, weight):
"""
change the weight of a device by passing in the hostname and
the device id
"""
if not isinstance(weight, six.integer_types):
raise ValueError('argument weight muse be an integer')
params = {'host': host, 'device': device, 'weight': weight}
return self.backend.do_request('set_weight', params)
def _get_slave_keys(self):
res = self.backend.do_request("server_setting", {"key": "slave_keys"})
if not res:
return {}
value = res['value']
slave_keys = {}
for slave in value.split(','):
key, weight = (slave.split("=", 1) + [None])[:2]
# Weight can be zero,
# so don't default to 1 if it's defined and longer than 0 characters.
try:
weight = int(weight)
if not weight:
weight = 1
except (TypeError, ValueError):
weight = 1
slave_keys[key] = weight
return slave_keys
def _set_slave_keys(self, keys):
buf = []
for key, weight in keys.items():
try:
weight = int(weight)
if weight != 1:
key = "%s=%d" % (key, weight)
except (TypeError, ValueError):
pass
buf.append(key)
self.backend.do_request("set_server_setting", {
'key': 'slave_keys',
'value': ''.join(buf)
})
def slave_list(self):
keys = self._get_slave_keys()
ret = {}
for key in keys:
res = self.backend.do_request('server_setting',
{'key': 'slave_%s' % key})
if not res:
continue
value = res['value']
dsn, username, password = (value.split('|') + ['', ''])[:3]
ret[key] = (dsn, username, password)
return ret
def slave_add(self, key, dsn, username, password):
keys = self._get_slave_keys()
if key in keys:
return
value = '|'.join([dsn, username, password])
self.backend.do_request("set_server_setting", {
'key': key,
'value': value
})
keys[key] = None
self._set_slave_keys(keys)
def slave_modify(self, key, **opts):
keys = self._get_slave_keys()
if key not in keys:
# slave not fuond
return
res = self.backend.do_request("server_setting",
{"key": "slave_%s" % key})
value = res['value']
dsn, username, password = (value.split('|') + ['', ''])[:3]
dsn = opts.get('dsn') or dsn
username = opts.get('username') or username
password = opts.get('password') or password
value = '|'.join([dsn, username, password])
res = self.backend.do_request('set_server_setting', {
'key': 'slave_%s' % key,
'value': value
})
def slave_delete(self, key):
slave_keys = self._get_slave_keys()
if not slave_keys:
return None
if key not in slave_keys:
return False
self.backend.do_request('set_server_setting', {key: "slave_%s" % key})
del slave_keys[key]
self._set_slave_keys(slave_keys)
def fsck_start(self):
return self.backend.do_request("fsck_start")
def fsck_stop(self):
return self.backend.do_request("fsck_stop")
def fsck_reset(self, policy_only, startpos):
return self.backend.do_request("fsck_reset", {
'policy_only': policy_only,
'startpos': startpos
})
def fsck_clearlog(self):
return self.backend.do_request("fsck_clearlog")
def fsck_status(self):
return self.backend.do_request("fsck_status")
def fsck_log_rows(self, after_logid=None):
params = {}
if after_logid:
params['after_logid'] = after_logid
res = self.backend.do_request("fsck_getlog", params)
row_count = int(res['row_count'])
ret = []
for x in range(1, row_count + 1):
rec = {}
for k in ("logid", "utime", "fid", "evcode", "devid"):
rec[k] = res.get("row_%d_%s" % (x, k))
ret.append(rec)
return ret
def set_server_setting(self, key, value):
params = {'key': key, 'value': value}
return self.backend.do_request("set_server_setting", params)
def server_settings(self):
res = self.backend.do_request("server_settings")
if not res:
return
ret = {}
for x in range(1, int(res["key_count"]) + 1):
key = res.get("key_%d" % x, '')
value = res.get("value_%d" % x, '')
ret[key] = value
return ret
def _modify_class(self, verb, domain, cls, mindevcount, replpolicy=None):
if self.readonly:
return False
params = {'domain': domain, 'class': cls, 'mindevcount': mindevcount}
if replpolicy:
params['replpolicy'] = replpolicy
res = self.backend.do_request("%s_class" % verb, params)
if res['class'] == cls:
return True
else:
return False
def _modify_host(self, verb, params):
if self.readonly:
return False
try:
return self.backend.do_request("%s_host" % verb, params)
except MogileFSError:
return False
## Extra
#
def get_freespace(self, devid=None):
"""Get the free space for the entire cluster, or a specific node"""
return sum([x['mb_free'] for x in self.get_devices(devid)])
def get_stats(self):
params = {'all': 1}
res = self.backend.do_request('stats', params)
ret = {}
# get replication statistics
if 'replicationcount' in res:
replication = ret.setdefault('replication', {})
for x in range(1, int(res['replicationcount']) + 1):
domain = res.get('replication%ddomain' % x, '')
cls = res.get('replication%dclass' % x, '')
devcount = res.get('replication%ddevcount' % x, '')
fields = res.get('replication%dfields' % x)
(replication.setdefault(domain,
{}).setdefault(cls,
{}))[devcount] = fields
# get file statistics
if 'filescount' in res:
files = ret.setdefault('files', {})
for x in range(1, int(res['filescount']) + 1):
domain = res.get('files%ddomain' % x, '')
cls = res.get('files%dclass' % x, '')
(files.setdefault(domain,
{}))[cls] = res.get('files%dfiles' % x)
# get device statistics
if 'devicescount' in res:
devices = ret.setdefault('devices', {})
for x in range(1, int(res['devicescount']) + 1):
key = res.get('devices%did' % x, '')
devices[key] = {
'host': res.get('devices%dhost' % x),
'status': res.get('devices%dstatus' % x),
'files': res.get('devices%dfiles' % x),
}
if 'fidmax' in res:
ret['fids'] = {'max': res['fidmax']}
# return the created response
return ret
def __init__(self, trackers, readonly=False, timeout=None): self.readonly = bool(readonly) self.backend = Backend(trackers, timeout)
class Admin(object):
def __init__(self, trackers, readonly=False, timeout=None):
self.readonly = bool(readonly)
self.backend = Backend(trackers, timeout)
def replicate_now(self):
return self.backend.do_request("replicate_now")
def get_hosts(self, hostid=None):
if hostid:
params = {'hostid': hostid}
else:
params = None
res = self.backend.do_request("get_hosts", params)
print(res)
results = []
fields = ["hostid",
"status",
"hostname",
"hostip",
"http_port",
"http_get_port",
"altip altmask"]
hosts = int(res['hosts']) + 1
for ct in range(1, hosts):
results.append(dict([(f, res.get('host%d_%s' % (ct, f))) for f in fields]))
return results
def get_devices(self, devid=None):
if devid:
params = {'devid': devid}
else:
params = None
res = self.backend.do_request("get_devices", params)
ret = []
for x in range(1, int(res['devices']) + 1):
device = {}
for k in ('devid', 'hostid', 'status', 'observed_state', 'utilization'):
device[k] = res.get('dev%d_%s' % (x, k))
for k in ('mb_total', 'mb_used', 'weight'):
value = res.get('dev%d_%s' % (x, k))
if value:
device[k] = int(value)
else:
device[k] = None
ret.append(device)
return ret
def list_fids(self, from_fid, to_fid):
"""
get raw information about fids, for enumerating the dataset
( from_fid, to_fid )
returns:
{fid => {dict with keys: domain, class, devcount, length, key}}
"""
res = self.backend.do_request('list_fids', {'from': from_fid, 'to': to_fid})
results = {}
for x in range(1, int(res['fid_count']) + 1):
key = 'fid_%d_fid' % x
results[key] = dict([(k, res['fid_%d_%s' % (x, k)]) \
for k in ('key', 'length', 'class', 'domain', 'devcount')])
return results
def clear_cache(self):
return self.backend.do_request('clear_cache')
def get_domains(self):
"""
get a dict of the domains we know about in the format of
{
"domain_name": {
"class_name": mindevcount,
"class_name": mindevcount,
...
},
...
}
"""
res = self.backend.do_request('get_domains')
domain_length = int(res['domains'])
ret = {}
for x in range(1, domain_length + 1):
domain_name = res['domain%d' % x]
ret.setdefault(domain_name, {})
class_length = int(res['domain%dclasses' % x])
for y in range(1, class_length + 1):
k = 'domain%dclass%dname' % (x, y)
v = 'domain%dclass%dmindevcount' % (x, y)
ret[domain_name][res[k]] = int(res[v])
return ret
def create_domain(self, domain):
"""
create a new domain
"""
if self.readonly:
return False
res = self.backend.do_request('create_domain', {'domain': domain})
if res['domain'] == domain:
return True
else:
return False
def delete_domain(self, domain):
"""
delete a domain
"""
if self.readonly:
return False
try:
res = self.backend.do_request('delete_domain', {'domain': domain})
except MogileFSError:
return False
if res['domain'] == domain:
return True
else:
return False
def create_class(self, domain, cls, mindevcount):
"""
create a class within a domain
"""
return self._modify_class('create', domain, cls, mindevcount)
def update_class(self, domain, cls, mindevcount):
"""
update a class's mindevcount within a domain
"""
return self._modify_class('update', domain, cls, mindevcount)
def delete_class(self, domain, cls):
"""
delete a class
"""
if self.readonly:
return False
res = self.backend.do_request("delete_class",
{'domain': domain, 'class': cls})
if res['class'] == cls:
return True
else:
return False
def create_host(self, host, ip, port, getport=None, status=None):
params = {'host': host, 'ip': ip, 'port': port, 'getport': getport}
if status:
params['status'] = status
return self._modify_host('create', params)
def update_host(self, host, ip=None, port=None, getport=None, status=None):
params = {'host': host}
if ip:
params['ip'] = ip
if port:
params['port'] = port
if getport:
params['getport'] = True
if status:
params['status'] = status
return self._modify_host('update', params)
def delete_host(self, host):
self.backend.do_request("delete_host", {'host': host})
def create_device(self, hostname, devid, hostip=None, state=None):
params = {'hostname': hostname, 'devid': devid}
if hostip:
params['hostip'] = hostip
if state:
params['state'] = state
return self.backend.do_request('create_device', params)
def update_device(self, host, device, status=None, weight=None):
if status:
self.change_device_state(host, device, status)
if weight:
self.change_device_weight(host, device, weight)
return True
def change_device_state(self, host, device, state):
"""
change the state of a device; pass in the hostname of the host the
device is located on, the device id number, and the state you want
the host to be set to.
"""
params = {'host': host, 'device': device, 'state': state}
return self.backend.do_request('set_state', params)
def change_device_weight(self, host, device, weight):
"""
change the weight of a device by passing in the hostname and
the device id
"""
if not isinstance(weight, six.integer_types):
raise ValueError('argument weight muse be an integer')
params = {'host': host, 'device': device, 'weight': weight}
return self.backend.do_request('set_weight', params)
def _get_slave_keys(self):
res = self.backend.do_request("server_setting", {"key": "slave_keys"})
if not res:
return {}
value = res['value']
slave_keys = {}
for slave in value.split(','):
key, weight = (slave.split("=", 1) + [None])[:2]
# Weight can be zero,
# so don't default to 1 if it's defined and longer than 0 characters.
try:
weight = int(weight)
if not weight:
weight = 1
except (TypeError, ValueError):
weight = 1
slave_keys[key] = weight
return slave_keys
def _set_slave_keys(self, keys):
buf = []
for key, weight in keys.items():
try:
weight = int(weight)
if weight != 1:
key = "%s=%d" % (key, weight)
except (TypeError, ValueError):
pass
buf.append(key)
self.backend.do_request("set_server_setting",
{'key': 'slave_keys', 'value': ''.join(buf)})
def slave_list(self):
keys = self._get_slave_keys()
ret = {}
for key in keys:
res = self.backend.do_request('server_setting',
{'key': 'slave_%s' % key})
if not res:
continue
value = res['value']
dsn, username, password = (value.split('|') + ['', ''])[:3]
ret[key] = (dsn, username, password)
return ret
def slave_add(self, key, dsn, username, password):
keys = self._get_slave_keys()
if key in keys:
return
value = '|'.join([dsn, username, password])
self.backend.do_request("set_server_setting",
{'key': key, 'value': value})
keys[key] = None
self._set_slave_keys(keys)
def slave_modify(self, key, **opts):
keys = self._get_slave_keys()
if key not in keys:
# slave not fuond
return
res = self.backend.do_request("server_setting", {"key": "slave_%s" % key})
value = res['value']
dsn, username, password = (value.split('|') + ['', ''])[:3]
dsn = opts.get('dsn') or dsn
username = opts.get('username') or username
password = opts.get('password') or password
value = '|'.join([dsn, username, password])
res = self.backend.do_request('set_server_setting',
{'key': 'slave_%s' % key, 'value': value})
def slave_delete(self, key):
slave_keys = self._get_slave_keys()
if not slave_keys:
return None
if key not in slave_keys:
return False
self.backend.do_request('set_server_setting',
{key: "slave_%s" % key})
del slave_keys[key]
self._set_slave_keys(slave_keys)
def fsck_start(self):
return self.backend.do_request("fsck_start")
def fsck_stop(self):
return self.backend.do_request("fsck_stop")
def fsck_reset(self, policy_only, startpos):
return self.backend.do_request("fsck_reset",
{'policy_only': policy_only, 'startpos': startpos})
def fsck_clearlog(self):
return self.backend.do_request("fsck_clearlog")
def fsck_status(self):
return self.backend.do_request("fsck_status")
def fsck_log_rows(self, after_logid=None):
params = {}
if after_logid:
params['after_logid'] = after_logid
res = self.backend.do_request("fsck_getlog", params)
row_count = int(res['row_count'])
ret = []
for x in range(1, row_count + 1):
rec = {}
for k in ("logid", "utime", "fid", "evcode", "devid"):
rec[k] = res.get("row_%d_%s" % (x, k))
ret.append(rec)
return ret
def set_server_setting(self, key, value):
params = {'key': key, 'value': value}
return self.backend.do_request("set_server_setting", params)
def server_settings(self):
res = self.backend.do_request("server_settings")
if not res:
return
ret = {}
for x in range(1, int(res["key_count"]) + 1):
key = res.get("key_%d" % x, '')
value = res.get("value_%d" % x, '')
ret[key] = value
return ret
def _modify_class(self, verb, domain, cls, mindevcount, replpolicy=None):
if self.readonly:
return False
params = {'domain': domain,
'class': cls,
'mindevcount': mindevcount}
if replpolicy:
params['replpolicy'] = replpolicy
res = self.backend.do_request("%s_class" % verb, params)
if res['class'] == cls:
return True
else:
return False
def _modify_host(self, verb, params):
if self.readonly:
return False
try:
return self.backend.do_request("%s_host" % verb, params)
except MogileFSError:
return False
## Extra
#
def get_freespace(self, devid=None):
"""Get the free space for the entire cluster, or a specific node"""
return sum([x['mb_free'] for x in self.get_devices(devid)])
def get_stats(self):
params = {'all': 1}
res = self.backend.do_request('stats', params)
ret = {}
# get replication statistics
if 'replicationcount' in res:
replication = ret.setdefault('replication', {})
for x in range(1, int(res['replicationcount']) + 1):
domain = res.get('replication%ddomain' % x, '')
cls = res.get('replication%dclass' % x, '')
devcount = res.get('replication%ddevcount' % x, '')
fields = res.get('replication%dfields' % x)
(replication.setdefault(domain, {}).setdefault(cls, {}))[devcount] = fields
# get file statistics
if 'filescount' in res:
files = ret.setdefault('files', {})
for x in range(1, int(res['filescount']) + 1):
domain = res.get('files%ddomain' % x, '')
cls = res.get('files%dclass' % x, '')
(files.setdefault(domain, {}))[cls] = res.get('files%dfiles' % x)
# get device statistics
if 'devicescount' in res:
devices = ret.setdefault('devices', {})
for x in range(1, int(res['devicescount']) + 1):
key = res.get('devices%did' % x, '')
devices[key] = {'host': res.get('devices%dhost' % x),
'status': res.get('devices%dstatus' % x),
'files': res.get('devices%dfiles' % x),
}
if 'fidmax' in res:
ret['fids'] = {'max': res['fidmax']}
# return the created response
return ret
def __init__(self, hosts, backend=None, readonly=False, timeout=None, hooks=None): self.readonly = bool(readonly) self.backend = Backend(hosts, timeout) self._hosts = hosts
class Client(object):
def __init__(self, domain, trackers, readonly=False):
"""Create new Client object with the given list of trackers."""
self.readonly = bool(readonly)
self.domain = domain
self.backend = Backend(trackers, timeout=3)
def run_hook(self, hookname, *args):
pass
def add_hook(self, hookname, *args):
pass
def add_backend_hook(self):
raise NotImplementedError()
def errstr(self):
raise NotImplementedError()
def errcode(self):
raise NotImplementedError()
@property
def last_tracker(self):
"""
Returns a tuple of (ip, port), representing the last mogilefsd
'tracker' server which was talked to.
"""
return self.backend.last_host_connected
def new_file(self,
key,
cls=None,
largefile=False,
content_length=0,
create_open_arg=None,
create_close_arg=None,
opts=None):
"""
Start creating a new filehandle with the given key,
and option given class and options.
Returns a filehandle you should then print to,
and later close to complete the operation.
NOTE: check the return value from close!
If your close didn't succeed, the file didn't get saved!
"""
self.run_hook('new_file_start', key, cls, opts)
create_open_arg = create_open_arg or {}
create_close_arg = create_close_arg or {}
# fid should be specified, or pass 0 meaning to auto-generate one
fid = 0
params = {
'domain': self.domain,
'key': key,
'fid': fid,
'multi_dest': 1
}
if cls is not None:
params['class'] = cls
res = self.backend.do_request('create_open', params)
if not res:
return None
# [(devid, path), (devid, path),... ]
dests = []
# determine old vs. new format to populate destinations
if 'dev_count' not in res:
dests.append((res['devid'], res['path']))
else:
for x in xrange(1, int(res['dev_count']) + 1):
devid_key = 'devid_%d' % x
path_key = 'path_%s' % x
dests.append((res[devid_key], res[path_key]))
main_dest = dests[0]
main_devid, main_path = main_dest
self.run_hook("new_file_end", key, cls, opts)
# TODO
if largefile:
file_class = LargeHTTPFile
else:
file_class = NormalHTTPFile
return file_class(
mg=self,
fid=res['fid'],
path=main_path,
devid=main_devid,
backup_dests=dests,
cls=cls,
key=key,
content_length=content_length,
create_close_arg=create_close_arg,
overwrite=1
)
def edit_file(self, key, overwrite=False):
"""Edit the file with the the given key.
NOTE: edit_file is currently EXPERIMENTAL and not recommended for production use.
MogileFS is primarily designed for storing files for later retrieval, rather than editing.
Use of this function may lead to poor performance and, until it has been proven mature, should be considered to also potentially cause data loss.
NOTE: use of this function requires support for the DAV 'MOVE' verb and partial PUT (i.e. Content-Range in PUT) on the back-end storage servers (e.g. apache with mod_dav).
Returns a seekable filehandle you can read/write to.
Calling this function may invalidate some or all URLs you currently have for this key, so you should call ->get_paths again afterwards if you need them.
On close of the filehandle, the new file contents will replace the previous contents (and again invalidate any existing URLs).
By default, the file contents are preserved on open, but you may specify the overwrite option to zero the file first.
The seek position is at the beginning of the file, but you may seek to the end to append.
"""
raise NotImplementedError()
def read_file(self, key):
"""
Read the file with the the given key.
Returns a seekable filehandle you can read() from.
Note that you cannot read line by line using <$fh> notation.
Takes the same options as get_paths
(which is called internally to get the URIs to read from).
"""
paths = self.get_paths(key)
if not paths:
return None
path = paths[0]
backup_dests = [(None, p) for p in paths[1:]]
return LargeHTTPFile(path=path, backup_dests=backup_dests, readonly=1)
def store_file(self, key, fp, cls=None, chunk_size=8192):
"""
Wrapper around new_file, print, and close.
Given a key, class, and a filehandle or filename, stores the file
contents in MogileFS. Returns the number of bytes stored on success,
undef on failure.
"""
if self.readonly:
return False
params = {}
if chunk_size:
params['chunk_size'] = chunk_size
if cls:
params['class'] = cls
params[key] = key
self.run_hook('store_file_start', params)
try:
new_file = self.new_file(key, cls)
except MogileFSError:
fp.close()
return False
try:
_bytes = 0
while True:
buf = fp.read(chunk_size)
if not buf:
break
_bytes += len(buf)
new_file.write(buf)
self.run_hook('store_file_end', params)
finally:
fp.close()
new_file.close()
return _bytes
def store_content(self, key, content, cls=None, **opts):
"""
Wrapper around new_file, print, and close. Given a key, class, and
file contents (scalar or scalarref), stores the file contents in
MogileFS. Returns the number of bytes stored on success, undef on
failure.
"""
if self.readonly:
return False
self.run_hook('store_content_start', key, cls, opts)
output = self.new_file(key, cls, None, **opts)
output.write(content)
output.close()
self.run_hook('store_content_end', key, cls, opts)
return len(content)
def get_paths(self, key, noverify=1, zone='alt', pathcount=2):
"""
Given a key, returns an array of all the locations (HTTP URLs) that the file
has been replicated to.
"""
self.run_hook('get_paths_start', key)
params = {
'domain': self.domain,
'key': key,
'noverify': noverify and 1 or 0,
'zone': zone,
'pathcount': pathcount
}
res = self.backend.do_request('get_paths', params)
paths = [res["path%d" % x] for x in xrange(1, int(res["paths"]) + 1)]
self.run_hook('get_paths_end', key)
return paths
def get_file_data(self, key, timeout=10):
"""
Returns scalarref of file contents in a scalarref.
Don't use for large data, as it all comes back to you in one string.
"""
fp = self.read_file(key)
if not fp:
return None
try:
content = fp.read()
return content
finally:
fp.close()
def delete(self, key):
"""
Delete a file from MogileFS
"""
try:
if self.readonly:
return False
self.backend.do_request(
'delete', {'domain': self.domain,
'key': key}
)
return True
except MogileFSError:
return False
def rename(self, from_key, to_key):
"""
Rename file (key) in MogileFS from oldkey to newkey.
Returns true on success, failure otherwise
"""
try:
if self.readonly:
return False
params = {
'domain': self.domain,
'from_key': from_key,
'to_key': to_key
}
self.backend.do_request('rename', params)
return True
except MogileFSError:
return False
def file_debug(self, **kwargs):
"""
Thoroughly search for any database notes about a particular fid.
Searchable by raw fid, or by domain and key. **Use sparingly**.
Command hits the master database numerous times, and if you're using
it in production something is likely very wrong.
To be used with troubleshooting broken/odd files and errors from
mogilefsd.
"""
if 'key' not in kwargs and 'fid' not in kwargs:
raise TypeError('file_debug() missing 1 required '
'positional argument: fid/key')
params = {
'domain': kwargs.get('domain', self.domain)
}
params.update(kwargs)
return self.backend.do_request('file_debug', params)
def file_info(self, key, devices=False):
"""
Used to return metadata about a file. Returns the domain, class,
expected length, devcount, etc. Optionally device ids (not paths)
can be returned as well.
Should be used for informational purposes, and not usually for
dynamically serving files.
"""
params = {
'domain': self.domain,
'key': key
}
if devices:
params['devices'] = True
info = self.backend.do_request('file_info', params)
info['devcount'] = int(info['devcount'])
info['length'] = int(info['length'])
if 'devids' in info:
info['devids'] = info['devids'].split(',')
return info
def list_keys(self, prefix=None, after=None, limit=None):
"""
Used to get a list of keys matching a certain prefix.
$prefix specifies what you want to get a list of. $after is the item
specified as a return value from this function last time you called
it. `$limit` is optional and defaults to 1000 keys returned.
In list context, returns ($after, $keys). In scalar context, returns
arrayref of keys. The value $after is to be used as $after when you
call this function again.
When there are no more keys in the list, you will get back undef or
an empty list.
"""
params = {'domain': self.domain}
if prefix:
params['prefix'] = prefix
if after:
params['after'] = after
if limit:
params['limit'] = limit
res = self.backend.do_request('list_keys', params)
results = []
for x in xrange(1, int(res['key_count']) + 1):
results.append(res['key_%d' % x])
return results
def keys(self, prefix=None):
"""
Get all keys matching a certain prefix
"""
params = {'domain': self.domain}
if prefix:
params['prefix'] = prefix
results = []
while True:
res = self.backend.do_request('list_keys', params)
for x in res.keys():
if x not in ['key_count', 'next_after']:
results.append(res[x])
if len(res) < 1002:
break
params['after'] = res['next_after']
return list(set(results))
def foreach_key(self, *args, **kwds):
"""
Functional interface/wrapper around list_keys.
Given some %OPTIONS (currently only one, "prefix"),
calls your callback for each key matching the provided prefix.
"""
raise NotImplementedError()
def update_class(self, key, new_class):
"""
Update the replication class of a pre-existing file,
causing the file to become more or less replicated.
"""
try:
if self.readonly:
return False
params = {"domain": self.domain, "key": key, "class": new_class}
res = self.backend.do_request("updateclass", params)
return res
except MogileFSError:
return False
def sleep(self, duration):
"""
just makes some sleeping happen. first and only argument is number of
seconds to instruct backend thread to sleep for.
"""
try:
self.backend.do_request("sleep", {'duration': duration})
return True
except MogileFSError:
return False
def set_pref_ip(self, *ips):
"""
Weird option for old, weird network architecture.
Sets a mapping table of preferred alternate IPs, if reachable.
For instance, if trying to connect to 10.0.0.2 in the above example,
the module would instead try to connect to 10.2.0.2 quickly first,
then then fall back to 10.0.0.2 if 10.2.0.2 wasn't reachable.
expects as argument a tuple of ("standard-ip", "preferred-ip")
"""
self.backend.set_pref_ip(*ips)
def __init__(self, domain, trackers, readonly=False): """Create new Client object with the given list of trackers.""" self.readonly = bool(readonly) self.domain = domain self.backend = Backend(trackers, timeout=3)
class Client(object):
def __init__(self, domain, trackers, readonly=False):
"""Create new Client object with the given list of trackers."""
self.readonly = bool(readonly)
self.domain = domain
self.backend = Backend(trackers, timeout=3)
def run_hook(self, hookname, *args):
pass
def add_hook(self, hookname, *args):
pass
def add_backend_hook(self):
raise NotImplementedError()
def errstr(self):
raise NotImplementedError()
def errcode(self):
raise NotImplementedError()
@property
def last_tracker(self):
"""
Returns a tuple of (ip, port), representing the last mogilefsd
'tracker' server which was talked to.
"""
return self.backend.last_host_connected
def new_file(self, key, cls=None, largefile=False, content_length=0,
create_open_arg=None, create_close_arg=None, opts=None):
"""
Start creating a new filehandle with the given key,
and option given class and options.
Returns a filehandle you should then print to,
and later close to complete the operation.
NOTE: check the return value from close!
If your close didn't succeed, the file didn't get saved!
"""
self.run_hook('new_file_start', key, cls, opts)
create_open_arg = create_open_arg or {}
create_close_arg = create_close_arg or {}
# fid should be specified, or pass 0 meaning to auto-generate one
fid = 0
params = {'domain': self.domain,
'key': key,
'fid': fid,
'multi_dest': 1}
if cls is not None:
params['class'] = cls
res = self.backend.do_request('create_open', params)
if not res:
return None
# [(devid, path), (devid, path),... ]
dests = []
# determine old vs. new format to populate destinations
if 'dev_count' not in res:
dests.append((res['devid'], res['path']))
else:
for x in range(1, int(res['dev_count']) + 1):
devid_key = 'devid_%d' % x
path_key = 'path_%s' % x
dests.append((res[devid_key], res[path_key]))
main_dest = dests[0]
main_devid, main_path = main_dest
self.run_hook("new_file_end", key, cls, opts)
# TODO
if largefile:
file_class = LargeHTTPFile
else:
file_class = NormalHTTPFile
return file_class(mg=self,
fid=res['fid'],
path=main_path,
devid=main_devid,
backup_dests=dests,
cls=cls,
key=key,
content_length=content_length,
create_close_arg=create_close_arg,
overwrite=1)
def edit_file(self, key, overwrite=False):
"""Edit the file with the the given key.
NOTE: edit_file is currently EXPERIMENTAL and not recommended for production use.
MogileFS is primarily designed for storing files for later retrieval, rather than editing.
Use of this function may lead to poor performance and, until it has been proven mature, should be considered to also potentially cause data loss.
NOTE: use of this function requires support for the DAV 'MOVE' verb and partial PUT (i.e. Content-Range in PUT) on the back-end storage servers (e.g. apache with mod_dav).
Returns a seekable filehandle you can read/write to.
Calling this function may invalidate some or all URLs you currently have for this key, so you should call ->get_paths again afterwards if you need them.
On close of the filehandle, the new file contents will replace the previous contents (and again invalidate any existing URLs).
By default, the file contents are preserved on open, but you may specify the overwrite option to zero the file first.
The seek position is at the beginning of the file, but you may seek to the end to append.
"""
raise NotImplementedError()
def read_file(self, key):
"""
Read the file with the the given key.
Returns a seekable filehandle you can read() from.
Note that you cannot read line by line using <$fh> notation.
Takes the same options as get_paths
(which is called internally to get the URIs to read from).
"""
paths = self.get_paths(key)
if not paths:
return None
path = paths[0]
backup_dests = [(None, p) for p in paths[1:]]
return LargeHTTPFile(path=path, backup_dests=backup_dests, readonly=1)
def store_file(self, key, fp, cls=None, chunk_size=8192):
"""
Wrapper around new_file, print, and close.
Given a key, class, and a filehandle or filename, stores the file
contents in MogileFS. Returns the number of bytes stored on success,
undef on failure.
"""
if self.readonly:
return False
params = {}
if chunk_size:
params['chunk_size'] = chunk_size
if cls:
params['class'] = cls
params[key] = key
self.run_hook('store_file_start', params)
try:
new_file = self.new_file(key, cls)
except MogileFSError:
fp.close()
return False
try:
_bytes = 0
while True:
buf = fp.read(chunk_size)
if not buf:
break
_bytes += len(buf)
new_file.write(buf)
self.run_hook('store_file_end', params)
finally:
fp.close()
new_file.close()
return _bytes
def store_content(self, key, content, cls=None, **opts):
"""
Wrapper around new_file, print, and close. Given a key, class, and
file contents (scalar or scalarref), stores the file contents in
MogileFS. Returns the number of bytes stored on success, undef on
failure.
"""
if self.readonly:
return False
self.run_hook('store_content_start', key, cls, opts)
output = self.new_file(key, cls, None, **opts)
output.write(content)
output.close()
self.run_hook('store_content_end', key, cls, opts)
return len(content)
def get_paths(self, key, noverify=1, zone='alt', pathcount=2):
"""
Given a key, returns an array of all the locations (HTTP URLs) that the file
has been replicated to.
"""
self.run_hook('get_paths_start', key)
params = {'domain': self.domain,
'key': key,
'noverify': noverify and 1 or 0,
'zone': zone,
'pathcount': pathcount}
res = self.backend.do_request('get_paths', params)
paths = [res["path%d" % x] for x in range(1, int(res["paths"]) + 1)]
self.run_hook('get_paths_end', key)
return paths
def get_file_data(self, key, timeout=10):
"""
Returns scalarref of file contents in a scalarref.
Don't use for large data, as it all comes back to you in one string.
"""
fp = self.read_file(key)
if not fp:
return None
try:
content = fp.read()
return content
finally:
fp.close()
def delete(self, key):
"""
Delete a file from MogileFS
"""
try:
if self.readonly:
return False
self.backend.do_request('delete', {'domain': self.domain, 'key': key})
return True
except MogileFSError:
return False
def rename(self, from_key, to_key):
"""
Rename file (key) in MogileFS from oldkey to newkey.
Returns true on success, failure otherwise
"""
try:
if self.readonly:
return False
params = {'domain': self.domain,
'from_key': from_key,
'to_key': to_key}
self.backend.do_request('rename', params)
return True
except MogileFSError:
return False
def list_keys(self, prefix=None, after=None, limit=None):
"""
Used to get a list of keys matching a certain prefix.
$prefix specifies what you want to get a list of.
$after is the item specified as a return value from this function last time
you called it.
$limit is optional and defaults to 1000 keys returned.
In list context, returns ($after, $keys).
In scalar context, returns arrayref of keys.
The value $after is to be used as $after when you call this function again.
When there are no more keys in the list,
you will get back undef or an empty list
"""
params = {'domain': self.domain}
if prefix:
params['prefix'] = prefix
if after:
params['after'] = after
if limit:
params['limit'] = limit
res = self.backend.do_request('list_keys', params)
results = []
for x in range(1, int(res['key_count']) + 1):
results.append(res['key_%d' % x])
return results
def keys(self, prefix=None):
"""
Get all keys matching a certain prefix
"""
params = {'domain': self.domain}
if prefix:
params['prefix'] = prefix
results = []
while True:
res = self.backend.do_request('list_keys', params)
for x in res.keys():
if x not in ['key_count', 'next_after']:
results.append(res[x])
if len(res) < 1002:
break
params['after'] = res['next_after']
return list(set(results))
def foreach_key(self, *args, **kwds):
"""
Functional interface/wrapper around list_keys.
Given some %OPTIONS (currently only one, "prefix"),
calls your callback for each key matching the provided prefix.
"""
raise NotImplementedError()
def update_class(self, key, new_class):
"""
Update the replication class of a pre-existing file,
causing the file to become more or less replicated.
"""
try:
if self.readonly:
return False
params = {"domain": self.domain,
"key": key,
"class": new_class}
res = self.backend.do_request("updateclass", params)
return res
except MogileFSError:
return False
def sleep(self, duration):
"""
just makes some sleeping happen. first and only argument is number of
seconds to instruct backend thread to sleep for.
"""
try:
self.backend.do_request("sleep", {'duration': duration})
return True
except MogileFSError:
return False
def set_pref_ip(self, *ips):
"""
Weird option for old, weird network architecture.
Sets a mapping table of preferred alternate IPs, if reachable.
For instance, if trying to connect to 10.0.0.2 in the above example,
the module would instead try to connect to 10.2.0.2 quickly first,
then then fall back to 10.0.0.2 if 10.2.0.2 wasn't reachable.
expects as argument a tuple of ("standard-ip", "preferred-ip")
"""
self.backend.set_pref_ip(*ips)
def setUp(self): self.backend = Backend(['127.0.0.1:7001'])
def test_do_request_one_tracker_down(self):
backend = Backend(["127.0.0.1:7001", "127.0.0.1:7012"])
try:
backend.do_request("get_domains")
except MogileFSError:
assert False
def __init__(self, domain, hosts, readonly=False): self.readonly = bool(readonly) self.domain = domain self.backend = Backend(hosts, timeout=3)
def test_do_request_one_tracker_down(self):
backend = Backend(["127.0.0.1:7001", "127.0.0.1:7012"])
try:
backend.do_request("get_domains")
except MogileFSError:
assert False
def setUp(self):
self.backend = Backend(['127.0.0.1:7001'])
def __init__(self, domain, trackers, readonly=False):
"""Create new Client object with the given list of trackers."""
self.readonly = bool(readonly)
self.domain = domain
self.backend = Backend(trackers, timeout=3)
class Admin(object):
def __init__(self, hosts, backend=None, readonly=False, timeout=None, hooks=None):
self.readonly = bool(readonly)
self.backend = Backend(hosts, timeout)
self._hosts = hosts
def replicate_row(self):
self.backend.do_request("replicate_row")
def get_hosts(self, hostid=None):
if hostid:
params = {'hostid': hostid}
else:
params = None
res = self.backend.do_request("get_hosts", params)
ret = []
fields = "hostid status hostname hostip http_port".split()
hosts = int(res['hosts']) + 1
for ct in xrange(1, hosts):
ret.append(dict([ (f, res['host%d_%s' % (ct, f)]) for f in fields]))
return ret
def get_devices(self, devid=None):
if devid:
params = {'devid': devid}
else:
params = None
res = self.backend.do_request("get_devices", params)
ret = []
for x in xrange(1, int(res['devices'])+1):
device = {}
for k in ('devid', 'hostid', 'status', 'observed_state', 'utilization'):
device[k] = res.get('dev%d_%s' % (x, k))
for k in ('mb_total', 'mb_used', 'weight'):
value = res.get('dev%d_%s' % (x, k))
if value:
device[k] = int(value)
else:
device[k] = None
ret.append(device)
return ret
def get_freespace(self, devid=None):
"""Get the free space for the entire cluster, or a specific node"""
return sum([x['mb_free'] for x in self.get_devices(devid)])
def list_fids(self, fromfid, tofid):
"""
get raw information about fids, for enumerating the dataset
( from_fid, to_fid )
returns:
{ fid => { dict with keys: domain, class, devcount, length, key } }
"""
res = self.backend.do_request('list_fids',
{ 'from': fromfid,
'to' : tofid,
})
ret = {}
for x in xrange(1, int(res['fid_count'])+1):
key = 'fid_%d_fid' % x
ret[key] = dict([(k, res['fid_%d_%s' % (x, k)]) for k in ('key', 'length', 'class', 'domain', 'devcount')])
return ret
def clear_cache(self, fromfid, tofid):
params = {}
return self.backend.do_request('clear_cache', params)
def get_stats(self):
params = { 'all': 1 }
res = self.backend.do_request('stats', params)
ret = {}
# get replication statistics
if 'replicationcount' in res:
replication = ret.setdefault('replication', {})
for x in xrange(1, int(res['replicationcount'])+1):
domain = res.get('replication%ddomain' % x, '')
cls = res.get('replication%dclass' % x, '')
devcount = res.get('replication%ddevcount' % x, '')
fields = res.get('replication%dfields' % x)
(replication.setdefault(domain, {}).setdefault(cls, {}))[devcount] = fields
# get file statistics
if 'filescount' in res:
files = ret.setdefault('files', {})
for x in xrange(1, int(res['filescount'])+1):
domain = res.get('files%ddomain' % x, '')
cls = res.get('files%dclass' % x, '')
(files.setdefault(domain, {}))[cls] = res.get('files%dfiles' % x)
# get device statistics
if 'devicescount' in res:
devices = ret.setdefault('devices', {})
for x in xrange(1, int(res['devicescount'])+1):
key = res.get('devices%did' % x, '')
devices[key] = { 'host' : res.get('devices%dhost' % x),
'status': res.get('devices%dstatus' % x),
'files' : res.get('devices%dfiles' % x),
}
if 'fidmax' in res:
ret['fids'] = { 'max': res['fidmax'],
}
# return the created response
return ret
def get_domains(self):
"""
get a dict of the domains we know about in the format of
{ domain_name : { class_name => mindevcount, class_name => mindevcount, ... }, ... }
"""
res = self.backend.do_request('get_domains')
## KeyError, ValueError, TypeError
domain_length = int(res['domains'])
ret = {}
for x in xrange(1, domain_length+1):
domain_name = res['domain%d' % x]
ret.setdefault(domain_name, {})
class_length = int(res['domain%dclasses' % x])
for y in xrange(1, class_length+1):
k = 'domain%dclass%dname' % (x, y)
v = 'domain%dclass%dmindevcount' % (x, y)
ret[domain_name][res[k]] = int(res[v])
return ret
def create_domain(self, domain):
"""
create a new domain
"""
if self.readonly:
return
res = self.backend.do_request('create_domain',
{ 'domain': domain })
return res['domain'] == domain
def delete_domain(self, domain):
"""
delete a domain
"""
_complain_ifreadonly(self.readonly)
res = self.backend.do_request('delete_domain',
{ 'domain': domain })
return res['domain'] == domain
def create_class(self, domain, cls, mindevcount):
"""
create a class within a domain
"""
try:
return self._modify_class('create', domain, cls, mindevcount)
except MogileFSTrackerError, e:
if e.err != 'class_exists':
raise e
