def test_do_request_host_not_exist():
backend = Backend(["127.0.0.1:7011", "127.0.0.1:7012"])
try:
backend.do_request("get_domains")
except MogileFSError:
pass
else:
assert False
def __init__(self, hosts, backend=None, readonly=False, timeout=None, hooks=None):
self.readonly = bool(readonly)
self.backend = Backend(hosts, timeout)
self._hosts = hosts
def __init__(self, domain, hosts, timeout=3, backend=None, readonly=False, hooks=None):
self.readonly = bool(readonly)
self.domain = domain
self.backend = Backend(hosts, timeout)
class Admin(object):
def __init__(self, hosts, backend=None, readonly=False, timeout=None, hooks=None):
self.readonly = bool(readonly)
self.backend = Backend(hosts, timeout)
self._hosts = hosts
def replicate_row(self):
self.backend.do_request("replicate_row")
def get_hosts(self, hostid=None):
if hostid:
params = { 'hostid': hostid }
else:
params = None
res = self.backend.do_request("get_hosts", params)
def get_devices(self, devid=None):
if devid:
params = { 'devid': devid }
else:
params = None
res = self.backend.do_request("get_devices", params)
ret = []
for x in xrange(1, int(res['devices'])+1):
device = {}
for k in ('devid', 'hostid', 'status', 'observed_state', 'utilization'):
device[k] = res.get('dev%d_%s' % (x, k))
for k in ('mb_total', 'mb_used', 'weight'):
value = res.get('dev%d_%s' % (x, k))
if value:
device[k] = int(value)
else:
device[k] = None
ret.append(device)
return ret
def list_fids(self, fromfid, tofid):
"""
get raw information about fids, for enumerating the dataset
( from_fid, to_fid )
returns:
{ fid => { dict with keys: domain, class, devcount, length, key } }
"""
res = self.backend.do_request('list_fids',
{ 'from': fromfid,
'to' : tofid,
})
ret = {}
for x in xrange(1, int(res['fid_count'])+1):
key = 'fid_%d_fid' % x
ret[key] = dict([(k, res['fid_%d_%s' % (x, k)]) for k in ('key', 'length', 'class', 'domain', 'devcount')])
return ret
def clear_cache(self, fromfid, tofid):
params = {}
res = self.backend.do_request('clear_cache', params)
def get_stats(self):
params = { 'all': 1 }
res = self.backend.do_request('stats', params)
ret = {}
# get replication statistics
if 'replicationcount' in res:
replication = ret.setdefault('replication', {})
for x in xrange(1, int(res['replicationcount'])+1):
domain = res.get('replication%ddomain' % x, '')
cls = res.get('replication%dclass' % x, '')
devcount = res.get('replication%ddevcount' % x, '')
fields = res.get('replication%dfields' % x)
(replication.setdefault(domain, {}).setdefault(cls, {}))[devcount] = fields
# get file statistics
if 'filescount' in res:
files = ret.setdefault('files', {})
for x in xrange(1, int(res['filescount'])+1):
domain = res.get('files%ddomain' % x, '')
cls = res.get('files%dclass' % x, '')
(files.setdefault(domain, {}))[cls] = res.get('files%dfiles' % x)
# get device statistics
if 'devicescount' in res:
devices = ret.setdefault('devices', {})
for x in xrange(1, int(res['devicescount'])+1):
key = res.get('devices%did' % x, '')
devices[key] = { 'host' : res.get('devices%dhost' % x),
'status': res.get('devices%dstatus' % x),
'files' : res.get('devices%dfiles' % x),
}
if 'fidmax' in res:
ret['fids'] = { 'max': res['fidmax'],
}
# return the created response
return ret
def get_domains(self):
"""
get a dict of the domains we know about in the format of
{ domain_name : { class_name => mindevcount, class_name => mindevcount, ... }, ... }
"""
res = self.backend.do_request('get_domains')
## KeyError, ValueError, TypeError
domain_length = int(res['domains'])
ret = {}
for x in xrange(1, domain_length+1):
domain_name = res['domain%d' % x]
ret.setdefault(domain_name, {})
class_length = int(res['domain%dclasses' % x])
for y in xrange(1, class_length+1):
k = 'domain%dclass%dname' % (x, y)
v = 'domain%dclass%dmindevcount' % (x, y)
ret[domain_name][res[k]] = int(res[v])
return ret
def create_domain(self, domain):
"""
create a new domain
"""
if self.readonly:
return
res = self.backend.do_request('create_domain',
{ 'domain': domain })
return res['domain'] == domain
def delete_domain(self, domain):
"""
delete a domain
"""
_complain_ifreadonly(self.readonly)
res = self.backend.do_request('delete_domain',
{ 'domain': domain })
return res['domain'] == domain
def create_class(self, domain, cls, mindevcount):
"""
create a class within a domain
"""
try:
return self._modify_class('create', domain, cls, mindevcount)
except MogileFSTrackerError, e:
if e.err != 'class_exists':
raise e
class Client(object):
def __init__(self, domain, hosts, timeout=3, backend=None, readonly=False, hooks=None):
self.readonly = bool(readonly)
self.domain = domain
self.backend = Backend(hosts, timeout)
def run_hook(self, hookname, *args):
pass
def add_hook(self, hookname, *args):
pass
def get_last_tracker(self):
"""
Returns a tuple of (ip, port), representing the last mogilefsd
'tracker' server which was talked to.
"""
return self.backend.get_last_tracker()
last_tracker = property(get_last_tracker)
def new_file(self, key, cls=None, bytes=0, largefile=False, create_open_arg=None, create_close_arg=None, opts=None):
"""
- class
- key
- fid
- largefile
- create_open_arg
- create_close_arg
"""
self.run_hook('new_file_start', key, cls, opts)
create_open_arg = create_open_arg or {}
create_close_arg = create_close_arg or {}
# fid should be specified, or pass 0 meaning to auto-generate one
fid = 0
params = { 'domain' : self.domain,
'key' : key,
'fid' : fid,
'multi_dest': 1,
}
if cls is not None:
params['class'] = cls
res = self.backend.do_request('create_open', params)
if not res:
raise IOError()
# [ (devid,path), (devid,path), ... ]
dests = []
# determine old vs. new format to populate destinations
if 'dev_count' not in res:
dests.append((res['devid'], res['path']))
else:
for x in xrange(1, int(res['dev_count']) + 1):
devid_key = 'devid_%d' % x
path_key = 'path_%s' % x
dests.append((res[devid_key], res[path_key]))
main_dest = dests[0]
main_devid, main_path = main_dest
# TODO
# create a MogileFS::NewHTTPFile object, based off of IO::File
if not main_path.startswith('http://'):
raise MogileFSError("This version of mogilefs.client no longer supports non-http storage URLs.")
self.run_hook("new_file_end", key, cls, opts)
# TODO
if largefile:
file_cls = ClientHttpFile
else:
file_cls = NewHttpFile
return file_cls(mg=self,
fid=res['fid'],
path=main_path,
devid=main_devid,
backup_dests=dests,
cls=cls,
key=key,
content_length=bytes,
create_close_arg=create_close_arg,
overwrite=1,
)
def edit_file(self, key, **opts):
"""
EdiCt the file with the the given key.
B<NOTE:> edit_file is currently EXPERIMENTAL and not recommended for
production use. MogileFS is primarily designed for storing files
for later retrieval, rather than editing. Use of this function may lead to
poor performance and, until it has been proven mature, should be
considered to also potentially cause data loss.
B<NOTE:> use of this function requires support for the DAV 'MOVE'
verb and partial PUT (i.e. Content-Range in PUT) on the back-end
storage servers (e.g. apache with mod_dav).
Returns a seekable filehandle you can read/write to. Calling this
function may invalidate some or all URLs you currently have for this
key, so you should call ->get_paths again afterwards if you need
them.
On close of the filehandle, the new file contents will replace the
previous contents (and again invalidate any existing URLs).
By default, the file contents are preserved on open, but you may
specify the overwrite option to zero the file first. The seek position
is at the beginning of the file, but you may seek to the end to append.
"""
_complain_ifreadonly(self.readonly)
res = self.backend.do_request('edit_file',
{ 'domain': self.domain,
'key' : key,
})
oldpath = res['oldpath']
newpath = res['newpath']
fid = res['fid']
devid = res['devid']
cls = res['class']
## TODO
import httplib2
conn = httplib2.Http()
res, content = conn.request(oldpath,
"MOVE",
headers={ 'Destination': newpath,
})
status = int(res['status'])
if not (res.status >= 200 and res.status < 300):
raise IOError("failed to MOVE %s to %s" % (newpath, oldpath))
return ClientHttpFile(mg=self, path=newpath, fid=fid, devid=devid, cls=cls,
key=key, overwrite=opts.get('overwrite'))
def read_file(self, *args, **kwds):
paths = self.get_paths(*args, **kwds)
path = paths[0]
backup_dests = [(None, p) for p in paths[1:]]
return ClientHttpFile(path=path, backup_dests=backup_dests, readonly=1)
def get_paths(self, key, noverify=1, zone='alt', pathcount=None):
self.run_hook('get_paths_start', key)
extra_params = {}
params = { 'domain' : self.domain,
'key' : key,
'noverify': noverify and 1 or 0,
'zone' : zone,
}
params.update(extra_params)
try:
res = self.backend.do_request('get_paths', params)
paths = [res["path%d" % x] for x in xrange(1, int(res["paths"])+1)]
except MogileFSTrackerError, e:
if e.err == 'unknown_key':
paths = []
else:
raise e
self.run_hook('get_paths_end', key)
return paths
