def test_do_request_host_not_exist(): backend = Backend(["127.0.0.1:7011", "127.0.0.1:7012"]) try: backend.do_request("get_domains") except MogileFSError: pass else: assert False
class Admin(object): def __init__(self, hosts, backend=None, readonly=False, timeout=None, hooks=None): self.readonly = bool(readonly) self.backend = Backend(hosts, timeout) self._hosts = hosts def replicate_row(self): self.backend.do_request("replicate_row") def get_hosts(self, hostid=None): if hostid: params = { 'hostid': hostid } else: params = None res = self.backend.do_request("get_hosts", params) def get_devices(self, devid=None): if devid: params = { 'devid': devid } else: params = None res = self.backend.do_request("get_devices", params) ret = [] for x in xrange(1, int(res['devices'])+1): device = {} for k in ('devid', 'hostid', 'status', 'observed_state', 'utilization'): device[k] = res.get('dev%d_%s' % (x, k)) for k in ('mb_total', 'mb_used', 'weight'): value = res.get('dev%d_%s' % (x, k)) if value: device[k] = int(value) else: device[k] = None ret.append(device) return ret def list_fids(self, fromfid, tofid): """ get raw information about fids, for enumerating the dataset ( from_fid, to_fid ) returns: { fid => { dict with keys: domain, class, devcount, length, key } } """ res = self.backend.do_request('list_fids', { 'from': fromfid, 'to' : tofid, }) ret = {} for x in xrange(1, int(res['fid_count'])+1): key = 'fid_%d_fid' % x ret[key] = dict([(k, res['fid_%d_%s' % (x, k)]) for k in ('key', 'length', 'class', 'domain', 'devcount')]) return ret def clear_cache(self, fromfid, tofid): params = {} res = self.backend.do_request('clear_cache', params) def get_stats(self): params = { 'all': 1 } res = self.backend.do_request('stats', params) ret = {} # get replication statistics if 'replicationcount' in res: replication = ret.setdefault('replication', {}) for x in xrange(1, int(res['replicationcount'])+1): domain = res.get('replication%ddomain' % x, '') cls = res.get('replication%dclass' % x, '') devcount = res.get('replication%ddevcount' % x, '') fields = res.get('replication%dfields' % x) (replication.setdefault(domain, {}).setdefault(cls, {}))[devcount] = fields # get file statistics if 'filescount' in res: files = ret.setdefault('files', {}) for x in xrange(1, int(res['filescount'])+1): domain = res.get('files%ddomain' % x, '') cls = res.get('files%dclass' % x, '') (files.setdefault(domain, {}))[cls] = res.get('files%dfiles' % x) # get device statistics if 'devicescount' in res: devices = ret.setdefault('devices', {}) for x in xrange(1, int(res['devicescount'])+1): key = res.get('devices%did' % x, '') devices[key] = { 'host' : res.get('devices%dhost' % x), 'status': res.get('devices%dstatus' % x), 'files' : res.get('devices%dfiles' % x), } if 'fidmax' in res: ret['fids'] = { 'max': res['fidmax'], } # return the created response return ret def get_domains(self): """ get a dict of the domains we know about in the format of { domain_name : { class_name => mindevcount, class_name => mindevcount, ... }, ... } """ res = self.backend.do_request('get_domains') ## KeyError, ValueError, TypeError domain_length = int(res['domains']) ret = {} for x in xrange(1, domain_length+1): domain_name = res['domain%d' % x] ret.setdefault(domain_name, {}) class_length = int(res['domain%dclasses' % x]) for y in xrange(1, class_length+1): k = 'domain%dclass%dname' % (x, y) v = 'domain%dclass%dmindevcount' % (x, y) ret[domain_name][res[k]] = int(res[v]) return ret def create_domain(self, domain): """ create a new domain """ if self.readonly: return res = self.backend.do_request('create_domain', { 'domain': domain }) return res['domain'] == domain def delete_domain(self, domain): """ delete a domain """ _complain_ifreadonly(self.readonly) res = self.backend.do_request('delete_domain', { 'domain': domain }) return res['domain'] == domain def create_class(self, domain, cls, mindevcount): """ create a class within a domain """ try: return self._modify_class('create', domain, cls, mindevcount) except MogileFSTrackerError, e: if e.err != 'class_exists': raise e
class Client(object): def __init__(self, domain, hosts, timeout=3, backend=None, readonly=False, hooks=None): self.readonly = bool(readonly) self.domain = domain self.backend = Backend(hosts, timeout) def run_hook(self, hookname, *args): pass def add_hook(self, hookname, *args): pass def get_last_tracker(self): """ Returns a tuple of (ip, port), representing the last mogilefsd 'tracker' server which was talked to. """ return self.backend.get_last_tracker() last_tracker = property(get_last_tracker) def new_file(self, key, cls=None, bytes=0, largefile=False, create_open_arg=None, create_close_arg=None, opts=None): """ - class - key - fid - largefile - create_open_arg - create_close_arg """ self.run_hook('new_file_start', key, cls, opts) create_open_arg = create_open_arg or {} create_close_arg = create_close_arg or {} # fid should be specified, or pass 0 meaning to auto-generate one fid = 0 params = { 'domain' : self.domain, 'key' : key, 'fid' : fid, 'multi_dest': 1, } if cls is not None: params['class'] = cls res = self.backend.do_request('create_open', params) if not res: raise IOError() # [ (devid,path), (devid,path), ... ] dests = [] # determine old vs. new format to populate destinations if 'dev_count' not in res: dests.append((res['devid'], res['path'])) else: for x in xrange(1, int(res['dev_count']) + 1): devid_key = 'devid_%d' % x path_key = 'path_%s' % x dests.append((res[devid_key], res[path_key])) main_dest = dests[0] main_devid, main_path = main_dest # TODO # create a MogileFS::NewHTTPFile object, based off of IO::File if not main_path.startswith('http://'): raise MogileFSError("This version of mogilefs.client no longer supports non-http storage URLs.") self.run_hook("new_file_end", key, cls, opts) # TODO if largefile: file_cls = ClientHttpFile else: file_cls = NewHttpFile return file_cls(mg=self, fid=res['fid'], path=main_path, devid=main_devid, backup_dests=dests, cls=cls, key=key, content_length=bytes, create_close_arg=create_close_arg, overwrite=1, ) def edit_file(self, key, **opts): """ EdiCt the file with the the given key. B<NOTE:> edit_file is currently EXPERIMENTAL and not recommended for production use. MogileFS is primarily designed for storing files for later retrieval, rather than editing. Use of this function may lead to poor performance and, until it has been proven mature, should be considered to also potentially cause data loss. B<NOTE:> use of this function requires support for the DAV 'MOVE' verb and partial PUT (i.e. Content-Range in PUT) on the back-end storage servers (e.g. apache with mod_dav). Returns a seekable filehandle you can read/write to. Calling this function may invalidate some or all URLs you currently have for this key, so you should call ->get_paths again afterwards if you need them. On close of the filehandle, the new file contents will replace the previous contents (and again invalidate any existing URLs). By default, the file contents are preserved on open, but you may specify the overwrite option to zero the file first. The seek position is at the beginning of the file, but you may seek to the end to append. """ _complain_ifreadonly(self.readonly) res = self.backend.do_request('edit_file', { 'domain': self.domain, 'key' : key, }) oldpath = res['oldpath'] newpath = res['newpath'] fid = res['fid'] devid = res['devid'] cls = res['class'] ## TODO import httplib2 conn = httplib2.Http() res, content = conn.request(oldpath, "MOVE", headers={ 'Destination': newpath, }) status = int(res['status']) if not (res.status >= 200 and res.status < 300): raise IOError("failed to MOVE %s to %s" % (newpath, oldpath)) return ClientHttpFile(mg=self, path=newpath, fid=fid, devid=devid, cls=cls, key=key, overwrite=opts.get('overwrite')) def read_file(self, *args, **kwds): paths = self.get_paths(*args, **kwds) path = paths[0] backup_dests = [(None, p) for p in paths[1:]] return ClientHttpFile(path=path, backup_dests=backup_dests, readonly=1) def get_paths(self, key, noverify=1, zone='alt', pathcount=None): self.run_hook('get_paths_start', key) extra_params = {} params = { 'domain' : self.domain, 'key' : key, 'noverify': noverify and 1 or 0, 'zone' : zone, } params.update(extra_params) try: res = self.backend.do_request('get_paths', params) paths = [res["path%d" % x] for x in xrange(1, int(res["paths"])+1)] except MogileFSTrackerError, e: if e.err == 'unknown_key': paths = [] else: raise e self.run_hook('get_paths_end', key) return paths