コード例 #1
0
ファイル: test_backend.py プロジェクト: felixtang/pymogile
 def test_do_request_trackers_not_exist(self):
     backend = Backend(["127.0.0.1:7011", "127.0.0.1:7012"])
     try:
         backend.do_request("get_domains")
     except MogileFSError:
         pass
     else:
         assert False
コード例 #2
0
ファイル: test_backend.py プロジェクト: pombredanne/pymogile
def test_do_request_host_not_exist():
    backend = Backend(["127.0.0.1:7011", "127.0.0.1:7012"])
    try:
        backend.do_request("get_domains")
    except MogileFSError:
        pass
    else:
        assert False
コード例 #3
0
ファイル: test_backend.py プロジェクト: s-tatus/pymogile
class TestBackend(unittest.TestCase):
    def setUp(self):
        self.backend = Backend(['127.0.0.1:7001'])

    def test_do_request_trackers_not_exist(self):
        backend = Backend(["127.0.0.1:7011", "127.0.0.1:7012"])
        try:
            backend.do_request("get_domains")
        except MogileFSError:
            pass
        else:
            assert False

    def test_do_request_one_tracker_down(self):
        backend = Backend(["127.0.0.1:7001", "127.0.0.1:7012"])
        try:
            backend.do_request("get_domains")
        except MogileFSError:
            assert False

    def test_do_request(self):
        res = self.backend.do_request("get_domains")
        assert res

    def test_do_request_cmd_not_exist(self):
        try:
            self.backend.do_request("asdfkljweioav")
        except MogileFSError:
            pass
        else:
            assert False

    def test_do_request_with_no_cmd(self):
        try:
            self.backend.do_request()  # pylint: disable-msg=E1120
        except TypeError:
            pass
        except Exception as e:
            assert False, "TypeError expected, actual %r" % e
        else:
            assert False
コード例 #4
0
ファイル: test_backend.py プロジェクト: middagj/pymogile
class TestBackend(unittest.TestCase):
  def setUp(self):
    self.backend = Backend(['127.0.0.1:7001'])

  def test_do_request_trackers_not_exist(self):
    backend = Backend(["127.0.0.1:7011", "127.0.0.1:7012"])
    try:
      backend.do_request("get_domains")
    except MogileFSError:
      pass
    else:
      assert False

  def test_do_request_one_tracker_down(self):
    backend = Backend(["127.0.0.1:7001", "127.0.0.1:7012"])
    try:
      backend.do_request("get_domains")
    except MogileFSError:
      assert False

  def test_do_request(self):
      res = self.backend.do_request("get_domains")
      assert res

  def test_do_request_cmd_not_exist(self):
    try:
      self.backend.do_request("asdfkljweioav")
    except MogileFSError:
      pass
    else:
      assert False

  def test_do_request_with_no_cmd(self):
    try:
      self.backend.do_request()   # pylint: disable-msg=E1120
    except TypeError:
      pass
    except Exception as e:
      assert False, "TypeError expected, actual %r" % e
    else:
      assert False
コード例 #5
0
ファイル: client.py プロジェクト: AloneRoad/pymogile
class Client(object):
    def __init__(self, domain, trackers, readonly=False):
        """Create new Client object with the given list of trackers."""
        self.readonly = bool(readonly)
        self.domain = domain
        self.backend = Backend(trackers, timeout=3)

    def run_hook(self, hookname, *args):
        pass

    def add_hook(self, hookname, *args):
        pass

    def add_backend_hook(self):
        raise NotImplementedError()

    def errstr(self):
        raise NotImplementedError()

    def errcode(self):
        raise NotImplementedError()

    @property
    def last_tracker(self):
        """
        Returns a tuple of (ip, port), representing the last mogilefsd
        'tracker' server which was talked to.
        """
        return self.backend.last_host_connected

    def new_file(self,
                 key,
                 cls=None,
                 largefile=False,
                 content_length=0,
                 create_open_arg=None,
                 create_close_arg=None,
                 opts=None):
        """
        Start creating a new filehandle with the given key,
        and option given class and options.

        Returns a filehandle you should then print to,
        and later close to complete the operation.

        NOTE: check the return value from close!
        If your close didn't succeed, the file didn't get saved!
        """
        self.run_hook('new_file_start', key, cls, opts)

        create_open_arg = create_open_arg or {}
        create_close_arg = create_close_arg or {}

        # fid should be specified, or pass 0 meaning to auto-generate one
        fid = 0
        params = {
            'domain': self.domain,
            'key': key,
            'fid': fid,
            'multi_dest': 1
        }
        if cls is not None:
            params['class'] = cls
        res = self.backend.do_request('create_open', params)
        if not res:
            return None

        # [(devid, path), (devid, path),... ]
        dests = []
        # determine old vs. new format to populate destinations
        if 'dev_count' not in res:
            dests.append((res['devid'], res['path']))
        else:
            for x in xrange(1, int(res['dev_count']) + 1):
                devid_key = 'devid_%d' % x
                path_key = 'path_%s' % x
                dests.append((res[devid_key], res[path_key]))

        main_dest = dests[0]
        main_devid, main_path = main_dest

        self.run_hook("new_file_end", key, cls, opts)

        # TODO
        if largefile:
            file_class = LargeHTTPFile
        else:
            file_class = NormalHTTPFile

        return file_class(
            mg=self,
            fid=res['fid'],
            path=main_path,
            devid=main_devid,
            backup_dests=dests,
            cls=cls,
            key=key,
            content_length=content_length,
            create_close_arg=create_close_arg,
            overwrite=1
        )

    def edit_file(self, key, overwrite=False):
        """Edit the file with the the given key.

        NOTE: edit_file is currently EXPERIMENTAL and not recommended for production use.
        MogileFS is primarily designed for storing files for later retrieval, rather than editing.
        Use of this function may lead to poor performance and, until it has been proven mature, should be considered to also potentially cause data loss.

        NOTE: use of this function requires support for the DAV 'MOVE' verb and partial PUT (i.e. Content-Range in PUT) on the back-end storage servers (e.g. apache with mod_dav).

        Returns a seekable filehandle you can read/write to.
        Calling this function may invalidate some or all URLs you currently have for this key, so you should call ->get_paths again afterwards if you need them.

        On close of the filehandle, the new file contents will replace the previous contents (and again invalidate any existing URLs).

        By default, the file contents are preserved on open, but you may specify the overwrite option to zero the file first.
        The seek position is at the beginning of the file, but you may seek to the end to append.
        """
        raise NotImplementedError()

    def read_file(self, key):
        """
        Read the file with the the given key.
        Returns a seekable filehandle you can read() from.
        Note that you cannot read line by line using <$fh> notation.

        Takes the same options as get_paths
        (which is called internally to get the URIs to read from).
        """
        paths = self.get_paths(key)
        if not paths:
            return None
        path = paths[0]
        backup_dests = [(None, p) for p in paths[1:]]
        return LargeHTTPFile(path=path, backup_dests=backup_dests, readonly=1)

    def store_file(self, key, fp, cls=None, chunk_size=8192):
        """
        Wrapper around new_file, print, and close.

        Given a key, class, and a filehandle or filename, stores the file
        contents in MogileFS.  Returns the number of bytes stored on success,
        undef on failure.
        """
        if self.readonly:
            return False

        params = {}
        if chunk_size:
            params['chunk_size'] = chunk_size
        if cls:
            params['class'] = cls
        params[key] = key

        self.run_hook('store_file_start', params)

        try:
            new_file = self.new_file(key, cls)
        except MogileFSError:
            fp.close()
            return False

        try:
            _bytes = 0
            while True:
                buf = fp.read(chunk_size)
                if not buf:
                    break
                _bytes += len(buf)
                new_file.write(buf)

            self.run_hook('store_file_end', params)
        finally:
            fp.close()
            new_file.close()

        return _bytes

    def store_content(self, key, content, cls=None, **opts):
        """
        Wrapper around new_file, print, and close.  Given a key, class, and
        file contents (scalar or scalarref), stores the file contents in
        MogileFS. Returns the number of bytes stored on success, undef on
        failure.
        """
        if self.readonly:
            return False

        self.run_hook('store_content_start', key, cls, opts)

        output = self.new_file(key, cls, None, **opts)
        output.write(content)
        output.close()

        self.run_hook('store_content_end', key, cls, opts)

        return len(content)

    def get_paths(self, key, noverify=1, zone='alt', pathcount=2):
        """
        Given a key, returns an array of all the locations (HTTP URLs) that the file
        has been replicated to.
        """
        self.run_hook('get_paths_start', key)

        params = {
            'domain': self.domain,
            'key': key,
            'noverify': noverify and 1 or 0,
            'zone': zone,
            'pathcount': pathcount
        }

        res = self.backend.do_request('get_paths', params)
        paths = [res["path%d" % x] for x in xrange(1, int(res["paths"]) + 1)]

        self.run_hook('get_paths_end', key)
        return paths

    def get_file_data(self, key, timeout=10):
        """
        Returns scalarref of file contents in a scalarref.
        Don't use for large data, as it all comes back to you in one string.
        """
        fp = self.read_file(key)
        if not fp:
            return None
        try:
            content = fp.read()
            return content
        finally:
            fp.close()

    def delete(self, key):
        """
        Delete a file from MogileFS
        """
        try:
            if self.readonly:
                return False
            self.backend.do_request(
                'delete', {'domain': self.domain,
                           'key': key}
            )
            return True
        except MogileFSError:
            return False

    def rename(self, from_key, to_key):
        """
        Rename file (key) in MogileFS from oldkey to newkey.
        Returns true on success, failure otherwise
        """
        try:
            if self.readonly:
                return False
            params = {
                'domain': self.domain,
                'from_key': from_key,
                'to_key': to_key
            }
            self.backend.do_request('rename', params)
            return True
        except MogileFSError:
            return False

    def file_debug(self, **kwargs):
        """
        Thoroughly search for any database notes about a particular fid.
        Searchable by raw fid, or by domain and key. **Use sparingly**.
        Command hits the master database numerous times, and if you're using
        it in production something is likely very wrong.

        To be used with troubleshooting broken/odd files and errors from
        mogilefsd.
        """
        if 'key' not in kwargs and 'fid' not in kwargs:
            raise TypeError('file_debug() missing 1 required '
                            'positional argument: fid/key')
        params = {
            'domain': kwargs.get('domain', self.domain)
        }
        params.update(kwargs)
        return self.backend.do_request('file_debug', params)

    def file_info(self, key, devices=False):
        """
        Used to return metadata about a file. Returns the domain, class,
        expected length, devcount, etc. Optionally device ids (not paths)
        can be returned as well.

        Should be used for informational purposes, and not usually for
        dynamically serving files.
        """
        params = {
            'domain': self.domain,
            'key': key
        }
        if devices:
            params['devices'] = True
        info = self.backend.do_request('file_info', params)
        info['devcount'] = int(info['devcount'])
        info['length'] = int(info['length'])
        if 'devids' in info:
            info['devids'] = info['devids'].split(',')
        return info

    def list_keys(self, prefix=None, after=None, limit=None):
        """
        Used to get a list of keys matching a certain prefix.

        $prefix specifies what you want to get a list of.  $after is the item
        specified as a return value from this function last time you called
        it. `$limit` is optional and defaults to 1000 keys returned.

        In list context, returns ($after, $keys).  In scalar context, returns
        arrayref of keys.  The value $after is to be used as $after when you
        call this function again.

        When there are no more keys in the list, you will get back undef or
        an empty list.
        """
        params = {'domain': self.domain}
        if prefix:
            params['prefix'] = prefix
        if after:
            params['after'] = after
        if limit:
            params['limit'] = limit

        res = self.backend.do_request('list_keys', params)
        results = []
        for x in xrange(1, int(res['key_count']) + 1):
            results.append(res['key_%d' % x])
        return results

    def keys(self, prefix=None):
        """
        Get all keys matching a certain prefix
        """
        params = {'domain': self.domain}
        if prefix:
            params['prefix'] = prefix
        results = []
        while True:
            res = self.backend.do_request('list_keys', params)
            for x in res.keys():
                if x not in ['key_count', 'next_after']:
                    results.append(res[x])
            if len(res) < 1002:
                break
            params['after'] = res['next_after']
        return list(set(results))

    def foreach_key(self, *args, **kwds):
        """
        Functional interface/wrapper around list_keys.

        Given some %OPTIONS (currently only one, "prefix"),
        calls your callback for each key matching the provided prefix.
        """
        raise NotImplementedError()

    def update_class(self, key, new_class):
        """
        Update the replication class of a pre-existing file,
        causing the file to become more or less replicated.
        """
        try:
            if self.readonly:
                return False
            params = {"domain": self.domain, "key": key, "class": new_class}
            res = self.backend.do_request("updateclass", params)
            return res
        except MogileFSError:
            return False

    def sleep(self, duration):
        """
        just makes some sleeping happen.  first and only argument is number of
        seconds to instruct backend thread to sleep for.
        """
        try:
            self.backend.do_request("sleep", {'duration': duration})
            return True
        except MogileFSError:
            return False

    def set_pref_ip(self, *ips):
        """
        Weird option for old, weird network architecture.
        Sets a mapping table of preferred alternate IPs, if reachable.
        For instance, if trying to connect to 10.0.0.2 in the above example,
        the module would instead try to connect to 10.2.0.2 quickly first,
        then then fall back to 10.0.0.2 if 10.2.0.2 wasn't reachable.
        expects as argument a tuple of ("standard-ip", "preferred-ip")
        """
        self.backend.set_pref_ip(*ips)
コード例 #6
0
ファイル: client.py プロジェクト: pombredanne/pymogile
class Client(object):
  def __init__(self, domain, hosts, readonly=False):
    self.readonly = bool(readonly)
    self.domain   = domain
    self.backend  = Backend(hosts, timeout=3)

  def run_hook(self, hookname, *args):
    pass

  def add_hook(self, hookname, *args):
    pass

  @property
  def last_tracker(self):
    """
    Returns a tuple of (ip, port), representing the last mogilefsd
    'tracker' server which was talked to.
    """
    return self.backend.get_last_tracker()

  def new_file(self, key, cls=None, bytes=0, largefile=False, 
               create_open_arg=None, create_close_arg=None, opts=None):
    """
    Start creating a new filehandle with the given key, 
    and option given class and options.
    
    Returns a filehandle you should then print to, 
    and later close to complete the operation. 
    
    NOTE: check the return value from close! 
    If your close didn't succeed, the file didn't get saved!
    """
    self.run_hook('new_file_start', key, cls, opts)

    create_open_arg = create_open_arg or {}
    create_close_arg = create_close_arg or {}

    # fid should be specified, or pass 0 meaning to auto-generate one
    fid = 0
    params = {'domain'  : self.domain,
              'key'     : key,
              'fid'     : fid,
              'multi_dest': 1}
    if cls is not None:
      params['class'] = cls
    res = self.backend.do_request('create_open', params)
    if not res:
      return None

    # [ (devid,path), (devid,path), ... ]
    dests = []
    # determine old vs. new format to populate destinations
    if 'dev_count' not in res:
      dests.append((res['devid'], res['path']))
    else:
      for x in xrange(1, int(res['dev_count']) + 1):
        devid_key = 'devid_%d' % x
        path_key = 'path_%s' % x
        dests.append((res[devid_key], res[path_key]))

    main_dest = dests[0]
    main_devid, main_path = main_dest

    self.run_hook("new_file_end", key, cls, opts)

    # TODO
    if largefile:
      file_class = ClientHttpFile
    else:
      file_class = NewHttpFile

    return file_class(mg=self,
                      fid=res['fid'],
                      path=main_path,
                      devid=main_devid,
                      backup_dests=dests,
                      cls=cls,
                      key=key,
                      content_length=bytes,
                      create_close_arg=create_close_arg,
                      overwrite=1)

  def read_file(self, *args, **kwds):
    """
    Read the file with the the given key.
    Returns a seekable filehandle you can read() from. 
    Note that you cannot read line by line using <$fh> notation.
    
    Takes the same options as get_paths 
    (which is called internally to get the URIs to read from).
    """
    paths = self.get_paths(*args, **kwds)
    path = paths[0]
    backup_dests = [(None, p) for p in paths[1:]]
    return ClientHttpFile(path=path, backup_dests=backup_dests, readonly=1)

  def get_paths(self, key, noverify=1, zone='alt', pathcount=None):
    """ 
    Given a key, returns an array of all the locations (HTTP URLs) that the file 
    has been replicated to.
    """
    self.run_hook('get_paths_start', key)

    if not pathcount:
      pathcount = 2
      
    params = {'domain'   : self.domain,
              'key'      : key,
              'noverify' : noverify and 1 or 0, 
              'zone'     : zone,
              'pathcount': pathcount}
    try:
      res = self.backend.do_request('get_paths', params)
      paths = [res["path%d" % x] for x in xrange(1, int(res["paths"]) + 1)]
    except (MogileFSTrackerError, MogileFSError):
      paths = []

    self.run_hook('get_paths_end', key)
    return paths

  def get_file_data(self, key, timeout=10):
    """
    Returns scalarref of file contents in a scalarref.
    Don't use for large data, as it all comes back to you in one string.
    """
    fp = self.read_file(key, noverify=1)
    try:
      content = fp.read()
      return content
    finally:
      fp.close()

  def rename(self, old_key, new_key):
    """
    Rename file (key) in MogileFS from oldkey to newkey. 
    Returns true on success, failure otherwise
    """
    _complain_ifreadonly(self.readonly)
    self.backend.do_request('rename', {'domain'  : self.domain,
                                       'from_key': old_key,
                                       'to_key'  : new_key})
    return True

  def list_keys(self, prefix=None, after=None, limit=None):
    """
    Used to get a list of keys matching a certain prefix.
    
    $prefix specifies what you want to get a list of. 
    
    $after is the item specified as a return value from this function last time 
          you called it. 
    
    $limit is optional and defaults to 1000 keys returned.
    
    In list context, returns ($after, $keys). 
    In scalar context, returns arrayref of keys. 
    The value $after is to be used as $after when you call this function again.
    
    When there are no more keys in the list, 
    you will get back undef or an empty list
    """
    params = {'domain': self.domain}
    if prefix:
      params['prefix'] = prefix
    if after:
      params['after'] = after
    if limit:
      params['limit'] = limit

    res = self.backend.do_request('list_keys', params)
    reslist = []
    for x in xrange(1, int(res['key_count']) + 1):
      reslist.append(res['key_%d' % x])
    return reslist

  def foreach_key(self, *args, **kwds):
    raise NotImplementedError()
  
  def update_class(self, *args, **kwds):
    raise NotImplementedError()

  def sleep(self, duration):
    """
    just makes some sleeping happen.  first and only argument is number of
    seconds to instruct backend thread to sleep for.
    """
    self.backend.do_request("sleep", {'duration': duration})
    return True

  def set_pref_ip(self, *ips):
    """
    Weird option for old, weird network architecture.  Sets a mapping
    table of preferred alternate IPs, if reachable.  For instance, if
    trying to connect to 10.0.0.2 in the above example, the module would
    instead try to connect to 10.2.0.2 quickly first, then then fall back
    to 10.0.0.2 if 10.2.0.2 wasn't reachable.
    expects as argument a tuple of ("standard-ip", "preferred-ip")
    """
    self.backend.set_pref_ip(*ips)

  def store_file(self, key, fp, cls=None, **opts):
    """
    Wrapper around new_file, print, and close.

    Given a key, class, and a filehandle or filename, stores the file
    contents in MogileFS.  Returns the number of bytes stored on success,
    undef on failure.
    """
    _complain_ifreadonly(self.readonly)

    self.run_hook('store_file_start', key, cls, opts)

    try:
      output = self.new_file(key, cls, largefile=1, **opts)
      bytes = 0
      while 1:
        buf = fp.read(1024 * 16)
        if not buf:
          break
        bytes += len(buf)
        output.write(buf)

      self.run_hook('store_file_end', key, cls, opts)
    finally:
      # finally
      fp.close()
      output.close()

    return bytes

  def store_content(self, key, content, cls=None, **opts):
    """
    Wrapper around new_file, print, and close.  Given a key, class, and
    file contents (scalar or scalarref), stores the file contents in
    MogileFS. Returns the number of bytes stored on success, undef on
    failure.
    """
    _complain_ifreadonly(self.readonly)

    self.run_hook('store_content_start', key, cls, opts)

    output = self.new_file(key, cls, None, **opts)
    try:
      output.write(content)
    finally:
      output.close()

    self.run_hook('store_content_end', key, cls, opts)

    return len(content)

  def delete(self, key):
    """ Delete a file from MogileFS """
    _complain_ifreadonly(self.readonly)
    self.backend.do_request('delete', {'domain': self.domain, 'key': key})
    return True
コード例 #7
0
class Admin(object):
    def __init__(self, trackers, readonly=False, timeout=None):
        self.readonly = bool(readonly)
        self.backend = Backend(trackers, timeout)

    def replicate_now(self):
        return self.backend.do_request("replicate_now")

    def get_hosts(self, hostid=None):
        if hostid:
            params = {'hostid': hostid}
        else:
            params = None
        res = self.backend.do_request("get_hosts", params)
        print(res)
        results = []
        fields = [
            "hostid", "status", "hostname", "hostip", "http_port",
            "http_get_port", "altip altmask"
        ]
        hosts = int(res['hosts']) + 1
        for ct in range(1, hosts):
            results.append(
                dict([(f, res.get('host%d_%s' % (ct, f))) for f in fields]))
        return results

    def get_devices(self, devid=None):
        if devid:
            params = {'devid': devid}
        else:
            params = None
        res = self.backend.do_request("get_devices", params)
        ret = []
        for x in range(1, int(res['devices']) + 1):
            device = {}
            for k in ('devid', 'hostid', 'status', 'observed_state',
                      'utilization'):
                device[k] = res.get('dev%d_%s' % (x, k))

            for k in ('mb_total', 'mb_used', 'weight'):
                value = res.get('dev%d_%s' % (x, k))
                if value:
                    device[k] = int(value)
                else:
                    device[k] = None
            ret.append(device)
        return ret

    def list_fids(self, from_fid, to_fid):
        """
    get raw information about fids, for enumerating the dataset
       ( from_fid, to_fid )
    returns:
       {fid => {dict with keys: domain, class, devcount, length, key}}
    """
        res = self.backend.do_request('list_fids', {
            'from': from_fid,
            'to': to_fid
        })
        results = {}
        for x in range(1, int(res['fid_count']) + 1):
            key = 'fid_%d_fid' % x
            results[key] = dict([(k, res['fid_%d_%s' % (x, k)]) \
                                 for k in ('key', 'length', 'class', 'domain', 'devcount')])
        return results

    def clear_cache(self):
        return self.backend.do_request('clear_cache')

    def get_domains(self):
        """
    get a dict of the domains we know about in the format of
       {
         "domain_name": {
           "class_name": mindevcount,
           "class_name": mindevcount,
           ...
                        },
         ...
      }
    """
        res = self.backend.do_request('get_domains')
        domain_length = int(res['domains'])
        ret = {}
        for x in range(1, domain_length + 1):
            domain_name = res['domain%d' % x]
            ret.setdefault(domain_name, {})
            class_length = int(res['domain%dclasses' % x])
            for y in range(1, class_length + 1):
                k = 'domain%dclass%dname' % (x, y)
                v = 'domain%dclass%dmindevcount' % (x, y)
                ret[domain_name][res[k]] = int(res[v])
        return ret

    def create_domain(self, domain):
        """
    create a new domain
    """
        if self.readonly:
            return False
        res = self.backend.do_request('create_domain', {'domain': domain})
        if res['domain'] == domain:
            return True
        else:
            return False

    def delete_domain(self, domain):
        """
    delete a domain
    """
        if self.readonly:
            return False
        try:
            res = self.backend.do_request('delete_domain', {'domain': domain})
        except MogileFSError:
            return False
        if res['domain'] == domain:
            return True
        else:
            return False

    def create_class(self, domain, cls, mindevcount):
        """
    create a class within a domain
    """
        return self._modify_class('create', domain, cls, mindevcount)

    def update_class(self, domain, cls, mindevcount):
        """
    update a class's mindevcount within a domain
    """
        return self._modify_class('update', domain, cls, mindevcount)

    def delete_class(self, domain, cls):
        """
    delete a class
    """
        if self.readonly:
            return False
        res = self.backend.do_request("delete_class", {
            'domain': domain,
            'class': cls
        })
        if res['class'] == cls:
            return True
        else:
            return False

    def create_host(self, host, ip, port, getport=None, status=None):
        params = {'host': host, 'ip': ip, 'port': port, 'getport': getport}
        if status:
            params['status'] = status
        return self._modify_host('create', params)

    def update_host(self, host, ip=None, port=None, getport=None, status=None):
        params = {'host': host}
        if ip:
            params['ip'] = ip
        if port:
            params['port'] = port
        if getport:
            params['getport'] = True
        if status:
            params['status'] = status
        return self._modify_host('update', params)

    def delete_host(self, host):
        self.backend.do_request("delete_host", {'host': host})

    def create_device(self, hostname, devid, hostip=None, state=None):
        params = {'hostname': hostname, 'devid': devid}
        if hostip:
            params['hostip'] = hostip
        if state:
            params['state'] = state
        return self.backend.do_request('create_device', params)

    def update_device(self, host, device, status=None, weight=None):
        if status:
            self.change_device_state(host, device, status)

        if weight:
            self.change_device_weight(host, device, weight)
        return True

    def change_device_state(self, host, device, state):
        """
    change the state of a device; pass in the hostname of the host the
    device is located on, the device id number, and the state you want
    the host to be set to.
    """
        params = {'host': host, 'device': device, 'state': state}
        return self.backend.do_request('set_state', params)

    def change_device_weight(self, host, device, weight):
        """
    change the weight of a device by passing in the hostname and
    the device id
    """
        if not isinstance(weight, six.integer_types):
            raise ValueError('argument weight muse be an integer')
        params = {'host': host, 'device': device, 'weight': weight}
        return self.backend.do_request('set_weight', params)

    def _get_slave_keys(self):
        res = self.backend.do_request("server_setting", {"key": "slave_keys"})
        if not res:
            return {}

        value = res['value']
        slave_keys = {}
        for slave in value.split(','):
            key, weight = (slave.split("=", 1) + [None])[:2]
            # Weight can be zero,
            # so don't default to 1 if it's defined and longer than 0 characters.
            try:
                weight = int(weight)
                if not weight:
                    weight = 1
            except (TypeError, ValueError):
                weight = 1

            slave_keys[key] = weight
        return slave_keys

    def _set_slave_keys(self, keys):
        buf = []
        for key, weight in keys.items():
            try:
                weight = int(weight)
                if weight != 1:
                    key = "%s=%d" % (key, weight)
            except (TypeError, ValueError):
                pass

            buf.append(key)

        self.backend.do_request("set_server_setting", {
            'key': 'slave_keys',
            'value': ''.join(buf)
        })

    def slave_list(self):
        keys = self._get_slave_keys()
        ret = {}
        for key in keys:
            res = self.backend.do_request('server_setting',
                                          {'key': 'slave_%s' % key})
            if not res:
                continue

            value = res['value']
            dsn, username, password = (value.split('|') + ['', ''])[:3]
            ret[key] = (dsn, username, password)

        return ret

    def slave_add(self, key, dsn, username, password):
        keys = self._get_slave_keys()
        if key in keys:
            return

        value = '|'.join([dsn, username, password])
        self.backend.do_request("set_server_setting", {
            'key': key,
            'value': value
        })
        keys[key] = None
        self._set_slave_keys(keys)

    def slave_modify(self, key, **opts):
        keys = self._get_slave_keys()
        if key not in keys:
            # slave not fuond
            return
        res = self.backend.do_request("server_setting",
                                      {"key": "slave_%s" % key})
        value = res['value']
        dsn, username, password = (value.split('|') + ['', ''])[:3]

        dsn = opts.get('dsn') or dsn
        username = opts.get('username') or username
        password = opts.get('password') or password

        value = '|'.join([dsn, username, password])
        res = self.backend.do_request('set_server_setting', {
            'key': 'slave_%s' % key,
            'value': value
        })

    def slave_delete(self, key):
        slave_keys = self._get_slave_keys()
        if not slave_keys:
            return None

        if key not in slave_keys:
            return False

        self.backend.do_request('set_server_setting', {key: "slave_%s" % key})
        del slave_keys[key]
        self._set_slave_keys(slave_keys)

    def fsck_start(self):
        return self.backend.do_request("fsck_start")

    def fsck_stop(self):
        return self.backend.do_request("fsck_stop")

    def fsck_reset(self, policy_only, startpos):
        return self.backend.do_request("fsck_reset", {
            'policy_only': policy_only,
            'startpos': startpos
        })

    def fsck_clearlog(self):
        return self.backend.do_request("fsck_clearlog")

    def fsck_status(self):
        return self.backend.do_request("fsck_status")

    def fsck_log_rows(self, after_logid=None):
        params = {}
        if after_logid:
            params['after_logid'] = after_logid
        res = self.backend.do_request("fsck_getlog", params)

        row_count = int(res['row_count'])
        ret = []
        for x in range(1, row_count + 1):
            rec = {}
            for k in ("logid", "utime", "fid", "evcode", "devid"):
                rec[k] = res.get("row_%d_%s" % (x, k))
            ret.append(rec)
        return ret

    def set_server_setting(self, key, value):
        params = {'key': key, 'value': value}
        return self.backend.do_request("set_server_setting", params)

    def server_settings(self):
        res = self.backend.do_request("server_settings")
        if not res:
            return
        ret = {}
        for x in range(1, int(res["key_count"]) + 1):
            key = res.get("key_%d" % x, '')
            value = res.get("value_%d" % x, '')
            ret[key] = value
        return ret

    def _modify_class(self, verb, domain, cls, mindevcount, replpolicy=None):
        if self.readonly:
            return False
        params = {'domain': domain, 'class': cls, 'mindevcount': mindevcount}
        if replpolicy:
            params['replpolicy'] = replpolicy
        res = self.backend.do_request("%s_class" % verb, params)
        if res['class'] == cls:
            return True
        else:
            return False

    def _modify_host(self, verb, params):
        if self.readonly:
            return False
        try:
            return self.backend.do_request("%s_host" % verb, params)
        except MogileFSError:
            return False

    ## Extra
    #
    def get_freespace(self, devid=None):
        """Get the free space for the entire cluster, or a specific node"""
        return sum([x['mb_free'] for x in self.get_devices(devid)])

    def get_stats(self):
        params = {'all': 1}
        res = self.backend.do_request('stats', params)

        ret = {}
        # get replication statistics
        if 'replicationcount' in res:
            replication = ret.setdefault('replication', {})
            for x in range(1, int(res['replicationcount']) + 1):
                domain = res.get('replication%ddomain' % x, '')
                cls = res.get('replication%dclass' % x, '')
                devcount = res.get('replication%ddevcount' % x, '')
                fields = res.get('replication%dfields' % x)
                (replication.setdefault(domain,
                                        {}).setdefault(cls,
                                                       {}))[devcount] = fields

        # get file statistics
        if 'filescount' in res:
            files = ret.setdefault('files', {})
            for x in range(1, int(res['filescount']) + 1):
                domain = res.get('files%ddomain' % x, '')
                cls = res.get('files%dclass' % x, '')
                (files.setdefault(domain,
                                  {}))[cls] = res.get('files%dfiles' % x)

        # get device statistics
        if 'devicescount' in res:
            devices = ret.setdefault('devices', {})
            for x in range(1, int(res['devicescount']) + 1):
                key = res.get('devices%did' % x, '')
                devices[key] = {
                    'host': res.get('devices%dhost' % x),
                    'status': res.get('devices%dstatus' % x),
                    'files': res.get('devices%dfiles' % x),
                }

        if 'fidmax' in res:
            ret['fids'] = {'max': res['fidmax']}

        # return the created response
        return ret
コード例 #8
0
ファイル: admin.py プロジェクト: middagj/pymogile
class Admin(object):
  def __init__(self, trackers, readonly=False, timeout=None):
    self.readonly = bool(readonly)
    self.backend = Backend(trackers, timeout)

  def replicate_now(self):
    return self.backend.do_request("replicate_now")

  def get_hosts(self, hostid=None):
    if hostid:
      params = {'hostid': hostid}
    else:
      params = None
    res = self.backend.do_request("get_hosts", params)
    print(res)
    results = []
    fields = ["hostid",
              "status",
              "hostname",
              "hostip",
              "http_port",
              "http_get_port",
              "altip altmask"]
    hosts = int(res['hosts']) + 1
    for ct in range(1, hosts):
      results.append(dict([(f, res.get('host%d_%s' % (ct, f))) for f in fields]))
    return results

  def get_devices(self, devid=None):
    if devid:
      params = {'devid': devid}
    else:
      params = None
    res = self.backend.do_request("get_devices", params)
    ret = []
    for x in range(1, int(res['devices']) + 1):
      device = {}
      for k in ('devid', 'hostid', 'status', 'observed_state', 'utilization'):
        device[k] = res.get('dev%d_%s' % (x, k))

      for k in ('mb_total', 'mb_used', 'weight'):
        value = res.get('dev%d_%s' % (x, k))
        if value:
          device[k] = int(value)
        else:
          device[k] = None
      ret.append(device)
    return ret

  def list_fids(self, from_fid, to_fid):
    """
    get raw information about fids, for enumerating the dataset
       ( from_fid, to_fid )
    returns:
       {fid => {dict with keys: domain, class, devcount, length, key}}
    """
    res = self.backend.do_request('list_fids', {'from': from_fid, 'to': to_fid})
    results = {}
    for x in range(1, int(res['fid_count']) + 1):
      key = 'fid_%d_fid' % x
      results[key] = dict([(k, res['fid_%d_%s' % (x, k)]) \
                           for k in ('key', 'length', 'class', 'domain', 'devcount')])
    return results

  def clear_cache(self):
    return self.backend.do_request('clear_cache')

  def get_domains(self):
    """
    get a dict of the domains we know about in the format of
       {
         "domain_name": {
           "class_name": mindevcount,
           "class_name": mindevcount,
           ...
                        },
         ...
      }
    """
    res = self.backend.do_request('get_domains')
    domain_length = int(res['domains'])
    ret = {}
    for x in range(1, domain_length + 1):
      domain_name = res['domain%d' % x]
      ret.setdefault(domain_name, {})
      class_length = int(res['domain%dclasses' % x])
      for y in range(1, class_length + 1):
        k = 'domain%dclass%dname' % (x, y)
        v = 'domain%dclass%dmindevcount' % (x, y)
        ret[domain_name][res[k]] = int(res[v])
    return ret

  def create_domain(self, domain):
    """
    create a new domain
    """
    if self.readonly:
      return False
    res = self.backend.do_request('create_domain', {'domain': domain})
    if res['domain'] == domain:
      return True
    else:
      return False

  def delete_domain(self, domain):
    """
    delete a domain
    """
    if self.readonly:
      return False
    try:
      res = self.backend.do_request('delete_domain', {'domain': domain})
    except MogileFSError:
      return False
    if res['domain'] == domain:
      return True
    else:
      return False

  def create_class(self, domain, cls, mindevcount):
    """
    create a class within a domain
    """
    return self._modify_class('create', domain, cls, mindevcount)

  def update_class(self, domain, cls, mindevcount):
    """
    update a class's mindevcount within a domain
    """
    return self._modify_class('update', domain, cls, mindevcount)

  def delete_class(self, domain, cls):
    """
    delete a class
    """
    if self.readonly:
      return False
    res = self.backend.do_request("delete_class",
                                  {'domain': domain, 'class': cls})
    if res['class'] == cls:
      return True
    else:
      return False

  def create_host(self, host, ip, port, getport=None, status=None):
    params = {'host': host, 'ip': ip, 'port': port, 'getport': getport}
    if status:
      params['status'] = status
    return self._modify_host('create', params)

  def update_host(self, host, ip=None, port=None, getport=None, status=None):
    params = {'host': host}
    if ip:
      params['ip'] = ip
    if port:
      params['port'] = port
    if getport:
      params['getport'] = True
    if status:
      params['status'] = status
    return self._modify_host('update', params)

  def delete_host(self, host):
    self.backend.do_request("delete_host", {'host': host})

  def create_device(self, hostname, devid, hostip=None, state=None):
    params = {'hostname': hostname, 'devid': devid}
    if hostip:
      params['hostip'] = hostip
    if state:
      params['state'] = state
    return self.backend.do_request('create_device', params)

  def update_device(self, host, device, status=None, weight=None):
    if status:
      self.change_device_state(host, device, status)

    if weight:
      self.change_device_weight(host, device, weight)
    return True

  def change_device_state(self, host, device, state):
    """
    change the state of a device; pass in the hostname of the host the
    device is located on, the device id number, and the state you want
    the host to be set to.
    """
    params = {'host': host, 'device': device, 'state': state}
    return self.backend.do_request('set_state', params)

  def change_device_weight(self, host, device, weight):
    """
    change the weight of a device by passing in the hostname and
    the device id
    """
    if not isinstance(weight, six.integer_types):
      raise ValueError('argument weight muse be an integer')
    params = {'host': host, 'device': device, 'weight': weight}
    return self.backend.do_request('set_weight', params)

  def _get_slave_keys(self):
    res = self.backend.do_request("server_setting", {"key": "slave_keys"})
    if not res:
      return {}

    value = res['value']
    slave_keys = {}
    for slave in value.split(','):
      key, weight = (slave.split("=", 1) + [None])[:2]
      # Weight can be zero,
      # so don't default to 1 if it's defined and longer than 0 characters.
      try:
        weight = int(weight)
        if not weight:
          weight = 1
      except (TypeError, ValueError):
        weight = 1

      slave_keys[key] = weight
    return slave_keys

  def _set_slave_keys(self, keys):
    buf = []
    for key, weight in keys.items():
      try:
        weight = int(weight)
        if weight != 1:
          key = "%s=%d" % (key, weight)
      except (TypeError, ValueError):
        pass

      buf.append(key)

    self.backend.do_request("set_server_setting",
                            {'key': 'slave_keys', 'value': ''.join(buf)})

  def slave_list(self):
    keys = self._get_slave_keys()
    ret = {}
    for key in keys:
      res = self.backend.do_request('server_setting',
                                    {'key': 'slave_%s' % key})
      if not res:
        continue

      value = res['value']
      dsn, username, password = (value.split('|') + ['', ''])[:3]
      ret[key] = (dsn, username, password)

    return ret

  def slave_add(self, key, dsn, username, password):
    keys = self._get_slave_keys()
    if key in keys:
      return

    value = '|'.join([dsn, username, password])
    self.backend.do_request("set_server_setting",
                            {'key': key, 'value': value})
    keys[key] = None
    self._set_slave_keys(keys)

  def slave_modify(self, key, **opts):
    keys = self._get_slave_keys()
    if key not in keys:
      # slave not fuond
      return
    res = self.backend.do_request("server_setting", {"key": "slave_%s" % key})
    value = res['value']
    dsn, username, password = (value.split('|') + ['', ''])[:3]

    dsn = opts.get('dsn') or dsn
    username = opts.get('username') or username
    password = opts.get('password') or password

    value = '|'.join([dsn, username, password])
    res = self.backend.do_request('set_server_setting',
                                  {'key': 'slave_%s' % key, 'value': value})

  def slave_delete(self, key):
    slave_keys = self._get_slave_keys()
    if not slave_keys:
      return None

    if key not in slave_keys:
      return False

    self.backend.do_request('set_server_setting',
                            {key: "slave_%s" % key})
    del slave_keys[key]
    self._set_slave_keys(slave_keys)

  def fsck_start(self):
    return self.backend.do_request("fsck_start")

  def fsck_stop(self):
    return self.backend.do_request("fsck_stop")

  def fsck_reset(self, policy_only, startpos):
    return self.backend.do_request("fsck_reset",
                            {'policy_only': policy_only, 'startpos': startpos})

  def fsck_clearlog(self):
    return self.backend.do_request("fsck_clearlog")

  def fsck_status(self):
    return self.backend.do_request("fsck_status")

  def fsck_log_rows(self, after_logid=None):
    params = {}
    if after_logid:
      params['after_logid'] = after_logid
    res = self.backend.do_request("fsck_getlog", params)

    row_count = int(res['row_count'])
    ret = []
    for x in range(1, row_count + 1):
      rec = {}
      for k in ("logid", "utime", "fid", "evcode", "devid"):
        rec[k] = res.get("row_%d_%s" % (x, k))
      ret.append(rec)
    return ret

  def set_server_setting(self, key, value):
    params = {'key': key, 'value': value}
    return self.backend.do_request("set_server_setting", params)

  def server_settings(self):
    res = self.backend.do_request("server_settings")
    if not res:
      return
    ret = {}
    for x in range(1, int(res["key_count"]) + 1):
      key = res.get("key_%d" % x, '')
      value = res.get("value_%d" % x, '')
      ret[key] = value
    return ret

  def _modify_class(self, verb, domain, cls, mindevcount, replpolicy=None):
    if self.readonly:
      return False
    params = {'domain': domain,
              'class': cls,
              'mindevcount': mindevcount}
    if replpolicy:
      params['replpolicy'] = replpolicy
    res = self.backend.do_request("%s_class" % verb, params)
    if res['class'] == cls:
      return True
    else:
      return False

  def _modify_host(self, verb, params):
    if self.readonly:
      return False
    try:
      return self.backend.do_request("%s_host" % verb, params)
    except MogileFSError:
      return False

  ## Extra
  #
  def get_freespace(self, devid=None):
    """Get the free space for the entire cluster, or a specific node"""
    return sum([x['mb_free'] for x in  self.get_devices(devid)])

  def get_stats(self):
    params = {'all': 1}
    res = self.backend.do_request('stats', params)

    ret = {}
    # get replication statistics
    if 'replicationcount' in res:
      replication = ret.setdefault('replication', {})
      for x in range(1, int(res['replicationcount']) + 1):
        domain = res.get('replication%ddomain' % x, '')
        cls = res.get('replication%dclass' % x, '')
        devcount = res.get('replication%ddevcount' % x, '')
        fields = res.get('replication%dfields' % x)
        (replication.setdefault(domain, {}).setdefault(cls, {}))[devcount] = fields

    # get file statistics
    if 'filescount' in res:
      files = ret.setdefault('files', {})
      for x in range(1, int(res['filescount']) + 1):
        domain = res.get('files%ddomain' % x, '')
        cls = res.get('files%dclass' % x, '')
        (files.setdefault(domain, {}))[cls] = res.get('files%dfiles' % x)

    # get device statistics
    if 'devicescount' in res:
      devices = ret.setdefault('devices', {})
      for x in range(1, int(res['devicescount']) + 1):
        key = res.get('devices%did' % x, '')
        devices[key] = {'host': res.get('devices%dhost' % x),
                        'status': res.get('devices%dstatus' % x),
                        'files': res.get('devices%dfiles' % x),
                }

    if 'fidmax' in res:
      ret['fids'] = {'max': res['fidmax']}

    # return the created response
    return ret
コード例 #9
0
ファイル: test_backend.py プロジェクト: felixtang/pymogile
 def test_do_request_one_tracker_down(self):
     backend = Backend(["127.0.0.1:7001", "127.0.0.1:7012"])
     try:
         backend.do_request("get_domains")
     except MogileFSError:
         assert False
コード例 #10
0
ファイル: client.py プロジェクト: s-tatus/pymogile
class Client(object):
  def __init__(self, domain, trackers, readonly=False):
    """Create new Client object with the given list of trackers."""
    self.readonly = bool(readonly)
    self.domain = domain
    self.backend = Backend(trackers, timeout=3)

  def run_hook(self, hookname, *args):
    pass

  def add_hook(self, hookname, *args):
    pass

  def add_backend_hook(self):
    raise NotImplementedError()

  def errstr(self):
    raise NotImplementedError()

  def errcode(self):
    raise NotImplementedError()


  @property
  def last_tracker(self):
    """
    Returns a tuple of (ip, port), representing the last mogilefsd
    'tracker' server which was talked to.
    """
    return self.backend.last_host_connected

  def new_file(self, key, cls=None, largefile=False, content_length=0,
               create_open_arg=None, create_close_arg=None, opts=None):
    """
    Start creating a new filehandle with the given key,
    and option given class and options.

    Returns a filehandle you should then print to,
    and later close to complete the operation.

    NOTE: check the return value from close!
    If your close didn't succeed, the file didn't get saved!
    """
    self.run_hook('new_file_start', key, cls, opts)

    create_open_arg = create_open_arg or {}
    create_close_arg = create_close_arg or {}

    # fid should be specified, or pass 0 meaning to auto-generate one
    fid = 0
    params = {'domain': self.domain,
              'key': key,
              'fid': fid,
              'multi_dest': 1}
    if cls is not None:
      params['class'] = cls
    res = self.backend.do_request('create_open', params)
    if not res:
      return None

    # [(devid, path), (devid, path),... ]
    dests = []
    # determine old vs. new format to populate destinations
    if 'dev_count' not in res:
      dests.append((res['devid'], res['path']))
    else:
      for x in range(1, int(res['dev_count']) + 1):
        devid_key = 'devid_%d' % x
        path_key = 'path_%s' % x
        dests.append((res[devid_key], res[path_key]))

    main_dest = dests[0]
    main_devid, main_path = main_dest

    self.run_hook("new_file_end", key, cls, opts)

    # TODO
    if largefile:
      file_class = LargeHTTPFile
    else:
      file_class = NormalHTTPFile

    return file_class(mg=self,
                      fid=res['fid'],
                      path=main_path,
                      devid=main_devid,
                      backup_dests=dests,
                      cls=cls,
                      key=key,
                      content_length=content_length,
                      create_close_arg=create_close_arg,
                      overwrite=1)

  def edit_file(self, key, overwrite=False):
    """Edit the file with the the given key.

    NOTE: edit_file is currently EXPERIMENTAL and not recommended for production use.
    MogileFS is primarily designed for storing files for later retrieval, rather than editing.
    Use of this function may lead to poor performance and, until it has been proven mature, should be considered to also potentially cause data loss.

    NOTE: use of this function requires support for the DAV 'MOVE' verb and partial PUT (i.e. Content-Range in PUT) on the back-end storage servers (e.g. apache with mod_dav).

    Returns a seekable filehandle you can read/write to.
    Calling this function may invalidate some or all URLs you currently have for this key, so you should call ->get_paths again afterwards if you need them.

    On close of the filehandle, the new file contents will replace the previous contents (and again invalidate any existing URLs).

    By default, the file contents are preserved on open, but you may specify the overwrite option to zero the file first.
    The seek position is at the beginning of the file, but you may seek to the end to append.
    """
    raise NotImplementedError()

  def read_file(self, key):
    """
    Read the file with the the given key.
    Returns a seekable filehandle you can read() from.
    Note that you cannot read line by line using <$fh> notation.

    Takes the same options as get_paths
    (which is called internally to get the URIs to read from).
    """
    paths = self.get_paths(key)
    if not paths:
      return None
    path = paths[0]
    backup_dests = [(None, p) for p in paths[1:]]
    return LargeHTTPFile(path=path, backup_dests=backup_dests, readonly=1)


  def store_file(self, key, fp, cls=None, chunk_size=8192):
    """
    Wrapper around new_file, print, and close.

    Given a key, class, and a filehandle or filename, stores the file
    contents in MogileFS.  Returns the number of bytes stored on success,
    undef on failure.
    """
    if self.readonly:
      return False

    params = {}
    if chunk_size:
      params['chunk_size'] = chunk_size
    if cls:
      params['class'] = cls
    params[key] = key

    self.run_hook('store_file_start', params)

    try:
      new_file = self.new_file(key, cls)
    except MogileFSError:
      fp.close()
      return False

    try:
      _bytes = 0
      while True:
        buf = fp.read(chunk_size)
        if not buf:
          break
        _bytes += len(buf)
        new_file.write(buf)

      self.run_hook('store_file_end', params)
    finally:
      fp.close()
      new_file.close()

    return _bytes

  def store_content(self, key, content, cls=None, **opts):
    """
    Wrapper around new_file, print, and close.  Given a key, class, and
    file contents (scalar or scalarref), stores the file contents in
    MogileFS. Returns the number of bytes stored on success, undef on
    failure.
    """
    if self.readonly:
      return False

    self.run_hook('store_content_start', key, cls, opts)

    output = self.new_file(key, cls, None, **opts)
    output.write(content)
    output.close()

    self.run_hook('store_content_end', key, cls, opts)

    return len(content)

  def get_paths(self, key, noverify=1, zone='alt', pathcount=2):
    """
    Given a key, returns an array of all the locations (HTTP URLs) that the file
    has been replicated to.
    """
    self.run_hook('get_paths_start', key)

    params = {'domain': self.domain,
              'key': key,
              'noverify': noverify and 1 or 0,
              'zone': zone,
              'pathcount': pathcount}

    res = self.backend.do_request('get_paths', params)
    paths = [res["path%d" % x] for x in range(1, int(res["paths"]) + 1)]

    self.run_hook('get_paths_end', key)
    return paths

  def get_file_data(self, key, timeout=10):
    """
    Returns scalarref of file contents in a scalarref.
    Don't use for large data, as it all comes back to you in one string.
    """
    fp = self.read_file(key)
    if not fp:
      return None
    try:
      content = fp.read()
      return content
    finally:
      fp.close()

  def delete(self, key):
    """
    Delete a file from MogileFS
    """
    try:
      if self.readonly:
        return False
      self.backend.do_request('delete', {'domain': self.domain, 'key': key})
      return True
    except MogileFSError:
      return False

  def rename(self, from_key, to_key):
    """
    Rename file (key) in MogileFS from oldkey to newkey.
    Returns true on success, failure otherwise
    """
    try:
      if self.readonly:
        return False
      params =  {'domain': self.domain,
                 'from_key': from_key,
                 'to_key': to_key}
      self.backend.do_request('rename', params)
      return True
    except MogileFSError:
      return False

  def list_keys(self, prefix=None, after=None, limit=None):
    """
    Used to get a list of keys matching a certain prefix.

    $prefix specifies what you want to get a list of.

    $after is the item specified as a return value from this function last time
          you called it.

    $limit is optional and defaults to 1000 keys returned.

    In list context, returns ($after, $keys).
    In scalar context, returns arrayref of keys.
    The value $after is to be used as $after when you call this function again.

    When there are no more keys in the list,
    you will get back undef or an empty list
    """
    params = {'domain': self.domain}
    if prefix:
      params['prefix'] = prefix
    if after:
      params['after'] = after
    if limit:
      params['limit'] = limit

    res = self.backend.do_request('list_keys', params)
    results = []
    for x in range(1, int(res['key_count']) + 1):
      results.append(res['key_%d' % x])
    return results

  def keys(self, prefix=None):
    """
    Get all keys matching a certain prefix
    """
    params = {'domain': self.domain}
    if prefix:
      params['prefix'] = prefix
    results = []
    while True:
      res = self.backend.do_request('list_keys', params)
      for x in res.keys():
        if x not in ['key_count', 'next_after']:
          results.append(res[x])
      if len(res) < 1002:
        break
      params['after'] = res['next_after']
    return list(set(results))


  def foreach_key(self, *args, **kwds):
    """
    Functional interface/wrapper around list_keys.

    Given some %OPTIONS (currently only one, "prefix"),
    calls your callback for each key matching the provided prefix.
    """
    raise NotImplementedError()

  def update_class(self, key, new_class):
    """
    Update the replication class of a pre-existing file,
    causing the file to become more or less replicated.
    """
    try:
      if self.readonly:
        return False
      params = {"domain": self.domain,
                "key": key,
                "class": new_class}
      res = self.backend.do_request("updateclass", params)
      return res
    except MogileFSError:
      return False

  def sleep(self, duration):
    """
    just makes some sleeping happen.  first and only argument is number of
    seconds to instruct backend thread to sleep for.
    """
    try:
      self.backend.do_request("sleep", {'duration': duration})
      return True
    except MogileFSError:
      return False

  def set_pref_ip(self, *ips):
    """
    Weird option for old, weird network architecture.
    Sets a mapping table of preferred alternate IPs, if reachable.
    For instance, if trying to connect to 10.0.0.2 in the above example,
    the module would instead try to connect to 10.2.0.2 quickly first,
    then then fall back to 10.0.0.2 if 10.2.0.2 wasn't reachable.
    expects as argument a tuple of ("standard-ip", "preferred-ip")
    """
    self.backend.set_pref_ip(*ips)
コード例 #11
0
ファイル: test_backend.py プロジェクト: AloneRoad/pymogile
 def test_do_request_one_tracker_down(self):
   backend = Backend(["127.0.0.1:7001", "127.0.0.1:7012"])
   try:
     backend.do_request("get_domains")
   except MogileFSError:
     assert False
コード例 #12
0
ファイル: admin.py プロジェクト: pombredanne/pymogile
class Admin(object):
  def __init__(self, hosts, backend=None, readonly=False, timeout=None, hooks=None):
    self.readonly = bool(readonly)
    self.backend = Backend(hosts, timeout)
    self._hosts = hosts

  def replicate_row(self):
    self.backend.do_request("replicate_row")

  def get_hosts(self, hostid=None):
    if hostid:
      params = {'hostid': hostid}
    else:
      params = None
    res = self.backend.do_request("get_hosts", params)
    ret = []
    fields = "hostid status hostname hostip http_port".split()
    hosts = int(res['hosts']) + 1
    for ct in xrange(1, hosts):
        ret.append(dict([ (f, res['host%d_%s' % (ct, f)]) for f in fields]))
    return ret

  def get_devices(self, devid=None):
    if devid:
      params = {'devid': devid}
    else:
      params = None
    res = self.backend.do_request("get_devices", params)
    ret = []
    for x in xrange(1, int(res['devices'])+1):
      device = {}
      for k in ('devid', 'hostid', 'status', 'observed_state', 'utilization'):
        device[k] = res.get('dev%d_%s' % (x, k))

      for k in ('mb_total', 'mb_used', 'weight'):
        value = res.get('dev%d_%s' % (x, k))
        if value:
          device[k] = int(value)
        else:
          device[k] = None
      ret.append(device)
    return ret

  def get_freespace(self, devid=None):
    """Get the free space for the entire cluster, or a specific node"""
    return sum([x['mb_free'] for x in  self.get_devices(devid)])

  def list_fids(self, fromfid, tofid):
    """
    get raw information about fids, for enumerating the dataset
       ( from_fid, to_fid )
    returns:
       { fid => { dict with keys: domain, class, devcount, length, key } }
    """
    res = self.backend.do_request('list_fids',
                    { 'from': fromfid,
                    'to'  : tofid,
                    })
    ret = {}
    for x in xrange(1, int(res['fid_count'])+1):
      key = 'fid_%d_fid' % x
      ret[key] = dict([(k, res['fid_%d_%s' % (x, k)]) for k in ('key', 'length', 'class', 'domain', 'devcount')])
    return ret

  def clear_cache(self, fromfid, tofid):
    params = {}
    return self.backend.do_request('clear_cache', params)

  def get_stats(self):
    params = { 'all': 1 }
    res = self.backend.do_request('stats', params)

    ret = {}
    # get replication statistics
    if 'replicationcount' in res:
      replication = ret.setdefault('replication', {})
      for x in xrange(1, int(res['replicationcount'])+1):
        domain = res.get('replication%ddomain' % x, '')
        cls = res.get('replication%dclass' % x, '')
        devcount = res.get('replication%ddevcount' % x, '')
        fields = res.get('replication%dfields' % x)
        (replication.setdefault(domain, {}).setdefault(cls, {}))[devcount] = fields

    # get file statistics
    if 'filescount' in res:
      files = ret.setdefault('files', {})
      for x in xrange(1, int(res['filescount'])+1):
        domain = res.get('files%ddomain' % x, '')
        cls = res.get('files%dclass' % x, '')
        (files.setdefault(domain, {}))[cls] = res.get('files%dfiles' % x)

    # get device statistics
    if 'devicescount' in res:
      devices = ret.setdefault('devices', {})
      for x in xrange(1, int(res['devicescount'])+1):
        key = res.get('devices%did' % x, '')
        devices[key] = { 'host'  : res.get('devices%dhost' % x),
                 'status': res.get('devices%dstatus' % x),
                 'files' : res.get('devices%dfiles' % x),
                 }

    if 'fidmax' in res:
      ret['fids'] = { 'max': res['fidmax'],
              }

    # return the created response
    return ret

  def get_domains(self):
    """
    get a dict of the domains we know about in the format of
       { domain_name : { class_name => mindevcount, class_name => mindevcount, ... }, ... }
    """
    res = self.backend.do_request('get_domains')
    ## KeyError, ValueError, TypeError
    domain_length = int(res['domains'])
    ret = {}
    for x in xrange(1, domain_length+1):
      domain_name = res['domain%d' % x]
      ret.setdefault(domain_name, {})
      class_length = int(res['domain%dclasses' % x])
      for y in xrange(1, class_length+1):
        k = 'domain%dclass%dname' % (x, y)
        v = 'domain%dclass%dmindevcount' % (x, y)
        ret[domain_name][res[k]] = int(res[v])
    return ret

  def create_domain(self, domain):
    """
    create a new domain
    """
    if self.readonly:
      return
    res = self.backend.do_request('create_domain',
                    { 'domain': domain })
    return res['domain'] == domain

  def delete_domain(self, domain):
    """
    delete a domain
    """
    _complain_ifreadonly(self.readonly)
    res = self.backend.do_request('delete_domain',
                    { 'domain': domain })
    return res['domain'] == domain

  def create_class(self, domain, cls, mindevcount):
    """
    create a class within a domain
    """
    try:
      return self._modify_class('create', domain, cls, mindevcount)
    except MogileFSTrackerError, e:
      if e.err != 'class_exists':
        raise e