예제 #1
0
    def lookup(self, volume_id):
        """
        lookup the urls of a volume
        return a json like below:
        -----------
        {
          "locations": [
            {
              "publicUrl": "localhost:8080",
              "url": "localhost:8080"
            }
          ]
        }
        -----------

        Arguments:
        - `self`:
        """
        result = None
        try:
            r = requests.get(self.url_lookup + '?volumeId=%s' % volume_id)
            result = json.loads(r.content)
        except Exception as e:
            LOGGER.error(
                'Could not get status of this volume: %s. Exception is: %s' %
                (self.url_status, e))
            result = None
        return result
예제 #2
0
    def lookup(self, volume_id):
        """
        lookup the urls of a volume
        return a json like below:
        -----------
        {
          "locations": [
            {
              "publicUrl": "localhost:8080",
              "url": "localhost:8080"
            }
          ]
        }
        -----------

        Arguments:
        - `self`:
        """
        result = None
        try:
            r = requests.get(self.url_lookup + '?volumeId=%s' % volume_id)
            result = json.loads(r.content)
        except Exception as e:
            LOGGER.error('Could not get status of this volume: %s. Exception is: %s' % (self.url_status, e))
            result = None
        return result
예제 #3
0
    def get_fid_full_url(self, fid):
        """
        get operatable full_url by fid.
        lookup info returns:
        -----------
        {
          "locations": [
            {
              "publicUrl": "localhost:8080",
              "url": "localhost:8080"
            }
          ]
        }
        -----------

        return something like: 'http://127.0.0.1:8080/3,0230203913'
        """
        result = None
        volume_id = fid.split(',')[0]
        full_url = ''
        try:
            r = self.master.lookup(volume_id)
            # _url = WEEDFS_MASTER_URL + '/dir/lookup?volumeId=%s' % volume_id
            # LOGGER.debug('lookup volume by url: %s' % _url)
            # r = requests.get(_url)
            result = json.loads(r.content)
            locations = result['locations']

            # choose a random location
            location = locations[random.randint(0, len(locations) - 1)]
            full_url = 'http://%s/%s' % (location['url'], fid)
        except Exception as e:
            LOGGER.error('Could not get volume location of this fid: %s. Exception is: %s' % (fid, e))
            result = None
        return full_url
예제 #4
0
 def update(self, fp, fid, fname=''):
     """
     update a file in weed-fs with @fid
     """
     fid_full_url = 'wrong_url'
     try:
         fid_full_url = self.get_fid_full_url(fid)
         LOGGER.debug('Updating file: fp: %s, fname: %s, fid_full_url: %s' % (fp, fname, fid_full_url))
         return WeedOperation.save_file_to_weed(fp, fid_full_url, fname)
     except Exception as e:
         err_msg = 'Could not Updating file: fp: %s, fname: %s, fid_full_url: %s, e: %s' % (fp, fname, fid_full_url, e)
         LOGGER.error(err_msg)
         return None
예제 #5
0
 def update_by_fid(self, dst_fid, src_fid, src_fname=''):
     """
     replace file@dst_fid with file@src_fid
     """
     try:
         src_file_rsp = self.read(src_fid, fname=src_fname, just_url=False)
         fp = StringIO.StringIO(src_file_rsp.content)
         LOGGER.debug('Updating file: dst_fid: %s, src_fid: %s, src_fname: %s,  fp: %s' % (dst_fid, src_fid, src_fname, fp))
         return self.update(fp, dst_fid, src_fname)
     except Exception as e:
         err_msg = 'Could not Updating file: dst_fid: %s, src_fid: %s, src_fname: %s. e: %s' % (dst_fid, src_fid, src_fname, e)
         LOGGER.error(err_msg)
         return None
예제 #6
0
    def delete_file(self, fid):
        ''' Delete a file by @fid

        Deprecated.

        Use WeedOperation().delete instead.
        '''
        url = urljoin(self.url_base, fid)
        try:
            r = requests.delete(url)
        except Exception as e:
            LOGGER.error("Could not delete file. Exception is: %s" % e)

        return r.content
예제 #7
0
    def delete_file(self, fid):
        ''' Delete a file by @fid

        Deprecated.

        Use WeedOperation().delete instead.
        '''
        url = urljoin(self.url_base, fid)
        try:
            r = requests.delete(url)
        except Exception as e:
            LOGGER.error("Could not delete file. Exception is: %s" % e)

        return r.content
예제 #8
0
    def get_status(self):
        """
        get status of this volume

        Arguments:
        - `self`:
        """
        r = requests.get(self.url_status)
        try:
            result = json.loads(r.content)
        except Exception as e:
            LOGGER.error('Could not get status of this volume: %s. Exception is: %s' % (self.url_status, e))
            result = None
        return result
예제 #9
0
 def create(self, fp, fname=''):
     """
     create a file in weed-fs with @fid
     """
     fid_full_url = 'wrong_url'
     try:
         wak = self.master.acquire_new_assign_key()
         LOGGER.debug('Creating file: fp: %s, fname: %s, fid_full_url: %s'
                      % (fp, fname, fid_full_url))
         return WeedOperation.save_file_to_weed(fp, wak.fid_full_url, fname)
     except Exception as e:
         err_msg = 'Could not create file: fp: %s, fname: %s, fid_full_url: %s, e: %s' % (fp, fname, fid_full_url, e)
         LOGGER.error(err_msg)
         return None
예제 #10
0
 def update(self, fp, fid, fname=''):
     """
     update a file in weed-fs with @fid
     """
     fid_full_url = 'wrong_url'
     try:
         fid_full_url = self.get_fid_full_url(fid)
         LOGGER.debug('Updating file: fp: %s, fname: %s, fid_full_url: %s' %
                      (fp, fname, fid_full_url))
         return WeedOperation.save_file_to_weed(fp, fid_full_url, fname)
     except Exception as e:
         err_msg = 'Could not Updating file: fp: %s, fname: %s, fid_full_url: %s, e: %s' % (
             fp, fname, fid_full_url, e)
         LOGGER.error(err_msg)
         return None
예제 #11
0
 def create(self, fp, fname=''):
     """
     create a file in weed-fs with @fid
     """
     fid_full_url = 'wrong_url'
     try:
         wak = self.master.acquire_new_assign_key()
         LOGGER.debug('Creating file: fp: %s, fname: %s, fid_full_url: %s' %
                      (fp, fname, fid_full_url))
         return WeedOperation.save_file_to_weed(fp, wak.fid_full_url, fname)
     except Exception as e:
         err_msg = 'Could not create file: fp: %s, fname: %s, fid_full_url: %s, e: %s' % (
             fp, fname, fid_full_url, e)
         LOGGER.error(err_msg)
         return None
예제 #12
0
 def update_by_fid(self, dst_fid, src_fid, src_fname=''):
     """
     replace file@dst_fid with file@src_fid
     """
     try:
         src_file_rsp = self.read(src_fid, fname=src_fname, just_url=False)
         fp = StringIO.StringIO(src_file_rsp.content)
         LOGGER.debug(
             'Updating file: dst_fid: %s, src_fid: %s, src_fname: %s,  fp: %s'
             % (dst_fid, src_fid, src_fname, fp))
         return self.update(fp, dst_fid, src_fname)
     except Exception as e:
         err_msg = 'Could not Updating file: dst_fid: %s, src_fid: %s, src_fname: %s. e: %s' % (
             dst_fid, src_fid, src_fname, e)
         LOGGER.error(err_msg)
         return None
예제 #13
0
    def get_status(self):
        """
        get status of this volume

        Arguments:
        - `self`:
        """
        r = requests.get(self.url_status)
        try:
            result = json.loads(r.content)
        except Exception as e:
            LOGGER.error(
                "Could not get status of this volume: %s. Exception is: %s" %
                (self.url_status, e))
            result = None
        return result
예제 #14
0
 def save_file_to_weed(self, fp, fid_full_url, fname=''):
     """
     save fp(file-pointer, file-description) to remote weed
     """
     tmp_uploading_file_name = fname or 'a.unknown'
     rsp = requests.post(fid_full_url, files={'file' : (tmp_uploading_file_name, fp)})
     # LOGGER.debug(rsp.request.headers)
     if not rsp.json().has_key('size'): # post new file fails
         err_msg = 'Could not save file on weed-fs with fid_full_url: %s' % (fid_full_url)
         LOGGER.error(err_msg)
         return (False, err_msg)
     data = {
         'fid_full_url' : fid_full_url,
         'fname' : fname,
         'storage_size' : rsp.json().get('size', 0),
         }
     return (True, data)
예제 #15
0
    def delete(self, fid, fname=''):
        """
        delete a file in weed-fs with @fid
        """
        fid_full_url = 'wrong_url'
        try:
            fid_full_url = self.get_fid_full_url(fid)
            LOGGER.debug('Deleting file: fid: %s, fname: %s, fid_full_url: %s' % (fid, fname, fid_full_url))

            r = requests.delete(fid_full_url)
            if r.json().has_key('size'):
                return True
        except Exception as e:
            err_msg = 'Deleting file: fid: %s, fname: %s, fid_full_url: %s, e: %s' % (fid, fname, fid_full_url, e)
            LOGGER.error(err_msg)
            return False
        return False
예제 #16
0
    def __init__(self, json_of_weed_response=None):

        self['fid'] = ''
        self['count'] = 0
        self['url'] = ''
        self['publicUrl'] = ''

        if json_of_weed_response:
            try:
                d = json.loads(json_of_weed_response)
                self.update(d)
            except Exception as e:
                LOGGER.error('Error for json.loads "%s".\nException: %s' %
                             (json_of_weed_response, e))

        for k, v in self.items():
            setattr(self, k, v)
        super(WeedAssignKey, self).__init__()
예제 #17
0
    def __init__(self, json_of_weed_response=None):

        self['fid'] = ''
        self['count'] = 0
        self['url'] = ''
        self['publicUrl'] = ''

        if json_of_weed_response:
            try:
                d = json.loads(json_of_weed_response)
                self.update(d)
            except Exception as e:
                LOGGER.error('Error for json.loads "%s".\nException: %s'
                             % (json_of_weed_response, e))

        for k, v in self.items():
            setattr(self, k, v)
        super(WeedAssignKey, self).__init__()
예제 #18
0
 def save_file_to_weed(self, fp, fid_full_url, fname=''):
     """
     save fp(file-pointer, file-description) to remote weed
     """
     tmp_uploading_file_name = fname or 'a.unknown'
     rsp = requests.post(fid_full_url,
                         files={'file': (tmp_uploading_file_name, fp)})
     # LOGGER.debug(rsp.request.headers)
     if not rsp.json().has_key('size'):  # post new file fails
         err_msg = 'Could not save file on weed-fs with fid_full_url: %s' % (
             fid_full_url)
         LOGGER.error(err_msg)
         return (False, err_msg)
     data = {
         'fid_full_url': fid_full_url,
         'fname': fname,
         'storage_size': rsp.json().get('size', 0),
     }
     return (True, data)
예제 #19
0
    def get_file(self, fid):
        ''' Get a file's content by @fid

        Deprecated.

        Use WeedOperation().get instead.
        '''
        url = urljoin(self.url_base, fid)
        try:
            r = requests.get(url)
        except Exception as e:
            LOGGER.error("Could not get file. Exception is: %s" % e)

        if r.status_code == 200:
            return r.content
        elif r.status_code == 404:
            LOGGER.error("File with fid %s not found" % fid)
            return None
        else:
            return None
예제 #20
0
    def delete(self, fid, fname=''):
        """
        delete a file in weed-fs with @fid
        """
        fid_full_url = 'wrong_url'
        try:
            fid_full_url = self.get_fid_full_url(fid)
            LOGGER.debug(
                'Deleting file: fid: %s, fname: %s, fid_full_url: %s' %
                (fid, fname, fid_full_url))

            r = requests.delete(fid_full_url)
            if r.json().has_key('size'):
                return True
        except Exception as e:
            err_msg = 'Deleting file: fid: %s, fname: %s, fid_full_url: %s, e: %s' % (
                fid, fname, fid_full_url, e)
            LOGGER.error(err_msg)
            return False
        return False
예제 #21
0
    def acquire_assign_info(self):
        """
        acquire an assign key from master-server.
        assign_key is in json format like below:
        -----------
        {"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"}
        -----------

        Arguments:
        - `self`:
        """
        result = None
        try:
            r = requests.get(self.url_assign)
            result = json.loads(r.content)
        except Exception as e:
            LOGGER.error('Could not get status of this volume: %s. Exception is: %s'
                         % (self.url_status, e))
            result = None
        return result
예제 #22
0
    def get_file(self, fid):
        ''' Get a file's content by @fid

        Deprecated.

        Use WeedOperation().get instead.
        '''
        url = urljoin(self.url_base, fid)
        try:
            r = requests.get(url)
        except Exception as e:
            LOGGER.error("Could not get file. Exception is: %s" % e)

        if r.status_code == 200:
            return r.content
        elif r.status_code == 404:
            LOGGER.error("File with fid %s not found" % fid)
            return None
        else:
            return None
예제 #23
0
    def read(self, fid, fname='', just_url=True):
        """
        read/get a file from weed-fs with @fid.

        @just_url:
          True -> just return fid_full_url (web-servers/browsers like nginx, chrome can get resource by this url)
          False -> return a http response of requests(package requests).
        """
        fid_full_url = 'wrong_url'
        try:
            fid_full_url = self.get_fid_full_url(fid)
            LOGGER.debug('Reading file(just_url:%s): fid: %s, fname: %s, fid_full_url: %s' % (just_url, fid, fname, fid_full_url))
            if just_url:
                return fid_full_url
            else:
                rsp = requests.get(fid_full_url)
                return rsp
        except Exception as e:
            err_msg = 'Could not read file(just_url:%s): fid: %s, fname: %s, fid_full_url: %s, e: %s' % (just_url, fid, fname, fid_full_url, e)
            LOGGER.error(err_msg)
            return None
예제 #24
0
    def acquire_assign_info(self):
        """
        acquire an assign key from master-server.
        assign_key is in json format like below:
        -----------
        {"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"}
        -----------

        Arguments:
        - `self`:
        """
        result = None
        try:
            r = requests.get(self.url_assign)
            result = json.loads(r.content)
        except Exception as e:
            LOGGER.error(
                'Could not get status of this volume: %s. Exception is: %s' %
                (self.url_status, e))
            result = None
        return result
예제 #25
0
    def acquire_new_assign_key(self, count=1):
        """
        get a new avalable new volume-file-location from from weed-master by getting a new-assign-key
        Arguments:
        - `self`:

        assign_key is in json format like below:
        -----------
        {"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"}
        -----------

        return a tuple(dst_file_fid, dst_volume_url) like below:
        ----------
        (dst_file_fid, http://{volume-url})
        (3,20392030920, http://127.0.0.1:8080)
        ----------

        """
        assign_key_url = self.url_assign + '?count=' + str(count)
        dst_volume_url = None
        wak = WeedAssignKeyExtended()
        try:
            LOGGER.debug('Getting new dst_volume_url with master-assign-key-url: %s' % assign_key_url)
            r = requests.get(assign_key_url)
            key_dict = json.loads(r.content)

            wak.update(key_dict)
            wak.update_full_urls()

            LOGGER.info('Successfuly got dst_volume_url: %s' % dst_volume_url)
        except Exception as e:
            LOGGER.error('Could not get new assign key from the assign url: %s. Exception is: %s'
                         % (assign_key_url, e))
        return wak
예제 #26
0
    def put_file(self, absolute_file_path, fid, headers=None):
        ''' you can put exact http-headers in @headers to help weed to clarify putting file.

        eg:
           @headers = {'content-type' : 'image/png'} or
           @headers = {'content-type' : 'image/jpeg'} or
           @headers = {'content-type' : 'text/xml'} or
           @headers = {'content-type' : 'application/json'}

        Deprecated.

        Use util.put_file instead.
        '''
        url = urljoin(self.url_base, fid)
        if headers and isinstance(headers, dict):
            files = {'file': (open(absolute_file_path, 'rb')), 'headers' : headers}
        else:
            files = {'file': (open(absolute_file_path, 'rb'))}
        try:
            r = requests.post(url, files=files)
        except Exception as e:
            LOGGER.error("Could not post file. Exception is: %s" % e)
            return None

        # weed-fs returns a 200 but the content may contain an error
        result = json.loads(r.content)
        if r.status_code == 200:
            if 'error' in result:
                LOGGER.error(result['error'])
            else:
                LOGGER.debug(result)

        return result
예제 #27
0
def put_file(fp, fid_full_url, fname='', http_headers=None):
    """
    save fp(file-pointer, file-description) to a remote weed volume.
    eg:
       PUT http://127.0.0.1:8080/3,20392030920

    addtional fname and http_headers can help weed-server to decide content-type and other infos.

    eg:
       @fname = 'hello.txt' or 'abc.jpg' or 'youknow.png',

       @http_headers = {'content-type' : 'image/png'} or
       @http_headers = {'content-type' : 'image/jpeg'} or
       @http_headers = {'content-type' : 'text/xml'} or
       @http_headers = {'content-type' : 'application/json'}


    """
    pos = fp.tell()
    tmp_uploading_file_name = fname or 'a.unknown'
    # print('fid_full_url is: "%s"' % fid_full_url)
    # print('fp position: %d' % fp.tell())
    # print('fp info: length: %d' % len(fp.read()))
    # fp.seek(0)
    if http_headers:
        rsp = requests.post(fid_full_url,
                            files={tmp_uploading_file_name: fp},
                            headers=http_headers)
    else:
        rsp = requests.post(fid_full_url, files={tmp_uploading_file_name: fp})

    # recove position of fp
    fp.seek(pos)

    # LOGGER.debug(rsp.request.headers)
    rsp_json = rsp.json()
    wor = WeedOperationResponse()
    wor.status = 'success'
    wor.url = fid_full_url
    wor.fname = fname
    wor.storage_size = rsp.json().get('size', 0)
    LOGGER.info('wor is: %s' % wor)

    if 'error' in rsp_json:
        LOGGER.error('Put file fails. Error returns from weedfs: "%s"' %
                     rsp_json['error'])
        wor.status = 'fail'
        wor.message = rsp_json['error']
    elif not rsp_json.has_key(
            'size') or rsp_json['size'] == 0:  # post new file fails
        err_msg = 'Could not save file on weed-fs with fid_full_url: %s' % (
            fid_full_url)
        LOGGER.error(err_msg)
        wor.status = 'fail'
        wor.message = err_msg
    else:
        pass

    return wor
예제 #28
0
    def get_fid_full_url(self, fid):
        """
        get operatable full_url by fid.
        lookup info returns:
        -----------
        {
          "locations": [
            {
              "publicUrl": "localhost:8080",
              "url": "localhost:8080"
            }
          ]
        }
        -----------

        return something like: 'http://127.0.0.1:8080/3,0230203913'
        """
        result = None
        volume_id = fid.split(',')[0]
        full_url = ''
        try:
            r = self.master.lookup(volume_id)
            # _url = WEEDFS_MASTER_URL + '/dir/lookup?volumeId=%s' % volume_id
            # LOGGER.debug('lookup volume by url: %s' % _url)
            # r = requests.get(_url)
            result = json.loads(r.content)
            locations = result['locations']

            # choose a random location
            location = locations[random.randint(0, len(locations) - 1)]
            full_url = 'http://%s/%s' % (location['url'], fid)
        except Exception as e:
            LOGGER.error(
                'Could not get volume location of this fid: %s. Exception is: %s'
                % (fid, e))
            result = None
        return full_url
예제 #29
0
    def read(self, fid, fname='', just_url=True):
        """
        read/get a file from weed-fs with @fid.

        @just_url:
          True -> just return fid_full_url (web-servers/browsers like nginx, chrome can get resource by this url)
          False -> return a http response of requests(package requests).
        """
        fid_full_url = 'wrong_url'
        try:
            fid_full_url = self.get_fid_full_url(fid)
            LOGGER.debug(
                'Reading file(just_url:%s): fid: %s, fname: %s, fid_full_url: %s'
                % (just_url, fid, fname, fid_full_url))
            if just_url:
                return fid_full_url
            else:
                rsp = requests.get(fid_full_url)
                return rsp
        except Exception as e:
            err_msg = 'Could not read file(just_url:%s): fid: %s, fname: %s, fid_full_url: %s, e: %s' % (
                just_url, fid, fname, fid_full_url, e)
            LOGGER.error(err_msg)
            return None
예제 #30
0
def put_file(fp, fid_full_url, fname='', http_headers=None):
    """
    save fp(file-pointer, file-description) to a remote weed volume.
    eg:
       PUT http://127.0.0.1:8080/3,20392030920

    addtional fname and http_headers can help weed-server to decide content-type and other infos.

    eg:
       @fname = 'hello.txt' or 'abc.jpg' or 'youknow.png',

       @http_headers = {'content-type' : 'image/png'} or
       @http_headers = {'content-type' : 'image/jpeg'} or
       @http_headers = {'content-type' : 'text/xml'} or
       @http_headers = {'content-type' : 'application/json'}


    """
    pos = fp.tell()
    tmp_uploading_file_name = fname or 'a.unknown'
    # print('fid_full_url is: "%s"' % fid_full_url)
    # print('fp position: %d' % fp.tell())
    # print('fp info: length: %d' % len(fp.read()))
    # fp.seek(0)
    if http_headers:
        rsp = requests.post(fid_full_url, files={tmp_uploading_file_name : fp}, headers=http_headers)
    else:
        rsp = requests.post(fid_full_url, files={tmp_uploading_file_name : fp})

    # recove position of fp
    fp.seek(pos)

    # LOGGER.debug(rsp.request.headers)
    rsp_json = rsp.json()
    wor = WeedOperationResponse()
    wor.status == 'success'
    wor.url = fid_full_url
    wor.fname = fname
    wor.storage_size = rsp.json().get('size', 0)
    LOGGER.info('wor is: %s' % wor)

    if 'error' in rsp_json:
        LOGGER.error('Put file fails. Error returns from weedfs: "%s"' % rsp_json['error'])
        wor.status = 'fail'
        wor.message = rsp_json['error']
    elif not rsp_json.has_key('size') or rsp_json['size'] == 0: # post new file fails
        err_msg = 'Could not save file on weed-fs with fid_full_url: %s' % (fid_full_url)
        LOGGER.error(err_msg)
        wor.status = 'fail'
        wor.message = err_msg
    else:
        pass

    return wor
예제 #31
0
    def acquire_new_assign_key(self, count=1):
        """
        get a new avalable new volume-file-location from from weed-master by getting a new-assign-key
        Arguments:
        - `self`:

        assign_key is in json format like below:
        -----------
        {"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"}
        -----------

        return a tuple(dst_file_fid, dst_volume_url) like below:
        ----------
        (dst_file_fid, http://{volume-url})
        (3,20392030920, http://127.0.0.1:8080)
        ----------

        """
        assign_key_url = self.url_assign + '?count=' + str(count)
        dst_volume_url = None
        wak = WeedAssignKeyExtended()
        try:
            LOGGER.debug(
                'Getting new dst_volume_url with master-assign-key-url: %s' %
                assign_key_url)
            r = requests.get(assign_key_url)
            key_dict = json.loads(r.content)

            wak.update(key_dict)
            wak.update_full_urls()

            LOGGER.info('Successfuly got dst_volume_url: %s' % dst_volume_url)
        except Exception as e:
            LOGGER.error(
                'Could not get new assign key from the assign url: %s. Exception is: %s'
                % (assign_key_url, e))
        return wak
예제 #32
0
    def put_file(self, absolute_file_path, fid, headers=None):
        ''' you can put exact http-headers in @headers to help weed to clarify putting file.

        eg:
           @headers = {'content-type' : 'image/png'} or
           @headers = {'content-type' : 'image/jpeg'} or
           @headers = {'content-type' : 'text/xml'} or
           @headers = {'content-type' : 'application/json'}

        Deprecated.

        Use util.put_file instead.
        '''
        url = urljoin(self.url_base, fid)
        if headers and isinstance(headers, dict):
            files = {
                'file': (open(absolute_file_path, 'rb')),
                'headers': headers
            }
        else:
            files = {'file': (open(absolute_file_path, 'rb'))}
        try:
            r = requests.post(url, files=files)
        except Exception as e:
            LOGGER.error("Could not post file. Exception is: %s" % e)
            return None

        # weed-fs returns a 200 but the content may contain an error
        result = json.loads(r.content)
        if r.status_code == 200:
            if 'error' in result:
                LOGGER.error(result['error'])
            else:
                LOGGER.debug(result)

        return result
예제 #33
0
    def vacuum(self):
        """
        Force Garbage Collection

        If your system has many deletions, the deleted file's disk space will not be synchronously re-claimed. There is a background job to check volume disk usage. If empty space is more than the threshold, default to 0.3, the vacuum job will make the volume readonly, create a new volume with only existing files, and switch on the new volume. If you are impatient or doing some testing, vacuum the unused spaces this way.

        > curl "http://localhost:9333/vol/vacuum"
        > curl "http://localhost:9333/vol/vacuum?garbageThreshold=0.4"

        return a json of refreshed status like below:
        -----------
        {
          "Topology": {
            "DataCenters": [
              {
                "Free": 93,
                "Max": 100,
                "Racks": [
                  {
                    "DataNodes": [
                      {
                        "Free": 93,
                        "Max": 100,
                        "PublicUrl": "127.0.0.1:8080",
                        "Url": "127.0.0.1:8080",
                        "Volumes": 7
                      }
                    ],
                    "Free": 93,
                    "Max": 100
                  }
                ]
              }
            ],
            "Free": 93,
            "Max": 100,
            "layouts": [
              {
                "replication": "000",
                "writables": [
                  2,
                  3,
                  5,
                  6,
                  7,
                  1,
                  4
                ]
              }
            ]
          },
          "Version": "0.37"
        }

                {
                  "locations": [
                    {
                      "publicUrl": "localhost:8080",
                      "url": "localhost:8080"
                    }
                  ]
                }
        -----------

        Arguments:
        - `self`:
        """
        result = None
        try:
            r = requests.get(self.url_vacuum)
            result = json.loads(r.content)
        except Exception as e:
            LOGGER.error(
                'Could not get status of this volume: %s. Exception is: %s' %
                (self.url_status, e))
            result = None
        return result
예제 #34
0
    def vacuum(self):
        """
        Force Garbage Collection

        If your system has many deletions, the deleted file's disk space will not be synchronously re-claimed. There is a background job to check volume disk usage. If empty space is more than the threshold, default to 0.3, the vacuum job will make the volume readonly, create a new volume with only existing files, and switch on the new volume. If you are impatient or doing some testing, vacuum the unused spaces this way.

        > curl "http://localhost:9333/vol/vacuum"
        > curl "http://localhost:9333/vol/vacuum?garbageThreshold=0.4"

        return a json of refreshed status like below:
        -----------
        {
          "Topology": {
            "DataCenters": [
              {
                "Free": 93,
                "Max": 100,
                "Racks": [
                  {
                    "DataNodes": [
                      {
                        "Free": 93,
                        "Max": 100,
                        "PublicUrl": "127.0.0.1:8080",
                        "Url": "127.0.0.1:8080",
                        "Volumes": 7
                      }
                    ],
                    "Free": 93,
                    "Max": 100
                  }
                ]
              }
            ],
            "Free": 93,
            "Max": 100,
            "layouts": [
              {
                "replication": "000",
                "writables": [
                  2,
                  3,
                  5,
                  6,
                  7,
                  1,
                  4
                ]
              }
            ]
          },
          "Version": "0.37"
        }

                {
                  "locations": [
                    {
                      "publicUrl": "localhost:8080",
                      "url": "localhost:8080"
                    }
                  ]
                }
        -----------

        Arguments:
        - `self`:
        """
        result = None
        try:
            r = requests.get(self.url_vacuum)
            result = json.loads(r.content)
        except Exception as e:
            LOGGER.error('Could not get status of this volume: %s. Exception is: %s' % (self.url_status, e))
            result = None
        return result