def lookup(self, volume_id): """ lookup the urls of a volume return a json like below: ----------- { "locations": [ { "publicUrl": "localhost:8080", "url": "localhost:8080" } ] } ----------- Arguments: - `self`: """ result = None try: r = requests.get(self.url_lookup + '?volumeId=%s' % volume_id) result = json.loads(r.content) except Exception as e: LOGGER.error('Could not get status of this volume: %s. Exception is: %s' % (self.url_status, e)) result = None return result
def lookup(self, volume_id): """ lookup the urls of a volume return a json like below: ----------- { "locations": [ { "publicUrl": "localhost:8080", "url": "localhost:8080" } ] } ----------- Arguments: - `self`: """ result = None try: r = requests.get(self.url_lookup + '?volumeId=%s' % volume_id) result = json.loads(r.content) except Exception as e: LOGGER.error( 'Could not get status of this volume: %s. Exception is: %s' % (self.url_status, e)) result = None return result
def acquire_new_assign_key(self, count=1): """ get a new avalable new volume-file-location from from weed-master by getting a new-assign-key Arguments: - `self`: assign_key is in json format like below: ----------- {"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"} ----------- return a tuple(dst_file_fid, dst_volume_url) like below: ---------- (dst_file_fid, http://{volume-url}) (3,20392030920, http://127.0.0.1:8080) ---------- """ assign_key_url = self.url_assign + '?count=' + str(count) dst_volume_url = None wak = WeedAssignKeyExtended() try: LOGGER.debug('Getting new dst_volume_url with master-assign-key-url: %s' % assign_key_url) r = requests.get(assign_key_url) key_dict = json.loads(r.content) wak.update(key_dict) wak.update_full_urls() LOGGER.info('Successfuly got dst_volume_url: %s' % dst_volume_url) except Exception as e: LOGGER.error('Could not get new assign key from the assign url: %s. Exception is: %s' % (assign_key_url, e)) return wak
def put_file(self, absolute_file_path, fid, headers=None): ''' you can put exact http-headers in @headers to help weed to clarify putting file. eg: @headers = {'content-type' : 'image/png'} or @headers = {'content-type' : 'image/jpeg'} or @headers = {'content-type' : 'text/xml'} or @headers = {'content-type' : 'application/json'} Deprecated. Use util.put_file instead. ''' url = urljoin(self.url_base, fid) if headers and isinstance(headers, dict): files = {'file': (open(absolute_file_path, 'rb')), 'headers' : headers} else: files = {'file': (open(absolute_file_path, 'rb'))} try: r = requests.post(url, files=files) except Exception as e: LOGGER.error("Could not post file. Exception is: %s" % e) return None # weed-fs returns a 200 but the content may contain an error result = json.loads(r.content) if r.status_code == 200: if 'error' in result: LOGGER.error(result['error']) else: LOGGER.debug(result) return result
def get_fid_full_url(self, fid): """ get operatable full_url by fid. lookup info returns: ----------- { "locations": [ { "publicUrl": "localhost:8080", "url": "localhost:8080" } ] } ----------- return something like: 'http://127.0.0.1:8080/3,0230203913' """ result = None volume_id = fid.split(',')[0] full_url = '' try: r = self.master.lookup(volume_id) # _url = WEEDFS_MASTER_URL + '/dir/lookup?volumeId=%s' % volume_id # LOGGER.debug('lookup volume by url: %s' % _url) # r = requests.get(_url) result = json.loads(r.content) locations = result['locations'] # choose a random location location = locations[random.randint(0, len(locations) - 1)] full_url = 'http://%s/%s' % (location['url'], fid) except Exception as e: LOGGER.error('Could not get volume location of this fid: %s. Exception is: %s' % (fid, e)) result = None return full_url
def put_file(fp, fid_full_url, fname='', http_headers=None): """ save fp(file-pointer, file-description) to a remote weed volume. eg: PUT http://127.0.0.1:8080/3,20392030920 addtional fname and http_headers can help weed-server to decide content-type and other infos. eg: @fname = 'hello.txt' or 'abc.jpg' or 'youknow.png', @http_headers = {'content-type' : 'image/png'} or @http_headers = {'content-type' : 'image/jpeg'} or @http_headers = {'content-type' : 'text/xml'} or @http_headers = {'content-type' : 'application/json'} """ pos = fp.tell() tmp_uploading_file_name = fname or 'a.unknown' # print('fid_full_url is: "%s"' % fid_full_url) # print('fp position: %d' % fp.tell()) # print('fp info: length: %d' % len(fp.read())) # fp.seek(0) if http_headers: rsp = requests.post(fid_full_url, files={tmp_uploading_file_name: fp}, headers=http_headers) else: rsp = requests.post(fid_full_url, files={tmp_uploading_file_name: fp}) # recove position of fp fp.seek(pos) # LOGGER.debug(rsp.request.headers) rsp_json = rsp.json() wor = WeedOperationResponse() wor.status = 'success' wor.url = fid_full_url wor.fname = fname wor.storage_size = rsp.json().get('size', 0) LOGGER.info('wor is: %s' % wor) if 'error' in rsp_json: LOGGER.error('Put file fails. Error returns from weedfs: "%s"' % rsp_json['error']) wor.status = 'fail' wor.message = rsp_json['error'] elif not rsp_json.has_key( 'size') or rsp_json['size'] == 0: # post new file fails err_msg = 'Could not save file on weed-fs with fid_full_url: %s' % ( fid_full_url) LOGGER.error(err_msg) wor.status = 'fail' wor.message = err_msg else: pass return wor
def put_file(fp, fid_full_url, fname='', http_headers=None): """ save fp(file-pointer, file-description) to a remote weed volume. eg: PUT http://127.0.0.1:8080/3,20392030920 addtional fname and http_headers can help weed-server to decide content-type and other infos. eg: @fname = 'hello.txt' or 'abc.jpg' or 'youknow.png', @http_headers = {'content-type' : 'image/png'} or @http_headers = {'content-type' : 'image/jpeg'} or @http_headers = {'content-type' : 'text/xml'} or @http_headers = {'content-type' : 'application/json'} """ pos = fp.tell() tmp_uploading_file_name = fname or 'a.unknown' # print('fid_full_url is: "%s"' % fid_full_url) # print('fp position: %d' % fp.tell()) # print('fp info: length: %d' % len(fp.read())) # fp.seek(0) if http_headers: rsp = requests.post(fid_full_url, files={tmp_uploading_file_name : fp}, headers=http_headers) else: rsp = requests.post(fid_full_url, files={tmp_uploading_file_name : fp}) # recove position of fp fp.seek(pos) # LOGGER.debug(rsp.request.headers) rsp_json = rsp.json() wor = WeedOperationResponse() wor.status == 'success' wor.url = fid_full_url wor.fname = fname wor.storage_size = rsp.json().get('size', 0) LOGGER.info('wor is: %s' % wor) if 'error' in rsp_json: LOGGER.error('Put file fails. Error returns from weedfs: "%s"' % rsp_json['error']) wor.status = 'fail' wor.message = rsp_json['error'] elif not rsp_json.has_key('size') or rsp_json['size'] == 0: # post new file fails err_msg = 'Could not save file on weed-fs with fid_full_url: %s' % (fid_full_url) LOGGER.error(err_msg) wor.status = 'fail' wor.message = err_msg else: pass return wor
def update_by_fid(self, dst_fid, src_fid, src_fname=''): """ replace file@dst_fid with file@src_fid """ try: src_file_rsp = self.read(src_fid, fname=src_fname, just_url=False) fp = StringIO.StringIO(src_file_rsp.content) LOGGER.debug('Updating file: dst_fid: %s, src_fid: %s, src_fname: %s, fp: %s' % (dst_fid, src_fid, src_fname, fp)) return self.update(fp, dst_fid, src_fname) except Exception as e: err_msg = 'Could not Updating file: dst_fid: %s, src_fid: %s, src_fname: %s. e: %s' % (dst_fid, src_fid, src_fname, e) LOGGER.error(err_msg) return None
def update(self, fp, fid, fname=''): """ update a file in weed-fs with @fid """ fid_full_url = 'wrong_url' try: fid_full_url = self.get_fid_full_url(fid) LOGGER.debug('Updating file: fp: %s, fname: %s, fid_full_url: %s' % (fp, fname, fid_full_url)) return WeedOperation.save_file_to_weed(fp, fid_full_url, fname) except Exception as e: err_msg = 'Could not Updating file: fp: %s, fname: %s, fid_full_url: %s, e: %s' % (fp, fname, fid_full_url, e) LOGGER.error(err_msg) return None
def delete_file(self, fid): ''' Delete a file by @fid Deprecated. Use WeedOperation().delete instead. ''' url = urljoin(self.url_base, fid) try: r = requests.delete(url) except Exception as e: LOGGER.error("Could not delete file. Exception is: %s" % e) return r.content
def get_status(self): """ get status of this volume Arguments: - `self`: """ r = requests.get(self.url_status) try: result = json.loads(r.content) except Exception as e: LOGGER.error('Could not get status of this volume: %s. Exception is: %s' % (self.url_status, e)) result = None return result
def create(self, fp, fname=''): """ create a file in weed-fs with @fid """ fid_full_url = 'wrong_url' try: wak = self.master.acquire_new_assign_key() LOGGER.debug('Creating file: fp: %s, fname: %s, fid_full_url: %s' % (fp, fname, fid_full_url)) return WeedOperation.save_file_to_weed(fp, wak.fid_full_url, fname) except Exception as e: err_msg = 'Could not create file: fp: %s, fname: %s, fid_full_url: %s, e: %s' % (fp, fname, fid_full_url, e) LOGGER.error(err_msg) return None
def update(self, fp, fid, fname=''): """ update a file in weed-fs with @fid """ fid_full_url = 'wrong_url' try: fid_full_url = self.get_fid_full_url(fid) LOGGER.debug('Updating file: fp: %s, fname: %s, fid_full_url: %s' % (fp, fname, fid_full_url)) return WeedOperation.save_file_to_weed(fp, fid_full_url, fname) except Exception as e: err_msg = 'Could not Updating file: fp: %s, fname: %s, fid_full_url: %s, e: %s' % ( fp, fname, fid_full_url, e) LOGGER.error(err_msg) return None
def create(self, fp, fname=''): """ create a file in weed-fs with @fid """ fid_full_url = 'wrong_url' try: wak = self.master.acquire_new_assign_key() LOGGER.debug('Creating file: fp: %s, fname: %s, fid_full_url: %s' % (fp, fname, fid_full_url)) return WeedOperation.save_file_to_weed(fp, wak.fid_full_url, fname) except Exception as e: err_msg = 'Could not create file: fp: %s, fname: %s, fid_full_url: %s, e: %s' % ( fp, fname, fid_full_url, e) LOGGER.error(err_msg) return None
def get_status(self): """ get status of this volume Arguments: - `self`: """ r = requests.get(self.url_status) try: result = json.loads(r.content) except Exception as e: LOGGER.error( "Could not get status of this volume: %s. Exception is: %s" % (self.url_status, e)) result = None return result
def update_by_fid(self, dst_fid, src_fid, src_fname=''): """ replace file@dst_fid with file@src_fid """ try: src_file_rsp = self.read(src_fid, fname=src_fname, just_url=False) fp = StringIO.StringIO(src_file_rsp.content) LOGGER.debug( 'Updating file: dst_fid: %s, src_fid: %s, src_fname: %s, fp: %s' % (dst_fid, src_fid, src_fname, fp)) return self.update(fp, dst_fid, src_fname) except Exception as e: err_msg = 'Could not Updating file: dst_fid: %s, src_fid: %s, src_fname: %s. e: %s' % ( dst_fid, src_fid, src_fname, e) LOGGER.error(err_msg) return None
def delete(self, fid, fname=''): """ delete a file in weed-fs with @fid """ fid_full_url = 'wrong_url' try: fid_full_url = self.get_fid_full_url(fid) LOGGER.debug('Deleting file: fid: %s, fname: %s, fid_full_url: %s' % (fid, fname, fid_full_url)) r = requests.delete(fid_full_url) if r.json().has_key('size'): return True except Exception as e: err_msg = 'Deleting file: fid: %s, fname: %s, fid_full_url: %s, e: %s' % (fid, fname, fid_full_url, e) LOGGER.error(err_msg) return False return False
def save_file_to_weed(self, fp, fid_full_url, fname=''): """ save fp(file-pointer, file-description) to remote weed """ tmp_uploading_file_name = fname or 'a.unknown' rsp = requests.post(fid_full_url, files={'file' : (tmp_uploading_file_name, fp)}) # LOGGER.debug(rsp.request.headers) if not rsp.json().has_key('size'): # post new file fails err_msg = 'Could not save file on weed-fs with fid_full_url: %s' % (fid_full_url) LOGGER.error(err_msg) return (False, err_msg) data = { 'fid_full_url' : fid_full_url, 'fname' : fname, 'storage_size' : rsp.json().get('size', 0), } return (True, data)
def __init__(self, json_of_weed_response=None): self['fid'] = '' self['count'] = 0 self['url'] = '' self['publicUrl'] = '' if json_of_weed_response: try: d = json.loads(json_of_weed_response) self.update(d) except Exception as e: LOGGER.error('Error for json.loads "%s".\nException: %s' % (json_of_weed_response, e)) for k, v in self.items(): setattr(self, k, v) super(WeedAssignKey, self).__init__()
def save_file_to_weed(self, fp, fid_full_url, fname=''): """ save fp(file-pointer, file-description) to remote weed """ tmp_uploading_file_name = fname or 'a.unknown' rsp = requests.post(fid_full_url, files={'file': (tmp_uploading_file_name, fp)}) # LOGGER.debug(rsp.request.headers) if not rsp.json().has_key('size'): # post new file fails err_msg = 'Could not save file on weed-fs with fid_full_url: %s' % ( fid_full_url) LOGGER.error(err_msg) return (False, err_msg) data = { 'fid_full_url': fid_full_url, 'fname': fname, 'storage_size': rsp.json().get('size', 0), } return (True, data)
def delete(self, fid, fname=''): """ delete a file in weed-fs with @fid """ fid_full_url = 'wrong_url' try: fid_full_url = self.get_fid_full_url(fid) LOGGER.debug( 'Deleting file: fid: %s, fname: %s, fid_full_url: %s' % (fid, fname, fid_full_url)) r = requests.delete(fid_full_url) if r.json().has_key('size'): return True except Exception as e: err_msg = 'Deleting file: fid: %s, fname: %s, fid_full_url: %s, e: %s' % ( fid, fname, fid_full_url, e) LOGGER.error(err_msg) return False return False
def get_file(self, fid): ''' Get a file's content by @fid Deprecated. Use WeedOperation().get instead. ''' url = urljoin(self.url_base, fid) try: r = requests.get(url) except Exception as e: LOGGER.error("Could not get file. Exception is: %s" % e) if r.status_code == 200: return r.content elif r.status_code == 404: LOGGER.error("File with fid %s not found" % fid) return None else: return None
def acquire_assign_info(self): """ acquire an assign key from master-server. assign_key is in json format like below: ----------- {"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"} ----------- Arguments: - `self`: """ result = None try: r = requests.get(self.url_assign) result = json.loads(r.content) except Exception as e: LOGGER.error('Could not get status of this volume: %s. Exception is: %s' % (self.url_status, e)) result = None return result
def read(self, fid, fname='', just_url=True): """ read/get a file from weed-fs with @fid. @just_url: True -> just return fid_full_url (web-servers/browsers like nginx, chrome can get resource by this url) False -> return a http response of requests(package requests). """ fid_full_url = 'wrong_url' try: fid_full_url = self.get_fid_full_url(fid) LOGGER.debug('Reading file(just_url:%s): fid: %s, fname: %s, fid_full_url: %s' % (just_url, fid, fname, fid_full_url)) if just_url: return fid_full_url else: rsp = requests.get(fid_full_url) return rsp except Exception as e: err_msg = 'Could not read file(just_url:%s): fid: %s, fname: %s, fid_full_url: %s, e: %s' % (just_url, fid, fname, fid_full_url, e) LOGGER.error(err_msg) return None
def acquire_assign_info(self): """ acquire an assign key from master-server. assign_key is in json format like below: ----------- {"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"} ----------- Arguments: - `self`: """ result = None try: r = requests.get(self.url_assign) result = json.loads(r.content) except Exception as e: LOGGER.error( 'Could not get status of this volume: %s. Exception is: %s' % (self.url_status, e)) result = None return result
def acquire_new_assign_key(self, count=1): """ get a new avalable new volume-file-location from from weed-master by getting a new-assign-key Arguments: - `self`: assign_key is in json format like below: ----------- {"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"} ----------- return a tuple(dst_file_fid, dst_volume_url) like below: ---------- (dst_file_fid, http://{volume-url}) (3,20392030920, http://127.0.0.1:8080) ---------- """ assign_key_url = self.url_assign + '?count=' + str(count) dst_volume_url = None wak = WeedAssignKeyExtended() try: LOGGER.debug( 'Getting new dst_volume_url with master-assign-key-url: %s' % assign_key_url) r = requests.get(assign_key_url) key_dict = json.loads(r.content) wak.update(key_dict) wak.update_full_urls() LOGGER.info('Successfuly got dst_volume_url: %s' % dst_volume_url) except Exception as e: LOGGER.error( 'Could not get new assign key from the assign url: %s. Exception is: %s' % (assign_key_url, e)) return wak
def get_fid_full_url(self, fid): """ get operatable full_url by fid. lookup info returns: ----------- { "locations": [ { "publicUrl": "localhost:8080", "url": "localhost:8080" } ] } ----------- return something like: 'http://127.0.0.1:8080/3,0230203913' """ result = None volume_id = fid.split(',')[0] full_url = '' try: r = self.master.lookup(volume_id) # _url = WEEDFS_MASTER_URL + '/dir/lookup?volumeId=%s' % volume_id # LOGGER.debug('lookup volume by url: %s' % _url) # r = requests.get(_url) result = json.loads(r.content) locations = result['locations'] # choose a random location location = locations[random.randint(0, len(locations) - 1)] full_url = 'http://%s/%s' % (location['url'], fid) except Exception as e: LOGGER.error( 'Could not get volume location of this fid: %s. Exception is: %s' % (fid, e)) result = None return full_url
def put_file(self, absolute_file_path, fid, headers=None): ''' you can put exact http-headers in @headers to help weed to clarify putting file. eg: @headers = {'content-type' : 'image/png'} or @headers = {'content-type' : 'image/jpeg'} or @headers = {'content-type' : 'text/xml'} or @headers = {'content-type' : 'application/json'} Deprecated. Use util.put_file instead. ''' url = urljoin(self.url_base, fid) if headers and isinstance(headers, dict): files = { 'file': (open(absolute_file_path, 'rb')), 'headers': headers } else: files = {'file': (open(absolute_file_path, 'rb'))} try: r = requests.post(url, files=files) except Exception as e: LOGGER.error("Could not post file. Exception is: %s" % e) return None # weed-fs returns a 200 but the content may contain an error result = json.loads(r.content) if r.status_code == 200: if 'error' in result: LOGGER.error(result['error']) else: LOGGER.debug(result) return result
def read(self, fid, fname='', just_url=True): """ read/get a file from weed-fs with @fid. @just_url: True -> just return fid_full_url (web-servers/browsers like nginx, chrome can get resource by this url) False -> return a http response of requests(package requests). """ fid_full_url = 'wrong_url' try: fid_full_url = self.get_fid_full_url(fid) LOGGER.debug( 'Reading file(just_url:%s): fid: %s, fname: %s, fid_full_url: %s' % (just_url, fid, fname, fid_full_url)) if just_url: return fid_full_url else: rsp = requests.get(fid_full_url) return rsp except Exception as e: err_msg = 'Could not read file(just_url:%s): fid: %s, fname: %s, fid_full_url: %s, e: %s' % ( just_url, fid, fname, fid_full_url, e) LOGGER.error(err_msg) return None
def vacuum(self): """ Force Garbage Collection If your system has many deletions, the deleted file's disk space will not be synchronously re-claimed. There is a background job to check volume disk usage. If empty space is more than the threshold, default to 0.3, the vacuum job will make the volume readonly, create a new volume with only existing files, and switch on the new volume. If you are impatient or doing some testing, vacuum the unused spaces this way. > curl "http://localhost:9333/vol/vacuum" > curl "http://localhost:9333/vol/vacuum?garbageThreshold=0.4" return a json of refreshed status like below: ----------- { "Topology": { "DataCenters": [ { "Free": 93, "Max": 100, "Racks": [ { "DataNodes": [ { "Free": 93, "Max": 100, "PublicUrl": "127.0.0.1:8080", "Url": "127.0.0.1:8080", "Volumes": 7 } ], "Free": 93, "Max": 100 } ] } ], "Free": 93, "Max": 100, "layouts": [ { "replication": "000", "writables": [ 2, 3, 5, 6, 7, 1, 4 ] } ] }, "Version": "0.37" } { "locations": [ { "publicUrl": "localhost:8080", "url": "localhost:8080" } ] } ----------- Arguments: - `self`: """ result = None try: r = requests.get(self.url_vacuum) result = json.loads(r.content) except Exception as e: LOGGER.error( 'Could not get status of this volume: %s. Exception is: %s' % (self.url_status, e)) result = None return result
def vacuum(self): """ Force Garbage Collection If your system has many deletions, the deleted file's disk space will not be synchronously re-claimed. There is a background job to check volume disk usage. If empty space is more than the threshold, default to 0.3, the vacuum job will make the volume readonly, create a new volume with only existing files, and switch on the new volume. If you are impatient or doing some testing, vacuum the unused spaces this way. > curl "http://localhost:9333/vol/vacuum" > curl "http://localhost:9333/vol/vacuum?garbageThreshold=0.4" return a json of refreshed status like below: ----------- { "Topology": { "DataCenters": [ { "Free": 93, "Max": 100, "Racks": [ { "DataNodes": [ { "Free": 93, "Max": 100, "PublicUrl": "127.0.0.1:8080", "Url": "127.0.0.1:8080", "Volumes": 7 } ], "Free": 93, "Max": 100 } ] } ], "Free": 93, "Max": 100, "layouts": [ { "replication": "000", "writables": [ 2, 3, 5, 6, 7, 1, 4 ] } ] }, "Version": "0.37" } { "locations": [ { "publicUrl": "localhost:8080", "url": "localhost:8080" } ] } ----------- Arguments: - `self`: """ result = None try: r = requests.get(self.url_vacuum) result = json.loads(r.content) except Exception as e: LOGGER.error('Could not get status of this volume: %s. Exception is: %s' % (self.url_status, e)) result = None return result