def put_file(self, absolute_file_path, fid, headers=None):
''' you can put exact http-headers in @headers to help weed to clarify putting file.
eg:
@headers = {'content-type' : 'image/png'} or
@headers = {'content-type' : 'image/jpeg'} or
@headers = {'content-type' : 'text/xml'} or
@headers = {'content-type' : 'application/json'}
Deprecated.
Use util.put_file instead.
'''
url = urljoin(self.url_base, fid)
if headers and isinstance(headers, dict):
files = {'file': (open(absolute_file_path, 'rb')), 'headers' : headers}
else:
files = {'file': (open(absolute_file_path, 'rb'))}
try:
r = requests.post(url, files=files)
except Exception as e:
LOGGER.error("Could not post file. Exception is: %s" % e)
return None
# weed-fs returns a 200 but the content may contain an error
result = json.loads(r.content)
if r.status_code == 200:
if 'error' in result:
LOGGER.error(result['error'])
else:
LOGGER.debug(result)
return result
def acquire_new_assign_key(self, count=1):
"""
get a new avalable new volume-file-location from from weed-master by getting a new-assign-key
Arguments:
- `self`:
assign_key is in json format like below:
-----------
{"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"}
-----------
return a tuple(dst_file_fid, dst_volume_url) like below:
----------
(dst_file_fid, http://{volume-url})
(3,20392030920, http://127.0.0.1:8080)
----------
"""
assign_key_url = self.url_assign + '?count=' + str(count)
dst_volume_url = None
wak = WeedAssignKeyExtended()
try:
LOGGER.debug('Getting new dst_volume_url with master-assign-key-url: %s' % assign_key_url)
r = requests.get(assign_key_url)
key_dict = json.loads(r.content)
wak.update(key_dict)
wak.update_full_urls()
LOGGER.info('Successfuly got dst_volume_url: %s' % dst_volume_url)
except Exception as e:
LOGGER.error('Could not get new assign key from the assign url: %s. Exception is: %s'
% (assign_key_url, e))
return wak
def update_by_fid(self, dst_fid, src_fid, src_fname=''):
"""
replace file@dst_fid with file@src_fid
"""
try:
src_file_rsp = self.read(src_fid, fname=src_fname, just_url=False)
fp = StringIO.StringIO(src_file_rsp.content)
LOGGER.debug('Updating file: dst_fid: %s, src_fid: %s, src_fname: %s, fp: %s' % (dst_fid, src_fid, src_fname, fp))
return self.update(fp, dst_fid, src_fname)
except Exception as e:
err_msg = 'Could not Updating file: dst_fid: %s, src_fid: %s, src_fname: %s. e: %s' % (dst_fid, src_fid, src_fname, e)
LOGGER.error(err_msg)
return None
def update(self, fp, fid, fname=''):
"""
update a file in weed-fs with @fid
"""
fid_full_url = 'wrong_url'
try:
fid_full_url = self.get_fid_full_url(fid)
LOGGER.debug('Updating file: fp: %s, fname: %s, fid_full_url: %s' % (fp, fname, fid_full_url))
return WeedOperation.save_file_to_weed(fp, fid_full_url, fname)
except Exception as e:
err_msg = 'Could not Updating file: fp: %s, fname: %s, fid_full_url: %s, e: %s' % (fp, fname, fid_full_url, e)
LOGGER.error(err_msg)
return None
def create(self, fp, fname=''):
"""
create a file in weed-fs with @fid
"""
fid_full_url = 'wrong_url'
try:
wak = self.master.acquire_new_assign_key()
LOGGER.debug('Creating file: fp: %s, fname: %s, fid_full_url: %s'
% (fp, fname, fid_full_url))
return WeedOperation.save_file_to_weed(fp, wak.fid_full_url, fname)
except Exception as e:
err_msg = 'Could not create file: fp: %s, fname: %s, fid_full_url: %s, e: %s' % (fp, fname, fid_full_url, e)
LOGGER.error(err_msg)
return None
def update(self, fp, fid, fname=''):
"""
update a file in weed-fs with @fid
"""
fid_full_url = 'wrong_url'
try:
fid_full_url = self.get_fid_full_url(fid)
LOGGER.debug('Updating file: fp: %s, fname: %s, fid_full_url: %s' %
(fp, fname, fid_full_url))
return WeedOperation.save_file_to_weed(fp, fid_full_url, fname)
except Exception as e:
err_msg = 'Could not Updating file: fp: %s, fname: %s, fid_full_url: %s, e: %s' % (
fp, fname, fid_full_url, e)
LOGGER.error(err_msg)
return None
def create(self, fp, fname=''):
"""
create a file in weed-fs with @fid
"""
fid_full_url = 'wrong_url'
try:
wak = self.master.acquire_new_assign_key()
LOGGER.debug('Creating file: fp: %s, fname: %s, fid_full_url: %s' %
(fp, fname, fid_full_url))
return WeedOperation.save_file_to_weed(fp, wak.fid_full_url, fname)
except Exception as e:
err_msg = 'Could not create file: fp: %s, fname: %s, fid_full_url: %s, e: %s' % (
fp, fname, fid_full_url, e)
LOGGER.error(err_msg)
return None
def update_by_fid(self, dst_fid, src_fid, src_fname=''):
"""
replace file@dst_fid with file@src_fid
"""
try:
src_file_rsp = self.read(src_fid, fname=src_fname, just_url=False)
fp = StringIO.StringIO(src_file_rsp.content)
LOGGER.debug(
'Updating file: dst_fid: %s, src_fid: %s, src_fname: %s, fp: %s'
% (dst_fid, src_fid, src_fname, fp))
return self.update(fp, dst_fid, src_fname)
except Exception as e:
err_msg = 'Could not Updating file: dst_fid: %s, src_fid: %s, src_fname: %s. e: %s' % (
dst_fid, src_fid, src_fname, e)
LOGGER.error(err_msg)
return None
def delete(self, fid, fname=''):
"""
delete a file in weed-fs with @fid
"""
fid_full_url = 'wrong_url'
try:
fid_full_url = self.get_fid_full_url(fid)
LOGGER.debug('Deleting file: fid: %s, fname: %s, fid_full_url: %s' % (fid, fname, fid_full_url))
r = requests.delete(fid_full_url)
if r.json().has_key('size'):
return True
except Exception as e:
err_msg = 'Deleting file: fid: %s, fname: %s, fid_full_url: %s, e: %s' % (fid, fname, fid_full_url, e)
LOGGER.error(err_msg)
return False
return False
def delete(self, fid, fname=''):
"""
delete a file in weed-fs with @fid
"""
fid_full_url = 'wrong_url'
try:
fid_full_url = self.get_fid_full_url(fid)
LOGGER.debug(
'Deleting file: fid: %s, fname: %s, fid_full_url: %s' %
(fid, fname, fid_full_url))
r = requests.delete(fid_full_url)
if r.json().has_key('size'):
return True
except Exception as e:
err_msg = 'Deleting file: fid: %s, fname: %s, fid_full_url: %s, e: %s' % (
fid, fname, fid_full_url, e)
LOGGER.error(err_msg)
return False
return False
def read(self, fid, fname='', just_url=True):
"""
read/get a file from weed-fs with @fid.
@just_url:
True -> just return fid_full_url (web-servers/browsers like nginx, chrome can get resource by this url)
False -> return a http response of requests(package requests).
"""
fid_full_url = 'wrong_url'
try:
fid_full_url = self.get_fid_full_url(fid)
LOGGER.debug('Reading file(just_url:%s): fid: %s, fname: %s, fid_full_url: %s' % (just_url, fid, fname, fid_full_url))
if just_url:
return fid_full_url
else:
rsp = requests.get(fid_full_url)
return rsp
except Exception as e:
err_msg = 'Could not read file(just_url:%s): fid: %s, fname: %s, fid_full_url: %s, e: %s' % (just_url, fid, fname, fid_full_url, e)
LOGGER.error(err_msg)
return None
def acquire_new_assign_key(self, count=1):
"""
get a new avalable new volume-file-location from from weed-master by getting a new-assign-key
Arguments:
- `self`:
assign_key is in json format like below:
-----------
{"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"}
-----------
return a tuple(dst_file_fid, dst_volume_url) like below:
----------
(dst_file_fid, http://{volume-url})
(3,20392030920, http://127.0.0.1:8080)
----------
"""
assign_key_url = self.url_assign + '?count=' + str(count)
dst_volume_url = None
wak = WeedAssignKeyExtended()
try:
LOGGER.debug(
'Getting new dst_volume_url with master-assign-key-url: %s' %
assign_key_url)
r = requests.get(assign_key_url)
key_dict = json.loads(r.content)
wak.update(key_dict)
wak.update_full_urls()
LOGGER.info('Successfuly got dst_volume_url: %s' % dst_volume_url)
except Exception as e:
LOGGER.error(
'Could not get new assign key from the assign url: %s. Exception is: %s'
% (assign_key_url, e))
return wak
def put_file(self, absolute_file_path, fid, headers=None):
''' you can put exact http-headers in @headers to help weed to clarify putting file.
eg:
@headers = {'content-type' : 'image/png'} or
@headers = {'content-type' : 'image/jpeg'} or
@headers = {'content-type' : 'text/xml'} or
@headers = {'content-type' : 'application/json'}
Deprecated.
Use util.put_file instead.
'''
url = urljoin(self.url_base, fid)
if headers and isinstance(headers, dict):
files = {
'file': (open(absolute_file_path, 'rb')),
'headers': headers
}
else:
files = {'file': (open(absolute_file_path, 'rb'))}
try:
r = requests.post(url, files=files)
except Exception as e:
LOGGER.error("Could not post file. Exception is: %s" % e)
return None
# weed-fs returns a 200 but the content may contain an error
result = json.loads(r.content)
if r.status_code == 200:
if 'error' in result:
LOGGER.error(result['error'])
else:
LOGGER.debug(result)
return result
def read(self, fid, fname='', just_url=True):
"""
read/get a file from weed-fs with @fid.
@just_url:
True -> just return fid_full_url (web-servers/browsers like nginx, chrome can get resource by this url)
False -> return a http response of requests(package requests).
"""
fid_full_url = 'wrong_url'
try:
fid_full_url = self.get_fid_full_url(fid)
LOGGER.debug(
'Reading file(just_url:%s): fid: %s, fname: %s, fid_full_url: %s'
% (just_url, fid, fname, fid_full_url))
if just_url:
return fid_full_url
else:
rsp = requests.get(fid_full_url)
return rsp
except Exception as e:
err_msg = 'Could not read file(just_url:%s): fid: %s, fname: %s, fid_full_url: %s, e: %s' % (
just_url, fid, fname, fid_full_url, e)
LOGGER.error(err_msg)
return None