def getfileinfo(self, path):
logger.debug("getfileinfo: %s" % path)
resp = self._retry(self._getfileinfo, path)
if resp.status_code == 200:
j = resp.json()
key = "fragment-info"
if key in j:
fragementsinfo = j[key]
max_range, fragementsinfo = self._check_fragments(
fragementsinfo)
if not max_range or not fragementsinfo:
raise exceptions.UnspecifiedError("fragment-info Error!")
return max_range, fragementsinfo
else:
raise exceptions.UnspecifiedError(
"fileinfo not contain fragment-info!")
elif resp.status_code == 404:
raise exceptions.FileNotFoundError("File Not Found!")
else:
raise exceptions.UnspecifiedError(
"getfileinfo UnKnow status code:%d" % resp.status_code)
def s_write(self, key, value, tags):
# Write data with given key
r = self._session.write_data(key, str(value))
r.wait()
err = r.error()
if err.code != 0:
raise exceptions.UnspecifiedError("Writing failed %s" % err)
# Set indexes
r = self._session.update_indexes(key, list(tags), [key] * len(tags))
r.wait()
err = r.error()
if err.code != 0:
raise exceptions.UnspecifiedError("Indexe setting failed %s" % err)
def _downloader(self, fragments):
logger.debug("begin download multi parts.")
for fragment in fragments:
fragment_index = fragment['Index']
fragment_begin = fragment['Start']
fragment_end = fragment['End']
try:
logger.debug("begin download part fragment_index: %d" %
fragment_index)
resp = self.conn.download(self.path,
fragment_index,
(fragment_begin, fragment_end),
stream=False)
if resp.status_code == 200:
content = resp.content
self._fragment_tempfiles[fragment_index] = _TempFile(
mode='w+b', prefix=self.tmpdir)
f = self._fragment_tempfiles[fragment_index].file
f.write(content)
# seek to 0, ready to be read
f.seek(0)
self._refresh_max_completed_byte(fragment_index,
fragment_end)
logger.debug(
"download part success!!! fragment_index: %d" %
fragment_index)
elif resp.status_code == 404:
logger.debug(
"download part, file not found, path: %s, frament_index: %d"
% (self.path, fragment_index))
raise exceptions.FileNotFoundError(
"fetch_part FileNotFound!")
else:
logger.debug("mark else code: %d" % resp.status_code)
raise exceptions.UnspecifiedError(
"unexcept status code: %d" % resp.status_code)
except:
logger.error("download part failed, path: %s, index: %d" %
(self.path, fragment_index))
self._set_error()
return
# docker read timeout
now = time.time()
if now - self._get_last_read_time() > 300:
logger.debug(
"speedy long time no read, reader maybe exited!!!")
self._set_error()
self.clear()
return
logger.debug("end download multi parts.")
def s_append(self, key, content):
session = self._session
session.ioflags = elliptics.io_flags.append
# set offset to resolve function overloading
r = session.write_data(key, content, offset=0)
r.wait()
err = r.error()
if err.code != 0:
raise exceptions.UnspecifiedError("Writing failed {0}".format(err))
def _retry(self, func, *args, **kwargs):
retry_times = 0
while True:
try:
rv = func(*args, **kwargs)
return rv
except:
retry_times += 1
if retry_times > self.max_retry_times:
raise exceptions.UnspecifiedError("retry %s failed" %
func.__name__)
def list_directory(self, path=None):
logger.debug("list directory path: %s" % path)
resp = self.speedy_conn.list_directory(path)
if resp.status_code == 200:
j = resp.json()
return j["file-list"]
elif resp.status_code == 404:
raise exceptions.FileNotFoundError("no such directory: %s" % path)
else:
raise exceptions.UnspecifiedError("unexcept status code: %d" %
path)
def exists(self, path):
logger.debug("exists, path: %s" % path)
resp = self._retry(self._exists, path)
if resp.status_code == 200:
return True
elif resp.status_code == 404:
return False
else:
logger.error("Unkown stauts code: %d" % resp.status_code)
raise exceptions.UnspecifiedError("unexcept status code: %d" %
resp.status_code)
def put_content(self, path, content):
logger.debug("put content path: %s" % path)
bytes_range = (0, len(content))
resp = self.speedy_conn.upload(path,
content,
0,
bytes_range,
is_last=True)
if resp.status_code != 200:
raise exceptions.UnspecifiedError(
"put speedy content failed: %s, status_code: %d" %
(path, resp.status_code))
def remove(self, path):
logger.debug("remove path: %s" % path)
resp = self.speedy_conn.delete(path)
if resp.status_code == 204:
logger.debug("speedy remove success: %s" % path)
elif resp.status_code == 404:
logger.warning("speedy remove, file not found: %s" % path)
raise exceptions.FileNotFoundError("%s is not here" % path)
else:
logger.error("speedy remove, unexcept status code %d" %
resp.status_code)
raise exceptions.UnspecifiedError(
"speedy remove, unexcept status code %d" % resp.status_code)
def get_content(self, path):
logger.debug("get speedy content path: %s" % path)
_, fragments = self.speedy_conn.getfileinfo(path)
if len(fragments) != 1:
logger.error('little file fragment error! %s' % path)
raise exceptions.UnspecifiedError(
"little file invalid fragment info!")
fragment = fragments[0]
resp = self.speedy_conn.download(path, 0,
(fragment['Start'], fragment['End']))
if resp.status_code == 200:
content = resp.content
logger.debug("get content len: %d" % len(content))
return content
elif resp.status_code == 404:
raise exceptions.FileNotFoundError('%s is not here' % path)
else:
logger.error("get content unexcept status code: %d" %
resp.status_code)
raise exceptions.UnspecifiedError(
"get content unexcept status code: %d" % resp.status_code)
def _check_fragments(self, fragments):
if not isinstance(fragments, list):
return None, None
fragcount = 0
max_range = 0
sorted_fragments = sorted(fragments, key=operator.itemgetter('Index'))
for fragment in sorted_fragments:
if fragment['Index'] != fragcount or fragment['Start'] != max_range:
return None, None
max_range = fragment['End']
fragcount += 1
last_fragment = sorted_fragments[-1]
if not last_fragment['IsLast']:
raise exceptions.UnspecifiedError(
"uncompleted file fragment infos")
return max_range, sorted_fragments
def _gen_http_conn(self):
"""
@choose url and generate connection
"""
global HealthUrls
global HeartBeatMutex
url = ""
HeartBeatMutex.acquire()
size = len(HealthUrls)
if size > 0:
index = random.randint(0, size - 1)
url = HealthUrls[index]
HeartBeatMutex.release()
if len(url) == 0:
logger.error("no healthy image server")
raise exceptions.UnspecifiedError("No Healthy ImageServer!!!")
return HTTPConnection(url)
def _check_error(self):
if self.failed_parts > 0:
self.conn.delete(self.path)
raise exceptions.UnspecifiedError("speedy upload failed")