def get_source(self, chunk, is_first):
if is_first:
return self.source
else:
return request.get(self.link_file,
cookie=self.cookie,
range=(chunk[START], None))
def get_source(self, chunk, is_first):
if is_first:
return self.source
else:
return request.get(self.link_file, cookie=self.cookie, range=(chunk[START], None))
def thread_download(self, fh, i, chunk_range):
"""
DONE: Thread Safe.
"""
buffer_list = []
len_buffer_data = 0
len_data = 0
complete = 0
def flush_buffer():
buf_data = "".join(buffer_list)
with self.lock1:
fh.seek(chunk_range[0] + complete - len_buffer_data)
fh.write(buf_data)
with self.lock2:
start = chunk_range[0] + complete
self.chunks[i] = (start, self.chunks[i][1])
del buffer_list[:]
while True:
try:
with URLClose(
request.get(self.link_file, cookie=self.cookie, range=(chunk_range[0] + complete, chunk_range[1]))
) as source:
# if (chunk_range[0] and chunk_range[1] is not None) and (self.size_file != self.get_content_size(source.info()) or source.info().getheader("Content-Range", None) is None):
info = source.info()
if self.size_file != self.get_content_size(info): # and content-len != self.size_file
raise EnvironmentError("Link expired, or cant download the requested range.")
# print "Link expired, or cant download the requested range."
# if not (chunk_range[0] == 0 and chunk_range[1] is None):
# if info.getheader("Content-Range", None) is None: #and content-len + chunk_range[0] != self.size_file
# raise EnvironmentError("Content-Range tag is missing.")
# print "Cant validate the requested range."
# Todo: if the above fails, check content-len + chunk_range[0] != self.size_file
# comprobar si la etiqueta content-len no existe al sacar el size_file, ya que quedaria en cero y luego si existe content-range sera distinto.
# tomar en cuenta cuando no exista content-len o content-range o ambos.
while True:
data = source.read(NT_BUFSIZ)
len_data = len(data)
buffer_list.append(data)
len_buffer_data += len_data
complete += len_data
if len_buffer_data >= DATA_BUFSIZ:
flush_buffer()
len_buffer_data = 0
with self.lock2:
self.size_complete += len_data
if self.bucket.fill_rate: # self.bucket.fill_rate != 0
time.sleep(self.bucket.consume(len_data))
if self.stop_flag or self.error_flag: # len(data) == 0
flush_buffer()
return
elif not len_data: # todo: check complete against range requested.
flush_buffer()
content_len = 0
if chunk_range[1] is not None:
content_len = chunk_range[1] - chunk_range[0] + 1
print content_len, complete # error
elif self.size_file and self.size_file > chunk_range[0]:
content_len = self.size_file - chunk_range[0]
print content_len, complete # error
if content_len and content_len > complete:
# print content_len, complete #error
raise EnvironmentError("Incomplete chunk")
return
# content_len = int(info.getheader("Content-Length", 0)) #some server send wrong sizes.
# if content_len and content_len != complete: #verification
# raise EnvironmentError("Incomplete chunk")
except (
urllib2.URLError,
httplib.HTTPException,
socket.error,
) as err: # httplib.HTTPException = badstatusline, etc
# TODO: capturar error "overflow" y dar status error para reiniciar resumiendo.
logger.warning("Conexion {0}: {1}".format(i, err))
time.sleep(3) # to avoid overflowing the server.
if self.stop_flag or self.error_flag:
flush_buffer()
return
# dont break, lets retry...
except EnvironmentError as err: # excepciones en Threads no pueden ser delegadas.
# FIXME: si la excepcion es por falta de espacio en disco, se tendria q eliminar la ultima pos del chunk
logger.exception(err)
self.error_flag = True
self.status_msg = "Error: {0}".format(err) # operaciones atomicas. No necesita lock.
return
def thread_download(self, fh, i, chunk_range):
"""
DONE: Thread Safe.
"""
buffer_list = []
len_buffer_data = 0
len_data = 0
complete = 0
def flush_buffer():
buf_data = ''.join(buffer_list)
with self.lock1:
fh.seek(chunk_range[0] + complete - len_buffer_data)
fh.write(buf_data)
with self.lock2:
start = chunk_range[0] + complete
self.chunks[i] = (start, self.chunks[i][1])
del buffer_list[:]
while True:
try:
with URLClose(
request.get(self.link_file,
cookie=self.cookie,
range=(chunk_range[0] + complete,
chunk_range[1]))) as source:
#if (chunk_range[0] and chunk_range[1] is not None) and (self.size_file != self.get_content_size(source.info()) or source.info().getheader("Content-Range", None) is None):
info = source.info()
if self.size_file != self.get_content_size(
info): #and content-len != self.size_file
raise EnvironmentError(
"Link expired, or cant download the requested range."
)
#print "Link expired, or cant download the requested range."
#if not (chunk_range[0] == 0 and chunk_range[1] is None):
#if info.getheader("Content-Range", None) is None: #and content-len + chunk_range[0] != self.size_file
#raise EnvironmentError("Content-Range tag is missing.")
#print "Cant validate the requested range."
#Todo: if the above fails, check content-len + chunk_range[0] != self.size_file
#comprobar si la etiqueta content-len no existe al sacar el size_file, ya que quedaria en cero y luego si existe content-range sera distinto.
#tomar en cuenta cuando no exista content-len o content-range o ambos.
while True:
data = source.read(NT_BUFSIZ)
len_data = len(data)
buffer_list.append(data)
len_buffer_data += len_data
complete += len_data
if len_buffer_data >= DATA_BUFSIZ:
flush_buffer()
len_buffer_data = 0
with self.lock2:
self.size_complete += len_data
if self.bucket.fill_rate: #self.bucket.fill_rate != 0
time.sleep(self.bucket.consume(len_data))
if self.stop_flag or self.error_flag: #len(data) == 0
flush_buffer()
return
elif not len_data: #todo: check complete against range requested.
flush_buffer()
content_len = 0
if chunk_range[1] is not None:
content_len = chunk_range[1] - chunk_range[
0] + 1
print content_len, complete #error
elif self.size_file and self.size_file > chunk_range[
0]:
content_len = self.size_file - chunk_range[0]
print content_len, complete #error
if content_len and content_len > complete:
#print content_len, complete #error
raise EnvironmentError("Incomplete chunk")
return
#content_len = int(info.getheader("Content-Length", 0)) #some server send wrong sizes.
#if content_len and content_len != complete: #verification
#raise EnvironmentError("Incomplete chunk")
except (urllib2.URLError, httplib.HTTPException, socket.error
) as err: #httplib.HTTPException = badstatusline, etc
#TODO: capturar error "overflow" y dar status error para reiniciar resumiendo.
logger.warning("Conexion {0}: {1}".format(i, err))
time.sleep(3) #to avoid overflowing the server.
if self.stop_flag or self.error_flag:
flush_buffer()
return
#dont break, lets retry...
except EnvironmentError as err: #excepciones en Threads no pueden ser delegadas.
#FIXME: si la excepcion es por falta de espacio en disco, se tendria q eliminar la ultima pos del chunk
logger.exception(err)
self.error_flag = True
self.status_msg = "Error: {0}".format(
err) #operaciones atomicas. No necesita lock.
return
