def queue_handling_loop(self):
any_download = False
while True:
if not is_alive(self.__host):
time.sleep(WAIT_2)
continue
any_download = False
if len(self.download_queue) > 0:
self.rwl.acquire_read()
logging.getLogger().debug(
"QUEUE Lenght: " + str(len(self.download_queue)))
elem = self.download_queue[0]
img_name = get_image_name(elem)
tick = timeit.default_timer()
self.api.download_file(elem["Path"],
os.path.join(
self.__destination,
img_name), skip_existing=True)
tock = timeit.default_timer()
logging.getLogger().info(
"Image downloaded " + str(tock - tick))
self.rwl.promote()
self.download_queue.remove(elem)
self.cache[uniq_identifier(elem)] = elem
self.rwl.demote()
self.rwl.release()
any_download = True
if any_download:
time.sleep(WAIT_3)
else:
time.sleep(WAIT_2)
def __append_download_queue(self, contents):
"""
SINGLE WRITER of File QUEUE
:param contents: list of files
:return:
"""
logging.getLogger().info("add_contents_to_download_queue ...")
logging.getLogger().debug("Acquiring rwl Read")
self.rwl.acquire_read()
for content in contents:
# chech cache first
uid = uniq_identifier(content)
if uid in self.cache:
# skip this.
continue
if content not in self.download_queue:
logging.getLogger().debug("Promoting rwl.")
self.rwl.promote()
print "Appending ", content
self.download_queue.append(content)
logging.getLogger().debug("Demoting rwl.")
self.rwl.demote()
logging.getLogger().debug("Releasing rwl Read")
self.rwl.release()