Example #1
0
    def queue_handling_loop(self):
        any_download = False
        while True:
            if not is_alive(self.__host):
                time.sleep(WAIT_2)
                continue

            any_download = False
            if len(self.download_queue) > 0:
                self.rwl.acquire_read()
                logging.getLogger().debug(
                    "QUEUE Lenght: " + str(len(self.download_queue)))
                elem = self.download_queue[0]
                img_name = get_image_name(elem)
                tick = timeit.default_timer()
                self.api.download_file(elem["Path"],
                                       os.path.join(
                                           self.__destination,
                                           img_name), skip_existing=True)
                tock = timeit.default_timer()
                logging.getLogger().info(
                    "Image downloaded " + str(tock - tick))
                self.rwl.promote()
                self.download_queue.remove(elem)
                self.cache[uniq_identifier(elem)] = elem
                self.rwl.demote()
                self.rwl.release()
                any_download = True

            if any_download:
                time.sleep(WAIT_3)
            else:
                time.sleep(WAIT_2)
Example #2
0
 def __append_download_queue(self, contents):
     """
     SINGLE WRITER of File QUEUE
     :param contents: list of files
     :return:
     """
     logging.getLogger().info("add_contents_to_download_queue ...")
     logging.getLogger().debug("Acquiring rwl Read")
     self.rwl.acquire_read()
     for content in contents:
         # chech cache first
         uid = uniq_identifier(content)
         if uid in self.cache:
             # skip this.
             continue
         if content not in self.download_queue:
             logging.getLogger().debug("Promoting rwl.")
             self.rwl.promote()
             print "Appending ", content
             self.download_queue.append(content)
             logging.getLogger().debug("Demoting rwl.")
             self.rwl.demote()
     logging.getLogger().debug("Releasing rwl Read")
     self.rwl.release()