def test_no_max_size(self): q = NoBlockQueue() q.put(1) q.put(2) q.put(3) q.put(4) self.assertEqual(q.get(), 1) self.assertEqual(q.get(), 2) self.assertEqual(q.get(), 3) self.assertEqual(q.get(), 4)
def multi_download(self): """ This performs the multipart download. It assigns ranges to get from s3 of particular object to a task.It creates a queue ``part_queue`` which is directly responsible with controlling the progress of the multipart download. It then creates ``DownloadPartTasks`` for threads to run via the ``executer``. This fucntion waits for all of the parts in the multipart download to finish, and then the last modification time is changed to the last modified time of the s3 object. This method waits on its parts to finish. So, threads are required to process the parts for this function to complete. """ part_queue = NoBlockQueue(self.interrupt) dest_queue = NoBlockQueue(self.interrupt) part_counter = MultiCounter() write_lock = threading.Lock() counter_lock = threading.Lock() d = os.path.dirname(self.dest) try: if not os.path.exists(d): os.makedirs(d) except Exception: pass size_uploads = self.chunksize num_uploads = int(self.size / size_uploads) with open(self.dest, 'wb') as f: for i in range(num_uploads): part = (self, i, size_uploads) part_queue.put(part) task = DownloadPartTask(session=self.session, executer=self.executer, part_queue=part_queue, dest_queue=dest_queue, f=f, region=self.region, printQueue=self.printQueue, write_lock=write_lock, part_counter=part_counter, counter_lock=counter_lock) self.executer.submit(task) part_queue.join() # The following ensures that if the multipart download is # in progress, all part uploads finish before releasing the # the file handle. This really only applies when an interrupt # signal is sent because the ``part_queue.join()`` ensures this # if the process is not interrupted. while part_counter.count: time.sleep(0.1) part_list = [] while not dest_queue.empty(): part = dest_queue.get() part_list.append(part) if len(part_list) != num_uploads: raise Exception() last_update_tuple = self.last_update.timetuple() mod_timestamp = time.mktime(last_update_tuple) os.utime(self.dest, (int(mod_timestamp), int(mod_timestamp)))
def multi_download(self): """ This performs the multipart download. It assigns ranges to get from s3 of particular object to a task.It creates a queue ``part_queue`` which is directly responsible with controlling the progress of the multipart download. It then creates ``DownloadPartTasks`` for threads to run via the ``executer``. This fucntion waits for all of the parts in the multipart download to finish, and then the last modification time is changed to the last modified time of the s3 object. This method waits on its parts to finish. So, threads are required to process the parts for this function to complete. """ part_queue = NoBlockQueue(self.interrupt) dest_queue = NoBlockQueue(self.interrupt) part_counter = MultiCounter() write_lock = threading.Lock() counter_lock = threading.Lock() d = os.path.dirname(self.dest) try: if not os.path.exists(d): os.makedirs(d) except Exception: pass size_uploads = self.chunksize num_uploads = int(self.size/size_uploads) with open(self.dest, 'wb') as f: for i in range(num_uploads): part = (self, i, size_uploads) part_queue.put(part) task = DownloadPartTask(session=self.session, executer=self.executer, part_queue=part_queue, dest_queue=dest_queue, f=f, region=self.region, printQueue=self.printQueue, write_lock=write_lock, part_counter=part_counter, counter_lock=counter_lock) self.executer.submit(task) part_queue.join() # The following ensures that if the multipart download is # in progress, all part uploads finish before releasing the # the file handle. This really only applies when an interrupt # signal is sent because the ``part_queue.join()`` ensures this # if the process is not interrupted. while part_counter.count: time.sleep(0.1) part_list = [] while not dest_queue.empty(): part = dest_queue.get() part_list.append(part) if len(part_list) != num_uploads: raise Exception() last_update_tuple = self.last_update.timetuple() mod_timestamp = time.mktime(last_update_tuple) os.utime(self.dest, (int(mod_timestamp), int(mod_timestamp)))