def test_upload_queue_persistence(self): item1 = athenad.UploadItem(path="_", url="_", headers={}, created_at=int(time.time()), id='id1') item2 = athenad.UploadItem(path="_", url="_", headers={}, created_at=int(time.time()), id='id2') athenad.upload_queue.put_nowait(item1) athenad.upload_queue.put_nowait(item2) # Ensure cancelled items are not persisted athenad.cancelled_uploads.add(item2.id) # serialize item athenad.UploadQueueCache.cache(athenad.upload_queue) # deserialize item athenad.upload_queue.queue.clear() athenad.UploadQueueCache.initialize(athenad.upload_queue) self.assertEqual(athenad.upload_queue.qsize(), 1) self.assertDictEqual(athenad.upload_queue.queue[-1]._asdict(), item1._asdict())
def test_do_upload(self, host): fn = os.path.join(athenad.ROOT, 'qlog.bz2') Path(fn).touch() item = athenad.UploadItem(path=fn, url="http://localhost:1238", headers={}, created_at=int(time.time()*1000), id='') with self.assertRaises(requests.exceptions.ConnectionError): athenad._do_upload(item) item = athenad.UploadItem(path=fn, url=f"{host}/qlog.bz2", headers={}, created_at=int(time.time()*1000), id='') resp = athenad._do_upload(item) self.assertEqual(resp.status_code, 201)
def test_cancelUpload(self): item = athenad.UploadItem(path="qlog.bz2", url="http://localhost:44444/qlog.bz2", headers={}, created_at=int(time.time() * 1000), id='id') athenad.upload_queue.put_nowait(item) dispatcher["cancelUpload"](item.id) self.assertIn(item.id, athenad.cancelled_uploads) end_event = threading.Event() thread = threading.Thread(target=athenad.upload_handler, args=(end_event, )) thread.start() try: now = time.time() while time.time() - now < 5: if athenad.upload_queue.qsize() == 0 and len( athenad.cancelled_uploads) == 0: break self.assertEqual(athenad.upload_queue.qsize(), 0) self.assertEqual(len(athenad.cancelled_uploads), 0) finally: end_event.set() athenad.upload_queue = queue.Queue()
def test_upload_handler(self, host): fn = os.path.join(athenad.ROOT, 'qlog.bz2') Path(fn).touch() item = athenad.UploadItem(path=fn, url=f"{host}/qlog.bz2", headers={}, created_at=int(time.time() * 1000), id='') end_event = threading.Event() thread = threading.Thread(target=athenad.upload_handler, args=(end_event, )) thread.start() athenad.upload_queue.put_nowait(item) try: now = time.time() while time.time() - now < 5: if athenad.upload_queue.qsize() == 0: break self.assertEqual(athenad.upload_queue.qsize(), 0) finally: end_event.set() athenad.upload_queue = queue.Queue() os.unlink(fn)
def test_upload_handler_timeout(self): """When an upload times out or fails to connect it should be placed back in the queue""" fn = os.path.join(athenad.ROOT, 'qlog.bz2') Path(fn).touch() item = athenad.UploadItem(path=fn, url="http://localhost:44444/qlog.bz2", headers={}, created_at=int(time.time() * 1000), id='') item_no_retry = item._replace(retry_count=MAX_RETRY_COUNT) end_event = threading.Event() thread = threading.Thread(target=athenad.upload_handler, args=(end_event, )) thread.start() try: athenad.upload_queue.put_nowait(item_no_retry) self.wait_for_upload() time.sleep(0.1) # Check that upload with retry count exceeded is not put back self.assertEqual(athenad.upload_queue.qsize(), 0) athenad.upload_queue.put_nowait(item) self.wait_for_upload() time.sleep(0.1) # Check that upload item was put back in the queue with incremented retry count self.assertEqual(athenad.upload_queue.qsize(), 1) self.assertEqual(athenad.upload_queue.get().retry_count, 1) finally: end_event.set()
def test_listUploadQueueCurrent(self, host): fn = os.path.join(athenad.ROOT, 'qlog.bz2') Path(fn).touch() item = athenad.UploadItem(path=fn, url=f"{host}/qlog.bz2", headers={}, created_at=int(time.time() * 1000), id='', allow_cellular=True) end_event = threading.Event() thread = threading.Thread(target=athenad.upload_handler, args=(end_event, )) thread.start() try: athenad.upload_queue.put_nowait(item) self.wait_for_upload() items = dispatcher["listUploadQueue"]() self.assertEqual(len(items), 1) self.assertTrue(items[0]['current']) finally: end_event.set()
def test_cancelExpiry(self): t_future = datetime.now() - timedelta(days=40) ts = int(t_future.strftime("%s")) * 1000 # Item that would time out if actually uploaded fn = os.path.join(athenad.ROOT, 'qlog.bz2') Path(fn).touch() item = athenad.UploadItem(path=fn, url="http://localhost:44444/qlog.bz2", headers={}, created_at=ts, id='', allow_cellular=True) end_event = threading.Event() thread = threading.Thread(target=athenad.upload_handler, args=(end_event, )) thread.start() try: athenad.upload_queue.put_nowait(item) self.wait_for_upload() time.sleep(0.1) self.assertEqual(athenad.upload_queue.qsize(), 0) finally: end_event.set()
def test_upload_handler_retry(self, host, mock_put): for status, retry in ((500, True), (412, False)): mock_put.return_value.status_code = status fn = os.path.join(athenad.ROOT, 'qlog.bz2') Path(fn).touch() item = athenad.UploadItem(path=fn, url=f"{host}/qlog.bz2", headers={}, created_at=int(time.time() * 1000), id='', allow_cellular=True) end_event = threading.Event() thread = threading.Thread(target=athenad.upload_handler, args=(end_event, )) thread.start() athenad.upload_queue.put_nowait(item) try: self.wait_for_upload() time.sleep(0.1) self.assertEqual(athenad.upload_queue.qsize(), 1 if retry else 0) finally: end_event.set() if retry: self.assertEqual(athenad.upload_queue.get().retry_count, 1)
def test_listUploadQueue(self): item = athenad.UploadItem(path="qlog.bz2", url="http://localhost:44444/qlog.bz2", headers={}, created_at=int(time.time()*1000), id='id') athenad.upload_queue.put_nowait(item) items = dispatcher["listUploadQueue"]() self.assertEqual(len(items), 1) self.assertDictEqual(items[0], item._asdict()) self.assertFalse(items[0]['current'])
def test_do_upload(self): fn = os.path.join(athenad.ROOT, 'qlog.bz2') Path(fn).touch() try: item = athenad.UploadItem(path=fn, url="http://localhost:1238", headers={}, created_at=int(time.time() * 1000), id='') try: athenad._do_upload(item) except requests.exceptions.ConnectionError: pass item = athenad.UploadItem(path=fn, url="http://localhost:44444/qlog.bz2", headers={}, created_at=int(time.time() * 1000), id='') resp = athenad._do_upload(item) self.assertEqual(resp.status_code, 201) finally: os.unlink(fn)
def test_cancelUpload(self): item = athenad.UploadItem(path="qlog.bz2", url="http://localhost:44444/qlog.bz2", headers={}, created_at=int(time.time()*1000), id='id', allow_cellular=True) athenad.upload_queue.put_nowait(item) dispatcher["cancelUpload"](item.id) self.assertIn(item.id, athenad.cancelled_uploads) end_event = threading.Event() thread = threading.Thread(target=athenad.upload_handler, args=(end_event,)) thread.start() try: self.wait_for_upload() time.sleep(0.1) self.assertEqual(athenad.upload_queue.qsize(), 0) self.assertEqual(len(athenad.cancelled_uploads), 0) finally: end_event.set()
def test_upload_handler(self, host): fn = os.path.join(athenad.ROOT, 'qlog.bz2') Path(fn).touch() item = athenad.UploadItem(path=fn, url=f"{host}/qlog.bz2", headers={}, created_at=int(time.time()*1000), id='', allow_cellular=True) end_event = threading.Event() thread = threading.Thread(target=athenad.upload_handler, args=(end_event,)) thread.start() athenad.upload_queue.put_nowait(item) try: self.wait_for_upload() time.sleep(0.1) # TODO: verify that upload actually succeeded self.assertEqual(athenad.upload_queue.qsize(), 0) finally: end_event.set()