def setUp(self):
self.luigiconn = DropboxClient(DROPBOX_APP_TOKEN)
self.dropbox_api = dropbox.dropbox.Dropbox(DROPBOX_APP_TOKEN)
self.dropbox_api.files_upload(b'hello', DROPBOX_TEST_SIMPLE_FILE)
self.dropbox_api.files_upload(b'hello2', DROPBOX_TEST_FILE_IN_DIR)
self.dropbox_api.files_upload(b'hello3', DROPBOX_TEST_FILE_TO_MOVE_ORIG)
self.dropbox_api.files_upload(b'hello4', DROPBOX_TEST_FILE_TO_COPY_ORIG)
self.dropbox_api.files_upload(b'hello3.1', DROPBOX_TEST_FILE_TO_DELETE_1)
self.dropbox_api.files_upload(b'hello3.2', DROPBOX_TEST_FILE_TO_DELETE_2)
def setUp(self):
self.luigiconn = DropboxClient(DROPBOX_APP_TOKEN)
self.dropbox_api = dropbox.dropbox.Dropbox(DROPBOX_APP_TOKEN)
self.initial_contents = b'\x00hello\xff\x00-\xe2\x82\x28' # Binary invalid-utf8 sequence
self.dropbox_api.files_upload(self.initial_contents,
DROPBOX_TEST_SIMPLE_FILE)
import os
import logging
import gspread
from luigi.contrib.dropbox import DropboxClient
DROPBOX_TOKEN = os.getenv("SWB_DROPBOX_TOKEN")
dropbox_client = DropboxClient(DROPBOX_TOKEN)
GSPREAD_CLIENT = None
DEFAULT_WORKSHEET_URL = "https://docs.google.com/spreadsheets/d/1HeTZKEXtSYFDNKmVEcRmF573k2ZraDb6DzgCOSXI0f0/edit#gid=0"
WORKSHEET_URL = os.getenv("SWB_WORKSHEET_URL", DEFAULT_WORKSHEET_URL)
try:
GSPREAD_CLIENT = gspread.service_account(
filename=os.getenv(
"GOOGLE_APPLICATION_CREDENTIALS", "~/.config/gspread/service_account.json"
)
)
except FileNotFoundError as e:
logging.error(f"Unable to create gspread client #{e}")
class TestClientDropbox(unittest.TestCase):
def setUp(self):
self.luigiconn = DropboxClient(DROPBOX_APP_TOKEN)
self.dropbox_api = dropbox.dropbox.Dropbox(DROPBOX_APP_TOKEN)
self.dropbox_api.files_upload(b'hello', DROPBOX_TEST_SIMPLE_FILE)
self.dropbox_api.files_upload(b'hello2', DROPBOX_TEST_FILE_IN_DIR)
self.dropbox_api.files_upload(b'hello3', DROPBOX_TEST_FILE_TO_MOVE_ORIG)
self.dropbox_api.files_upload(b'hello4', DROPBOX_TEST_FILE_TO_COPY_ORIG)
self.dropbox_api.files_upload(b'hello3.1', DROPBOX_TEST_FILE_TO_DELETE_1)
self.dropbox_api.files_upload(b'hello3.2', DROPBOX_TEST_FILE_TO_DELETE_2)
def tearDown(self):
self.dropbox_api.files_delete_v2(DROPBOX_TEST_PATH)
self.dropbox_api._session.close()
def test_exists(self):
self.assertTrue(self.luigiconn.exists('/'))
self.assertTrue(self.luigiconn.exists(DROPBOX_TEST_PATH))
self.assertTrue(self.luigiconn.exists(DROPBOX_TEST_SIMPLE_DIR))
self.assertTrue(self.luigiconn.exists(DROPBOX_TEST_SIMPLE_DIR + '/'))
self.assertTrue(self.luigiconn.exists(DROPBOX_TEST_SIMPLE_FILE))
self.assertFalse(self.luigiconn.exists(DROPBOX_TEST_SIMPLE_FILE + '/'))
self.assertFalse(self.luigiconn.exists(DROPBOX_TEST_NON_EXISTING_FILE))
def test_listdir_simple(self):
list_of_dirs = self.luigiconn.listdir(DROPBOX_TEST_PATH)
self.assertTrue('/' not in list_of_dirs)
self.assertTrue(DROPBOX_TEST_PATH in list_of_dirs)
self.assertTrue(DROPBOX_TEST_SIMPLE_FILE in list_of_dirs) # we verify recursivity
def test_listdir_simple_with_one_slash(self):
list_of_dirs = self.luigiconn.listdir(DROPBOX_TEST_PATH + '/')
self.assertTrue('/' not in list_of_dirs)
self.assertTrue(DROPBOX_TEST_PATH in list_of_dirs)
self.assertTrue(DROPBOX_TEST_SIMPLE_FILE in list_of_dirs) # we verify recursivity
def test_listdir_multiple(self):
list_of_dirs = self.luigiconn.listdir(DROPBOX_TEST_PATH, limit=2)
self.assertTrue('/' not in list_of_dirs)
self.assertTrue(DROPBOX_TEST_PATH in list_of_dirs)
self.assertTrue(DROPBOX_TEST_SIMPLE_FILE in list_of_dirs) # we verify recursivity
def test_listdir_nonexisting(self):
with self.assertRaises(dropbox.exceptions.ApiError):
self.luigiconn.listdir(DROPBOX_TEST_NON_EXISTING_FILE)
def test_remove(self):
# We remove File_to_delete_1. We make sure it is the only file that gets deleted
self.assertTrue(self.luigiconn.exists(DROPBOX_TEST_FILE_TO_DELETE_1))
self.assertTrue(self.luigiconn.exists(DROPBOX_TEST_FILE_TO_DELETE_2))
self.assertTrue(self.luigiconn.remove(DROPBOX_TEST_FILE_TO_DELETE_1))
self.assertFalse(self.luigiconn.exists(DROPBOX_TEST_FILE_TO_DELETE_1))
self.assertTrue(self.luigiconn.exists(DROPBOX_TEST_FILE_TO_DELETE_2))
# We remove a directory, we make sure that the files that were in the directory are also deleted
self.luigiconn.remove(DROPBOX_TEST_DIR_TO_DELETE)
self.assertFalse(self.luigiconn.exists(DROPBOX_TEST_FILE_TO_DELETE_2))
# We make sure that we return False when we fail to remove a non-existing path
self.assertFalse(self.luigiconn.remove(DROPBOX_TEST_NON_EXISTING_FILE))
self.assertFalse(self.luigiconn.remove(DROPBOX_TEST_NON_EXISTING_FILE + '/'))
def test_mkdir_new_dir(self):
self.assertFalse(self.luigiconn.exists(DROPBOX_TEST_DIR_TO_CREATE))
self.assertFalse(self.luigiconn.exists(DROPBOX_TEST_OUTER_DIR_TO_CREATE))
self.luigiconn.mkdir(DROPBOX_TEST_DIR_TO_CREATE)
self.assertTrue(self.luigiconn.isdir(DROPBOX_TEST_OUTER_DIR_TO_CREATE))
self.assertTrue(self.luigiconn.isdir(DROPBOX_TEST_DIR_TO_CREATE))
self.assertTrue(self.luigiconn.isdir(DROPBOX_TEST_DIR_TO_CREATE))
def aux_lifecycle_of_directory(self, path):
# Initially, the directory does not exists
self.assertFalse(self.luigiconn.exists(path))
self.assertFalse(self.luigiconn.isdir(path))
# Now we create the directory and verify that it exists
self.luigiconn.mkdir(path)
self.assertTrue(self.luigiconn.exists(path))
self.assertTrue(self.luigiconn.isdir(path))
# Now we remote the directory and verify that it no longer exists
self.luigiconn.remove(path)
self.assertFalse(self.luigiconn.exists(path))
self.assertFalse(self.luigiconn.isdir(path))
def test_lifecycle_of_dirpath(self):
self.aux_lifecycle_of_directory(DROPBOX_TEST_SIMPLE_DIR_TO_CREATE_AND_DELETE)
def test_lifecycle_of_dirpath_with_trailing_slash(self):
self.aux_lifecycle_of_directory(DROPBOX_TEST_SIMPLE_DIR_TO_CREATE_AND_DELETE + '/')
def test_lifecycle_of_dirpath_with_several_trailing_mixed(self):
self.luigiconn.mkdir(DROPBOX_TEST_SIMPLE_DIR_TO_CREATE_AND_DELETE + '/')
self.assertTrue(self.luigiconn.exists(DROPBOX_TEST_SIMPLE_DIR_TO_CREATE_AND_DELETE))
self.luigiconn.remove(DROPBOX_TEST_SIMPLE_DIR_TO_CREATE_AND_DELETE)
self.assertFalse(self.luigiconn.exists(DROPBOX_TEST_SIMPLE_DIR_TO_CREATE_AND_DELETE + '/'))
def test_lifecycle_of_dirpath_with_several_trailing_mixed_2(self):
self.luigiconn.mkdir(DROPBOX_TEST_SIMPLE_DIR_TO_CREATE_AND_DELETE)
self.assertTrue(self.luigiconn.exists(DROPBOX_TEST_SIMPLE_DIR_TO_CREATE_AND_DELETE + '/'))
self.luigiconn.remove(DROPBOX_TEST_SIMPLE_DIR_TO_CREATE_AND_DELETE + '/')
self.assertFalse(self.luigiconn.exists(DROPBOX_TEST_SIMPLE_DIR_TO_CREATE_AND_DELETE))
def test_mkdir_new_dir_two_slashes(self):
with self.assertRaises(dropbox.dropbox.ApiError):
self.luigiconn.mkdir(DROPBOX_TEST_SIMPLE_DIR_TO_CREATE_AND_DELETE + '//')
def test_mkdir_recreate_dir(self):
try:
self.luigiconn.mkdir(DROPBOX_TEST_SIMPLE_DIR)
except Exception as ex:
self.fail("mkdir with default options raises Exception:" + str(ex))
try:
self.luigiconn.mkdir(DROPBOX_TEST_SIMPLE_DIR, raise_if_exists=False)
except Exception as ex:
self.fail("mkdir with 'raise_if_exists=False' raises Exception:" + str(ex))
with self.assertRaises(luigi.target.FileAlreadyExists):
self.luigiconn.mkdir(DROPBOX_TEST_SIMPLE_DIR, raise_if_exists=True)
def test_mkdir_recreate_slashed_dir(self):
try:
self.luigiconn.mkdir(DROPBOX_TEST_SIMPLE_DIR + '/')
except Exception as ex:
self.fail("mkdir with default options raises Exception:" + str(ex))
try:
self.luigiconn.mkdir(DROPBOX_TEST_SIMPLE_DIR + '/', raise_if_exists=False)
except Exception as ex:
self.fail("mkdir with 'raise_if_exists=False' raises Exception:" + str(ex))
with self.assertRaises(luigi.target.FileAlreadyExists):
self.luigiconn.mkdir(DROPBOX_TEST_SIMPLE_DIR + '/', raise_if_exists=True)
def test_mkdir_recreate_file(self):
with self.assertRaises(luigi.target.NotADirectory):
self.luigiconn.mkdir(DROPBOX_TEST_SIMPLE_FILE)
with self.assertRaises(luigi.target.NotADirectory):
self.luigiconn.mkdir(DROPBOX_TEST_SIMPLE_FILE, raise_if_exists=True)
with self.assertRaises(luigi.target.NotADirectory):
self.luigiconn.mkdir(DROPBOX_TEST_SIMPLE_FILE, raise_if_exists=False)
def test_isdir(self):
self.assertTrue(self.luigiconn.isdir('/'))
self.assertTrue(self.luigiconn.isdir(DROPBOX_TEST_PATH))
self.assertTrue(self.luigiconn.isdir(DROPBOX_TEST_SIMPLE_DIR))
self.assertTrue(self.luigiconn.isdir(DROPBOX_TEST_SIMPLE_DIR + '/'))
self.assertFalse(self.luigiconn.isdir(DROPBOX_TEST_SIMPLE_FILE))
self.assertFalse(self.luigiconn.isdir(DROPBOX_TEST_NON_EXISTING_FILE))
self.assertFalse(self.luigiconn.isdir(DROPBOX_TEST_NON_EXISTING_FILE + '/'))
def test_move(self):
md, res = self.dropbox_api.files_download(DROPBOX_TEST_FILE_TO_MOVE_ORIG)
initial_contents = res.content
self.luigiconn.move(DROPBOX_TEST_FILE_TO_MOVE_ORIG, DROPBOX_TEST_FILE_TO_MOVE_DEST)
md, res = self.dropbox_api.files_download(DROPBOX_TEST_FILE_TO_MOVE_DEST)
after_moving_contents = res.content
self.assertEqual(initial_contents, after_moving_contents)
self.assertFalse(self.luigiconn.exists(DROPBOX_TEST_FILE_TO_MOVE_ORIG))
self.assertTrue(self.luigiconn.exists(DROPBOX_TEST_FILE_TO_MOVE_DEST))
def test_copy(self):
md, res = self.dropbox_api.files_download(DROPBOX_TEST_FILE_TO_COPY_ORIG)
initial_contents = res.content
self.luigiconn.copy(DROPBOX_TEST_FILE_TO_COPY_ORIG, DROPBOX_TEST_FILE_TO_COPY_DEST)
md, res = self.dropbox_api.files_download(DROPBOX_TEST_FILE_TO_COPY_DEST)
after_copyng_contents = res.content
self.assertEqual(initial_contents, after_copyng_contents)
self.assertTrue(self.luigiconn.exists(DROPBOX_TEST_FILE_TO_COPY_ORIG))
self.assertTrue(self.luigiconn.exists(DROPBOX_TEST_FILE_TO_COPY_DEST))