def test_is_dir(self):
ftp = ftp_client(self.host_download, "{0}/file".format(BASE_DIR),
self.user_name, self.password)
self.assertFalse(ftp.is_directory(), "Path is dir")
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
self.assertTrue(ftp.is_directory(), "Path is file")
def test_is_exist(self):
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR), self.user_name, self.password)
self.assertTrue(ftp.exists(), "not correct root")
ftp = ftp_client(self.host_download, "{0}/folder/s".format(BASE_DIR), self.user_name, self.password)
self.assertFalse(ftp.exists(), "correct path")
ftp = ftp_client(self.host_download, "asd/qwe", self.user_name, self.password)
self.assertFalse(ftp.exists(), "correct path")
def test_basedir(self):
ftp = ftp_client(self.host_download, "{0}/file".format(BASE_DIR),
self.user_name, self.password)
self.assertEqual("{0}".format(BASE_DIR), ftp.base_dir().get_description().name)
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
self.assertEqual("", ftp.base_dir().get_description().name)
def test_is_dir(self):
ftp = ftp_client(self.host_download, "{0}/file".format(BASE_DIR),
self.user_name, self.password)
self.assertFalse(ftp.is_directory(), "Path is dir")
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
self.assertTrue(ftp.is_directory(), "Path is file")
def load_file_from_ftp_to_local(context):
for _file in context['new_files']:
ftp_client(host=HOST_DOWNLOAD,
login=USER_NAME,
password=PASSWORD,
path=_file.name).download_file(local_path=os.path.join(
os.path.dirname(__file__), "resources/tmp/"))
def setUp(self):
super(TestFTPConnector, self).setUp()
config = RawConfigParser()
config.read(os.path.join(os.path.dirname(__file__), "resources/ftp/ftp_config.ini"))
self.host_download = config.get("ftp", "host.download")
self.user_name = config.get("ftp", "user.name")
self.password = config.get("ftp", "password")
self.path = config.get("ftp", "path")
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR), self.user_name, self.password)
self.assertFalse(ftp.exists())
if not ftp.exists():
ftp.create(make_dir=True, create_parents=True)
self.assertTrue(ftp.exists())
self.assertTrue(ftp.is_directory())
ftp = ftp_client(self.host_download, "{0}/file".format(BASE_DIR), self.user_name, self.password)
self.assertFalse(ftp.exists())
if not ftp.exists():
ftp.create(make_dir=False, create_parents=True)
self.assertTrue(ftp.exists())
self.assertFalse(ftp.is_directory())
ftp = ftp_client(self.host_download, "{0}/folder/file".format(BASE_DIR), self.user_name, self.password)
self.assertFalse(ftp.exists())
if not ftp.exists():
ftp.create(make_dir=False, create_parents=True)
self.assertTrue(ftp.exists())
self.assertFalse(ftp.is_directory())
def test_basedir(self):
ftp = ftp_client(self.host_download, "{0}/file".format(BASE_DIR),
self.user_name, self.password)
self.assertEqual("{0}".format(BASE_DIR),
ftp.base_dir().get_description().name)
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
self.assertEqual("", ftp.base_dir().get_description().name)
def load_file_from_ftp_to_local(context):
for _file in context['new_files']:
ftp_client(
host=HOST_DOWNLOAD,
login=USER_NAME,
password=PASSWORD,
path=_file.name).download_file(local_path=os.path.join(os.path.dirname(__file__),
"resources/tmp/"))
def test_get_description(self):
ftp = ftp_client(self.host_download, "{0}/file".format(BASE_DIR),
self.user_name, self.password)
self.assertEqual(ftp.get_description().name, "{0}/file".format(BASE_DIR))
self.assertNotEqual(ftp.get_description().update_date, None)
self.assertEqual(ftp.get_description().owner, None)
self.assertEqual(ftp.get_description().size, 0)
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
self.assertEqual(ftp.get_description().update_date, None)
def test_list_files(self):
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
self.assertEqual(2, len(ftp.list_files()), "bad count")
self.assertEqual(list,
ftp.list_files().__class__,
"list files has return bad type")
ftp = ftp_client(self.host_download, "{0}/asd".format(BASE_DIR),
self.user_name, self.password)
self.assertRaises(FileNotFoundException, ftp.list_files)
def test_is_exist(self):
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
self.assertTrue(ftp.exists(), "not correct root")
ftp = ftp_client(self.host_download, "{0}/folder/s".format(BASE_DIR),
self.user_name, self.password)
self.assertFalse(ftp.exists(), "correct path")
ftp = ftp_client(self.host_download, "asd/qwe", self.user_name,
self.password)
self.assertFalse(ftp.exists(), "correct path")
def test_size(self):
ftp = ftp_client(self.host_download, "/", self.user_name, self.password)
self.assertEqual(0, ftp.size(), "Folder has size")
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR), self.user_name, self.password)
self.assertEqual(0, ftp.size(), "Folder has size")
ftp = ftp_client(self.host_download, "{0}/file".format(BASE_DIR), self.user_name, self.password)
self.assertEqual(0, ftp.size(), "File has size")
ftp = ftp_client(self.host_download, "{0}/folder/file".format(BASE_DIR),
self.user_name, self.password)
self.assertEqual(0, ftp.size(), "not equal size")
def test_get_description(self):
ftp = ftp_client(self.host_download, "{0}/file".format(BASE_DIR),
self.user_name, self.password)
self.assertEqual(ftp.get_description().name,
"{0}/file".format(BASE_DIR))
self.assertNotEqual(ftp.get_description().update_date, None)
self.assertEqual(ftp.get_description().owner, None)
self.assertEqual(ftp.get_description().size, 0)
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
self.assertEqual(ftp.get_description().update_date, None)
def test_upload(self):
ftp = ftp_client(self.host_download, "{0}/file2".format(BASE_DIR), self.user_name, self.password)
ftp.upload(os.path.join(os.path.dirname(__file__), "resources/ftp/ftp_config.ini"))
ftp = ftp_client(self.host_download, "{0}/file2".format(BASE_DIR), self.user_name, self.password)
ftp.upload(os.path.join(os.path.dirname(__file__), "resources/ftp/ftp_config.ini"), update=True)
ftp = ftp_client(self.host_download, "{0}/file2".format(BASE_DIR), self.user_name, self.password)
self.assertTrue(ftp.exists())
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR), self.user_name, self.password)
ftp.upload(os.path.join(os.path.dirname(__file__), "resources/ftp/ftp_config.ini"))
ftp = ftp_client(self.host_download, "{0}/ftp_config.ini".format(BASE_DIR), self.user_name, self.password)
self.assertTrue(ftp.exists())
def test_download_file(self):
tmp = tempfile.gettempdir()
path = os.path.join(tmp, "file")
try:
ftp = ftp_client(self.host_download, "{0}/".format(BASE_DIR), self.user_name, self.password)
self.assertRaises(Exception, ftp.download_file, tmp)
ftp = ftp_client(self.host_download, "{0}/file".format(BASE_DIR),
self.user_name, self.password)
ftp.download_file(tmp)
self.assertTrue(os.path.exists(path))
finally:
os.remove(path)
def test_download_file(self):
tmp = tempfile.gettempdir()
path = os.path.join(tmp, "file")
try:
ftp = ftp_client(self.host_download, "{0}/".format(BASE_DIR),
self.user_name, self.password)
self.assertRaises(Exception, ftp.download_file, tmp)
ftp = ftp_client(self.host_download, "{0}/file".format(BASE_DIR),
self.user_name, self.password)
ftp.download_file(tmp)
self.assertTrue(os.path.exists(path))
finally:
os.remove(path)
def test_download_dir(self):
tmp = tempfile.gettempdir()
path = os.path.join(tmp, "{0}".format(BASE_DIR))
try:
ftp = ftp_client(self.host_download, "{0}/file".format(BASE_DIR), self.user_name, self.password)
self.assertRaises(Exception, ftp.download_file(tmp))
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
ftp.download_dir(tmp, recursive=True)
self.assertTrue(os.path.exists(path))
finally:
import shutil
shutil.rmtree(path)
def test_size(self):
ftp = ftp_client(self.host_download, "/", self.user_name,
self.password)
self.assertEqual(0, ftp.size(), "Folder has size")
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
self.assertEqual(0, ftp.size(), "Folder has size")
ftp = ftp_client(self.host_download, "{0}/file".format(BASE_DIR),
self.user_name, self.password)
self.assertEqual(0, ftp.size(), "File has size")
ftp = ftp_client(self.host_download,
"{0}/folder/file".format(BASE_DIR), self.user_name,
self.password)
self.assertEqual(0, ftp.size(), "not equal size")
def test_download_dir(self):
tmp = tempfile.gettempdir()
path = os.path.join(tmp, "{0}".format(BASE_DIR))
try:
ftp = ftp_client(self.host_download, "{0}/file".format(BASE_DIR),
self.user_name, self.password)
self.assertRaises(Exception, ftp.download_file(tmp))
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
ftp.download_dir(tmp, recursive=True)
self.assertTrue(os.path.exists(path))
finally:
import shutil
shutil.rmtree(path)
def load_file_on_ftp(context):
context['files_on_FTP'] = []
for ftp_file in ftp_client(host=HOST_DOWNLOAD,
login=USER_NAME,
password=PASSWORD,
path=PATH):
context['files_on_FTP'].append(ftp_file.get_description())
def load_file_on_ftp(context):
context['files_on_FTP'] = []
for ftp_file in ftp_client(
host=HOST_DOWNLOAD,
login=USER_NAME,
password=PASSWORD,
path=PATH):
context['files_on_FTP'].append(ftp_file.get_description())
def test_download_dir_with_predicate(self):
tmp = tempfile.gettempdir()
path = os.path.join(tmp, "{0}".format(BASE_DIR))
try:
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
ftp.download_dir(tmp, self.predicate, recursive=True)
self.assertTrue(os.path.exists(path))
finally:
import shutil
shutil.rmtree(path)
def test_download_dir_with_predicate(self):
tmp = tempfile.gettempdir()
path = os.path.join(tmp, "{0}".format(BASE_DIR))
try:
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
ftp.download_dir(tmp, self.predicate, recursive=True)
self.assertTrue(os.path.exists(path))
finally:
import shutil
shutil.rmtree(path)
def test_upload(self):
ftp = ftp_client(self.host_download, "{0}/file2".format(BASE_DIR),
self.user_name, self.password)
ftp.upload(
os.path.join(os.path.dirname(__file__),
"resources/ftp/ftp_config.ini"))
ftp = ftp_client(self.host_download, "{0}/file2".format(BASE_DIR),
self.user_name, self.password)
ftp.upload(os.path.join(os.path.dirname(__file__),
"resources/ftp/ftp_config.ini"),
update=True)
ftp = ftp_client(self.host_download, "{0}/file2".format(BASE_DIR),
self.user_name, self.password)
self.assertTrue(ftp.exists())
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
ftp.upload(
os.path.join(os.path.dirname(__file__),
"resources/ftp/ftp_config.ini"))
ftp = ftp_client(self.host_download,
"{0}/ftp_config.ini".format(BASE_DIR), self.user_name,
self.password)
self.assertTrue(ftp.exists())
def setUp(self):
super(TestFTPConnector, self).setUp()
config = RawConfigParser()
config.read(
os.path.join(os.path.dirname(__file__),
"resources/ftp/ftp_config.ini"))
self.host_download = config.get("ftp", "host.download")
self.user_name = config.get("ftp", "user.name")
self.password = config.get("ftp", "password")
self.path = config.get("ftp", "path")
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
self.assertFalse(ftp.exists())
if not ftp.exists():
ftp.create(make_dir=True, create_parents=True)
self.assertTrue(ftp.exists())
self.assertTrue(ftp.is_directory())
ftp = ftp_client(self.host_download, "{0}/file".format(BASE_DIR),
self.user_name, self.password)
self.assertFalse(ftp.exists())
if not ftp.exists():
ftp.create(make_dir=False, create_parents=True)
self.assertTrue(ftp.exists())
self.assertFalse(ftp.is_directory())
ftp = ftp_client(self.host_download,
"{0}/folder/file".format(BASE_DIR), self.user_name,
self.password)
self.assertFalse(ftp.exists())
if not ftp.exists():
ftp.create(make_dir=False, create_parents=True)
self.assertTrue(ftp.exists())
self.assertFalse(ftp.is_directory())
def test_list_files(self):
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR), self.user_name, self.password)
self.assertEqual(2, len(ftp.list_files()), "bad count")
self.assertEqual(list, ftp.list_files().__class__, "list files has return bad type")
ftp = ftp_client(self.host_download, "{0}/asd".format(BASE_DIR), self.user_name, self.password)
self.assertRaises(FileNotFoundException, ftp.list_files)
def tearDown(self):
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR),
self.user_name, self.password)
ftp.delete(recursive=True)
self.assertFalse(ftp.exists())
BASE_DIR = "/tmp"
if __name__ == "__main__":
hdfs_file = HDFS("{0}/raw".format(BASE_DIR))
if hdfs_file.exists():
hdfs_file.delete(recursive=True)
config = RawConfigParser()
config.read(os.path.join(os.path.dirname(__file__), "resources/ftp_config.ini"))
host_download = config.get("ftp", "host.download")
user_name = config.get("ftp", "user.name")
password = config.get("ftp", "password")
path = config.get("ftp", "path")
ftp = ftp_client(host=host_download,
login=user_name,
password=password,
path="/tmp")
if ftp.exists():
ftp.delete(recursive=True)
local_file = LocalFS(path=os.path.join(os.path.dirname(__file__),
'resources/tmp'))
if local_file.exists():
local_file.delete_directory()
hive = Hive.load_queries_from_string(query="DROP DATABASE IF EXISTS hive_monitoring CASCADE;")
hive.run()
def tearDown(self):
ftp = ftp_client(self.host_download, "{0}".format(BASE_DIR), self.user_name, self.password)
ftp.delete(recursive=True)
self.assertFalse(ftp.exists())