def test_single(self):
""" Test upload file with creation of intermediate folders """
ftp_path = "/test/nest/luigi-test"
local_filepath = "/tmp/luigi-test-ftp"
# create local temp file
with open(local_filepath, 'w') as outfile:
outfile.write("something to fill")
rfs = RemoteFileSystem(HOST, USER, PWD)
rfs.put(local_filepath, ftp_path)
# manually connect to ftp
ftp = ftplib.FTP(HOST, USER, PWD)
ftp.cwd("/test/nest")
list_dir = ftp.nlst()
# file is successfuly created
self.assertTrue("luigi-test" in list_dir)
# delete tmp files
ftp.delete("luigi-test")
ftp.cwd("/")
ftp.rmd("/test/nest")
ftp.rmd("test")
os.remove(local_filepath)
ftp.close()
def test_single(self):
""" Test upload file with creation of intermediate folders """
ftp_path = "/test/nest/luigi-test"
local_filepath = "/tmp/luigi-test-ftp"
# create local temp file
with open(local_filepath, 'w') as outfile:
outfile.write("something to fill")
rfs = RemoteFileSystem(HOST, USER, PWD)
rfs.put(local_filepath, ftp_path)
# manually connect to ftp
ftp = ftplib.FTP(HOST, USER, PWD)
ftp.cwd("/test/nest")
list_dir = ftp.nlst()
# file is successfuly created
self.assertTrue("luigi-test" in list_dir)
# delete tmp files
ftp.delete("luigi-test")
ftp.cwd("/")
ftp.rmd("/test/nest")
ftp.rmd("test")
os.remove(local_filepath)
ftp.close()
def test_recursive_remove(self):
""" Test FTP filesystem removing files recursive """
rfs = RemoteFileSystem(HOST, USER, PWD)
rfs.remove('/test')
ftp = ftplib.FTP(HOST, USER, PWD)
list_dir = ftp.nlst()
self.assertFalse("test" in list_dir)
def test_recursive_remove(self):
""" Test FTP filesystem removing files recursive """
rfs = RemoteFileSystem(HOST, USER, PWD)
rfs.remove('/test')
ftp = ftplib.FTP(HOST, USER, PWD)
list_dir = ftp.nlst()
self.assertFalse("test" in list_dir)
def test_file_remove(self):
""" Delete with recursive deactivated """
rfs = RemoteFileSystem(HOST, USER, PWD)
rfs.remove('/test/hola/file3', recursive=False)
rfs.remove('/test/hola/file2', recursive=False)
rfs.remove('/test/hola', recursive=False)
rfs.remove('/test/file1', recursive=False)
rfs.remove('/test', recursive=False)
ftp = ftplib.FTP(HOST, USER, PWD)
list_dir = ftp.nlst()
self.assertFalse("test" in list_dir)
def test_file_remove(self):
""" Delete with recursive deactivated """
rfs = RemoteFileSystem(HOST, USER, PWD)
rfs.remove('/test/hola/file3', recursive=False)
rfs.remove('/test/hola/file2', recursive=False)
rfs.remove('/test/hola', recursive=False)
rfs.remove('/test/file1', recursive=False)
rfs.remove('/test', recursive=False)
ftp = ftplib.FTP(HOST, USER, PWD)
list_dir = ftp.nlst()
self.assertFalse("test" in list_dir)
def output(self):
rmfs = RemoteFileSystem(FTP_HOST, username=FTP_USER, password=FTP_PASS)
files = None
if rmfs.exists(FTP_IN_PATH):
lst = rmfs.listdir(FTP_IN_PATH)
files = fnmatch.filter([basename(l) for l in lst], self.bins_fname_tmp)
else:
NoBinsToParseTaxPayments('Could not find directory with bins')
if not files:
raise NoBinsToParseTaxPayments('Could not find any file with bins')
# bins_fpath = join(FTP_IN_PATH, last_file_with_bins(files))
bins_fpath = FTP_IN_PATH + '/' + last_file_with_bins(files)
return RemoteTarget(bins_fpath, FTP_HOST,
username=FTP_USER, password=FTP_PASS)
def output(self):
rmfs = RemoteFileSystem(FTP_HOST, username=FTP_USER, password=FTP_PASS)
files = None
if rmfs.exists(FTP_IN_PATH):
lst = rmfs.listdir(FTP_IN_PATH)
files = fnmatch.filter([basename(l) for l in lst],
self.fname_pattern)
else:
NoDataflowExportFile('Could not find directory with bins')
if not files:
raise NoDataflowExportFile('Could not find any file with bins')
bins_fpath = FTP_IN_PATH + '/' + current_file_to_process(files)
return RemoteTarget(bins_fpath,
FTP_HOST,
username=FTP_USER,
password=FTP_PASS)