def test_get(self):
""" Test Remote target get method downloading a file from ftp """
local_filepath = "/tmp/luigi-remotetarget-read-test"
tmp_filepath = "/tmp/tmp-luigi-remotetarget-read-test"
remote_file = "/test/example.get.file"
# create local temp file
with open(tmp_filepath, 'w') as outfile:
outfile.write("something to fill")
# manualy upload to ftp
ftp = ftplib.FTP(HOST, USER, PWD)
ftp.mkd("test")
ftp.storbinary('STOR %s' % remote_file, open(tmp_filepath, 'rb'))
ftp.close()
# execute command
remotetarget = RemoteTarget(remote_file, HOST, username=USER, password=PWD)
remotetarget.get(local_filepath)
# file is successfuly created
self.assertTrue(os.path.exists(local_filepath))
# clean
os.remove(local_filepath)
os.remove(tmp_filepath)
ftp = ftplib.FTP(HOST, USER, PWD)
ftp.delete(remote_file)
ftp.cwd("/")
ftp.rmd("test")
ftp.close()
def test_get(self):
""" Test Remote target get method downloading a file from ftp """
local_filepath = "/tmp/luigi-remotetarget-read-test"
tmp_filepath = "/tmp/tmp-luigi-remotetarget-read-test"
remote_file = "/test/example.get.file"
# create local temp file
with open(tmp_filepath, 'w') as outfile:
outfile.write("something to fill")
# manualy upload to ftp
ftp = ftplib.FTP(HOST, USER, PWD)
ftp.mkd("test")
ftp.storbinary('STOR %s' % remote_file, open(tmp_filepath, 'rb'))
ftp.close()
# execute command
remotetarget = RemoteTarget(remote_file,
HOST,
username=USER,
password=PWD)
remotetarget.get(local_filepath)
# file is successfuly created
self.assertTrue(os.path.exists(local_filepath))
# clean
os.remove(local_filepath)
os.remove(tmp_filepath)
ftp = ftplib.FTP(HOST, USER, PWD)
ftp.delete(remote_file)
ftp.cwd("/")
ftp.rmd("test")
ftp.close()
def test_put(self):
""" Test RemoteTarget put method with uploading to an FTP """
local_filepath = "/tmp/luigi-remotetarget-write-test"
remote_file = "/test/example.put.file"
# create local temp file
with open(local_filepath, 'w') as outfile:
outfile.write("something to fill")
remotetarget = RemoteTarget(remote_file, HOST, username=USER, password=PWD)
remotetarget.put(local_filepath)
# manually connect to ftp
ftp = ftplib.FTP(HOST, USER, PWD)
ftp.cwd("/test")
list_dir = ftp.nlst()
# file is successfuly created
self.assertTrue(remote_file.split("/")[-1] in list_dir)
# clean
os.remove(local_filepath)
ftp.delete(remote_file)
ftp.cwd("/")
ftp.rmd("test")
ftp.close()
def test_put(self):
""" Test RemoteTarget put method with uploading to an FTP """
local_filepath = "/tmp/luigi-remotetarget-write-test"
remote_file = "/test/example.put.file"
# create local temp file
with open(local_filepath, 'w') as outfile:
outfile.write("something to fill")
remotetarget = RemoteTarget(remote_file,
HOST,
username=USER,
password=PWD)
remotetarget.put(local_filepath)
# manually connect to ftp
ftp = ftplib.FTP(HOST, USER, PWD)
ftp.cwd("/test")
list_dir = ftp.nlst()
# file is successfuly created
self.assertTrue(remote_file.split("/")[-1] in list_dir)
# clean
os.remove(local_filepath)
ftp.delete(remote_file)
ftp.cwd("/")
ftp.rmd("test")
ftp.close()
def output(self):
fname = replace_fext(self.input().path, self.gzext).name
ftp_path = self.ftp_fs_sep.join([self.ftp_directory, fname])
return RemoteTarget(ftp_path,
self.ftp_host,
username=self.ftp_user,
password=self.ftp_pass)
def output(self):
"""
Returns the target output for this task.
In this case, a successful execution of this task will create a file that will be created in a FTP server.
:return: the target output for this task.
:rtype: object (:py:class:`~luigi.target.Target`)
"""
return RemoteTarget('/experiment/output1.txt', HOST, username=USER, password=PWD)
def output(self):
# fix from 30.07.2020
# requested by Esmukhanov to avoid probability getting no data for DataFlow
# here we just use tomorrow date to build output file name
dt = datetime.today() + timedelta(days=1)
ftp_prs_gzip_fpath = os.path.join(FTP_PATH, gziped_fname(self.input()[0].path,
suff=date_for_fname(dt)))
ftp_notax_gzip_fpath = os.path.join(FTP_PATH, gziped_fname(self.input()[1].path,
suff=date_for_fname(dt)))
ftp_data_gzip_fpath = os.path.join(FTP_PATH, gziped_fname(self.input()[2].path,
suff=date_for_fname(dt)))
return [
RemoteTarget(ftp_prs_gzip_fpath, FTP_HOST,
username=FTP_USER, password=FTP_PASS),
RemoteTarget(ftp_notax_gzip_fpath, FTP_HOST,
username=FTP_USER, password=FTP_PASS),
RemoteTarget(ftp_data_gzip_fpath, FTP_HOST,
username=FTP_USER, password=FTP_PASS)
]
def output(self):
r = []
for inp in self.input():
fname = gziped_fname(inp.path)
r.append(os.path.join(self.ftp_directory, fname))
return [
RemoteTarget(ftp_fpath,
FTP_HOST,
username=FTP_USER,
password=FTP_PASS) for ftp_fpath in r
]
def output(self):
job_file = os.path.join(JOBS_CONFIG_DIR, str(self.jobfile))
job_conf = Utils.read_file(job_file)
base = os.path.basename(self.input().path)
day = dt.today().strftime("%Y%m%d")
ftp_path = os.path.join(
FTP_REMOTE_PATH, "{}_{}.{}".format(
os.path.splitext(base)[0], day, Utils.ext(self.input().path)))
if Box(json.loads(job_conf)).gzip:
ftp_path += '.gzip'
return RemoteTarget(ftp_path,
FTP_HOST,
username=FTP_USER,
password=FTP_PASS)
def output(self):
# convert luigi.Parameter to string
directory = str(self.ftp_directory)
sep = str(self.ftp_os_sep)
# build full path to target directory
if self.ftp_directory:
path = sep.join([self.ftp_path, directory])
else:
path = self.ftp_path
dt = date_for_fname(self.date, for_month=self.monthly)
fname = gziped_fname(self.input().path, suff=dt)
ftp_path = sep.join([path, fname])
return RemoteTarget(ftp_path, self.ftp_host,
username=self.ftp_user, password=self.ftp_pass)
def test_get(self):
""" Test Remote target get method downloading a file from ftp """
local_filepath = "/tmp/luigi-remotetarget-read-test"
tmp_filepath = "/tmp/tmp-luigi-remotetarget-read-test"
remote_file = "/test/example.get.file"
# create local temp file
with open(tmp_filepath, 'w') as outfile:
outfile.write("something to fill")
# manualy upload to ftp
ftp = ftplib.FTP(HOST, USER, PWD)
ftp.mkd("test")
ftp.storbinary('STOR %s' % remote_file, open(tmp_filepath, 'rb'))
ftp.close()
# execute command
remotetarget = RemoteTarget(remote_file, HOST, username=USER, password=PWD)
remotetarget.get(local_filepath)
# make sure that it can open file
with remotetarget.open('r') as fin:
self.assertEqual(fin.read(), "something to fill")
# file is successfuly created
self.assertTrue(os.path.exists(local_filepath))
# test RemoteTarget with mtime
ts = datetime.datetime.now() - datetime.timedelta(minutes=2)
delayed_remotetarget = RemoteTarget(remote_file, HOST, username=USER, password=PWD, mtime=ts)
self.assertTrue(delayed_remotetarget.exists())
ts = datetime.datetime.now() + datetime.timedelta(days=2) # who knows what timezone it is in
delayed_remotetarget = RemoteTarget(remote_file, HOST, username=USER, password=PWD, mtime=ts)
self.assertFalse(delayed_remotetarget.exists())
# clean
os.remove(local_filepath)
os.remove(tmp_filepath)
ftp = ftplib.FTP(HOST, USER, PWD)
ftp.delete(remote_file)
ftp.cwd("/")
ftp.rmd("test")
ftp.close()
def output(self):
rmfs = RemoteFileSystem(FTP_HOST, username=FTP_USER, password=FTP_PASS)
files = None
if rmfs.exists(FTP_IN_PATH):
lst = rmfs.listdir(FTP_IN_PATH)
files = fnmatch.filter([basename(l) for l in lst], self.bins_fname_tmp)
else:
NoBinsToParseTaxPayments('Could not find directory with bins')
if not files:
raise NoBinsToParseTaxPayments('Could not find any file with bins')
# bins_fpath = join(FTP_IN_PATH, last_file_with_bins(files))
bins_fpath = FTP_IN_PATH + '/' + last_file_with_bins(files)
return RemoteTarget(bins_fpath, FTP_HOST,
username=FTP_USER, password=FTP_PASS)
def output(self):
rmfs = RemoteFileSystem(FTP_HOST, username=FTP_USER, password=FTP_PASS)
files = None
if rmfs.exists(FTP_IN_PATH):
lst = rmfs.listdir(FTP_IN_PATH)
files = fnmatch.filter([basename(l) for l in lst],
self.fname_pattern)
else:
NoDataflowExportFile('Could not find directory with bins')
if not files:
raise NoDataflowExportFile('Could not find any file with bins')
bins_fpath = FTP_IN_PATH + '/' + current_file_to_process(files)
return RemoteTarget(bins_fpath,
FTP_HOST,
username=FTP_USER,
password=FTP_PASS)
def output(self):
return RemoteTarget('/experiment/output1.txt',
HOST,
username=USER,
password=PWD)
def output(self):
return RemoteTarget(self.date.strftime('us-stocks.%Y%m%d.gz'),
csi_ftp_server,
username=csi_ftp_username,
password=csi_ftp_password,
port=csi_ftp_port)
def output(self):
return RemoteTarget(self.date.strftime('Econ%Y%m%d.txt'),
csi_ftp_server,
username=csi_ftp_username,
password=csi_ftp_password,
port=csi_ftp_port)
def test_get(self):
""" Test Remote target get method downloading a file from ftp """
local_filepath = "/tmp/luigi-remotetarget-read-test"
tmp_filepath = "/tmp/tmp-luigi-remotetarget-read-test"
remote_file = "/test/example.get.file"
# create local temp file
with open(tmp_filepath, 'w') as outfile:
outfile.write("something to fill")
# manualy upload to ftp
ftp = ftplib.FTP(HOST, USER, PWD)
ftp.mkd("test")
ftp.storbinary('STOR %s' % remote_file, open(tmp_filepath, 'rb'))
ftp.close()
# execute command
remotetarget = RemoteTarget(remote_file,
HOST,
username=USER,
password=PWD)
remotetarget.get(local_filepath)
# make sure that it can open file
with remotetarget.open('r') as fin:
self.assertEqual(fin.read(), "something to fill")
# file is successfuly created
self.assertTrue(os.path.exists(local_filepath))
# test RemoteTarget with mtime
ts = datetime.datetime.now() - datetime.timedelta(minutes=2)
delayed_remotetarget = RemoteTarget(remote_file,
HOST,
username=USER,
password=PWD,
mtime=ts)
self.assertTrue(delayed_remotetarget.exists())
ts = datetime.datetime.now() + datetime.timedelta(
days=2) # who knows what timezone it is in
delayed_remotetarget = RemoteTarget(remote_file,
HOST,
username=USER,
password=PWD,
mtime=ts)
self.assertFalse(delayed_remotetarget.exists())
# clean
os.remove(local_filepath)
os.remove(tmp_filepath)
ftp = ftplib.FTP(HOST, USER, PWD)
ftp.delete(remote_file)
ftp.cwd("/")
ftp.rmd("test")
ftp.close()
