Esempio n. 1
0
    def test_get(self):
        """ Test Remote target get method downloading a file from ftp """
        local_filepath = "/tmp/luigi-remotetarget-read-test"
        tmp_filepath = "/tmp/tmp-luigi-remotetarget-read-test"
        remote_file = "/test/example.get.file"

        # create local temp file
        with open(tmp_filepath, 'w') as outfile:
            outfile.write("something to fill")

        # manualy upload to ftp
        ftp = ftplib.FTP(HOST, USER, PWD)
        ftp.mkd("test")
        ftp.storbinary('STOR %s' % remote_file, open(tmp_filepath, 'rb'))
        ftp.close()

        # execute command
        remotetarget = RemoteTarget(remote_file, HOST, username=USER, password=PWD)
        remotetarget.get(local_filepath)

        # file is successfuly created
        self.assertTrue(os.path.exists(local_filepath))

        # clean
        os.remove(local_filepath)
        os.remove(tmp_filepath)
        ftp = ftplib.FTP(HOST, USER, PWD)
        ftp.delete(remote_file)
        ftp.cwd("/")
        ftp.rmd("test")
        ftp.close()
Esempio n. 2
0
    def test_get(self):
        """ Test Remote target get method downloading a file from ftp """
        local_filepath = "/tmp/luigi-remotetarget-read-test"
        tmp_filepath = "/tmp/tmp-luigi-remotetarget-read-test"
        remote_file = "/test/example.get.file"

        # create local temp file
        with open(tmp_filepath, 'w') as outfile:
            outfile.write("something to fill")

        # manualy upload to ftp
        ftp = ftplib.FTP(HOST, USER, PWD)
        ftp.mkd("test")
        ftp.storbinary('STOR %s' % remote_file, open(tmp_filepath, 'rb'))
        ftp.close()

        # execute command
        remotetarget = RemoteTarget(remote_file,
                                    HOST,
                                    username=USER,
                                    password=PWD)
        remotetarget.get(local_filepath)

        # file is successfuly created
        self.assertTrue(os.path.exists(local_filepath))

        # clean
        os.remove(local_filepath)
        os.remove(tmp_filepath)
        ftp = ftplib.FTP(HOST, USER, PWD)
        ftp.delete(remote_file)
        ftp.cwd("/")
        ftp.rmd("test")
        ftp.close()
Esempio n. 3
0
    def test_put(self):
        """ Test RemoteTarget put method with uploading to an FTP """
        local_filepath = "/tmp/luigi-remotetarget-write-test"
        remote_file = "/test/example.put.file"

        # create local temp file
        with open(local_filepath, 'w') as outfile:
            outfile.write("something to fill")

        remotetarget = RemoteTarget(remote_file, HOST, username=USER, password=PWD)
        remotetarget.put(local_filepath)

        # manually connect to ftp
        ftp = ftplib.FTP(HOST, USER, PWD)
        ftp.cwd("/test")
        list_dir = ftp.nlst()

        # file is successfuly created
        self.assertTrue(remote_file.split("/")[-1] in list_dir)

        # clean
        os.remove(local_filepath)
        ftp.delete(remote_file)
        ftp.cwd("/")
        ftp.rmd("test")
        ftp.close()
Esempio n. 4
0
    def test_put(self):
        """ Test RemoteTarget put method with uploading to an FTP """
        local_filepath = "/tmp/luigi-remotetarget-write-test"
        remote_file = "/test/example.put.file"

        # create local temp file
        with open(local_filepath, 'w') as outfile:
            outfile.write("something to fill")

        remotetarget = RemoteTarget(remote_file,
                                    HOST,
                                    username=USER,
                                    password=PWD)
        remotetarget.put(local_filepath)

        # manually connect to ftp
        ftp = ftplib.FTP(HOST, USER, PWD)
        ftp.cwd("/test")
        list_dir = ftp.nlst()

        # file is successfuly created
        self.assertTrue(remote_file.split("/")[-1] in list_dir)

        # clean
        os.remove(local_filepath)
        ftp.delete(remote_file)
        ftp.cwd("/")
        ftp.rmd("test")
        ftp.close()
Esempio n. 5
0
 def output(self):
     fname = replace_fext(self.input().path, self.gzext).name
     ftp_path = self.ftp_fs_sep.join([self.ftp_directory, fname])
     return RemoteTarget(ftp_path,
                         self.ftp_host,
                         username=self.ftp_user,
                         password=self.ftp_pass)
Esempio n. 6
0
    def output(self):
        """
        Returns the target output for this task.
        In this case, a successful execution of this task will create a file that will be created in a FTP server.

        :return: the target output for this task.
        :rtype: object (:py:class:`~luigi.target.Target`)
        """
        return RemoteTarget('/experiment/output1.txt', HOST, username=USER, password=PWD)
Esempio n. 7
0
    def output(self):
        # fix from 30.07.2020
        # requested by Esmukhanov to avoid probability getting no data for DataFlow
        # here we just use tomorrow date to build output file name
        dt = datetime.today() + timedelta(days=1)

        ftp_prs_gzip_fpath = os.path.join(FTP_PATH, gziped_fname(self.input()[0].path,
                                                                 suff=date_for_fname(dt)))
        ftp_notax_gzip_fpath = os.path.join(FTP_PATH, gziped_fname(self.input()[1].path,
                                                                   suff=date_for_fname(dt)))
        ftp_data_gzip_fpath = os.path.join(FTP_PATH, gziped_fname(self.input()[2].path,
                                                                  suff=date_for_fname(dt)))

        return [
            RemoteTarget(ftp_prs_gzip_fpath, FTP_HOST,
                         username=FTP_USER, password=FTP_PASS),
            RemoteTarget(ftp_notax_gzip_fpath, FTP_HOST,
                         username=FTP_USER, password=FTP_PASS),
            RemoteTarget(ftp_data_gzip_fpath, FTP_HOST,
                         username=FTP_USER, password=FTP_PASS)
        ]
Esempio n. 8
0
    def output(self):
        r = []
        for inp in self.input():
            fname = gziped_fname(inp.path)
            r.append(os.path.join(self.ftp_directory, fname))

        return [
            RemoteTarget(ftp_fpath,
                         FTP_HOST,
                         username=FTP_USER,
                         password=FTP_PASS) for ftp_fpath in r
        ]
 def output(self):
     job_file = os.path.join(JOBS_CONFIG_DIR, str(self.jobfile))
     job_conf = Utils.read_file(job_file)
     base = os.path.basename(self.input().path)
     day = dt.today().strftime("%Y%m%d")
     ftp_path = os.path.join(
         FTP_REMOTE_PATH, "{}_{}.{}".format(
             os.path.splitext(base)[0], day, Utils.ext(self.input().path)))
     if Box(json.loads(job_conf)).gzip:
         ftp_path += '.gzip'
     return RemoteTarget(ftp_path,
                         FTP_HOST,
                         username=FTP_USER,
                         password=FTP_PASS)
Esempio n. 10
0
    def output(self):
        # convert luigi.Parameter to string
        directory = str(self.ftp_directory)
        sep = str(self.ftp_os_sep)

        # build full path to target directory
        if self.ftp_directory:
            path = sep.join([self.ftp_path, directory])
        else:
            path = self.ftp_path

        dt = date_for_fname(self.date, for_month=self.monthly)
        fname = gziped_fname(self.input().path, suff=dt)
        ftp_path = sep.join([path, fname])
        return RemoteTarget(ftp_path, self.ftp_host,
                            username=self.ftp_user, password=self.ftp_pass)
Esempio n. 11
0
    def test_get(self):
        """ Test Remote target get method downloading a file from ftp """
        local_filepath = "/tmp/luigi-remotetarget-read-test"
        tmp_filepath = "/tmp/tmp-luigi-remotetarget-read-test"
        remote_file = "/test/example.get.file"

        # create local temp file
        with open(tmp_filepath, 'w') as outfile:
            outfile.write("something to fill")

        # manualy upload to ftp
        ftp = ftplib.FTP(HOST, USER, PWD)
        ftp.mkd("test")
        ftp.storbinary('STOR %s' % remote_file, open(tmp_filepath, 'rb'))
        ftp.close()

        # execute command
        remotetarget = RemoteTarget(remote_file, HOST, username=USER, password=PWD)
        remotetarget.get(local_filepath)

        # make sure that it can open file
        with remotetarget.open('r') as fin:
            self.assertEqual(fin.read(), "something to fill")

        # file is successfuly created
        self.assertTrue(os.path.exists(local_filepath))

        # test RemoteTarget with mtime
        ts = datetime.datetime.now() - datetime.timedelta(minutes=2)
        delayed_remotetarget = RemoteTarget(remote_file, HOST, username=USER, password=PWD, mtime=ts)
        self.assertTrue(delayed_remotetarget.exists())

        ts = datetime.datetime.now() + datetime.timedelta(days=2)  # who knows what timezone it is in
        delayed_remotetarget = RemoteTarget(remote_file, HOST, username=USER, password=PWD, mtime=ts)
        self.assertFalse(delayed_remotetarget.exists())

        # clean
        os.remove(local_filepath)
        os.remove(tmp_filepath)
        ftp = ftplib.FTP(HOST, USER, PWD)
        ftp.delete(remote_file)
        ftp.cwd("/")
        ftp.rmd("test")
        ftp.close()
Esempio n. 12
0
    def output(self):
        rmfs = RemoteFileSystem(FTP_HOST, username=FTP_USER, password=FTP_PASS)

        files = None

        if rmfs.exists(FTP_IN_PATH):
            lst = rmfs.listdir(FTP_IN_PATH)
            files = fnmatch.filter([basename(l) for l in lst], self.bins_fname_tmp)
        else:
            NoBinsToParseTaxPayments('Could not find directory with bins')

        if not files:
            raise NoBinsToParseTaxPayments('Could not find any file with bins')

        # bins_fpath = join(FTP_IN_PATH, last_file_with_bins(files))
        bins_fpath = FTP_IN_PATH + '/' + last_file_with_bins(files)
        return RemoteTarget(bins_fpath, FTP_HOST,
                            username=FTP_USER, password=FTP_PASS)
Esempio n. 13
0
    def output(self):
        rmfs = RemoteFileSystem(FTP_HOST, username=FTP_USER, password=FTP_PASS)

        files = None

        if rmfs.exists(FTP_IN_PATH):
            lst = rmfs.listdir(FTP_IN_PATH)
            files = fnmatch.filter([basename(l) for l in lst],
                                   self.fname_pattern)
        else:
            NoDataflowExportFile('Could not find directory with bins')

        if not files:
            raise NoDataflowExportFile('Could not find any file with bins')

        bins_fpath = FTP_IN_PATH + '/' + current_file_to_process(files)
        return RemoteTarget(bins_fpath,
                            FTP_HOST,
                            username=FTP_USER,
                            password=FTP_PASS)
Esempio n. 14
0
 def output(self):
     return RemoteTarget('/experiment/output1.txt',
                         HOST,
                         username=USER,
                         password=PWD)
 def output(self):
     return RemoteTarget(self.date.strftime('us-stocks.%Y%m%d.gz'),
                         csi_ftp_server,
                         username=csi_ftp_username,
                         password=csi_ftp_password,
                         port=csi_ftp_port)
 def output(self):
     return RemoteTarget(self.date.strftime('Econ%Y%m%d.txt'),
                         csi_ftp_server,
                         username=csi_ftp_username,
                         password=csi_ftp_password,
                         port=csi_ftp_port)
Esempio n. 17
0
    def test_get(self):
        """ Test Remote target get method downloading a file from ftp """
        local_filepath = "/tmp/luigi-remotetarget-read-test"
        tmp_filepath = "/tmp/tmp-luigi-remotetarget-read-test"
        remote_file = "/test/example.get.file"

        # create local temp file
        with open(tmp_filepath, 'w') as outfile:
            outfile.write("something to fill")

        # manualy upload to ftp
        ftp = ftplib.FTP(HOST, USER, PWD)
        ftp.mkd("test")
        ftp.storbinary('STOR %s' % remote_file, open(tmp_filepath, 'rb'))
        ftp.close()

        # execute command
        remotetarget = RemoteTarget(remote_file,
                                    HOST,
                                    username=USER,
                                    password=PWD)
        remotetarget.get(local_filepath)

        # make sure that it can open file
        with remotetarget.open('r') as fin:
            self.assertEqual(fin.read(), "something to fill")

        # file is successfuly created
        self.assertTrue(os.path.exists(local_filepath))

        # test RemoteTarget with mtime
        ts = datetime.datetime.now() - datetime.timedelta(minutes=2)
        delayed_remotetarget = RemoteTarget(remote_file,
                                            HOST,
                                            username=USER,
                                            password=PWD,
                                            mtime=ts)
        self.assertTrue(delayed_remotetarget.exists())

        ts = datetime.datetime.now() + datetime.timedelta(
            days=2)  # who knows what timezone it is in
        delayed_remotetarget = RemoteTarget(remote_file,
                                            HOST,
                                            username=USER,
                                            password=PWD,
                                            mtime=ts)
        self.assertFalse(delayed_remotetarget.exists())

        # clean
        os.remove(local_filepath)
        os.remove(tmp_filepath)
        ftp = ftplib.FTP(HOST, USER, PWD)
        ftp.delete(remote_file)
        ftp.cwd("/")
        ftp.rmd("test")
        ftp.close()