Ejemplo n.º 1
0
 def test_delete_dir(self):
     local = LocalFS(os.path.dirname(os.path.realpath(__file__)))
     hdfs_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
     local.copy_to_hdfs(hdfs_file.path)
     self.assertTrue(hdfs_file.exists(), "Target HDFS dir does not exists")
     hdfs_file.delete(recursive=True)
     self.assertFalse(hdfs_file.exists(), "Target HDFS dir was not deleted")
Ejemplo n.º 2
0
 def test_delete_dir(self):
     local = LocalFS(os.path.dirname(os.path.realpath(__file__)))
     hdfs_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
     local.copy_to_hdfs(hdfs_file.path)
     self.assertTrue(hdfs_file.exists(), "Target HDFS dir does not exists")
     hdfs_file.delete(recursive=True)
     self.assertFalse(hdfs_file.exists(), "Target HDFS dir was not deleted")
Ejemplo n.º 3
0
 def test_file_size(self):
     local = LocalFS(os.path.realpath(__file__))
     hdfs_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
     try:
         local.copy_to_hdfs(hdfs_file.path)
         self.assertTrue(hdfs_file.exists(), "Local file was not copied to HDFS")
         self.assertEqual(hdfs_file.size(), local.size())
     finally:
         hdfs_file.delete()
Ejemplo n.º 4
0
 def test_file_size(self):
     local = LocalFS(os.path.realpath(__file__))
     hdfs_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
     try:
         local.copy_to_hdfs(hdfs_file.path)
         self.assertTrue(hdfs_file.exists(), "Local file was not copied to HDFS")
         self.assertEqual(hdfs_file.size(), local.size())
     finally:
         hdfs_file.delete()
Ejemplo n.º 5
0
 def test_dir_size(self):
     local_basedir = os.path.dirname(os.path.realpath(__file__))
     local = LocalFS(os.path.join(local_basedir, "resources", "test_dir_size"))
     hdfs_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
     try:
         local.copy_to_hdfs(hdfs_file.path)
         self.assertTrue(hdfs_file.exists(), "Local file was not copied to HDFS")
         expected_fsize = local.size()
         self.assertEqual(hdfs_file.size(), expected_fsize)
     finally:
         hdfs_file.delete(recursive=True)
Ejemplo n.º 6
0
 def test_dir_size(self):
     local_basedir = os.path.dirname(os.path.realpath(__file__))
     local = LocalFS(os.path.join(local_basedir, "resources", "test_dir_size"))
     hdfs_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
     try:
         local.copy_to_hdfs(hdfs_file.path)
         self.assertTrue(hdfs_file.exists(), "Local file was not copied to HDFS")
         expected_fsize = local.size()
         self.assertEqual(hdfs_file.size(), expected_fsize)
     finally:
         hdfs_file.delete(recursive=True)
Ejemplo n.º 7
0
 def test_merge(self):
     basedir = os.path.dirname(os.path.realpath(__file__))
     local = LocalFS(os.path.join(basedir, "resources", "test_merge"))
     hdfs_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
     merged_file = LocalFS(os.path.join(basedir, "resources", "merged.txt"))
     try:
         local.copy_to_hdfs(hdfs_file.path)
         self.assertTrue(hdfs_file.exists(), "Local file was not copied to HDFS")
         hdfs_file.merge(merged_file.path)
         self.assertTrue(merged_file.exists(), "merged file was not copied to local fs")
     finally:
         hdfs_file.delete_directory()
         merged_file.delete()
Ejemplo n.º 8
0
 def test_merge(self):
     basedir = os.path.dirname(os.path.realpath(__file__))
     local = LocalFS(os.path.join(basedir, "resources", "test_merge"))
     hdfs_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
     merged_file = LocalFS(os.path.join(basedir, "resources", "merged.txt"))
     try:
         local.copy_to_hdfs(hdfs_file.path)
         self.assertTrue(hdfs_file.exists(), "Local file was not copied to HDFS")
         hdfs_file.merge(merged_file.path)
         self.assertTrue(merged_file.exists(), "merged file was not copied to local fs")
     finally:
         hdfs_file.delete_directory()
         merged_file.delete()
Ejemplo n.º 9
0
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# See the NOTICE file and the LICENSE file distributed with this work
# for additional information regarding copyright ownership and licensing.
#

import os

from merlin.fs.hdfs import HDFS
from merlin.fs.localfs import LocalFS

if __name__ == "__main__":
    _basedir = HDFS(os.path.join('/tmp', 'scd.active'))

    _basedir.create_directory()
    _scd_active_snapshot = LocalFS(
        os.path.join(os.path.dirname(__file__), 'resources', 'scd.active.csv'))
    _scd_active_snapshot.copy_to_hdfs(_basedir.path)
Ejemplo n.º 10
0
Archivo: setup.py Proyecto: epam/Merlin
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# See the NOTICE file and the LICENSE file distributed with this work
# for additional information regarding copyright ownership and licensing.
#

import os

from merlin.fs.hdfs import HDFS
from merlin.fs.localfs import LocalFS


if __name__ == "__main__":
    _basedir = HDFS(os.path.join('/tmp', 'scd.active'))

    _basedir.create_directory()
    _scd_active_snapshot = LocalFS(os.path.join(os.path.dirname(__file__), 'resources', 'scd.active.csv'))
    _scd_active_snapshot.copy_to_hdfs(_basedir.path)

Ejemplo n.º 11
0
    ftp.create(make_dir=True)

    # upload file to directory on FTP
    ftp.upload(local_path=os.path.join(os.path.dirname(__file__),
                                       "resources/file_12.11.2014_.txt"))
    ftp.upload(local_path=os.path.join(os.path.dirname(__file__),
                                       "resources/file_13.11.2014_.txt"))
    ftp.upload(local_path=os.path.join(os.path.dirname(__file__),
                                       "resources/file_14.11.2014_.txt"))

    # upload file to HDFS/ create directories
    hdfs_file = HDFS("{0}/raw/12.11.2014".format(BASE_DIR))
    hdfs_file.create(directory=True)
    local_file = LocalFS(path=os.path.join(os.path.dirname(__file__),
                                           'resources/file_12.11.2014_.txt'))
    local_file.copy_to_hdfs(hdfs_path="{0}/raw/12.11.2014".format(BASE_DIR))

    hdfs_file = HDFS("{0}/raw/13.11.2014".format(BASE_DIR))
    hdfs_file.create(directory=True)
    local_file = LocalFS(path=os.path.join(os.path.dirname(__file__),
                                           'resources/file_13.11.2014_.txt'))
    local_file.copy_to_hdfs(hdfs_path="{0}/raw/13.11.2014".format(BASE_DIR))

    # create empty local directory 'tmp' in folder 'resources'
    local_file = LocalFS(
        path=os.path.join(os.path.dirname(__file__), 'resources/tmp'))
    if local_file.exists():
        local_file.delete_directory()
    local_file.create(directory=True)

    # create HIVE external table with partition
Ejemplo n.º 12
0
Archivo: setup.py Proyecto: epam/Merlin
    if ftp.exists():
        ftp.delete(recursive=True)
    ftp.create(make_dir=True)

    # upload file to directory on FTP
    ftp.upload(local_path=os.path.join(os.path.dirname(__file__), "resources/file_12.11.2014_.txt"))
    ftp.upload(local_path=os.path.join(os.path.dirname(__file__), "resources/file_13.11.2014_.txt"))
    ftp.upload(local_path=os.path.join(os.path.dirname(__file__), "resources/file_14.11.2014_.txt"))

    # upload file to HDFS/ create directories
    hdfs_file = HDFS("{0}/raw/12.11.2014".format(BASE_DIR))
    hdfs_file.create(directory=True)
    local_file = LocalFS(path=os.path.join(os.path.dirname(__file__),
                                           'resources/file_12.11.2014_.txt'))
    local_file.copy_to_hdfs(hdfs_path="{0}/raw/12.11.2014".format(BASE_DIR))

    hdfs_file = HDFS("{0}/raw/13.11.2014".format(BASE_DIR))
    hdfs_file.create(directory=True)
    local_file = LocalFS(path=os.path.join(os.path.dirname(__file__),
                                           'resources/file_13.11.2014_.txt'))
    local_file.copy_to_hdfs(hdfs_path="{0}/raw/13.11.2014".format(BASE_DIR))

    # create empty local directory 'tmp' in folder 'resources'
    local_file = LocalFS(path=os.path.join(os.path.dirname(__file__),
                                           'resources/tmp'))
    if local_file.exists():
        local_file.delete_directory()
    local_file.create(directory=True)

    # create HIVE external table with partition