Esempio n. 1
0
    def setupClass(cls):
        if not cls.cluster:
            testfiles_path = os.path.join(
                os.path.dirname(os.path.abspath(__file__)), "testfiles")
            cls.cluster = MiniCluster(testfiles_path)
            cls.cluster.put("/test1", "/test1")
            cls.cluster.put("/test1", "/test2")
            cls.cluster.put("/test3", "/test3")  #1024 bytes

            cls.cluster.mkdir("/zipped")
            cls.cluster.put("/zipped/test1.gz", "/zipped")
            cls.cluster.put("/zipped/test1.bz2", "/zipped")

            cls.cluster.put("/zerofile", "/")

            cls.cluster.mkdir("/dir1")
            cls.cluster.put("/zerofile", "/dir1")
            cls.cluster.mkdir("/dir2")
            cls.cluster.mkdir("/dir2/dir3")
            cls.cluster.put("/test1", "/dir2/dir3")
            cls.cluster.put("/test3", "/dir2/dir3")

            cls.cluster.mkdir("/foo/bar/baz", ['-p'])
            cls.cluster.put("/zerofile", "/foo/bar/baz/qux")
            cls.cluster.mkdir("/bar/baz/foo", ['-p'])
            cls.cluster.put("/zerofile", "/bar/baz/foo/qux")
            cls.cluster.mkdir("/bar/foo/baz", ['-p'])
            cls.cluster.put("/zerofile", "/bar/foo/baz/qux")
            cls.cluster.put("/log", "/")
Esempio n. 2
0
    def setupClass(cls):
        if not cls.cluster:
            # Prevent running tests if a hadoop cluster is reachable. This guard
            # is in place because the MiniCluster java class can break things on
            # a production cluster. The MiniCluster python class is used, but doesn't
            # start an actual cluster. We only use convenience methods to call java
            # hadoop.

            c = MiniCluster(None, start_cluster=False)
            result = c.ls("/")
            if result:
                raise Exception(
                    "An active Hadoop cluster is found! Not running tests!")

            testfiles_path = os.path.join(
                os.path.dirname(os.path.abspath(__file__)), "testfiles")
            cls.cluster = MiniCluster(testfiles_path)
            cls.cluster.put("/test1", "/test1")
            cls.cluster.put("/test1", "/test2")
            cls.cluster.put("/test3", "/test3")  #1024 bytes
            cls.cluster.put("/test1", "/test4")

            cls.cluster.mkdir("/zipped")
            cls.cluster.put("/zipped/test1.gz", "/zipped")
            cls.cluster.put("/zipped/test1.bz2", "/zipped")

            cls.cluster.put("/zerofile", "/")

            cls.cluster.mkdir("/dir1")
            cls.cluster.put("/zerofile", "/dir1")
            cls.cluster.mkdir("/dir2")
            cls.cluster.mkdir("/dir2/dir3")
            cls.cluster.put("/test1", "/dir2/dir3")
            cls.cluster.put("/test3", "/dir2/dir3")

            cls.cluster.mkdir("/foo/bar/baz", ['-p'])
            cls.cluster.put("/zerofile", "/foo/bar/baz/qux")
            cls.cluster.mkdir("/bar/baz/foo", ['-p'])
            cls.cluster.put("/zerofile", "/bar/baz/foo/qux")
            cls.cluster.mkdir("/bar/foo/baz", ['-p'])
            cls.cluster.put("/zerofile", "/bar/foo/baz/qux")
            cls.cluster.put("/log", "/")

            cls.cluster.mkdir("/sticky_dir")
Esempio n. 3
0
    def setUpClass(cls):
        if 'HADOOP_HOME' not in os.environ or not os.environ['HADOOP_HOME']:
            raise SkipTest("Hadoop not installed")

        if not cls.cluster:
            c = MiniCluster(None, start_cluster=False)
            result = c.ls("/")
            if result:
                raise Exception("An active Hadoop cluster is found! Not running tests!")

            cls.cluster = MiniCluster(cls.testfiles_path)
            result = cls.cluster.ls("/")
            if result:
                raise Exception("An active Hadoop cluster is found! Not running tests!")
            cls.cluster.put("/A", "/A")
            cls.cluster.put("/A", "/file_to_be_deleted")
            cls.cluster.mkdir("/dir")
            cls.cluster.mkdir("/dir_to_be_deleted")
            cls.cluster.put("/A", "/dir_to_be_deleted/inside_dir")
Esempio n. 4
0
    def setupClass(cls):
        if not cls.cluster:
            # Prevent running tests if a hadoop cluster is reachable. This guard
            # is in place because the MiniCluster java class can break things on
            # a production cluster. The MiniCluster python class is used, but doesn't
            # start an actual cluster. We only use convenience methods to call java
            # hadoop.

            c = MiniCluster(None, start_cluster=False)
            result = c.ls("/")
            if result:
                raise Exception("An active Hadoop cluster is found! Not running tests!")

            testfiles_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "testfiles")
            cls.cluster = MiniCluster(testfiles_path)
            cls.cluster.put("/test1", "/test1")
            cls.cluster.put("/test1", "/test2")
            cls.cluster.put("/test3", "/test3") #1024 bytes
            cls.cluster.put("/test1", "/test4")

            cls.cluster.mkdir("/zipped")
            cls.cluster.put("/zipped/test1.gz", "/zipped")
            cls.cluster.put("/zipped/test1.bz2", "/zipped")

            cls.cluster.put("/zerofile", "/")

            cls.cluster.mkdir("/dir1")
            cls.cluster.put("/zerofile", "/dir1")
            cls.cluster.mkdir("/dir2")
            cls.cluster.mkdir("/dir2/dir3")
            cls.cluster.put("/test1", "/dir2/dir3")
            cls.cluster.put("/test3", "/dir2/dir3")

            cls.cluster.mkdir("/foo/bar/baz", ['-p'])
            cls.cluster.put("/zerofile", "/foo/bar/baz/qux")
            cls.cluster.mkdir("/bar/baz/foo", ['-p'])
            cls.cluster.put("/zerofile", "/bar/baz/foo/qux")
            cls.cluster.mkdir("/bar/foo/baz", ['-p'])
            cls.cluster.put("/zerofile", "/bar/foo/baz/qux")
            cls.cluster.put("/log", "/")

            cls.cluster.mkdir("/sticky_dir")
Esempio n. 5
0
 def setupClass(cls):
     if not cls.cluster:
         cls.cluster = MiniCluster(None, nnport=50030)
     cls.cluster.mkdir("/tmp")
Esempio n. 6
0
 def test_explicit_port(self):
     c = MiniCluster(None, nnport=50050)
     self.assertEqual(50050, c.port)
     c.terminate()
Esempio n. 7
0
 def instantiate_cluster(cls):
     return MiniCluster(None, nnport=50030)
Esempio n. 8
0
 def setupClass(cls):
     if not cls.cluster:
         cls.cluster = MiniCluster(None, nnport=50030)
 def test_explicit_port(self):
     c = MiniCluster(None, nnport=50050)
     self.assertEqual(50050, c.port)
     c.terminate()
def HDFS_Upload(uploadfile):
    if uploadfile:
        connect = MiniCluster(None, start_cluster=False)  
        result = connect.ls("/")
        cluster = MiniCluster("/")
        if result:
            #raise Exception("An active Hadoop cluster is found! Not running tests!")
            if cluster.exists(uploadfile):
                hadoop_home = os.getenv('HADOOP_HOME')
                if hadoop_home is None:
                    print "Can't find hadoop path!!!"
                    cluster.terminate()
                    sys.exit()
                upload_command =  hadoop_home + "/bin/hadoop" + " dfs -put " + filename + " " + uploadfile
                #print upload_command
                os.system(upload_command)
            else:
                cluster.terminate()
                print "No path in HDFS"
        else:
            print "Hadoop is not running"
        cluster.terminate()
    else:
        sys.exit()