def setupClass(cls):
if not cls.cluster:
testfiles_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "testfiles")
cls.cluster = MiniCluster(testfiles_path)
cls.cluster.put("/test1", "/test1")
cls.cluster.put("/test1", "/test2")
cls.cluster.put("/test3", "/test3") #1024 bytes
cls.cluster.mkdir("/zipped")
cls.cluster.put("/zipped/test1.gz", "/zipped")
cls.cluster.put("/zipped/test1.bz2", "/zipped")
cls.cluster.put("/zerofile", "/")
cls.cluster.mkdir("/dir1")
cls.cluster.put("/zerofile", "/dir1")
cls.cluster.mkdir("/dir2")
cls.cluster.mkdir("/dir2/dir3")
cls.cluster.put("/test1", "/dir2/dir3")
cls.cluster.put("/test3", "/dir2/dir3")
cls.cluster.mkdir("/foo/bar/baz", ['-p'])
cls.cluster.put("/zerofile", "/foo/bar/baz/qux")
cls.cluster.mkdir("/bar/baz/foo", ['-p'])
cls.cluster.put("/zerofile", "/bar/baz/foo/qux")
cls.cluster.mkdir("/bar/foo/baz", ['-p'])
cls.cluster.put("/zerofile", "/bar/foo/baz/qux")
cls.cluster.put("/log", "/")
def setupClass(cls):
if not cls.cluster:
# Prevent running tests if a hadoop cluster is reachable. This guard
# is in place because the MiniCluster java class can break things on
# a production cluster. The MiniCluster python class is used, but doesn't
# start an actual cluster. We only use convenience methods to call java
# hadoop.
c = MiniCluster(None, start_cluster=False)
result = c.ls("/")
if result:
raise Exception(
"An active Hadoop cluster is found! Not running tests!")
testfiles_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "testfiles")
cls.cluster = MiniCluster(testfiles_path)
cls.cluster.put("/test1", "/test1")
cls.cluster.put("/test1", "/test2")
cls.cluster.put("/test3", "/test3") #1024 bytes
cls.cluster.put("/test1", "/test4")
cls.cluster.mkdir("/zipped")
cls.cluster.put("/zipped/test1.gz", "/zipped")
cls.cluster.put("/zipped/test1.bz2", "/zipped")
cls.cluster.put("/zerofile", "/")
cls.cluster.mkdir("/dir1")
cls.cluster.put("/zerofile", "/dir1")
cls.cluster.mkdir("/dir2")
cls.cluster.mkdir("/dir2/dir3")
cls.cluster.put("/test1", "/dir2/dir3")
cls.cluster.put("/test3", "/dir2/dir3")
cls.cluster.mkdir("/foo/bar/baz", ['-p'])
cls.cluster.put("/zerofile", "/foo/bar/baz/qux")
cls.cluster.mkdir("/bar/baz/foo", ['-p'])
cls.cluster.put("/zerofile", "/bar/baz/foo/qux")
cls.cluster.mkdir("/bar/foo/baz", ['-p'])
cls.cluster.put("/zerofile", "/bar/foo/baz/qux")
cls.cluster.put("/log", "/")
cls.cluster.mkdir("/sticky_dir")
def setUpClass(cls):
if 'HADOOP_HOME' not in os.environ or not os.environ['HADOOP_HOME']:
raise SkipTest("Hadoop not installed")
if not cls.cluster:
c = MiniCluster(None, start_cluster=False)
result = c.ls("/")
if result:
raise Exception("An active Hadoop cluster is found! Not running tests!")
cls.cluster = MiniCluster(cls.testfiles_path)
result = cls.cluster.ls("/")
if result:
raise Exception("An active Hadoop cluster is found! Not running tests!")
cls.cluster.put("/A", "/A")
cls.cluster.put("/A", "/file_to_be_deleted")
cls.cluster.mkdir("/dir")
cls.cluster.mkdir("/dir_to_be_deleted")
cls.cluster.put("/A", "/dir_to_be_deleted/inside_dir")
def setupClass(cls):
if not cls.cluster:
# Prevent running tests if a hadoop cluster is reachable. This guard
# is in place because the MiniCluster java class can break things on
# a production cluster. The MiniCluster python class is used, but doesn't
# start an actual cluster. We only use convenience methods to call java
# hadoop.
c = MiniCluster(None, start_cluster=False)
result = c.ls("/")
if result:
raise Exception("An active Hadoop cluster is found! Not running tests!")
testfiles_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "testfiles")
cls.cluster = MiniCluster(testfiles_path)
cls.cluster.put("/test1", "/test1")
cls.cluster.put("/test1", "/test2")
cls.cluster.put("/test3", "/test3") #1024 bytes
cls.cluster.put("/test1", "/test4")
cls.cluster.mkdir("/zipped")
cls.cluster.put("/zipped/test1.gz", "/zipped")
cls.cluster.put("/zipped/test1.bz2", "/zipped")
cls.cluster.put("/zerofile", "/")
cls.cluster.mkdir("/dir1")
cls.cluster.put("/zerofile", "/dir1")
cls.cluster.mkdir("/dir2")
cls.cluster.mkdir("/dir2/dir3")
cls.cluster.put("/test1", "/dir2/dir3")
cls.cluster.put("/test3", "/dir2/dir3")
cls.cluster.mkdir("/foo/bar/baz", ['-p'])
cls.cluster.put("/zerofile", "/foo/bar/baz/qux")
cls.cluster.mkdir("/bar/baz/foo", ['-p'])
cls.cluster.put("/zerofile", "/bar/baz/foo/qux")
cls.cluster.mkdir("/bar/foo/baz", ['-p'])
cls.cluster.put("/zerofile", "/bar/foo/baz/qux")
cls.cluster.put("/log", "/")
cls.cluster.mkdir("/sticky_dir")
def setupClass(cls):
if not cls.cluster:
cls.cluster = MiniCluster(None, nnport=50030)
cls.cluster.mkdir("/tmp")
def test_explicit_port(self):
c = MiniCluster(None, nnport=50050)
self.assertEqual(50050, c.port)
c.terminate()
def instantiate_cluster(cls):
return MiniCluster(None, nnport=50030)
def setupClass(cls):
if not cls.cluster:
cls.cluster = MiniCluster(None, nnport=50030)
def test_explicit_port(self):
c = MiniCluster(None, nnport=50050)
self.assertEqual(50050, c.port)
c.terminate()
def HDFS_Upload(uploadfile):
if uploadfile:
connect = MiniCluster(None, start_cluster=False)
result = connect.ls("/")
cluster = MiniCluster("/")
if result:
#raise Exception("An active Hadoop cluster is found! Not running tests!")
if cluster.exists(uploadfile):
hadoop_home = os.getenv('HADOOP_HOME')
if hadoop_home is None:
print "Can't find hadoop path!!!"
cluster.terminate()
sys.exit()
upload_command = hadoop_home + "/bin/hadoop" + " dfs -put " + filename + " " + uploadfile
#print upload_command
os.system(upload_command)
else:
cluster.terminate()
print "No path in HDFS"
else:
print "Hadoop is not running"
cluster.terminate()
else:
sys.exit()
