def test_check_hdfs_path_dest_file(self): """ unit testing that check_hdfs_path method returns correct value when file does not exist in hdfs. """ self.opts.update({'destination': VALID_DEST_FILE}) t_hadoopcluster = HadoopCluster(self.opts) status = t_hadoopcluster.check_hdfs_path(self.master_IP, self.opts['destination'], '-e') self.assertNotEqual(status, 0)
def test_check_hdfs_path_dest_dir(self): """ unit testing that check_hdfs_path method returns correct value when directory does exist in hdfs. """ self.opts.update({'destination': VALID_DEST_DIR}) t_hadoopcluster = HadoopCluster(self.opts) status = t_hadoopcluster.check_hdfs_path(self.master_IP, self.opts['destination'], '-d') self.assertEqual(status, 0)
def test_check_hdfs_path_dest_dir(self): """ unit testing that check_hdfs_path method returns correct value when directory does exist in hdfs. """ self.opts.update({'destination': self.VALID_DEST_DIR}) t_hadoopcluster = HadoopCluster(self.opts) status = t_hadoopcluster.check_hdfs_path(self.master_IP, self.opts['destination'], '-d') self.assertEqual(status, 0)
def test_put_from_local_recursive(self): """ functional test to put files inside a folder from local to hdfs and check all the files now exist in hdfs and is not zero size. """ list_of_files = [] for i in range(10): subprocess.call('echo "this is the unit test file {0} for local to hdfs orka-cli put." > {0}{1}'.format(i, SOURCE_LOCAL_TO_HDFS_FILE), stderr=FNULL, shell=True) list_of_files.append('{0}{1}'.format(i, SOURCE_LOCAL_TO_HDFS_FILE)) list_of_files.append('/user/hduser') list_of_files.remove('0{0}'.format(SOURCE_LOCAL_TO_HDFS_FILE)) self.opts.update({'source': '0{0}'.format(SOURCE_LOCAL_TO_HDFS_FILE), 'destination': list_of_files, 'fileput': True}) HadoopCluster(self.opts).file_action() list_of_files.insert(0, '0{0}'.format(SOURCE_LOCAL_TO_HDFS_FILE)) for file in list_of_files[:-1]: exist_check_status = ssh_call_hadoop(self.user, self.master_IP, ' dfs -test -e {0}'.format(file), hadoop_path=self.hdfs_path) zero_check_status = ssh_call_hadoop(self.user, self.master_IP, ' dfs -test -z {0}'.format(file), hadoop_path=self.hdfs_path) self.assertEqual(exist_check_status, 0) and self.assertEqual(zero_check_status, 1) self.addCleanup(self.delete_hdfs_files, '/user/hduser/{0}'.format(file)) self.addCleanup(self.delete_local_files, file)
def test_error_source_file(self): """ unit testing if correct exception is raised when input file is invalid. """ self.opts.update({'source': INVALID_SOURCE_FILE}) t_hadoopcluster = HadoopCluster(self.opts) self.assertRaises(IOError, t_hadoopcluster.put_from_local, self.active_cluster)
def test_check_hdfs_path_dest_error_dir(self): """ unit testing that check_hdfs_path method raises correct exception when directory does exist in hdfs. """ self.opts.update({'destination': INVALID_DEST_DIR}) t_hadoopcluster = HadoopCluster(self.opts) self.assertRaises(SystemExit, t_hadoopcluster.check_hdfs_path, self.master_IP, self.opts['destination'], '-d')
def test_file_size_put_from_local(self): """ Testing, using mock, if the put from Local to Hdfs method raises correct exception when file is bigger than available space. """ subprocess.call('echo "this is a unit test file for local to hdfs orka-cli put." > {0}'.format(SOURCE_LOCAL_TO_HDFS_FILE), stderr=FNULL, shell=True) self.opts.update({'source': SOURCE_LOCAL_TO_HDFS_FILE, 'destination': DEST_LOCAL_TO_HDFS_FILE}) t_hadoopcluster = HadoopCluster(self.opts) self.assertRaises(SystemExit, t_hadoopcluster.put_from_local, self.active_cluster) self.addCleanup(self.delete_local_files, self.opts['source'])
def test_get_from_hdfs_to_local(self): """ functional test to get file from Hdfs and check that file now exists in local filesystem. """ self.put_file_to_hdfs('/tmp/{0}'.format(SOURCE_HDFS_TO_LOCAL_FILE)) self.opts.update({'source': SOURCE_HDFS_TO_LOCAL_FILE, 'destination': DEST_HDFS_TO_LOCAL_FILE}) HadoopCluster(self.opts).get_from_hadoop_to_local(self.active_cluster) exist_check_status = os.system('ls {0}'.format(self.opts['destination'])) self.assertEqual(exist_check_status, 0) self.addCleanup(self.delete_hdfs_files, self.opts['source']) self.addCleanup(self.delete_local_files, self.opts['destination']) self.addCleanup(self.hadoop_local_fs_action, 'rm /tmp/{0}'.format(SOURCE_HDFS_TO_LOCAL_FILE))
def test_get_from_hdfs_to_pithos(self): """ functional test to get a test file from hdfs to pithos and check that file exists. """ self.put_file_to_hdfs('/tmp/{0}'.format(SOURCE_HDFS_TO_PITHOS_FILE)) self.opts.update({'source': SOURCE_HDFS_TO_PITHOS_FILE, 'destination': DEST_HDFS_TO_PITHOS_FILE}) HadoopCluster(self.opts).get_from_hadoop_to_pithos(self.active_cluster, self.opts['destination']) exist_check_status = os.system('kamaki file list | grep {0}'.format(self.opts['destination'])) self.assertEqual(exist_check_status, 0) self.addCleanup(self.delete_pithos_files, self.opts['destination']) self.addCleanup(self.delete_hdfs_files, self.opts['source']) self.addCleanup(self.hadoop_local_fs_action, 'rm /tmp/{0}'.format(SOURCE_HDFS_TO_PITHOS_FILE))
def test_put_from_remote(self): """ functional test to put file from remote server to Hdfs and check that file now exists in Hdfs and is not zero size. """ self.opts.update({'source': SOURCE_REMOTE_TO_HDFS_FILE, 'destination': DEST_REMOTE_TO_HDFS_FILE, 'user': '', 'password': ''}) HadoopCluster(self.opts).put_from_server() exist_check_status = ssh_call_hadoop(self.user, self.master_IP, ' dfs -test -e {0}'.format(self.opts['destination']), hadoop_path=self.hdfs_path) zero_check_status = ssh_call_hadoop(self.user, self.master_IP, ' dfs -test -z {0}'.format(self.opts['destination']), hadoop_path=self.hdfs_path) self.assertEqual(exist_check_status, 0) and self.assertEqual(zero_check_status, 1) self.addCleanup(self.delete_hdfs_files, self.opts['destination'])
def test_put_from_local(self): """ functional test to put file from local to hdfs and check that file now exists in hdfs and is not zero size. """ subprocess.call('echo "this is a unit test file for local to hdfs orka-cli put." > {0}'.format(SOURCE_LOCAL_TO_HDFS_FILE), stderr=FNULL, shell=True) self.opts.update({'source': SOURCE_LOCAL_TO_HDFS_FILE, 'destination': [DEST_LOCAL_TO_HDFS_FILE], 'fileput': True}) HadoopCluster(self.opts).file_action() exist_check_status = ssh_call_hadoop(self.user, self.master_IP, ' dfs -test -e {0}'.format(DEST_LOCAL_TO_HDFS_FILE), hadoop_path=self.hdfs_path) zero_check_status = ssh_call_hadoop(self.user, self.master_IP, ' dfs -test -z {0}'.format(DEST_LOCAL_TO_HDFS_FILE), hadoop_path=self.hdfs_path) self.assertEqual(exist_check_status, 0) and self.assertEqual(zero_check_status, 1) self.addCleanup(self.delete_hdfs_files, DEST_LOCAL_TO_HDFS_FILE) self.addCleanup(self.delete_local_files, self.opts['source'])
def test_put_from_pithos(self): """ functional test to put file from Pithos to Hdfs and check that file now exists in Hdfs and is not zero size. """ subprocess.call('echo "this is a test file for pithos to hdfs orka-cli put" > {0}'.format(SOURCE_PITHOS_TO_HDFS_FILE), stderr=FNULL, shell=True) subprocess.call('kamaki file upload {0}'.format(SOURCE_PITHOS_TO_HDFS_FILE), stderr=FNULL, shell=True) self.opts.update({'destination': DEST_PITHOS_TO_HDFS_FILE}) HadoopCluster(self.opts).put_from_pithos(self.active_cluster, SOURCE_PITHOS_TO_HDFS_FILE) exist_check_status = ssh_call_hadoop(self.user, self.master_IP, ' dfs -test -e {0}'.format(self.opts['destination']), hadoop_path=self.hdfs_path) zero_check_status = ssh_call_hadoop(self.user, self.master_IP, ' dfs -test -z {0}'.format(self.opts['destination']), hadoop_path=self.hdfs_path) self.assertEqual(exist_check_status, 0) and self.assertEqual(zero_check_status, 1) self.addCleanup(self.delete_hdfs_files, self.opts['destination']) self.addCleanup(self.delete_pithos_files, SOURCE_PITHOS_TO_HDFS_FILE) self.addCleanup(self.delete_local_files, SOURCE_PITHOS_TO_HDFS_FILE)