def test_check_hdfs_path_dest_file(self):
     """
     unit testing that check_hdfs_path method returns correct value when file does not exist in hdfs.
     """
     self.opts.update({'destination': VALID_DEST_FILE})
     t_hadoopcluster = HadoopCluster(self.opts)
     status = t_hadoopcluster.check_hdfs_path(self.master_IP, self.opts['destination'], '-e')
     self.assertNotEqual(status, 0)
 def test_check_hdfs_path_dest_dir(self):
     """
     unit testing that check_hdfs_path method returns correct value when directory does exist in hdfs.
     """
     self.opts.update({'destination': VALID_DEST_DIR})
     t_hadoopcluster = HadoopCluster(self.opts)
     status = t_hadoopcluster.check_hdfs_path(self.master_IP, self.opts['destination'], '-d')
     self.assertEqual(status, 0)
 def test_check_hdfs_path_dest_file(self):
     """
     unit testing that check_hdfs_path method returns correct value when file does not exist in hdfs.
     """
     self.opts.update({'destination': VALID_DEST_FILE})
     t_hadoopcluster = HadoopCluster(self.opts)
     status = t_hadoopcluster.check_hdfs_path(self.master_IP, self.opts['destination'], '-e')
     self.assertNotEqual(status, 0)
 def test_check_hdfs_path_dest_dir(self):
     """
     unit testing that check_hdfs_path method returns correct value when directory does exist in hdfs.
     """
     self.opts.update({'destination': self.VALID_DEST_DIR})
     t_hadoopcluster = HadoopCluster(self.opts)
     status = t_hadoopcluster.check_hdfs_path(self.master_IP, self.opts['destination'], '-d')
     self.assertEqual(status, 0)
 def test_put_from_local_recursive(self):
     """
     functional test to put files inside a folder from local to hdfs and check all the files now exist in hdfs and is not zero size.
     """
     list_of_files = []
     for i in range(10):
         subprocess.call('echo "this is the unit test file {0} for local to hdfs orka-cli put." > {0}{1}'.format(i, SOURCE_LOCAL_TO_HDFS_FILE),
                         stderr=FNULL, shell=True)
         list_of_files.append('{0}{1}'.format(i, SOURCE_LOCAL_TO_HDFS_FILE))
     list_of_files.append('/user/hduser')
     list_of_files.remove('0{0}'.format(SOURCE_LOCAL_TO_HDFS_FILE))
     self.opts.update({'source': '0{0}'.format(SOURCE_LOCAL_TO_HDFS_FILE), 'destination': list_of_files,
                      'fileput': True})
     HadoopCluster(self.opts).file_action()
     list_of_files.insert(0, '0{0}'.format(SOURCE_LOCAL_TO_HDFS_FILE))
     for file in list_of_files[:-1]:
         exist_check_status = ssh_call_hadoop(self.user, self.master_IP,
                                              ' dfs -test -e {0}'.format(file),
                                              hadoop_path=self.hdfs_path)
         zero_check_status = ssh_call_hadoop(self.user, self.master_IP,
                                             ' dfs -test -z {0}'.format(file),
                                             hadoop_path=self.hdfs_path)
         self.assertEqual(exist_check_status, 0) and self.assertEqual(zero_check_status, 1)
         self.addCleanup(self.delete_hdfs_files, '/user/hduser/{0}'.format(file))
         self.addCleanup(self.delete_local_files, file)
 def test_error_source_file(self):
     """
     unit testing if correct exception is raised when input file is invalid.
     """
     self.opts.update({'source': INVALID_SOURCE_FILE})
     t_hadoopcluster = HadoopCluster(self.opts)
     self.assertRaises(IOError, t_hadoopcluster.put_from_local, self.active_cluster)
 def test_check_hdfs_path_dest_error_dir(self):
     """
     unit testing that check_hdfs_path method raises correct exception when directory does exist in hdfs.
     """
     self.opts.update({'destination': INVALID_DEST_DIR})
     t_hadoopcluster = HadoopCluster(self.opts)
     self.assertRaises(SystemExit, t_hadoopcluster.check_hdfs_path, self.master_IP, self.opts['destination'], '-d')
 def test_file_size_put_from_local(self):
     """
     Testing, using mock, if the put from Local to Hdfs method raises correct exception when file is bigger than
     available space.
     """
     subprocess.call('echo "this is a unit test file for local to hdfs orka-cli put." > {0}'.format(SOURCE_LOCAL_TO_HDFS_FILE),
                     stderr=FNULL, shell=True)
     self.opts.update({'source': SOURCE_LOCAL_TO_HDFS_FILE, 'destination': DEST_LOCAL_TO_HDFS_FILE})
     t_hadoopcluster = HadoopCluster(self.opts)
     self.assertRaises(SystemExit, t_hadoopcluster.put_from_local, self.active_cluster)
     self.addCleanup(self.delete_local_files, self.opts['source'])
 def test_get_from_hdfs_to_local(self):
     """
     functional test to get file from Hdfs and check that file now exists in local filesystem.
     """
     self.put_file_to_hdfs('/tmp/{0}'.format(SOURCE_HDFS_TO_LOCAL_FILE))
     self.opts.update({'source': SOURCE_HDFS_TO_LOCAL_FILE, 'destination': DEST_HDFS_TO_LOCAL_FILE})
     HadoopCluster(self.opts).get_from_hadoop_to_local(self.active_cluster)
     exist_check_status = os.system('ls {0}'.format(self.opts['destination']))
     self.assertEqual(exist_check_status, 0)
     self.addCleanup(self.delete_hdfs_files, self.opts['source'])
     self.addCleanup(self.delete_local_files, self.opts['destination'])
     self.addCleanup(self.hadoop_local_fs_action, 'rm /tmp/{0}'.format(SOURCE_HDFS_TO_LOCAL_FILE))
 def test_get_from_hdfs_to_pithos(self):
     """
     functional test to get a test file from hdfs to pithos and check that file exists.
     """
     self.put_file_to_hdfs('/tmp/{0}'.format(SOURCE_HDFS_TO_PITHOS_FILE))
     self.opts.update({'source': SOURCE_HDFS_TO_PITHOS_FILE, 'destination': DEST_HDFS_TO_PITHOS_FILE})
     HadoopCluster(self.opts).get_from_hadoop_to_pithos(self.active_cluster, self.opts['destination'])
     exist_check_status = os.system('kamaki file list | grep {0}'.format(self.opts['destination']))
     self.assertEqual(exist_check_status, 0)
     self.addCleanup(self.delete_pithos_files, self.opts['destination'])
     self.addCleanup(self.delete_hdfs_files, self.opts['source'])
     self.addCleanup(self.hadoop_local_fs_action, 'rm /tmp/{0}'.format(SOURCE_HDFS_TO_PITHOS_FILE))
 def test_put_from_remote(self):
     """
     functional test to put file from remote server to Hdfs and check that file now exists in Hdfs and
     is not zero size.
     """
     self.opts.update({'source': SOURCE_REMOTE_TO_HDFS_FILE, 'destination': DEST_REMOTE_TO_HDFS_FILE, 'user': '',
                       'password': ''})
     HadoopCluster(self.opts).put_from_server()
     exist_check_status = ssh_call_hadoop(self.user, self.master_IP,
                                          ' dfs -test -e {0}'.format(self.opts['destination']),
                                          hadoop_path=self.hdfs_path)
     zero_check_status = ssh_call_hadoop(self.user, self.master_IP,
                                         ' dfs -test -z {0}'.format(self.opts['destination']),
                                         hadoop_path=self.hdfs_path)
     self.assertEqual(exist_check_status, 0) and self.assertEqual(zero_check_status, 1)
     self.addCleanup(self.delete_hdfs_files, self.opts['destination'])
 def test_put_from_local(self):
     """
     functional test to put file from local to hdfs and check that file now exists in hdfs and is not zero size.
     """
     subprocess.call('echo "this is a unit test file for local to hdfs orka-cli put." > {0}'.format(SOURCE_LOCAL_TO_HDFS_FILE),
                     stderr=FNULL, shell=True)
     self.opts.update({'source': SOURCE_LOCAL_TO_HDFS_FILE, 'destination': [DEST_LOCAL_TO_HDFS_FILE],
                      'fileput': True})
     HadoopCluster(self.opts).file_action()
     exist_check_status = ssh_call_hadoop(self.user, self.master_IP,
                                          ' dfs -test -e {0}'.format(DEST_LOCAL_TO_HDFS_FILE),
                                          hadoop_path=self.hdfs_path)
     zero_check_status = ssh_call_hadoop(self.user, self.master_IP,
                                         ' dfs -test -z {0}'.format(DEST_LOCAL_TO_HDFS_FILE),
                                         hadoop_path=self.hdfs_path)
     self.assertEqual(exist_check_status, 0) and self.assertEqual(zero_check_status, 1)
     self.addCleanup(self.delete_hdfs_files, DEST_LOCAL_TO_HDFS_FILE)
     self.addCleanup(self.delete_local_files, self.opts['source'])
 def test_put_from_pithos(self):
     """
     functional test to put file from Pithos to Hdfs and check that file now exists in Hdfs and
     is not zero size.
     """
     subprocess.call('echo "this is a test file for pithos to hdfs orka-cli put" > {0}'.format(SOURCE_PITHOS_TO_HDFS_FILE),
                     stderr=FNULL, shell=True)
     subprocess.call('kamaki file upload {0}'.format(SOURCE_PITHOS_TO_HDFS_FILE), stderr=FNULL, shell=True)
     self.opts.update({'destination': DEST_PITHOS_TO_HDFS_FILE})
     HadoopCluster(self.opts).put_from_pithos(self.active_cluster, SOURCE_PITHOS_TO_HDFS_FILE)
     exist_check_status = ssh_call_hadoop(self.user, self.master_IP,
                                          ' dfs -test -e {0}'.format(self.opts['destination']),
                                          hadoop_path=self.hdfs_path)
     zero_check_status = ssh_call_hadoop(self.user, self.master_IP,
                                         ' dfs -test -z {0}'.format(self.opts['destination']),
                                         hadoop_path=self.hdfs_path)
     self.assertEqual(exist_check_status, 0) and self.assertEqual(zero_check_status, 1)
     self.addCleanup(self.delete_hdfs_files, self.opts['destination'])
     self.addCleanup(self.delete_pithos_files, SOURCE_PITHOS_TO_HDFS_FILE)
     self.addCleanup(self.delete_local_files, SOURCE_PITHOS_TO_HDFS_FILE)