def test_bootstrap_resample(): """Tests the resample method of the Bootstrap class Tests comprise of simple resampling cases for which we can evalue the exact answer by hand. """ # All data is the same, variance should be zero. OLS = LeastSquares() DM = DesignMatrix('polynomial', 3) bootstrap = Bootstrap(OLS, DM) x = np.ones(10) * 2.25 y = x**3 #bootstrap.resample(x, y, 10) # This fails with an raise LinAlgError("Singular matrix") error on # TravisCI, but passes locally. Removing for now. #assert bootstrap.betaVariance == pytest.approx(np.zeros(4), abs=1e-15) # Ensure that larger noise in the data set gives larger computed # variance in the beta values from resampling. functions = { 0: lambda x: np.sin(x), 1: lambda x: np.cos(x), 2: lambda x: np.sin(2 * x), 3: lambda x: np.cos(2 * x), 4: lambda x: np.sin(3 * x), 5: lambda x: np.cos(3 * x), 6: lambda x: np.sin(4 * x), 7: lambda x: np.cos(4 * x), 8: lambda x: np.sin(5 * x), 9: lambda x: np.cos(5 * x), 10: lambda x: np.sin(x)**2, 11: lambda x: np.cos(x)**2, 12: lambda x: np.sin(2 * x)**2, 13: lambda x: np.cos(2 * x)**2, 14: lambda x: np.sin(3 * x)**2, 15: lambda x: np.cos(3 * x)**2, } DM = DesignMatrix(lambda j, x: functions[j](x), 9) OLS = LeastSquares() bootstrap = Bootstrap(OLS, DM) N = 100 x = np.linspace(0, 2 * np.pi, N) meanBetaVariance = np.zeros(6) ind = 0 for noiseScale in [0.0, 0.1, 1.0]: y = np.sin(1.5 * x) - 0.5 * np.cos(2 * x)**2 + np.random.normal( 0, noiseScale, N) bootstrap.resample(x, y, 100) meanBetaVariance[ind] = np.mean(bootstrap.betaVariance) if ind > 0: assert meanBetaVariance[ind - 1] < meanBetaVariance[ind] ind += 1
def test_getAmbariPort(self): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) self.assertEquals(bootstrap_obj.getAmbariPort(), "8440") shared_state.server_port = None bootstrap_obj = Bootstrap("hostname", shared_state) self.assertEquals(bootstrap_obj.getAmbariPort(), "null")
def test_try_to_execute(self, write_mock): expected = 43 shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) # Normal case def act_normal_return_int(): return 43 ret = bootstrap_obj.try_to_execute(act_normal_return_int) self.assertEqual(ret["exitstatus"], expected) self.assertFalse(write_mock.called) write_mock.reset_mock() def act_normal_return(): return {"exitstatus": 43} ret = bootstrap_obj.try_to_execute(act_normal_return) self.assertEqual(ret["exitstatus"], expected) self.assertFalse(write_mock.called) write_mock.reset_mock() # Exception scenario def act(): raise IOError() ret = bootstrap_obj.try_to_execute(act) self.assertEqual(ret["exitstatus"], 177) self.assertTrue(write_mock.called)
def test_copyPasswordFile(self, write_mock, ssh_run_mock, ssh_init_mock, scp_run_mock, scp_init_mock, getPasswordFile_mock): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", password_file="PasswordFile") bootstrap_obj = Bootstrap("hostname", shared_state) getPasswordFile_mock.return_value = "PasswordFile" # Testing max retcode return expected1 = {"exitstatus": 42, "log": "log42", "errormsg": "errorMsg"} expected2 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"} scp_init_mock.return_value = None scp_run_mock.return_value = expected1 ssh_init_mock.return_value = None ssh_run_mock.return_value = expected2 res = bootstrap_obj.copyPasswordFile() self.assertEquals(res, expected1["exitstatus"]) input_file = str(scp_init_mock.call_args[0][3]) remote_file = str(scp_init_mock.call_args[0][4]) self.assertEqual(input_file, "PasswordFile") self.assertEqual(remote_file, "PasswordFile") command = str(ssh_init_mock.call_args[0][3]) self.assertEqual(command, "chmod 600 PasswordFile") # Another order expected1 = {"exitstatus": 0, "log": "log0", "errormsg": "errorMsg"} expected2 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"} scp_run_mock.return_value = expected1 ssh_run_mock.return_value = expected2
def test_interruptBootstrap(self, write_mock, createDoneFile_mock): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) bootstrap_obj.interruptBootstrap() self.assertTrue(createDoneFile_mock.called)
def handlerCpinfo(self): # 导入放在这里是防止模块初始化时的导入循环,也可以放到方法体中 from bootstrap import Bootstrap for index,tag in enumerate(self.classFile.cp_tag): # Class ,Fieldref ,Methodref ,InterfaceMethodref if tag not in [7,9,10,11]: continue # print tag,constant_type.get(tag),self.classFile.cp_info[index] # TODO 其他模块的getContent方法可以删掉 res = self.__searchContent(index) # print res # class eg:java/lang/System if tag == 7: # if not Base.methodArea.has_key(_class_path): # print '=======has Bootstrap========',_class_path # # TODO 处理class文件之后还需要加载class进方法区,bootstrap # print '+++++++++++++++++ClassInfo.handlerCpinfo+++++++++++++++%s' % res Bootstrap(res) # Fieldref eg:['java/lang/System', ['out', 'Ljava/io/PrintStream;']] # class name_and_type elif tag == 9: pass # Methodref eg: ['java/lang/Object', ['<init>', '()V']] # class name_and_type elif tag == 10: pass # InterfaceMethodref elif tag == 11: pass
def test_getOsCheckScript(self): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) ocs = bootstrap_obj.getOsCheckScript() self.assertEquals(ocs, "scriptDir/os_check_type.py")
def test_getRepoFile(self, is_redhat_family, is_ubuntu_family, is_suse_family, hasPassword_mock, getRemoteName_mock): shared_state = SharedState("root", "123", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") is_redhat_family.return_value = True is_ubuntu_family.return_value = False is_suse_family.return_value = False bootstrap_obj = Bootstrap("hostname", shared_state) # Without password hasPassword_mock.return_value = False getRemoteName_mock.return_value = "RemoteName" rf = bootstrap_obj.getMoveRepoFileCommand("target") self.assertEquals( rf, "/var/lib/ambari-agent/tmp/ambari-sudo.sh mv RemoteName target/ambari.repo" ) # With password hasPassword_mock.return_value = True getRemoteName_mock.return_value = "RemoteName" rf = bootstrap_obj.getMoveRepoFileCommand("target") self.assertEquals( rf, "/var/lib/ambari-agent/tmp/ambari-sudo.sh -S mv RemoteName target/ambari.repo < RemoteName" )
def test_generateRandomFileName(self): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) self.assertTrue( bootstrap_obj.generateRandomFileName(None) == bootstrap_obj.getUtime())
def test_getRepoFile(self, is_suse_mock): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) is_suse_mock.return_value = False rf = bootstrap_obj.getRepoFile() self.assertEquals(rf, "/etc/yum.repos.d/ambari.repo")
def test_agent_setup_command_without_project_version(self): os.environ[AMBARI_PASSPHRASE_VAR_NAME] = "" version = None shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", version, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) runSetupCommand = bootstrap_obj.getRunSetupCommand("hostname") self.assertTrue(runSetupCommand.endswith(" 8440"))
def test_is_suse(self, open_mock, isfile_mock): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440") bootstrap_obj = Bootstrap("hostname", shared_state) isfile_mock.return_value = True f = open_mock.return_value f.read.return_value = " suse " self.assertTrue(bootstrap_obj.is_suse())
def test_getOsCheckScriptRemoteLocation(self, getRemoteName_mock): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) v = "/tmp/os_check_type1374259902.py" getRemoteName_mock.return_value = v ocs = bootstrap_obj.getOsCheckScriptRemoteLocation() self.assertEquals(ocs, v)
def fit(self, data): ''' Given a dataset, train the Random Forest model ''' data.reset_index(drop=True, inplace=True) bootstrap = Bootstrap(self.n_trees) n_baggings, test_index = bootstrap.generate_bootstraps(data) self.__generate_trees(data, n_baggings) result, result_index = self.__majority_votation(data, test_index) target = data[self.target_attribute].iloc[result_index.tolist()] print('Validation Accuracy: ', self.__calculate_accuracy(result_index,target,result)[0])
def test_checkSudoPackage(self, write_mock, run_mock, init_mock): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440") bootstrap_obj = Bootstrap("hostname", shared_state) expected = 42 init_mock.return_value = None run_mock.return_value = expected res = bootstrap_obj.checkSudoPackage() self.assertEquals(res, expected) command = str(init_mock.call_args[0][3]) self.assertEqual(command, "rpm -qa | grep sudo")
def get_main(): # check and/or update the source code bootstrap = Bootstrap(ROOT, local) if bootstrap.update(): # src updated correctly, load it bootstrap.load() from ristomele.gui.app import main return main else: # probably we cannot connect to the sync server. Start a simple app to # allow the user to change the settings return bootstrap_main
def test_getRunSetupWithPasswordCommand(self, environ_mock): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") environ_mock.__getitem__.return_value = "TEST_PASSPHRASE" bootstrap_obj = Bootstrap("hostname", shared_state) utime = 1234 bootstrap_obj.getUtime = MagicMock(return_value=utime) ret = bootstrap_obj.getRunSetupWithPasswordCommand("hostname") expected = "sudo -S python /var/lib/ambari-agent/data/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \ "ambariServer root 8440 < /var/lib/ambari-agent/data/tmp/host_pass{0}".format(utime) self.assertEquals(ret, expected)
def get_forest(self, size): bootstrap = Bootstrap(copy.deepcopy(self.data)) bootstrap_sets = [] for i in range(0, size): bootstrap_sets.append(bootstrap.get_partition()) trees = [] for bootstrap in bootstrap_sets: trees.append(Tree(copy.deepcopy(bootstrap[0]), True)) return trees
def test_createDoneFile(self, write_mock): tmp_dir = tempfile.gettempdir() shared_state = SharedState("root", "sshkey_file", "scriptDir", tmp_dir, "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) done_file = os.path.join(tmp_dir, "hostname.done") expected = 42 bootstrap_obj.createDoneFile(expected) with open(done_file) as df: res = df.read() self.assertEqual(res, str(expected)) os.unlink(done_file)
def test_getRepoDir(self, is_suse_mock): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440") bootstrap_obj = Bootstrap("hostname", shared_state) # Suse is_suse_mock.return_value = True res = bootstrap_obj.getRepoDir() self.assertEquals(res, "/etc/zypp/repos.d") # non-Suse is_suse_mock.return_value = False res = bootstrap_obj.getRepoDir() self.assertEquals(res, "/etc/yum.repos.d")
def test_copyNeededFiles(self, write_mock, ssh_run_mock, ssh_init_mock, scp_run_mock, scp_init_mock, getRemoteName_mock, getRepoFile_mock, getRepoDir, getMoveRepoFileCommand, is_redhat_family, is_ubuntu_family, is_suse_family): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") is_redhat_family.return_value = True is_ubuntu_family.return_value = False is_suse_family.return_value = False bootstrap_obj = Bootstrap("hostname", shared_state) getMoveRepoFileCommand.return_value = "MoveRepoFileCommand" getRepoDir.return_value = "RepoDir" getRemoteName_mock.return_value = "RemoteName" getRepoFile_mock.return_value = "RepoFile" expected1 = {"exitstatus": 42, "log": "log42", "errormsg": "errorMsg"} expected2 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"} expected3 = {"exitstatus": 1, "log": "log1", "errormsg": "errorMsg"} expected4 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"} scp_init_mock.return_value = None ssh_init_mock.return_value = None # Testing max retcode return scp_run_mock.side_effect = [expected1, expected3] ssh_run_mock.side_effect = [expected2, expected4] res = bootstrap_obj.copyNeededFiles() self.assertEquals(res, expected1["exitstatus"]) input_file = str(scp_init_mock.call_args[0][3]) remote_file = str(scp_init_mock.call_args[0][4]) self.assertEqual(input_file, "setupAgentFile") self.assertEqual(remote_file, "RemoteName") command = str(ssh_init_mock.call_args[0][3]) self.assertEqual(command, "sudo chmod 644 RepoFile") # Another order expected1 = {"exitstatus": 0, "log": "log0", "errormsg": "errorMsg"} expected2 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"} expected3 = {"exitstatus": 1, "log": "log1", "errormsg": "errorMsg"} expected4 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"} scp_run_mock.side_effect = [expected1, expected3] ssh_run_mock.side_effect = [expected2, expected4] res = bootstrap_obj.copyNeededFiles() self.assertEquals(res, expected2["exitstatus"]) # yet another order expected1 = {"exitstatus": 33, "log": "log33", "errormsg": "errorMsg"} expected2 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"} expected3 = {"exitstatus": 42, "log": "log42", "errormsg": "errorMsg"} expected4 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"} scp_run_mock.side_effect = [expected1, expected3] ssh_run_mock.side_effect = [expected2, expected4] res = bootstrap_obj.copyNeededFiles() self.assertEquals(res, expected3["exitstatus"])
def test_checkSudoPackageDebian(self, write_mock, run_mock, init_mock, server_family_mock): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "debian12", None, "8440") bootstrap_obj = Bootstrap("hostname", shared_state) expected = 42 init_mock.return_value = None run_mock.return_value = expected server_family_mock.return_value = ["debian", "12"] res = bootstrap_obj.checkSudoPackage() self.assertEquals(res, expected) command = str(init_mock.call_args[0][3]) self.assertEqual(command, "dpkg --get-selections|grep -e ^sudo")
def test_changePasswordFileModeOnHost(self, getPasswordFile_mock, write_mock, run_mock, init_mock): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) expected = 42 getPasswordFile_mock.return_value = "PasswordFile" init_mock.return_value = None run_mock.return_value = expected res = bootstrap_obj.changePasswordFileModeOnHost() self.assertEquals(res, expected) command = str(init_mock.call_args[0][3]) self.assertEqual(command, "chmod 600 PasswordFile")
def main(self,argv): methodName ="main" self.rc = 0 self.bootstrap = Bootstrap() try: beginTime = Utilities.currentTimeMillis() cmdLineArgs = Utilities.getInputArgs(self.ArgsSignature,argv[1:]) trace, logFile = self.bootstrap._configureTraceAndLogging(cmdLineArgs) TR.info(methodName,"BOOT0101I BEGIN Bootstrap AWS ICPD Quickstart version 1.0.0.") if (trace): TR.info(methodName,"BOOT0102I Tracing with specification: '%s' to log file: '%s'" % (trace,logFile)) #endIf region = cmdLineArgs.get('region') self.bootstrap.region = region self.bootstrap.role = cmdLineArgs.get('role') self.bootstrap.fqdn = socket.getfqdn() self.bootStackId = cmdLineArgs.get('stackid') self.bootstrap.rootStackName = cmdLineArgs.get('stack-name') self.bootstrap._init(self.bootstrap.rootStackName,self.bootStackId) self.logExporter = LogExporter(region=self.bootstrap.region, bucket=self.bootstrap.ICPDeploymentLogsBucketName, keyPrefix='logs/%s' % self.bootstrap.rootStackName, role=self.bootstrap.role, fqdn=self.bootstrap.fqdn ) self.icpHome = "/opt/icp/%s" % self.bootstrap.ICPVersion installMapPath = os.path.join(self.home,"maps","icpd-install-artifact-map.yaml") self.installMap = self.bootstrap.loadInstallMap(mapPath=installMapPath, version=self.bootstrap.ICPDVersion, region=self.bootstrap.region) icpdS3Path = "{version}/{object}".format(version=self.installMap['version'],object=self.installMap['icpd-base-install-archive']) destPath = "/tmp/icp4d.tar" storageClassCmd = "kubectl get storageclass | nl | grep aws-efs | awk '{print $1}'" TR.info(methodName,"check_output Get StorageClass value from kubectl %s"%(storageClassCmd)) self.storageclassValue=check_output(['bash','-c', storageClassCmd]) TR.info(methodName,"check_output StorageclassValue returned : %s"%(self.storageclassValue)) self.bootstrap.getS3Object(self.bootstrap.ICPDArchiveBucketName, icpdS3Path, destPath) self.stackIds = self.bootstrap._getStackIds(self.bootStackId) self.bootstrap._getHosts(self.stackIds) self.installICPD() except ExitException: pass # ExitException is used as a "goto" end of program after emitting help info except Exception, e: TR.error(methodName,"ERROR: %s" % e, e) self.rc = 1
def test_runSetupAgent(self, write_mock, run_mock, getRunSetupCommand_mock, init_mock): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) getRunSetupCommand_mock.return_value = "RunSetupCommand" expected = 42 init_mock.return_value = None run_mock.return_value = expected res = bootstrap_obj.runSetupAgent() self.assertEquals(res, expected) command = str(init_mock.call_args[0][3]) self.assertEqual(command, "RunSetupCommand")
def test_createTargetDir(self, write_mock, run_mock, init_mock): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) expected = 42 init_mock.return_value = None run_mock.return_value = expected res = bootstrap_obj.createTargetDir() self.assertEquals(res, expected) command = str(init_mock.call_args[0][3]) self.assertEqual( command, "sudo mkdir -p /var/lib/ambari-agent/data/tmp ; " "sudo chown -R root /var/lib/ambari-agent/data/tmp")
def test_checkSudoPackage(self, write_mock, run_mock, init_mock, is_redhat_family, is_ubuntu_family, is_suse_family): shared_state = SharedState("root", "123", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) expected = 42 init_mock.return_value = None run_mock.return_value = expected is_redhat_family.return_value = True is_ubuntu_family.return_value = False is_suse_family.return_value = False res = bootstrap_obj.checkSudoPackage() self.assertEquals(res, expected) command = str(init_mock.call_args[0][4]) self.assertEqual(command, "[ \"$EUID\" -eq 0 ] || rpm -qa | grep -e '^sudo\-'")
def test_runOsCheckScript(self, write_mock, run_mock, init_mock, getOsCheckScriptRemoteLocation_mock): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440") bootstrap_obj = Bootstrap("hostname", shared_state) getOsCheckScriptRemoteLocation_mock.return_value = "OsCheckScriptRemoteLocation" expected = 42 init_mock.return_value = None run_mock.return_value = expected res = bootstrap_obj.runOsCheckScript() self.assertEquals(res, expected) command = str(init_mock.call_args[0][3]) self.assertEqual( command, "chmod a+x OsCheckScriptRemoteLocation &&" " OsCheckScriptRemoteLocation centos6")
def test_getRepoFile(self, hasPassword_mock, getRemoteName_mock): shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440") bootstrap_obj = Bootstrap("hostname", shared_state) # Without password hasPassword_mock.return_value = False getRemoteName_mock.return_value = "RemoteName" rf = bootstrap_obj.getMoveRepoFileCommand("target") self.assertEquals(rf, "sudo mv RemoteName target/ambari.repo") # With password hasPassword_mock.return_value = True getRemoteName_mock.return_value = "RemoteName" rf = bootstrap_obj.getMoveRepoFileCommand("target") self.assertEquals( rf, "sudo -S mv RemoteName target/ambari.repo < RemoteName")
def test_getRemoteName(self): shared_state = SharedState("root", "123", "sshkey_file", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") res = bootstrap_obj = Bootstrap("hostname", shared_state) utime1 = 1234 utime2 = 12345 bootstrap_obj.getUtime = MagicMock(return_value=utime1) remote1 = bootstrap_obj.getRemoteName("/tmp/setupAgent.sh") self.assertEquals(remote1, "/tmp/setupAgent{0}.sh".format(utime1)) bootstrap_obj.getUtime.return_value=utime2 remote1 = bootstrap_obj.getRemoteName("/tmp/setupAgent.sh") self.assertEquals(remote1, "/tmp/setupAgent{0}.sh".format(utime1)) remote2 = bootstrap_obj.getRemoteName("/tmp/host_pass") self.assertEquals(remote2, "/tmp/host_pass{0}".format(utime2))