def test_C_prostate(self):
        h2o.nodes[0].log_view()
        namelist = h2o.nodes[0].log_download()
        print "\nStarting prostate.csv"
        # columns start at 0
        y = "1"
        x = ""
        csvFilename = "prostate.csv"
        csvPathname = "logreg" + "/" + csvFilename
        parseResult = h2i.import_parse(bucket="smalldata", path=csvPathname, hex_key=csvFilename + ".hex", schema="put")

        for maxx in range(2, 6):
            x = range(maxx)
            x.remove(0)  # 0 is member ID. not used
            x.remove(1)  # 1 is output
            x = ",".join(map(str, x))
            print "\nx:", x
            print "y:", y

            kwargs = {"x": x, "y": y, "n_folds": 5}
            glm = h2o_cmd.runGLM(parseResult=parseResult, timeoutSecs=15, **kwargs)
            # ID,CAPSULE,AGE,RACE,DPROS,DCAPS,PSA,VOL,GLEASON
            h2o_glm.simpleCheckGLM(self, glm, "AGE", **kwargs)
            sys.stdout.write(".")
            sys.stdout.flush()

        # now redo it all thru the browser
        # three times!
        for i in range(3):
            h2b.browseJsonHistoryAsUrl()

        h2o.nodes[0].log_view()
        namelist = h2o.nodes[0].log_download()
    def test_B_benign(self):
        h2o.nodes[0].log_view()
        namelist = h2o.nodes[0].log_download()

        print "\nStarting benign.csv"
        csvFilename = "benign.csv"
        csvPathname = "logreg" + "/" + csvFilename
        parseResult = h2i.import_parse(bucket="smalldata", path=csvPathname, hex_key=csvFilename + ".hex", schema="put")
        # columns start at 0
        y = "3"
        # cols 0-13. 3 is output
        # no member id in this one
        for maxx in range(11, 14):
            x = range(maxx)
            x.remove(3)  # 3 is output
            x = ",".join(map(str, x))
            print "\nx:", x
            print "y:", y

            kwargs = {"x": x, "y": y}
            glm = h2o_cmd.runGLM(parseResult=parseResult, timeoutSecs=15, **kwargs)
            # no longer look at STR?
            h2o_glm.simpleCheckGLM(self, glm, None, **kwargs)
            sys.stdout.write(".")
            sys.stdout.flush()

        # now redo it all thru the browser
        h2b.browseJsonHistoryAsUrl()
    def test_B_benign(self):
        h2o.nodes[0].log_view()
        namelist = h2o.nodes[0].log_download()

        print "\nStarting benign.csv"
        csvFilename = "benign.csv"
        csvPathname = 'logreg' + '/' + csvFilename
        parseResult = h2i.import_parse(bucket='smalldata',
                                       path=csvPathname,
                                       hex_key=csvFilename + ".hex",
                                       schema='put')
        # columns start at 0
        y = "3"
        # cols 0-13. 3 is output
        # no member id in this one
        for maxx in range(11, 14):
            x = range(maxx)
            x.remove(3)  # 3 is output
            x = ",".join(map(str, x))
            print "\nx:", x
            print "y:", y

            kwargs = {'x': x, 'y': y}
            glm = h2o_cmd.runGLM(parseResult=parseResult,
                                 timeoutSecs=15,
                                 **kwargs)
            # no longer look at STR?
            h2o_glm.simpleCheckGLM(self, glm, None, **kwargs)
            sys.stdout.write('.')
            sys.stdout.flush()

        # now redo it all thru the browser
        h2b.browseJsonHistoryAsUrl()
    def test_rf_tnc3_fvec(self):
        h2o.beta_features = True
        csvPathname = 'tnc3.csv'
        print "\n" + csvPathname
        hex_key = "tnc3.hex"
        ### h2b.browseTheCloud()

        parseResult = h2i.import_parse(bucket='smalldata', path=csvPathname, hex_key=hex_key, schema='put', 
            timeoutSecs=10, retryDelaySecs=0.25, header=1)
        print "Parse result['Key']:", parseResult['destination_key']
        inspect = h2o_cmd.runInspect(None, parseResult['destination_key'])
        h2b.browseJsonHistoryAsUrlLastMatch("Inspect")

        if 1==1:
            lenNodes = len(h2o.nodes)
            colResultList = h2e.exec_expr_list_across_cols(lenNodes, numExprList, hex_key, maxCol=10,
                incrementingResult=False, timeoutSecs=10)
            print "\ncolResultList after num swap", colResultList

        if (1==1):
            print "\nWe're not CM data getting back from RFView.json that we can check!. so look at the browser"
            print 'The good case with ignore="boat,body"'
            rfv = h2o_cmd.runRF(parseResult=parseResult, trees=5, timeoutSecs=10, retryDelaySecs=0.25,
                ignored_cols_by_name="boat,body")

        inspect = h2o_cmd.runInspect(None, parseResult['destination_key'])
        ### h2b.browseJsonHistoryAsUrlLastMatch("Inspect")
        ### time.sleep(3600)
        h2b.browseJsonHistoryAsUrl(retryDelaySecs=0.5)

        #******************
        if 1==0:
            colResultList = h2e.exec_expr_list_across_cols(lenNodes, charExprList, hex_key, maxCol=10,
                incrementingResult=False, timeoutSecs=10, retryDelaySecs=0.25)
            print "\ncolResultList after char swap", colResultList

        if 1==1:
            print "\nNow the bad case (no ignore)"
            rfv = h2o_cmd.runRF(parseResult=parseResult, trees=5, timeoutSecs=10, retryDelaySecs=0.25)

        inspect = h2o_cmd.runInspect(None, parseResult['destination_key'])
        ### h2b.browseJsonHistoryAsUrlLastMatch("Inspect")
        ### time.sleep(3600)
        h2b.browseJsonHistoryAsUrl(retryDelaySecs=0.5)
    def test_C_prostate(self):
        h2o.nodes[0].log_view()
        namelist = h2o.nodes[0].log_download()
        print "\nStarting prostate.csv"
        # columns start at 0
        y = "1"
        x = ""
        csvFilename = "prostate.csv"
        csvPathname = 'logreg' + '/' + csvFilename
        parseResult = h2i.import_parse(bucket='smalldata',
                                       path=csvPathname,
                                       hex_key=csvFilename + ".hex",
                                       schema='put')

        for maxx in range(2, 6):
            x = range(maxx)
            x.remove(0)  # 0 is member ID. not used
            x.remove(1)  # 1 is output
            x = ",".join(map(str, x))
            print "\nx:", x
            print "y:", y

            kwargs = {'x': x, 'y': y, 'n_folds': 5}
            glm = h2o_cmd.runGLM(parseResult=parseResult,
                                 timeoutSecs=15,
                                 **kwargs)
            # ID,CAPSULE,AGE,RACE,DPROS,DCAPS,PSA,VOL,GLEASON
            h2o_glm.simpleCheckGLM(self, glm, 'AGE', **kwargs)
            sys.stdout.write('.')
            sys.stdout.flush()

        # now redo it all thru the browser
        # three times!
        for i in range(3):
            h2b.browseJsonHistoryAsUrl()

        h2o.nodes[0].log_view()
        namelist = h2o.nodes[0].log_download()
    def test_GLM2_basic_browser(self):
        h2b.browseTheCloud()

        importFolderPath = "logreg"
        csvFilename = 'prostate.csv'
        csvPathname = importFolderPath + "/" + csvFilename
        hex_key = csvFilename + ".hex"

        parseResult = h2i.import_parse(bucket='smalldata', path=csvPathname, schema='local', hex_key=hex_key, timeoutSecs=180)
        inspect = h2o_cmd.runInspect(None, parseResult['destination_key'])
        print inspect
        print "\n" + csvPathname, \
            "    numRows:", "{:,}".format(inspect['numRows']), \
            "    numCols:", "{:,}".format(inspect['numCols'])

        x         = 'ID'
        y         = 'CAPSULE'
        family    = 'binomial'
        alpha     = '0.5'
        lambda_   = '1E-4'
        nfolds    = '0'
        f         = 'prostate'
        modelKey  = 'GLM_' + f

        kwargs = {       'response'           : y,
                         'ignored_cols'       : x,
                         'family'             : family,
                         'lambda'             : lambda_,
                         'alpha'              : alpha,
                         'n_folds'            : nfolds, # passes if 0, fails otherwise
                         'destination_key'    : modelKey,
                 }

        timeoutSecs = 60
        start = time.time()
        glmResult = h2o_cmd.runGLM(parseResult=parseResult, timeoutSecs=timeoutSecs, retryDelaySecs=0.25, pollTimeoutSecs=180, **kwargs)

        # this stuff was left over from when we got the result after polling the jobs list
        # okay to do it again
        # GLM2: when it redirects to the model view, we no longer have the job_key! (unlike the first response and polling)
        if 1==0:
            job_key = glmResult['job_key']
            # is the job finishing before polling would say it's done?
            params = {'job_key': job_key, 'destination_key': modelKey}
            glm = h2o.nodes[0].completion_redirect(jsonRequest="2/GLMProgressPage2.json", params=params)
            print "GLM result from completion_redirect:", h2o.dump_json(a)
        if 1==1:
            glm = h2o.nodes[0].glm_view(_modelKey=modelKey)
            ### print "GLM result from glm_view:", h2o.dump_json(a)

        h2o_glm.simpleCheckGLM(self, glm, None, **kwargs)

        glm_model = glm['glm_model']
        _names = glm_model['_names']
        coefficients_names = glm_model['coefficients_names']
        submodels = glm_model['submodels'][0]

        beta = submodels['beta']
        norm_beta = submodels['norm_beta']
        iteration = submodels['iteration']

        validation = submodels['validation']        
        auc = validation['auc']
        aic = validation['aic']
        null_deviance = validation['null_deviance']
        residual_deviance = validation['residual_deviance']

        print '_names', _names
        print 'coefficients_names', coefficients_names
        # did beta get shortened? the simple check confirms names/beta/norm_beta are same length
        print 'beta', beta
        print 'iteration', iteration        
        print 'auc', auc

        # now redo it all thru the browser
        h2b.browseJsonHistoryAsUrl()
    def test_GLM2_basic_browser(self):
        h2b.browseTheCloud()

        h2o.beta_features=True
        importFolderPath = "logreg"
        csvFilename = 'prostate.csv'
        csvPathname = importFolderPath + "/" + csvFilename
        hex_key = csvFilename + ".hex"

        parseResult = h2i.import_parse(bucket='smalldata', path=csvPathname, schema='local', hex_key=hex_key, timeoutSecs=180)
        inspect = h2o_cmd.runInspect(None, parseResult['destination_key'])
        print inspect
        print "\n" + csvPathname, \
            "    numRows:", "{:,}".format(inspect['numRows']), \
            "    numCols:", "{:,}".format(inspect['numCols'])

        x         = 'ID'
        y         = 'CAPSULE'
        family    = 'binomial'
        alpha     = '0.5'
        lambda_   = '1E-4'
        nfolds    = '0'
        f         = 'prostate'
        modelKey  = 'GLM_' + f

        kwargs = {       'response'           : y,
                         'ignored_cols'       : x,
                         'family'             : family,
                         'lambda'             : lambda_,
                         'alpha'              : alpha,
                         'n_folds'            : nfolds, # passes if 0, fails otherwise
                         'destination_key'    : modelKey,
                 }

        timeoutSecs = 60
        start = time.time()
        glmResult = h2o_cmd.runGLM(parseResult=parseResult, timeoutSecs=timeoutSecs, retryDelaySecs=0.25, pollTimeoutSecs=180, **kwargs)

        # this stuff was left over from when we got the result after polling the jobs list
        # okay to do it again
        # GLM2: when it redirects to the model view, we no longer have the job_key! (unlike the first response and polling)
        if 1==0:
            job_key = glmResult['job_key']
            # is the job finishing before polling would say it's done?
            params = {'job_key': job_key, 'destination_key': modelKey}
            glm = h2o.nodes[0].completion_redirect(jsonRequest="2/GLMProgressPage2.json", params=params)
            print "GLM result from completion_redirect:", h2o.dump_json(a)
        if 1==1:
            glm = h2o.nodes[0].glm_view(_modelKey=modelKey)
            ### print "GLM result from glm_view:", h2o.dump_json(a)

        h2o_glm.simpleCheckGLM(self, glm, None, **kwargs)

        glm_model = glm['glm_model']
        _names = glm_model['_names']
        coefficients_names = glm_model['coefficients_names']
        submodels = glm_model['submodels'][0]

        beta = submodels['beta']
        norm_beta = submodels['norm_beta']
        iteration = submodels['iteration']

        validation = submodels['validation']        
        auc = validation['auc']
        aic = validation['aic']
        null_deviance = validation['null_deviance']
        residual_deviance = validation['residual_deviance']

        print '_names', _names
        print 'coefficients_names', coefficients_names
        # did beta get shortened? the simple check confirms names/beta/norm_beta are same length
        print 'beta', beta
        print 'iteration', iteration        
        print 'auc', auc

        # now redo it all thru the browser
        h2b.browseJsonHistoryAsUrl()