def test_B_hdfs_files(self): print "\nLoad a list of files from HDFS, parse and do 1 RF tree" print "\nYou can try running as hduser/hduser if fail" # larger set in my local dir # fails because classes aren't integers # "allstate_claim_prediction_train_set.zip", csvFilenameList = [ "airlines_88_08_100lines.csv", ] h2b.browseTheCloud() timeoutSecs = 200 # save the first, for all comparisions, to avoid slow drift with each iteration firstglm = {} h2i.setupImportHdfs() for csvFilename in csvFilenameList: # creates csvFilename.hex from file in hdfs dir print "Loading", csvFilename, 'from HDFS' parseKey = h2i.parseImportHdfsFile(csvFilename=csvFilename, path='/datasets', timeoutSecs=1000) print csvFilename, 'parse time:', parseKey['response']['time'] print "parse result:", parseKey['destination_key'] print "\n" + csvFilename start = time.time() RFview = h2o_cmd.runRFOnly(trees=1,parseKey=parseKey,timeoutSecs=2000)
def test_B_hdfs_files(self): print "\nLoad a list of files from HDFS, parse and do 1 RF tree" print "\nYou can try running as hduser/hduser if fail" # larger set in my local dir # fails because classes aren't integers # "allstate_claim_prediction_train_set.zip", csvFilenameList = [ "airlines_88_08_100lines.csv", ] h2b.browseTheCloud() timeoutSecs = 200 # save the first, for all comparisions, to avoid slow drift with each iteration firstglm = {} h2i.setupImportHdfs() for csvFilename in csvFilenameList: # creates csvFilename.hex from file in hdfs dir print "Loading", csvFilename, 'from HDFS' parseKey = h2i.parseImportHdfsFile(csvFilename=csvFilename, path='/datasets', timeoutSecs=1000) print csvFilename, 'parse time:', parseKey['response']['time'] print "parse result:", parseKey['destination_key'] print "\n" + csvFilename start = time.time() RFview = h2o_cmd.runRFOnly(trees=1, parseKey=parseKey, timeoutSecs=2000)
def test_B_hdfs_files(self): print "\nLoad a list of files from HDFS, parse and do 1 RF tree" print "\nYou can try running as hduser/hduser if fail" # larger set in my local dir # fails because classes aren't integers # "allstate_claim_prediction_train_set.zip", csvFilenameAll = [ "TEST-poker1000.csv", "leads.csv", "and-testing.data", "arcene2_train.both", "arcene_train.both", # these can't RF ..output classes not integer? # "bestbuy_test.csv", # "bestbuy_train.csv", "covtype.data", "covtype.4x.shuffle.data", "covtype4x.shuffle.data", "covtype.13x.data", "covtype.13x.shuffle.data", # "covtype.169x.data", # "prostate_2g.csv", # "prostate_long.csv.gz", "prostate_long_1G.csv", "hhp.unbalanced.012.1x11.data.gz", "hhp.unbalanced.012.data.gz", "hhp.unbalanced.data.gz", "hhp2.os.noisy.0_1.data", "hhp2.os.noisy.9_4.data", "hhp_9_14_12.data", # "poker_c1s1_testing_refresh.csv", # "3G_poker_shuffle", # "billion_rows.csv.gz", # "poker-hand.1244M.shuffled311M.full.txt", ] # pick 8 randomly! if (1==0): csvFilenameList = random.sample(csvFilenameAll,8) # Alternatively: do the list in order! Note the order is easy to hard else: csvFilenameList = csvFilenameAll # pop open a browser on the cloud h2b.browseTheCloud() timeoutSecs = 200 # save the first, for all comparisions, to avoid slow drift with each iteration firstglm = {} h2i.setupImportHdfs() for csvFilename in csvFilenameList: # creates csvFilename.hex from file in hdfs dir print "Loading", csvFilename, 'from HDFS' parseKey = h2i.parseImportHdfsFile(csvFilename=csvFilename, path='/datasets', timeoutSecs=1000) print csvFilename, 'parse time:', parseKey['response']['time'] print "parse result:", parseKey['destination_key'] print "\n" + csvFilename start = time.time() RFview = h2o_cmd.runRFOnly(trees=1,parseKey=parseKey,timeoutSecs=2000)
def test_GLM_hdfs_YearPredictionMSD(self): if localhost: csvFilenameList = [ 'YearPredictionMSD.txt', 'YearPredictionMSD.txt' ] else: csvFilenameList = [ 'YearPredictionMSD.txt', 'YearPredictionMSD.txt' ] # a browser window too, just because we can h2b.browseTheCloud() validations1= {} coefficients1= {} for csvFilename in csvFilenameList: # creates csvFilename.hex from file in importFolder dir h2i.setupImportHdfs() parseKey = h2i.parseImportHdfsFile(csvFilename=csvFilename, path='/datasets', timeoutSecs=60) print csvFilename, 'parse time:', parseKey['response']['time'] print "Parse result['destination_key']:", parseKey['destination_key'] # We should be able to see the parse result? inspect = h2o_cmd.runInspect(key=parseKey['destination_key']) print "\n" + csvFilename start = time.time() # can't pass lamba as kwarg because it's a python reserved word # FIX! just look at X=0:1 for speed, for now kwargs = {'y': 54, 'n_folds': 2, 'family': "binomial", 'case': 1} glm = h2o_cmd.runGLMOnly(parseKey=parseKey, timeoutSecs=500, **kwargs) # different when n_foldsidation is used? No trainingErrorDetails? h2o.verboseprint("\nglm:", glm) ### h2b.browseJsonHistoryAsUrlLastMatch("GLM") GLMModel = glm['GLMModel'] print "GLM time", GLMModel['time'] coefficients = GLMModel['coefficients'] validationsList = GLMModel['validations'] validations = validationsList.pop() # validations['err'] if validations1: h2o_glm.compareToFirstGlm(self, 'err', validations, validations1) else: validations1 = copy.deepcopy(validations) if coefficients1: h2o_glm.compareToFirstGlm(self, '0', coefficients, coefficients1) else: coefficients1 = copy.deepcopy(coefficients) sys.stdout.write('.') sys.stdout.flush()
def test_GLM_hdfs_YearPredictionMSD(self): if localhost: csvFilenameList = [ 'YearPredictionMSD.txt', 'YearPredictionMSD.txt' ] else: csvFilenameList = [ 'YearPredictionMSD.txt', 'YearPredictionMSD.txt' ] # a browser window too, just because we can h2b.browseTheCloud() validations1= {} coefficients1= {} for csvFilename in csvFilenameList: # creates csvFilename.hex from file in importFolder dir h2i.setupImportHdfs() parseKey = h2i.parseImportHdfsFile(csvFilename=csvFilename, path='/datasets', timeoutSecs=60) print csvFilename, 'parse time:', parseKey['response']['time'] print "Parse result['destination_key']:", parseKey['destination_key'] # We should be able to see the parse result? inspect = h2o_cmd.runInspect(key=parseKey['destination_key']) print "\n" + csvFilename start = time.time() # can't pass lamba as kwarg because it's a python reserved word # FIX! just look at X=0:1 for speed, for now kwargs = {'y': 54, 'n_folds': 2, 'family': "binomial", 'case': 1} glm = h2o_cmd.runGLMOnly(parseKey=parseKey, timeoutSecs=500, **kwargs) # different when n_foldsidation is used? No trainingErrorDetails? h2o.verboseprint("\nglm:", glm) h2b.browseJsonHistoryAsUrlLastMatch("GLM") GLMModel = glm['GLMModel'] print "GLM time", GLMModel['time'] coefficients = GLMModel['coefficients'] validationsList = GLMModel['validations'] validations = validationsList.pop() # validations['err'] if validations1: h2o_glm.compareToFirstGlm(self, 'err', validations, validations1) else: validations1 = copy.deepcopy(validations) if coefficients1: h2o_glm.compareToFirstGlm(self, '0', coefficients, coefficients1) else: coefficients1 = copy.deepcopy(coefficients) sys.stdout.write('.') sys.stdout.flush()
def test_hdfs_multi_bad_csv(self): print "\nUse the new regex capabilities for selecting hdfs: try *csv* at /datasets" # pop open a browser on the cloud h2b.browseTheCloud() # defaults to /datasets h2i.setupImportHdfs() parseKey = h2o.nodes[0].parse('*airlines_all*csv', key2='random_csv.hex', exclude=None, header=None, timeoutSecs=600) print "*csv* regex to hdfs /datasets", 'parse time:', parseKey['response']['time'] print "parse result:", parseKey['destination_key'] sys.stdout.flush()
def test_hdfs_multi_copies(self): print "\nUse the new regex capabilities for selecting hdfs: try *copies* at /datasets" print "This should match to a folder with about twenty covtype10x?" # pop open a browser on the cloud h2b.browseTheCloud() # defaults to /datasets h2i.setupImportHdfs() parseKey = h2i.parseImportHdfsFile(csvFilename='*covtype10x_copies*', key2='copies.hex', exclude=None, header=None, timeoutSecs=600) print "*copies* regex to hdfs /datasets", 'parse time:', parseKey['response']['time'] print "parse result:", parseKey['destination_key'] sys.stdout.flush()
def test_hdfs_multi_copies(self): print "\nUse the new regex capabilities for selecting hdfs: try *copies* at /datasets" print "This should match to a folder with about twenty covtype10x?" # pop open a browser on the cloud h2b.browseTheCloud() # defaults to /datasets h2i.setupImportHdfs() parseKey = h2o.nodes[0].parse( "*covtype10x_copies*", key2="copies.hex", exclude=None, header=None, timeoutSecs=600 ) print "*copies* regex to hdfs /datasets", "parse time:", parseKey["response"]["time"] print "parse result:", parseKey["destination_key"] sys.stdout.flush()
def test_B_hdfs_files(self): print "\nLoad a list of files from HDFS, parse and do 1 RF tree" print "\nYou can try running as hduser/hduser if fail" # larger set in my local dir # fails because classes aren't integers # "allstate_claim_prediction_train_set.zip", csvFilenameAll = [ "allyears2k.csv", "billion_rows.csv.gz", "covtype.data", "covtype.shuffled.data", "covtype200x.data", "covtype20x.data", "kddcup_1999.data.gz", "rand_logreg_100000000x70.csv.gz", ] # pick 8 randomly! if (1==0): csvFilenameList = random.sample(csvFilenameAll,8) # Alternatively: do the list in order! Note the order is easy to hard else: csvFilenameList = csvFilenameAll # pop open a browser on the cloud # h2b.browseTheCloud() timeoutSecs = 200 # save the first, for all comparisions, to avoid slow drift with each iteration firstglm = {} h2i.setupImportHdfs( path='/datasets/standard', schema='maprfs') for csvFilename in csvFilenameList: # creates csvFilename.hex from file in hdfs dir print "Loading", csvFilename, 'from HDFS' parseKey = h2i.parseImportHdfsFile( csvFilename=csvFilename, path='/datasets/standard', schema='maprfs', timeoutSecs=1000) print csvFilename, 'parse time:', parseKey['response']['time'] print "parse result:", parseKey['destination_key'] print "\n" + csvFilename start = time.time() RFview = h2o_cmd.runRFOnly(trees=1,parseKey=parseKey,timeoutSecs=2000)
def test_hdfs_multi_bad_csv(self): print "\nUse the new regex capabilities for selecting hdfs: try *csv* at /datasets" # pop open a browser on the cloud h2b.browseTheCloud() # defaults to /datasets h2i.setupImportHdfs() parseKey = h2o.nodes[0].parse('*airlines_all*csv', key2='random_csv.hex', exclude=None, header=None, timeoutSecs=600) print "*csv* regex to hdfs /datasets", 'parse time:', parseKey[ 'response']['time'] print "parse result:", parseKey['destination_key'] sys.stdout.flush()
def test_B_hdfs_files(self): print "\nLoad a list of files from HDFS, parse and do 1 RF tree" print "\nYou can try running as hduser/hduser if fail" # larger set in my local dir # fails because classes aren't integers # "allstate_claim_prediction_train_set.zip", csvFilenameAll = [ "allyears2k.csv", "billion_rows.csv.gz", "covtype.data", "covtype.shuffled.data", "covtype200x.data", "covtype20x.data", "kddcup_1999.data.gz", "rand_logreg_100000000x70.csv.gz", ] # pick 8 randomly! if (1 == 0): csvFilenameList = random.sample(csvFilenameAll, 8) # Alternatively: do the list in order! Note the order is easy to hard else: csvFilenameList = csvFilenameAll # pop open a browser on the cloud # h2b.browseTheCloud() timeoutSecs = 200 # save the first, for all comparisions, to avoid slow drift with each iteration firstglm = {} h2i.setupImportHdfs(path='/datasets/standard', schema='maprfs') for csvFilename in csvFilenameList: # creates csvFilename.hex from file in hdfs dir print "Loading", csvFilename, 'from HDFS' parseKey = h2i.parseImportHdfsFile(csvFilename=csvFilename, path='/datasets/standard', schema='maprfs', timeoutSecs=1000) print csvFilename, 'parse time:', parseKey['response']['time'] print "parse result:", parseKey['destination_key'] print "\n" + csvFilename start = time.time() RFview = h2o_cmd.runRFOnly(trees=1, parseKey=parseKey, timeoutSecs=2000)
def test_B_hdfs_files(self): print "\nLoad a list of files from HDFS, parse and do 1 RF tree" print "\nYou can try running as hduser/hduser if fail" # larger set in my local dir # fails because classes aren't integers # "allstate_claim_prediction_train_set.zip", csvFilenameAll = [ "TEST-poker1000.csv", ] # pick 8 randomly! if (1==0): csvFilenameList = random.sample(csvFilenameAll,8) # Alternatively: do the list in order! Note the order is easy to hard else: csvFilenameList = csvFilenameAll # pop open a browser on the cloud # h2b.browseTheCloud() timeoutSecs = 200 # save the first, for all comparisions, to avoid slow drift with each iteration firstglm = {} h2i.setupImportHdfs( path='/datasets', schema='maprfs') for csvFilename in csvFilenameList: # creates csvFilename.hex from file in hdfs dir print "Loading", csvFilename, 'from HDFS' parseKey = h2i.parseImportHdfsFile( csvFilename=csvFilename, path='/datasets', schema='maprfs', timeoutSecs=1000) print csvFilename, 'parse time:', parseKey['response']['time'] print "parse result:", parseKey['destination_key'] print "\n" + csvFilename start = time.time() RFview = h2o_cmd.runRFOnly(trees=1,parseKey=parseKey,timeoutSecs=2000) h2b.browseJsonHistoryAsUrlLastMatch("RFView") # wait in case it recomputes it time.sleep(10) sys.stdout.write('.') sys.stdout.flush()
def test_hdfs_multi_copies(self): print "\nUse the new regex capabilities for selecting hdfs: try *copies* at /datasets" print "This should match to a folder with about twenty covtype10x?" # pop open a browser on the cloud h2b.browseTheCloud() # defaults to /datasets h2i.setupImportHdfs() parseKey = h2o.nodes[0].parse('*covtype10x_copies*', key2='copies.hex', exclude=None, header=None, timeoutSecs=600) print "*copies* regex to hdfs /datasets", 'parse time:', parseKey[ 'response']['time'] print "parse result:", parseKey['destination_key'] sys.stdout.flush()
def test_import_nflx_parse_loop(self): print "Using the -.gz files from hdfs" # hdfs://<name node>/datasets/manyfiles-nflx-gz/file_1.dat.gz csvFilename = "file_10.dat.gz" csvFilepattern = "file_1[0-9].dat.gz" trialMax = 2 for tryHeap in [24]: print "\n", tryHeap,"GB heap, 1 jvm per host, import 192.168.1.176 hdfs, then parse" localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(node_count=1, java_heap_GB=tryHeap, use_hdfs=True, hdfs_name_node='192.168.1.176', hdfs_version='cdh3') else: h2o_hosts.build_cloud_with_hosts(node_count=1, java_heap_GB=tryHeap, use_hdfs=True, hdfs_name_node='192.168.1.176', hdfs_version='cdh3') # don't raise exception if we find something bad in h2o stdout/stderr? # h2o.nodes[0].sandbox_ignore_errors = True timeoutSecs = 500 importFolderPath = "/datasets/manyfiles-nflx-gz" for trial in range(trialMax): # since we delete the key, we have to re-import every iteration, to get it again importHdfsResult = h2i.setupImportHdfs(path=importFolderPath) hdfsFullList = importHdfsResult['succeeded'] for k in hdfsFullList: key = k['key'] # just print the first tile if 'nflx' in key and 'file_1.dat.gz' in key: # should be hdfs://home-0xdiag-datasets/manyfiles-nflx-gz/file_1.dat.gz print "example file we'll use:", key ### print "hdfsFullList:", h2o.dump_json(hdfsFullList) # error if none? self.assertGreater(len(hdfsFullList),8,"Didn't see more than 8 files in hdfs?") key2 = csvFilename + "_" + str(trial) + ".hex" csvFilePattern = 'file_1.dat.gz' # "key": "hdfs://192.168.1.176/datasets/manyfiles-nflx-gz/file_99.dat.gz", time.sleep(5) print "Loading from hdfs:", importFolderPath + "/" + csvFilePattern start = time.time() parseKey = h2i.parseImportHdfsFile(csvFilename=csvFilePattern, path=importFolderPath, key2=key2, timeoutSecs=timeoutSecs, retryDelaySecs=10, pollTimeoutSecs=60) elapsed = time.time() - start print hdfsKey, 'parse time:', parseKey['response']['time'] print "parse result:", parseKey['destination_key'] print "Parse #", trial, "completed in", "%6.2f" % elapsed, "seconds.", \ "%d pct. of timeout" % ((elapsed*100)/timeoutSecs) h2o_cmd.runStoreView() h2o.tear_down_cloud() # sticky ports? wait a bit. time.sleep(5)
def test_hdfs_multi_bad_csv(self): print "\nUse the new regex capabilities for selecting hdfs: try *csv* at /datasets" # pop open a browser on the cloud h2b.browseTheCloud() # defaults to /datasets h2i.setupImportHdfs() # path should default to /datasets # One .gz in with non .gz seems to cause a stack trace..so don't match to all (*airlines*). # no..maybe it's just the zero length gz file?. No it doesn't show up in the list of keys? # drwxr-xr-x - earl supergroup 0 2013-07-24 17:55 /datasets/airline.gz # -rw-r--r-- 3 hduser supergroup 12155501626 2013-02-22 17:13 /datasets/airline_116M.csv # -rw-r--r-- 3 hduser supergroup 11349125129 2013-05-03 15:45 /datasets/airlines_1988_2008.csv # -rw-r--r-- 3 hduser supergroup 11349125429 2013-05-01 12:52 /datasets/airlines_1988_2008_shuffled.csv # -rw-r--r-- 3 hduser supergroup 9936 2013-05-01 11:49 /datasets/airlines_88_08_100lines.csv # -rw-r--r-- 3 hduser supergroup 12155501626 2013-02-23 15:59 /datasets/airlines_all.csv # -rw-r--r-- 3 hduser supergroup 133710514626 2013-02-23 15:21 /datasets/airlines_all_11x.csv parseKey = h2i.parseImportHdfsFile(csvFilename="airline_116M.csv", key2="random_csv.hex", timeoutSecs=600) print "*csv* regex to hdfs /datasets", "parse time:", parseKey["response"]["time"] print "parse result:", parseKey["destination_key"] sys.stdout.flush()
def test_hdfs_multi_bad_csv(self): print "\nUse the new regex capabilities for selecting hdfs: try *csv* at /datasets" # pop open a browser on the cloud h2b.browseTheCloud() # defaults to /datasets h2i.setupImportHdfs() # path should default to /datasets # One .gz in with non .gz seems to cause a stack trace..so don't match to all (*airlines*). # no..maybe it's just the zero length gz file?. No it doesn't show up in the list of keys? # drwxr-xr-x - earl supergroup 0 2013-07-24 17:55 /datasets/airline.gz # -rw-r--r-- 3 hduser supergroup 12155501626 2013-02-22 17:13 /datasets/airline_116M.csv # -rw-r--r-- 3 hduser supergroup 11349125129 2013-05-03 15:45 /datasets/airlines_1988_2008.csv # -rw-r--r-- 3 hduser supergroup 11349125429 2013-05-01 12:52 /datasets/airlines_1988_2008_shuffled.csv # -rw-r--r-- 3 hduser supergroup 9936 2013-05-01 11:49 /datasets/airlines_88_08_100lines.csv # -rw-r--r-- 3 hduser supergroup 12155501626 2013-02-23 15:59 /datasets/airlines_all.csv # -rw-r--r-- 3 hduser supergroup 133710514626 2013-02-23 15:21 /datasets/airlines_all_11x.csv parseKey = h2i.parseImportHdfsFile(csvFilename='airline_116M.csv', key2='random_csv.hex', timeoutSecs=600) print "*csv* regex to hdfs /datasets", 'parse time:', parseKey['response']['time'] print "parse result:", parseKey['destination_key'] sys.stdout.flush()
def test_KMeans_sphere15_180GB(self): csvFilename = 'syn_sphere15_2711545732row_6col_180GB_from_7x.csv' totalBytes = 183538602156 if FROM_HDFS: importFolderPath = "/datasets/kmeans_big" csvPathname = "hdfs://" + importFolderPath + '/' + csvFilename else: importFolderPath = "/home3/0xdiag/datasets/kmeans_big" csvPathname = importFolderPath + '/' + csvFilename # FIX! put right values in # will there be different expected for random vs the other inits? expected = [ ([ 0.0, -113.00566692375459, -89.99595447985321, -455.9970643424373, 4732.0, 49791778.0, 36800.0 ], 248846122, 1308149283316.2988), ([ 0.0, 1.0, 1.0, -525.0093818313685, 2015.001629398412, 25654042.00592703, 28304.0 ], 276924291, 1800760152555.98), ([ 0.0, 5.0, 2.0, 340.0, 1817.995920197288, 33970406.992053084, 31319.99486705394 ], 235089554, 375419158808.3253), ([ 0.0, 10.0, -72.00113070337981, -171.0198611715457, 4430.00952228909, 37007399.0, 29894.0 ], 166180630, 525423632323.6474), ([ 0.0, 11.0, 3.0, 578.0043558141306, 1483.0163188052604, 22865824.99639042, 5335.0 ], 167234179, 1845362026223.1094), ([ 0.0, 12.0, 3.0, 168.0, -4066.995950679284, 41077063.00269915, -47537.998050740985 ], 195420925, 197941282992.43475), ([ 0.0, 19.00092954923767, -10.999565572612255, 90.00028669073289, 1928.0, 39967190.0, 27202.0 ], 214401768, 11868360232.658035), ([ 0.0, 20.0, 0.0, 141.0, -3263.0030236302937, 6163210.990273981, 30712.99115201907 ], 258853406, 598863991074.3276), ([ 0.0, 21.0, 114.01584574295777, 242.99690338815898, 1674.0029079209912, 33089556.0, 36415.0 ], 190979054, 1505088759456.314), ([ 0.0, 25.0, 1.0, 614.0032787274755, -2275.9931284021022, -48473733.04122273, 47343.0 ], 87794427, 1124697008162.3955), ([ 0.0, 39.0, 3.0, 470.0, -3337.9880599007597, 28768057.98852736, 16716.003410920028 ], 78226988, 1151439441529.0215), ([ 0.0, 40.0, 1.0, 145.0, 950.9990795199593, 14602680.991458317, -14930.007919032574 ], 167273589, 693036940951.0249), ([ 0.0, 42.0, 4.0, 479.0, -3678.0033024834297, 8209673.001421165, 11767.998552236539 ], 148426180, 35942838893.32379), ([ 0.0, 48.0, 4.0, 71.0, -951.0035145455234, 49882273.00063991, -23336.998167498707 ], 157533313, 88431531357.62982), ([ 0.0, 147.00394564757505, 122.98729664236723, 311.0047920137008, 2320.0, 46602185.0, 11212.0 ], 118361306, 1111537045743.7646), ] benchmarkLogging = ['cpu', 'disk', 'network', 'iostats', 'jstack'] benchmarkLogging = ['cpu', 'disk', 'network', 'iostats'] # IOStatus can hang? benchmarkLogging = ['cpu', 'disk', 'network'] benchmarkLogging = [] for trial in range(6): # IMPORT********************************************** # since H2O deletes the source key, re-import every iteration. if FROM_HDFS: importFolderResult = h2i.setupImportHdfs( None, importFolderPath) else: importFolderResult = h2i.setupImportFolder( None, importFolderPath) # PARSE **************************************** print "Parse starting: " + csvFilename key2 = csvFilename + "_" + str(trial) + ".hex" start = time.time() timeoutSecs = 2 * 3600 kwargs = {} if FROM_HDFS: parseKey = h2i.parseImportHdfsFile( None, csvFilename, importFolderPath, key2=key2, timeoutSecs=timeoutSecs, pollTimeoutsecs=60, retryDelaySecs=2, benchmarkLogging=benchmarkLogging, **kwargs) else: parseKey = h2i.parseImportFolderFile( None, csvFilename, importFolderPath, key2=key2, timeoutSecs=timeoutSecs, pollTimeoutsecs=60, retryDelaySecs=2, benchmarkLogging=benchmarkLogging, **kwargs) elapsed = time.time() - start fileMBS = (totalBytes / 1e6) / elapsed l = '{!s} jvms, {!s}GB heap, {:s} {:s} {:6.2f} MB/sec for {:.2f} secs'.format( len(h2o.nodes), h2o.nodes[0].java_heap_GB, 'Parse', csvPathname, fileMBS, elapsed) print "\n" + l h2o.cloudPerfH2O.message(l) # KMeans **************************************** print "col 0 is enum in " + csvFilename + " but KMeans should skip that automatically?? or no?" kwargs = { 'k': 15, 'initialization': 'Furthest', 'epsilon': 1e-6, 'cols': None, 'destination_key': 'junk.hex', # reuse the same seed, to get deterministic results 'seed': 265211114317615310, } if (trial % 3) == 0: kwargs['initialization'] = 'PlusPlus' elif (trial % 3) == 1: kwargs['initialization'] = 'Furthest' else: kwargs['initialization'] = None timeoutSecs = 4 * 3600 params = kwargs paramsString = json.dumps(params) start = time.time() kmeans = h2o_cmd.runKMeansOnly(parseKey=parseKey, timeoutSecs=timeoutSecs, benchmarkLogging=benchmarkLogging, **kwargs) elapsed = time.time() - start print "kmeans end on ", csvPathname, 'took', elapsed, 'seconds.', "%d pct. of timeout" % ( (elapsed / timeoutSecs) * 100) l = '{!s} jvms, {!s}GB heap, {:s} {:s} {:s} for {:.2f} secs {:s}'.format( len(h2o.nodes), h2o.nodes[0].java_heap_GB, "KMeans", "trial " + str(trial), csvFilename, elapsed, paramsString) print l h2o.cloudPerfH2O.message(l) (centers, tupleResultList) = h2o_kmeans.bigCheckResults( self, kmeans, csvPathname, parseKey, 'd', **kwargs) # all are multipliers of expected tuple value allowedDelta = (0.01, 0.01, 0.01) h2o_kmeans.compareResultsToExpected(self, tupleResultList, expected, allowedDelta, allowError=True, trial=trial)
def test_B_load_hdfs_and_store_hex_to_hdfs(self): print "\nLoad a list of files from 0xdata hdfs, parse, and store the .hex to hdfs" print "\nYou can try running as hduser/hduser if fail" # larger set in my local dir # fails because classes aren't integers # "allstate_claim_prediction_train_set.zip", csvFilenameAll = [ "covtype.data", "TEST-poker1000.csv", "leads.csv", "and-testing.data", "arcene2_train.both", "arcene_train.both", "bestbuy_test.csv", "bestbuy_train.csv", "covtype.4x.shuffle.data", "covtype4x.shuffle.data", "covtype.13x.data", "covtype.13x.shuffle.data", "covtype.169x.data", "prostate_2g.csv", "prostate_long.csv.gz", "prostate_long_1G.csv", "hhp.unbalanced.012.1x11.data.gz", "hhp.unbalanced.012.data.gz", "hhp.unbalanced.data.gz", "hhp2.os.noisy.0_1.data", "hhp2.os.noisy.9_4.data", "hhp_9_14_12.data", "poker_c1s1_testing_refresh.csv", "3G_poker_shuffle", "billion_rows.csv.gz", "poker-hand.1244M.shuffled311M.full.txt", ] # pick 8 randomly! if (1==0): csvFilenameList = random.sample(csvFilenameAll,8) # Alternatively: do the list in order! Note the order is easy to hard else: csvFilenameList = csvFilenameAll # pop open a browser on the cloud h2b.browseTheCloud() timeoutSecs = 200 # save the first, for all comparisions, to avoid slow drift with each iteration firstglm = {} h2i.setupImportHdfs() for csvFilename in csvFilenameList: # creates csvFilename.hex from file in hdfs dir print "Loading", csvFilename, 'from HDFS' parseKey = h2i.parseImportHdfsFile(csvFilename=csvFilename, path='/datasets', timeoutSecs=1000) print csvFilename, 'parse time:', parseKey['response']['time'] print "parse result:", parseKey['destination_key'] print "\n" + csvFilename start = time.time() print "Storing", parseKey['destination_key'], 'to HDFS' ### print "FIX! temporarily disabling since it causes HDFS corruption" storeKey = h2o_cmd.runStore2HDFS(key=parseKey['destination_key'], timeoutSecs=1000) h2b.browseJsonHistoryAsUrlLastMatch("Parse") sys.stdout.write('.') sys.stdout.flush()
def test_B_hdfs_files(self): # larger set in my local dir # fails because classes aren't integers # "allstate_claim_prediction_train_set.zip", csvFilenameAll = [ "3G_poker_shuffle", "TEST-poker1000.csv", # corrupt zip file? # "allstate_claim_prediction_train_set.zip", "and-testing.data", "arcene2_train.both", "arcene_train.both", "bestbuy_test.csv", "bestbuy_train.csv", "billion_rows.csv.gz", "covtype.13x.data", "covtype.13x.shuffle.data", "covtype.169x.data", "covtype.4x.shuffle.data", "covtype.data", "covtype4x.shuffle.data", "hhp.unbalanced.012.1x11.data.gz", "hhp.unbalanced.012.data.gz", "hhp.unbalanced.data.gz", "hhp2.os.noisy.0_1.data", "hhp2.os.noisy.9_4.data", "hhp_9_14_12.data", "leads.csv", "prostate_long_1G.csv", ] # pick 8 randomly! if (1==0): csvFilenameList = random.sample(csvFilenameAll,8) # Alternatively: do the list in order! Note the order is easy to hard else: csvFilenameList = csvFilenameAll # pop open a browser on the cloud h2b.browseTheCloud() timeoutSecs = 1000 # save the first, for all comparisions, to avoid slow drift with each iteration firstglm = {} h2i.setupImportHdfs() for csvFilename in csvFilenameList: # creates csvFilename.hex from file in hdfs dir start = time.time() print 'Parsing', csvFilename parseKey = h2i.parseImportHdfsFile( csvFilename=csvFilename, path='/datasets', timeoutSecs=timeoutSecs, retryDelaySecs=1.0) print csvFilename, '\nparse time (python)', time.time() - start, 'seconds' print csvFilename, '\nparse time (h2o):', parseKey['response']['time'] ### print h2o.dump_json(parseKey['response']) print "parse result:", parseKey['destination_key'] # I use this if i want the larger set in my localdir inspect = h2o_cmd.runInspect(None, parseKey['destination_key']) ### print h2o.dump_json(inspect) cols = inspect['cols'] # look for nonzero num_missing_values count in each col for i, colDict in enumerate(cols): num_missing_values = colDict['num_missing_values'] if num_missing_values != 0: ### print "%s: col: %d, num_missing_values: %d" % (csvFilename, i, num_missing_values) pass ### print h2o.dump_json(cols[0]) num_cols = inspect['num_cols'] num_rows = inspect['num_rows'] row_size = inspect['row_size'] ptype = inspect['type'] value_size_bytes = inspect['value_size_bytes'] response = inspect['response'] ptime = response['time'] print "num_cols: %s, num_rows: %s, row_size: %s, ptype: %s, \ value_size_bytes: %s, response: %s, time: %s" % \ (num_cols, num_rows, row_size, ptype, value_size_bytes, response, ptime) h2b.browseJsonHistoryAsUrlLastMatch("Inspect") print "\n" + csvFilename # start = time.time() # RFview = h2o_cmd.runRFOnly(trees=1,parseKey=parseKey,timeoutSecs=2000) # h2b.browseJsonHistoryAsUrlLastMatch("RFView") # # wait in case it recomputes it # time.sleep(10) sys.stdout.write('.') sys.stdout.flush()
def test_KMeans_sphere15_180GB(self): csvFilename = 'syn_sphere15_2711545732row_6col_180GB_from_7x.csv' totalBytes = 183538602156 if FROM_HDFS: importFolderPath = "/datasets/kmeans_big" csvPathname = "hdfs://" + importFolderPath + '/' + csvFilename else: importFolderPath = "/home3/0xdiag/datasets/kmeans_big" csvPathname = importFolderPath + '/' + csvFilename # FIX! put right values in # will there be different expected for random vs the other inits? expected = [ ([0.0, -113.00566692375459, -89.99595447985321, -455.9970643424373, 4732.0, 49791778.0, 36800.0], 248846122, 1308149283316.2988) , ([0.0, 1.0, 1.0, -525.0093818313685, 2015.001629398412, 25654042.00592703, 28304.0], 276924291, 1800760152555.98) , ([0.0, 5.0, 2.0, 340.0, 1817.995920197288, 33970406.992053084, 31319.99486705394], 235089554, 375419158808.3253) , ([0.0, 10.0, -72.00113070337981, -171.0198611715457, 4430.00952228909, 37007399.0, 29894.0], 166180630, 525423632323.6474) , ([0.0, 11.0, 3.0, 578.0043558141306, 1483.0163188052604, 22865824.99639042, 5335.0], 167234179, 1845362026223.1094) , ([0.0, 12.0, 3.0, 168.0, -4066.995950679284, 41077063.00269915, -47537.998050740985], 195420925, 197941282992.43475) , ([0.0, 19.00092954923767, -10.999565572612255, 90.00028669073289, 1928.0, 39967190.0, 27202.0], 214401768, 11868360232.658035) , ([0.0, 20.0, 0.0, 141.0, -3263.0030236302937, 6163210.990273981, 30712.99115201907], 258853406, 598863991074.3276) , ([0.0, 21.0, 114.01584574295777, 242.99690338815898, 1674.0029079209912, 33089556.0, 36415.0], 190979054, 1505088759456.314) , ([0.0, 25.0, 1.0, 614.0032787274755, -2275.9931284021022, -48473733.04122273, 47343.0], 87794427, 1124697008162.3955) , ([0.0, 39.0, 3.0, 470.0, -3337.9880599007597, 28768057.98852736, 16716.003410920028], 78226988, 1151439441529.0215) , ([0.0, 40.0, 1.0, 145.0, 950.9990795199593, 14602680.991458317, -14930.007919032574], 167273589, 693036940951.0249) , ([0.0, 42.0, 4.0, 479.0, -3678.0033024834297, 8209673.001421165, 11767.998552236539], 148426180, 35942838893.32379) , ([0.0, 48.0, 4.0, 71.0, -951.0035145455234, 49882273.00063991, -23336.998167498707], 157533313, 88431531357.62982) , ([0.0, 147.00394564757505, 122.98729664236723, 311.0047920137008, 2320.0, 46602185.0, 11212.0], 118361306, 1111537045743.7646) , ] benchmarkLogging = ['cpu','disk', 'network', 'iostats', 'jstack'] benchmarkLogging = ['cpu','disk', 'network', 'iostats'] # IOStatus can hang? benchmarkLogging = ['cpu', 'disk', 'network'] benchmarkLogging = [] for trial in range(6): # IMPORT********************************************** # since H2O deletes the source key, re-import every iteration. if FROM_HDFS: importFolderResult = h2i.setupImportHdfs(None, importFolderPath) else: importFolderResult = h2i.setupImportFolder(None, importFolderPath) # PARSE **************************************** print "Parse starting: " + csvFilename key2 = csvFilename + "_" + str(trial) + ".hex" start = time.time() timeoutSecs = 2 * 3600 kwargs = {} if FROM_HDFS: parseKey = h2i.parseImportHdfsFile(None, csvFilename, importFolderPath, key2=key2, timeoutSecs=timeoutSecs, pollTimeoutsecs=60, retryDelaySecs=2, benchmarkLogging=benchmarkLogging, **kwargs) else: parseKey = h2i.parseImportFolderFile(None, csvFilename, importFolderPath, key2=key2, timeoutSecs=timeoutSecs, pollTimeoutsecs=60, retryDelaySecs=2, benchmarkLogging=benchmarkLogging, **kwargs) elapsed = time.time() - start fileMBS = (totalBytes/1e6)/elapsed l = '{!s} jvms, {!s}GB heap, {:s} {:s} {:6.2f} MB/sec for {:.2f} secs'.format( len(h2o.nodes), h2o.nodes[0].java_heap_GB, 'Parse', csvPathname, fileMBS, elapsed) print "\n"+l h2o.cloudPerfH2O.message(l) # KMeans **************************************** print "col 0 is enum in " + csvFilename + " but KMeans should skip that automatically?? or no?" kwargs = { 'k': 15, 'initialization': 'Furthest', 'epsilon': 1e-6, 'cols': None, 'destination_key': 'junk.hex', # reuse the same seed, to get deterministic results 'seed': 265211114317615310, } if (trial%3)==0: kwargs['initialization'] = 'PlusPlus' elif (trial%3)==1: kwargs['initialization'] = 'Furthest' else: kwargs['initialization'] = None timeoutSecs = 4 * 3600 params = kwargs paramsString = json.dumps(params) start = time.time() kmeans = h2o_cmd.runKMeansOnly(parseKey=parseKey, timeoutSecs=timeoutSecs, benchmarkLogging=benchmarkLogging, **kwargs) elapsed = time.time() - start print "kmeans end on ", csvPathname, 'took', elapsed, 'seconds.', "%d pct. of timeout" % ((elapsed/timeoutSecs) * 100) l = '{!s} jvms, {!s}GB heap, {:s} {:s} {:s} for {:.2f} secs {:s}' .format( len(h2o.nodes), h2o.nodes[0].java_heap_GB, "KMeans", "trial "+str(trial), csvFilename, elapsed, paramsString) print l h2o.cloudPerfH2O.message(l) (centers, tupleResultList) = h2o_kmeans.bigCheckResults(self, kmeans, csvPathname, parseKey, 'd', **kwargs) # all are multipliers of expected tuple value allowedDelta = (0.01, 0.01, 0.01) h2o_kmeans.compareResultsToExpected(self, tupleResultList, expected, allowedDelta, allowError=True, trial=trial)
def test_B_load_hdfs_and_store_hex_to_hdfs(self): print "\nLoad a list of files from 0xdata hdfs, parse, and store the .hex to hdfs" print "\nYou can try running as hduser/hduser if fail" # larger set in my local dir # fails because classes aren't integers # "allstate_claim_prediction_train_set.zip", csvFilenameAll = [ "covtype.data", "TEST-poker1000.csv", "leads.csv", "and-testing.data", "arcene2_train.both", "arcene_train.both", "bestbuy_test.csv", "bestbuy_train.csv", "covtype.4x.shuffle.data", "covtype4x.shuffle.data", "covtype.13x.data", "covtype.13x.shuffle.data", "covtype.169x.data", "prostate_2g.csv", "prostate_long.csv.gz", "prostate_long_1G.csv", "hhp.unbalanced.012.1x11.data.gz", "hhp.unbalanced.012.data.gz", "hhp.unbalanced.data.gz", "hhp2.os.noisy.0_1.data", "hhp2.os.noisy.9_4.data", "hhp_9_14_12.data", "poker_c1s1_testing_refresh.csv", "3G_poker_shuffle", "billion_rows.csv.gz", "poker-hand.1244M.shuffled311M.full.txt", ] # pick 8 randomly! if (1 == 0): csvFilenameList = random.sample(csvFilenameAll, 8) # Alternatively: do the list in order! Note the order is easy to hard else: csvFilenameList = csvFilenameAll # pop open a browser on the cloud h2b.browseTheCloud() timeoutSecs = 200 # save the first, for all comparisions, to avoid slow drift with each iteration firstglm = {} h2i.setupImportHdfs() for csvFilename in csvFilenameList: # creates csvFilename.hex from file in hdfs dir print "Loading", csvFilename, 'from HDFS' parseKey = h2i.parseImportHdfsFile(csvFilename=csvFilename, path='/datasets', timeoutSecs=1000) print csvFilename, 'parse time:', parseKey['response']['time'] print "parse result:", parseKey['destination_key'] print "\n" + csvFilename start = time.time() print "Storing", parseKey['destination_key'], 'to HDFS' ### print "FIX! temporarily disabling since it causes HDFS corruption" storeKey = h2o_cmd.runStore2HDFS(key=parseKey['destination_key'], timeoutSecs=1000) h2b.browseJsonHistoryAsUrlLastMatch("Parse") sys.stdout.write('.') sys.stdout.flush()
def test_B_hdfs_files(self): # larger set in my local dir # fails because classes aren't integers # "allstate_claim_prediction_train_set.zip", csvFilenameAll = [ "3G_poker_shuffle", "TEST-poker1000.csv", # corrupt zip file? # "allstate_claim_prediction_train_set.zip", "and-testing.data", "arcene2_train.both", "arcene_train.both", "bestbuy_test.csv", "bestbuy_train.csv", "billion_rows.csv.gz", "covtype.13x.data", "covtype.13x.shuffle.data", "covtype.169x.data", "covtype.4x.shuffle.data", "covtype.data", "covtype4x.shuffle.data", "hhp.unbalanced.012.1x11.data.gz", "hhp.unbalanced.012.data.gz", "hhp.unbalanced.data.gz", "hhp2.os.noisy.0_1.data", "hhp2.os.noisy.9_4.data", "hhp_9_14_12.data", "leads.csv", "prostate_long_1G.csv", ] # pick 8 randomly! if (1 == 0): csvFilenameList = random.sample(csvFilenameAll, 8) # Alternatively: do the list in order! Note the order is easy to hard else: csvFilenameList = csvFilenameAll # pop open a browser on the cloud h2b.browseTheCloud() timeoutSecs = 1000 # save the first, for all comparisions, to avoid slow drift with each iteration firstglm = {} h2i.setupImportHdfs() for csvFilename in csvFilenameList: # creates csvFilename.hex from file in hdfs dir start = time.time() print 'Parsing', csvFilename parseKey = h2i.parseImportHdfsFile(csvFilename=csvFilename, path='/datasets', timeoutSecs=timeoutSecs, retryDelaySecs=1.0) print csvFilename, '\nparse time (python)', time.time( ) - start, 'seconds' print csvFilename, '\nparse time (h2o):', parseKey['response'][ 'time'] ### print h2o.dump_json(parseKey['response']) print "parse result:", parseKey['destination_key'] # I use this if i want the larger set in my localdir inspect = h2o_cmd.runInspect(None, parseKey['destination_key']) ### print h2o.dump_json(inspect) cols = inspect['cols'] # look for nonzero num_missing_values count in each col for i, colDict in enumerate(cols): num_missing_values = colDict['num_missing_values'] if num_missing_values != 0: ### print "%s: col: %d, num_missing_values: %d" % (csvFilename, i, num_missing_values) pass ### print h2o.dump_json(cols[0]) num_cols = inspect['num_cols'] num_rows = inspect['num_rows'] row_size = inspect['row_size'] ptype = inspect['type'] value_size_bytes = inspect['value_size_bytes'] response = inspect['response'] ptime = response['time'] print "num_cols: %s, num_rows: %s, row_size: %s, ptype: %s, \ value_size_bytes: %s, response: %s, time: %s" % \ (num_cols, num_rows, row_size, ptype, value_size_bytes, response, ptime) # h2b.browseJsonHistoryAsUrlLastMatch("Inspect") print "\n" + csvFilename
def test_import_nflx_parse_loop(self): print "Using the -.gz files from hdfs" # hdfs://<name node>/datasets/manyfiles-nflx-gz/file_1.dat.gz csvFilename = "file_10.dat.gz" csvFilepattern = "file_1[0-9].dat.gz" trialMax = 2 for tryHeap in [24]: print "\n", tryHeap, "GB heap, 1 jvm per host, import 192.168.1.176 hdfs, then parse" localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(node_count=1, java_heap_GB=tryHeap, use_hdfs=True, hdfs_name_node='192.168.1.176', hdfs_version='cdh3') else: h2o_hosts.build_cloud_with_hosts( node_count=1, java_heap_GB=tryHeap, use_hdfs=True, hdfs_name_node='192.168.1.176', hdfs_version='cdh3') # don't raise exception if we find something bad in h2o stdout/stderr? # h2o.nodes[0].sandbox_ignore_errors = True timeoutSecs = 500 importFolderPath = "/datasets/manyfiles-nflx-gz" for trial in range(trialMax): # since we delete the key, we have to re-import every iteration, to get it again importHdfsResult = h2i.setupImportHdfs(path=importFolderPath) hdfsFullList = importHdfsResult['succeeded'] for k in hdfsFullList: key = k['key'] # just print the first tile if 'nflx' in key and 'file_1.dat.gz' in key: # should be hdfs://home-0xdiag-datasets/manyfiles-nflx-gz/file_1.dat.gz print "example file we'll use:", key ### print "hdfsFullList:", h2o.dump_json(hdfsFullList) # error if none? self.assertGreater(len(hdfsFullList), 8, "Didn't see more than 8 files in hdfs?") key2 = csvFilename + "_" + str(trial) + ".hex" csvFilePattern = 'file_1.dat.gz' # "key": "hdfs://192.168.1.176/datasets/manyfiles-nflx-gz/file_99.dat.gz", time.sleep(5) print "Loading from hdfs:", importFolderPath + "/" + csvFilePattern start = time.time() parseKey = h2i.parseImportHdfsFile(csvFilename=csvFilePattern, path=importFolderPath, key2=key2, timeoutSecs=timeoutSecs, retryDelaySecs=10, pollTimeoutSecs=60) elapsed = time.time() - start print hdfsKey, 'parse time:', parseKey['response']['time'] print "parse result:", parseKey['destination_key'] print "Parse #", trial, "completed in", "%6.2f" % elapsed, "seconds.", \ "%d pct. of timeout" % ((elapsed*100)/timeoutSecs) h2o_cmd.runStoreView() h2o.tear_down_cloud() # sticky ports? wait a bit. time.sleep(5)
def test_B_hdfs_files(self): print "\nLoad a list of files from HDFS, parse and do 1 RF tree" print "\nYou can try running as hduser/hduser if fail" # larger set in my local dir # fails because classes aren't integers # "allstate_claim_prediction_train_set.zip", csvFilenameAll = [ "TEST-poker1000.csv", "leads.csv", "and-testing.data", "arcene2_train.both", "arcene_train.both", # these can't RF ..output classes not integer? # "bestbuy_test.csv", # "bestbuy_train.csv", "covtype.data", "covtype.4x.shuffle.data", "covtype4x.shuffle.data", "covtype.13x.data", "covtype.13x.shuffle.data", # "covtype.169x.data", # "prostate_2g.csv", # "prostate_long.csv.gz", "prostate_long_1G.csv", "hhp.unbalanced.012.1x11.data.gz", "hhp.unbalanced.012.data.gz", "hhp.unbalanced.data.gz", "hhp2.os.noisy.0_1.data", "hhp2.os.noisy.9_4.data", "hhp_9_14_12.data", # "poker_c1s1_testing_refresh.csv", # "3G_poker_shuffle", # "billion_rows.csv.gz", # "poker-hand.1244M.shuffled311M.full.txt", ] # pick 8 randomly! if (1 == 0): csvFilenameList = random.sample(csvFilenameAll, 8) # Alternatively: do the list in order! Note the order is easy to hard else: csvFilenameList = csvFilenameAll # pop open a browser on the cloud h2b.browseTheCloud() timeoutSecs = 200 # save the first, for all comparisions, to avoid slow drift with each iteration firstglm = {} h2i.setupImportHdfs() for csvFilename in csvFilenameList: # creates csvFilename.hex from file in hdfs dir print "Loading", csvFilename, 'from HDFS' parseKey = h2i.parseImportHdfsFile(csvFilename=csvFilename, path='/datasets', timeoutSecs=1000) print csvFilename, 'parse time:', parseKey['response']['time'] print "parse result:", parseKey['destination_key'] print "\n" + csvFilename start = time.time() RFview = h2o_cmd.runRFOnly(trees=1, parseKey=parseKey, timeoutSecs=2000) h2b.browseJsonHistoryAsUrlLastMatch("RFView") # wait in case it recomputes it time.sleep(10) sys.stdout.write('.') sys.stdout.flush()