def setUpClass(cls):
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(2,java_heap_GB=4,java_extra_args='-XX:+PrintCompilation')
else:
h2o_hosts.build_cloud_with_hosts(java_extra_args='-XX:+PrintCompilation')
def setUpClass(cls):
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(3,
use_hdfs=True, hdfs_version='cdh3', hdfs_name_node='192.168.1.176')
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
global localhost
localhost = h2o.decide_if_localhost()
if localhost:
h2o.build_cloud()
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
localhost = h2o.decide_if_localhost()
if (localhost):
# maybe fails more reliably with just 2 jvms?
h2o.build_cloud(2,java_heap_GB=5)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1,java_heap_GB=4)
else:
h2o_hosts.build_cloud_with_hosts()
def test_import_covtype_parse_loop(self):
csvFilename = "covtype.data"
importFolderPath = "/home/0xdiag/datasets/standard"
trialMax = 2
localhost = h2o.decide_if_localhost()
for tryHeap in [4, 3, 2, 1]:
print "\n", tryHeap, "GB heap, 1 jvms, import folder, then loop parsing 'covtype.data' to unique keys"
if (localhost):
h2o.build_cloud(node_count=1, java_heap_GB=tryHeap)
else:
h2o_hosts.build_cloud_with_hosts(
node_count=1, java_heap_GB=tryHeap)
for trial in range(trialMax):
# import each time, because h2o deletes source file after parse
h2i.setupImportFolder(None, importFolderPath)
key2 = csvFilename + "_" + str(trial) + ".hex"
parseKey = h2i.parseImportFolderFile(
None,
csvFilename,
importFolderPath,
key2=key2,
timeoutSecs=20)
# sticky ports?
h2o.tear_down_cloud()
time.sleep(2)
def setUpClass(cls):
localhost = h2o.decide_if_localhost()
if localhost:
# h2o.build_cloud(3, java_heap_GB=4, base_port=54323)
h2o.build_cloud(3, java_heap_GB=12, base_port=54323)
else:
h2o_hosts.build_cloud_with_hosts(base_port=54323)
def setUpClass(cls):
global SEED, localhost
localhost = h2o.decide_if_localhost()
if localhost:
h2o.build_cloud(java_heap_GB=10)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
print "Will build_cloud() with random heap size and do overlapped import folder/parse (groups)"
global SEED, localhost
SEED = h2o.setup_random_seed()
if RANDOM_HEAP:
tryHeap = random.randint(4, 28)
else:
tryHeap = 28
# print "\n", tryHeap,"GB heap, 1 jvm per host, import 192.168.1.176 hdfs, then parse"
print "\n", tryHeap, "GB heap, 1 jvm per host, import, then parse"
localhost = h2o.decide_if_localhost()
h2o.beta_features = True # for the beta tab in the browser
if localhost:
h2o.build_cloud(
node_count=3,
java_heap_GB=4,
base_port=54323,
# use_hdfs=True, hdfs_name_node='192.168.1.176', hdfs_version='cdh3'
)
else:
h2o_hosts.build_cloud_with_hosts(
node_count=1,
java_heap_GB=tryHeap,
base_port=54321,
# use_hdfs=True, hdfs_name_node='192.168.1.176', hdfs_version='cdh3'
)
def setUpClass(cls):
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(node_count=1, java_heap_GB=10, base_port=54333)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
# assume we're at 0xdata with it's hdfs namenode
global localhost
localhost = h2o.decide_if_localhost()
# hdfs_config='/opt/mapr/conf/mapr-clusters.conf',
# # hdfs_name_node='mr-0x1.0xdata.loc:7222')
# hdfs_version='mapr2.1.3',
if localhost:
h2o.build_cloud(
1,
java_heap_GB=15,
enable_benchmark_log=True,
use_maprfs=True,
hdfs_version="mapr3.0.1",
hdfs_name_node="192.168.1.171:7222",
)
else:
h2o_hosts.build_cloud_with_hosts(
1,
java_heap_GB=15,
enable_benchmark_log=True,
use_maprfs=True,
hdfs_version="mapr3.0.1",
hdfs_name_node="192.168.1.171:7222",
)
def setUpClass(cls):
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(2,java_heap_GB=10,use_flatfile=True)
else:
import h2o_hosts
h2o_hosts.build_cloud_with_hosts()
def test_import_covtype_parse_3jvm_fvec(self):
h2o.beta_features = True
csvFilename = "covtype.data"
importFolderPath = "standard"
trialMax = 2
for tryHeap in [1]:
print "\n", tryHeap,"GB heap, 3 jvms, import folder, then loop parsing 'covtype.data' to unique keys"
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(node_count=3, java_heap_GB=tryHeap)
else:
h2o_hosts.build_cloud_with_hosts(node_count=3, java_heap_GB=tryHeap)
for trial in range(trialMax):
# import each time, because h2o deletes source file after parse
csvPathname = importFolderPath + "/" + csvFilename
hex_key = csvFilename + "_" + str(trial) + ".hex"
parseResult = h2i.import_parse(bucket='home-0xdiag-datasets', path=csvPathname, hex_key=hex_key, timeoutSecs=20)
# sticky ports?
h2o.tear_down_cloud()
time.sleep(5)
# print "Waiting 60 secs for TIME_WAIT sockets to go away"
# time.sleep(60)
time.sleep(2)
def setUpClass(cls):
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1,java_heap_GB=14, enable_benchmark_log=True)
else:
h2o_hosts.build_cloud_with_hosts(enable_benchmark_log=True)
def setUpClass(cls):
global localhost
localhost = h2o.decide_if_localhost()
if localhost:
h2o.build_cloud(node_count=1, java_heap_GB=10)
else:
h2o_hosts.build_cloud_with_hosts(node_count=1, java_heap_GB=10)
def setUpClass(cls):
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(node_count=1)
else:
h2o_hosts.build_cloud_with_hosts(node_count=1)
def setUpClass(cls):
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(3)
else:
h2o_hosts.build_cloud_with_hosts()
h2b.browseTheCloud()
def setUpClass(cls):
localhost = h2o.decide_if_localhost()
h2o.beta_features = True
if (localhost):
h2o.build_cloud(3, java_heap_GB=1, use_hdfs=True)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
localhost = h2o.decide_if_localhost()
h2o.beta_features = True
if localhost:
h2o.build_cloud(3, java_heap_GB=1, use_hdfs=True, base_port=54321)
else:
h2o_hosts.build_cloud_with_hosts(base_port=54321)
def setUpClass(cls):
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(3)
else:
h2o_hosts.build_cloud_with_hosts()
h2o.beta_features = True
def setUpClass(cls):
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(2, java_heap_GB=5)
else:
print "1 jvms per node, 28GB heap each"
h2o_hosts.build_cloud_with_hosts(node_count=1, java_heap_GB=28)
def setUpClass(cls):
global SEED, localhost
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1, java_heap_GB=10, enable_benchmark_log=True)
else:
h2o_hosts.build_cloud_with_hosts(enable_benchmark_log=True)
def setUpClass(cls):
global SEED, localhost
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1, use_flatfile=True)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
global SEED, localhost
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(node_count=1, java_heap_GB=10)
else:
h2o_hosts.build_cloud_with_hosts(node_count=1, java_heap_GB=10)
def setUpClass(cls):
# assume we're at 0xdata with it's hdfs namenode
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
global SEED, localhost
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(CLOUD_SIZE, java_heap_GB=12 / CLOUD_SIZE)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
params = collectConf(cls)
h2o.build_cloud(**params)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
global SEED, localhost
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(node_count=1, base_port=54327)
else:
h2o_hosts.build_cloud_with_hosts(node_count=1)
def setUpClass(cls):
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(2,java_heap_GB=5)
else:
h2o_hosts.build_cloud_with_hosts()
global SYNDATASETS_DIR
SYNDATASETS_DIR = h2o.make_syn_dir()
def setUpClass(cls):
global SEED, localhost
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(2,java_heap_GB=7)
else:
h2o_hosts.build_cloud_with_hosts() # uses import Hdfs for s3n instead of import folder
def setUpClass(cls):
# assume we're at 0xdata with it's hdfs namenode
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1, use_hdfs=True, hdfs_version='cdh4', hdfs_name_node='172.16.2.176')
else:
h2o_hosts.build_cloud_with_hosts(1, use_hdfs=True, hdfs_version='cdh4', hdfs_name_node='172.16.2.176')
def setUpClass(cls):
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1)
else:
h2o_hosts.build_cloud_with_hosts(1)
h2b.browseTheCloud()
def setUpClass(cls):
global SEED, localhost
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(3, java_heap_GB=1) # enum processing is more interesting with multiple jvms
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
# assume we're at 0xdata with it's hdfs namenode
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1, use_hdfs=True, hdfs_version='cdh3u5', hdfs_name_node='192.168.1.176')
else:
h2o_hosts.build_cloud_with_hosts(1, use_hdfs=True, hdfs_version='cdh3u5', hdfs_name_node='192.168.1.176')
def setUpClass(cls):
global SEED, localhost
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(3, java_heap_GB=1) # enum processing is more interesting with multiple jvms
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
global SEED, localhost
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(node_count=1, base_port=54327)
else:
h2o_hosts.build_cloud_with_hosts(node_count=1)
def setUpClass(cls):
# assume we're at 0xdata with it's hdfs namenode
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1, java_heap_GB=14)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
# done in build_cloud now
### h2o.write_flatfile(node_count=3)
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(node_count=3,use_flatfile=True)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
global SEED, localhost
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1,java_heap_GB=1)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
global SEED, localhost
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(2,java_heap_MB=1300,use_flatfile=True)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(java_heap_GB=4, random_udp_drop=RANDOM_UDP_DROP,
use_hdfs=True, hdfs_version='cdh4', hdfs_name_node='mr-0x6')
else:
h2o_hosts.build_cloud_with_hosts(random_udp_drop=RANDOM_UDP_DROP)
def setUpClass(cls):
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1)
else:
h2o_hosts.build_cloud_with_hosts(1)
h2o.beta_features = True # fvec
def setUpClass(cls):
global SEED, localhost
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(3,java_heap_GB=4)
else:
h2o_hosts.build_cloud_with_hosts() # uses import Hdfs for s3n instead of import folder
def setUpClass(cls):
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1)
else:
h2o_hosts.build_cloud_with_hosts(1)
global SYNDATASETS_DIR
SYNDATASETS_DIR = h2o.make_syn_dir()
def setUpClass(cls):
localhost = h2o.decide_if_localhost()
# want detail on the cloud building to see what node fails
h2o.verbose = True
if (localhost):
h2o.build_cloud(1, java_heap_GB=2, base_port=54323)
else:
h2o_hosts.build_cloud_with_hosts(base_port=54323)
h2o.verbose = False
def setUpClass(cls):
# assume we're at 0xdata with it's hdfs namenode
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1, java_heap_GB=14)
else:
# all hdfs info is done thru the hdfs_config michal's ec2 config sets up?
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
global SEED
SEED = h2o.setup_random_seed()
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(node_count=1)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
global SEED, localhost
# SEED = h2o.setup_random_seed()
SEED = 8977501266014959103
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(node_count=1)
else:
h2o_hosts.build_cloud_with_hosts(node_count=1)
def setUpClass(cls):
global SEED, localhost
# use the known bad seed if it's set. otherwise should be None
SEED = h2o.setup_random_seed(seed=BAD_SEED)
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(3)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
global SEED, localhost
# SEED = h2o.setup_random_seed()
SEED = 6019110937119320453
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1, java_heap_GB=4)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
# fails with 3
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(3, java_heap_GB=4, use_flatfile=True)
else:
h2o_hosts.build_cloud_with_hosts()
h2b.browseTheCloud()
def setUpClass(cls):
global SEED, localhost
# SEED = h2o.setup_random_seed()
SEED = 6204672511291494176
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1)
else:
h2o_hosts.build_cloud_with_hosts(1)
def setUpClass(cls):
global SEED, localhost
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(2, java_heap_MB=1300, use_flatfile=True)
else:
h2o_hosts.build_cloud_with_hosts()
h2o.beta_features = True
def test_parse_nflx_loop_hdfs_fvec(self):
h2o.beta_features = True
print "Using the -.gz files from hdfs"
# hdfs://<name node>/datasets/manyfiles-nflx-gz/file_1.dat.gz
# default
csvFilename = "hex_10"
csvFilePattern = '*' # all files in the folder
for tryHeap in [24]:
print "\n", tryHeap,"GB heap, 1 jvm per host, import mr-0x6 hdfs, then parse"
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(java_heap_GB=tryHeap, random_udp_drop=RANDOM_UDP_DROP, base_port=55930,
use_hdfs=True, hdfs_name_node=NAME_NODE, hdfs_version=VERSION)
else:
h2o_hosts.build_cloud_with_hosts(java_heap_GB=tryHeap, random_udp_drop=RANDOM_UDP_DROP, base_port=55600,
use_hdfs=True, hdfs_name_node=NAME_NODE, hdfs_version=VERSION)
# don't raise exception if we find something bad in h2o stdout/stderr?
# h2o.nodes[0].sandboxIgnoreErrors = True
timeoutSecs = 500
importFolderPath = "datasets/airlines_multi"
csvPathname = importFolderPath + "/" + csvFilePattern
parseResult = h2i.import_only(path=csvPathname, schema='hdfs',
timeoutSecs=timeoutSecs, retryDelaySecs=10, pollTimeoutSecs=60)
for trial in range(TRIAL_MAX):
# each parse now just does one
csvFilePattern = "*%s.csv" % trial
# if we want multifile
# csvFilePattern = "*"
hex_key = csvFilename + "_" + str(trial) + ".hex"
csvPathname = importFolderPath + "/" + csvFilePattern
start = time.time()
# print "Don't wait for completion. Just load things up!"
print "Drat. the source file is locked if we noPoll. Would have to increment across the individual files?"
print "Drat. We can't re-import the folder, if there's a parse using one of the source files?"
parseResult = h2i.parse_only(pattern=csvFilePattern, hex_key=hex_key, noPoll=True, delete_on_done=0,
timeoutSecs=timeoutSecs, retryDelaySecs=10, pollTimeoutSecs=60)
elapsed = time.time() - start
print "parse result:", parseResult['destination_key']
print "Parse #", trial, "completed in", "%6.2f" % elapsed, "seconds.", \
"%d pct. of timeout" % ((elapsed*100)/timeoutSecs)
h2o_cmd.runStoreView()
# we don't delete the hex key. it will start spilling? slow
h2j.pollWaitJobs(timeoutSecs=300, pollTimeoutSecs=30)
h2o.tear_down_cloud()
# sticky ports? wait a bit.
time.sleep(5)
def setUpClass(cls):
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(3,
use_hdfs=True,
hdfs_version='cdh3',
hdfs_name_node='192.168.1.176')
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
global SEED, localhost
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(2, java_heap_GB=4, use_flatfile=True)
else:
h2o_hosts.build_cloud_with_hosts()
h2b.browseTheCloud()
def setUpClass(cls):
h2o.beta_features = True
global SEED, localhost
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1, java_heap_GB=12)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
# assume we're at 0xdata with it's hdfs namenode
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(1, java_heap_GB=14)
else:
h2o_hosts.build_cloud_with_hosts()
h2o.beta_features = True
def setUpClass(cls):
global SEED
SEED = h2o.setup_random_seed()
localhost = h2o.decide_if_localhost()
print "WARNING: won't work for remote h2o, because syn_datasets is created locally only, for import"
if (localhost):
h2o.build_cloud(1, java_heap_GB=14)
else:
h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls):
global SEED, localhost
SEED = h2o.setup_random_seed()
global localhost
localhost = h2o.decide_if_localhost()
if (localhost):
h2o.build_cloud(3, java_heap_GB=4)
else:
h2o_hosts.build_cloud_with_hosts()
