def setUpClass(cls): localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(3, use_hdfs=True, hdfs_version='cdh3', hdfs_name_node='192.168.1.176') else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): global SEED, localhost SEED = h2o.setup_random_seed() if localhost: h2o.build_cloud(2) else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): localhost = h2o.decide_if_localhost() h2o.beta_features = True if localhost: h2o.build_cloud(3, java_heap_GB=1, use_hdfs=True, base_port=54321) else: h2o_hosts.build_cloud_with_hosts(base_port=54321)
def test_C_no_mc_rcv(self): print "\nwith flatfile, with multicast disabled" allAcceptIptables() multicastDropReceiveIptables() showIptables() h2o_hosts.build_cloud_with_hosts(nodes_per_host, use_flatfile=True) h2o.tear_down_cloud()
def test_import_billion_rows_parse_loop(self): print "Apparently we can't handle 1B rows .gzed" csvFilename = "billion_rows.csv.gz" importFolderPath = "standard" csvPathname = importFolderPath + "/" + csvFilename trialMax = 3 for tryHeap in [4,16]: print "\n", tryHeap,"GB heap, 1 jvm per host, import folder,", \ "then loop parsing 'billion_rows.csv' to unique keys" h2o_hosts.build_cloud_with_hosts(1, java_heap_GB=tryHeap) timeoutSecs=800 for trial in range(trialMax): hex_key = csvFilename + "_" + str(trial) + ".hex" start = time.time() parseResult = h2i.import_parse(bucket='home-0xdiag-datasets', path=csvPathname, hex_key=hex_key, timeoutSecs=timeoutSecs, retryDelaySecs=4, pollTimeoutSecs=60) elapsed = time.time() - start print "Trial #", trial, "completed in", elapsed, "seconds.", \ "%d pct. of timeout" % ((elapsed*100)/timeoutSecs) print "Deleting key in H2O so we get it from S3 (if ec2) or nfs again. ", \ "Otherwise it would just parse the cached key." storeView = h2o.nodes[0].store_view() ### print "storeView:", h2o.dump_json(storeView) # sticky ports? h2o.tear_down_cloud() time.sleep(5)
def setUpClass(cls): localhost = h2o.decide_if_localhost() if (localhost): # maybe fails more reliably with just 2 jvms? h2o.build_cloud(2,java_heap_GB=5) else: h2o_hosts.build_cloud_with_hosts()
def test_D_no_mc_snd(self): print "\nwith flatfile, with multicast disabled" allAcceptIptables() multicastBlockSendIptables() showIptables() h2o_hosts.build_cloud_with_hosts(nodes_per_host, use_flatfile=True) h2o.tear_down_cloud()
def test_parse_covtype20x_loop(self): csvFilename = "covtype20x.data" importFolderPath = "/home/0xdiag/datasets" trialMax = 2 for tryJvms in [1,2,3,4]: for tryHeap in [1,3]: print "\n", tryHeap,"GB heap,", tryJvms, "jvm per host, import folder,", \ "then loop parsing 'covtype20x.data' to unique keys" h2o_hosts.build_cloud_with_hosts(node_count=tryJvms, java_heap_GB=tryHeap) timeoutSecs=300 for trial in range(trialMax): # since we delete the key, we have to re-import every iteration, to get it again h2i.setupImportFolder(None, importFolderPath) key2 = csvFilename + "_" + str(trial) + ".hex" start = time.time() parseKey = h2i.parseImportFolderFile(None, csvFilename, importFolderPath, key2=key2, timeoutSecs=timeoutSecs, retryDelaySecs=4, pollTimeoutSecs=60) elapsed = time.time() - start print "Trial #", trial, "completed in", elapsed, "seconds.", \ "%d pct. of timeout" % ((elapsed*100)/timeoutSecs) print "Deleting key in H2O so we get it from S3 (if ec2) or nfs again.", \ "Otherwise it would just parse the cached key." storeView = h2o.nodes[0].store_view() ### print "storeView:", h2o.dump_json(storeView) # h2o removes key after parse now ## print "Removing", parseKey['source_key'] ## removeKeyResult = h2o.nodes[0].remove_key(key=parseKey['source_key']) ### print "removeKeyResult:", h2o.dump_json(removeKeyResult) # sticky ports? h2o.tear_down_cloud() time.sleep(tryJvms * 5)
def setUpClass(cls): localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(3) else: h2o_hosts.build_cloud_with_hosts() h2b.browseTheCloud()
def setUpClass(cls): localhost = h2o.decide_if_localhost() h2o.beta_features = True if (localhost): h2o.build_cloud(3, java_heap_GB=1, use_hdfs=True) else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): global localhost localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(1,java_heap_GB=4) else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): localhost = h2o.decide_if_localhost() if localhost: # h2o.build_cloud(3, java_heap_GB=4, base_port=54323) h2o.build_cloud(3, java_heap_GB=12, base_port=54323) else: h2o_hosts.build_cloud_with_hosts(base_port=54323)
def setUpClass(cls): print "Will build_cloud() with random heap size and do overlapped import folder/parse (groups)" global SEED, localhost SEED = h2o.setup_random_seed() if RANDOM_HEAP: tryHeap = random.randint(4, 28) else: tryHeap = 28 # print "\n", tryHeap,"GB heap, 1 jvm per host, import 192.168.1.176 hdfs, then parse" print "\n", tryHeap, "GB heap, 1 jvm per host, import, then parse" localhost = h2o.decide_if_localhost() h2o.beta_features = True # for the beta tab in the browser if localhost: h2o.build_cloud( node_count=3, java_heap_GB=4, base_port=54323, # use_hdfs=True, hdfs_name_node='192.168.1.176', hdfs_version='cdh3' ) else: h2o_hosts.build_cloud_with_hosts( node_count=1, java_heap_GB=tryHeap, base_port=54321, # use_hdfs=True, hdfs_name_node='192.168.1.176', hdfs_version='cdh3' )
def setUpClass(cls): global localhost localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(node_count=1) else: h2o_hosts.build_cloud_with_hosts(node_count=1)
def test_import_covtype_parse_3jvm_fvec(self): h2o.beta_features = True csvFilename = "covtype.data" importFolderPath = "standard" trialMax = 2 for tryHeap in [1]: print "\n", tryHeap,"GB heap, 3 jvms, import folder, then loop parsing 'covtype.data' to unique keys" localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(node_count=3, java_heap_GB=tryHeap) else: h2o_hosts.build_cloud_with_hosts(node_count=3, java_heap_GB=tryHeap) for trial in range(trialMax): # import each time, because h2o deletes source file after parse csvPathname = importFolderPath + "/" + csvFilename hex_key = csvFilename + "_" + str(trial) + ".hex" parseResult = h2i.import_parse(bucket='home-0xdiag-datasets', path=csvPathname, hex_key=hex_key, timeoutSecs=20) # sticky ports? h2o.tear_down_cloud() time.sleep(5) # print "Waiting 60 secs for TIME_WAIT sockets to go away" # time.sleep(60) time.sleep(2)
def setUpClass(cls): global localhost localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(1,java_heap_GB=14, enable_benchmark_log=True) else: h2o_hosts.build_cloud_with_hosts(enable_benchmark_log=True)
def setUpClass(cls): global local_host local_host = not 'hosts' in os.getcwd() if (local_host): h2o.build_cloud(1,java_heap_GB=4) else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): global localhost localhost = h2o.decide_if_localhost() if localhost: h2o.build_cloud() else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): global localhost localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(node_count=1, java_heap_GB=10, base_port=54333) else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(2,java_heap_GB=10,use_flatfile=True) else: import h2o_hosts h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(3) else: h2o_hosts.build_cloud_with_hosts() h2o.beta_features = True
def setUpClass(cls): global SEED, localhost localhost = h2o.decide_if_localhost() if localhost: h2o.build_cloud(java_heap_GB=10) else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): start = time.time() h2o_hosts.build_cloud_with_hosts(node_count, base_port=base_port, use_flatfile=True, java_heap_GB=1) print "Cloud of", len(h2o.nodes), "built in", time.time()-start, "seconds" h2o.verify_cloud_size() h2o.check_sandbox_for_errors()
def setUpClass(cls): global localhost localhost = h2o.decide_if_localhost() if localhost: h2o.build_cloud(node_count=1, java_heap_GB=10) else: h2o_hosts.build_cloud_with_hosts(node_count=1, java_heap_GB=10)
def test_import_covtype_parse_loop(self): csvFilename = "covtype.data" importFolderPath = "/home/0xdiag/datasets/standard" trialMax = 2 localhost = h2o.decide_if_localhost() for tryHeap in [4, 3, 2, 1]: print "\n", tryHeap, "GB heap, 1 jvms, import folder, then loop parsing 'covtype.data' to unique keys" if (localhost): h2o.build_cloud(node_count=1, java_heap_GB=tryHeap) else: h2o_hosts.build_cloud_with_hosts( node_count=1, java_heap_GB=tryHeap) for trial in range(trialMax): # import each time, because h2o deletes source file after parse h2i.setupImportFolder(None, importFolderPath) key2 = csvFilename + "_" + str(trial) + ".hex" parseKey = h2i.parseImportFolderFile( None, csvFilename, importFolderPath, key2=key2, timeoutSecs=20) # sticky ports? h2o.tear_down_cloud() time.sleep(2)
def setUpClass(cls): # Uses your username specific json: pytest_config-<username>.json # do what my json says, but with my hdfs. hdfs_name_node from the json # I'll set use_hdfs to False here, because H2O won't start if it can't talk to the hdfs # h2o_hosts.build_cloud_with_hosts(use_hdfs=False) h2o_hosts.build_cloud_with_hosts(use_hdfs=True)
def setUpClass(cls): global local_host local_host = not "hosts" in os.getcwd() if local_host: h2o.build_cloud(1, java_heap_GB=1) else: h2o_hosts.build_cloud_with_hosts()
def test_parse_covtype20x_loop_s3n_hdfs(self): bucket = 'home-0xdiag-datasets' importFolderPath = "standard" csvFilename = "covtype20x.data" csvPathname = importFolderPath + "/" + csvFilename timeoutSecs = 500 trialMax = 3 for tryHeap in [4,12]: print "\n", tryHeap,"GB heap, 1 jvm per host, import folder,", \ "then parse 'covtype20x.data'" h2o_hosts.build_cloud_with_hosts(node_count=1, java_heap_GB=tryHeap) # don't raise exception if we find something bad in h2o stdout/stderr? h2o.nodes[0].sandboxIgnoreErrors = True for trial in range(trialMax): hex_key = csvFilename + ".hex" start = time.time() parseResult = h2i.import_parse(bucket=bucket, path=csvPathname, schema='s3n', hex_key=hex_key, timeoutSecs=timeoutSecs, retryDelaySecs=10, pollTimeoutSecs=60) elapsed = time.time() - start print "parse result:", parseResult['destination_key'] print "Trial #", trial, "completed in", elapsed, "seconds.", \ "%d pct. of timeout" % ((elapsed*100)/timeoutSecs) removeKeyResult = h2o.nodes[0].remove_key(key=hex_key) h2o.tear_down_cloud() # sticky ports? wait a bit. time.sleep(5)
def setUpClass(cls): global local_host local_host = not 'hosts' in os.getcwd() if (local_host): h2o.build_cloud(2,java_heap_GB=4,java_extra_args='-XX:+PrintCompilation') else: h2o_hosts.build_cloud_with_hosts(java_extra_args='-XX:+PrintCompilation')
def setUpClass(cls): # assume we're at 0xdata with it's hdfs namenode global localhost localhost = h2o.decide_if_localhost() # hdfs_config='/opt/mapr/conf/mapr-clusters.conf', # # hdfs_name_node='mr-0x1.0xdata.loc:7222') # hdfs_version='mapr2.1.3', if localhost: h2o.build_cloud( 1, java_heap_GB=15, enable_benchmark_log=True, use_maprfs=True, hdfs_version="mapr3.0.1", hdfs_name_node="192.168.1.171:7222", ) else: h2o_hosts.build_cloud_with_hosts( 1, java_heap_GB=15, enable_benchmark_log=True, use_maprfs=True, hdfs_version="mapr3.0.1", hdfs_name_node="192.168.1.171:7222", )
def setUpClass(cls): global SEED, localhost SEED = h2o.setup_random_seed() localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(node_count=1, base_port=54321) else: h2o_hosts.build_cloud_with_hosts() global SYNDATASETS_DIR SYNDATASETS_DIR = h2o.make_syn_dir()
def setUpClass(cls): global localhost localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(node_count=1) else: h2o_hosts.build_cloud_with_hosts(node_count=1) global SYNDATASETS_DIR SYNDATASETS_DIR = h2o.make_syn_dir() h2o.beta_features = True
def setUpClass(cls): global SEED, localhost SEED = h2o.setup_random_seed() localhost = h2o.decide_if_localhost() if (localhost): # h2o.nodes[0].delete_keys_at_teardown should cause the testdir_release # tests to delete keys after each test completion (not cloud teardown, don't care then) h2o.build_cloud(3, create_json=True, java_heap_GB=4, delete_keys_at_teardown=True) else: h2o_hosts.build_cloud_with_hosts(create_json=True, delete_keys_at_teardown=True)
def setUpClass(cls): global SEED SEED = random.randint(0, sys.maxint) random.seed(SEED) print "\nUsing random seed:", SEED localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(1, use_flatfile=True) else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): global SEED, localhost SEED = h2o.setup_random_seed() localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(1, java_heap_GB=4) else: h2o_hosts.build_cloud_with_hosts() # h2b.browseTheCloud() h2o.beta_features = True
def setUpClass(cls): global localhost localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(1,java_heap_GB=4, # use_hdfs=True, hdfs_version='cdh3', hdfs_name_node='192.168.1.176') # use_hdfs=True) use_hdfs=True) else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): global SEED, localhost SEED = h2o.setup_random_seed() localhost = h2o.decide_if_localhost() h2o.beta_features = True # to get the browser page special tab if (localhost): h2o.build_cloud(node_count=1, base_port=54321) else: h2o_hosts.build_cloud_with_hosts(node_count=1) h2o.beta_features = False
def setUpClass(cls): # default to local unless you've got hosts somewhere in the path to cwd local_host = h2o.decide_if_localhost() if (local_host): h2o.build_cloud(3, java_heap_GB=7) else: # could have different config jsons, in different dirs (and execute out of those dirs) # Uses your username specific json: pytest_config-<username>.json # copy pytest_config-simple.json and modify to your needs. h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): start = time.time() h2o_hosts.build_cloud_with_hosts(node_count, base_port=base_port, use_flatfile=True, java_heap_GB=1) print "Cloud of", len( h2o.nodes), "built in", time.time() - start, "seconds" h2o.verify_cloud_size() h2o.check_sandbox_for_errors()
def setUpClass(cls): global SEED, localhost SEED = h2o.setup_random_seed() # SEED = h2o.setup_random_seed(8968685305521902318) localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(2, java_heap_MB=1300, use_flatfile=True) else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): global SEED, localhost SEED = h2o.setup_random_seed() localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud( 3, java_heap_GB=1 ) # enum processing is more interesting with multiple jvms else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): global SEED, localhost SEED = h2o.setup_random_seed() localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(1, java_heap_GB=14, base_port=54321) else: h2o_hosts.build_cloud_with_hosts(1, java_heap_GB=28, base_port=54321)
def setUpClass(cls): global SEED, localhost SEED = h2o.setup_random_seed() localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(java_heap_GB=14, java_extra_args='-XX:+PrintGCDetails') else: h2o_hosts.build_cloud_with_hosts( java_heap_GB=28, java_extra_args='-XX:+PrintGCDetails')
def setUpClass(cls): global SEED, localhost SEED = h2o.setup_random_seed() localhost = h2o.decide_if_localhost() if (localhost): pass h2o.build_cloud(2, java_heap_GB=10, use_flatfile=True) else: h2o_hosts.build_cloud_with_hosts() h2b.browseTheCloud()
def setUpClass(cls): global SEED, localhost SEED = h2o.setup_random_seed() global localhost localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(1) else: h2o_hosts.build_cloud_with_hosts(1)
def test_parse_nflx_loop_hdfs_fvec(self): h2o.beta_features = True print "Using the -.gz files from hdfs" # hdfs://<name node>/datasets/manyfiles-nflx-gz/file_1.dat.gz csvFilename = "file_10.dat.gz" csvFilepattern = "file_1[0-9].dat.gz" trialMax = 2 for tryHeap in [24]: print "\n", tryHeap, "GB heap, 1 jvm per host, import 192.168.1.176 hdfs, then parse" localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(node_count=1, java_heap_GB=tryHeap, use_hdfs=True, hdfs_name_node='192.168.1.176', hdfs_version='cdh3') else: h2o_hosts.build_cloud_with_hosts( node_count=1, java_heap_GB=tryHeap, use_hdfs=True, hdfs_name_node='192.168.1.176', hdfs_version='cdh3') # don't raise exception if we find something bad in h2o stdout/stderr? # h2o.nodes[0].sandboxIgnoreErrors = True timeoutSecs = 500 importFolderPath = "datasets/manyfiles-nflx-gz" for trial in range(trialMax): hex_key = csvFilename + "_" + str(trial) + ".hex" csvFilePattern = 'file_1.dat.gz' # "key": "hdfs://192.168.1.176/datasets/manyfiles-nflx-gz/file_99.dat.gz", time.sleep(5) csvPathname = importFolderPath + "/" + csvFilePattern start = time.time() parseResult = h2i.import_parse(path=csvPathname, schema='hdfs', hex_key=hex_key, timeoutSecs=timeoutSecs, retryDelaySecs=10, pollTimeoutSecs=60) elapsed = time.time() - start print "parse result:", parseResult['destination_key'] print "Parse #", trial, "completed in", "%6.2f" % elapsed, "seconds.", \ "%d pct. of timeout" % ((elapsed*100)/timeoutSecs) h2o_cmd.runStoreView() h2o.tear_down_cloud() # sticky ports? wait a bit. time.sleep(5)
def test_parse_airline_multi_hdfs_many(self): h2o.beta_features = True # default csvFilename = "hex_10" csvFilePattern = '*' # all files in the folder for tryHeap in [24]: print "\n", tryHeap,"GB heap, 1 jvm per host, import mr-0x6 hdfs, then parse" localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(java_heap_GB=tryHeap, random_udp_drop=RANDOM_UDP_DROP, base_port=55930, use_hdfs=True, hdfs_name_node=NAME_NODE, hdfs_version=VERSION) else: h2o_hosts.build_cloud_with_hosts(java_heap_GB=tryHeap, random_udp_drop=RANDOM_UDP_DROP, base_port=55600, disable_assertions=False, use_hdfs=True, hdfs_name_node=NAME_NODE, hdfs_version=VERSION) # don't raise exception if we find something bad in h2o stdout/stderr? # h2o.nodes[0].sandboxIgnoreErrors = True timeoutSecs = 500 importFolderPath = "datasets/airlines_multi" csvPathname = importFolderPath + "/" + csvFilePattern parseResult = h2i.import_only(path=csvPathname, schema='hdfs', timeoutSecs=timeoutSecs, retryDelaySecs=10, pollTimeoutSecs=60) for trial in range(TRIAL_MAX): # each parse now just does one csvFilePattern = "*%s.csv" % trial # if we want multifile # csvFilePattern = "*" hex_key = csvFilename + "_" + str(trial) + ".hex" csvPathname = importFolderPath + "/" + csvFilePattern start = time.time() # print "Don't wait for completion. Just load things up!" print "Drat. the source file is locked if we noPoll. Would have to increment across the individual files?" print "Drat. We can't re-import the folder, if there's a parse using one of the source files?" parseResult = h2i.parse_only(pattern=csvFilePattern, hex_key=hex_key, noPoll=True, delete_on_done=0, timeoutSecs=timeoutSecs, retryDelaySecs=10, pollTimeoutSecs=60) elapsed = time.time() - start print "parse result:", parseResult['destination_key'] print "Parse #", trial, "completed in", "%6.2f" % elapsed, "seconds.", \ "%d pct. of timeout" % ((elapsed*100)/timeoutSecs) h2o_cmd.runStoreView() # we don't delete the hex key. it will start spilling? slow h2j.pollWaitJobs(timeoutSecs=300, pollTimeoutSecs=30) h2o.tear_down_cloud() # sticky ports? wait a bit. time.sleep(5)
def test_parse_nflx_loop_hdfs_fvec(self): h2o.beta_features = True print "Using the -.gz files from hdfs" # hdfs://<name node>/datasets/manyfiles-nflx-gz/file_1.dat.gz csvFilename = "hex_10" csvFilePattern = '*' # all files in the folder trialMax = 2 for tryHeap in [24]: print "\n", tryHeap, "GB heap, 1 jvm per host, import mr-0x6 hdfs, then parse" localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(java_heap_GB=tryHeap, random_udp_drop=RANDOM_UDP_DROP, base_port=55930, use_hdfs=True, hdfs_name_node=NAME_NODE, hdfs_version=VERSION) else: h2o_hosts.build_cloud_with_hosts( java_heap_GB=tryHeap, random_udp_drop=RANDOM_UDP_DROP, base_port=55600, use_hdfs=True, hdfs_name_node=NAME_NODE, hdfs_version=VERSION) # don't raise exception if we find something bad in h2o stdout/stderr? # h2o.nodes[0].sandboxIgnoreErrors = True timeoutSecs = 500 importFolderPath = "datasets/airlines_multi" for trial in range(trialMax): hex_key = csvFilename + "_" + str(trial) + ".hex" csvPathname = importFolderPath + "/" + csvFilePattern start = time.time() parseResult = h2i.import_parse(path=csvPathname, schema='hdfs', hex_key=hex_key, timeoutSecs=timeoutSecs, retryDelaySecs=10, pollTimeoutSecs=60) elapsed = time.time() - start print "parse result:", parseResult['destination_key'] print "Parse #", trial, "completed in", "%6.2f" % elapsed, "seconds.", \ "%d pct. of timeout" % ((elapsed*100)/timeoutSecs) h2o_cmd.runStoreView() # we don't delete the hex key. it will start spilling? slow h2o.tear_down_cloud() # sticky ports? wait a bit. time.sleep(5)
def invoke_hosts_action(action, hosts_config, args, ec2_reservation=None): ids = [inst['id'] for inst in hosts_config['ec2_instances']] ips = [ inst['private_ip_address'] for inst in hosts_config['ec2_instances'] ] region = hosts_config['ec2_region'] if (action == 'terminate'): terminate_instances(ids, region) elif (action == 'stop'): stop_instances(ids, region) elif (action == 'reboot'): reboot_instances(ids, region) wait_for_ssh(ips, skipAlive=False, requiredsuccess=10) elif (action == 'start'): start_instances(ids, region) # FIXME after start instances receive new IPs: wait_for_ssh(ips) elif (action == 'distribute_h2o'): pass elif (action == 'start_h2o'): try: h2o.config_json = args.hosts log("Starting H2O cloud...") h2o_hosts.build_cloud_with_hosts(timeoutSecs=120, retryDelaySecs=5) h2o.touch_cloud() log("Cloud started. Let's roll!") log("You can start for example here \033[93mhttp://{0}:{1}\033[0m". format(hosts_config['ec2_instances'][0]['public_dns_name'], hosts_config['base_port'])) if args.timeout: log("Cloud will shutdown after {0} seconds or use Ctrl+C to shutdown it." .format(args.timeout)) time.sleep(args.timeout) else: log("To kill the cloud please use Ctrl+C as usual.") while (True): time.sleep(3600) except: print traceback.format_exc() finally: log("Goodbye H2O cloud...") h2o.tear_down_cloud() log("Cloud is gone.") elif (action == 'stop_h2o'): pass elif (action == 'clean_tmp'): execute_using_ssh_commands(hosts_config, ec2_reservation, command_string='sudo rm -rf /tmp/*; df') elif (action == 'nexec'): execute_using_ssh_commands(hosts_config, ec2_reservation, command_string=args.cmd)
def setUpClass(cls): global SEED, localhost SEED = h2o.setup_random_seed() localhost = h2o.decide_if_localhost() h2o.beta_features = True # for the beta tab in the browser if (localhost): h2o.build_cloud(node_count=NODES, java_heap_GB=4) # use_hdfs=True, hdfs_name_node='172.16.2.176', hdfs_version='cdh4' else: h2o_hosts.build_cloud_with_hosts(java_heap_GB=4)
def setUpClass(cls): global SEED SEED = random.randint(0, sys.maxint) # SEED = random.seed(SEED) print "\nUsing random seed:", SEED localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(1, java_heap_GB=10) else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(3, use_hdfs=True, hdfs_version='cdh3', hdfs_name_node='192.168.1.176') else: h2o_hosts.build_cloud_with_hosts(use_hdfs=True, hdfs_version='cdh3', hdfs_name_node='192.168.1.176')
def setUpClass(cls): global SEED, localhost SEED = h2o.setup_random_seed() localhost = h2o.decide_if_localhost() # just to get a browser with beta tab h2o.beta_features = True if (localhost): h2o.build_cloud(node_count=1, base_port=54327) else: h2o_hosts.build_cloud_with_hosts(node_count=1) h2o.beta_features = False
def setUpClass(cls): SEED = random.randint(0, sys.maxint) # if you have to force to redo a test # SEED = random.seed(SEED) print "\nUsing random seed:", SEED localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(2,java_heap_GB=1) else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): global localhost localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(base_port=59300, java_heap_GB=4, random_udp_drop=True, use_hdfs=True, hdfs_version='cdh4', hdfs_name_node='mr-0x6') else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): global SEED, localhost, tryHeap tryHeap = 4 SEED = h2o.setup_random_seed() localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(1, java_heap_GB=tryHeap, enable_benchmark_log=True, base_port=54321) else: h2o_hosts.build_cloud_with_hosts(enable_benchmark_log=True)
def setUpClass(cls): localhost = h2o.decide_if_localhost() # want detail on the cloud building to see what node fails h2o.verbose = True if (localhost): h2o.build_cloud(java_heap_GB=2, base_port=54323, random_udp_drop=RANDOM_UDP_DROP) else: h2o_hosts.build_cloud_with_hosts(base_port=54323, random_udp_drop=RANDOM_UDP_DROP) h2o.verbose = False
def setUpClass(cls): global SEED ### SEED = random.randint(0, sys.maxint) ### SEED = 8389506152467586392 SEED = 2437856391921621805 random.seed(SEED) print "\nUsing random seed:", SEED localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(1, use_flatfile=True) else: h2o_hosts.build_cloud_with_hosts()
def setUpClass(cls): # assume we're at 0xdata with it's hdfs namenode global localhost localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(1) else: # all hdfs info is done thru the hdfs_config michal's ec2 config sets up? h2o_hosts.build_cloud_with_hosts(1, # this is for our amazon ec hdfs # see https://github.com/0xdata/h2o/wiki/H2O-and-s3n hdfs_name_node='10.78.14.235:9000', hdfs_version='0.20.2')
def setUpClass(cls): # assume we're at 0xdata with it's hdfs namenode global localhost localhost = h2o.decide_if_localhost() if (localhost): h2o.build_cloud(1, use_hdfs=True, hdfs_version='cdh3', hdfs_name_node='192.168.1.176') else: h2o_hosts.build_cloud_with_hosts(1, use_hdfs=True, hdfs_version='cdh3', hdfs_name_node='192.168.1.176')