def write_flatfile(node_count=2, base_port=None, hosts=None, rand_shuffle=True):
# too bad this must be in two places..here and build_cloud()..could do a global above?
base_port = get_base_port(base_port)
# always create the flatfile.
ports_per_node = 2
pff = open(flatfile_pathname(), "w+")
# doing this list outside the loops so we can shuffle for better test variation
hostPortList = []
if hosts is None:
ip = h2o_args.python_cmd_ip
for i in range(node_count):
hostPortList.append(ip + ":" + str(base_port + ports_per_node * i))
else:
for h in hosts:
for i in range(node_count):
# removed leading "/"
hostPortList.append(h.h2o_addr + ":" + str(base_port + ports_per_node * i))
# note we want to shuffle the full list of host+port
if rand_shuffle:
random.shuffle(hostPortList)
for hp in hostPortList:
pff.write(hp + "\n")
pff.close()
def write_flatfile(node_count=2, base_port=None, hosts=None, rand_shuffle=True):
# too bad this must be in two places..here and build_cloud()..could do a global above?
base_port = get_base_port(base_port)
# always create the flatfile.
ports_per_node = 2
pff = open(flatfile_pathname(), "w+")
# doing this list outside the loops so we can shuffle for better test variation
hostPortList = []
if hosts is None:
ip = h2o_args.python_cmd_ip
for i in range(node_count):
hostPortList.append(ip + ":" + str(base_port + ports_per_node * i))
else:
for h in hosts:
for i in range(node_count):
# removed leading "/"
hostPortList.append(h.h2o_addr + ":" + str(base_port + ports_per_node * i))
# note we want to shuffle the full list of host+port
if rand_shuffle:
random.shuffle(hostPortList)
for hp in hostPortList:
pff.write(hp + "\n")
pff.close()
def upload_jar_to_remote_hosts(hosts, slow_connection=False):
def prog(sofar, total):
# output is bad for jenkins.
username = getpass.getuser()
if username != 'jenkins':
p = int((10.0*sofar)/total)
sys.stdout.write('\rUploading jar [%s%s] %02d%%' % ('#'*p, ' ' * (10-p), (100*sofar)/total))
sys.stdout.flush()
if not slow_connection:
for h in hosts:
f = find_file('target/h2o.jar')
h.upload_file(f, progress=prog)
# skipping progress indicator for the flatfile
h.upload_file(flatfile_pathname())
else:
f = find_file('target/h2o.jar')
hosts[0].upload_file(f, progress=prog)
hosts[0].push_file_to_remotes(f, hosts[1:])
f = find_file(flatfile_pathname())
hosts[0].upload_file(f, progress=prog)
hosts[0].push_file_to_remotes(f, hosts[1:])
def upload_jar_to_remote_hosts(hosts, slow_connection=False):
def prog(sofar, total):
# output is bad for jenkins.
username = getpass.getuser()
if username != 'jenkins':
p = int((10.0 * sofar) / total)
sys.stdout.write('\rUploading jar [%s%s] %02d%%' %
('#' * p, ' ' * (10 - p), (100 * sofar) / total))
sys.stdout.flush()
if not slow_connection:
for h in hosts:
f = find_file('target/h2o.jar')
h.upload_file(f, progress=prog)
# skipping progress indicator for the flatfile
h.upload_file(flatfile_pathname())
else:
f = find_file('target/h2o.jar')
hosts[0].upload_file(f, progress=prog)
hosts[0].push_file_to_remotes(f, hosts[1:])
f = find_file(flatfile_pathname())
hosts[0].upload_file(f, progress=prog)
hosts[0].push_file_to_remotes(f, hosts[1:])
def __init__(self, *args, **kwargs):
super(LocalH2O, self).__init__(*args, **kwargs)
self.rc = None
# FIX! no option for local /home/username ..always the sandbox (LOG_DIR)
self.ice = tmp_dir('ice.')
self.flatfile = flatfile_pathname()
# so we can tell if we're remote or local. Apparently used in h2o_import.py
self.remoteH2O = False
h2o_os_util.check_port_group(self.port)
h2o_os_util.show_h2o_processes()
if self.node_id is not None:
logPrefix = 'local-h2o-' + str(self.node_id)
else:
logPrefix = 'local-h2o'
spawn = spawn_cmd(logPrefix, cmd=self.get_args(), capture_output=self.capture_output)
self.ps = spawn[0]
def __init__(self, *args, **kwargs):
super(LocalH2O, self).__init__(*args, **kwargs)
self.rc = None
# FIX! no option for local /home/username ..always the sandbox (LOG_DIR)
self.ice = tmp_dir('ice.')
self.flatfile = flatfile_pathname()
# so we can tell if we're remote or local. Apparently used in h2o_import.py
self.remoteH2O = False
h2o_os_util.check_port_group(self.port)
h2o_os_util.show_h2o_processes()
if self.node_id is not None:
logPrefix = 'local-h2o-' + str(self.node_id)
else:
logPrefix = 'local-h2o'
# see https://docs.python.org/2/library/subprocess.html#subprocess.Popen
# for why I'm using shlex to split the cmd string into a sequence
# confusing issues, especially when thinking about windows too
# OS/build | os.name | platform.system()
# -------------+---------+-----------------------
# Win32 native | nt | Windows
# Win32 cygwin | posix | CYGWIN_NT-5.1*
# Win64 native | nt | Windows
# Win64 cygwin | posix | CYGWIN_NT-6.1-WOW64*
# Linux | posix | Linux
# make it a string if cygwin or windows
# in unix, the args was created with split by space. (assumption is no pathname has space)
# need to pass string in windows..doesn't seem to assemble string from args list correctly
# could make unix use shell and pass string?
pf = platform.system()
print "System is %s" % pf
cmd = self.get_args()
if re.match('win', pf, re.IGNORECASE): # covers cygwin and windows
cmd = " ".join(cmd)
spawn = spawn_cmd(logPrefix,
cmd=cmd,
capture_output=self.capture_output)
self.ps = spawn[0]
def __init__(self, *args, **kwargs):
super(LocalH2O, self).__init__(*args, **kwargs)
self.rc = None
# FIX! no option for local /home/username ..always the sandbox (LOG_DIR)
self.ice = tmp_dir('ice.')
self.flatfile = flatfile_pathname()
# so we can tell if we're remote or local. Apparently used in h2o_import.py
self.remoteH2O = False
h2o_os_util.check_port_group(self.port)
h2o_os_util.show_h2o_processes()
if self.node_id is not None:
logPrefix = 'local-h2o-' + str(self.node_id)
else:
logPrefix = 'local-h2o'
spawn = spawn_cmd(logPrefix,
cmd=self.get_args(),
capture_output=self.capture_output)
self.ps = spawn[0]
def __init__(self, *args, **kwargs):
super(LocalH2O, self).__init__(*args, **kwargs)
self.rc = None
# FIX! no option for local /home/username ..always the sandbox (LOG_DIR)
self.ice = tmp_dir('ice.')
self.flatfile = flatfile_pathname()
# so we can tell if we're remote or local. Apparently used in h2o_import.py
self.remoteH2O = False
h2o_os_util.check_port_group(self.port)
h2o_os_util.show_h2o_processes()
if self.node_id is not None:
logPrefix = 'local-h2o-' + str(self.node_id)
else:
logPrefix = 'local-h2o'
# see https://docs.python.org/2/library/subprocess.html#subprocess.Popen
# for why I'm using shlex to split the cmd string into a sequence
# confusing issues, especially when thinking about windows too
# OS/build | os.name | platform.system()
# -------------+---------+-----------------------
# Win32 native | nt | Windows
# Win32 cygwin | posix | CYGWIN_NT-5.1*
# Win64 native | nt | Windows
# Win64 cygwin | posix | CYGWIN_NT-6.1-WOW64*
# Linux | posix | Linux
# make it a string if cygwin or windows
# in unix, the args was created with split by space. (assumption is no pathname has space)
# need to pass string in windows..doesn't seem to assemble string from args list correctly
# could make unix use shell and pass string?
pf = platform.system()
print "System is %s" % pf
cmd = self.get_args()
if re.match('win', pf, re.IGNORECASE): # covers cygwin and windows
cmd = " ".join(cmd)
spawn = spawn_cmd(logPrefix, cmd=cmd, capture_output=self.capture_output)
self.ps = spawn[0]
def __init__(self, *args, **kwargs):
super(LocalH2O, self).__init__(*args, **kwargs)
self.rc = None
# FIX! no option for local /home/username ..always the sandbox (LOG_DIR)
self.ice = tmp_dir('ice.')
self.flatfile = flatfile_pathname()
# so we can tell if we're remote or local. Apparently used in h2o_import.py
self.remoteH2O = False
h2o_os_util.check_port_group(self.port)
h2o_os_util.show_h2o_processes()
if self.node_id is not None:
logPrefix = 'local-h2o-' + str(self.node_id)
else:
logPrefix = 'local-h2o'
# see https://docs.python.org/2/library/subprocess.html#subprocess.Popen
# for why I'm using shlex to split the cmd string into a sequence
# confusing issues, especially when thinking about windows too
cmd = " ".join(self.get_args())
cmdSplit = shlex.split(cmd)
spawn = spawn_cmd(logPrefix, cmd=cmdSplit, capture_output=self.capture_output)
self.ps = spawn[0]
def __init__(self, host, *args, **kwargs):
super(RemoteH2O, self).__init__(*args, **kwargs)
# it gets set True if an address is specified for LocalH2o init. Override.
if 'force_ip' in kwargs:
self.force_ip = kwargs['force_ip']
self.remoteH2O = True # so we can tell if we're remote or local
self.jar = host.upload_file('build/h2o.jar')
# need to copy the flatfile. We don't always use it (depends on h2o args)
self.flatfile = host.upload_file(flatfile_pathname())
# distribute AWS credentials
if self.aws_credentials:
self.aws_credentials = host.upload_file(self.aws_credentials)
if self.hdfs_config:
self.hdfs_config = host.upload_file(self.hdfs_config)
if self.use_home_for_ice:
# this will be the username used to ssh to the host
self.ice = "/home/" + host.username + '/ice.%d.%s' % (self.port, time.time())
else:
self.ice = '/tmp/ice.%d.%s' % (self.port, time.time())
self.channel = host.open_channel()
### FIX! TODO...we don't check on remote hosts yet
# this fires up h2o over there
cmd = ' '.join(self.get_args())
# UPDATE: somehow java -jar on cygwin target (xp) can't handle /tmp/h2o*jar
# because it's a windows executable and expects windows style path names.
# but if we cd into /tmp, it can do java -jar h2o*jar.
# So just split out the /tmp (pretend we don't know) and the h2o jar file name
# Newer windows may not have this problem? Do the ls (this goes into the local stdout
# files) so we can see the file is really where we expect.
# This hack only works when the dest is /tmp/h2o*jar. It's okay to execute
# with pwd = /tmp. If /tmp/ isn't in the jar path, I guess things will be the same as
# normal.
if 1 == 0: # enable if you want windows remote machines
cmdList = ["cd /tmp"] # separate by ;<space> when we join
cmdList += ["ls -ltr " + self.jar]
cmdList += [re.sub("/tmp/", "", cmd)]
self.channel.exec_command("; ".join(cmdList))
else:
self.channel.exec_command(cmd)
if self.capture_output:
if self.node_id is not None:
logPrefix = 'remote-h2o-' + str(self.node_id)
else:
logPrefix = 'remote-h2o'
logPrefix += '-' + host.h2o_addr
outfd, outpath = tmp_file(logPrefix + '.stdout.', '.log')
errfd, errpath = tmp_file(logPrefix + '.stderr.', '.log')
drain(self.channel.makefile(), outfd)
drain(self.channel.makefile_stderr(), errfd)
comment = 'Remote on %s, stdout %s, stderr %s' % (
self.h2o_addr, os.path.basename(outpath), os.path.basename(errpath))
else:
drain(self.channel.makefile(), sys.stdout)
drain(self.channel.makefile_stderr(), sys.stderr)
comment = 'Remote on %s' % self.h2o_addr
log(cmd, comment=comment)