def write_flatfile(node_count=2, base_port=None, hosts=None, rand_shuffle=True): # too bad this must be in two places..here and build_cloud()..could do a global above? base_port = get_base_port(base_port) # always create the flatfile. ports_per_node = 2 pff = open(flatfile_pathname(), "w+") # doing this list outside the loops so we can shuffle for better test variation hostPortList = [] if hosts is None: ip = h2o_args.python_cmd_ip for i in range(node_count): hostPortList.append(ip + ":" + str(base_port + ports_per_node * i)) else: for h in hosts: for i in range(node_count): # removed leading "/" hostPortList.append(h.h2o_addr + ":" + str(base_port + ports_per_node * i)) # note we want to shuffle the full list of host+port if rand_shuffle: random.shuffle(hostPortList) for hp in hostPortList: pff.write(hp + "\n") pff.close()
def upload_jar_to_remote_hosts(hosts, slow_connection=False): def prog(sofar, total): # output is bad for jenkins. username = getpass.getuser() if username != 'jenkins': p = int((10.0*sofar)/total) sys.stdout.write('\rUploading jar [%s%s] %02d%%' % ('#'*p, ' ' * (10-p), (100*sofar)/total)) sys.stdout.flush() if not slow_connection: for h in hosts: f = find_file('target/h2o.jar') h.upload_file(f, progress=prog) # skipping progress indicator for the flatfile h.upload_file(flatfile_pathname()) else: f = find_file('target/h2o.jar') hosts[0].upload_file(f, progress=prog) hosts[0].push_file_to_remotes(f, hosts[1:]) f = find_file(flatfile_pathname()) hosts[0].upload_file(f, progress=prog) hosts[0].push_file_to_remotes(f, hosts[1:])
def upload_jar_to_remote_hosts(hosts, slow_connection=False): def prog(sofar, total): # output is bad for jenkins. username = getpass.getuser() if username != 'jenkins': p = int((10.0 * sofar) / total) sys.stdout.write('\rUploading jar [%s%s] %02d%%' % ('#' * p, ' ' * (10 - p), (100 * sofar) / total)) sys.stdout.flush() if not slow_connection: for h in hosts: f = find_file('target/h2o.jar') h.upload_file(f, progress=prog) # skipping progress indicator for the flatfile h.upload_file(flatfile_pathname()) else: f = find_file('target/h2o.jar') hosts[0].upload_file(f, progress=prog) hosts[0].push_file_to_remotes(f, hosts[1:]) f = find_file(flatfile_pathname()) hosts[0].upload_file(f, progress=prog) hosts[0].push_file_to_remotes(f, hosts[1:])
def __init__(self, *args, **kwargs): super(LocalH2O, self).__init__(*args, **kwargs) self.rc = None # FIX! no option for local /home/username ..always the sandbox (LOG_DIR) self.ice = tmp_dir('ice.') self.flatfile = flatfile_pathname() # so we can tell if we're remote or local. Apparently used in h2o_import.py self.remoteH2O = False h2o_os_util.check_port_group(self.port) h2o_os_util.show_h2o_processes() if self.node_id is not None: logPrefix = 'local-h2o-' + str(self.node_id) else: logPrefix = 'local-h2o' spawn = spawn_cmd(logPrefix, cmd=self.get_args(), capture_output=self.capture_output) self.ps = spawn[0]
def __init__(self, *args, **kwargs): super(LocalH2O, self).__init__(*args, **kwargs) self.rc = None # FIX! no option for local /home/username ..always the sandbox (LOG_DIR) self.ice = tmp_dir('ice.') self.flatfile = flatfile_pathname() # so we can tell if we're remote or local. Apparently used in h2o_import.py self.remoteH2O = False h2o_os_util.check_port_group(self.port) h2o_os_util.show_h2o_processes() if self.node_id is not None: logPrefix = 'local-h2o-' + str(self.node_id) else: logPrefix = 'local-h2o' # see https://docs.python.org/2/library/subprocess.html#subprocess.Popen # for why I'm using shlex to split the cmd string into a sequence # confusing issues, especially when thinking about windows too # OS/build | os.name | platform.system() # -------------+---------+----------------------- # Win32 native | nt | Windows # Win32 cygwin | posix | CYGWIN_NT-5.1* # Win64 native | nt | Windows # Win64 cygwin | posix | CYGWIN_NT-6.1-WOW64* # Linux | posix | Linux # make it a string if cygwin or windows # in unix, the args was created with split by space. (assumption is no pathname has space) # need to pass string in windows..doesn't seem to assemble string from args list correctly # could make unix use shell and pass string? pf = platform.system() print "System is %s" % pf cmd = self.get_args() if re.match('win', pf, re.IGNORECASE): # covers cygwin and windows cmd = " ".join(cmd) spawn = spawn_cmd(logPrefix, cmd=cmd, capture_output=self.capture_output) self.ps = spawn[0]
def __init__(self, *args, **kwargs): super(LocalH2O, self).__init__(*args, **kwargs) self.rc = None # FIX! no option for local /home/username ..always the sandbox (LOG_DIR) self.ice = tmp_dir('ice.') self.flatfile = flatfile_pathname() # so we can tell if we're remote or local. Apparently used in h2o_import.py self.remoteH2O = False h2o_os_util.check_port_group(self.port) h2o_os_util.show_h2o_processes() if self.node_id is not None: logPrefix = 'local-h2o-' + str(self.node_id) else: logPrefix = 'local-h2o' # see https://docs.python.org/2/library/subprocess.html#subprocess.Popen # for why I'm using shlex to split the cmd string into a sequence # confusing issues, especially when thinking about windows too cmd = " ".join(self.get_args()) cmdSplit = shlex.split(cmd) spawn = spawn_cmd(logPrefix, cmd=cmdSplit, capture_output=self.capture_output) self.ps = spawn[0]
def __init__(self, host, *args, **kwargs): super(RemoteH2O, self).__init__(*args, **kwargs) # it gets set True if an address is specified for LocalH2o init. Override. if 'force_ip' in kwargs: self.force_ip = kwargs['force_ip'] self.remoteH2O = True # so we can tell if we're remote or local self.jar = host.upload_file('build/h2o.jar') # need to copy the flatfile. We don't always use it (depends on h2o args) self.flatfile = host.upload_file(flatfile_pathname()) # distribute AWS credentials if self.aws_credentials: self.aws_credentials = host.upload_file(self.aws_credentials) if self.hdfs_config: self.hdfs_config = host.upload_file(self.hdfs_config) if self.use_home_for_ice: # this will be the username used to ssh to the host self.ice = "/home/" + host.username + '/ice.%d.%s' % (self.port, time.time()) else: self.ice = '/tmp/ice.%d.%s' % (self.port, time.time()) self.channel = host.open_channel() ### FIX! TODO...we don't check on remote hosts yet # this fires up h2o over there cmd = ' '.join(self.get_args()) # UPDATE: somehow java -jar on cygwin target (xp) can't handle /tmp/h2o*jar # because it's a windows executable and expects windows style path names. # but if we cd into /tmp, it can do java -jar h2o*jar. # So just split out the /tmp (pretend we don't know) and the h2o jar file name # Newer windows may not have this problem? Do the ls (this goes into the local stdout # files) so we can see the file is really where we expect. # This hack only works when the dest is /tmp/h2o*jar. It's okay to execute # with pwd = /tmp. If /tmp/ isn't in the jar path, I guess things will be the same as # normal. if 1 == 0: # enable if you want windows remote machines cmdList = ["cd /tmp"] # separate by ;<space> when we join cmdList += ["ls -ltr " + self.jar] cmdList += [re.sub("/tmp/", "", cmd)] self.channel.exec_command("; ".join(cmdList)) else: self.channel.exec_command(cmd) if self.capture_output: if self.node_id is not None: logPrefix = 'remote-h2o-' + str(self.node_id) else: logPrefix = 'remote-h2o' logPrefix += '-' + host.h2o_addr outfd, outpath = tmp_file(logPrefix + '.stdout.', '.log') errfd, errpath = tmp_file(logPrefix + '.stderr.', '.log') drain(self.channel.makefile(), outfd) drain(self.channel.makefile_stderr(), errfd) comment = 'Remote on %s, stdout %s, stderr %s' % ( self.h2o_addr, os.path.basename(outpath), os.path.basename(errpath)) else: drain(self.channel.makefile(), sys.stdout) drain(self.channel.makefile_stderr(), sys.stderr) comment = 'Remote on %s' % self.h2o_addr log(cmd, comment=comment)