def platform_string(): return jobset.platform_string()
submodule = spec[0]
branch = 'master'
elif len(spec) == 2:
submodule = spec[0]
branch = spec[1]
cwd = 'third_party/%s' % submodule
def git(cmd, cwd=cwd):
print 'in %s: git %s' % (cwd, cmd)
subprocess.check_call('git %s' % cmd, cwd=cwd, shell=True)
git('fetch')
git('checkout %s' % branch)
git('pull origin %s' % branch)
if os.path.exists('src/%s/gen_build_yaml.py' % submodule):
need_to_regenerate_projects = True
if need_to_regenerate_projects:
if jobset.platform_string() == 'linux':
subprocess.check_call('tools/buildgen/generate_projects.sh', shell=True)
else:
print 'WARNING: may need to regenerate projects, but since we are not on'
print ' Linux this step is being skipped. Compilation MAY fail.'
# grab config
run_config = _CONFIGS[args.config]
build_config = run_config.build_config
if args.travis:
_FORCE_ENVIRON_FOR_WRAPPERS = {'GRPC_TRACE': 'api'}
if 'all' in args.language:
lang_list = _LANGUAGES.keys()
def start_port_server(port_server_port):
# check if a compatible port server is running
# if incompatible (version mismatch) ==> start a new one
# if not running ==> start a new one
# otherwise, leave it up
try:
version = int(urllib.request.urlopen(
'http://localhost:%d/version_number' % port_server_port,
timeout=10).read())
print('detected port server running version %d' % version)
running = True
except Exception as e:
print('failed to detect port server: %s' % sys.exc_info()[0])
print(e.strerror)
running = False
if running:
current_version = int(subprocess.check_output(
[sys.executable, os.path.abspath('tools/run_tests/python_utils/port_server.py'),
'dump_version']))
print('my port server is version %d' % current_version)
running = (version >= current_version)
if not running:
print('port_server version mismatch: killing the old one')
urllib.request.urlopen('http://localhost:%d/quitquitquit' % port_server_port).read()
time.sleep(1)
if not running:
fd, logfile = tempfile.mkstemp()
os.close(fd)
print('starting port_server, with log file %s' % logfile)
args = [sys.executable, os.path.abspath('tools/run_tests/python_utils/port_server.py'),
'-p', '%d' % port_server_port, '-l', logfile]
env = dict(os.environ)
env['BUILD_ID'] = 'pleaseDontKillMeJenkins'
if jobset.platform_string() == 'windows':
# Working directory of port server needs to be outside of Jenkins
# workspace to prevent file lock issues.
tempdir = tempfile.mkdtemp()
port_server = subprocess.Popen(
args,
env=env,
cwd=tempdir,
creationflags = 0x00000008, # detached process
close_fds=True)
else:
port_server = subprocess.Popen(
args,
env=env,
preexec_fn=os.setsid,
close_fds=True)
time.sleep(1)
# ensure port server is up
waits = 0
while True:
if waits > 10:
print('killing port server due to excessive start up waits')
port_server.kill()
if port_server.poll() is not None:
print('port_server failed to start')
# try one final time: maybe another build managed to start one
time.sleep(1)
try:
urllib.request.urlopen('http://localhost:%d/get' % port_server_port,
timeout=1).read()
print('last ditch attempt to contact port server succeeded')
break
except:
traceback.print_exc()
port_log = open(logfile, 'r').read()
print(port_log)
sys.exit(1)
try:
urllib.request.urlopen('http://localhost:%d/get' % port_server_port,
timeout=1).read()
print('port server is up and ready')
break
except socket.timeout:
print('waiting for port_server: timeout')
traceback.print_exc();
time.sleep(1)
waits += 1
except urllib.error.URLError:
print('waiting for port_server: urlerror')
traceback.print_exc();
time.sleep(1)
waits += 1
except:
traceback.print_exc()
port_server.kill()
raise
def start_port_server():
# check if a compatible port server is running
# if incompatible (version mismatch) ==> start a new one
# if not running ==> start a new one
# otherwise, leave it up
try:
version = int(
request.urlopen('http://localhost:%d/version_number' %
_PORT_SERVER_PORT).read())
logging.info('detected port server running version %d', version)
running = True
except Exception as e:
logging.exception('failed to detect port server')
running = False
if running:
current_version = int(
subprocess.check_output([
sys.executable, # use the same python binary as this process
os.path.abspath('tools/run_tests/python_utils/port_server.py'),
'dump_version'
]).decode())
logging.info('my port server is version %d', current_version)
running = (version >= current_version)
if not running:
logging.info('port_server version mismatch: killing the old one')
request.urlopen('http://localhost:%d/quitquitquit' %
_PORT_SERVER_PORT).read()
time.sleep(1)
if not running:
fd, logfile = tempfile.mkstemp()
os.close(fd)
logging.info('starting port_server, with log file %s', logfile)
args = [
sys.executable,
os.path.abspath('tools/run_tests/python_utils/port_server.py'),
'-p',
'%d' % _PORT_SERVER_PORT, '-l', logfile
]
env = dict(os.environ)
env['BUILD_ID'] = 'pleaseDontKillMeJenkins'
if jobset.platform_string() == 'windows':
# Working directory of port server needs to be outside of Jenkins
# workspace to prevent file lock issues.
tempdir = tempfile.mkdtemp()
if sys.version_info.major == 2:
creationflags = 0x00000008 # detached process
else:
creationflags = 0 # DETACHED_PROCESS doesn't seem to work with python3
port_server = subprocess.Popen(args,
env=env,
cwd=tempdir,
creationflags=creationflags,
close_fds=True)
else:
port_server = subprocess.Popen(args,
env=env,
preexec_fn=os.setsid,
close_fds=True)
time.sleep(1)
# ensure port server is up
waits = 0
while True:
if waits > 10:
logging.warning(
'killing port server due to excessive start up waits')
port_server.kill()
if port_server.poll() is not None:
logging.error('port_server failed to start')
# try one final time: maybe another build managed to start one
time.sleep(1)
try:
request.urlopen('http://localhost:%d/get' %
_PORT_SERVER_PORT).read()
logging.info(
'last ditch attempt to contact port server succeeded')
break
except:
logging.exception(
'final attempt to contact port server failed')
port_log = open(logfile, 'r').read()
print(port_log)
sys.exit(1)
try:
port_server_url = 'http://localhost:%d/get' % _PORT_SERVER_PORT
request.urlopen(port_server_url).read()
logging.info('port server is up and ready')
break
except socket.timeout:
logging.exception('while waiting for port_server')
time.sleep(1)
waits += 1
except IOError:
logging.exception('while waiting for port_server')
time.sleep(1)
waits += 1
except:
logging.exception(
'error while contacting port server at "%s".'
'Will try killing it.', port_server_url)
port_server.kill()
raise
def start_port_server(port_server_port):
# check if a compatible port server is running
# if incompatible (version mismatch) ==> start a new one
# if not running ==> start a new one
# otherwise, leave it up
try:
version = int(
urllib.request.urlopen('http://localhost:%d/version_number' %
port_server_port,
timeout=10).read())
print('detected port server running version %d' % version)
running = True
except Exception as e:
print('failed to detect port server: %s' % sys.exc_info()[0])
print(e.strerror)
running = False
if running:
current_version = int(
subprocess.check_output([
sys.executable,
os.path.abspath('tools/run_tests/python_utils/port_server.py'),
'dump_version'
]))
print('my port server is version %d' % current_version)
running = (version >= current_version)
if not running:
print('port_server version mismatch: killing the old one')
urllib.request.urlopen('http://localhost:%d/quitquitquit' %
port_server_port).read()
time.sleep(1)
if not running:
fd, logfile = tempfile.mkstemp()
os.close(fd)
print('starting port_server, with log file %s' % logfile)
args = [
sys.executable,
os.path.abspath('tools/run_tests/python_utils/port_server.py'),
'-p',
'%d' % port_server_port, '-l', logfile
]
env = dict(os.environ)
env['BUILD_ID'] = 'pleaseDontKillMeJenkins'
if jobset.platform_string() == 'windows':
# Working directory of port server needs to be outside of Jenkins
# workspace to prevent file lock issues.
tempdir = tempfile.mkdtemp()
port_server = subprocess.Popen(
args,
env=env,
cwd=tempdir,
creationflags=0x00000008, # detached process
close_fds=True)
else:
port_server = subprocess.Popen(args,
env=env,
preexec_fn=os.setsid,
close_fds=True)
time.sleep(1)
# ensure port server is up
waits = 0
while True:
if waits > 10:
print('killing port server due to excessive start up waits')
port_server.kill()
if port_server.poll() is not None:
print('port_server failed to start')
# try one final time: maybe another build managed to start one
time.sleep(1)
try:
urllib.request.urlopen('http://localhost:%d/get' %
port_server_port,
timeout=1).read()
print(
'last ditch attempt to contact port server succeeded')
break
except:
traceback.print_exc()
port_log = open(logfile, 'r').read()
print(port_log)
sys.exit(1)
try:
urllib.request.urlopen('http://localhost:%d/get' %
port_server_port,
timeout=1).read()
print('port server is up and ready')
break
except socket.timeout:
print('waiting for port_server: timeout')
traceback.print_exc()
time.sleep(1)
waits += 1
except urllib.error.URLError:
print('waiting for port_server: urlerror')
traceback.print_exc()
time.sleep(1)
waits += 1
except:
traceback.print_exc()
port_server.kill()
raise
elif len(spec) == 2:
submodule = spec[0]
branch = spec[1]
cwd = "third_party/%s" % submodule
def git(cmd, cwd=cwd):
print "in %s: git %s" % (cwd, cmd)
subprocess.check_call("git %s" % cmd, cwd=cwd, shell=True)
git("fetch")
git("checkout %s" % branch)
git("pull origin %s" % branch)
if os.path.exists("src/%s/gen_build_yaml.py" % submodule):
need_to_regenerate_projects = True
if need_to_regenerate_projects:
if jobset.platform_string() == "linux":
subprocess.check_call("tools/buildgen/generate_projects.sh", shell=True)
else:
print "WARNING: may need to regenerate projects, but since we are not on"
print " Linux this step is being skipped. Compilation MAY fail."
# grab config
run_configs = set(
_CONFIGS[cfg]
for cfg in itertools.chain.from_iterable(_CONFIGS.iterkeys() if x == "all" else [x] for x in args.config)
)
build_configs = set(cfg.build_config for cfg in run_configs)
if args.travis:
_FORCE_ENVIRON_FOR_WRAPPERS = {"GRPC_TRACE": "api"}