def test_url_retrieve(self):
# Successfully reads the data.
@contextlib.contextmanager
def fake_open(_filepath, _mode):
yield StringIO.StringIO()
self.mock(__builtin__, 'open', fake_open)
self.mock(net, 'url_open',
lambda url, **_kwargs: net_utils.make_fake_response('111', url))
self.assertEqual(
True, net.url_retrieve('filepath', 'https://localhost/test'))
# Respects url_open connection errors.
self.mock(net, 'url_open', lambda _url, **_kwargs: None)
self.assertEqual(
False, net.url_retrieve('filepath', 'https://localhost/test'))
# Respects read timeout errors.
def timeouting_http_response(url):
def iter_content_mock(_size=None):
raise net.TimeoutError()
response = net_utils.make_fake_response('', url)
self.mock(response, 'iter_content', iter_content_mock)
return response
removed = []
self.mock(os, 'remove', removed.append)
self.mock(net, 'url_open',
lambda url, **_kwargs: timeouting_http_response(url))
self.assertEqual(
False, net.url_retrieve('filepath', 'https://localhost/test'))
self.assertEqual(['filepath'], removed)
def test_url_retrieve(self):
# Successfully reads the data.
@contextlib.contextmanager
def fake_open(_filepath, _mode):
yield StringIO.StringIO()
self.mock(__builtin__, 'open', fake_open)
self.mock(net, 'url_open',
lambda url, **_kwargs: net.HttpResponse.get_fake_response('111', url))
self.assertEqual(
True, net.url_retrieve('filepath', 'https://localhost/test'))
# Respects url_open connection errors.
self.mock(net, 'url_open', lambda _url, **_kwargs: None)
self.assertEqual(
False, net.url_retrieve('filepath', 'https://localhost/test'))
# Respects read timeout errors.
def timeouting_http_response(url):
def read_mock(_size=None):
raise net.TimeoutError()
response = net.HttpResponse.get_fake_response('', url)
self.mock(response, 'read', read_mock)
return response
removed = []
self.mock(os, 'remove', removed.append)
self.mock(net, 'url_open',
lambda url, **_kwargs: timeouting_http_response(url))
self.assertEqual(
False, net.url_retrieve('filepath', 'https://localhost/test'))
self.assertEqual(['filepath'], removed)
def update_bot(botobj, version):
"""Downloads the new version of the bot code and then runs it.
Use alternating files; first load swarming_bot.1.zip, then swarming_bot.2.zip,
never touching swarming_bot.zip which was the originally bootstrapped file.
Does not return.
TODO(maruel): Create LKGBC:
https://code.google.com/p/swarming/issues/detail?id=112
"""
# Alternate between .1.zip and .2.zip.
new_zip = 'swarming_bot.1.zip'
if os.path.basename(THIS_FILE) == new_zip:
new_zip = 'swarming_bot.2.zip'
# Download as a new file.
url = botobj.remote.url + '/swarming/api/v1/bot/bot_code/%s' % version
if not net.url_retrieve(new_zip, url):
# Try without a specific version. It can happen when a server is rapidly
# updated multiple times in a row.
botobj.post_error(
'Unable to download %s from %s; first tried version %s' %
(new_zip, url, version))
# Poll again, this may work next time. To prevent busy-loop, sleep a little.
time.sleep(2)
return
logging.info('Restarting to %s.', new_zip)
sys.stdout.flush()
sys.stderr.flush()
cmd = [sys.executable, new_zip, 'start_slave', '--survive']
# Do not call on_bot_shutdown.
if sys.platform in ('cygwin', 'win32'):
# (Tentative) It is expected that subprocess.Popen() behaves a tad better
# on Windows than os.exec*(), which has to be emulated since there's no OS
# provided implementation. This means processes will accumulate as the bot
# is restarted, which could be a problem long term.
try:
subprocess.Popen(cmd)
except Exception as e:
logging.exception('failed to respawn: %s', e)
else:
sys.exit(0)
else:
# On OSX, launchd will be unhappy if we quit so the old code bot process
# has to outlive the new code child process. Launchd really wants the main
# process to survive, and it'll restart it if it disappears. os.exec*()
# replaces the process so this is fine.
os.execv(sys.executable, cmd)
# This code runs only if bot failed to respawn itself.
botobj.post_error('Bot failed to respawn after update')
def update_bot(botobj, version):
"""Downloads the new version of the bot code and then runs it.
Use alternating files; first load swarming_bot.1.zip, then swarming_bot.2.zip,
never touching swarming_bot.zip which was the originally bootstrapped file.
Does not return.
TODO(maruel): Create LKGBC:
https://code.google.com/p/swarming/issues/detail?id=112
"""
# Alternate between .1.zip and .2.zip.
new_zip = 'swarming_bot.1.zip'
if os.path.basename(THIS_FILE) == new_zip:
new_zip = 'swarming_bot.2.zip'
# Download as a new file.
url = botobj.remote.url + '/swarming/api/v1/bot/bot_code/%s' % version
if not net.url_retrieve(new_zip, url):
# Try without a specific version. It can happen when a server is rapidly
# updated multiple times in a row.
botobj.post_error(
'Unable to download %s from %s; first tried version %s' %
(new_zip, url, version))
# Poll again, this may work next time. To prevent busy-loop, sleep a little.
time.sleep(2)
return
logging.info('Restarting to %s.', new_zip)
sys.stdout.flush()
sys.stderr.flush()
cmd = [sys.executable, new_zip, 'start_slave', '--survive']
# Do not call on_bot_shutdown.
if sys.platform in ('cygwin', 'win32'):
# (Tentative) It is expected that subprocess.Popen() behaves a tad better
# on Windows than os.exec*(), which has to be emulated since there's no OS
# provided implementation. This means processes will accumulate as the bot
# is restarted, which could be a problem long term.
try:
subprocess.Popen(cmd)
except Exception as e:
logging.exception('failed to respawn: %s', e)
else:
sys.exit(0)
else:
# On OSX, launchd will be unhappy if we quit so the old code bot process
# has to outlive the new code child process. Launchd really wants the main
# process to survive, and it'll restart it if it disappears. os.exec*()
# replaces the process so this is fine.
os.execv(sys.executable, cmd)
# This code runs only if bot failed to respawn itself.
botobj.post_error('Bot failed to respawn after update')
def update_bot(botobj, version):
"""Downloads the new version of the bot code and then runs it.
Use alternating files; first load swarming_bot.1.zip, then swarming_bot.2.zip,
never touching swarming_bot.zip which was the originally bootstrapped file.
Does not return.
TODO(maruel): Create LKGBC:
https://code.google.com/p/swarming/issues/detail?id=112
"""
# Alternate between .1.zip and .2.zip.
new_zip = 'swarming_bot.1.zip'
if os.path.basename(THIS_FILE) == new_zip:
new_zip = 'swarming_bot.2.zip'
new_zip = os.path.join(os.path.dirname(THIS_FILE), new_zip)
# Download as a new file.
url = botobj.remote.url + '/swarming/api/v1/bot/bot_code/%s' % version
if not net.url_retrieve(new_zip, url):
# Try without a specific version. It can happen when a server is rapidly
# updated multiple times in a row.
botobj.post_error(
'Unable to download %s from %s; first tried version %s' %
(new_zip, url, version))
# Poll again, this may work next time. To prevent busy-loop, sleep a little.
time.sleep(2)
return
logging.info('Restarting to %s.', new_zip)
sys.stdout.flush()
sys.stderr.flush()
# Don't forget to release the singleton before restarting itself.
SINGLETON.release()
# Do not call on_bot_shutdown.
# On OSX, launchd will be unhappy if we quit so the old code bot process has
# to outlive the new code child process. Launchd really wants the main process
# to survive, and it'll restart it if it disappears. os.exec*() replaces the
# process so this is fine.
ret = common.exec_python([new_zip, 'start_slave', '--survive'])
if ret not in (0, 1073807364):
# 1073807364 is returned when the process is killed due to shutdown. No need
# to alert anyone in that case.
botobj.post_error('Bot failed to respawn after update: %s' % ret)
sys.exit(ret)
def _url_retrieve(self, filepath, url_path):
"""Fetches the file from the given URL path on the server."""
return net.url_retrieve(filepath,
self._server + url_path,
headers=self.get_headers(include_auth=True),
timeout=NET_CONNECTION_TIMEOUT_SEC)
def update_bot(botobj, version):
"""Downloads the new version of the bot code and then runs it.
Use alternating files; first load swarming_bot.1.zip, then swarming_bot.2.zip,
never touching swarming_bot.zip which was the originally bootstrapped file.
LKGBC is handled by update_lkgbc().
Does not return.
"""
# Alternate between .1.zip and .2.zip.
new_zip = 'swarming_bot.1.zip'
if os.path.basename(THIS_FILE) == new_zip:
new_zip = 'swarming_bot.2.zip'
new_zip = os.path.join(os.path.dirname(THIS_FILE), new_zip)
# Download as a new file.
url = botobj.server + '/swarming/api/v1/bot/bot_code/%s' % version
if not net.url_retrieve(new_zip, url):
# It can happen when a server is rapidly updated multiple times in a row.
botobj.post_error(
'Unable to download %s from %s; first tried version %s' %
(new_zip, url, version))
# Poll again, this may work next time. To prevent busy-loop, sleep a little.
time.sleep(2)
return
s = os.stat(new_zip)
logging.info('Restarting to %s; %d bytes.', new_zip, s.st_size)
sys.stdout.flush()
sys.stderr.flush()
proc = subprocess42.Popen([sys.executable, new_zip, 'is_fine'],
stdout=subprocess42.PIPE,
stderr=subprocess42.STDOUT)
output, _ = proc.communicate()
if proc.returncode:
botobj.post_error('New bot code is bad: proc exit = %s. stdout:\n%s' %
(proc.returncode, output))
# Poll again, the server may have better code next time. To prevent
# busy-loop, sleep a little.
time.sleep(2)
return
# Don't forget to release the singleton before restarting itself.
SINGLETON.release()
# Do not call on_bot_shutdown.
# On OSX, launchd will be unhappy if we quit so the old code bot process has
# to outlive the new code child process. Launchd really wants the main process
# to survive, and it'll restart it if it disappears. os.exec*() replaces the
# process so this is fine.
ret = common.exec_python([new_zip, 'start_slave', '--survive'])
if ret in (1073807364, -1073741510):
# 1073807364 is returned when the process is killed due to shutdown. No need
# to alert anyone in that case.
# -1073741510 is returned when rebooting too. This can happen when the
# parent code was running the old version and gets confused and decided to
# poll again.
# In any case, zap out the error code.
ret = 0
elif ret:
botobj.post_error('Bot failed to respawn after update: %s' % ret)
sys.exit(ret)
def update_bot(botobj, version):
"""Downloads the new version of the bot code and then runs it.
Use alternating files; first load swarming_bot.1.zip, then swarming_bot.2.zip,
never touching swarming_bot.zip which was the originally bootstrapped file.
LKGBC is handled by update_lkgbc().
Does not return.
"""
# Alternate between .1.zip and .2.zip.
new_zip = 'swarming_bot.1.zip'
if os.path.basename(THIS_FILE) == new_zip:
new_zip = 'swarming_bot.2.zip'
new_zip = os.path.join(os.path.dirname(THIS_FILE), new_zip)
# Download as a new file.
url = botobj.server + '/swarming/api/v1/bot/bot_code/%s' % version
if not net.url_retrieve(new_zip, url):
# It can happen when a server is rapidly updated multiple times in a row.
botobj.post_error(
'Unable to download %s from %s; first tried version %s' %
(new_zip, url, version))
# Poll again, this may work next time. To prevent busy-loop, sleep a little.
time.sleep(2)
return
s = os.stat(new_zip)
logging.info('Restarting to %s; %d bytes.', new_zip, s.st_size)
sys.stdout.flush()
sys.stderr.flush()
proc = subprocess42.Popen(
[sys.executable, new_zip, 'is_fine'],
stdout=subprocess42.PIPE, stderr=subprocess42.STDOUT)
output, _ = proc.communicate()
if proc.returncode:
botobj.post_error(
'New bot code is bad: proc exit = %s. stdout:\n%s' %
(proc.returncode, output))
# Poll again, the server may have better code next time. To prevent
# busy-loop, sleep a little.
time.sleep(2)
return
# Don't forget to release the singleton before restarting itself.
SINGLETON.release()
# Do not call on_bot_shutdown.
# On OSX, launchd will be unhappy if we quit so the old code bot process has
# to outlive the new code child process. Launchd really wants the main process
# to survive, and it'll restart it if it disappears. os.exec*() replaces the
# process so this is fine.
ret = common.exec_python([new_zip, 'start_slave', '--survive'])
if ret in (1073807364, -1073741510):
# 1073807364 is returned when the process is killed due to shutdown. No need
# to alert anyone in that case.
# -1073741510 is returned when rebooting too. This can happen when the
# parent code was running the old version and gets confused and decided to
# poll again.
# In any case, zap out the error code.
ret = 0
elif ret:
botobj.post_error('Bot failed to respawn after update: %s' % ret)
sys.exit(ret)