def update_pantheon(first_boot=False):
"""Update pantheon code and server configurations.
first_boot: bool. If this is being called from the configure job. If it
is the first boot, we don't need to wait for jenkins or send back update
data.
Otherwise:
This script is run from a cron job because it may update Jenkins (and
therefor cannot be run inside jenkins.)
If the script is successful, a known message will be printed to
stdout which will be redirected to a log file. The jenkins job
post_update_pantheon will check this log file for the message to
determine if it was successful.
"""
try:
try:
# Put jenkins into quietDown mode so no more jobs are started.
urllib2.urlopen('http://localhost:8090/quietDown')
# Find out if this server is using a testing branch.
branch = 'master'
if os.path.exists('/opt/branch.txt'):
branch = open('/opt/branch.txt').read().strip() or 'master'
# Update from repo
with cd('/opt/pantheon'):
local('git fetch --prune origin', capture=False)
local('git checkout --force %s' % branch, capture=False)
local('git reset --hard origin/%s' % branch, capture=False)
# Update from BCFG2
local('/usr/sbin/bcfg2 -vqed', capture=False)
except:
print(traceback.format_exc())
raise
finally:
# wait a min for jenkins to respond.
for x in range(12):
if pantheon.jenkins_running():
break
else:
time.sleep(10)
else:
print("Jenkins not ready after 2 minutes.")
raise Exception("ABORTING: Jenkins not ready after 2 minutes.")
# Restart Jenkins
local('curl -X POST http://localhost:8090/safeRestart', capture=False)
# If this is not the first boot, send back update data.
if not first_boot:
"""
We have to check for both queued jobs then the jenkins restart.
This is because jobs could have been queued before the update
was started, and a check on 'jenkins_running' would return True
because the restart hasn't occured yet (safeRestart). This way,
we first make sure the queue is 0 or jenkins is unreachable, then
wait until it is back up.
"""
# wait for any jobs that were queued to finish.
while True:
queued = pantheon.jenkins_queued()
if queued == 0:
# No more jobs, give jenkins a few seconds to begin restart.
time.sleep(5)
break
# Jenkins is unreachable (already in restart process)
elif queued == -1:
break
else:
time.sleep(5)
# wait for jenkins to restart.
for x in range(30):
if pantheon.jenkins_running():
break
else:
time.sleep(10)
else:
raise Exception("ABORTING: Jenkins did not restart after 5 minutes.")
# stdout is redirected in cron, so this will go to log file.
print "UPDATE COMPLETED SUCCESSFULLY"
except:
print(traceback.format_exc())
raise
def update_pantheon(postback=True):
"""Update pantheon code and server configurations.
postback: bool. If this is being called from the configure job, then it
is the first boot, we don't need to wait for jenkins or send back update
data.
Otherwise:
This script is run from a cron job because it may update Jenkins (and
therefor cannot be run inside jenkins.)
"""
log = logger.logging.getLogger('pantheon.update')
if postback:
log.info('Initiated pantheon update.')
try:
# Ensure the JDK is properly installed.
local('apt-get install -y default-jdk')
# Nightly security package updates disabled.
try:
log.debug('Putting jenkins into quietDown mode.')
pantheon.jenkins_quiet()
# TODO: Actually get security upgrades.
# Get package security updates.
#log.debug('Checking for security releases')
#local('aptitude update')
# Update pantheon code.
log.debug('Checking which branch to use.')
log.debug('Using branch %s.' % MERCURY_BRANCH)
log.debug('Updating from repo.')
with cd('/opt/pantheon'):
local('git fetch --prune origin', capture=False)
local('git checkout --force %s' % MERCURY_BRANCH,
capture=False)
local('git reset --hard origin/%s' % MERCURY_BRANCH,
capture=False)
# Run bcfg2.
local('/usr/sbin/bcfg2 -vqed', capture=False)
except:
log.exception('Pantheon update encountered a fatal error.')
raise
finally:
for x in range(12):
if pantheon.jenkins_running():
break
else:
log.debug('Waiting for jenkins to respond.')
time.sleep(10)
else:
log.error(
"ABORTING: Jenkins hasn't responded after 2 minutes.")
raise Exception("ABORTING: Jenkins not responding.")
log.debug('Restarting jenkins.')
pantheon.jenkins_restart()
# If this is not the first boot, send back update data.
if postback:
"""
We have to check for both queued jobs then the jenkins restart.
This is because jobs could have been queued before the update
was started, and a check on 'jenkins_running' would return True
because the restart hasn't occured yet (safeRestart). This way,
we first make sure the queue is 0 or jenkins is unreachable, then
wait until it is back up.
"""
log.debug('Not first boot, recording update data.')
while True:
queued = pantheon.jenkins_queued()
if queued == 0:
# No more jobs, give jenkins a few seconds to begin restart.
time.sleep(5)
break
# Jenkins is unreachable (already in restart process)
elif queued == -1:
break
else:
log.debug('Waiting for queued jobs to finish.')
time.sleep(5)
# wait for jenkins to restart.
for x in range(30):
if pantheon.jenkins_running():
break
else:
log.debug('Waiting for jenkins to respond.')
time.sleep(10)
else:
log.error(
"ABORTING: Jenkins hasn't responded after 5 minutes.")
raise Exception("ABORTING: Jenkins not responding.")
log.info('Pantheon update completed successfully.')
else:
log.info('Pantheon update completed successfully.')
except:
log.exception('Pantheon update encountered a fatal error.')
raise
def update_pantheon(postback=True):
"""Update pantheon code and server configurations.
postback: bool. If this is being called from the configure job, then it
is the first boot, we don't need to wait for jenkins or send back update
data.
Otherwise:
This script is run from a cron job because it may update Jenkins (and
therefor cannot be run inside jenkins.)
"""
log = logger.logging.getLogger("pantheon.update")
if postback:
log.info("Initiated pantheon update.")
try:
# Ensure the JDK is properly installed.
local("apt-get install -y default-jdk")
# Nightly security package updates disabled.
try:
log.debug("Putting jenkins into quietDown mode.")
pantheon.jenkins_quiet()
# TODO: Actually get security upgrades.
# Get package security updates.
# log.debug('Checking for security releases')
# local('aptitude update')
# Update pantheon code.
log.debug("Checking which branch to use.")
log.debug("Using branch %s." % MERCURY_BRANCH)
log.debug("Updating from repo.")
with cd("/opt/pantheon"):
local("git fetch --prune origin", capture=False)
local("git checkout --force %s" % MERCURY_BRANCH, capture=False)
local("git reset --hard origin/%s" % MERCURY_BRANCH, capture=False)
# Run bcfg2.
local("/usr/sbin/bcfg2 -vqed", capture=False)
except:
log.exception("Pantheon update encountered a fatal error.")
raise
finally:
for x in range(12):
if pantheon.jenkins_running():
break
else:
log.debug("Waiting for jenkins to respond.")
time.sleep(10)
else:
log.error("ABORTING: Jenkins hasn't responded after 2 minutes.")
raise Exception("ABORTING: Jenkins not responding.")
log.debug("Restarting jenkins.")
pantheon.jenkins_restart()
# If this is not the first boot, send back update data.
if postback:
"""
We have to check for both queued jobs then the jenkins restart.
This is because jobs could have been queued before the update
was started, and a check on 'jenkins_running' would return True
because the restart hasn't occured yet (safeRestart). This way,
we first make sure the queue is 0 or jenkins is unreachable, then
wait until it is back up.
"""
log.debug("Not first boot, recording update data.")
while True:
queued = pantheon.jenkins_queued()
if queued == 0:
# No more jobs, give jenkins a few seconds to begin restart.
time.sleep(5)
break
# Jenkins is unreachable (already in restart process)
elif queued == -1:
break
else:
log.debug("Waiting for queued jobs to finish.")
time.sleep(5)
# wait for jenkins to restart.
for x in range(30):
if pantheon.jenkins_running():
break
else:
log.debug("Waiting for jenkins to respond.")
time.sleep(10)
else:
log.error("ABORTING: Jenkins hasn't responded after 5 minutes.")
raise Exception("ABORTING: Jenkins not responding.")
log.info("Pantheon update completed successfully.")
else:
log.info("Pantheon update completed successfully.")
except:
log.exception("Pantheon update encountered a fatal error.")
raise
def update_pantheon(first_boot=False):
"""Update pantheon code and server configurations.
first_boot: bool. If this is being called from the configure job. If it
is the first boot, we don't need to wait for jenkins or send back update
data.
Otherwise:
This script is run from a cron job because it may update Jenkins (and
therefor cannot be run inside jenkins.)
If the script is successful, a known message will be printed to
stdout which will be redirected to a log file. The jenkins job
post_update_pantheon will check this log file for the message to
determine if it was successful.
"""
try:
try:
# Put jenkins into quietDown mode so no more jobs are started.
urllib2.urlopen('http://localhost:8090/quietDown')
# Find out if this server is using a testing branch.
branch = 'master'
if os.path.exists('/opt/branch.txt'):
branch = open('/opt/branch.txt').read().strip() or 'master'
# Update from repo
with cd('/opt/pantheon'):
local('git fetch --prune origin', capture=False)
local('git checkout --force %s' % branch, capture=False)
local('git reset --hard origin/%s' % branch, capture=False)
# Update from BCFG2
local('/usr/sbin/bcfg2 -vqed', capture=False)
except:
print(traceback.format_exc())
raise
finally:
# wait a min for jenkins to respond.
for x in range(12):
if pantheon.jenkins_running():
break
else:
time.sleep(10)
else:
print("Jenkins not ready after 2 minutes.")
raise Exception("ABORTING: Jenkins not ready after 2 minutes.")
# Restart Jenkins
local('curl -X POST http://localhost:8090/safeRestart',
capture=False)
# If this is not the first boot, send back update data.
if not first_boot:
"""
We have to check for both queued jobs then the jenkins restart.
This is because jobs could have been queued before the update
was started, and a check on 'jenkins_running' would return True
because the restart hasn't occured yet (safeRestart). This way,
we first make sure the queue is 0 or jenkins is unreachable, then
wait until it is back up.
"""
# wait for any jobs that were queued to finish.
while True:
queued = pantheon.jenkins_queued()
if queued == 0:
# No more jobs, give jenkins a few seconds to begin restart.
time.sleep(5)
break
# Jenkins is unreachable (already in restart process)
elif queued == -1:
break
else:
time.sleep(5)
# wait for jenkins to restart.
for x in range(30):
if pantheon.jenkins_running():
break
else:
time.sleep(10)
else:
raise Exception(
"ABORTING: Jenkins did not restart after 5 minutes.")
# stdout is redirected in cron, so this will go to log file.
print "UPDATE COMPLETED SUCCESSFULLY"
except:
print(traceback.format_exc())
raise