def main():
debug = int(os.environ["GNT_DEBUG"])
logname = pathutils.GetLogFilename("jobs")
utils.SetupLogging(logname, "job-startup", debug=debug)
(job_id, livelock_name) = _GetMasterInfo()
utils.SetupLogging(logname, "job-%s" % (job_id, ), debug=debug)
exit_code = 1
try:
logging.debug("Preparing the context and the configuration")
context = masterd.GanetiContext(livelock_name)
logging.debug("Registering a SIGTERM handler")
cancel = [False]
def _TermHandler(signum, _frame):
logging.info("Killed by signal %d", signum)
cancel[0] = True
signal.signal(signal.SIGTERM, _TermHandler)
logging.debug("Picking up job %d", job_id)
context.jobqueue.PickupJob(job_id)
# waiting for the job to finish
time.sleep(1)
while not context.jobqueue.HasJobBeenFinalized(job_id):
if cancel[0]:
logging.debug("Got cancel request, cancelling job %d", job_id)
r = context.jobqueue.CancelJob(job_id)
logging.debug("CancelJob result for job %d: %s", job_id, r)
cancel[0] = False
time.sleep(1)
# wait until the queue finishes
logging.debug("Waiting for the queue to finish")
while context.jobqueue.PrepareShutdown():
time.sleep(1)
logging.debug("Shutting the queue down")
context.jobqueue.Shutdown()
exit_code = 0
except Exception: # pylint: disable=W0703
logging.exception("Exception when trying to run job %d", job_id)
finally:
logging.debug("Job %d finalized", job_id)
logging.debug("Removing livelock file %s", livelock_name.GetPath())
os.remove(livelock_name.GetPath())
sys.exit(exit_code)
def main():
debug = int(os.environ["GNT_DEBUG"])
logname = pathutils.GetLogFilename("jobs")
utils.SetupLogging(logname, "job-post-hooks-startup", debug=debug)
job_id = _GetMasterInfo()
utils.SetupLogging(logname, "job-%s-post-hooks" % (job_id, ), debug=debug)
try:
job = JobQueue.SafeLoadJobFromDisk(None,
job_id,
try_archived=False,
writable=False)
assert job.id == job_id, "The job id received %d differs " % job_id + \
"from the serialized one %d" % job.id
target_op = None
for op in job.ops:
if op.start_timestamp is None:
break
target_op = op
# We should run post hooks only if opcode execution has been started.
# Note that currently the opcodes inside a job execute sequentially.
if target_op is None:
sys.exit(0)
livelock_name = livelock.LiveLockName("post-hooks-executor-%d" %
job_id)
context = masterd.GanetiContext(livelock_name)
cfg_tmp = context.GetConfig(job_id)
# Get static snapshot of the config and release it in order to prevent
# further synchronizations.
cfg = cfg_tmp.GetDetachedConfig()
cfg_tmp.OutDate()
hooksmaster.ExecGlobalPostHooks(
target_op.input.OP_ID, cfg.GetMasterNodeName(),
context.GetRpc(cfg).call_hooks_runner, logging.warning,
cfg.GetClusterName(), cfg.GetMasterNode(), job_id,
constants.POST_HOOKS_STATUS_DISAPPEARED)
except Exception: # pylint: disable=W0703
logging.exception("Exception when trying to run post hooks of job %d",
job_id)
finally:
logging.debug("Post hooks exec for disappeared job %d finalized",
job_id)
logging.debug("Removing livelock file %s", livelock_name.GetPath())
os.remove(livelock_name.GetPath())
sys.exit(0)
from qa import qa_performance
from qa import qa_job
from qa import qa_rapi
from qa import qa_tags
from qa import qa_utils
from ganeti import utils
from ganeti import rapi # pylint: disable=W0611
from ganeti import constants
from ganeti import netutils
from ganeti import pathutils
import ganeti.rapi.client # pylint: disable=W0611
from ganeti.rapi.client import UsesRapiClient
_QA_PROFILE = pathutils.GetLogFilename("qa-profile")
_PROFILE_LOG_INDENT = ""
def _FormatHeader(line, end=72, mark="-", color=None):
"""Fill a line up to the end column.
"""
line = (mark * 4) + " " + line + " "
line += "-" * (end - len(line))
line = line.rstrip()
line = colors.colorize(line, color=color)
return line
def _DescriptionOf(fn):
from ganeti import pathutils
from ganeti import vcluster
import colors
import qa_config
import qa_error
from qa_logging import FormatInfo
_MULTIPLEXERS = {}
#: Unique ID per QA run
_RUN_UUID = utils.NewUUID()
#: Path to the QA query output log file
_QA_OUTPUT = pathutils.GetLogFilename("qa-output")
(INST_DOWN, INST_UP) = range(500, 502)
(FIRST_ARG, RETURN_VALUE) = range(1000, 1002)
def _RaiseWithInfo(msg, error_desc):
"""Raises a QA error with the given content, and adds a message if present.
"""
if msg:
output = "%s: %s" % (msg, error_desc)
else:
output = error_desc
raise qa_error.Error(output)
def main():
debug = int(os.environ["GNT_DEBUG"])
logname = pathutils.GetLogFilename("jobs")
utils.SetupLogging(logname, "job-startup", debug=debug)
(job_id, llock, secret_params_serialized) = _SetupJob()
secret_params = ""
if secret_params_serialized:
secret_params_json = serializer.LoadJson(secret_params_serialized)
secret_params = RestorePrivateValueWrapping(secret_params_json)
utils.SetupLogging(logname, "job-%s" % (job_id,), debug=debug)
try:
logging.debug("Preparing the context and the configuration")
context = masterd.GanetiContext(llock)
logging.debug("Registering signal handlers")
cancel = [False]
prio_change = [False]
def _TermHandler(signum, _frame):
logging.info("Killed by signal %d", signum)
cancel[0] = True
signal.signal(signal.SIGTERM, _TermHandler)
def _HupHandler(signum, _frame):
logging.debug("Received signal %d, old flag was %s, will set to True",
signum, mcpu.sighupReceived)
mcpu.sighupReceived[0] = True
signal.signal(signal.SIGHUP, _HupHandler)
def _User1Handler(signum, _frame):
logging.info("Received signal %d, indicating priority change", signum)
prio_change[0] = True
signal.signal(signal.SIGUSR1, _User1Handler)
job = context.jobqueue.SafeLoadJobFromDisk(job_id, False)
job.SetPid(os.getpid())
if secret_params:
for i in range(0, len(secret_params)):
if hasattr(job.ops[i].input, "osparams_secret"):
job.ops[i].input.osparams_secret = secret_params[i]
execfun = mcpu.Processor(context, job_id, job_id).ExecOpCode
proc = _JobProcessor(context.jobqueue, execfun, job)
result = _JobProcessor.DEFER
while result != _JobProcessor.FINISHED:
result = proc()
if result == _JobProcessor.WAITDEP and not cancel[0]:
# Normally, the scheduler should avoid starting a job where the
# dependencies are not yet finalised. So warn, but wait an continue.
logging.warning("Got started despite a dependency not yet finished")
time.sleep(5)
if cancel[0]:
logging.debug("Got cancel request, cancelling job %d", job_id)
r = context.jobqueue.CancelJob(job_id)
job = context.jobqueue.SafeLoadJobFromDisk(job_id, False)
proc = _JobProcessor(context.jobqueue, execfun, job)
logging.debug("CancelJob result for job %d: %s", job_id, r)
cancel[0] = False
if prio_change[0]:
logging.debug("Received priority-change request")
try:
fname = os.path.join(pathutils.LUXID_MESSAGE_DIR, "%d.prio" % job_id)
new_prio = int(utils.ReadFile(fname))
utils.RemoveFile(fname)
logging.debug("Changing priority of job %d to %d", job_id, new_prio)
r = context.jobqueue.ChangeJobPriority(job_id, new_prio)
job = context.jobqueue.SafeLoadJobFromDisk(job_id, False)
proc = _JobProcessor(context.jobqueue, execfun, job)
logging.debug("Result of changing priority of %d to %d: %s", job_id,
new_prio, r)
except Exception: # pylint: disable=W0703
logging.warning("Informed of priority change, but could not"
" read new priority")
prio_change[0] = False
except Exception: # pylint: disable=W0703
logging.exception("Exception when trying to run job %d", job_id)
finally:
logging.debug("Job %d finalized", job_id)
logging.debug("Removing livelock file %s", llock.GetPath())
os.remove(llock.GetPath())
sys.exit(0)
def GetPaths():
"""Returns a tuple of path objects to process.
"""
getent = runtime.GetEnts()
masterd_log = constants.DAEMONS_LOGFILES[constants.MASTERD]
noded_log = constants.DAEMONS_LOGFILES[constants.NODED]
confd_log = constants.DAEMONS_LOGFILES[constants.CONFD]
wconfd_log = constants.DAEMONS_LOGFILES[constants.WCONFD]
luxid_log = constants.DAEMONS_LOGFILES[constants.LUXID]
rapi_log = constants.DAEMONS_LOGFILES[constants.RAPI]
mond_log = constants.DAEMONS_LOGFILES[constants.MOND]
metad_log = constants.DAEMONS_LOGFILES[constants.METAD]
mond_extra_log = constants.DAEMONS_EXTRA_LOGFILES[constants.MOND]
metad_extra_log = constants.DAEMONS_EXTRA_LOGFILES[constants.METAD]
jobs_log = pathutils.GetLogFilename("jobs")
rapi_dir = os.path.join(pathutils.DATA_DIR, "rapi")
cleaner_log_dir = os.path.join(pathutils.LOG_DIR, "cleaner")
master_cleaner_log_dir = os.path.join(pathutils.LOG_DIR, "master-cleaner")
# A note on the ordering: The parent directory (type C{DIR}) must always be
# listed before files (type C{FILE}) in that directory. Once the directory is
# set, only files directly in that directory can be listed.
paths = [
(pathutils.DATA_DIR, DIR, 0o755, getent.masterd_uid,
getent.masterd_gid),
(pathutils.CLUSTER_DOMAIN_SECRET_FILE, FILE, 0o640, getent.masterd_uid,
getent.masterd_gid, False),
(pathutils.CLUSTER_CONF_FILE, FILE, 0o640, getent.masterd_uid,
getent.confd_gid, False),
(pathutils.LOCK_STATUS_FILE, FILE, 0o640, getent.masterd_uid,
getent.confd_gid, False),
(pathutils.TEMP_RES_STATUS_FILE, FILE, 0o640, getent.masterd_uid,
getent.confd_gid, False),
(pathutils.CONFD_HMAC_KEY, FILE, 0o440, getent.confd_uid,
getent.masterd_gid, False),
(pathutils.SSH_KNOWN_HOSTS_FILE, FILE, 0o644, getent.masterd_uid,
getent.masterd_gid, False),
(pathutils.RAPI_CERT_FILE, FILE, 0o440, getent.rapi_uid,
getent.masterd_gid, False),
(pathutils.SPICE_CERT_FILE, FILE, 0o440, getent.noded_uid,
getent.masterd_gid, False),
(pathutils.SPICE_CACERT_FILE, FILE, 0o440, getent.noded_uid,
getent.masterd_gid, False),
(pathutils.NODED_CERT_FILE, FILE, pathutils.NODED_CERT_MODE,
getent.masterd_uid, getent.masterd_gid, False),
(pathutils.NODED_CLIENT_CERT_FILE, FILE, pathutils.NODED_CERT_MODE,
getent.masterd_uid, getent.masterd_gid, False),
(pathutils.WATCHER_PAUSEFILE, FILE, 0o644, getent.masterd_uid,
getent.masterd_gid, False),
]
ss = ssconf.SimpleStore()
for ss_path in ss.GetFileList():
paths.append((ss_path, FILE, constants.SS_FILE_PERMS, getent.noded_uid,
getent.noded_gid, False))
paths.extend([
(pathutils.QUEUE_DIR, DIR, 0o750, getent.masterd_uid,
getent.daemons_gid),
(pathutils.QUEUE_DIR, QUEUE_DIR, constants.JOB_QUEUE_FILES_PERMS,
getent.masterd_uid, getent.daemons_gid),
(pathutils.JOB_QUEUE_DRAIN_FILE, FILE, 0o644, getent.masterd_uid,
getent.daemons_gid, False),
(pathutils.JOB_QUEUE_LOCK_FILE, FILE, constants.JOB_QUEUE_FILES_PERMS,
getent.masterd_uid, getent.daemons_gid, False),
(pathutils.JOB_QUEUE_SERIAL_FILE, FILE,
constants.JOB_QUEUE_FILES_PERMS, getent.masterd_uid,
getent.daemons_gid, False),
(pathutils.JOB_QUEUE_VERSION_FILE, FILE,
constants.JOB_QUEUE_FILES_PERMS, getent.masterd_uid,
getent.daemons_gid, False),
(pathutils.JOB_QUEUE_ARCHIVE_DIR, DIR, 0o750, getent.masterd_uid,
getent.daemons_gid),
(rapi_dir, DIR, 0o750, getent.rapi_uid, getent.masterd_gid),
(pathutils.RAPI_USERS_FILE, FILE, 0o640, getent.rapi_uid,
getent.masterd_gid, False),
(pathutils.RUN_DIR, DIR, 0o775, getent.masterd_uid,
getent.daemons_gid),
(pathutils.SOCKET_DIR, DIR, 0o770, getent.masterd_uid,
getent.daemons_gid),
(pathutils.MASTER_SOCKET, FILE, 0o660, getent.masterd_uid,
getent.daemons_gid, False),
(pathutils.QUERY_SOCKET, FILE, 0o660, getent.luxid_uid,
getent.daemons_gid, False),
(pathutils.BDEV_CACHE_DIR, DIR, 0o755, getent.noded_uid,
getent.masterd_gid),
(pathutils.UIDPOOL_LOCKDIR, DIR, 0o750, getent.noded_uid,
getent.masterd_gid),
(pathutils.DISK_LINKS_DIR, DIR, 0o755, getent.noded_uid,
getent.masterd_gid),
(pathutils.CRYPTO_KEYS_DIR, DIR, 0o700, getent.noded_uid,
getent.masterd_gid),
(pathutils.IMPORT_EXPORT_DIR, DIR, 0o755, getent.noded_uid,
getent.masterd_gid),
(pathutils.LOG_DIR, DIR, 0o770, getent.masterd_uid,
getent.daemons_gid),
(masterd_log, FILE, 0o600, getent.masterd_uid, getent.masterd_gid,
False),
(confd_log, FILE, 0o600, getent.confd_uid, getent.masterd_gid, False),
(wconfd_log, FILE, 0o600, getent.wconfd_uid, getent.masterd_gid,
False),
(luxid_log, FILE, 0o600, getent.luxid_uid, getent.masterd_gid, False),
(noded_log, FILE, 0o600, getent.noded_uid, getent.masterd_gid, False),
(rapi_log, FILE, 0o600, getent.rapi_uid, getent.masterd_gid, False),
(mond_log, FILE, 0o600, getent.mond_uid, getent.masterd_gid, False),
(mond_extra_log["access"], FILE, 0o600, getent.mond_uid,
getent.masterd_gid, False),
(mond_extra_log["error"], FILE, 0o600, getent.mond_uid,
getent.masterd_gid, False),
(metad_log, FILE, 0o600, getent.metad_uid, getent.metad_gid, False),
(metad_extra_log["access"], FILE, 0o600, getent.metad_uid,
getent.metad_gid, False),
(metad_extra_log["error"], FILE, 0o600, getent.metad_uid,
getent.metad_gid, False),
(jobs_log, FILE, 0o600, getent.luxid_uid, getent.luxid_gid, False),
(pathutils.LOG_OS_DIR, DIR, 0o750, getent.noded_uid,
getent.daemons_gid),
(pathutils.LOG_XEN_DIR, DIR, 0o750, getent.noded_uid,
getent.daemons_gid),
(pathutils.LOG_KVM_DIR, DIR, 0o750, getent.noded_uid,
getent.daemons_gid),
(cleaner_log_dir, DIR, 0o750, getent.noded_uid, getent.noded_gid),
(master_cleaner_log_dir, DIR, 0o750, getent.masterd_uid,
getent.masterd_gid),
(pathutils.INSTANCE_REASON_DIR, DIR, 0o755, getent.noded_uid,
getent.noded_gid),
(pathutils.LIVELOCK_DIR, DIR, 0o750, getent.masterd_uid,
getent.daemons_gid),
(pathutils.LUXID_MESSAGE_DIR, DIR, 0o750, getent.masterd_uid,
getent.daemons_gid),
])
return paths
def main():
debug = int(os.environ["GNT_DEBUG"])
logname = pathutils.GetLogFilename("jobs")
utils.SetupLogging(logname, "job-startup", debug=debug)
(job_id, livelock_name) = _GetMasterInfo()
utils.SetupLogging(logname, "job-%s" % (job_id,), debug=debug)
exit_code = 1
try:
logging.debug("Preparing the context and the configuration")
context = masterd.GanetiContext(livelock_name)
logging.debug("Registering signal handlers")
cancel = [False]
prio_change = [False]
def _TermHandler(signum, _frame):
logging.info("Killed by signal %d", signum)
cancel[0] = True
signal.signal(signal.SIGTERM, _TermHandler)
def _HupHandler(signum, _frame):
logging.debug("Received signal %d, old flag was %s, will set to True",
signum, mcpu.sighupReceived)
mcpu.sighupReceived[0] = True
signal.signal(signal.SIGHUP, _HupHandler)
def _User1Handler(signum, _frame):
logging.info("Received signal %d, indicating priority change", signum)
prio_change[0] = True
signal.signal(signal.SIGUSR1, _User1Handler)
logging.debug("Picking up job %d", job_id)
context.jobqueue.PickupJob(job_id)
# waiting for the job to finish
time.sleep(1)
while not context.jobqueue.HasJobBeenFinalized(job_id):
if cancel[0]:
logging.debug("Got cancel request, cancelling job %d", job_id)
r = context.jobqueue.CancelJob(job_id)
logging.debug("CancelJob result for job %d: %s", job_id, r)
cancel[0] = False
if prio_change[0]:
logging.debug("Received priority-change request")
try:
fname = os.path.join(pathutils.LUXID_MESSAGE_DIR, "%d.prio" % job_id)
new_prio = int(utils.ReadFile(fname))
utils.RemoveFile(fname)
logging.debug("Changing priority of job %d to %d", job_id, new_prio)
r = context.jobqueue.ChangeJobPriority(job_id, new_prio)
logging.debug("Result of changing priority of %d to %d: %s", job_id,
new_prio, r)
except Exception: # pylint: disable=W0703
logging.warning("Informed of priority change, but could not"
" read new priority")
prio_change[0] = False
time.sleep(1)
# wait until the queue finishes
logging.debug("Waiting for the queue to finish")
while context.jobqueue.PrepareShutdown():
time.sleep(1)
logging.debug("Shutting the queue down")
context.jobqueue.Shutdown()
exit_code = 0
except Exception: # pylint: disable=W0703
logging.exception("Exception when trying to run job %d", job_id)
finally:
logging.debug("Job %d finalized", job_id)
logging.debug("Removing livelock file %s", livelock_name.GetPath())
os.remove(livelock_name.GetPath())
sys.exit(exit_code)
# to be re-exported but pylint complains because the imported names
# are not actually used in this module.
import re
import socket
from ganeti._constants import *
from ganeti._vcsversion import *
from ganeti import compat
from ganeti import pathutils
ALLOCATABLE_KEY = "allocatable"
FAILED_KEY = "failed"
DAEMONS_LOGFILES = \
dict((daemon, pathutils.GetLogFilename(DAEMONS_LOGBASE[daemon]))
for daemon in DAEMONS_LOGBASE)
DAEMONS_EXTRA_LOGFILES = \
dict((daemon, dict((extra,
pathutils.GetLogFilename(DAEMONS_EXTRA_LOGBASE[daemon][extra]))
for extra in DAEMONS_EXTRA_LOGBASE[daemon]))
for daemon in DAEMONS_EXTRA_LOGBASE)
# When the Xen toolstack used is "xl", live migration requires the source host
# to connect to the target host via ssh (xl runs this command). We need to pass
# the command xl runs some extra info so that it can use Ganeti's key
# verification and not fail. Note that this string is incomplete: it must be
# filled with the cluster name before being used.
XL_SSH_CMD = ("ssh -l %s -oGlobalKnownHostsFile=%s"
" -oUserKnownHostsFile=/dev/null"