def Create(cls, unique_id, children, size, spindles, params, excl_stor, dyn_params, **kwargs): """Create a new rbd device. Provision a new rbd volume inside a RADOS pool. """ if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2: raise errors.ProgrammerError("Invalid configuration data %s" % str(unique_id)) if excl_stor: raise errors.ProgrammerError("RBD device requested with" " exclusive_storage") rbd_pool = params[constants.LDP_POOL] rbd_name = unique_id[1] # Provision a new rbd volume (Image) inside the RADOS cluster. cmd = cls.MakeRbdCmd(params, ["create", "-p", rbd_pool, rbd_name, "--size", str(size)]) result = utils.RunCmd(cmd) if result.failed: base.ThrowError("rbd creation failed (%s): %s", result.fail_reason, result.output) return RADOSBlockDevice(unique_id, children, size, params, dyn_params, **kwargs)
def Create(cls, unique_id, children, size, spindles, params, excl_stor, dyn_params, *args): """Create a new extstorage device. Provision a new volume using an extstorage provider, which will then be mapped to a block device. """ (name, uuid) = args if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2: raise errors.ProgrammerError("Invalid configuration data %s" % str(unique_id)) if excl_stor: raise errors.ProgrammerError("extstorage device requested with" " exclusive_storage") # Call the External Storage's create script, # to provision a new Volume inside the External Storage _ExtStorageAction(constants.ES_ACTION_CREATE, unique_id, params, size=size, name=name, uuid=uuid) return ExtStorageDevice(unique_id, children, size, params, dyn_params, *args)
def AddManyTasks(self, tasks, priority=_DEFAULT_PRIORITY, task_id=None): """Add a list of tasks to the queue. @type tasks: list of tuples @param tasks: list of args passed to L{BaseWorker.RunTask} @type priority: number or list of numbers @param priority: Priority for all added tasks or a list with the priority for each task @type task_id: list @param task_id: List with the ID for each task @note: See L{AddTask} for a note on task IDs. """ assert compat.all(isinstance(task, (tuple, list)) for task in tasks), \ "Each task must be a sequence" assert (isinstance(priority, (int, long)) or compat.all(isinstance(prio, (int, long)) for prio in priority)), \ "Priority must be numeric or be a list of numeric values" assert task_id is None or isinstance(task_id, (tuple, list)), \ "Task IDs must be in a sequence" if isinstance(priority, (int, long)): priority = [priority] * len(tasks) elif len(priority) != len(tasks): raise errors.ProgrammerError( "Number of priorities (%s) doesn't match" " number of tasks (%s)" % (len(priority), len(tasks))) if task_id is None: task_id = [None] * len(tasks) elif len(task_id) != len(tasks): raise errors.ProgrammerError( "Number of task IDs (%s) doesn't match" " number of tasks (%s)" % (len(task_id), len(tasks))) self._lock.acquire() try: self._WaitWhileQuiescingUnlocked() assert compat.all( isinstance(prio, (int, long)) for prio in priority) assert len(tasks) == len(priority) assert len(tasks) == len(task_id) for (args, prio, tid) in zip(tasks, priority, task_id): self._AddTaskUnlocked(args, prio, tid) finally: self._lock.release()
def PathJoin(*args): """Safe-join a list of path components. Requirements: - the first argument must be an absolute path - no component in the path must have backtracking (e.g. /../), since we check for normalization at the end @param args: the path components to be joined @raise ValueError: for invalid paths """ # ensure we're having at least two paths passed in if len(args) <= 1: raise errors.ProgrammerError("PathJoin requires two arguments") # ensure the first component is an absolute and normalized path name root = args[0] if not IsNormAbsPath(root): raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0])) result = os.path.join(*args) # ensure that the whole path is normalized if not IsNormAbsPath(result): raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args)) # check that we're still under the original prefix if not IsBelowDir(root, result): raise ValueError("Error: path joining resulted in different prefix" " (%s != %s)" % (result, root)) return result
def ListVisibleFiles(path, _is_mountpoint=os.path.ismount): """Returns a list of visible files in a directory. @type path: str @param path: the directory to enumerate @rtype: list @return: the list of all files not starting with a dot @raise ProgrammerError: if L{path} is not an absolue and normalized path """ if not IsNormAbsPath(path): raise errors.ProgrammerError("Path passed to ListVisibleFiles is not" " absolute/normalized: '%s'" % path) mountpoint = _is_mountpoint(path) def fn(name): """File name filter. Ignores files starting with a dot (".") as by Unix convention they're considered hidden. The "lost+found" directory found at the root of some filesystems is also hidden. """ return not (name.startswith(".") or (mountpoint and name == _LOST_AND_FOUND and os.path.isdir(os.path.join(path, name)))) return filter(fn, os.listdir(path))
def CreateBackup(file_name): """Creates a backup of a file. @type file_name: str @param file_name: file to be backed up @rtype: str @return: the path to the newly created backup @raise errors.ProgrammerError: for invalid file names """ if not os.path.isfile(file_name): raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" % file_name) prefix = ("%s.backup-%s." % (os.path.basename(file_name), TimestampForFilename())) dir_name = os.path.dirname(file_name) fsrc = open(file_name, "rb") try: (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name) fdst = os.fdopen(fd, "wb") try: logging.debug("Backing up %s at %s", file_name, backup_name) shutil.copyfileobj(fsrc, fdst) finally: fdst.close() finally: fsrc.close() return backup_name
def GetRequest(self, cfg): """Request an relocation of an instance The checks for the completeness of the opcode must have already been done. """ instance = cfg.GetInstanceInfo(self.inst_uuid) disks = cfg.GetInstanceDisks(self.inst_uuid) if instance is None: raise errors.ProgrammerError("Unknown instance '%s' passed to" " IAllocator" % self.inst_uuid) if not utils.AllDiskOfType(disks, constants.DTS_MIRRORED): raise errors.OpPrereqError("Can't relocate non-mirrored instances", errors.ECODE_INVAL) secondary_nodes = cfg.GetInstanceSecondaryNodes(instance.uuid) if (utils.AnyDiskOfType(disks, constants.DTS_INT_MIRROR) and len(secondary_nodes) != 1): raise errors.OpPrereqError("Instance has not exactly one secondary node", errors.ECODE_STATE) disk_sizes = [{constants.IDISK_SIZE: disk.size, constants.IDISK_TYPE: disk.dev_type} for disk in disks] disk_space = gmi.ComputeDiskSize(disk_sizes) return { "name": instance.name, "disk_space_total": disk_space, "required_nodes": 1, "relocate_from": cfg.GetNodeNames(self.relocate_from_node_uuids), }
def _UnlockedDetachInstanceDisk(self, inst_uuid, disk_uuid): """Detach a disk from an instance. @type inst_uuid: string @param inst_uuid: The UUID of the instance object @type disk_uuid: string @param disk_uuid: The UUID of the disk object """ instance = self._UnlockedGetInstanceInfo(inst_uuid) if instance is None: raise errors.ConfigurationError("Instance %s doesn't exist" % inst_uuid) if disk_uuid not in self._ConfigData().disks: raise errors.ConfigurationError("Disk %s doesn't exist" % disk_uuid) # Check if disk is attached to the instance if disk_uuid not in instance.disks: raise errors.ProgrammerError( "Disk %s is not attached to an instance" % disk_uuid) idx = instance.disks.index(disk_uuid) instance.disks.remove(disk_uuid) instance_disks = self._UnlockedGetInstanceDisks(inst_uuid) _UpdateIvNames(idx, instance_disks[idx:]) instance.serial_no += 1 instance.mtime = time.time()
def TcpPing(target, port, timeout=10, live_port_needed=False, source=None): """Simple ping implementation using TCP connect(2). Check if the given IP is reachable by doing attempting a TCP connect to it. @type target: str @param target: the IP to ping @type port: int @param port: the port to connect to @type timeout: int @param timeout: the timeout on the connection attempt @type live_port_needed: boolean @param live_port_needed: whether a closed port will cause the function to return failure, as if there was a timeout @type source: str or None @param source: if specified, will cause the connect to be made from this specific source address; failures to bind other than C{EADDRNOTAVAIL} will be ignored """ logging.debug("Attempting to reach TCP port %s on target %s with a timeout" " of %s seconds", port, target, timeout) try: family = IPAddress.GetAddressFamily(target) except errors.IPAddressError, err: raise errors.ProgrammerError("Family of IP address given in parameter" " 'target' can't be determined: %s" % err)
def Create(cls, unique_id, children, size, spindles, params, excl_stor, dyn_params, *args): """Create a new file. @param size: the size of file in MiB @rtype: L{bdev.FileStorage} @return: an instance of FileStorage """ if excl_stor: raise errors.ProgrammerError("FileStorage device requested with" " exclusive_storage") if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2: raise ValueError("Invalid configuration data %s" % str(unique_id)) full_path = unique_id[1] server_addr = params[constants.GLUSTER_HOST] port = params[constants.GLUSTER_PORT] volume = params[constants.GLUSTER_VOLUME] volume_obj = GlusterVolume(server_addr, port, volume) full_path = io.PathJoin(volume_obj.mount_point, full_path) # Possible optimization: defer actual creation to first Attach, rather # than mounting and unmounting here, then remounting immediately after. with volume_obj.Mount(): FileDeviceHelper.CreateFile(full_path, size, create_folders=True) return GlusterStorage(unique_id, children, size, params, dyn_params, *args)
def FormatUnit(value, units): """Formats an incoming number of MiB with the appropriate unit. @type value: int @param value: integer representing the value in MiB (1048576) @type units: char @param units: the type of formatting we should do: - 'h' for automatic scaling - 'm' for MiBs - 'g' for GiBs - 't' for TiBs @rtype: str @return: the formatted value (with suffix) """ if units not in ("m", "g", "t", "h"): raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units)) suffix = "" if units == "m" or (units == "h" and value < 1024): if units == "h": suffix = "M" return "%d%s" % (round(value, 0), suffix) elif units == "g" or (units == "h" and value < (1024 * 1024)): if units == "h": suffix = "G" return "%0.1f%s" % (round(float(value) / 1024, 1), suffix) else: if units == "h": suffix = "T" return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
def VerifyCertificate(filename): """Verifies a SSL certificate. @type filename: string @param filename: Path to PEM file """ try: cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, io.ReadFile(filename)) except Exception as err: # pylint: disable=W0703 return (constants.CV_ERROR, "Failed to load X509 certificate %s: %s" % (filename, err)) (errcode, msg) = \ x509.VerifyX509Certificate(cert, constants.SSL_CERT_EXPIRATION_WARN, constants.SSL_CERT_EXPIRATION_ERROR) if msg: fnamemsg = "While verifying %s: %s" % (filename, msg) else: fnamemsg = None if errcode is None: return (None, fnamemsg) elif errcode == x509.CERT_WARNING: return (constants.CV_WARNING, fnamemsg) elif errcode == x509.CERT_ERROR: return (constants.CV_ERROR, fnamemsg) raise errors.ProgrammerError("Unhandled certificate error code %r" % errcode)
def run(self): # Hypothesis: the messages we receive contain only a complete QMP message # encoded in JSON. conn, addr = self.socket.accept() # Send the banner as the first thing conn.send(self.encode_string(self._QMP_BANNER_DATA)) # Expect qmp_capabilities and return an empty response conn.recv(4096) conn.send(self.encode_string(self._EMPTY_RESPONSE)) while True: # We ignore the expected message, as the purpose of this object is not # to verify the correctness of the communication but to act as a # partner for the SUT (System Under Test, that is QmpConnection) msg = conn.recv(4096) if not msg: break if not self.script: break response = self.script.pop(0) if isinstance(response, str): conn.send(response) elif isinstance(response, list): for chunk in response: conn.send(chunk) else: raise errors.ProgrammerError("Unknown response type for %s" % response) conn.close()
def _check_connection(self): """Make sure that the connection is established. """ if not self._connected: raise errors.ProgrammerError("To use a MonitorSocket you need to first" " invoke connect() on it")
def KillProcess(pid, signal_=signal.SIGTERM, timeout=30, waitpid=False): """Kill a process given by its pid. @type pid: int @param pid: The PID to terminate. @type signal_: int @param signal_: The signal to send, by default SIGTERM @type timeout: int @param timeout: The timeout after which, if the process is still alive, a SIGKILL will be sent. If not positive, no such checking will be done @type waitpid: boolean @param waitpid: If true, we should waitpid on this process after sending signals, since it's our own child and otherwise it would remain as zombie """ def _helper(pid, signal_, wait): """Simple helper to encapsulate the kill/waitpid sequence""" if utils_wrapper.IgnoreProcessNotFound(os.kill, pid, signal_) and wait: try: os.waitpid(pid, os.WNOHANG) except OSError: pass if pid <= 0: # kill with pid=0 == suicide raise errors.ProgrammerError("Invalid pid given '%s'" % pid) if not IsProcessAlive(pid): return _helper(pid, signal_, waitpid) if timeout <= 0: return def _CheckProcess(): if not IsProcessAlive(pid): return try: (result_pid, _) = os.waitpid(pid, os.WNOHANG) except OSError: raise utils_retry.RetryAgain() if result_pid > 0: return raise utils_retry.RetryAgain() try: # Wait up to $timeout seconds utils_retry.Retry(_CheckProcess, (0.01, 1.5, 0.1), timeout) except utils_retry.RetryTimeout: pass if IsProcessAlive(pid): # Kill process if it's still alive _helper(pid, signal.SIGKILL, waitpid)
def RunWatcherHooks(): """Run the watcher hooks. """ hooks_dir = utils.PathJoin(pathutils.HOOKS_BASE_DIR, constants.HOOKS_NAME_WATCHER) if not os.path.isdir(hooks_dir): return try: results = utils.RunParts(hooks_dir) except Exception as err: # pylint: disable=W0703 logging.exception("RunParts %s failed: %s", hooks_dir, err) return for (relname, status, runresult) in results: if status == constants.RUNPARTS_SKIP: logging.debug("Watcher hook %s: skipped", relname) elif status == constants.RUNPARTS_ERR: logging.warning("Watcher hook %s: error (%s)", relname, runresult) elif status == constants.RUNPARTS_RUN: if runresult.failed: logging.warning( "Watcher hook %s: failed (exit: %d) (output: %s)", relname, runresult.exit_code, runresult.output) else: logging.debug("Watcher hook %s: success (output: %s)", relname, runresult.output) else: raise errors.ProgrammerError( "Unknown status %s returned by RunParts", status)
def __init__(self, hmac_key, peers, callback, port=None, logger=None): """Constructor for ConfdClient @type hmac_key: string @param hmac_key: hmac key to talk to confd @type peers: list @param peers: list of peer nodes @type callback: f(L{ConfdUpcallPayload}) @param callback: function to call when getting answers @type port: integer @param port: confd port (default: use GetDaemonPort) @type logger: logging.Logger @param logger: optional logger for internal conditions """ if not callable(callback): raise errors.ProgrammerError("callback must be callable") self.UpdatePeerList(peers) self._SetPeersAddressFamily() self._hmac_key = hmac_key self._socket = ConfdAsyncUDPClient(self, self._family) self._callback = callback self._confd_port = port self._logger = logger self._requests = {} if self._confd_port is None: self._confd_port = netutils.GetDaemonPort(constants.CONFD)
def _Send(self, message): """Encodes and sends a message to KVM using QMP. @type message: QmpMessage @param message: message to send to KVM @raise errors.HypervisorError: when there are communication errors @raise errors.ProgrammerError: when there are data serialization errors """ self._check_connection() try: message_str = str(message) except Exception as err: raise errors.ProgrammerError("QMP data deserialization error: %s" % err) try: self.sock.sendall(message_str) except socket.timeout as err: raise errors.HypervisorError( "Timeout while sending a QMP message: " "%s" % err) except socket.error as err: raise errors.HypervisorError( "Unable to send data from KVM using the" " QMP protocol: %s" % err)
def Exec(self, feedback_fn): """Run the allocator test. """ if self.op.mode == constants.IALLOCATOR_MODE_ALLOC: req = iallocator.IAReqInstanceAlloc( name=self.op.name, memory=self.op.memory, disks=self.op.disks, disk_template=self.op.disk_template, group_name=self.op.group_name, os=self.op.os, tags=self.op.tags, nics=self.op.nics, vcpus=self.op.vcpus, spindle_use=self.op.spindle_use, hypervisor=self.op.hypervisor, node_whitelist=None) elif self.op.mode == constants.IALLOCATOR_MODE_RELOC: req = iallocator.IAReqRelocate(inst_uuid=self.inst_uuid, relocate_from_node_uuids=list( self.relocate_from_node_uuids)) elif self.op.mode == constants.IALLOCATOR_MODE_CHG_GROUP: req = iallocator.IAReqGroupChange( instances=self.op.instances, target_groups=self.op.target_groups) elif self.op.mode == constants.IALLOCATOR_MODE_NODE_EVAC: req = iallocator.IAReqNodeEvac(instances=self.op.instances, evac_mode=self.op.evac_mode, ignore_soft_errors=False) elif self.op.mode == constants.IALLOCATOR_MODE_MULTI_ALLOC: disk_template = self.op.disk_template insts = [ iallocator.IAReqInstanceAlloc( name="%s%s" % (self.op.name, idx), memory=self.op.memory, disks=self.op.disks, disk_template=disk_template, group_name=self.op.group_name, os=self.op.os, tags=self.op.tags, nics=self.op.nics, vcpus=self.op.vcpus, spindle_use=self.op.spindle_use, hypervisor=self.op.hypervisor, node_whitelist=None) for idx in range(self.op.count) ] req = iallocator.IAReqMultiInstanceAlloc(instances=insts) else: raise errors.ProgrammerError( "Uncatched mode %s in" " LUTestAllocator.Exec", self.op.mode) ial = iallocator.IAllocator(self.cfg, self.rpc, req) if self.op.direction == constants.IALLOCATOR_DIR_IN: result = ial.in_text else: ial.Run(self.op.iallocator, validate=False) result = ial.out_text return result
def _GetVolumeInfo(lvm_cmd, fields): """Returns LVM Volume infos using lvm_cmd @param lvm_cmd: Should be one of "pvs", "vgs" or "lvs" @param fields: Fields to return @return: A list of dicts each with the parsed fields """ if not fields: raise errors.ProgrammerError("No fields specified") sep = "|" cmd = [lvm_cmd, "--noheadings", "--nosuffix", "--units=m", "--unbuffered", "--separator=%s" % sep, "-o%s" % ",".join(fields)] result = utils.RunCmd(cmd) if result.failed: raise errors.CommandError("Can't get the volume information: %s - %s" % (result.fail_reason, result.output)) data = [] for line in result.stdout.splitlines(): splitted_fields = line.strip().split(sep) if len(fields) != len(splitted_fields): raise errors.CommandError("Can't parse %s output: line '%s'" % (lvm_cmd, line)) data.append(splitted_fields) return data
def Rename(self, new_id): """Rename a device. This is not supported for drbd devices. """ raise errors.ProgrammerError("Can't rename a drbd device")
def Rename(self, new_id): """Rename this logical volume. """ if not isinstance(new_id, (tuple, list)) or len(new_id) != 2: raise errors.ProgrammerError("Invalid new logical id '%s'" % new_id) new_vg, new_name = new_id if new_vg != self._vg_name: raise errors.ProgrammerError("Can't move a logical volume across" " volume groups (from %s to to %s)" % (self._vg_name, new_vg)) result = utils.RunCmd(["lvrename", new_vg, self._lv_name, new_name]) if result.failed: base.ThrowError("Failed to rename the logical volume: %s", result.output) self._lv_name = new_name self.dev_path = utils.PathJoin("/dev", self._vg_name, self._lv_name)
def ReportLogMessage(self, job_id, serial, timestamp, log_type, log_msg): """Handles a log message. """ if self._job_id is None: self._job_id = job_id elif self._job_id != job_id: raise errors.ProgrammerError("The same reporter instance was used for" " more than one job") if log_type == constants.ELOG_JQUEUE_TEST: (sockname, test, arg) = log_msg return self._ProcessTestMessage(job_id, sockname, test, arg) elif (log_type == constants.ELOG_MESSAGE and log_msg.startswith(constants.JQT_MSGPREFIX)): if self._testmsgs is None: raise errors.OpExecError("Received test message without a preceding" " start message") testmsg = log_msg[len(constants.JQT_MSGPREFIX):] self._testmsgs.append(testmsg) self._all_testmsgs.append(testmsg) return return cli.StdioJobPollReportCb.ReportLogMessage(self, job_id, serial, timestamp, log_type, log_msg)
def perform(self): method = self._opts[pycurl.CUSTOMREQUEST] url = self._opts[pycurl.URL] request_body = self._opts[pycurl.POSTFIELDS] writefn = self._opts[pycurl.WRITEFUNCTION] if pycurl.HTTPHEADER in self._opts: baseheaders = _FormatHeaders(self._opts[pycurl.HTTPHEADER]) else: baseheaders = "" headers = http.ParseHeaders(StringIO(baseheaders)) if request_body: headers[http.HTTP_CONTENT_LENGTH] = str(len(request_body)) if self._opts.get(pycurl.HTTPAUTH, 0) & pycurl.HTTPAUTH_BASIC: try: userpwd = self._opts[pycurl.USERPWD] except KeyError: raise errors.ProgrammerError( "Basic authentication requires username" " and password") headers[http.HTTP_AUTHORIZATION] = \ "%s %s" % (http.auth.HTTP_BASIC_AUTH, base64.b64encode(userpwd)) path = _GetPathFromUri(url) (code, _, resp_body) = \ self._handler.FetchResponse(path, method, headers, request_body) self._info[pycurl.RESPONSE_CODE] = code if resp_body is not None: writefn(resp_body)
def _CheckLocksEnabled(self): """Checks if locking is enabled. @raise errors.ProgrammerError: In case locking is not enabled """ if not self._enable_locks: raise errors.ProgrammerError("Attempted to use disabled locks")
def _VerifyDiskParams(disk): """Verifies if all disk parameters are set. """ missing = set(constants.DISK_LD_DEFAULTS[disk.dev_type]) - set(disk.params) if missing: raise errors.ProgrammerError("Block device is missing disk parameters: %s" % missing)
def _FormatStatus(value): """Formats a job status. """ try: return _USER_JOB_STATUS[value] except KeyError: raise errors.ProgrammerError("Unknown job status code '%s'" % value)
def GetECId(self): """Returns the current execution context ID. """ if not self._ec_id: raise errors.ProgrammerError("Tried to use execution context id when" " not set") return self._ec_id
def FormatJobID(job_id): """Convert a job ID to int format. Currently this just is a no-op that performs some checks, but if we want to change the job id format this will abstract this change. @type job_id: int or long @param job_id: the numeric job id @rtype: int @return: the formatted job id """ if not isinstance(job_id, (int, long)): raise errors.ProgrammerError("Job ID '%s' not numeric" % job_id) if job_id < 0: raise errors.ProgrammerError("Job ID %s is negative" % job_id) return job_id
def GetArchInfo(): """Returns previsouly initialized architecture information. """ if _arch is None: raise errors.ProgrammerError("Architecture information hasn't been" " initialized") return _arch