def fioresult(initiator, check=True, expectation=False):
"""
Return the fio result. True: fio stdout. False: fio stderr.
"""
wait_nofiorunning(initiator, expectation=expectation)
wait_file_exists(initiator, 'out')
result = initiator.run_and_check('cat out', expectation=expectation)
if not result:
return result
if not result.message:
result = initiator.run_and_check('cat err', expectation=expectation)
if not result:
return result
if not result.mesage:
return ReturnCode(False, 'err: no output')
else:
return ReturnCode(False, 'err: %s' % result.message)
logger.info('fio result:\n%s' % result.message)
j = simplejson.loads(result.message)
if check:
for i in range(len(j['jobs'])):
if j['jobs'][i]['error'] != 0:
return ReturnCode(
False,
'fio[%d] error code: %s' % (i, j['jobs'][i]['error']))
return ReturnCode(True, j)
def make_spare(sr, num_spares=1, min_size=None):
"""
This function will create any numbre of spare drives in the specified appliance, provided drives are available.
:param sr: it's an appliance.srx.Srx object
:param num_spares: the number of spare drives you want to create, by default 1 will be created.
:returns: ReturnCode indicating if successful.
"""
d = sr.disks
slots = d.keys()
availableDisks = list()
for x in slots:
if min_size:
if d[x]['size'] == 'missing' or not int(
d[x]['size'].strip('*').split('.')[0]) >= int(min_size):
continue
if slot_is_available(sr, x):
availableDisks.append(str(sr.shelf) + '.' + x)
if len(availableDisks) >= num_spares:
for i in range(num_spares):
try:
# sr.run_and_check('spare %s' % availableDisks[i])
sr.spare(availableDisks[i])
except ApplianceError as e:
logger.error('Failed to create spare drive: %s', e)
else:
return ReturnCode(
False,
'Not enough disk to complete request to create %d spare drives' %
num_spares)
return ReturnCode(True, 'Spare drives were created successfully')
def is_down(self):
try:
self.run_and_check('echo waiting for ssh to shutdown', timeout=5)
except (OSError, TIMEOUT):
logger.info('ssh appears to have shutdown')
return ReturnCode(True, "host appears down")
return ReturnCode(False)
def pool_is_empty(vsx, pool):
"""
Verifypool makes sure that the pool has no extents allocated.
"""
if not isinstance(vsx, Vsx):
e = ReturnCode(False)
e.message = "object is not a Vsx instance"
return e
total = 0
free = 0
unique = 0
shelf = vsx.shelf
ret = vsx.run_and_check("pools -a %s" %
pool) # TODO: teach this that pools has -a
if not ret.status:
return ret
p = ret.message
ret = ReturnCode(True)
m = re.search(r"Total[ \t]+Exts[ \t]+:[ \t]+([0-9]+)[ \t]+", p)
if m:
total = int(m.group(1))
m = re.search(r"Free[ \t]+Exts[ \t]+:[ \t]+([0-9]+)[ \t]+", p)
if m:
free = int(m.group(1))
m = re.search(r"Unique[ \t]+Exts[ \t]+:[ \t]+([0-9]+)[ \t]+", p)
if m:
unique = int(m.group(1))
if total:
e = "Empty pool %s on %s has %d total extents:\n%s" % (pool, shelf,
total, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
if free:
e = "Empty pool %s on %s has %d free extents:\n%s" % (pool, shelf,
free, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
if unique:
e = "Empty pool %s on %s has %d unique extents:\n%s" % (pool, shelf,
unique, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
m = re.search(r"PVs[ \t]+:[ \t]+[0-9]+\.[0-9]+", p)
if m:
e = "Empty pool %s on %s has PVs:\n%s" % (pool, shelf, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
m = re.search(r"LVs[ \t]+:[ \t]+(.*)[ \t]+", p)
if m:
e = "Empty pool %s on %s has LVs:\n%s" % (pool, shelf, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
return ret
def wget(initiator, path, fname, loops=10, sleeptime=1):
initiator.run_and_check('rm %s' % fname, expectation=False)
initiator.run('wget -q %s/%s' % (path, fname), wait=False)
for i in range(loops):
if exists(initiator, fname):
return ReturnCode(True)
time.sleep(sleeptime)
return ReturnCode(False, 'wget: %s/%s not available' % (path, fname))
def cmp_hba_ports(h, p):
checks = (h['port'] != str(p.index), h['mac'] != p.ea, h['type'] != p.name,
h['link']['max'] != str(p.maxlink),
h['link']['speed'] != str(p.currentlink))
if True in checks:
return ReturnCode(False, 'hba %s does not match ports %s' % (h, p))
return ReturnCode(True)
def get_elstats(initiator):
"""
This function will translate data from /dev/ethdrv/elstat into a dictionary.
"""
elstat = OrderedDict()
if initiator.os == 'solaris':
fname = '/dev/ethdrv/elstats'
elif initiator.os == 'linux':
fname = '/proc/ethdrv/elstats'
else:
raise (NotImplementedError('%s does not support elstats' %
initiator.os))
if isinstance(initiator, otto.connections.ssh.Client):
sftpsession = initiator.open_sftp()
try:
fh = sftpsession.open(fname)
result = ReturnCode(True)
result.message = fh.read()
except Exception as e:
result = ReturnCode(False)
result.message = str(e)
return result
else:
cmd = 'cat %s' % fname
result = initiator.run_and_check(cmd)
if result:
for line in result.message.splitlines():
if line:
if line.startswith('Listen'):
if not elstat.get('Listen'):
elstat['Listen'] = dict()
k = line.split()[1].strip(
'[]') # This will extract just the number from [0]
v = line.split()[2:]
elstat['Listen'][k] = v
elif line.startswith('Closed'):
if not elstat.get('Closed'):
elstat['Closed'] = dict()
k = line.split()[1].strip(
'[]') # This will extract just the number from [0]
v = line.split()[2:]
elstat['Listen'][k] = v
else:
kvpair = line.split(':')
k = kvpair[0]
if len(kvpair) < 2:
continue
v = kvpair[1].strip()
elstat[k] = v
return elstat
def connect(self, timeout=10, args=None, nolog=False):
"""Connect to and authenticate with host."""
cmd = "ssh"
if args is None:
args = [
"-q", "-o PubkeyAuthentication no",
"-o UserKnownHostsFile=/dev/null",
"-o UserKnownHostsFile2=/dev/null",
"-o StrictHostKeyChecking=no"
]
args.append("-l" + self.user)
args.append(self.host)
if not nolog:
logger.debug("%s %s" % (cmd, str(args)))
try:
spawn.__init__(self, cmd, args, timeout)
prompt = self.expect(["(?i)password: "******"(?i)password"])
if prompt in [0, 1]:
self.sendline(self.password)
else:
self.close()
return False
try:
if self.prompt == ':\>':
self.expect_exact(self.prompt)
else:
self.expect(self.prompt)
except TIMEOUT:
raise ConnectionError(
"Connected but didn't find prompt '%s'\n instead self.before was:\n%s"
% (self.prompt, self.before))
self.connected = True
except KeyError as e:
if not nolog:
logger.critical("Couldn't complete connection")
if e.message:
logger.error(e.message)
return False
except (EOF, TIMEOUT) as e:
if not nolog:
logger.critical("Couldn't complete connection to %s@%s" %
(self.user, self.host))
if e.message:
logger.error(e.message)
return ReturnCode(False, message=e.message)
return ReturnCode(True, message=self.before)
def get_ethdrv(self, fname):
"""
Required function for Ethdrv class
"""
sftpsession = self.open_sftp()
try:
fh = sftpsession.open('/dev/ethdrv/%s' % fname, 'r')
ret = ReturnCode(True)
ret.message = fh.read()
except Exception as e:
ret = ReturnCode(False, str(e))
return ret
def lun_exists(self, lun, flush=True):
"""
Returns lun's aoestat dict::
{'device': 'sd379', 'port': ['1'], 'target': '91.1', 'size': '2000.398GB'}
or False in ReturnCode format
"""
if flush:
self.aoeflush()
n = self.aoestat
if lun in n:
return ReturnCode(True, n[lun])
return ReturnCode(False, '%s not found' % lun)
def __checkasync(self, buf, cmd):
"""
__checkasync checks pexpect buffers for async CEC
messages which often cause timeouts. If found,
the function returns a ReturnCode which is True and
consists of the async message. If an expected async
message is not found, the ReturnCode is False and its
message is the pexpect buffer.
"""
logger.debug("CHECKASYNC: '%s'\nbuf:\n'%s'" % (cmd, buf))
ret = ""
msg = buf
for msg in self.amsgs:
match = re.search(r"(%s)(\r\n)*" % msg, buf)
if not match:
continue
logger.debug("CHECKASYNC: FOUND MATCH: '%s'" % msg)
start = buf.find(match.group(1))
end = len(match.group(1))
if match.lastindex == 3:
# include the matched newline
end += len(match.group(3))
# what surrounds the match (hopefully the expected cmd)
ret = buf[:start] + buf[start + end:]
# the matched async message
msg = buf[start:start + end]
break
if not ret:
# prevent infinite recursion
return ReturnCode(False)
if ret.find(cmd) != -1:
logger.debug("CHECKASYNC: successfully matched and removed '%s'" %
msg)
return ReturnCode(True, msg)
# we have had instances where two async msgs were output while pexpect
# was waiting for the echo of a cmd ... recurse to handle any multiples
r = self.__checkasync(ret, cmd)
if r:
return r
e = "CHECKASYNC: '%s' timed-out, but no asynchronous output found in:\n" \
"'%s'\nret: '%s'" % (cmd, buf, ret)
logger.error(e)
return ReturnCode(False, e)
def save_sos(sr, loc='.'):
"""
Save a copy of sos output into a local file
:param sr: an srx object
:param loc: file destination drectory
filename will be in the following format::
sos_1521_15_1387393873.113084.txt
"""
result = ReturnCode(False)
sos_output = sr.sos
# File name will be a combination of model+shelfID+time
file_name = 'sos_%s_%s_%s.txt' % (sr.model, sr.shelf, str(time()))
# full path + file name where sos will be saved.
sos_file = '%s/%s' % (loc, file_name)
try:
f = open(sos_file, 'w')
for line in sos_output.split('\r\n'):
f.write('%s\n' % line)
f.close()
result.status = True
result.message = 'sos output stores in %s' % sos_file
except IOError as e:
result.message = e
return result
def add_ssd_to_fc(sr, nssds=1):
"""
Add SSD a number to flashcache pool
:param sr: an srx object
:param nssds: number of drives ot add
"""
ret = ReturnCode(False)
disks = sr.disks.keys()
available_disks = [
"%s.%s" % (sr.shelf, x) for x in disks
if slot_is_available(sr, x, cache=True)
]
ssds = ["%s" % d for d in available_disks if is_ssd(sr, d)]
if not ssds or len(ssds) < nssds:
ret.message = 'No more ssds to add to flashcache'
ret.status = False
return ret
ssds.reverse() # implicitly sorts ssds wrt slot numbers
for _ in range(nssds):
ret = sr.fcconfig(ssds.pop())
if not ret:
break
return ret
def fail_disk(sr, lun, element=None, raid='0'):
"""
Fail a single disk on a lun, if not specified, the function will choose one for you.
:param sr: an srx object
:param lun: this is the lun id from which we want to fail an element
:param element: if this is passed, the element we want to fail, it's in the form '1','3' or '5'
:param raid: this value was for concat raids and is not used anymore the default value is 0
:returns: A ReturnCode object the message contains the disk that was failed or if no disk was failed at all.
"""
result = ReturnCode(False, 'err: No disk was failed in lun %s' % lun)
# get a list of disks for lun
luns = sr.list
# We search among the components of the lun for one to fail.
for l in luns[lun]['raids'][int(raid)]['components']:
if element is None or element == l['position']:
if l['stat'] == 'normal':
try:
drive = '%s.%s.%s' % (lun, raid, l['position'])
result = sr.fail(drive)
result.message = drive
return result
except ApplianceError as e:
result.message = e
return result
return result
def cycle(self, outlet, expectation=True):
"""
Power cycle a particular port in the PDU specified by the outlet number.
"""
outlet_state = self.state
result = ReturnCode(True)
outlet = str(outlet)
logger.info("cycle %s" % outlet)
self.send("1")
self.expect("Control Sub Menu")
self.send("1")
self.expect("Outlet State Sub Menu")
if int(outlet) in range(1, 9):
self.send("1")
elif int(outlet) in range(9, 17):
self.send("2")
elif int(outlet) in range(17, 25):
self.send("3")
self.expect("Outlet Control Sub Menu")
self.send(outlet)
self.expect("%s Command Choices" % outlet_state[outlet]['name'])
self.send("3")
self.expect("%s Requested Command is Reboot" %
outlet_state[outlet]['name'])
self.send("\r")
self.expect("Outlet State Sub Menu")
if not expectation:
return result
elif re.search("error|fail", self.before, re.MULTILINE | re.I):
raise ApplianceError("pdu cycle failed")
# get back to main menu
for i in range(0, 2):
self.send('\x1b')
self.expect("Select Item Number")
def rdunfail(sr, disk):
"""
unset the rdfail flag on a drive
:param sr: an srx object
:param disk: drive on which to set the rdfail flag, we test the flag to verify it is on, before setting it
Return: a ReturnCode True if flag was set to off False if failed (drive was already off or missing.
"""
result = ReturnCode(False)
d = sr.disks.get(disk)
if d and d[
'rdfail'] == 'on': # Let's test for the result of sr.disks.get, just in case that disk is missing.
cmd = "echo rdfail off > /raiddev/%s/ctl" % disk
try:
if sr.version >= 7:
result = sr.expert_run(cmd)
else:
result = sr.run_and_check(cmd)
except ApplianceError, e:
logger.error('A problem was found trying to rdunfail disk %s: %s',
disk, e)
def run_and_check(self, cmd, expectation=True):
"""
Run a command check the result. If the caller cares about failure
and the command fails we raise a generic exception.
"""
result = ReturnCode(True)
logger.info(cmd + " called")
self.pdu.connect()
result.message = self.pdu.run(cmd)
self.pdu.disconnect()
e = Exception()
if not result.message.find('E000: Success'):
if result.message.find('E101: Command Not Found') > -1:
logger.error(result.message)
result.status = False
failmsg = cmd + " failed"
e = ApplianceError(failmsg)
elif result.message.startswith('E102: Parameter Error'):
logger.critical(result.message)
result.status = False
failmsg = cmd + " failed"
e = ApplianceUsage(failmsg)
result.status = False
if not expectation:
return result
elif not result.status:
raise e
return result
def _zpool_create(self, pname, targets, ptype='', spares=None):
"""
This gets called from abstraction zpool_create and does zpool creation.
"""
if type(targets) == list:
devices = [self._target2device(target) for target in targets]
spare_devices = []
if spares:
spare_devices = [
self._target2device(spare) for spare in spares
]
if len(devices) > 0:
cmd = 'zpool create -f %s %s' % (pname, ptype)
for d in devices:
if d:
cmd += ' %s' % d
if spares:
cmd += ' spare'
for s in spare_devices:
if s:
cmd += ' %s' % s
return self.run_and_check(cmd, False)
# it could be a question being asked during the creation.
else:
return ReturnCode(False, 'Devices not found at initiator')
elif type(targets) == str:
device = self._target2device(targets)
spare_devices = []
if spares:
spare_devices = [
self._target2device(spare) for spare in spares
]
if device:
cmd = 'zpool create -f %s %s %s' % (pname, ptype, device)
if spares:
cmd += ' spare'
for s in spare_devices:
if s:
cmd += ' %s' % s
return self.run_and_check(
cmd, False) # FIXME, same as above run command.
else:
return ReturnCode(False,
'Device for target %s not found' % targets)
def aoerevalidate(self, shelf_lun): # STUB
"""
Calls aoe-revalidate e{shelf}.{lun} on initiator
Accept a string e.g. 'e4.1' or AoEAddress
Returns ReturnCode object
"""
ret = ReturnCode(False)
return ret
def loadaoe(self): # STUB
"""
Loads the AoE driver module defined in self.coraidmodule:
either the HBA driver 'ethdrv', or the software initiator 'aoe'.
"""
ret = ReturnCode(False)
return ret
def unloadaoe(self): # STUB
"""
Unload the AoE driver module defined in self.coraid_module
Returns a ReturnCode object
"""
ret = ReturnCode(False)
return ret
def claim(self, lun): # STUB
"""
claim a LUN
accept string or AoEAddress type
Return : ReturnCode
"""
ret = ReturnCode(False)
return ret
def run(self, cmd, timeout=None, bufsize=-1):
"""
:param cmd: command to run on remote host
:type cmd: str
:param timeout: timeout on blocking read/write operations when exceeded socket error will be raised
:type timeout: float
:param bufsize: byte size of the buffer for the filehandle returned
:type bufsize: int
:rtype: ReturnCode
"""
ret = ReturnCode(False)
if not self.connected:
raise ConnectionError(
"Run was called on an unconnected host. Did you check the result of connect()?"
)
try:
if self.environmentals:
envstring = str()
for var, value in self.environmentals.items():
statement = "%s=%s " % (var, value)
envstring += statement
cmd = "%s%s" % (envstring, cmd)
if self.cwd:
cmd = "cd %s && %s" % (self.cwd, cmd)
self._log(logging.DEBUG, 'running command: "%s"' % cmd)
stdin, stdout, stderr = self.exec_command(command=cmd,
timeout=timeout,
bufsize=bufsize)
except paramiko.SSHException as e:
err = "Couldn't complete the command: %s" % str(e)
logger.critical(err)
ret.message = err
return ret
# we must read stderr _before_ stdout
# otherwise paramiko losses the stdout data
try:
ret.raw = Data(ret.raw.status, ret.raw.stdout, stderr.read())
except socket.timeout:
ret.message = "Timeout"
return ret
status = stdout.channel.recv_exit_status()
ret.raw = Data(status, stdout.read(), ret.raw.stderr)
if status != 0:
ret.message = ret.raw.stderr
else:
ret.status = True
ret.message = ret.raw.stdout
stdin.close()
return ret
def aoediscover(self):
"""
Call the driver's discover command. Returns ReturnCode object
"""
cmd = "%s discover" % ethdrvadm
output = self.run(cmd)
if output.find("error") > -1: # I don't think this will ever happen
status = False
else:
status = True
return ReturnCode(status=status, message=output)
def claim_targets(self, target_list):
"""
Claim the luns
"""
lun_list = target_list.split(' ')
result = ReturnCode(False, message="empty target list")
for lun in lun_list:
cmd = mkcmdstr('esxcli ethdrv claim -t', lun)
logger.info(lun)
result = self.run_and_check(cmd)
return result
def lv_is_empty(vsx, lv):
"""
Verify there are no snap extents left on this LV.
"""
su = get_snap_used(vsx, lv)
r = ReturnCode(False)
if su == "0.000":
r.status = True
else:
r.message = "Stray snap extents on LV %s: %s" % (lv, su)
logger.error(r.message)
return r
def slot_is_available(sr, slot, cache=False):
"""
check slots' role as part of a lun, or if is marked as an spare or cache disk or if it missing.
:param slot: the slot we want to check, it's a str or int in the form: '3', '12' or '34'.
:param cache: is it ok to assume that sr.disks is unchanged while we are in this function?
:return: a ReturnCode object, message contains the output of disks command for the specified slot
"""
# !!this is a surprisingly complicated function!! You have been warned.
if sr.use_slots and slot not in sr.use_slots:
return ReturnCode(status=False, message="masked by use_slots")
t = sr.use_slots
sr.use_slots = [slot]
diskcache = sr.cache.get('disks')
if not ((cache and diskcache) and diskcache.get(slot)):
disk = getattr(sr, "s%s" % slot)
if not disk.get('model'):
sr.use_slots = t
return ReturnCode(status=False, message=str(disk))
else:
disk = diskcache.get(slot)
sr.use_slots = t
if disk['role'] in ['', None]:
if disk['size'] != 'missing':
status = True
else:
status = False
else:
status = False
return ReturnCode(status=status, message=disk)
def pv_is_empty(vsx, pv=None): # TODO: use pv to filter results
"""
Check if most of the metadata is empty and that extents are
correctly accounted for. pv parameter is ignored for now
"""
if not isinstance(vsx, Vsx):
e = ReturnCode(False)
e.message = "object is not a Vsx instance"
return e
total = 0
free = 0
dirty = 0
meta = 0
sh = vsx.shelf
ret = vsx.pvs
if not ret.status:
return ret
p = ret.message
ret = ReturnCode(True)
used = calculate_metaext(total, 4096)
if used != dirty:
e = "pv has too many dirty extents, should be %d on VSX %s:\n%s" % (
used, sh, p)
logger.error(e)
ret.status = False
ret.message = e
if meta:
e = "meta extents not zero on VSX %s:\n%s" % (sh, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
if total != free + dirty:
e = "pv accounting error on VSX %s:\n%s" % (sh, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
return ret
def cmp_aoestat_targets(a, t):
# Confirm aoestats.paths in targets.ea
for l in a.port:
for m in a.paths[l].address:
found = False
for n in t:
mask = bin(n.ports)[2:][::-1]
if a.paths[l].port < len(mask) and mask[
a.paths[l].port] == '1' and m == n.ea:
found = True
if not found:
return ReturnCode(
False, 'aoestat %s does not match targets %s' % (a, t))
# Confirm targets.ea in aoestats.paths
for l in t:
mask = bin(l.ports)[2:][::-1]
for m in range(len(mask)):
if mask[m] == '1':
if l.ea not in a.paths[m].address:
return ReturnCode(
False, 'targets %s does not match aoestat %s' % (t, a))
return ReturnCode(True)
def fio(init1, args):
"""
Pull out fio configuration options from fc and execute
"""
init1.run_and_check('set +m; rm -f *out *err')
cmd = 'fio --output-format=json %s> out 2> err &' % args
logger.info(cmd)
init1.run(cmd, wait=False)
for i in range(10):
n = nofiorunning(init1)
if not n:
return ReturnCode(True)
time.sleep(1)
return fioresult(init1)
