def add_ssd_to_fc(sr, nssds=1):
"""
Add SSD a number to flashcache pool
:param sr: an srx object
:param nssds: number of drives ot add
"""
ret = ReturnCode(False)
disks = sr.disks.keys()
available_disks = [
"%s.%s" % (sr.shelf, x) for x in disks
if slot_is_available(sr, x, cache=True)
]
ssds = ["%s" % d for d in available_disks if is_ssd(sr, d)]
if not ssds or len(ssds) < nssds:
ret.message = 'No more ssds to add to flashcache'
ret.status = False
return ret
ssds.reverse() # implicitly sorts ssds wrt slot numbers
for _ in range(nssds):
ret = sr.fcconfig(ssds.pop())
if not ret:
break
return ret
def make_spare(sr, num_spares=1, min_size=None):
"""
This function will create any numbre of spare drives in the specified appliance, provided drives are available.
:param sr: it's an appliance.srx.Srx object
:param num_spares: the number of spare drives you want to create, by default 1 will be created.
:returns: ReturnCode indicating if successful.
"""
d = sr.disks
slots = d.keys()
availableDisks = list()
for x in slots:
if min_size:
if d[x]['size'] == 'missing' or not int(
d[x]['size'].strip('*').split('.')[0]) >= int(min_size):
continue
if slot_is_available(sr, x):
availableDisks.append(str(sr.shelf) + '.' + x)
if len(availableDisks) >= num_spares:
for i in range(num_spares):
try:
# sr.run_and_check('spare %s' % availableDisks[i])
sr.spare(availableDisks[i])
except ApplianceError as e:
logger.error('Failed to create spare drive: %s', e)
else:
return ReturnCode(
False,
'Not enough disk to complete request to create %d spare drives' %
num_spares)
return ReturnCode(True, 'Spare drives were created successfully')
def add_ssd_to_fc(sr, nssds=1):
"""
Add SSD a number to flashcache pool
:param sr: an srx object
:param nssds: number of drives ot add
"""
ret = ReturnCode(False)
disks = sr.disks.keys()
available_disks = ["%s.%s" % (sr.shelf, x) for x in disks if slot_is_available(sr, x, cache=True)]
ssds = ["%s" % d for d in available_disks if is_ssd(sr, d)]
if not ssds or len(ssds) < nssds:
ret.message = 'No more ssds to add to flashcache'
ret.status = False
return ret
ssds.reverse() # implicitly sorts ssds wrt slot numbers
for _ in range(nssds):
ret = sr.fcconfig(ssds.pop())
if not ret:
break
return ret
def save_sos(sr, loc='.'):
"""
Save a copy of sos output into a local file
:param sr: an srx object
:param loc: file destination drectory
filename will be in the following format::
sos_1521_15_1387393873.113084.txt
"""
result = ReturnCode(False)
sos_output = sr.sos
# File name will be a combination of model+shelfID+time
file_name = 'sos_%s_%s_%s.txt' % (sr.model, sr.shelf, str(time()))
# full path + file name where sos will be saved.
sos_file = '%s/%s' % (loc, file_name)
try:
f = open(sos_file, 'w')
for line in sos_output.split('\r\n'):
f.write('%s\n' % line)
f.close()
result.status = True
result.message = 'sos output stores in %s' % sos_file
except IOError as e:
result.message = e
return result
def fail_disk(sr, lun, element=None, raid='0'):
"""
Fail a single disk on a lun, if not specified, the function will choose one for you.
:param sr: an srx object
:param lun: this is the lun id from which we want to fail an element
:param element: if this is passed, the element we want to fail, it's in the form '1','3' or '5'
:param raid: this value was for concat raids and is not used anymore the default value is 0
:returns: A ReturnCode object the message contains the disk that was failed or if no disk was failed at all.
"""
result = ReturnCode(False, 'err: No disk was failed in lun %s' % lun)
# get a list of disks for lun
luns = sr.list
# We search among the components of the lun for one to fail.
for l in luns[lun]['raids'][int(raid)]['components']:
if element is None or element == l['position']:
if l['stat'] == 'normal':
try:
drive = '%s.%s.%s' % (lun, raid, l['position'])
result = sr.fail(drive)
result.message = drive
return result
except ApplianceError as e:
result.message = e
return result
return result
def save_sos(sr, loc='.'):
"""
Save a copy of sos output into a local file
:param sr: an srx object
:param loc: file destination drectory
filename will be in the following format::
sos_1521_15_1387393873.113084.txt
"""
result = ReturnCode(False)
sos_output = sr.sos
# File name will be a combination of model+shelfID+time
file_name = 'sos_%s_%s_%s.txt' % (sr.model, sr.shelf, str(time()))
# full path + file name where sos will be saved.
sos_file = '%s/%s' % (loc, file_name)
try:
f = open(sos_file, 'w')
for line in sos_output.split('\r\n'):
f.write('%s\n' % line)
f.close()
result.status = True
result.message = 'sos output stores in %s' % sos_file
except IOError as e:
result.message = e
return result
def run_and_check(self, cmd, expectation=True):
"""
Run a command check the result. If the caller cares about failure
and the command fails we raise a generic exception.
"""
result = ReturnCode(True)
logger.info(cmd + " called")
self.pdu.connect()
result.message = self.pdu.run(cmd)
self.pdu.disconnect()
e = Exception()
if not result.message.find('E000: Success'):
if result.message.find('E101: Command Not Found') > -1:
logger.error(result.message)
result.status = False
failmsg = cmd + " failed"
e = ApplianceError(failmsg)
elif result.message.startswith('E102: Parameter Error'):
logger.critical(result.message)
result.status = False
failmsg = cmd + " failed"
e = ApplianceUsage(failmsg)
result.status = False
if not expectation:
return result
elif not result.status:
raise e
return result
def fail_disk(sr, lun, element=None, raid='0'):
"""
Fail a single disk on a lun, if not specified, the function will choose one for you.
:param sr: an srx object
:param lun: this is the lun id from which we want to fail an element
:param element: if this is passed, the element we want to fail, it's in the form '1','3' or '5'
:param raid: this value was for concat raids and is not used anymore the default value is 0
:returns: A ReturnCode object the message contains the disk that was failed or if no disk was failed at all.
"""
result = ReturnCode(False, 'err: No disk was failed in lun %s' % lun)
# get a list of disks for lun
luns = sr.list
# We search among the components of the lun for one to fail.
for l in luns[lun]['raids'][int(raid)]['components']:
if element is None or element == l['position']:
if l['stat'] == 'normal':
try:
drive = '%s.%s.%s' % (lun, raid, l['position'])
result = sr.fail(drive)
result.message = drive
return result
except ApplianceError as e:
result.message = e
return result
return result
def fioresult(initiator, check=True, expectation=False):
"""
Return the fio result. True: fio stdout. False: fio stderr.
"""
wait_nofiorunning(initiator, expectation=expectation)
wait_file_exists(initiator, 'out')
result = initiator.run_and_check('cat out', expectation=expectation)
if not result:
return result
if not result.message:
result = initiator.run_and_check('cat err', expectation=expectation)
if not result:
return result
if not result.mesage:
return ReturnCode(False, 'err: no output')
else:
return ReturnCode(False, 'err: %s' % result.message)
logger.info('fio result:\n%s' % result.message)
j = simplejson.loads(result.message)
if check:
for i in range(len(j['jobs'])):
if j['jobs'][i]['error'] != 0:
return ReturnCode(
False,
'fio[%d] error code: %s' % (i, j['jobs'][i]['error']))
return ReturnCode(True, j)
def is_down(self):
try:
self.run_and_check('echo waiting for ssh to shutdown', timeout=5)
except (OSError, TIMEOUT):
logger.info('ssh appears to have shutdown')
return ReturnCode(True, "host appears down")
return ReturnCode(False)
def pool_is_empty(vsx, pool):
"""
Verifypool makes sure that the pool has no extents allocated.
"""
if not isinstance(vsx, Vsx):
e = ReturnCode(False)
e.message = "object is not a Vsx instance"
return e
total = 0
free = 0
unique = 0
shelf = vsx.shelf
ret = vsx.run_and_check("pools -a %s" %
pool) # TODO: teach this that pools has -a
if not ret.status:
return ret
p = ret.message
ret = ReturnCode(True)
m = re.search(r"Total[ \t]+Exts[ \t]+:[ \t]+([0-9]+)[ \t]+", p)
if m:
total = int(m.group(1))
m = re.search(r"Free[ \t]+Exts[ \t]+:[ \t]+([0-9]+)[ \t]+", p)
if m:
free = int(m.group(1))
m = re.search(r"Unique[ \t]+Exts[ \t]+:[ \t]+([0-9]+)[ \t]+", p)
if m:
unique = int(m.group(1))
if total:
e = "Empty pool %s on %s has %d total extents:\n%s" % (pool, shelf,
total, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
if free:
e = "Empty pool %s on %s has %d free extents:\n%s" % (pool, shelf,
free, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
if unique:
e = "Empty pool %s on %s has %d unique extents:\n%s" % (pool, shelf,
unique, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
m = re.search(r"PVs[ \t]+:[ \t]+[0-9]+\.[0-9]+", p)
if m:
e = "Empty pool %s on %s has PVs:\n%s" % (pool, shelf, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
m = re.search(r"LVs[ \t]+:[ \t]+(.*)[ \t]+", p)
if m:
e = "Empty pool %s on %s has LVs:\n%s" % (pool, shelf, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
return ret
def wget(initiator, path, fname, loops=10, sleeptime=1):
initiator.run_and_check('rm %s' % fname, expectation=False)
initiator.run('wget -q %s/%s' % (path, fname), wait=False)
for i in range(loops):
if exists(initiator, fname):
return ReturnCode(True)
time.sleep(sleeptime)
return ReturnCode(False, 'wget: %s/%s not available' % (path, fname))
def cmp_hba_ports(h, p):
checks = (h['port'] != str(p.index), h['mac'] != p.ea, h['type'] != p.name,
h['link']['max'] != str(p.maxlink),
h['link']['speed'] != str(p.currentlink))
if True in checks:
return ReturnCode(False, 'hba %s does not match ports %s' % (h, p))
return ReturnCode(True)
def connect(self, timeout=10, args=None, nolog=False):
"""Connect to and authenticate with host."""
cmd = "ssh"
if args is None:
args = [
"-q", "-o PubkeyAuthentication no",
"-o UserKnownHostsFile=/dev/null",
"-o UserKnownHostsFile2=/dev/null",
"-o StrictHostKeyChecking=no"
]
args.append("-l" + self.user)
args.append(self.host)
if not nolog:
logger.debug("%s %s" % (cmd, str(args)))
try:
spawn.__init__(self, cmd, args, timeout)
prompt = self.expect(["(?i)password: "******"(?i)password"])
if prompt in [0, 1]:
self.sendline(self.password)
else:
self.close()
return False
try:
if self.prompt == ':\>':
self.expect_exact(self.prompt)
else:
self.expect(self.prompt)
except TIMEOUT:
raise ConnectionError(
"Connected but didn't find prompt '%s'\n instead self.before was:\n%s"
% (self.prompt, self.before))
self.connected = True
except KeyError as e:
if not nolog:
logger.critical("Couldn't complete connection")
if e.message:
logger.error(e.message)
return False
except (EOF, TIMEOUT) as e:
if not nolog:
logger.critical("Couldn't complete connection to %s@%s" %
(self.user, self.host))
if e.message:
logger.error(e.message)
return ReturnCode(False, message=e.message)
return ReturnCode(True, message=self.before)
def lv_is_empty(vsx, lv):
"""
Verify there are no snap extents left on this LV.
"""
su = get_snap_used(vsx, lv)
r = ReturnCode(False)
if su == "0.000":
r.status = True
else:
r.message = "Stray snap extents on LV %s: %s" % (lv, su)
logger.error(r.message)
return r
def get_ethdrv(self, fname):
"""
Required function for Ethdrv class
"""
sftpsession = self.open_sftp()
try:
fh = sftpsession.open('/dev/ethdrv/%s' % fname, 'r')
ret = ReturnCode(True)
ret.message = fh.read()
except Exception as e:
ret = ReturnCode(False, str(e))
return ret
def lv_is_empty(vsx, lv):
"""
Verify there are no snap extents left on this LV.
"""
su = get_snap_used(vsx, lv)
r = ReturnCode(False)
if su == "0.000":
r.status = True
else:
r.message = "Stray snap extents on LV %s: %s" % (lv, su)
logger.error(r.message)
return r
def get_ethdrv(self, fname):
"""
Required function for Ethdrv class
"""
sftpsession = self.open_sftp()
try:
fh = sftpsession.open('/dev/ethdrv/%s' % fname, 'r')
ret = ReturnCode(True)
ret.message = fh.read()
except Exception as e:
ret = ReturnCode(False, str(e))
return ret
def get_elstats(initiator):
"""
This function will translate data from /dev/ethdrv/elstat into a dictionary.
"""
elstat = OrderedDict()
if initiator.os == 'solaris':
fname = '/dev/ethdrv/elstats'
elif initiator.os == 'linux':
fname = '/proc/ethdrv/elstats'
else:
raise (NotImplementedError('%s does not support elstats' % initiator.os))
if isinstance(initiator, otto.connections.ssh.Client):
sftpsession = initiator.open_sftp()
try:
fh = sftpsession.open(fname)
result = ReturnCode(True)
result.message = fh.read()
except Exception as e:
result = ReturnCode(False)
result.message = str(e)
return result
else:
cmd = 'cat %s' % fname
result = initiator.run_and_check(cmd)
if result:
for line in result.message.splitlines():
if line:
if line.startswith('Listen'):
if not elstat.get('Listen'):
elstat['Listen'] = dict()
k = line.split()[1].strip('[]') # This will extract just the number from [0]
v = line.split()[2:]
elstat['Listen'][k] = v
elif line.startswith('Closed'):
if not elstat.get('Closed'):
elstat['Closed'] = dict()
k = line.split()[1].strip('[]') # This will extract just the number from [0]
v = line.split()[2:]
elstat['Listen'][k] = v
else:
kvpair = line.split(':')
k = kvpair[0]
if len(kvpair) < 2:
continue
v = kvpair[1].strip()
elstat[k] = v
return elstat
def verifylun(obj, lun):
obj.lnx1.aoeflush(aflag=False)
aoestat = obj.lnx1.aoestat
shelflun = '{0}.{1}'.format(obj.srx1_shelf, lun)
if shelflun in aoestat and not re.search('init', aoestat[shelflun]['path']):
result = ReturnCode(True)
result.message = aoestat[shelflun]['path']
# DEBUG
obj.log.write("VERIFY LUN: Success for {0}".format(shelflun))
return result
# DEBUG PRINT
obj.log.write(
"VERIFY LUN: Did not find {0} yet, sleeping {1} seconds to retry...".format(shelflun, verifylun.sleeptime))
return False
def lun_exists(self, lun, flush=True):
"""
Returns lun's aoestat dict::
{'device': 'sd379', 'port': ['1'], 'target': '91.1', 'size': '2000.398GB'}
or False in ReturnCode format
"""
if flush:
self.aoeflush()
n = self.aoestat
if lun in n:
return ReturnCode(True, n[lun])
return ReturnCode(False, '%s not found' % lun)
def run(self, cmd, timeout=None, bufsize=-1):
"""
:param cmd: command to run on remote host
:type cmd: str
:param timeout: timeout on blocking read/write operations when exceeded socket error will be raised
:type timeout: float
:param bufsize: byte size of the buffer for the filehandle returned
:type bufsize: int
:rtype: ReturnCode
"""
ret = ReturnCode(False)
if not self.connected:
raise ConnectionError(
"Run was called on an unconnected host. Did you check the result of connect()?"
)
try:
if self.environmentals:
envstring = str()
for var, value in self.environmentals.items():
statement = "%s=%s " % (var, value)
envstring += statement
cmd = "%s%s" % (envstring, cmd)
if self.cwd:
cmd = "cd %s && %s" % (self.cwd, cmd)
self._log(logging.DEBUG, 'running command: "%s"' % cmd)
stdin, stdout, stderr = self.exec_command(command=cmd,
timeout=timeout,
bufsize=bufsize)
except paramiko.SSHException as e:
err = "Couldn't complete the command: %s" % str(e)
logger.critical(err)
ret.message = err
return ret
# we must read stderr _before_ stdout
# otherwise paramiko losses the stdout data
try:
ret.raw = Data(ret.raw.status, ret.raw.stdout, stderr.read())
except socket.timeout:
ret.message = "Timeout"
return ret
status = stdout.channel.recv_exit_status()
ret.raw = Data(status, stdout.read(), ret.raw.stderr)
if status != 0:
ret.message = ret.raw.stderr
else:
ret.status = True
ret.message = ret.raw.stdout
stdin.close()
return ret
def __checkasync(self, buf, cmd):
"""
__checkasync checks pexpect buffers for async CEC
messages which often cause timeouts. If found,
the function returns a ReturnCode which is True and
consists of the async message. If an expected async
message is not found, the ReturnCode is False and its
message is the pexpect buffer.
"""
logger.debug("CHECKASYNC: '%s'\nbuf:\n'%s'" % (cmd, buf))
ret = ""
msg = buf
for msg in self.amsgs:
match = re.search(r"(%s)(\r\n)*" % msg, buf)
if not match:
continue
logger.debug("CHECKASYNC: FOUND MATCH: '%s'" % msg)
start = buf.find(match.group(1))
end = len(match.group(1))
if match.lastindex == 3:
# include the matched newline
end += len(match.group(3))
# what surrounds the match (hopefully the expected cmd)
ret = buf[:start] + buf[start + end:]
# the matched async message
msg = buf[start:start + end]
break
if not ret:
# prevent infinite recursion
return ReturnCode(False)
if ret.find(cmd) != -1:
logger.debug("CHECKASYNC: successfully matched and removed '%s'" %
msg)
return ReturnCode(True, msg)
# we have had instances where two async msgs were output while pexpect
# was waiting for the echo of a cmd ... recurse to handle any multiples
r = self.__checkasync(ret, cmd)
if r:
return r
e = "CHECKASYNC: '%s' timed-out, but no asynchronous output found in:\n" \
"'%s'\nret: '%s'" % (cmd, buf, ret)
logger.error(e)
return ReturnCode(False, e)
def check_margins(curr, prev, delta):
ret = ReturnCode(True, 'All parameters are under the expected error margin of %s' % delta)
for value in ['iops', 'bw', 'lat']:
for stat in ['deviation', 'average', 'median']:
logger.info('value: %s stat: %s', value, stat)
currstat = curr[value][stat]
prevstat = prev[value][stat]
newdelta = (abs(currstat - prevstat) * 100) / prevstat
logger.info('current: %f previous: %f delta: %f', currstat, prevstat, newdelta)
if newdelta > delta:
ret.status = False
ret.message = 'Margin on %s: current: %f previous: %f outside of margin: %f' % \
(value, currstat, prevstat, delta)
logger.info(ret.message)
return ret
def is_failed(sr, lun):
result = ReturnCode(False)
l = sr.list.get(lun)
if l:
if sr.version >= 7:
if l['state'].find('failed') != -1:
result.status = True
result.message = l['state']
else:
for component in l.get('raids'):
if component['state'].find('failed') != -1:
result.status = True
result.message = component['state']
else:
raise ApplianceError('trying to query status of a non-existing lun: %s' % lun)
return result
def rdunfail(sr, disk):
"""
unset the rdfail flag on a drive
:param sr: an srx object
:param disk: drive on which to set the rdfail flag, we test the flag to verify it is on, before setting it
Return: a ReturnCode True if flag was set to off False if failed (drive was already off or missing.
"""
result = ReturnCode(False)
d = sr.disks.get(disk)
if d and d[
'rdfail'] == 'on': # Let's test for the result of sr.disks.get, just in case that disk is missing.
cmd = "echo rdfail off > /raiddev/%s/ctl" % disk
try:
if sr.version >= 7:
result = sr.expert_run(cmd)
else:
result = sr.run_and_check(cmd)
except ApplianceError, e:
logger.error('A problem was found trying to rdunfail disk %s: %s',
disk, e)
def cycle(self, outlet, expectation=True):
"""
Power cycle a particular port in the PDU specified by the outlet number.
"""
outlet_state = self.state
result = ReturnCode(True)
outlet = str(outlet)
logger.info("cycle %s" % outlet)
self.send("1")
self.expect("Control Sub Menu")
self.send("1")
self.expect("Outlet State Sub Menu")
if int(outlet) in range(1, 9):
self.send("1")
elif int(outlet) in range(9, 17):
self.send("2")
elif int(outlet) in range(17, 25):
self.send("3")
self.expect("Outlet Control Sub Menu")
self.send(outlet)
self.expect("%s Command Choices" % outlet_state[outlet]['name'])
self.send("3")
self.expect("%s Requested Command is Reboot" %
outlet_state[outlet]['name'])
self.send("\r")
self.expect("Outlet State Sub Menu")
if not expectation:
return result
elif re.search("error|fail", self.before, re.MULTILINE | re.I):
raise ApplianceError("pdu cycle failed")
# get back to main menu
for i in range(0, 2):
self.send('\x1b')
self.expect("Select Item Number")
def _zpool_create(self, pname, targets, ptype='', spares=None):
"""
This gets called from abstraction zpool_create and does zpool creation.
"""
if type(targets) == list:
devices = [self._target2device(target) for target in targets]
spare_devices = []
if spares:
spare_devices = [
self._target2device(spare) for spare in spares
]
if len(devices) > 0:
cmd = 'zpool create -f %s %s' % (pname, ptype)
for d in devices:
if d:
cmd += ' %s' % d
if spares:
cmd += ' spare'
for s in spare_devices:
if s:
cmd += ' %s' % s
return self.run_and_check(cmd, False)
# it could be a question being asked during the creation.
else:
return ReturnCode(False, 'Devices not found at initiator')
elif type(targets) == str:
device = self._target2device(targets)
spare_devices = []
if spares:
spare_devices = [
self._target2device(spare) for spare in spares
]
if device:
cmd = 'zpool create -f %s %s %s' % (pname, ptype, device)
if spares:
cmd += ' spare'
for s in spare_devices:
if s:
cmd += ' %s' % s
return self.run_and_check(
cmd, False) # FIXME, same as above run command.
else:
return ReturnCode(False,
'Device for target %s not found' % targets)
def loadaoe(self): # STUB
"""
Loads the AoE driver module defined in self.coraidmodule:
either the HBA driver 'ethdrv', or the software initiator 'aoe'.
"""
ret = ReturnCode(False)
return ret
def unloadaoe(self): # STUB
"""
Unload the AoE driver module defined in self.coraid_module
Returns a ReturnCode object
"""
ret = ReturnCode(False)
return ret
def claim(self, lun): # STUB
"""
claim a LUN
accept string or AoEAddress type
Return : ReturnCode
"""
ret = ReturnCode(False)
return ret
def aoerevalidate(self, shelf_lun): # STUB
"""
Calls aoe-revalidate e{shelf}.{lun} on initiator
Accept a string e.g. 'e4.1' or AoEAddress
Returns ReturnCode object
"""
ret = ReturnCode(False)
return ret
def pool_is_empty(vsx, pool):
"""
Verifypool makes sure that the pool has no extents allocated.
"""
if not isinstance(vsx, Vsx):
e = ReturnCode(False)
e.message = "object is not a Vsx instance"
return e
total = 0
free = 0
unique = 0
shelf = vsx.shelf
ret = vsx.run_and_check("pools -a %s" % pool) # TODO: teach this that pools has -a
if not ret.status:
return ret
p = ret.message
ret = ReturnCode(True)
m = re.search(r"Total[ \t]+Exts[ \t]+:[ \t]+([0-9]+)[ \t]+", p)
if m:
total = int(m.group(1))
m = re.search(r"Free[ \t]+Exts[ \t]+:[ \t]+([0-9]+)[ \t]+", p)
if m:
free = int(m.group(1))
m = re.search(r"Unique[ \t]+Exts[ \t]+:[ \t]+([0-9]+)[ \t]+", p)
if m:
unique = int(m.group(1))
if total:
e = "Empty pool %s on %s has %d total extents:\n%s" % (pool, shelf, total, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
if free:
e = "Empty pool %s on %s has %d free extents:\n%s" % (pool, shelf, free, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
if unique:
e = "Empty pool %s on %s has %d unique extents:\n%s" % (pool, shelf, unique, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
m = re.search(r"PVs[ \t]+:[ \t]+[0-9]+\.[0-9]+", p)
if m:
e = "Empty pool %s on %s has PVs:\n%s" % (pool, shelf, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
m = re.search(r"LVs[ \t]+:[ \t]+(.*)[ \t]+", p)
if m:
e = "Empty pool %s on %s has LVs:\n%s" % (pool, shelf, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
return ret
def pv_is_mirrored(vsx, pv):
"""
Check whether or not a pv is mirrored and
is done silvering.
"""
if not isinstance(vsx, Vsx):
e = ReturnCode(False)
e.message = "Object is not a Vsx instance"
return e
pv = vsx.pvs.get(pv)
if not pv:
e = ReturnCode(False)
e.message = "pv not found"
return e
if pv.get('stat') != 'mirrored':
e = ReturnCode(False)
e.message = "pv not mirrored"
return e
pool = pv.get('pool')
pv = pv.get('pv')
tprompt = vsx.prompt
vsx.prompt = 'VSX EXPERTMODE# '
vsx.run_and_check('/expertmode')
cmd = "ls /n/xlate/pool/%s/pv/" % pool
ls = vsx.run_and_check(cmd)
if not ls:
return ls
ret = ReturnCode(False)
files = ls.message.split('\r\n')
for fname in files:
status = vsx.run_and_check("cat %s/status" % fname)
fields = status.message.split()
if fields[0] == "single" and fields[5] == pv:
ret = ReturnCode(False, fields[0])
break
elif fields[0] == "mirrored" and fields[5] == pv:
ret = ReturnCode(True, fields[0])
break
else:
ret.message = str(fields)
vsx.prompt = tprompt
vsx.run("exit")
return ret
def target_is_available(self, target):
"""
Check if a target is visible
"""
if self.coraid_module == "aoe":
self.aoediscover()
stat = self.aoestat
else:
stat = self._uniqscsi(target)
ret = ReturnCode(False)
for targ in stat:
if stat[targ]["target"] == target:
ret.message = stat[targ]
ret.status = True
return ret
def rdfail_preboot(sr, lun, disk=None):
"""
**WARNING: usage of rdfail can cause an un-responsive appliance.**
Set the rdfail flag on a drive. Values stored inside /rc/bin/srlocalpreboot,
to prevent shield errors during boot.
:param lun: the lun id we want to affect, we will select an element from that lun and fail it
:param disk: if we are looking to fail a particular element inside the lun
:return: a ReturnCode object, status of True if successful with message indicating which device
in the appliance was affected otherwise a False value will be returned.
"""
result = ReturnCode(False)
# The following command will allow us to modify content inside files in the SRX
if not _kfscmd(sr, 'allow'):
return result
# Lets get the info from the lun we want to fail a disk
l = sr.list.get(lun)
for c in l['raids'][0]['components']:
regExp = re.search(r'd+.(d+)', c['device'])
if regExp and (regExp.group(1) == disk or disk is None):
disk2fail = regExp.group(1)
if sr.version >= 7:
cmd = "echo \'echo rdfail on > /n/raiddev/%s/ctl\' > /n/kfs/srx/srlocal0" % disk2fail
else:
cmd = "echo \'echo rdfail on > /raiddev/%s/ctl\' > /rc/bin/srlocalpreboot" % disk2fail
try:
result = sr.run_and_check(cmd)
result.message = '/raiddev/%s/ctl' % disk2fail
except ApplianceError as e:
result.messsage = e
break
return result
def rdfail_preboot(sr, lun, disk=None):
"""
**WARNING: usage of rdfail can cause an un-responsive appliance.**
Set the rdfail flag on a drive. Values stored inside /rc/bin/srlocalpreboot,
to prevent shield errors during boot.
:param lun: the lun id we want to affect, we will select an element from that lun and fail it
:param disk: if we are looking to fail a particular element inside the lun
:return: a ReturnCode object, status of True if successful with message indicating which device
in the appliance was affected otherwise a False value will be returned.
"""
result = ReturnCode(False)
# The following command will allow us to modify content inside files in the SRX
if not _kfscmd(sr, 'allow'):
return result
# Lets get the info from the lun we want to fail a disk
l = sr.list.get(lun)
for c in l['raids'][0]['components']:
regExp = re.search(r'd+.(d+)', c['device'])
if regExp and (regExp.group(1) == disk or disk is None):
disk2fail = regExp.group(1)
if sr.version >= 7:
cmd = "echo \'echo rdfail on > /n/raiddev/%s/ctl\' > /n/kfs/srx/srlocal0" % disk2fail
else:
cmd = "echo \'echo rdfail on > /raiddev/%s/ctl\' > /rc/bin/srlocalpreboot" % disk2fail
try:
result = sr.run_and_check(cmd)
result.message = '/raiddev/%s/ctl' % disk2fail
except ApplianceError as e:
result.messsage = e
break
return result
def claim_targets(self, target_list):
"""
Claim the luns
"""
lun_list = target_list.split(' ')
result = ReturnCode(False, message="empty target list")
for lun in lun_list:
cmd = mkcmdstr('esxcli ethdrv claim -t', lun)
logger.info(lun)
result = self.run_and_check(cmd)
return result
def aoediscover(self):
"""
Call the driver's discover command. Returns ReturnCode object
"""
cmd = "%s discover" % ethdrvadm
output = self.run(cmd)
if output.find("error") > -1: # I don't think this will ever happen
status = False
else:
status = True
return ReturnCode(status=status, message=output)
def wrapper(*args, **kwargs):
result = ReturnCode(not case)
if timeout is None:
while bool(result) != bool(case):
result = function(*args, **kwargs)
if bool(result) != case: # no need to sleep if case is met
sleep(wrapper.sleeptime)
else:
starttime = now()
while now() - starttime < float(wrapper.timeout):
result = function(*args, **kwargs)
if bool(result) == case:
break
sleep(wrapper.sleeptime)
if bool(result) != case:
result = ReturnCode(False)
result.message = "Timed out : {0} seconds".format(timeout)
return result
def run(self, cmd, timeout=None, bufsize=-1):
"""
:param cmd: command to run on remote host
:type cmd: str
:param timeout: timeout on blocking read/write operations when exceeded socket error will be raised
:type timeout: float
:param bufsize: byte size of the buffer for the filehandle returned
:type bufsize: int
:rtype: ReturnCode
"""
ret = ReturnCode(False)
if not self.connected:
raise ConnectionError("Run was called on an unconnected host. Did you check the result of connect()?")
try:
if self.environmentals:
envstring = str()
for var, value in self.environmentals.items():
statement = "%s=%s " % (var, value)
envstring += statement
cmd = "%s%s" % (envstring, cmd)
if self.cwd:
cmd = "cd %s && %s" % (self.cwd, cmd)
self._log(logging.DEBUG, 'running command: "%s"' % cmd)
stdin, stdout, stderr = self.exec_command(command=cmd, timeout=timeout, bufsize=bufsize)
except paramiko.SSHException as e:
err = "Couldn't complete the command: %s" % str(e)
logger.critical(err)
ret.message = err
return ret
# we must read stderr _before_ stdout
# otherwise paramiko losses the stdout data
try:
ret.raw = Data(ret.raw.status, ret.raw.stdout, stderr.read())
except socket.timeout:
ret.message = "Timeout"
return ret
status = stdout.channel.recv_exit_status()
ret.raw = Data(status, stdout.read(), ret.raw.stderr)
if status != 0:
ret.message = ret.raw.stderr
else:
ret.status = True
ret.message = ret.raw.stdout
stdin.close()
return ret
def run_and_check(self, cmd, expectation=True):
"""
Run a command check the result. If the caller cares about failure
and the command fails we raise a generic exception.
"""
result = ReturnCode(True)
logger.info(cmd + " called")
result.message = self.run(cmd)
if result.message.find('Invalid command or parameters.') > -1:
# this will never work since the error is after the
# prompt added except KeyError: to catch problems
# below I suspect "expectation" might have a problem too.
logger.error(result.message)
result.status = False
if not expectation:
return result
elif not result.status:
raise ApplianceError(cmd + " failed")
return result
def run_and_check(self, cmd, expectation=True):
"""
Run a command check the result. If the caller cares about failure
and the command fails we raise a generic exception.
"""
result = ReturnCode(True)
logger.info(cmd + " called")
result.message = self.run(cmd)
if result.message.find('Invalid command or parameters.') > -1:
# this will never work since the error is after the
# prompt added except KeyError: to catch problems
# below I suspect "expectation" might have a problem too.
logger.error(result.message)
result.status = False
if not expectation:
return result
elif not result.status:
raise ApplianceError(cmd + " failed")
return result
def run_and_check(self, cmd, expectation=True):
"""
Run a command check the result. If the caller cares about failure
and the command fails we raise a generic exception.
"""
result = ReturnCode(True)
logger.info(cmd + " called")
self.pdu.connect()
result.message = self.pdu.run(cmd)
self.pdu.disconnect()
e = Exception()
if not result.message.find('E000: Success'):
if result.message.find('E101: Command Not Found') > -1:
logger.error(result.message)
result.status = False
failmsg = cmd + " failed"
e = ApplianceError(failmsg)
elif result.message.startswith('E102: Parameter Error'):
logger.critical(result.message)
result.status = False
failmsg = cmd + " failed"
e = ApplianceUsage(failmsg)
result.status = False
if not expectation:
return result
elif not result.status:
raise e
return result
def slot_is_available(sr, slot, cache=False):
"""
check slots' role as part of a lun, or if is marked as an spare or cache disk or if it missing.
:param slot: the slot we want to check, it's a str or int in the form: '3', '12' or '34'.
:param cache: is it ok to assume that sr.disks is unchanged while we are in this function?
:return: a ReturnCode object, message contains the output of disks command for the specified slot
"""
# !!this is a surprisingly complicated function!! You have been warned.
if sr.use_slots and slot not in sr.use_slots:
return ReturnCode(status=False, message="masked by use_slots")
t = sr.use_slots
sr.use_slots = [slot]
diskcache = sr.cache.get('disks')
if not ((cache and diskcache) and diskcache.get(slot)):
disk = getattr(sr, "s%s" % slot)
if not disk.get('model'):
sr.use_slots = t
return ReturnCode(status=False, message=str(disk))
else:
disk = diskcache.get(slot)
sr.use_slots = t
if disk['role'] in ['', None]:
if disk['size'] != 'missing':
status = True
else:
status = False
else:
status = False
return ReturnCode(status=status, message=disk)
def cmp_aoestat_targets(a, t):
# Confirm aoestats.paths in targets.ea
for l in a.port:
for m in a.paths[l].address:
found = False
for n in t:
mask = bin(n.ports)[2:][::-1]
if a.paths[l].port < len(mask) and mask[
a.paths[l].port] == '1' and m == n.ea:
found = True
if not found:
return ReturnCode(
False, 'aoestat %s does not match targets %s' % (a, t))
# Confirm targets.ea in aoestats.paths
for l in t:
mask = bin(l.ports)[2:][::-1]
for m in range(len(mask)):
if mask[m] == '1':
if l.ea not in a.paths[m].address:
return ReturnCode(
False, 'targets %s does not match aoestat %s' % (t, a))
return ReturnCode(True)
def fio(init1, args):
"""
Pull out fio configuration options from fc and execute
"""
init1.run_and_check('set +m; rm -f *out *err')
cmd = 'fio --output-format=json %s> out 2> err &' % args
logger.info(cmd)
init1.run(cmd, wait=False)
for i in range(10):
n = nofiorunning(init1)
if not n:
return ReturnCode(True)
time.sleep(1)
return fioresult(init1)
def is_recovering(sr, lun):
"""
Is this lun recovering?
:param sr: an srx object
:param lun: a lun number as str
:return: returnCode with lun state as message
"""
result = ReturnCode(False)
l = sr.list.get(lun)
if l:
if sr.version >= 7:
if l['state'].find('recovering') != -1:
result.status = True
result.message = l['state']
else:
for component in l.get('raids'):
if component['state'].find('recovering') != -1:
result.status = True
result.message = component['state']
else:
raise ApplianceError('trying to query status of a non-existing lun: %s' % lun)
return result
def rdfail(sr, lun, disk=None):
"""
**NOTE: the usage of rdfail can produce an un-responsive appliance.**
set the rdfail flag of a drive to 'on'
:param lun: the lun id we want to affect, we will select an element from that lun and fail it
:param disk: if passed, the drive we want to set the rdfail flag.
:return: a return code of True if the flag was reset back to on; False otherwise.
"""
result = ReturnCode(False)
# Lets get the info from the lun we want to fail a disk
l = sr.list.get(lun)
if not l:
result.message = "Seems like lun %s does not exist" % lun
return result
for c in l['raids'][0]['components']:
regExp = re.search(r'd+.(d+)', c['device'])
if regExp and (regExp.group(1) == disk or disk is None):
disk2fail = regExp.group(1)
cmd = "echo rdfail on > /raiddev/%s/ctl" % disk2fail
if sr.version >= 7:
result = sr.expert_run(cmd)
else:
result = sr.run_and_check(cmd)
result.message = '/raiddev/%s/ctl' % disk2fail
return result
return result
def pv_is_empty(vsx, pv=None): # TODO: use pv to filter results
"""
Check if most of the metadata is empty and that extents are
correctly accounted for. pv parameter is ignored for now
"""
if not isinstance(vsx, Vsx):
e = ReturnCode(False)
e.message = "object is not a Vsx instance"
return e
total = 0
free = 0
dirty = 0
meta = 0
sh = vsx.shelf
ret = vsx.pvs
if not ret.status:
return ret
p = ret.message
ret = ReturnCode(True)
used = calculate_metaext(total, 4096)
if used != dirty:
e = "pv has too many dirty extents, should be %d on VSX %s:\n%s" % (used, sh, p)
logger.error(e)
ret.status = False
ret.message = e
if meta:
e = "meta extents not zero on VSX %s:\n%s" % (sh, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
if total != free + dirty:
e = "pv accounting error on VSX %s:\n%s" % (sh, p)
logger.error(e)
ret.status = False
ret.message += "\n%s" % e
return ret
def run_and_check(self, cmd, expectation=True, force=False):
"""
Run a command check the result. If the caller cares about failure
and the command fails we raise a generic exception.
"""
result = ReturnCode(True)
confirm = re.compile("Enter[ \t]*\'y\'*.")
logger.info(cmd + " called")
if force:
t = self.prompt
self.prompt = confirm
self.run(cmd)
self.prompt = t
result.message = self.run('y')
else:
result.message = self.run(cmd)
e = Exception()
if result.message.startswith('Error:'):
logger.error(result.message)
result.status = False
failmsg = cmd + " failed: " + result.message
e = ApplianceError(failmsg)
elif result.message.startswith('Usage:'):
logger.critical(result.message)
result.status = False
failmsg = cmd + " failed: " + result.message
e = ApplianceUsage(failmsg)
result.status = False
if not expectation:
return result
elif not result.status:
raise e
# logger.info( result)
return result
def is_inited(sr, lun):
"""
check if a lun's parity is built
:param sr: an srx object
:param lun: a lun number as int or str
:return: return code with number of seconds until done in message field
"""
result = ReturnCode(False)
lun = str(lun)
l = sr.list.get(lun)
if not l:
raise ApplianceUsage("lun not found '%s'" % lun)
if l.get('state') == 'initing':
t = sr.when.get(lun)
if t:
t = t['time'] # mmmmm
result = ReturnCode(False, get_sec(t))
if l.get('state') == 'normal':
result.status = True
for component in l.get('raids'): # SRX 6
if component.get('state') == 'initing':
result.status = False
lc = "%s.%s" % (lun, component['number'])
wt = sr.when.get(lc)
if wt:
t = wt['time'] or '0:0:0'
else:
t = '0:0:0'
result.message = get_sec(t)
break
elif component.get('state') == 'normal':
result.status = True
return result
def wait_something_timeout_param(para):
ret = ReturnCode(False)
ret.message = "looking for timeout param passed to inner func = %s" % para
print ("waiting for timeout")
return ret
def make_lun(sr, luns, fcenable=False, cache=False):
"""
Creates one or a series of luns based on a dictionary containing their details.
:param luns: a dictionary of lun descriptions are created based on a dictionary
:param fcenable: enable flashcache on the lun after making it
:param cache: sent to slot_is_available to mean it is ok to only read the disks command once
and assume they didn't change in this function. This rate limits log messages and
is much faster, especially with cec.
example::
{
'0':{'num_disks':2,'type':'raid1','size':'-c','version':'1','iomode':'random'},
'1':{'num_disks':3,'type':'raid5','size':'2G','version':'0','clean':'False'},
'2':{'num_disks':4,'type':'raid6rs','size':'-c','version':'0','iomode':'sequential'},
'3':{'num_disks':4,'type':'raid10','size':'10G','version':'1'},
'4':{'num_disks':2,'type':'raid0','size':'-c','version':'1'},
'5':{'num_disks':1,'type':'jbod','size':'4G','version':'1'}
}
The dictionary should contain the details of the luns to create and the following categories are mandatory:
- num_disks: how many disks in the lun
- type: raid type, or jbod or raw are also valid
- version: either 0 or 1 are valid
The following categories are optional in the dictionary:
- iomode: the default value is based on the lun type
- clean : if you want the lun to avoid parity initialization, default is to do parity initialization
- size: the usable size of the disk using the setsize command; if not specified, the current size,
'setesiz'd or not, of the disk will be used;
-c is valid and will indicate restore the disk to its actual size.
"""
# pylint: disable=R0912
result = ReturnCode(False)
all_nil = False not in [lun.get('type') in ['nil', 'fnil'] for lun in luns.values()]
if all_nil:
for lunid, cfg in luns.items:
result = sr.make(lunid, cfg.get('type'))
if not result:
break
else:
result = sr.online(lunid)
return result
else:
disks = ["%s" % x for x in range(sr.slots)]
# Lets get a list of drives available inside the shelf.
available_disks = ["%s.%s" % (sr.shelf, x) for x in disks if slot_is_available(sr, x, cache=cache)]
for lun in luns.keys():
l = luns[lun] # Creating a reference to make the code more readable
# Check if we have enough disk to create the lun
if len(available_disks) >= l['num_disks']:
if 'size' in l: # if size is provided, each disk on that lun will be resized to that size
for d in available_disks[:l['num_disks']]:
if l['size']:
sr.setsize(l['size'], d)
else:
sr.setsize('-c', d)
# Should we skip or not the lun initialization
if 'clean' in l:
skipInit = l['clean']
else:
skipInit = False
sr.make(lun, l['type'], available_disks[:l['num_disks']], l['version'], skipInit)
# if iomode has been specified also, we set the mode next
if 'iomode' in l:
sr.setiomode(l['iomode'], lun)
sr.online(lun)
if fcenable is True:
sr.fcenable(lun)
# Remove used disks from the available list
available_disks = available_disks[l['num_disks']:]
else:
result.message = 'Not enough disks to complete creation of lun: %s' % lun
result.status = False
return result
result.status = True
return result
def wait_something():
ret = ReturnCode(choice([True, False, False, False]))
ret.message = "%s: looking for True" % ret.status
return ret
def wait_something_sleeptime_param(para):
ret = ReturnCode(True)
ret.message = para
return ret
def wait_something_timeout():
ret = ReturnCode(False)
ret.message = "set to False"
print ("sleeping")
return ret
def wait_something_case():
ret = ReturnCode(choice([True, True, True, False]))
ret.message = "looking for False"
print ("sleeping")
return ret
