def buildNodesFromTestbedFile(testbed):
Logger.debug("building nodes from testbed file {0}".format(testbed))
nodes = []
try:
with open(testbed, "r") as topoFile:
data = json.load(topoFile)
if 'Vars' in data['Provision'] and getattr(
data['Provision']['Vars'], "EsxUsername", None):
username = data['Provision']['Vars']['EsxUsername']
password = data['Provision']['Vars']['EsxPassword']
else:
username = data['Provision']['Username']
password = data['Provision']['Password']
for inst in data["Instances"]:
if inst.get(
"Type", ""
) == "bm" and "NodeMgmtIP" in inst and "Name" in inst:
node = CollectLogNode(inst["Name"], inst["NodeMgmtIP"],
username, password)
node.cimcIp = inst.get("NodeCimcIP", None)
node.cimcUsername = inst.get("NodeCimcUsername", None)
node.cimcPassword = inst.get("NodeCimcPassword", None)
nodes.append(node)
except:
msg = "failed to build nodes from testbed file. error was:"
msg += traceback.format_exc()
print(msg)
Logger.debug(msg)
return nodes
def __load_bundle(self):
pkg_base = self.__parent.GetPackages()[0].replace(".", "/")
fullpath = "%s/%s/testbundles/%s" % (api.GetTopDir(), pkg_base,
self.__bunfile)
Logger.debug("Importing Testbundle %s" % fullpath)
self.__spec = self.__read_spec(fullpath)
return
def __resolve_teardown(self):
teardown_spec = getattr(self.__spec, 'teardown', [])
if teardown_spec is None:
return types.status.SUCCESS
for s in self.__spec.teardown:
Logger.debug("Resolving teardown module: %s" % s.step)
s.step = loader.Import(s.step, self.__spec.packages)
return types.status.SUCCESS
def __discover_testsuites():
suites = []
expr = GlobalOptions.topdir + '/iota/test/%s/**/*.testsuite' %\
(GlobalOptions.pipeline)
for filename in glob.iglob(expr, recursive=True):
Logger.debug("Reading testsuite: %s" % filename)
data = parser.YmlParse(filename)
suites.append(data)
return suites
def Main(self):
if GlobalOptions.testcases and self.Name(
) not in GlobalOptions.testcases:
Logger.info("Skipping Testcase: %s due to cmdline filter." %
self.Name())
self.__enable = False
return types.status.SUCCESS
if GlobalOptions.markers_present:
if self.Name() == GlobalOptions.testcase_begin:
Logger.debug("Match found for Testcase starting marker %s" %
self.Name())
GlobalOptions.inb_markers = True
if GlobalOptions.markers_present and not GlobalOptions.inb_markers:
Logger.info(
"Skipping Testcase: %s due to cmdline testcase begin/end markers."
% self.Name())
self.__enable = False
return types.status.SUCCESS
if self.__enable:
Logger.SetTestcase(self.Name())
Logger.info("Starting Testcase: %s" % self.Name())
Logger.info("Testcase {0} timestamp: {1}".format(
self.Name(), time.asctime()))
self.__timer.Start()
try:
self.status = self.__execute()
except OfflineTestbedException:
utils.LogException(Logger)
Logger.error(
"EXCEPTION: Aborting Testcase Execution. Reason: testbed failure"
)
raise
except:
utils.LogException(Logger)
Logger.error("EXCEPTION: Aborting Testcase Execution.")
self.status = types.status.ERROR
if self.__ignored:
Logger.error(
"IGNORE: testcase in ignore mode, ignoring exception.")
self.status = types.status.SUCCESS
self.__timer.Stop()
Logger.info("Testcase %s FINAL RESULT = %d" %
(self.Name(), self.status))
else:
self.status = types.status.SUCCESS
if GlobalOptions.markers_present:
if self.Name() == GlobalOptions.testcase_end:
Logger.debug("Match found for Testcase ending marker %s" %
self.Name())
GlobalOptions.inb_markers = False
return self.status
def CollectCores(self):
try:
destCoreDir = "corefiles/{0}".format(re.sub('[\W]+','_',self.Name()))
print("Searching for corefiles.....")
core_collector.CollectCores(GlobalOptions.testbed_json, destCoreDir,
store.GetTestbed().GetProvisionUsername(),
store.GetTestbed().GetProvisionPassword(), Logger)
except:
Logger.debug("failed to collect cores. error was {0}".format(traceback.format_exc()))
def writeTestResults(self):
filename = "testsuite_{0}_results.json".format(self.Name())
try:
tsr = TestSuiteResults()
for tbun in self.__testbundles:
if tbun.getTestBundleResults():
tsr.Testcases.extend(tbun.getTestBundleResults())
with open(filename,'w') as outfile:
json.dump(tsr, outfile, default=lambda x: x.__dict__, sort_keys=True)
except:
Logger.debug("failed to save test results to file {0}. error was: {1}".format(filename,traceback.format_exc()))
def __exit_cleanup():
global gl_srv_process
if not engine.SkipSetupValid():
Logger.debug("ATEXIT: Stopping IOTA Server as setup was not complete")
gl_srv_process.Stop()
if glopts.GlobalOptions.dryrun or glopts.GlobalOptions.skip_logs:
return
packageCoresDirectory()
Logger.info("Saving logs to iota_sanity_logs.tar.gz")
os.system("%s/iota/scripts/savelogs.sh %s" % (topdir, topdir))
return
def GetPipelineFromProvisionInfo(self, nodeName, nicName):
try:
prov_spec = parser.YmlParse(self.ProvisionInfo())
nodes = getattr(prov_spec, 'nodes', [])
for node in nodes:
if node.node.name == nodeName:
for nic in node.node.nics:
if nic.nic.name == nicName:
return nic.nic.pipeline
except:
Logger.debug("failed to find pipeline for {0}:{1}".format(nodeName, nicName))
def __resolve_setup_config(self):
cfgspec = getattr(self.__spec.setup, 'config', None)
if not cfgspec:
return types.status.SUCCESS
for s in self.__spec.setup.config:
Logger.debug("Resolving config step: %s" % s.step)
s.name = s.step
args = getattr(s, "args", None)
s.step = loader.Import(s.step, self.__spec.packages)
s.args = getattr(s, "args", None)
return types.status.SUCCESS
def Init():
server = 'localhost:' + str(GlobalOptions.svcport)
Logger.info("Creating GRPC Channel to IOTA Service %s" % server)
IotaSvcChannel = grpc.insecure_channel(server)
Logger.debug("Waiting for IOTA Service to be UP")
grpc.channel_ready_future(IotaSvcChannel).result()
Logger.info("Connected to IOTA Service")
global TopoSvcStub
TopoSvcStub = topo_svc.TopologyApiStub(IotaSvcChannel)
return
def __resolve(self):
Logger.debug("Resolving testcase teardown module: %s" %
self.__spec.step)
self.__mod = loader.Import(self.__spec.step, self.__spec.packages)
if hasattr(self.__spec, 'args') and hasattr(self.__spec.args,
'trigger'):
self.__trigger = self.__spec.args.trigger
if hasattr(self.__spec, 'args') and hasattr(self.__spec.args,
'background'):
self.__background = self.__spec.args.background
return
def GetImageManifestFile(self):
if GlobalOptions.compat_test:
Logger.debug("Compat-testing for driver: %s and firmware: %s" % (GlobalOptions.driver_version, GlobalOptions.fw_version))
# assert(GlobalOptions.driver_version != GlobalOptions.fw_version)
return self.__build_new_image_manifest()
else:
Logger.debug("Using testsuite spec for image-manifest")
if hasattr(self.__spec, 'image_manifest'):
path = getattr(self.__spec.image_manifest, 'file', 'images/latest.json')
else:
path = 'images/latest.json'
return os.path.join(GlobalOptions.topdir, path)
def TriggerBackgroundTasks(self, tasks, iter_data):
result = types.status.SUCCESS
for task in tasks:
bt = self.__background_tasks[task_name]
assert (bt)
Logger.debug("Triggering BackgroundTask %s - manual trigger" %
task_name)
bt_trigger_result = bt.StartTask(self.__tc, iter_data)
if bt_trigger_result != types.status.SUCCESS:
result = bt_trigger_result
return result
def StopBackgroundTasks(self, tasks):
result = types.status.SUCCESS
for task in tasks:
bt = self.__background_tasks[task_name]
assert (bt)
Logger.debug("Triggering BackgroundTask %s - manual trigger" %
task_name)
bt_trigger_result = bt.CollectTask()
if bt_trigger_result != types.status.SUCCESS:
result = bt_trigger_result
return result
def __execute(self, iter_data):
if self.__background:
Logger.debug("Running common teardown module in background: %s" %
self.__spec.step)
bt_inst = TestcaseBackgroundTrigger('Main', self.__trigger,
self.__run,
self.__sleep_interval,
self.__terminate)
return bt_inst.StartTask(self.__mod, iter_data)
else:
Logger.debug("Running common teardown module in foreground: %s" %
self.__spec.step)
return loader.RunCallback(self.__mod, 'Main', True, iter_data)
def __start_server():
Logger.debug("setting default SIGINT handler")
signal.signal(signal.SIGINT, signal.default_int_handler)
global gl_srv_process
srv_binary = "VENICE_DEV=1 nohup %s/iota/bin/server/iota_server" % topdir
srv_logfile = "%s/server.log" % glopts.GlobalOptions.logdir
srv_args = "--port %d" % glopts.GlobalOptions.svcport
if glopts.GlobalOptions.dryrun:
srv_args += " --stubmode"
gl_srv_process = procs.IotaProcess("%s %s" % (srv_binary, srv_args),
srv_logfile)
gl_srv_process.Start()
return
def CollectTechSupport(tsName):
try:
#global __CURREN_TECHSUPPORT_CNT
#__CURREN_TECHSUPPORT_CNT = __CURREN_TECHSUPPORT_CNT + 1
if GlobalOptions.pipeline in ["apulu"]:
return types.status.SUCCESS
Logger.info("Collecting techsupport for testsuite {0}".format(tsName))
tsName = re.sub('\W', '_', tsName)
logDir = GlobalOptions.logdir
if not logDir.endswith('/'):
logDir += '/'
logDir += 'techsupport/'
if not os.path.exists(logDir):
os.mkdir(logDir)
nodes = api.GetNaplesHostnames()
req = api.Trigger_CreateExecuteCommandsRequest()
for n in nodes:
Logger.info("Techsupport for node: %s" % n)
common.AddPenctlCommand(
req, n, "system tech-support -b %s-tech-support" % (n))
resp = api.Trigger(req)
result = types.status.SUCCESS
for n, cmd in zip(nodes, resp.commands):
#api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
Logger.error(
"Failed to execute penctl system tech-support on node: %s. err: %d"
% (n, cmd.exit_code))
result = types.status.FAILURE
continue
# Copy tech support tar out
# TAR files are created at: pensando/iota/entities/node1_host/<test_case>
ntsn = "%s-tech-support.tar.gz" % (n)
resp = api.CopyFromHost(n, [ntsn], logDir)
if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK:
Logger.error(
"Failed to copy techsupport file %s from node: %s" %
(ntsn, n))
result = types.status.FAILURE
continue
os.rename(logDir + ntsn, logDir + tsName + '_' + ntsn)
#if __CURREN_TECHSUPPORT_CNT > __MAX_TECHSUPPORT_PER_RUN:
# return types.status.CRITICAL
return result
except AttributeError:
Logger.debug('failed to collect tech support. node list not setup yet')
except:
Logger.debug('failed to collect tech support. error was: {0}'.format(
traceback.format_exc()))
return types.status.CRITICAL
def Main(self):
if self.__skip:
Logger.debug("Skipping Testsuite: %s due to filters." % self.Name())
return types.status.SUCCESS
atexit.register(self.ExitHandler)
# Start the testsuite timer
self.__timer.Start()
# Update logger
Logger.SetTestsuite(self.Name())
Logger.info("Starting Testsuite: %s" % self.Name())
Logger.info("Testsuite {0} timestamp: {1}".format(self.Name(), time.asctime()))
if self.GetFirmwareType() == types.firmware.GOLD:
Logger.debug("setting global firmware type to gold")
GlobalOptions.use_gold_firmware = True
# Initialize Testbed for this testsuite
status = store.GetTestbed().InitForTestsuite(self)
if status != types.status.SUCCESS:
self.__timer.Stop()
return status
status = self.__parse_setup()
if status != types.status.SUCCESS:
self.__timer.Stop()
return status
self.__import_testbundles()
self.__resolve_teardown()
self.__expand_iterators()
self.__resolve_calls()
status = self.__setup()
if status != types.status.SUCCESS:
self.__timer.Stop()
return status
status = self.checkPci()
if status != types.status.SUCCESS:
self.__timer.Stop()
return status
self.result = self.__execute_testbundles()
self.__update_stats()
Logger.info("Testsuite %s FINAL STATUS = %d" % (self.Name(), self.result))
self.__timer.Stop()
return self.result
def checkPci(self):
result = types.status.SUCCESS
if GlobalOptions.dryrun:
return result
for node in self.GetTopology().GetNodes():
msg = "calling verify_pci.verify_error_lspci() for node {0}".format(node.Name())
Logger.debug(msg)
result = verify_pci.verify_errors_lspci(node.Name(), node.GetOs())
if result != types.status.SUCCESS:
msg = "PCIe Failure detected on node {0} with OS {1}".format(node.Name(), node.GetOs())
print(msg)
Logger.error(msg)
return result
# raise OfflineTestbedException
return result
def __init_testbed(self):
self.__tbid = getattr(self.__tbspec, 'TestbedID', 1)
self.__vlan_base = getattr(self.__tbspec, 'TestbedVlanBase', 1)
self.__vlan_allocator = resmgr.TestbedVlanAllocator(self.__vlan_base, self.curr_ts.GetDefaultNicMode())
#TODO: merge single allocator into list below
self.__multi_vlan_allocators = []
self.__nextVlanAllocator = 0
self.__image_manifest_file = self.curr_ts.GetImageManifestFile()
self.curr_ts.DownloadReleaseImages()
resp = None
msg = self.__prepare_TestBedMsg(self.curr_ts)
if not GlobalOptions.skip_setup:
status = self.CleanupTestbed()
if status != types.status.SUCCESS:
return status
try:
self.__recover_testbed(self.__image_manifest_file)
except:
utils.LogException(Logger)
Logger.error("Failed to recover testbed")
Logger.debug(traceback.format_exc())
return types.status.TESTBED_INIT_FAILURE
if GlobalOptions.dryrun:
status = types.status.SUCCESS
resp = api.InitTestbed(msg)
else:
resp = api.GetTestbed(msg)
if resp is None:
Logger.error("Failed to initialize testbed: ")
raise OfflineTestbedException
#return types.status.FAILURE
if not api.IsApiResponseOk(resp):
Logger.error("Failed to initialize testbed: ")
raise OfflineTestbedException
#return types.status.FAILURE
for instance,node in zip(self.__tbspec.Instances, resp.nodes):
if getattr(instance, 'NodeOs', None) == "esx":
instance.esx_ctrl_vm_ip = node.esx_ctrl_node_ip_address
Logger.info("Testbed allocated vlans {}".format(resp.allocated_vlans))
if resp.allocated_vlans:
tbvlans = []
for vlan in resp.allocated_vlans:
tbvlans.append(vlan)
self.__vlan_allocator = resmgr.TestbedVlanManager(tbvlans)
self.__instpool = copy.deepcopy(self.__tbspec.Instances)
return types.status.SUCCESS
def PrintReport(self):
if self.__skip:
return types.status.SUCCESS
try:
self.GetTopology().PrintTestbedInfo()
except Exception as e:
Logger.debug("Error while collecting testbed info %s", e)
print("\nTestSuite: %s" % self.__spec.meta.name)
print(types.HEADER_SUMMARY)
print(types.FORMAT_TESTCASE_SUMMARY %\
("Testbundle", "Testcase", "Owner", "Result", "Duration"))
print(types.HEADER_SUMMARY)
for tbun in self.__testbundles:
tbun.PrintReport()
print(types.HEADER_SUMMARY)
return
def __checkNodeInitLogs(self, logfile):
checkList = [
"segmentation fault",
]
Logger.debug("checking logfile {0} for errors".format(logfile))
for check in checkList:
try:
subprocess.check_call("grep \"{0}\" {1}".format(check, logfile), shell=True, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
if e.returncode == 1:
pass #nothing found to ok
elif e.returncode == 2:
Logger.warn("failed to find logfile {0}".format(logfile))
else:
Logger.warn("failed to grep logfile {0}. error was: {1}".format(logfile, traceback.format_exc()))
else:
Logger.warn("found string {0} in logfile {1}".format(check, logfile))
def __apply_skip_filters(self):
if GlobalOptions.testsuites and self.Name() not in GlobalOptions.testsuites:
Logger.debug("Skipping Testsuite: %s because of command-line filters." % self.Name())
return True
#if self.GetTestbedType() != types.tbtype.ANY and\
# self.GetTestbedType() != store.GetTestbed().GetTestbedType()\
# and not GlobalOptions.dryrun:
# Logger.debug("Skipping Testsuite: %s due to testbed type mismatch." % self.Name())
# return True
if not store.GetTestbed().IsSimulation() and not store.GetTestbed().GetOs().intersection(self.__get_oss()) and not GlobalOptions.dryrun:
Logger.debug("Skipping Testsuite: %s due to OS mismatch." % self.Name())
return True
enable = getattr(self.__spec.meta, 'enable', True)
return not enable
def __resolve_testcase(self):
Logger.debug("Resolving testcase module: %s" % self.__spec.testcase)
self.__tc = loader.Import(self.__spec.testcase, self.__spec.packages)
if getattr(self.__spec, "selector", None):
self.__sel_module = loader.Import(self.__spec.selector.module,
self.__spec.packages)
self.__sel_module_args = self.__spec.selector.args
else:
self.__sel_module = None
setattr(self.__tc, 'parent', self)
setups_spec = getattr(self.__spec, 'setups', [])
if setups_spec is None:
return types.status.SUCCESS
for v in setups_spec:
v.packages = self.__spec.packages
setup = SetupStep(v)
self.__setups.append(setup)
verifs_spec = getattr(self.__spec, 'verifs', [])
if verifs_spec is None:
return types.status.SUCCESS
for v in verifs_spec:
v.packages = self.__spec.packages
verif = VerifStep(v)
self.__verifs.append(verif)
debugs_spec = getattr(self.__spec, 'debugs', [])
if debugs_spec is None:
return types.status.SUCCESS
for d in debugs_spec:
d.packages = self.__spec.packages
debug = DebugStep(d)
self.__debugs.append(debug)
common_triggers_spec = getattr(self.__spec, 'triggers', {})
if common_triggers_spec is None:
return types.status.SUCCESS
for t in common_triggers_spec:
t.packages = self.__spec.packages
trigger = TriggerStep(t)
self.__triggers.append(trigger)
common_teardowns_spec = getattr(self.__spec, 'teardowns', [])
if common_teardowns_spec is None:
return types.status.SUCCESS
for t in common_teardowns_spec:
t.packages = self.__spec.packages
teardown = TeardownStep(t)
self.__teardowns.append(teardown)
return types.status.SUCCESS
def __mk_testcase_directory(self, newdir):
Logger.debug("Creating Testcase directory: %s" % newdir)
command = "mkdir -p %s && chmod 777 %s" % (newdir, newdir)
req = api.Trigger_CreateAllParallelCommandsRequest()
for nodename in api.GetWorkloadNodeHostnames():
api.Trigger_AddHostCommand(req, nodename, command)
for wl in api.GetWorkloads():
if api.IsWorkloadRunning(wl.workload_name):
api.Trigger_AddCommand(req,
wl.node_name,
wl.workload_name,
command,
timeout=60)
resp = api.Trigger(req)
if not api.Trigger_IsSuccess(resp):
Logger.error("Failed to create destination directory %s" % newdir)
return types.status.FAILURE
return types.status.SUCCESS
def __setup_simple_iters(self, spec):
Logger.debug("Setting up simple iterators.")
min_len = 0
for k, v in spec.__dict__.items():
if isinstance(v, list):
iter_len = len(v)
if min_len == 0 or min_len > iter_len: min_len = iter_len
spec.__dict__[k] = iter(v)
else:
spec.__dict__[k] = None
for inst in range(min_len):
new_data = self.__new_TestcaseData()
for k, v in spec.__dict__.items():
if not v: continue
new_data.iterators.AddKV(k, next(v))
Logger.debug("- Adding K:%s V:" % k, v)
self.__iters.append(new_data)
return
def UpdateNaplesPipelines(self, pipelines=[]):
if GlobalOptions.skip_firmware_upgrade:
Logger.debug("user requested to skip firmware upgrade so skipping naples pipeline install")
return
if not pipelines:
pipelines = self.GetNaplesPipelines()
if not pipelines:
Logger.debug("no pipelines found")
return
nwarmd = "{0}/iota/warmd.json".format(api.GetTopDir())
with open(GlobalOptions.testbed_json, "r") as warmdFile:
updated = False
alreadyDownloaded = []
warmd = json.load(warmdFile)
for pl in pipelines:
if not types.nicModes.valid(pl.mode.upper()):
raise ValueError("nic mode {0} is not valid. must be one of: {1}".format(pl.mode, types.nicModes.str_enums.values()))
if not types.pipelines.valid(pl.pipeline.upper()):
raise ValueError("nic pipeline {0} is not valid. must be one of: {1}".format(pl.pipeline, types.pipelines.str_enums.values()))
if pl.nicNumber < 1:
raise ValueError("nic number must be >= 1. value from testsuite files was {0}".format(pl.nicNumber))
Logger.debug("checking pipeline info for {0}".format(pl))
topoNode = self.__getNodeByName(pl.node)
if not topoNode:
Logger.warn("failed to find node {0} in topology node list".format(node.name))
continue
instId = topoNode.GetNodeInfo()["InstanceID"]
for node in warmd['Instances']:
if instId == node.get('ID',None):
device = topoNode.GetDeviceByNicNumber(pl.nicNumber)
device.SetMode(pl.mode)
device.SetNaplesPipeline(pl.pipeline)
nic = node['Nics'][pl.nicNumber-1]
nic['version'] = pl.version
nic['pipeline'] = pl.pipeline
nic['mode'] = pl.mode
updated = True
if pl.version not in alreadyDownloaded:
api.DownloadAssets(pl.version)
alreadyDownloaded.append(pl.version)
Logger.info("upgrading node:nic {0}:{1}".format(topoNode.MgmtIpAddress(),pl.nicNumber))
devices = {instId : { "nics":[pl.nicNumber], "nodeName":pl.node, "pipeline":pl.pipeline} }
Logger.debug("writing updated warmd.json to {0}".format(nwarmd))
with open(nwarmd,'w') as outfile:
json.dump(warmd,outfile,indent=4)
resp = api.ReInstallImage(fw_version=pl.version, dr_version=pl.version, devices=devices)
if resp != api.types.status.SUCCESS:
Logger.error(f"Failed to install images on the node:nic {0}:{1}".format(topoNode.MgmtIpAddress(),pl.nicNumber))
break
else:
Logger.warn("failed to find node {0} / id {1} in warmd".format(topoNode.MgmtIpAddress(),instId))
def CollectLogs():
if GlobalOptions.dryrun or GlobalOptions.skip_logs: return
nodes = []
try:
for node in store.GetTopology().GetNodes():
node.username = node.MgmtUserName()
node.password = node.MgmtPassword()
node.ip = node.MgmtIpAddress()
cimcInfo = node.GetCimcInfo()
if cimcInfo:
node.cimcIp = cimcInfo.GetIp()
node.cimcUsername = cimcInfo.GetUsername()
node.cimcPassword = cimcInfo.GetPassword()
nodes.append(node)
except Exception as e:
Logger.debug(
'topo not setup yet, gathering node info from testbed json file: %s'
% str(e))
nodes = buildNodesFromTestbedFile(GlobalOptions.testbed_json)
pool = ThreadPool(len(nodes))
results = pool.map(__collect_onenode, nodes)
def packageCoresDirectory():
Logger.debug("checking for cores in corefiles/")
if not os.path.exists("corefiles"):
Logger.debug("no corefiles directory found")
else:
try:
files = subprocess.check_output(
'find corefiles/ -type f -name "*.tgz"', shell=True)
files = str(files.decode("utf-8"))
if not files:
Logger.debug("no corefiles found")
return
files = re.sub('[\s]+', ' ', files)
cmd = "tar -zcf all_corefiles.tgz {0}".format(files)
msg = "create main corefile tgz with {0}".format(cmd)
print(msg)
Logger.debug(msg)
subprocess.check_call(cmd, stderr=subprocess.PIPE, shell=True)
subprocess.check_call("rm -f {0}".format(files), shell=True)
except:
Logger.debug(
"failed to process corefiles directory. error was: {0}".format(
traceback.format_exc()))