def addAttachment(self, id, rev, value, name=None, contentType=None, checksum=None, add_checksum=False):
"""
Add an attachment stored in value to a document identified by id at revision rev.
If specified the attachement will be uploaded as name, other wise the attachment is
named "attachment".
If not set CouchDB will try to determine contentType and default to text/plain.
If checksum is specified pass this to CouchDB, it will refuse if the MD5 checksum
doesn't match the one provided. If add_checksum is True calculate the checksum of
the attachment and pass that into CouchDB for validation. The checksum should be the
base64 encoded binary md5 (as returned by hashlib.md5().digest())
"""
if name is None:
name = "attachment"
req_headers = {}
if add_checksum:
# calculate base64 encoded MD5
keyhash = hashlib.md5()
value_str = str(value) if not isinstance(value, (newstr, newbytes)) else value
keyhash.update(encodeUnicodeToBytes(value_str))
content_md5 = base64.b64encode(keyhash.digest())
req_headers['Content-MD5'] = decodeBytesToUnicode(content_md5) if PY3 else content_md5
elif checksum:
req_headers['Content-MD5'] = decodeBytesToUnicode(checksum) if PY3 else checksum
return self.put('/%s/%s/%s?rev=%s' % (self.name, id, name, rev),
value, encode=False,
contentType=contentType,
incoming_headers=req_headers)
def __setitem__(self, key, value):
"""
make exception look like a dictionary
"""
# using unicode sandwich pattern
key = decodeBytesToUnicode(key, "ignore")
value = decodeBytesToUnicode(value, "ignore")
self.data[key] = value
def _sanitise_input(self, input_args=[], input_kwargs={}, method=None):
"""
Pull out the necessary input from kwargs (by name) and, failing that,
pulls out the number required args from args, which assumes the
arguments are positional.
_sanitise_input is called automatically if you use the _addMethod/_addDAO
convenience functions. If you add your method to the methods dictionary
by hand you should call _sanitise_input explicitly.
In all but the most basic cases you'll likely want to over-ride this, or
at least treat its outcome with deep suspicion.
TODO: Would be nice to loose the method argument and derive it in this method.
Returns a dictionary of validated, sanitised input data.
"""
verb = request.method.upper()
if len(input_args):
input_args = list(input_args)
if (len(input_args) + len(input_kwargs)) > len(
self.methods[verb][method]['args']):
self.debug('%s to %s expects %s argument(s), got %s' %
(verb, method, len(self.methods[verb][method]['args']),
(len(input_args) + len(input_kwargs))))
raise HTTPError(
400, 'Invalid input: Input arguments failed sanitation.')
input_data = {}
# VK, we must read input kwargs/args as string types
# rather then unicode one. This is important for cx_Oracle
# driver which will place parameters into binded queries
# due to mixmatch (string vs unicode) between python and Oracle
# we must pass string parameters.
for a in self.methods[verb][method]['args']:
if a in input_kwargs:
v = input_kwargs[a]
input_data[a] = decodeBytesToUnicode(
v) if PY3 else encodeUnicodeToBytes(v)
input_kwargs.pop(a)
else:
if len(input_args):
v = input_args.pop(0)
input_data[a] = decodeBytesToUnicode(
v) if PY3 else encodeUnicodeToBytes(v)
if input_kwargs:
raise HTTPError(
400, 'Invalid input: Input arguments failed sanitation.')
self.debug('%s raw data: %s' % (method, {
'args': input_args,
'kwargs': input_kwargs
}))
self.debug('%s sanitised input_data: %s' % (method, input_data))
return self._validate_input(input_data, verb, method)
def addInfo(self, **data):
"""
_addInfo_
Add key=value information pairs to an
exception instance
"""
for key, value in viewitems(data):
# assumption: value is not iterable (list, dict, tuple, ...)
# using unicode sandwich pattern
key = decodeBytesToUnicode(key, "ignore")
value = decodeBytesToUnicode(value, "ignore")
self.data[key] = value
return
def parse(self, response):
"""Parse response header and assign class member data"""
startRegex = r"^HTTP/\d.\d \d{3}"
continueRegex = r"^HTTP/\d.\d 100" # Continue: client should continue its request
replaceRegex = r"^HTTP/\d.\d"
response = decodeBytesToUnicode(response)
for row in response.split('\r'):
row = row.replace('\n', '')
if not row:
continue
if re.search(startRegex, row):
if re.search(continueRegex, row):
continue
res = re.sub(replaceRegex, "", row).strip()
status, reason = res.split(' ', 1)
self.status = int(status)
self.reason = reason
continue
try:
key, val = row.split(':', 1)
self.header[key.strip()] = val.strip()
except:
pass
def decode(self, data):
"""
decode data to some appropriate format, for now make it a string...
"""
if PY3:
return decodeBytesToUnicode(data)
return data.__str__()
def execute_command(command, logger, timeout, redirect=True):
"""
_execute_command_
Function to manage commands.
"""
stdout, stderr, rc = None, None, 99999
if redirect:
proc = subprocess.Popen(
command,
shell=True,
cwd=os.environ['PWD'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
)
else:
proc = subprocess.Popen(command, shell=True, cwd=os.environ['PWD'])
t_beginning = time.time()
while True:
if proc.poll() is not None:
break
seconds_passed = time.time() - t_beginning
if timeout and seconds_passed > timeout:
proc.terminate()
logger.error('Timeout in %s execution.' % command)
return stdout, rc
time.sleep(0.1)
stdout, stderr = proc.communicate()
stdout = decodeBytesToUnicode(stdout) if PY3 else stdout
stderr = decodeBytesToUnicode(stderr) if PY3 else stderr
rc = proc.returncode
logger.debug(
'Executing : \n command : %s\n output : %s\n error: %s\n retcode : %s'
% (command, stdout, stderr, rc))
return stdout, stderr, rc
def listRunLumis(self, dataset=None, block=None):
"""
It gets a list of DBSRun objects and returns the number of lumisections per run
DbsRun (RunNumber,
NumberOfEvents,
NumberOfLumiSections,
TotalLuminosity,
StoreNumber,
StartOfRungetLong,
EndOfRun,
CreationDate,
CreatedBy,
LastModificationDate,
LastModifiedBy
)
"""
# Pointless code in python3
block = decodeBytesToUnicode(block)
dataset = decodeBytesToUnicode(dataset)
try:
if block:
results = self.dbs.listRuns(block_name=block)
else:
results = self.dbs.listRuns(dataset=dataset)
except dbsClientException as ex:
msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block)
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
# send runDict format as result, this format is for sync with dbs2 call
# which has {run_number: num_lumis} but dbs3 call doesn't return num Lumis
# So it returns {run_number: None}
# TODO: After DBS2 is completely removed change the return format more sensible one
runDict = {}
for x in results:
for runNumber in x["run_num"]:
runDict[runNumber] = None
return runDict
def runCommand(cmd, shell=True, timeout=None):
"""
Run generic command
This is NOT secure and hence NOT recommended
It does however have the timeout functions built into it
timeout must be an int
Note, setting timeout = 0 does nothing!
"""
if timeout:
if not isinstance(timeout, int):
timeout = None
logging.error(
"SubprocessAlgo.runCommand expected int timeout, got %s",
timeout)
else:
signal.signal(signal.SIGALRM, alarmHandler)
signal.alarm(timeout)
try:
pipe = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=shell)
stdout, stderr = pipe.communicate()
if PY3:
stdout = decodeBytesToUnicode(stdout)
stderr = decodeBytesToUnicode(stderr)
returnCode = pipe.returncode
except Alarm:
msg = "Alarm sounded while running command after %s seconds.\n" % timeout
msg += "Command: %s\n" % cmd
msg += "Raising exception"
logging.error(msg)
raise SubprocessAlgoException(msg)
if timeout:
signal.alarm(0)
return stdout, stderr, returnCode
def decode(self, data):
"""
decode the data to python from json
"""
if data:
decoder = JSONDecoder()
thunker = JSONThunker()
if PY3:
data = decodeBytesToUnicode(data)
data = decoder.decode(data)
unthunked = thunker.unthunk(data)
return unthunked
return {}
def addBasicAuth(self, username, password):
"""Add basic auth headers to request"""
## TODO: base64.encodestring is deprecated
# https://docs.python.org/3.8/library/base64.html#base64.encodestring
# change to base64.encodebytes after we drop python2
username = encodeUnicodeToBytes(username)
password = encodeUnicodeToBytes(password)
encodedauth = base64.encodestring(b'%s:%s' %
(username, password)).strip()
if PY3:
encodedauth = decodeBytesToUnicode(encodedauth)
auth_string = "Basic %s" % encodedauth
self.additionalHeaders["Authorization"] = auth_string
def getUserAttributes(self):
"""
_getUserAttributes_
Retrieve the user's attributes from the voms-proxy-info call.
"""
vomsProxyInfoCall = subprocess.Popen(["voms-proxy-info", "-fqan"],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
if vomsProxyInfoCall.wait() != 0:
return None
stdout, _ = vomsProxyInfoCall.communicate()
stdout = decodeBytesToUnicode(stdout) if PY3 else stdout
return stdout[0:-1]
def addConfig(self, newConfig, psetHash=None):
"""
_addConfig_
"""
# The newConfig parameter is a URL suitable for passing to urlopen.
with closing(urllib.request.urlopen(newConfig)) as f:
configString = f.read(-1)
configMD5 = hashlib.md5(configString).hexdigest()
if PY3:
configString = decodeBytesToUnicode(configString)
self.document['md5_hash'] = configMD5
self.document['pset_hash'] = psetHash
self.attachments['configFile'] = configString
return
def unpersist(self, filename, reportname=None):
"""
_unpersist_
Load a pickled FWJR from disk.
"""
if PY3:
with open(filename, 'rb') as handle:
self.data = decodeBytesToUnicode(pickle.load(handle))
else:
with open(filename, 'r') as handle:
self.data = pickle.load(handle)
# old self.report (if it existed) became unattached
if reportname:
self.report = getattr(self.data, reportname)
return
def upload(self, url, args, filename):
"""
_upload_
Perform a file upload to the dqm server using HTTPS auth with the
service proxy provided
"""
ident = "WMAgent python/%d.%d.%d" % sys.version_info[:3]
uploadProxy = self.step.upload.proxy or os.environ.get(
'X509_USER_PROXY', None)
logging.info("Using proxy file: %s", uploadProxy)
logging.info("Using CA certificate path: %s",
os.environ.get('X509_CERT_DIR'))
msg = "HTTP POST upload arguments:\n"
for arg in args:
msg += " ==> %s: %s\n" % (arg, args[arg])
logging.info(msg)
handler = HTTPSAuthHandler(key=uploadProxy, cert=uploadProxy)
opener = OpenerDirector()
opener.add_handler(handler)
# setup the request object
url = decodeBytesToUnicode(url) if PY3 else encodeUnicodeToBytes(url)
datareq = Request(url + '/data/put')
datareq.add_header('Accept-encoding', 'gzip')
datareq.add_header('User-agent', ident)
self.marshall(args, {'file': filename}, datareq)
if 'https://' in url:
result = opener.open(datareq)
else:
opener.add_handler(ProxyHandler({}))
result = opener.open(datareq)
data = result.read()
if result.headers.get('Content-encoding', '') == 'gzip':
data = GzipFile(fileobj=BytesIO(data)).read()
return (result.headers, data)
def getFileBlockWithParents(self, fileBlockName):
"""
Retrieve a list of parent files in the block; a flag whether the
block is still open or not; and it used to resolve the block
location via PhEDEx.
:return: a dictionary in the format of:
{"PhEDExNodeNames" : [],
"Files" : { LFN : Events },
"IsOpen" : True|False}
"""
fileBlockName = decodeBytesToUnicode(fileBlockName)
if not self.blockExists(fileBlockName):
msg = "DBSReader.getFileBlockWithParents(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
result = {"PhEDExNodeNames": [], # FIXME: we better get rid of this line!
"Files": self.listFilesInBlockWithParents(fileBlockName),
"IsOpen": self.blockIsOpen(fileBlockName)}
return result
def scramArchtoRequiredArch(scramArch=None):
"""
Converts a given ScramArch to a unique target CPU architecture.
Note that an architecture precedence is enforced in case there are
multiple matches.
In case no scramArch is defined, leave the architecture undefined.
:param scramArch: can be either a string or a list of ScramArchs
:return: a string with the matched architecture
"""
defaultArch = "X86_64"
requiredArchs = set()
if scramArch is None:
return None
elif isinstance(scramArch, (str, bytes)):
scramArch = [scramArch]
for item in scramArch:
item = decodeBytesToUnicode(item)
arch = item.split("_")[1]
if arch not in SCRAM_TO_ARCH:
msg = "Job configured to a ScramArch: '{}' not supported in BossAir".format(
item)
raise BossAirPluginException(msg)
requiredArchs.add(SCRAM_TO_ARCH.get(arch))
# now we have the final list of architectures, return only 1 of them
if len(requiredArchs) == 1:
return requiredArchs.pop()
elif "X86_64" in requiredArchs:
return "X86_64"
elif "ppc64le" in requiredArchs:
return "ppc64le"
elif "aarch64" in requiredArchs:
return "aarch64"
else: # should never get here!
return defaultArch
def addError(self,
stepName,
exitCode,
errorType,
errorDetails,
siteName=None):
"""
_addError_
Add an error report with an exitCode, type/class of error and
details of the error as a string.
Also, report attempted site if error happened before landing on it.
"""
if self.retrieveStep(stepName) is None:
# Create a step and set it to failed
# Assumption: Adding an error fails a step
self.addStep(stepName, status=1)
if exitCode is not None:
exitCode = int(exitCode)
setExitCodes = self.getStepExitCodes(stepName)
if exitCode in setExitCodes:
logging.warning(
"Exit code: %s has been already added to the job report",
exitCode)
return
stepSection = self.retrieveStep(stepName)
errorCount = getattr(stepSection.errors, "errorCount", 0)
errEntry = "error%s" % errorCount
stepSection.errors.section_(errEntry)
errDetails = getattr(stepSection.errors, errEntry)
errDetails.exitCode = exitCode
errDetails.type = str(errorType)
try:
if isinstance(errorDetails, newstr):
errDetails.details = errorDetails
elif isinstance(errorDetails, bytes):
errDetails.details = decodeBytesToUnicode(
errorDetails, 'ignore')
else:
errDetails.details = newstr(errorDetails)
except UnicodeEncodeError as ex:
msg = "Failed to encode the job error details for job ID: %s." % self.getJobID(
)
msg += "\nException message: %s\nOriginal error details: %s" % (
str(ex), errorDetails)
logging.error(msg)
msg = "DEFAULT ERROR MESSAGE, because it failed to UTF-8 encode the original message."
errDetails.details = msg
except UnicodeDecodeError as ex:
msg = "Failed to decode the job error details for job ID: %s." % self.getJobID(
)
msg += "\nException message: %s\nOriginal error details: %s" % (
str(ex), errorDetails)
logging.error(msg)
msg = "DEFAULT ERROR MESSAGE, because it failed to UTF-8 decode the original message."
errDetails.details = msg
setattr(stepSection.errors, "errorCount", errorCount + 1)
self.setStepStatus(stepName=stepName, status=exitCode)
if siteName:
self._setSiteName(site=siteName)
return
def __init__(self, message, errorNo=None, **data):
self.name = str(self.__class__.__name__)
# Fix for the unicode encoding issue, see #8056 and #8403
# interprets this string using utf-8 codec and ignoring any errors.
# using unicode sandwich pattern
message = decodeBytesToUnicode(message, "ignore")
Exception.__init__(self, self.name, message)
# //
# // Init data dictionary with defaults
# //
self.data = {}
self.data.setdefault("ClassName", None)
self.data.setdefault("ModuleName", None)
self.data.setdefault("MethodName", None)
self.data.setdefault("ClassInstance", None)
self.data.setdefault("FileName", None)
self.data.setdefault("LineNumber", None)
if errorNo is None:
self.data.setdefault("ErrorNr", 0)
else:
self.data.setdefault("ErrorNr", errorNo)
self._message = message
self.addInfo(**data)
# //
# // Automatically determine the module name
# // if not set
if self.data['ModuleName'] is None:
try:
frame = inspect.currentframe()
lastframe = inspect.getouterframes(frame)[1][0]
excepModule = inspect.getmodule(lastframe)
if excepModule is not None:
modName = excepModule.__name__
self.data['ModuleName'] = modName
finally:
frame = None
# //
# // Find out where the exception came from
# //
try:
stack = inspect.stack(1)[1]
self.data['FileName'] = stack[1]
self.data['LineNumber'] = stack[2]
self.data['MethodName'] = stack[3]
finally:
stack = None
# //
# // ClassName if ClassInstance is passed
# //
try:
if self.data['ClassInstance'] is not None:
self.data['ClassName'] = self.data[
'ClassInstance'].__class__.__name__
except Exception:
pass
# Determine the traceback at time of __init__
try:
self.traceback = "\n".join(traceback.format_tb(sys.exc_info()[2]))
except Exception:
self.traceback = "WMException error: Couldn't get traceback\n"
# using unicode sandwich pattern
self.traceback = decodeBytesToUnicode(self.traceback, "ignore")
def handlePileup(self):
"""
_handlePileup_
Handle pileup settings.
There has been stored pileup configuration stored in a JSON file
as a result of DBS querrying when running PileupFetcher,
this method loads this configuration from sandbox and returns it
as dictionary.
The PileupFetcher was called by WorkQueue which creates job's sandbox
and sandbox gets migrated to the worker node.
External script iterates over all modules and over all pileup configuration types.
The only considered types are "data" and "mc" (input to this method).
If other pileup types are specified by the user, the method doesn't
modify anything.
The method considers only files which are present on this local PNN.
The job will use only those, unless it was told to trust the PU site
location (trustPUSitelists=True), in this case ALL the blocks/files
will be added to the PSet and files will be read via AAA.
Dataset, divided into blocks, may not have all blocks present on a
particular PNN. However, all files belonging into a block will be
present when reported by DBS.
The structure of the pileupDict: PileupFetcher._queryDbsAndGetPileupConfig
"""
# find out local site SE name
siteConfig = loadSiteLocalConfig()
PhEDExNodeName = siteConfig.localStageOut["phedex-node"]
self.logger.info("Running on site '%s', local PNN: '%s'",
siteConfig.siteName, PhEDExNodeName)
jsonPileupConfig = os.path.join(self.stepSpace.location,
"pileupconf.json")
# Load pileup json
try:
with open(jsonPileupConfig) as jdata:
pileupDict = json.load(jdata)
except IOError:
m = "Could not read pileup JSON configuration file: '%s'" % jsonPileupConfig
raise RuntimeError(m)
# Create a json with a list of files and events available
# after dealing with PhEDEx/AAA logic
newPileupDict = {}
fileList = []
eventsAvailable = 0
for pileupType in self.step.data.pileup.listSections_():
pileupType = decodeBytesToUnicode(pileupType)
useAAA = True if getattr(self.jobBag, 'trustPUSitelists',
False) else False
self.logger.info("Pileup set to read data remotely: %s", useAAA)
for blockName in sorted(pileupDict[pileupType].keys()):
blockDict = pileupDict[pileupType][blockName]
if PhEDExNodeName in blockDict["PhEDExNodeNames"] or useAAA:
eventsAvailable += int(blockDict.get('NumberOfEvents', 0))
for fileLFN in blockDict["FileList"]:
fileList.append(decodeBytesToUnicode(fileLFN))
newPileupDict[pileupType] = {
"eventsAvailable": eventsAvailable,
"FileList": fileList
}
newJsonPileupConfig = os.path.join(self.stepSpace.location,
"CMSSWPileupConfig.json")
self.logger.info("Generating json for CMSSW pileup script")
try:
# If it's a python2 unicode, cmssw_handle_pileup will cast it to str
with open(newJsonPileupConfig, 'w') as f:
json.dump(newPileupDict, f)
except Exception as ex:
self.logger.exception("Error writing out process filelist json:")
raise ex
procScript = "cmssw_handle_pileup.py"
cmd = "%s --input_pkl %s --output_pkl %s --pileup_dict %s" % (
procScript, os.path.join(self.stepSpace.location,
self.configPickle),
os.path.join(self.stepSpace.location,
self.configPickle), newJsonPileupConfig)
if getattr(self.jobBag, "skipPileupEvents", None):
randomSeed = self.job['task']
skipPileupEvents = self.jobBag.skipPileupEvents
cmd += " --skip_pileup_events %s --random_seed %s" % (
skipPileupEvents, randomSeed)
self.scramRun(cmd)
return
def testExitCode(self):
"""
_testExitCode_
Test and see if we can get an exit code out of a report
Note: Errors without a return code return 99999
getStepExitCode: returns the first valid and non-zero exit code
getExitCode: uses the method above to get an exit code
getStepExitCodes: returns a set of all exit codes within the step
"""
report = Report("cmsRun1")
self.assertEqual(report.getExitCode(), 0)
self.assertEqual(report.getStepExitCode(stepName="cmsRun1"), 0)
self.assertItemsEqual(report.getStepExitCodes(stepName="cmsRun1"), {})
self.assertItemsEqual(report.getStepErrors(stepName="cmsRun1"), {})
report.addError(stepName="cmsRun1", exitCode=None, errorType="test", errorDetails="test")
# None is not a valid exitCode, but it will get mapped to 99999
self.assertEqual(report.getExitCode(), 99999)
self.assertEqual(report.getStepExitCode(stepName="cmsRun1"), 99999)
self.assertItemsEqual(report.getStepExitCodes(stepName="cmsRun1"), {99999})
self.assertEqual(report.getStepErrors(stepName="cmsRun1")['errorCount'], 1)
report.addError(stepName="cmsRun1", exitCode=102, errorType="test", errorDetails="test")
self.assertEqual(report.getExitCode(), 102)
self.assertEqual(report.getStepExitCode(stepName="cmsRun1"), 102)
self.assertItemsEqual(report.getStepExitCodes(stepName="cmsRun1"), {99999, 102})
self.assertEqual(report.getStepErrors(stepName="cmsRun1")['errorCount'], 2)
report.addError(stepName="cmsRun1", exitCode=103, errorType="test", errorDetails="test")
self.assertEqual(report.getExitCode(), 102)
self.assertEqual(report.getStepExitCode(stepName="cmsRun1"), 102)
self.assertItemsEqual(report.getStepExitCodes(stepName="cmsRun1"), {99999, 102, 103})
self.assertEqual(report.getStepErrors(stepName="cmsRun1")['errorCount'], 3)
# now try to record the same exit code once again
report.addError(stepName="cmsRun1", exitCode=104, errorType="test", errorDetails="test")
self.assertEqual(report.getExitCode(), 102)
self.assertEqual(report.getStepExitCode(stepName="cmsRun1"), 102)
self.assertItemsEqual(report.getStepExitCodes(stepName="cmsRun1"), {99999, 102, 103, 104})
self.assertEqual(report.getStepErrors(stepName="cmsRun1")['errorCount'], 4)
# and once again, but different type and details (which does not matter)
report.addError(stepName="cmsRun1", exitCode=105, errorType="testEE", errorDetails="testAA")
self.assertEqual(report.getExitCode(), 102)
self.assertEqual(report.getStepExitCode(stepName="cmsRun1"), 102)
self.assertItemsEqual(report.getStepExitCodes(stepName="cmsRun1"), {99999, 102, 103, 104, 105})
self.assertEqual(report.getStepErrors(stepName="cmsRun1")['errorCount'], 5)
# and once again, but different type and details - testing unicode handling
report.addError(stepName="cmsRun1", exitCode=106, errorType="test", errorDetails="1 тℯṧт")
self.assertEqual(report.getExitCode(), 102)
self.assertEqual(report.getStepExitCode(stepName="cmsRun1"), 102)
self.assertItemsEqual(report.getStepExitCodes(stepName="cmsRun1"), {99999, 102, 103, 104, 105, 106})
self.assertEqual(report.getStepErrors(stepName="cmsRun1")['errorCount'], 6)
# and once again, but different type and details - testing unicode handling
report.addError(stepName="cmsRun1", exitCode=107, errorType="test", errorDetails="2 тℯṧт \x95")
self.assertEqual(report.getExitCode(), 102)
self.assertEqual(report.getStepExitCode(stepName="cmsRun1"), 102)
self.assertItemsEqual(report.getStepExitCodes(stepName="cmsRun1"), {99999, 102, 103, 104, 105, 106, 107})
self.assertEqual(report.getStepErrors(stepName="cmsRun1")['errorCount'], 7)
# and once again, but different type and details - testing unicode handling
report.addError(stepName="cmsRun1", exitCode=108, errorType="test", errorDetails=encodeUnicodeToBytes("3 тℯṧт"))
self.assertEqual(report.getExitCode(), 102)
self.assertEqual(report.getStepExitCode(stepName="cmsRun1"), 102)
self.assertItemsEqual(report.getStepExitCodes(stepName="cmsRun1"), {99999, 102, 103, 104, 105, 106, 107, 108})
self.assertEqual(report.getStepErrors(stepName="cmsRun1")['errorCount'], 8)
# and once again, but different type and details - testing unicode handling
report.addError(stepName="cmsRun1", exitCode=109, errorType="test", errorDetails=decodeBytesToUnicode("4 тℯṧт"))
self.assertEqual(report.getExitCode(), 102)
self.assertEqual(report.getStepExitCode(stepName="cmsRun1"), 102)
self.assertItemsEqual(report.getStepExitCodes(stepName="cmsRun1"), {99999, 102, 103, 104, 105, 106, 107, 108, 109})
self.assertEqual(report.getStepErrors(stepName="cmsRun1")['errorCount'], 9)
# and once again, but different type and details - testing unicode handling
report.addError(stepName="cmsRun1", exitCode=110, errorType="test", errorDetails={"нεʟʟ◎": 3.14159})
self.assertEqual(report.getExitCode(), 102)
self.assertEqual(report.getStepExitCode(stepName="cmsRun1"), 102)
self.assertItemsEqual(report.getStepExitCodes(stepName="cmsRun1"), {99999, 102, 103, 104, 105, 106, 107, 108, 109, 110})
self.assertEqual(report.getStepErrors(stepName="cmsRun1")['errorCount'], 10)
# and once again, but different type and details - testing unicode handling
report.addError(stepName="cmsRun1", exitCode=111, errorType="test", errorDetails={"нεʟʟ◎ \x95": "ẘøґℓ∂ \x95"})
self.assertEqual(report.getExitCode(), 102)
self.assertEqual(report.getStepExitCode(stepName="cmsRun1"), 102)
self.assertItemsEqual(report.getStepExitCodes(stepName="cmsRun1"), {99999, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111})
self.assertEqual(report.getStepErrors(stepName="cmsRun1")['errorCount'], 11)
