def _specializedLoadXMLandCSV(self, filenameRoot, options):
"""
Function to load the xml additional file of the csv for data
(it contains metadata, etc). It must be implemented by the specialized classes
@ In, filenameRoot, string, file name root
@ In, options, dict, dictionary -> options for loading
@ Out, None
"""
#For Pointset it will create an XML file and one CSV file.
#The CSV file will have a header with the input names and output
#names, and multiple lines of data with the input and output
#numeric values, one line for each input.
if options is not None and 'fileToLoad' in options.keys():
name = os.path.join(options['fileToLoad'].getPath(),
options['fileToLoad'].getBase())
else:
name = self.name
filenameLocal = os.path.join(filenameRoot, name)
if os.path.isfile(filenameLocal + '.xml'):
xmlData = self._loadXMLFile(filenameLocal)
assert (xmlData["fileType"] == "Pointset")
if "metadata" in xmlData:
self._dataContainer['metadata'] = xmlData["metadata"]
mainCSV = os.path.join(filenameRoot, xmlData["filenameCSV"])
else:
mainCSV = os.path.join(filenameRoot, name + '.csv')
myFile = open(mainCSV, "rU")
header = myFile.readline().rstrip()
inoutKeys = header.split(",")
inoutValues = [[] for _ in range(len(inoutKeys))]
for line in myFile.readlines():
lineList = line.rstrip().split(",")
for i in range(len(inoutKeys)):
inoutValues[i].append(utils.partialEval(lineList[i]))
#extend the expected size of this point set
self.numAdditionalLoadPoints = len(
inoutValues[0]) #used in checkConsistency
self._dataContainer['inputs'] = {}
self._dataContainer['outputs'] = {}
inoutDict = {}
for key, value in zip(inoutKeys, inoutValues):
inoutDict[key] = value
for key in self.getParaKeys('inputs'):
self._dataContainer["inputs"][key] = c1darray(
values=np.array(inoutDict[key]))
for key in self.getParaKeys('outputs'):
self._dataContainer["outputs"][key] = c1darray(
values=np.array(inoutDict[key]))
def convert(cls, value):
"""
Converts value from string to a listi with string, integer, or float type.
@ In, value, string, the value to convert
@ Out, convert, list, the converted value
"""
values = value.split(",")
base = utils.partialEval(values[0].strip())
# three possibilities: string, integer, or float
if utils.isAString(base):
conv = str
elif utils.isAnInteger(base):
conv = int
else: #float
conv = float
return [conv(x.strip()) for x in values]
def _specializedLoadXMLandCSV(self, filenameRoot, options):
"""
Function to load the xml additional file of the csv for data
(it contains metadata, etc). It must be implemented by the specialized classes
@ In, filenameRoot, string, file name root
@ In, options, dict, dictionary -> options for loading
@ Out, None
"""
#For Pointset it will create an XML file and one CSV file.
#The CSV file will have a header with the input names and output
#names, and multiple lines of data with the input and output
#numeric values, one line for each input.
if options is not None and 'fileToLoad' in options.keys():
name = os.path.join(options['fileToLoad'].getPath(),
options['fileToLoad'].getBase())
else:
name = self.name
filenameLocal = os.path.join(filenameRoot, name)
if os.path.isfile(filenameLocal + '.xml'):
xmlData = self._loadXMLFile(filenameLocal)
assert (xmlData["fileType"] == "Pointset")
if "metadata" in xmlData:
self._dataContainer['metadata'] = xmlData["metadata"]
mainCSV = os.path.join(filenameRoot, xmlData["filenameCSV"])
else:
mainCSV = os.path.join(filenameRoot, name + '.csv')
myFile = open(mainCSV, "rU")
header = myFile.readline().rstrip()
inoutKeys = header.split(",")
inoutValues = [[] for _ in range(len(inoutKeys))]
for lineNumber, line in enumerate(myFile.readlines(), 2):
lineList = line.rstrip().split(",")
for i in range(len(inoutKeys)):
datum = utils.partialEval(lineList[i])
if datum == '':
self.raiseAnError(
IOError,
'Invalid data in input file: {} at line {}: "{}"'.
format(filenameLocal, lineNumber, line.rstrip()))
inoutValues[i].append(utils.partialEval(lineList[i]))
#extend the expected size of this point set
self.numAdditionalLoadPoints += len(
inoutValues[0]) #used in checkConsistency
## Do not reset these containers because it will wipe whatever information
## already exists in this PointSet. This is not one of the use cases for our
## data objects. We claim in the manual to construct or update information.
## These should be non-destructive operations. -- DPM 6/26/17
# self._dataContainer['inputs'] = {}
# self._dataContainer['outputs'] = {}
inoutDict = {}
for key, value in zip(inoutKeys, inoutValues):
inoutDict[key] = value
for key in self.getParaKeys('inputs'):
## Again, in order to be non-destructive we should only initialize on the
## first go-around, subsequent loads should append to the existing list.
## -- DPM 6/26/17
if key not in self._dataContainer["inputs"]:
self._dataContainer["inputs"][key] = c1darray(
values=np.array(inoutDict[key]))
else:
self._dataContainer["inputs"][key].append(
c1darray(values=np.array(inoutDict[key])))
for key in self.getParaKeys('outputs'):
## Again, in order to be non-destructive we should only initialize on the
## first go-around, subsequent loads should append to the existing list.
## -- DPM 6/26/17
if key not in self._dataContainer["outputs"]:
self._dataContainer["outputs"][key] = c1darray(
values=np.array(inoutDict[key]))
else:
self._dataContainer["outputs"][key].append(
c1darray(values=np.array(inoutDict[key])))
def _readMoreXMLbase(self, xmlNode):
"""
Function to read the portion of the xml input that belongs to the base sampler only
and initialize some stuff based on the inputs got
The text is supposed to contain the info where and which variable to change.
In case of a code the syntax is specified by the code interface itself
@ In, xmlNode, xml.etree.ElementTree.Element, Xml element node1
@ Out, None
"""
for child in xmlNode:
prefix = ""
if child.tag == 'Distribution':
for childChild in child:
if childChild.tag == 'distribution':
prefix = "<distribution>"
tobesampled = childChild.text
self.toBeSampled[prefix + child.attrib['name']] = tobesampled
#if child.attrib['name'] != tobesampled:self.raiseAnError(IOError,"name of the <Distribution> node and <distribution> mismatches for node named "+ child.attrib['name'])
elif child.tag == 'variable':
foundDistOrFunc = False
for childChild in child:
if childChild.tag == 'distribution':
if not foundDistOrFunc: foundDistOrFunc = True
else:
self.raiseAnError(
IOError,
'A sampled variable cannot have both a distribution and a function!'
)
tobesampled = childChild.text
varData = {}
varData['name'] = childChild.text
if childChild.get('dim') == None:
dim = 1
else:
dim = childChild.attrib['dim']
varData['dim'] = int(dim)
self.variables2distributionsMapping[
child.attrib['name']] = varData
self.toBeSampled[prefix +
child.attrib['name']] = tobesampled
elif childChild.tag == 'function':
if not foundDistOrFunc: foundDistOrFunc = True
else:
self.raiseAnError(
IOError,
'A sampled variable cannot have both a distribution and a function!'
)
tobesampled = childChild.text
self.dependentSample[
prefix + child.attrib['name']] = tobesampled
if not foundDistOrFunc:
self.raiseAnError(
IOError, 'Sampled variable', child.attrib['name'],
'has neither a <distribution> nor <function> node specified!'
)
elif child.tag == "variablesTransformation":
transformationDict = {}
listIndex = None
for childChild in child:
if childChild.tag == "latentVariables":
transformationDict[childChild.tag] = list(
inp.strip()
for inp in childChild.text.strip().split(','))
elif childChild.tag == "manifestVariables":
transformationDict[childChild.tag] = list(
inp.strip()
for inp in childChild.text.strip().split(','))
elif childChild.tag == "manifestVariablesIndex":
# the index provided by the input file starts from 1, but the index used by the code starts from 0.
listIndex = list(
int(inp.strip()) - 1
for inp in childChild.text.strip().split(','))
elif childChild.tag == "method":
self.transformationMethod[
child.attrib['distribution']] = childChild.text
if listIndex == None:
self.raiseAWarning(
'Index is not provided for manifestVariables, default index will be used instead!'
)
listIndex = range(
len(transformationDict["manifestVariables"]))
transformationDict["manifestVariablesIndex"] = listIndex
self.variablesTransformationDict[
child.attrib['distribution']] = transformationDict
elif child.tag == "constant":
value = utils.partialEval(child.text)
if value is None:
self.raiseAnError(
IOError,
'The body of "constant" XML block should be a number. Got: '
+ child.text)
try:
self.constants[child.attrib['name']] = value
except KeyError:
self.raiseAnError(
KeyError,
child.tag + ' must have the attribute "name"!!!')
elif child.tag == "restartTolerance":
self.restartTolerance = float(child.text)
if len(self.constants) > 0:
# check if constant variables are also part of the sampled space. In case, error out
if not set(self.toBeSampled.keys()).isdisjoint(
self.constants.keys()):
self.raiseAnError(
IOError,
"Some constant variables are also in the sampling space:" +
' '.join([
i if i in self.toBeSampled.keys() else ""
for i in self.constants.keys()
]))
if self.initSeed == None:
self.initSeed = Distributions.randomIntegers(0, 2**31, self)
# Creation of the self.distributions2variablesMapping dictionary: {'distName': ({'variable_name1': dim1}, {'variable_name2': dim2})}
for variable in self.variables2distributionsMapping.keys():
distName = self.variables2distributionsMapping[variable]['name']
dim = self.variables2distributionsMapping[variable]['dim']
listElement = {}
listElement[variable] = dim
if (distName in self.distributions2variablesMapping.keys()):
self.distributions2variablesMapping[distName].append(
listElement)
else:
self.distributions2variablesMapping[distName] = [listElement]
# creation of the self.distributions2variablesIndexList dictionary:{'distName':[dim1,dim2,...,dimN]}
self.distributions2variablesIndexList = {}
for distName in self.distributions2variablesMapping.keys():
positionList = []
for var in self.distributions2variablesMapping[distName]:
position = utils.first(var.values())
positionList.append(position)
positionList = list(set(positionList))
positionList.sort()
self.distributions2variablesIndexList[distName] = positionList
for key in self.variables2distributionsMapping.keys():
distName = self.variables2distributionsMapping[key]['name']
dim = self.variables2distributionsMapping[key]['dim']
reducedDim = self.distributions2variablesIndexList[distName].index(
dim) + 1
self.variables2distributionsMapping[key][
'reducedDim'] = reducedDim # the dimension of variable in the transformed space
self.variables2distributionsMapping[key]['totDim'] = max(
self.distributions2variablesIndexList[distName]
) # We will reset the value if the node <variablesTransformation> exist in the raven input file
if not self.variablesTransformationDict and self.variables2distributionsMapping[
key]['totDim'] > 1:
if self.variables2distributionsMapping[key]['totDim'] != len(
self.distributions2variablesIndexList[distName]):
self.raiseAnError(
IOError,
'The "dim" assigned to the variables insider Sampler are not correct! the "dim" should start from 1, and end with the full dimension of given distribution'
)
#Checking the variables transformation
if self.variablesTransformationDict:
for dist, varsDict in self.variablesTransformationDict.items():
maxDim = len(varsDict['manifestVariables'])
listLatentElement = varsDict['latentVariables']
if len(set(listLatentElement)) != len(listLatentElement):
dups = set(var for var in listLatentElement
if listLatentElement.count(var) > 1)
self.raiseAnError(
IOError,
'The following are duplicated variables listed in the latentVariables: '
+ str(dups))
if len(set(varsDict['manifestVariables'])) != len(
varsDict['manifestVariables']):
dups = set(var for var in varsDict['manifestVariables']
if varsDict['manifestVariables'].count(var) > 1)
self.raiseAnError(
IOError,
'The following are duplicated variables listed in the manifestVariables: '
+ str(dups))
if len(set(varsDict['manifestVariablesIndex'])) != len(
varsDict['manifestVariablesIndex']):
dups = set(
var + 1 for var in varsDict['manifestVariablesIndex']
if varsDict['manifestVariablesIndex'].count(var) > 1)
self.raiseAnError(
IOError,
'The following are duplicated variables indices listed in the manifestVariablesIndex: '
+ str(dups))
listElement = self.distributions2variablesMapping[dist]
for var in listElement:
self.variables2distributionsMapping[var.keys()[0]][
'totDim'] = maxDim #reset the totDim to reflect the totDim of original input space
tempListElement = {
k.strip(): v
for x in listElement for ks, v in x.items()
for k in list(ks.strip().split(','))
}
listIndex = []
for var in listLatentElement:
if var not in set(tempListElement.keys()):
self.raiseAnError(
IOError,
'The variable listed in latentVariables ' + var +
' is not listed in the given distribution: ' +
dist)
listIndex.append(tempListElement[var] - 1)
if max(listIndex) > maxDim:
self.raiseAnError(
IOError, 'The maximum dim = ' + str(max(listIndex)) +
' defined for latent variables is exceeded the dimension of the problem '
+ str(maxDim))
if len(set(listIndex)) != len(listIndex):
dups = set(var + 1 for var in listIndex
if listIndex.count(var) > 1)
self.raiseAnError(
IOError,
'Each of the following dimensions are assigned to multiple latent variables in Samplers: '
+ str(dups))
# update the index for latentVariables according to the 'dim' assigned for given var defined in Sampler
self.variablesTransformationDict[dist][
'latentVariablesIndex'] = listIndex
def _specializedLoadXMLandCSV(self, filenameRoot, options):
"""
Function to load the xml additional file of the csv for data
(it contains metadata, etc). It must be implemented by the specialized classes
@ In, filenameRoot, string, file name root
@ In, options, dict, dictionary -> options for loading
@ Out, None
"""
#For HistorySet, create an XML file, and multiple CSV
#files. The first CSV file has a header with the input names,
#and a column for the filenames. There is one CSV file for each
#data line in the first CSV and they are named with the
#filename. They have the output names for a header, a column
#for time, and the rest of the file is data for different times.
if options is not None and 'fileToLoad' in options.keys():
name = os.path.join(options['fileToLoad'].getPath(),
options['fileToLoad'].getBase())
else:
name = self.name
filenameLocal = os.path.join(filenameRoot, name)
if os.path.isfile(filenameLocal + '.xml'):
xmlData = self._loadXMLFile(filenameLocal)
assert (xmlData["fileType"] == "HistorySet")
if "metadata" in xmlData:
self._dataContainer['metadata'] = xmlData["metadata"]
mainCSV = os.path.join(filenameRoot, xmlData["filenameCSV"])
else:
mainCSV = os.path.join(filenameRoot, name + '.csv')
myFile = open(mainCSV, "rU")
header = myFile.readline().rstrip()
inpKeys = header.split(",")[:-1]
inpValues = []
outKeys = []
outValues = []
allLines = myFile.readlines()
for mainLine in allLines:
mainLineList = mainLine.rstrip().split(",")
inpValues_h = [utils.partialEval(a) for a in mainLineList[:-1]]
inpValues.append(inpValues_h)
dataFilename = mainLineList[-1]
subCSVFilename = os.path.join(filenameRoot, dataFilename)
subCSVFile = Files.returnInstance("CSV", self)
subCSVFile.setFilename(subCSVFilename)
self._toLoadFromList.append(subCSVFile)
with open(subCSVFilename, "rU") as myDataFile:
header = myDataFile.readline().rstrip()
outKeys_h = header.split(",")
outValues_h = [[] for a in range(len(outKeys_h))]
for lineNumber, line in enumerate(myDataFile.readlines(), 2):
lineList = line.rstrip().split(",")
for i in range(len(outKeys_h)):
datum = utils.partialEval(lineList[i])
if datum == '':
self.raiseAnError(
IOError,
'Invalid data in input file: {} at line {}: "{}"'
.format(subCSVFilename, lineNumber,
line.rstrip()))
outValues_h[i].append(datum)
myDataFile.close()
outKeys.append(outKeys_h)
outValues.append(outValues_h)
## Do not reset these containers because it will wipe whatever information
## already exists in this HistorySet. This is not one of the use cases for
## our data objects. We claim in the manual to construct or update
## information. These should be non-destructive operations. -- DPM 6/26/17
# self._dataContainer['inputs'] = {} #XXX these are indexed by 1,2,...
# self._dataContainer['outputs'] = {} #XXX these are indexed by 1,2,...
startKey = len(self._dataContainer['inputs'].keys())
for i in range(len(inpValues)):
mainKey = startKey + i + 1
subInput = {}
subOutput = {}
for key, value in zip(inpKeys, inpValues[i]):
#subInput[key] = c1darray(values=np.array([value]*len(outValues[0][0])))
if key in self.getParaKeys('inputs'):
subInput[key] = c1darray(values=np.array([value]))
for key, value in zip(outKeys[i], outValues[i]):
if key in self.getParaKeys('outputs'):
subOutput[key] = c1darray(values=np.array(value))
self._dataContainer['inputs'][mainKey] = subInput
self._dataContainer['outputs'][mainKey] = subOutput
#extend the expected size of this point set
self.numAdditionalLoadPoints += len(
allLines) #used in checkConsistency
self.checkConsistency()
def _readMoreXMLbase(self, xmlNode):
"""
Function to read the portion of the xml input that belongs to the base optimizer only
and initialize some stuff based on the inputs got
@ In, xmlNode, xml.etree.ElementTree.Element, Xml element node1
@ Out, None
"""
for child in xmlNode:
if child.tag == "variable":
if self.optVars == None: self.optVars = []
varname = str(child.attrib['name'])
self.optVars.append(varname)
for childChild in child:
if childChild.tag == "upperBound":
self.optVarsInit['upperBound'][varname] = float(
childChild.text)
elif childChild.tag == "lowerBound":
self.optVarsInit['lowerBound'][varname] = float(
childChild.text)
elif childChild.tag == "initial":
self.optVarsInit['initial'][varname] = {}
temp = childChild.text.split(',')
for trajInd, initVal in enumerate(temp):
try:
self.optVarsInit['initial'][varname][
trajInd] = float(initVal)
except ValueError:
self.raiseAnError(
ValueError,
"Unable to convert to float the intial value for variable "
+ varname + " in trajectory " +
str(trajInd))
if self.optTraj == None:
self.optTraj = range(
len(self.optVarsInit['initial']
[varname].keys()))
elif child.tag == "constant":
value = utils.partialEval(child.text)
if value is None:
self.raiseAnError(
IOError,
'The body of "constant" XML block should be a number. Got: '
+ child.text)
try:
self.constants[child.attrib['name']] = value
except KeyError:
self.raiseAnError(
KeyError,
child.tag + ' must have the attribute "name"!!!')
elif child.tag == "objectVar":
self.objVar = child.text
elif child.tag == "initialization":
self.initSeed = Distributions.randomIntegers(0, 2**31, self)
for childChild in child:
if childChild.tag == "limit":
self.limit['mdlEval'] = int(childChild.text)
elif childChild.tag == "type":
self.optType = childChild.text
if self.optType not in ['min', 'max']:
self.raiseAnError(
IOError, 'Unknown optimization type ' +
childChild.text + '. Available: mix or max')
elif childChild.tag == "initialSeed":
self.initSeed = int(childChild.text)
elif childChild.tag == 'thresholdTrajRemoval':
self.thresholdTrajRemoval = float(childChild.text)
else:
self.raiseAnError(
IOError, 'Unknown tag ' + childChild.tag +
' .Available: limit, type, initialSeed!')
elif child.tag == "convergence":
for childChild in child:
if childChild.tag == "iterationLimit":
self.limit['varsUpdate'] = int(childChild.text)
if childChild.tag == "absoluteThreshold":
self.absConvergenceTol = float(childChild.text)
if childChild.tag == "relativeThreshold":
self.relConvergenceTol = float(childChild.text)
elif child.tag == "restartTolerance":
self.restartTolerance = float(child.text)
elif child.tag == 'parameter':
for childChild in child:
self.paramDict[childChild.tag] = childChild.text
if self.optType == None: self.optType = 'min'
if self.thresholdTrajRemoval == None: self.thresholdTrajRemoval = 0.05
if self.initSeed == None:
self.initSeed = Distributions.randomIntegers(0, 2**31, self)
if self.objVar == None:
self.raiseAnError(
IOError, 'Object variable is not specified for optimizer!')
if self.optVars == None:
self.raiseAnError(
IOError, 'Decision variable is not specified for optimizer!')
else:
self.optVars.sort()
if self.optTraj == None: self.optTraj = [0]
for varname in self.optVars:
if varname not in self.optVarsInit['upperBound'].keys():
self.raiseAnError(
IOError, 'Upper bound for ' + varname + ' is not provided')
if varname not in self.optVarsInit['lowerBound'].keys():
self.raiseAnError(
IOError, 'Lower bound for ' + varname + ' is not provided')
if varname not in self.optVarsInit['initial'].keys():
self.optVarsInit['initial'][varname] = {}
for trajInd in self.optTraj:
self.optVarsInit['initial'][varname][trajInd] = (
self.optVarsInit['upperBound'][varname] +
self.optVarsInit['lowerBound'][varname]) / 2.0
else:
for trajInd in self.optTraj:
initVal = self.optVarsInit['initial'][varname][trajInd]
if initVal < self.optVarsInit['lowerBound'][
varname] or initVal > self.optVarsInit[
'upperBound'][varname]:
self.raiseAnError(
IOError, "The initial value for variable " +
varname + " and trajectory " + str(trajInd) +
" is outside the domain identified by the lower and upper bounds!"
)
if len(self.optTraj) != len(
self.optVarsInit['initial'][varname].keys()):
self.raiseAnError(
ValueError,
'Number of initial values does not equal to the number of parallel optimization trajectories'
)
self.optTrajLive = copy.deepcopy(self.optTraj)
def _specializedLoadXMLandCSV(self, filenameRoot, options):
"""
Function to load the xml additional file of the csv for data
(it contains metadata, etc). It must be implemented by the specialized classes
@ In, filenameRoot, string, file name root
@ In, options, dict, dictionary -> options for loading
@ Out, None
"""
#For HistorySet, create an XML file, and multiple CSV
#files. The first CSV file has a header with the input names,
#and a column for the filenames. There is one CSV file for each
#data line in the first CSV and they are named with the
#filename. They have the output names for a header, a column
#for time, and the rest of the file is data for different times.
if options is not None and 'fileToLoad' in options.keys():
name = os.path.join(options['fileToLoad'].getPath(),
options['fileToLoad'].getBase())
else:
name = self.name
filenameLocal = os.path.join(filenameRoot, name)
if os.path.isfile(filenameLocal + '.xml'):
xmlData = self._loadXMLFile(filenameLocal)
assert (xmlData["fileType"] == "HistorySet")
if "metadata" in xmlData:
self._dataContainer['metadata'] = xmlData["metadata"]
mainCSV = os.path.join(filenameRoot, xmlData["filenameCSV"])
else:
mainCSV = os.path.join(filenameRoot, name + '.csv')
myFile = open(mainCSV, "rU")
header = myFile.readline().rstrip()
inpKeys = header.split(",")[:-1]
inpValues = []
outKeys = []
outValues = []
allLines = myFile.readlines()
for mainLine in allLines:
mainLineList = mainLine.rstrip().split(",")
inpValues_h = [utils.partialEval(a) for a in mainLineList[:-1]]
inpValues.append(inpValues_h)
dataFilename = mainLineList[-1]
subCSVFilename = os.path.join(filenameRoot, dataFilename)
myDataFile = open(subCSVFilename, "rU")
subCSVFile = Files.returnInstance("CSV", self)
subCSVFile.setFilename(subCSVFilename)
self._toLoadFromList.append(subCSVFile)
header = myDataFile.readline().rstrip()
outKeys_h = header.split(",")
outValues_h = [[] for a in range(len(outKeys_h))]
for line in myDataFile.readlines():
lineList = line.rstrip().split(",")
for i in range(len(outKeys_h)):
outValues_h[i].append(utils.partialEval(lineList[i]))
myDataFile.close()
outKeys.append(outKeys_h)
outValues.append(outValues_h)
self._dataContainer['inputs'] = {} #XXX these are indexed by 1,2,...
self._dataContainer['outputs'] = {} #XXX these are indexed by 1,2,...
for i in range(len(inpValues)):
mainKey = i + 1
subInput = {}
subOutput = {}
for key, value in zip(inpKeys, inpValues[i]):
#subInput[key] = c1darray(values=np.array([value]*len(outValues[0][0])))
if key in self.getParaKeys('inputs'):
subInput[key] = c1darray(values=np.array([value]))
for key, value in zip(outKeys[i], outValues[i]):
if key in self.getParaKeys('outputs'):
subOutput[key] = c1darray(values=np.array(value))
self._dataContainer['inputs'][mainKey] = subInput
self._dataContainer['outputs'][mainKey] = subOutput
#extend the expected size of this point set
self.numAdditionalLoadPoints = len(allLines) #used in checkConsistency
self.checkConsistency()