def read(self):
'''
reading rules configuration
returns string of parsed rules delimited by System.Environment.NewLine
if the parser made any alterations the editedByParser property will be
set to True
'''
s1 = []
s = []
myParser = parser()
self.editedByParser = False
if File.Exists(self.theFile):
File.Copy(self.theFile, globalvars.BAKFILE,
True) # just in case something bad happens
s1 = File.ReadAllLines(self.theFile)
s1 = [line for line in s1 if str.Trim(line) <> '']
for line in s1:
myParser.validate(unicode(line))
if myParser.err:
self.editedByParser = True
pre = myParser.commentedLine(line)
else:
pre = line
s.Add(pre)
elif File.Exists(globalvars.SAMPLEFILE):
s = File.ReadAllLines(globalvars.SAMPLEFILE)
tmp = str('')
s = [line for line in s if str.Trim(line) <> '']
for line in s:
tmp += '%s%s' % (line, System.Environment.NewLine)
return tmp
def ReadAllLines(filePath):
lines = File.ReadAllLines(filePath)
if len(
lines
) > 0: # Workaround for a potential lack of detection of Unicode txt files.
if lines[0].Contains("\x00"):
lines = File.ReadAllLines(filePath, Encoding.Unicode)
return lines
def write(self, myKey, myVal):
'''
writes the key myKey and value myVal to the ini-file
ini file is build like this:
myKey = myValue
if the file does not exist the first call of method write creates it
'''
if File.Exists(self.theFile):
linefound = False
newConfig = []
myLines = File.ReadAllLines(self.theFile)
for line in myLines:
s = str.split(line, '=')
if str.lower(str.Trim(s[0])) == str.lower(myKey):
line = '%s = %s' % (myKey, myVal)
linefound = True
newConfig.append(line)
if linefound == False:
newConfig.append('%s = %s' % (myKey, myVal))
File.WriteAllLines(self.theFile, newConfig)
else:
File.AppendAllText(
self.theFile,
'%s = %s%s' % (myKey, myVal, System.Environment.NewLine))
return
def LoadFile(iniFilePath):
dictKeys = dict()
if File.Exists(iniFilePath):
KeyLines = File.ReadAllLines(iniFilePath)
for line in KeyLines:
if not line.startswith(';') and not line.strip() == '':
dictKeys.Add(
line.split(' = ')[0].strip(),
line.split('=')[1].strip())
return dictKeys
def groupHeaders(self, theFile=globalvars.DATFILE):
'''
returns a list of group headers in the rule set
'''
headers = []
if File.Exists(theFile):
s1 = File.ReadAllLines(theFile)
s1 = [line for line in s1 if String.StartsWith(line, '#@ GROUP')]
for line in s1:
headers.Add(String.Replace(line, '#@ GROUP ', ''))
return headers
def read(self, myKey):
'''
retrieves the value of myKey in Ini-file theFile
returns '' if key myKey was not found
'''
if File.Exists(self.theFile):
myLines = File.ReadAllLines(self.theFile)
for line in myLines:
s = str.split(unicode(line), '=')
if str.Trim(s[0]) == myKey:
return str.Trim(s[1])
return ''
def readFile(theFile):
if File.Exists(theFile):
s = File.ReadAllLines(theFile)
else:
return str('')
tmp = str('')
# s = [line for line in s if str.Trim(line) <> '']
for line in s:
tmp += '%s%s' % (line, System.Environment.NewLine)
if len(s) == 0 and theFile == globalvars.LOGFILE:
tmp = 'Your criteria matched no book. No data was touched by the Data Manager.'
return tmp
def _runscript():
try:
# loading config
iniFile = Path.ChangeExtension(sys.argv[0],".ini")
configLines = File.ReadAllLines(iniFile)
config = {}
for line in configLines:
kv = line.split("=")
config[kv[0].upper()] = kv[1]
print "construct"
app = Application()
print "set connection"
app.SetConnection("Host=" + config["HOST"]+";Database="+config["DATABASE"]+";port="+config["PORT"]+";dbflavour="+ config["DBFLAVOUR"]);
print "login"
app.Login(config["USER"],config["PW"],config["WORKSPACE"]);
print "startup"
app.StartUp();
print "ready"
scriptPath = sys.argv[1]
print "scriptPath=" + scriptPath
scr = app.Modules.Get("Script Manager")
script = scr.ScriptList.Fetch(scriptPath)
print "Setting up parameters ... "
dictParam = Dictionary[String, Object]()
for i in range(2,len(sys.argv)):
kv = sys.argv[i].split('=')
k = kv[0].strip()
v = kv[1].strip()
if v.startswith('"') and v.endswith('"'): v = v[1:-1]
print k,"=",v
dictParam.Add(k,v)
print "Running " + scr.ScriptList.GetEntityDescriptor(script)
output = clr.Reference[String]()
# avoid conflicting overloaded versions of ExecuteScript by supplying arguments by name
result = scr.ExecuteScript(script=script, arguments=dictParam, tag=None, output=output)
print "result=",result
print "output:"
print output.Value
print "done"
except Exception as e:
print "ERROR"
print str(e)
def replaceData(books):
ERROR_LEVEL = 0
if not crVersion(): return # ComicRack version ok?
s = File.ReadAllLines(globalvars.DATFILE)
if not s[0].startswith('#@ VERSION'):
MessageBox.Show(
'Your configuration needs conversion.\nPlease use the configurator first.',
'Data Manager %s' % globalvars.VERSION)
return
ini = dmutils.iniFile(globalvars.USERINI)
if ini.read('ShowStartupDialog') == 'False':
pass
else:
theForm = startupForm()
theForm.ShowDialog()
theForm.Dispose()
if theForm.DialogResult == DialogResult.Yes: # closed with Yes button
pass
elif theForm.DialogResult == DialogResult.Cancel: # closed with window close button
return
elif theForm.DialogResult == DialogResult.No: # closed with No button
return
elif theForm.DialogResult == DialogResult.Retry: # closed with configure button
dmConfig()
return
try: # delete temporary files from last data manager run
File.Delete(globalvars.TMPFILE)
File.Delete(globalvars.ERRFILE)
File.Delete(globalvars.LOGFILE)
except Exception, err:
MessageBox.Show(
'One of the temporary files of the Data Manager could not be deleted.\nPlease restart ComicRack.'
)
return
def RunICM(Adapter):
"""
<Script>
<Author>ANK</Author>
<Description>run an ICM model</Description>
<Parameters>
<Parameter name="Adapter" type="IRuntimeAdapter">handle to the adapter</Parameter>
</Parameters>
</Script>
"""
tsMgr = app.Modules.Get("Time series Manager")
# get the adapter root folder
rootfolder = Adapter.RootFolderPath
# convert dfs0-input files to Dummy_Inflow.csv
# open dummy inflow and read all lines
inflowPath = Path.Combine(rootfolder, "MODEL_SETUP\\import\\Dummy Inflow.csv")
inflowLines = File.ReadAllLines(inflowPath)
# read dfs0-files one by one
l=8
tslist = []
while inflowLines[l].split(",")[0]!="P_DATETIME" :
tsname = inflowLines[l].split(",")[0]
tsFile = Path.Combine(rootfolder, "INPUT_DATA", tsname+".dfs0")
tslist.extend(TsUtilities.ReadFromDFS0(tsMgr, tsFile))
l = l + 1;
# make new lines
newLines = []
i=0
while True:
if i==6:
# replace startdate
line = tslist[0].Start.ToString("dd/MM/yyyy HH:mm:ss") + inflowLines[i][19:]
else:
line = inflowLines[i]
newLines.append(line)
if inflowLines[i].split(",")[0]=="P_DATETIME" :
break;
i = i + 1;
# continue with timesteps
for t in range(tslist[0].Count):
line = tslist[0].Get(t).XValue.ToString("dd/MM/yyyy HH:mm:ss")
for ds in tslist:
line += "," + ds.Get(t).YValue.ToString(CultureInfo.InvariantCulture)
newLines.append(line)
# rewrite the input file
File.WriteAllLines(inflowPath, newLines)
# run the adapter bat-file
startInfo = ProcessStartInfo()
startInfo.CreateNoWindow = False;
startInfo.UseShellExecute = False;
startInfo.FileName = Path.Combine(rootfolder, "MODEL_SETUP", "run.bat");
startInfo.WorkingDirectory = Path.Combine(rootfolder, "MODEL_SETUP")
with Process.Start(startInfo) as exeProcess :
exeProcess.WaitForExit();
# convert exported csv-files to dfs0
# convert node depths
dslist = []
for fil in Directory.GetFiles(Path.Combine(rootfolder,"MODEL_SETUP\\export"),"Node_*_depnod.csv"):
lines = File.ReadAllLines(fil)
headers = lines[0].split(",")
for h in headers[2:]:
ds = tsMgr.TimeSeriesList.CreateNew()
ds.Name = h.strip()
ds.YAxisVariable = "Water Level"
dslist.append(ds)
for line in lines[1:] :
if not "[" in line:
values = line.split(",")
t = DateTime.ParseExact(values[0].strip(),"dd/MM/yyyy HH:mm:ss", CultureInfo.InvariantCulture)
for v in range(2,values.Count):
ds = dslist[v-2]
vp = ds.CreateNew()
vp.XValue = t
vp.YValue = Double.Parse(values[v].strip(), CultureInfo.InvariantCulture)
ds.Add(vp)
# save to dfs0
for ds in dslist:
fil = Path.Combine(rootfolder,"OUTPUT_DATA",ds.Name+".dfs0")
TsUtilities.ExportToDfs0(ds,fil)
def ReadFromCSVFile(csvFilePath, delimiter, encoding):
lines = File.ReadAllLines(csvFilePath, encoding)
rows = [[value for value in line.Split(delimiter)] for line in lines]
return rows
def BackgroundWorker1DoWork(self, sender, e):
theLog = ''
parserErrors = 0
if self.theProcess == 0: # just for testing
i = 0
while i <= 100:
i += 1
# Report progress to 'UI' thread
self._backgroundWorker1.ReportProgress(i)
# Simulate long task
System.Threading.Thread.Sleep(100)
return
# ------------------------------------------------------
# run the parsed code over the books:
# ------------------------------------------------------
userIni = iniFile(globalvars.USERINI)
dtStarted = System.DateTime.Now
if self.theProcess == globalvars.PROCESS_BOOKS:
self.maxVal = self.theBooks.Length
self._progressBar.Maximum = self.maxVal
self._progressBar.Step = 1
f = open(globalvars.LOGFILE, "w") # open logfile
s = File.ReadAllLines(globalvars.DATFILE)
if not s[0].startswith('#@ VERSION'):
MessageBox.Show(
'The configuration needs to be converted. Please start the configurator first.'
)
return
lines = []
for line in s:
if line == '#@ END_RULES': break
if (not line.startswith('#')) and line.strip() <> '':
lines.append(line)
# no books or values were touched until now:
allBooksTouched = 0
allValuesChanged = 0
for book in self.theBooks:
if not self._backgroundWorker1.CancellationPending:
self.stepsPerformed += 1
self._backgroundWorker1.ReportProgress(
self.stepsPerformed / self.maxVal * 100)
# we use this for custom fields only because some other fields
# might not be stored by book.Clone():
theOriginalBook = book.Clone(
) # to retrieve the original values later
# for the remaining we use:
for field in self.allVals:
if field <> 'Custom':
setattr(self, 'field_' + field,
getattr(book, field))
for line in lines:
thisBookTouched = 0 # for this book no value have changed yet
if self.stop_the_Worker == True: break
try:
if dmParser.matchAllRules(line, book):
if thisBookTouched == 0:
allBooksTouched += 1 # increment counter for books that have been touched
thisBookTouched += 1 # mark that this book has been touched
book.SetCustomValue(
'DataManager.processed',
System.DateTime.ToString(
System.DateTime.Now,
'yyyy-MM-dd HH:mm:ss'))
joinChar = ' ' + dmParser.ruleMode + ' '
# if LogBookOnlyWhenValuesChanged <> True write touched book
# to log even when no value have changed:
if userIni.read('LogBookOnlyWhenValuesChanged'
) <> 'True':
theLog += '%s (%s) #%s was touched\t[%s]\n' % (
unicode(book.Series), book.Volume,
unicode(book.Number),
joinChar.join(dmParser.rules))
dmParser.executeAllActions(book)
theLog += dmParser.theLog
parserErrors += dmParser.errCount
for fieldTouched in dmParser.fieldsTouched:
if fieldTouched.startswith('Custom'):
cField = fieldTouched.replace(
'Custom(', '').strip(')')
beforeTouch = theOriginalBook.GetCustomValue(
cField)
afterTouch = book.GetCustomValue(
cField)
else:
beforeTouch = getattr(
self, 'field_' + fieldTouched)
afterTouch = getattr(
book, fieldTouched)
if afterTouch <> beforeTouch:
# if LogBookOnlyWhenValuesChanged is True and this book was touched for the first time:
if userIni.read(
'LogBookOnlyWhenValuesChanged'
) == 'True' and thisBookTouched == 1:
theLog += '%s (%s) #%s was touched\t[%s]\n' % (
unicode(
book.Series), book.Volume,
unicode(book.Number),
joinChar.join(dmParser.rules))
theLog += '\t%s old: %s\n' % (
fieldTouched, unicode(beforeTouch))
theLog += '\t%s new: %s\n' % (
fieldTouched, unicode(afterTouch))
allValuesChanged += 1
if parserErrors > 0 and userIni.read(
"BreakAfterFirstError") == 'True':
self.stop_the_Worker = True
break
except Exception, err:
print str(Exception.args)
MessageBox.Show(
'An unhandled error occurred while executing the rules. \n%s\nPlease check your rule: %s'
% (str(err), line),
'Data Manager - Version %s' %
globalvars.VERSION)
self.errorLevel = 1
self.stop_the_Worker = True
break
else:
theLog += ('\n\nExcecution cancelled by user.')
self.cancelledByUser = True
break
if self.stop_the_Worker == True: break
if allBooksTouched > 0:
theLog += '\n\n' + '*' * 60 + '\n'
theLog += '%d books were touched, %d values were changed\n' % (
allBooksTouched, allValuesChanged)
theLog += '*' * 60 + '\n'
dtEnded = System.DateTime.Now
dtDuration = dtEnded - dtStarted
userIni.write('ParserStarted', str(dtStarted))
userIni.write('ParserEnded', str(dtEnded))
userIni.write('ParserDuration', str(dtDuration))
if parserErrors > 0:
MessageBox.Show(
'There were errors in your rules. You really should check the logfile!'
)
f.write(theLog)
f.close()
def _get_file(self, filename):
filename = Path.GetFullPath(filename)
if not filename in self.source_files:
self.source_files[filename] = File.ReadAllLines(filename)
return self.source_files[filename]
