def doprocesstest(self):
self.processtestclear.emit()
print("Processtest!")
testinfodict = {
str(k): str(v)
for k, v in self.test.info._asdict().items()
}
debug(testinfodict)
# debug(self.fs)
shallowfs = copy.copy(self.fs)
shallowfs._testinfo = None
self.test.timer = QTimer(self._parent)
self.test.timer.timeout.connect(self.doprocessorupdate)
# self.test.timer.start(.200)
self.test.timer.setInterval(0.2 * 1000)
self.test.timer.start()
stdOutLog, stdErrLog = self.test.logs
stdOutLog.reset(), stdErrLog.reset()
self.test.queues = stdOutLog.path, stdErrLog.path
debug(self.test.logs)
# guiimportraws.importrawsdialog(self.test, projectfolder=self.fs, parent=self)
testconfmethods = [(list(m.keys())[0], True)
for m in self.fs["projdesc"]["methods"]]
testconfmethods_results = forms.fedit(
testconfmethods, comment="<h3>Methods to run:</h3>")
skip_methods = ','.join([
methodname
for ((methodname, _),
result) in zip(testconfmethods, testconfmethods_results)
if result == False
])
self.args.options["dataprocessor", "testconfs"][
"skip_methods"] = skip_methods + "," + self.args.options[
"dataprocessor", "testconfs"].get("skip_methods", "")
print("Skipping: ", self.args.options["dataprocessor", "testconfs",
"skip_methods"])
print("starting queue: ", self.test.queues[0])
processargs = DataTree(testinfodict=testinfodict,
fs=shallowfs,
args=self.args,
logFileNames=self.test.queues)
self.pool.map_async(_process, [processargs])
def docreatenewtest(self):
def getvalues():
return json.loads(
self._parent.settings.value("dialog/createdialog", "{}"))
def setvalues(values):
self._parent.settings.setValue("dialog/createdialog",
json.dumps(values))
TestInfo = self.fs._testinfo
fieldnames = TestInfo._fields
regexes = TestInfo._regexfields
priorvalues = getvalues()
fielddata = [("<b>%s</b> [<i>%s</i>]" % (name, regexes[name].pattern),
priorvalues.get(name, "")) for name in fieldnames]
datalist = forms.fedit(fielddata)
# debug(datalist)
if not fielddata:
return
datadict = DataTree({k: v for k, v in zip(fieldnames, datalist)})
setvalues(datadict)
try:
# debug(datadict)
ti = TestInfo(**TestInfo.createfields(valuedict=datadict))
print(repr(ti))
tf = self.fs.makenewfolder(**ti._asdict())
self.projectrefresh.emit()
# TODO: emit testitem changed with new test name?
self._parent.testitemchanged.emit(DataTree(folder=tf.main,
test=ti))
except Exception as err:
logging.exception(err)
self.showErrorMessage("Error parsing: ", ex=traceback.format_exc())
raise err
def docreatenewtest(self):
def getvalues():
return json.loads(self._parent.settings.value("dialog/createdialog", "{}"))
def setvalues(values):
self._parent.settings.setValue("dialog/createdialog", json.dumps(values))
TestInfo = self.fs._testinfo
fieldnames = TestInfo._fields
regexes = TestInfo._regexfields
priorvalues = getvalues()
fielddata = [ ("<b>%s</b> [<i>%s</i>]"%(name, regexes[name].pattern), priorvalues.get(name,""))
for name in fieldnames ]
datalist = forms.fedit(fielddata)
# debug(datalist)
if not fielddata:
return
datadict = DataTree({ k:v for k,v in zip(fieldnames, datalist) })
setvalues(datadict)
try:
# debug(datadict)
ti = TestInfo(**TestInfo.createfields(valuedict=datadict))
print(repr(ti))
tf = self.fs.makenewfolder(**ti._asdict())
self.projectrefresh.emit()
# TODO: emit testitem changed with new test name?
self._parent.testitemchanged.emit(DataTree(folder=tf.main, test=ti))
except Exception as err:
logging.exception(err)
self.showErrorMessage("Error parsing: ",
ex=traceback.format_exc())
raise err
def getargs(self):
options = self.defaultoptions()
# Convert options to forms data format (lists of data, postfixed with name of tab)
datalist = [([(k, v) for k, v in sorted(flatten(v, ).items())], k,
"Options for %s" % str(k))
for k, v in sorted(options.items())]
print(datalist)
for i, j in enumerate(datalist):
print("i:%s: `%s`" % (i, j))
updatedargs = forms.fedit(datalist)
if not updatedargs:
return
# debug(updatedargs)
# join data down into groups
for dgroup, ugroup in zip(datalist, updatedargs):
dgroup_name, dgroup_comment = dgroup[1:]
for (dkey, ditem), uitem in zip(dgroup[0], ugroup):
# print("{dgroup_name}.{dkey}.{ditem} -> {dgroup_name}.{dkey}.{uitem} ".format(**locals()))
key = (dgroup_name, ) + tuple(dkey.split("."))
# debug(key)
options[key[:-1]][key[-1]] = uitem
# debug(options)
def setvalues(values):
self._parent.settings.setValue("dialog/optionsdialog",
json.dumps(values))
setvalues(options)
args = DataTree()
args.options = options
self.args = args
return args
def getargs(self):
options = self.defaultoptions()
# Convert options to forms data format (lists of data, postfixed with name of tab)
datalist = [ ( [ (k,v) for k,v in sorted(flatten(v, ).items()) ], k, "Options for %s"%str(k) )
for k,v in sorted(options.items()) ]
print(datalist)
for i,j in enumerate(datalist):
print("i:%s: `%s`"%(i, j))
updatedargs = forms.fedit(datalist)
if not updatedargs:
return
# debug(updatedargs)
# join data down into groups
for dgroup, ugroup in zip(datalist, updatedargs):
dgroup_name, dgroup_comment = dgroup[1:]
for (dkey, ditem), uitem in zip(dgroup[0], ugroup):
# print("{dgroup_name}.{dkey}.{ditem} -> {dgroup_name}.{dkey}.{uitem} ".format(**locals()))
key = (dgroup_name,) + tuple(dkey.split("."))
# debug(key)
options[key[:-1]][key[-1]] = uitem
# debug(options)
def setvalues(values):
self._parent.settings.setValue("dialog/optionsdialog", json.dumps(values))
setvalues(options)
args = DataTree()
args.options = options
self.args = args
return args
def doprocesstest(self):
self.processtestclear.emit()
print("Processtest!")
testinfodict = { str(k): str(v) for k,v in self.test.info._asdict().items() }
debug(testinfodict)
# debug(self.fs)
shallowfs = copy.copy(self.fs)
shallowfs._testinfo = None
self.test.timer = QTimer(self._parent)
self.test.timer.timeout.connect(self.doprocessorupdate)
# self.test.timer.start(.200)
self.test.timer.setInterval(0.2*1000)
self.test.timer.start()
stdOutLog, stdErrLog = self.test.logs
stdOutLog.reset(), stdErrLog.reset()
self.test.queues = stdOutLog.path, stdErrLog.path
debug(self.test.logs)
# guiimportraws.importrawsdialog(self.test, projectfolder=self.fs, parent=self)
testconfmethods = [ (list(m.keys())[0], True) for m in self.fs["projdesc"]["methods"] ]
testconfmethods_results = forms.fedit(testconfmethods, comment="<h3>Methods to run:</h3>")
skip_methods = ','.join([ methodname for ((methodname, _), result) in zip(testconfmethods, testconfmethods_results) if result==False ])
self.args.options["dataprocessor", "testconfs"]["skip_methods"] = skip_methods + ","+self.args.options["dataprocessor", "testconfs"].get("skip_methods","")
print("Skipping: ", self.args.options["dataprocessor", "testconfs", "skip_methods"])
print("starting queue: ", self.test.queues[0])
processargs = DataTree(testinfodict=testinfodict, fs=shallowfs, args=self.args, logFileNames=self.test.queues)
self.pool.map_async(_process, [processargs])
def showFileDialog(inputchoices, parent, **kwargs):
testrawdir = inputchoices["testraws"]["value"]
isfile = inputchoices["File?"]
if isfile:
projrawdir = QFileDialog.getOpenFileName(
parent._parent,
'Choose Raws File [%s]' % (inputchoices["projectraws"]["key"]),
str())
else:
projrawdir = QFileDialog.getExistingDirectory(
parent._parent, 'Choose Raws Folder [%s]' %
(inputchoices["projectraws"]["key"]),
str(inputchoices["projectraws"]["value"]))
#debug(testrawdir, projrawdir)
tgt_testrawtgt, src_projrawdir = Path(testrawdir), Path(
projrawdir).resolve()
#debug(str(tgt_testrawtgt), str(src_projrawdir))
if not tgt_testrawtgt or not src_projrawdir:
return
results = forms.fedit(
[('Input', [0, 'Yes', 'No'])],
title="Copy Raw Files",
comment="""
<h4>Copying:</h4>
<table>
<tr><td><b>From Directory:</b></td><td><small>{srcb}</small></td></tr> \n
<tr><td><b>From Name:</b></td><td>{srcn}</td></tr> \n
<tr><td><b>To Directory:</b></td><td><small>{tgtb}</small></td></tr> \n
<tr><td><b>To Name:</b></td><td>{tgtn}</td></tr> \n
""".format(srcb=str(src_projrawdir.parent),
tgtb=str(tgt_testrawtgt.parent),
srcn=str(src_projrawdir.name),
tgtn=str(tgt_testrawtgt.name)),
# apply=apply_dialog,
)
#debug(results)
if not results:
forms.fedit([], comment="Caneled import. ")
return
try:
# if tgt_testrawtgt.exists():
# print("Deleting")
# shutil.rmtree(str(tgt_testrawtgt))
# shutil.copytree(src=str(src_projrawdir), dst=str(tgt_testrawtgt), ignore_dangling_symlinks=True)
src_projrawdir_files = list(
src_projrawdir.glob("*")) if src_projrawdir.is_dir() else [
src_projrawdir
]
for src_file in src_projrawdir_files:
print("Copying {src} -> {dst} ".format(
src=str(src_projrawdir),
dst=str(tgt_testrawtgt).encode('utf-8')))
shutil.copy(src=str(src_file), dst=str(tgt_testrawtgt))
forms.fedit(
[],
comment=
"Done importing! Files copied:<br>\n <table>{files}</table>".
format(files='\n'.join("<tr><td>{}</td></tr>".format(f)
for f in src_projrawdir_files)))
except Exception as err:
ex = traceback.format_exc()
print(ex)
parent.showErrorMessage("Error copying files: `%s` -> `%s`" %
(str(tgt_testrawtgt), str(src_projrawdir)),
ex=ex)
def importrawsdialog(test, **kwargs):
import datetime
print("[Import Raws]")
testraws = test.folder["raws"]
projectraws = kwargs["projectfolder"]["raws"]
testfiles = {
k: v
for k, v in kwargs["projectfolder"]["projdesc"]["experiment_config"]
["testfolder"]["filestructure"].items() if "images" in k
}
testimages = kwargs["projectfolder"]["projdesc"]["experiment_config"][
"testfolder"]["files"]
#debug(list(test.keys()))
#debug(testraws)
#debug(kwargs)
getnames = lambda d: [k for k in sorted(d.keys())]
testrawsnames = getnames(testraws) + getnames(testimages)
projectrawsnames = getnames(projectraws) + getnames(testfiles)
testraws.update(testimages)
projectraws.update(testfiles)
rawsdatainput = [('Project Raws', [0] + projectrawsnames),
('Test Raw', [0] + testrawsnames), ('File?', False)]
def showFileDialog(inputchoices, parent, **kwargs):
testrawdir = inputchoices["testraws"]["value"]
isfile = inputchoices["File?"]
if isfile:
projrawdir = QFileDialog.getOpenFileName(
parent._parent,
'Choose Raws File [%s]' % (inputchoices["projectraws"]["key"]),
str())
else:
projrawdir = QFileDialog.getExistingDirectory(
parent._parent, 'Choose Raws Folder [%s]' %
(inputchoices["projectraws"]["key"]),
str(inputchoices["projectraws"]["value"]))
#debug(testrawdir, projrawdir)
tgt_testrawtgt, src_projrawdir = Path(testrawdir), Path(
projrawdir).resolve()
#debug(str(tgt_testrawtgt), str(src_projrawdir))
if not tgt_testrawtgt or not src_projrawdir:
return
results = forms.fedit(
[('Input', [0, 'Yes', 'No'])],
title="Copy Raw Files",
comment="""
<h4>Copying:</h4>
<table>
<tr><td><b>From Directory:</b></td><td><small>{srcb}</small></td></tr> \n
<tr><td><b>From Name:</b></td><td>{srcn}</td></tr> \n
<tr><td><b>To Directory:</b></td><td><small>{tgtb}</small></td></tr> \n
<tr><td><b>To Name:</b></td><td>{tgtn}</td></tr> \n
""".format(srcb=str(src_projrawdir.parent),
tgtb=str(tgt_testrawtgt.parent),
srcn=str(src_projrawdir.name),
tgtn=str(tgt_testrawtgt.name)),
# apply=apply_dialog,
)
#debug(results)
if not results:
forms.fedit([], comment="Caneled import. ")
return
try:
# if tgt_testrawtgt.exists():
# print("Deleting")
# shutil.rmtree(str(tgt_testrawtgt))
# shutil.copytree(src=str(src_projrawdir), dst=str(tgt_testrawtgt), ignore_dangling_symlinks=True)
src_projrawdir_files = list(
src_projrawdir.glob("*")) if src_projrawdir.is_dir() else [
src_projrawdir
]
for src_file in src_projrawdir_files:
print("Copying {src} -> {dst} ".format(
src=str(src_projrawdir),
dst=str(tgt_testrawtgt).encode('utf-8')))
shutil.copy(src=str(src_file), dst=str(tgt_testrawtgt))
forms.fedit(
[],
comment=
"Done importing! Files copied:<br>\n <table>{files}</table>".
format(files='\n'.join("<tr><td>{}</td></tr>".format(f)
for f in src_projrawdir_files)))
except Exception as err:
ex = traceback.format_exc()
print(ex)
parent.showErrorMessage("Error copying files: `%s` -> `%s`" %
(str(tgt_testrawtgt), str(src_projrawdir)),
ex=ex)
def apply_dialog(data):
print("[[apply_dialog]]")
#debug(data)
projectrawsidx, testrawsidx, isfile = data
inputchoices = DataTree()
inputchoices['testraws'] = DataTree(
key=testrawsnames[testrawsidx],
value=testraws[testrawsnames[testrawsidx]])
inputchoices['projectraws'] = DataTree(
key=projectrawsnames[projectrawsidx],
value=projectraws[projectrawsnames[projectrawsidx]])
inputchoices['File?'] = isfile
debug(inputchoices)
showFileDialog(inputchoices, **kwargs)
# rawsdata = forms.fedit(rawsdatainput)
results = forms.fedit(
rawsdatainput,
title="Copy Raw Files",
# comment="Choose input raws folder and the output test-raw:",
# apply=apply_dialog,
)
if not results:
return
apply_dialog(results)