class TestIteratorMap:
def __init__(self, dynamic, allApps):
self.dict = OrderedDict()
self.dynamic = dynamic
self.parentApps = {}
for app in allApps:
for extra in [ app ] + app.extras:
self.parentApps[extra] = app
def getKey(self, test):
if self.dynamic:
return test
elif test is not None:
return self.parentApps.get(test.app, test.app), test.getRelPath()
def store(self, test, iter):
self.dict[self.getKey(test)] = iter
def updateIterator(self, test, oldRelPath):
# relative path of test has changed
key = self.parentApps.get(test.app, test.app), oldRelPath
iter = self.dict.get(key)
if iter is not None:
self.store(test, iter)
del self.dict[key]
return iter
else:
return self.getIterator(test)
def getIterator(self, test):
return self.dict.get(self.getKey(test))
def remove(self, test):
key = self.getKey(test)
if self.dict.has_key(key):
del self.dict[key]
def run(self):
results = OrderedDict()
for key, qs in self.get_responsesets().items():
if isinstance(qs, list):
if len(qs) == 0:
results[key] = plugins.Vector([])
continue
results[key] = self.scorecard.get_values(qs)
if isinstance(qs,dict):
result = []
for entity, data in qs.items():
result.append((entity, self.scorecard.get_values(data)))
results[key] = plugins.Vector(result)
if self.reverse_axis:
if self.aggregate_on and self.aggregate_by_entity:
for entity, data in results.items():
flipped = OrderedDict()
for key, scorecard in data.get_values():
for operation, values in scorecard:
flipped[operation] = flipped.get(operation, [])
flipped[operation].append((key, values))
results[entity] = plugins.Vector(flipped.items())
else:
flipped = OrderedDict()
for key, scorecard in results.items():
for operation, values in scorecard:
flipped[operation] = flipped.get(operation, [])
flipped[operation].append((key, values))
results = flipped
return results
def info(muxed_data):
"""
Returns a list of information derived from the data returned by
the `feed_mux` function. Useful for templates.
"""
x = OrderedDict({'subscribed': 0, 'unsubscribed': 0})
# {'grouper': [{'key': val, 'key2': val2}, {'key': ...}]}
for feed_info in muxed_data:
for key in ['total', 'total_read', 'total_unread']:
x[key.replace('_', ' ')] = x.get(key, 0) + feed_info[key]
key = 'subscribed' if feed_info['subscribed'] else 'unsubscribed'
x[key] = x.get(key, 0) + 1
x['can_subscribe_all'] = x['unsubscribed'] != 0
x['can_unsubscribe_all'] = x['subscribed'] != 0
return x
class ActionRunner(BaseActionRunner):
def __init__(self, optionMap, *args):
BaseActionRunner.__init__(self, optionMap, logging.getLogger("Action Runner"))
self.currentTestRunner = None
self.previousTestRunner = None
self.appRunners = OrderedDict()
def addSuite(self, suite):
plugins.log.info("Using " + suite.app.description(includeCheckout=True))
appRunner = ApplicationRunner(suite, self.diag)
self.appRunners[suite.app] = appRunner
def notifyAllReadAndNotified(self):
# kicks off processing. Don't use notifyAllRead as we end up running all the tests before
# everyone's been notified of the reading.
self.runAllTests()
def notifyRerun(self, test):
if self.currentTestRunner and self.currentTestRunner.test is test:
self.diag.info("Got rerun notification for " + repr(test) + ", resetting actions")
self.currentTestRunner.resetActionSequence()
def runTest(self, test):
# We have the lock coming in to here...
appRunner = self.appRunners.get(test.app)
if appRunner:
self.lock.acquire()
self.currentTestRunner = TestRunner(test, appRunner, self.diag, self.exited, self.killSignal)
self.lock.release()
self.currentTestRunner.performActions(self.previousTestRunner)
self.previousTestRunner = self.currentTestRunner
self.lock.acquire()
self.currentTestRunner = None
self.notifyComplete(test)
self.lock.release()
def killTests(self):
if self.currentTestRunner:
self.currentTestRunner.kill(self.killSignal)
def killOrCancel(self, test):
if self.currentTestRunner and self.currentTestRunner.test is test:
self.currentTestRunner.kill()
else:
self.cancel(test)
def getAllActionClasses(self):
classes = set()
for appRunner in self.appRunners.values():
for action in appRunner.actionSequence:
classes.add(action.__class__)
return classes
def cleanup(self):
for actionClass in self.getAllActionClasses():
actionClass.finalise()
for appRunner in self.appRunners.values():
appRunner.cleanActions()
def hook(pairs):
""" Method for simplejson object_pairs_hook
pairs is ordered list of key value duples
"""
dct = OrderedDict(pairs)
hint = dct.get('@class')
for cls in classes:
if cls.__name__ == hint:
return brinify(cls,
propertied=propertied,
safed=safed,
hinted=hinted,
extendable=extendable)()._update(dct)
return dct
class TestIteratorMap:
def __init__(self, dynamic, allApps):
self.dict = OrderedDict()
self.dynamic = dynamic
self.parentApps = {}
for app in allApps:
for extra in [app] + app.extras:
self.parentApps[extra] = app
def getKey(self, test):
if self.dynamic:
return test
elif test is not None:
return self.parentApps.get(test.app, test.app), test.getRelPath()
def store(self, test, iter):
self.dict[self.getKey(test)] = iter
def updateIterator(self, test, oldRelPath):
# relative path of test has changed
key = self.parentApps.get(test.app, test.app), oldRelPath
iter = self.dict.get(key)
if iter is not None:
self.store(test, iter)
del self.dict[key]
return iter
else:
return self.getIterator(test)
def getIterator(self, test):
return self.dict.get(self.getKey(test))
def remove(self, test):
key = self.getKey(test)
if self.dict.has_key(key):
del self.dict[key]
def hook(pairs):
""" Method for simplejson object_pairs_hook
pairs is ordered list of key value duples
"""
dct = OrderedDict(pairs)
hint = dct.get('@class')
for cls in classes:
if cls.__name__ == hint:
return brinify( cls,
propertied=propertied,
safed=safed,
hinted=hinted,
extendable=extendable)()._update(dct)
return dct
class DictCache(AbstractCache):
"""Implementation of a cache in a memory dictionary
"""
def __init__(self, ttl=60):
"""Creates a new instance
params:
``ttl``
Time to live of the data.
"""
super(DictCache, self).__init__(ttl=ttl)
try:
self._ich = collections.OrderedDict()
self._ttds = collections.OrderedDict()
except AttributeError:
#This version of python does not support OrderedDict
from ordereddict import OrderedDict
self._ich = OrderedDict()
self._ttds = OrderedDict()
def store_data(self, k, ttl, v):
self._ich[k] = v
self._ttds[k] = (
time.time() + ttl if ttl != None else None
)
def retrieve_data(self, k):
ttd = self._ttds.get(k, 0)
if ttd == None or time.time() < ttd:
return self._ich[k]
elif ttd:
self._ttds.pop(k)
self._ich.pop(k)
def clear_expired(self):
for k, ttd in self._ttds.items():
if ttd != None and ttd < time.time():
self._ttds.pop(k)
self._ich.pop(k)
else:
break
def clear(self):
self._ich.clear()
self._ttds.clear()
def check_unique_columns(self):
"""Check columns which should contain unique values
actually do."""
for colhead in self.UNIQUES:
col = self.HEADINGS.index(colhead)
rowsdata = [(i + self.HEADING_ROW, c.value) for i, c in \
enumerate(self.sheet.col_slice(
col, self.HEADING_ROW, self.sheet.nrows))]
datarows = OrderedDict()
for i, key in rowsdata:
item = datarows.get(key)
# don't count empty fields
if key is None or key == "":
continue
if item is None:
datarows[key] = [i]
else:
datarows[key].append(i)
for key, rows in datarows.iteritems():
if len(rows) > 1:
header = self.sheet.cell(self.HEADING_ROW, col).value
self.add_error(
rows[0], "Duplicate on unique column: %s: '%s' %s" % (
header, key, [r+1 for r in rows[1:]]))
class QueueSystemServer(BaseActionRunner):
instance = None
def __init__(self, optionMap, allApps):
BaseActionRunner.__init__(self, optionMap, logging.getLogger("Queue System Submit"))
# queue for putting tests when we couldn't reuse the originals
self.reuseFailureQueue = Queue()
self.testCount = 0
self.testsSubmitted = 0
self.maxCapacity = 100000 # infinity, sort of
self.allApps = allApps
for app in allApps:
currCap = app.getConfigValue("queue_system_max_capacity")
if currCap is not None and currCap < self.maxCapacity:
self.maxCapacity = currCap
self.jobs = OrderedDict()
self.submissionRules = {}
self.killedJobs = {}
self.queueSystems = {}
self.reuseOnly = False
self.submitAddress = None
self.slaveLogDirs = set()
self.delayedTestsForAdd = []
self.remainingForApp = OrderedDict()
capacityPerSuite = self.maxCapacity / len(allApps)
for app in allApps:
self.remainingForApp[app.name] = capacityPerSuite
self.getQueueSystem(app) # populate cache
QueueSystemServer.instance = self
def addSuites(self, suites):
for suite in suites:
self.slaveLogDirs.add(suite.app.makeWriteDirectory("slavelogs"))
plugins.log.info("Using " + queueSystemName(suite.app) + " queues for " +
suite.app.description(includeCheckout=True))
def setSlaveServerAddress(self, address):
self.submitAddress = os.getenv("CAPTUREMOCK_SERVER", address)
self.testQueue.put("TextTest slave server started on " + address)
def addTest(self, test):
capacityForApp = self.remainingForApp[test.app.name]
if capacityForApp > 0:
self.addTestToQueues(test)
self.remainingForApp[test.app.name] = capacityForApp - 1
else:
if test.app.name == self.remainingForApp.keys()[-1]:
self.addTestToQueues(test) # For the last app (which may be the only one) there is no point in delaying
else:
self.delayedTestsForAdd.append(test)
def addTestToQueues(self, test):
self.testCount += 1
queue = self.findQueueForTest(test)
if queue:
queue.put(test)
def addDelayedTests(self):
for test in self.delayedTestsForAdd:
self.addTestToQueues(test)
self.delayedTestsForAdd = []
def notifyAllRead(self, suites):
self.addDelayedTests()
BaseActionRunner.notifyAllRead(self, suites)
def run(self): # picked up by core to indicate running in a thread
self.runAllTests()
if len(self.jobs):
self.diag.info("All jobs submitted, polling the queue system now.")
if self.canPoll():
self.pollQueueSystem()
def pollQueueSystem(self):
# Start by polling after 5 seconds, ever after try every 15
attempts = int(os.getenv("TEXTTEST_QS_POLL_WAIT", "5")) * 2 # Amount of time to wait before initiating polling of SGE
if attempts >= 0:
while True:
for i in range(attempts):
time.sleep(0.5)
if self.allComplete or self.exited:
return
self.updateJobStatus()
attempts = 30
# In case any tests have had reruns triggered since we stopped submitting
self.runQueue(self.getTestForRun, self.runTest, "rerunning", block=False)
def canPoll(self):
queueSystem = self.getQueueSystem(self.jobs.keys()[0])
return queueSystem.supportsPolling()
def updateJobStatus(self):
queueSystem = self.getQueueSystem(self.jobs.keys()[0])
statusInfo = queueSystem.getStatusForAllJobs()
self.diag.info("Got status for all jobs : " + repr(statusInfo))
if statusInfo is not None: # queue system not available for some reason
for test, jobs in self.jobs.items():
if not test.state.isComplete():
for jobId, _ in jobs:
status = statusInfo.get(jobId)
if status and test.state.hasStarted() and test.state.briefText:
# Only do this to test jobs (might make a difference for derived configurations)
# Ignore filtering states for now, which have empty 'briefText'.
self.updateRunStatus(test, status)
elif not status and not self.jobStarted(test):
# Do this to any jobs
self.setSlaveFailed(test, False, True)
def updateRunStatus(self, test, status):
newRunStatus, newExplanation = status
newState = test.state.makeModifiedState(newRunStatus, newExplanation, "grid status update")
if newState:
test.changeState(newState)
def findQueueForTest(self, test):
# If we've gone into reuse mode and there are no active tests for reuse, use the "reuse failure queue"
if self.reuseOnly and self.testsSubmitted == 0:
self.diag.info("Putting " + test.uniqueName + " in reuse failure queue " + self.remainStr())
return self.reuseFailureQueue
else:
self.diag.info("Putting " + test.uniqueName + " in normal queue " + self.remainStr())
return self.testQueue
def handleLocalError(self, test, previouslySubmitted):
self.handleErrorState(test, previouslySubmitted)
if self.testCount == 0 or (self.reuseOnly and self.testsSubmitted == 0):
self.diag.info("Submitting terminators after local error")
self.submitTerminators()
def submitTerminators(self):
# snap out of our loop if this was the last one. Rely on others to manage the test queue
self.reuseFailureQueue.put(None)
def getTestForReuse(self, test, state, tryReuse):
# Pick up any test that matches the current one's resource requirements
if not self.exited:
# Don't allow this to use up the terminator
newTest = self.getTest(block=False, replaceTerminators=True)
if newTest:
if tryReuse and self.allowReuse(test, state, newTest):
self.jobs[newTest] = self.getJobInfo(test)
if self.testCount > 1:
self.testCount -= 1
postText = self.remainStr()
else:
# Don't allow test count to drop to 0 here, can cause race conditions
self.submitTerminators()
postText = ": submitting terminators as final test"
self.diag.info("Reusing slave from " + test.uniqueName + " for " + newTest.uniqueName + postText)
return newTest
else:
self.diag.info("Adding to reuse failure queue : " + newTest.uniqueName)
self.reuseFailureQueue.put(newTest)
else:
self.diag.info("No tests available for reuse : " + test.uniqueName)
# Allowed a submitted job to terminate
self.testsSubmitted -= 1
self.diag.info("No reuse for " + test.uniqueName + " : " + repr(self.testsSubmitted) + " tests still submitted")
if self.exited and self.testsSubmitted == 0:
self.diag.info("Forcing termination")
self.submitTerminators()
def allowReuse(self, oldTest, oldState, newTest):
# Don't reuse jobs that have been killed
if newTest.state.isComplete() or oldState.category == "killed":
return False
if oldTest.getConfigValue("queue_system_proxy_executable") or \
newTest.getConfigValue("queue_system_proxy_executable"):
return False
# Jobs maintain the same virtual display instance where possible, if they require different settings they can't be reused
if oldTest.getConfigValue("virtual_display_extra_args") != newTest.getConfigValue("virtual_display_extra_args"):
return False
oldRules = self.getSubmissionRules(oldTest)
newRules = self.getSubmissionRules(newTest)
return oldRules.allowsReuse(newRules)
def getJobSubmissionRules(self, test):
proxyRules = self.getProxySubmissionRules(test)
if proxyRules:
return proxyRules
else:
return self.getSubmissionRules(test)
def getProxySubmissionRules(self, test):
proxyResources = test.getConfigValue("queue_system_proxy_resource")
if proxyResources:
return test.app.getProxySubmissionRulesClass()(test, proxyResources)
def getSubmissionRules(self, test):
if self.submissionRules.has_key(test):
return self.submissionRules[test]
else:
submissionRules = test.app.getSubmissionRules(test)
self.submissionRules[test] = submissionRules
return submissionRules
def getTest(self, block, replaceTerminators=False):
testOrStatus = self.getItemFromQueue(self.testQueue, block, replaceTerminators)
if not testOrStatus:
return
if type(testOrStatus) == StringType:
self.sendServerState(testOrStatus)
return self.getTest(block)
else:
return testOrStatus
def sendServerState(self, state):
self.diag.info("Sending server state '" + state + "'")
mimServAddr = os.getenv("CAPTUREMOCK_SERVER")
if mimServAddr:
host, port = mimServAddr.split(":")
serverAddress = (host, int(port))
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(serverAddress)
sock.sendall("SUT_SERVER:" + state + "\n")
sock.close()
def getTestForRunNormalMode(self, block):
self.reuseOnly = False
reuseFailure = self.getItemFromQueue(self.reuseFailureQueue, block=False)
if reuseFailure:
self.diag.info("Found a reuse failure...")
return reuseFailure
else:
self.diag.info("Waiting for new tests...")
newTest = self.getTest(block=block)
if newTest:
return newTest
else:
# Make sure we pick up anything that failed in reuse while we were submitting the final test...
self.diag.info("No normal test found, checking reuse failures...")
return self.getItemFromQueue(self.reuseFailureQueue, block=False)
def getTestForRunReuseOnlyMode(self, block):
self.reuseOnly = True
self.diag.info("Waiting for reuse failures...")
reuseFailure = self.getItemFromQueue(self.reuseFailureQueue, block=block)
if reuseFailure:
return reuseFailure
elif self.testCount > 0 and self.testsSubmitted < self.maxCapacity:
# Try again, the capacity situation has changed...
return self.getTestForRunNormalMode(block)
def getTestForRun(self, block=True):
if self.testCount == 0 or (self.testsSubmitted < self.maxCapacity):
return self.getTestForRunNormalMode(block)
else:
return self.getTestForRunReuseOnlyMode(block)
def notifyAllComplete(self):
BaseActionRunner.notifyAllComplete(self)
errors = {}
errorFiles = []
for logDir in self.slaveLogDirs:
errorFiles += filter(os.path.getsize, glob(os.path.join(logDir, "*.errors")))
if len(errorFiles) == 0:
return
for fileName in errorFiles:
contents = None
# Take the shortest (i.e. most filtered) one
for app in self.allApps:
currContent = app.filterErrorText(fileName)
if contents is None or len(currContent) < len(contents):
contents = currContent
if contents:
errors[contents] = os.path.basename(fileName)[:-7]
for msg, jobName in errors.items():
sys.stderr.write("WARNING: error produced by slave job '" + jobName + "'\n" + msg)
def cleanup(self):
self.sendServerState("Completed submission of all tests")
def remainStr(self):
return " : " + str(self.testCount) + " tests remain, " + str(self.testsSubmitted) + " are submitted."
def runTest(self, test):
submissionRules = self.getSubmissionRules(test)
command = self.getSlaveCommand(test, submissionRules)
plugins.log.info("Q: Submitting " + repr(test) + submissionRules.getSubmitSuffix())
sys.stdout.flush()
self.jobs[test] = [] # Preliminary jobs aren't interesting any more
if not self.submitJob(test, submissionRules, command, self.getSlaveEnvironment()):
return
self.testCount -= 1
self.testsSubmitted += 1
self.diag.info("Submission successful" + self.remainStr())
if not test.state.hasStarted():
test.changeState(self.getPendingState(test))
if self.testsSubmitted == self.maxCapacity:
self.sendServerState("Completed submission of tests up to capacity")
def getSlaveVarsToBlock(self):
"""Make sure we clear out the master scripts so the slave doesn't use them too,
otherwise just use the environment as is.
If we're being run via SSH, don't pass this on to the slave jobs
This has been known to trip up shell starter scripts, e.g. on SuSE 10
making them believe that the SGE job is an SSH login and setting things wrongly
as a result."""
return [ "USECASE_REPLAY_SCRIPT", "USECASE_RECORD_SCRIPT", "SSH_TTY" ]
def getSlaveEnvironment(self):
return plugins.copyEnvironment(ignoreVars=self.getSlaveVarsToBlock())
def fixDisplay(self, env):
# Must make sure SGE jobs don't get a locally referencing DISPLAY
display = env.get("DISPLAY")
if display and display.startswith(":"):
env["DISPLAY"] = plugins.gethostname() + display
def getPendingState(self, test):
freeText = "Job pending in " + queueSystemName(test.app)
return plugins.TestState("pending", freeText=freeText, briefText="PEND", lifecycleChange="become pending")
def shellWrap(self, command):
# Must use exec so as not to create extra processes: SGE's qdel isn't very clever when
# it comes to noticing extra shells
return "exec $SHELL -c \"exec " + command + "\""
def getSlaveCommand(self, test, submissionRules):
cmdArgs = [ plugins.getTextTestProgram(), "-d", ":".join(self.optionMap.rootDirectories),
"-a", test.app.name + test.app.versionSuffix(),
"-l", "-tp", plugins.quote(test.getRelPath()) ] + \
self.getSlaveArgs(test) + self.getRunOptions(test.app, submissionRules)
return " ".join(cmdArgs)
def getSlaveArgs(self, test):
return [ "-slave", test.app.writeDirectory, "-servaddr", self.submitAddress ]
def getRunOptions(self, app, submissionRules):
runOptions = []
for slaveSwitch in app.getSlaveSwitches():
if self.optionMap.has_key(slaveSwitch):
option = "-" + slaveSwitch
runOptions.append(option)
value = self.optionMap.get(slaveSwitch)
if value:
runOptions.append(value)
if self.optionMap.has_key("x"):
runOptions.append("-xr")
runOptions.append(self.optionMap.get("xr", os.path.expandvars("$TEXTTEST_PERSONAL_LOG/logging.debug")))
runOptions.append("-xw")
runOptions.append(os.path.expandvars("$TEXTTEST_PERSONAL_LOG/" + submissionRules.getJobName()))
return runOptions
def getSlaveLogDir(self, test):
return os.path.join(test.app.writeDirectory, "slavelogs")
def getSubmitCmdArgs(self, test, submissionRules):
queueSystem = self.getQueueSystem(test)
extraArgs = submissionRules.getExtraSubmitArgs()
cmdArgs = queueSystem.getSubmitCmdArgs(submissionRules)
if extraArgs:
cmdArgs += plugins.splitcmd(extraArgs)
return cmdArgs
def getQueueSystemCommand(self, test):
submissionRules = self.getSubmissionRules(test)
cmdArgs = self.getSubmitCmdArgs(test, submissionRules)
text = queueSystemName(test) + " Command : " + plugins.commandLineString(cmdArgs) + " ...\n" + \
"Slave Command : " + self.getSlaveCommand(test, submissionRules) + "\n"
proxyArgs = self.getProxyCmdArgs(test)
if proxyArgs:
return queueSystemName(test) + " Proxy Command : " + plugins.commandLineString(proxyArgs) + "\n" + text
else:
return text
def getProxyCmdArgs(self, test):
proxyCmd = test.getConfigValue("queue_system_proxy_executable")
if proxyCmd:
proxyOptions = test.getCommandLineOptions("proxy_options")
fullProxyCmd = proxyCmd + " " + " ".join(proxyOptions)
proxyRules = self.getJobSubmissionRules(test)
proxyArgs = self.getSubmitCmdArgs(test, proxyRules)
proxyArgs.append(self.shellWrap(fullProxyCmd))
return proxyArgs
else:
return []
def createSubmitProcess(self, test, cmdArgs, slaveEnv):
logDir = self.getSlaveLogDir(test)
proxyArgs = self.getProxyCmdArgs(test)
if proxyArgs:
cmdArgs[1:1] = [ "-sync", "y" ] # must synchronise in the proxy
slaveEnv["TEXTTEST_SUBMIT_COMMAND_ARGS"] = repr(cmdArgs) # Exact command arguments to run TextTest slave, for use by proxy
cmdArgs = proxyArgs
return subprocess.Popen(cmdArgs, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
cwd=logDir, env=slaveEnv)
def submitJob(self, test, submissionRules, command, slaveEnv):
self.diag.info("Submitting job at " + plugins.localtime() + ":" + command)
self.diag.info("Creating job at " + plugins.localtime())
cmdArgs = self.getSubmitCmdArgs(test, submissionRules)
cmdArgs.append(self.shellWrap(command))
jobName = submissionRules.getJobName()
self.fixDisplay(slaveEnv)
self.diag.info("Creating job " + jobName + " with command arguments : " + repr(cmdArgs))
self.lock.acquire()
if self.exited:
self.cancel(test)
self.lock.release()
plugins.log.info("Q: Submission cancelled for " + repr(test) + " - exit underway")
return False
self.lockDiag.info("Got lock for submission")
queueSystem = self.getQueueSystem(test)
try:
process = self.createSubmitProcess(test, cmdArgs, slaveEnv)
stdout, stderr = process.communicate()
errorMessage = self.findErrorMessage(stderr, queueSystem)
except OSError:
errorMessage = "local machine is not a submit host: running '" + cmdArgs[0] + "' failed."
if not errorMessage:
jobId = queueSystem.findJobId(stdout)
self.diag.info("Job created with id " + jobId)
self.jobs.setdefault(test, []).append((jobId, jobName))
self.lockDiag.info("Releasing lock for submission...")
self.lock.release()
return True
else:
self.lock.release()
self.diag.info("Job not created : " + errorMessage)
fullError = self.getFullSubmitError(test, errorMessage, cmdArgs)
test.changeState(plugins.Unrunnable(fullError, "NOT SUBMITTED"))
self.handleErrorState(test)
return False
def findErrorMessage(self, stderr, queueSystem):
if len(stderr) > 0:
return queueSystem.findSubmitError(stderr)
def getFullSubmitError(self, test, errorMessage, cmdArgs):
qname = queueSystemName(test.app)
return "Failed to submit to " + qname + " (" + errorMessage.strip() + ")\n" + \
"Submission command was '" + " ".join(cmdArgs[:-1]) + " ... '\n"
def handleErrorState(self, test, previouslySubmitted=False):
if previouslySubmitted:
self.testsSubmitted -= 1
else:
self.testCount -= 1
self.diag.info(repr(test) + " in error state" + self.remainStr())
bugchecker = CheckForBugs()
self.setUpSuites(bugchecker, test)
bugchecker(test)
test.actionsCompleted()
def setUpSuites(self, bugchecker, test):
if test.parent:
bugchecker.setUpSuite(test.parent)
self.setUpSuites(bugchecker, test.parent)
def _getJobFailureInfo(self, test):
jobInfo = self.getJobInfo(test)
if len(jobInfo) == 0:
return "No job has been submitted to " + queueSystemName(test)
queueSystem = self.getQueueSystem(test)
# Take the most recent job, it's hopefully the most interesting
jobId = jobInfo[-1][0]
return queueSystem.getJobFailureInfo(jobId)
def getSlaveErrors(self, test, name):
slaveErrFile = self.getSlaveErrFile(test)
if slaveErrFile:
errors = open(slaveErrFile).read()
if errors:
return "-" * 10 + " Error messages written by " + name + " job " + "-" * 10 + \
"\n" + errors
def getSlaveErrFile(self, test):
for _, jobName in self.getJobInfo(test):
errFile = os.path.join(self.getSlaveLogDir(test), jobName + ".errors")
if os.path.isfile(errFile):
return errFile
def getJobInfo(self, test):
return self.jobs.get(test, [])
def killJob(self, test, jobId, jobName):
prevTest, prevJobExisted = self.killedJobs.get(jobId, (None, False))
# Killing the same job for other tests should result in the cached result being returned
if prevTest and test is not prevTest:
return prevJobExisted
self.describeJob(test, jobId, jobName)
queueSystem = self.getQueueSystem(test)
jobExisted = queueSystem.killJob(jobId)
self.killedJobs[jobId] = test, jobExisted
return jobExisted
def getQueueSystem(self, test):
queueModuleText = queueSystemName(test)
if queueModuleText is None:
return None
queueModule = queueModuleText.lower()
if self.queueSystems.has_key(queueModule):
return self.queueSystems[queueModule]
command = "from " + queueModule + " import QueueSystem as _QueueSystem"
exec command
system = _QueueSystem()
self.queueSystems[queueModule] = system
return system
def changeState(self, test, newState, previouslySubmitted=True):
test.changeState(newState)
self.handleLocalError(test, previouslySubmitted)
def killTests(self):
# If we've been killed with some sort of limit signal, wait here until we know
# all tests terminate. Otherwise we rely on them terminating naturally, and if they don't
wantStatus = self.killSignal and self.killSignal != signal.SIGINT
killedTests = []
for test, jobList in self.jobs.items():
if not test.state.isComplete():
for jobId, jobName in jobList:
if self.killTest(test, jobId, jobName, wantStatus):
killedTests.append((test, jobId))
if wantStatus:
self.waitForKill(killedTests)
def waitForKill(self, killedTests):
# Wait for a minute for the kill to take effect, otherwise give up
stillRunning = killedTests
for attempt in range(1, 61):
stillRunning = filter(lambda (test, jobId): not test.state.isComplete(), stillRunning)
if len(stillRunning) == 0:
return
time.sleep(1)
for test, jobId in stillRunning:
plugins.log.info("T: Cancellation in progress for " + repr(test) +
", waited " + str(attempt) + " seconds so far.")
for test, jobId in stillRunning:
name = queueSystemName(test.app)
freeText = "Could not delete test in " + name + " (job " + jobId + "): have abandoned it"
self.changeState(test, Abandoned(freeText))
def killOrCancel(self, test):
# Explicitly chose test to kill (from the GUI)
jobInfo = self.getJobInfo(test)
if len(jobInfo) > 0:
for jobId, jobName in jobInfo:
self.killTest(test, jobId, jobName, wantStatus=True)
else:
self.diag.info("No job info found from queue system server, changing state to cancelled")
return self.cancel(test)
def killTest(self, test, jobId, jobName, wantStatus):
self.diag.info("Killing test " + repr(test) + " in state " + test.state.category)
jobExisted = self.killJob(test, jobId, jobName)
startNotified = self.jobStarted(test)
if jobExisted:
if startNotified:
self.diag.info("Job " + jobId + " was running.")
return True
else:
self.diag.info("Job " + jobId + " was pending.")
self.setKilledPending(test, jobId)
return False
else:
self.diag.info("Job " + jobId + " did not exist.")
# might get here when the test completed since we checked...
if not test.state.isComplete():
self.setSlaveFailed(test, startNotified, wantStatus)
return False
def setSuspendStateForTests(self, tests, newState):
for test in tests:
queueSystem = self.getQueueSystem(test)
for jobId, jobName in self.getJobInfo(test):
queueSystem.setSuspendState(jobId, newState)
def jobStarted(self, test):
return test.state.hasStarted()
def setKilledPending(self, test, jobId):
timeStr = plugins.localtime("%H:%M")
briefText = "cancelled pending job at " + timeStr
freeText = "Test job " + jobId + " was cancelled (while still pending in " + queueSystemName(test.app) +\
") at " + timeStr
self.cancel(test, briefText, freeText)
def getJobFailureInfo(self, test, name, wantStatus):
if wantStatus:
return "-" * 10 + " Full accounting info from " + name + " " + "-" * 10 + "\n" + \
self._getJobFailureInfo(test)
else:
# Job accounting info can take ages to find, don't do it from GUI quit
return "No accounting info found as quitting..."
def setSlaveFailed(self, test, startNotified, wantStatus):
failReason, fullText = self.getSlaveFailure(test, startNotified, wantStatus)
fullText = failReason + "\n" + fullText
self.changeState(test, self.getSlaveFailureState(startNotified, failReason, fullText))
def getSlaveFailure(self, test, startNotified, wantStatus):
fullText = ""
name = queueSystemName(test.app)
slaveErrors = self.getSlaveErrors(test, name)
if slaveErrors:
fullText += slaveErrors
fullText += self.getJobFailureInfo(test, name, wantStatus)
return self.getSlaveFailureBriefText(name, startNotified), fullText
def getSlaveFailureBriefText(self, name, startNotified):
if startNotified:
return "no report, possibly killed with SIGKILL"
else:
return name + " job exited"
def getSlaveFailureState(self, startNotified, failReason, fullText):
if startNotified:
return plugins.TestState("killed", briefText=failReason, \
freeText=fullText, completed=1, lifecycleChange="complete")
else:
return plugins.Unrunnable(briefText=failReason, freeText=fullText, lifecycleChange="complete")
def getPostText(self, test, jobId):
name = queueSystemName(test.app)
return "in " + name + " (job " + jobId + ")"
def describeJob(self, test, jobId, *args):
postText = self.getPostText(test, jobId)
plugins.log.info("T: Cancelling " + repr(test) + " " + postText)
class HasParameters(object):
"""This class provides an implementation of the IHasParameters interface."""
_do_not_promote = ['get_expr_depends', 'get_referenced_compnames',
'get_referenced_varpaths', 'get_metadata']
def __init__(self, parent):
self._parameters = OrderedDict()
self._allowed_types = ['continuous']
if obj_has_interface(parent, ISolver):
self._allowed_types.append('unbounded')
self._parent = None if parent is None else weakref.ref(parent)
def __getstate__(self):
state = self.__dict__.copy()
state['_parent'] = self.parent
return state
def __setstate__(self, state):
self.__dict__.update(state)
parent = state['_parent']
self._parent = None if parent is None else weakref.ref(parent)
@property
def parent(self):
""" The object we are a delegate of. """
return None if self._parent is None else self._parent()
def _item_count(self):
"""This is used by the replace function to determine if a delegate from
the target object is 'empty' or not. If it's empty, it's not an error
if the replacing object doesn't have this delegate.
"""
return len(self._parameters)
def add_parameter(self, target, low=None, high=None,
scaler=None, adder=None, start=None,
fd_step=None, name=None, scope=None):
"""Adds a parameter or group of parameters to the driver.
target: string or iter of strings or Parameter
What the driver should vary during execution. A *target* is an
expression that can reside on the left-hand side of an assignment
statement, so typically it will be the name of a variable or
possibly a subscript expression indicating an entry within an array
variable, e.g., x[3]. If an iterator of targets is given, then the
driver will set all targets given to the same value whenever it
varies this parameter during execution. If a Parameter instance is
given, then that instance is copied into the driver with any other
arguments specified, overiding the values in the given parameter.
low: float (optional)
Minimum allowed value of the parameter. If scaler and/or adder
is supplied, use the transformed value here. If target is an array,
this may also be an array, but must have the same size.
high: float (optional)
Maximum allowed value of the parameter. If scaler and/or adder
is supplied, use the transformed value here. If target is an array,
this may also be an array, but must have the same size.
scaler: float (optional)
Value to multiply the possibly offset parameter value by. If target
is an array, this may also be an array, but must have the same size.
adder: float (optional)
Value to add to parameter prior to possible scaling. If target is
an array, this may also be an array, but must have the same size.
start: any (optional)
Value to set into the target or targets of a parameter before
starting any executions. If not given, analysis will start with
whatever values are in the target or targets at that time. If target
is an array, this may also be an array, but must have the same size.
fd_step: float (optional)
Step-size to use for finite difference calculation. If no value is
given, the differentiator will use its own default. If target is an
array, this may also be an array, but must have the same size.
name: str (optional)
Name used to refer to the parameter in place of the name of the
variable referred to in the parameter string.
This is sometimes useful if, for example, multiple entries in the
same array variable are declared as parameters.
scope: object (optional)
The object to be used as the scope when evaluating the expression.
If neither "low" nor "high" is specified, the min and max will
default to the values in the metadata of the variable being
referenced.
"""
if isinstance(target, (ParameterBase, ParameterGroup)):
self._parameters[target.name] = target
target.override(low, high, scaler, adder, start, fd_step, name)
else:
if isinstance(target, basestring):
names = [target]
key = target
else:
names = target
key = tuple(target)
if name is not None:
key = name
dups = set(self.list_param_targets()).intersection(names)
if len(dups) == 1:
self.parent.raise_exception("'%s' is already a Parameter"
" target" % dups.pop(), ValueError)
elif len(dups) > 1:
self.parent.raise_exception("%s are already Parameter targets"
% sorted(list(dups)), ValueError)
if key in self._parameters:
self.parent.raise_exception("%s is already a Parameter" % key,
ValueError)
try:
_scope = self._get_scope(scope)
if len(names) == 1:
target = self._create(names[0], low, high, scaler, adder,
start, fd_step, key, _scope)
else: # defining a ParameterGroup
parameters = [self._create(n, low, high, scaler, adder,
start, fd_step, key, _scope)
for n in names]
types = set([p.valtypename for p in parameters])
if len(types) > 1:
raise ValueError("Can't add parameter %s because "
"%s are not all of the same type" %
(key, " and ".join(names)))
target = ParameterGroup(parameters)
self._parameters[key] = target
except Exception:
self.parent.reraise_exception()
self.parent.config_changed()
def _create(self, target, low, high, scaler, adder, start, fd_step,
key, scope):
""" Create one Parameter or ArrayParameter. """
try:
expreval = ExprEvaluator(target, scope)
except Exception as err:
raise err.__class__("Can't add parameter: %s" % err)
if not expreval.is_valid_assignee():
raise ValueError("Can't add parameter: '%s' is not a"
" valid parameter expression"
% expreval.text)
try:
val = expreval.evaluate()
except Exception as err:
val = None # Let Parameter code sort out why.
name = key[0] if isinstance(key, tuple) else key
if isinstance(val, ndarray):
return ArrayParameter(target, low=low, high=high,
scaler=scaler, adder=adder,
start=start, fd_step=fd_step,
name=name, scope=scope,
_expreval=expreval, _val=val,
_allowed_types=self._allowed_types)
else:
return Parameter(target, low=low, high=high,
scaler=scaler, adder=adder,
start=start, fd_step=fd_step,
name=name, scope=scope,
_expreval=expreval, _val=val,
_allowed_types=self._allowed_types)
def remove_parameter(self, name):
"""Removes the parameter with the given name."""
param = self._parameters.get(name)
if param:
del self._parameters[name]
else:
self.parent.raise_exception("Trying to remove parameter '%s' "
"that is not in this driver."
% (name,), AttributeError)
self.parent.config_changed()
def config_parameters(self):
"""Reconfigure parameters from potentially changed targets."""
for param in self._parameters.values():
param.configure()
def get_references(self, name):
"""Return references to component `name` in preparation for subsequent
:meth:`restore_references` call.
name: string
Name of component being removed.
"""
refs = OrderedDict()
for pname, param in self._parameters.items():
if name in param.get_referenced_compnames():
refs[pname] = param
return refs
def remove_references(self, name):
"""Remove references to component `name`.
name: string
Name of component being removed.
"""
to_remove = []
for pname, param in self._parameters.items():
if name in param.get_referenced_compnames():
to_remove.append(pname)
for pname in to_remove:
self.remove_parameter(pname)
def restore_references(self, refs):
"""Restore references to component `name` from `refs`.
refs: object
Value returned by :meth:`get_references`.
"""
for pname, param in refs.items():
try:
self.add_parameter(param)
except Exception as err:
self.parent._logger.warning("Couldn't restore parameter '%s': %s"
% (pname, str(err)))
def list_param_targets(self):
"""Returns a list of parameter targets. Note that this
list may contain more entries than the list of Parameter,
ParameterGroup, and ArrayParameter objects since ParameterGroup
instances have multiple targets.
"""
targets = []
for param in self._parameters.values():
targets.extend(param.targets)
return targets
def list_param_group_targets(self):
"""Returns a list of tuples that contain the targets for each
parameter group.
"""
targets = []
for param in self.get_parameters().values():
targets.append(tuple(param.targets))
return targets
def clear_parameters(self):
"""Removes all parameters."""
for name in self._parameters.keys():
self.remove_parameter(name)
self._parameters = OrderedDict()
def get_parameters(self):
"""Returns an ordered dict of parameter objects."""
return self._parameters
def total_parameters(self):
"""Returns the total number of values to be set."""
return sum([param.size for param in self._parameters.values()])
def init_parameters(self):
"""Sets all parameters to their start value if a
start value is given
"""
scope = self._get_scope()
for param in self._parameters.itervalues():
if param.start is not None:
param.set(param.start, scope)
def set_parameter_by_name(self, name, value, case=None, scope=None):
"""Sets a single parameter by its name attribute.
name: str
Name of the parameter. This is either the name alias given when
the parameter was added or the variable path of the parameter's
target if no name was given.
value: object (typically a float)
Value of the parameter to be set.
case: Case (optional)
If supplied, the values will be associated with their corresponding
targets and added as inputs to the Case instead of being set
directly into the model.
"""
param = self._parameters[name]
if case is None:
param.set(value, self._get_scope(scope))
else:
for target in param.targets:
case.add_input(target, value)
return case
def set_parameters(self, values, case=None, scope=None):
"""Pushes the values in the iterator 'values' into the corresponding
variables in the model. If the 'case' arg is supplied, the values
will be set into the case and not into the model.
values: iterator
Iterator of input values with an order defined to match the
order of parameters returned by the get_parameters method. All
'values' must support the len() function.
case: Case (optional)
If supplied, the values will be associated with their corresponding
targets and added as inputs to the Case instead of being set
directly into the model.
"""
if len(values) != self.total_parameters():
raise ValueError("number of input values (%s) != expected number of"
" values (%s)" %
(len(values), self.total_parameters()))
if case is None:
scope = self._get_scope(scope)
start = 0
for param in self._parameters.values():
size = param.size
if size == 1:
param.set(values[start], scope)
start += 1
else:
end = start + size
param.set(values[start:end], scope)
start = end
else:
start = 0
for param in self._parameters.values():
size = param.size
if size == 1:
for target in param.targets:
case.add_input(target, values[start])
start += 1
else:
end = start + size
for target in param.targets:
case.add_input(target, values[start:end])
start = end
return case
def eval_parameters(self, scope=None, dtype='d'):
"""Return evaluated parameter values.
dtype: string or None
If not None, return an array of this dtype. Otherwise just return
a list (useful if parameters may be of different types).
"""
result = []
for param in self._parameters.values():
result.extend(param.evaluate(scope))
if dtype:
result = array(result, dtype)
return result
def get_lower_bounds(self, dtype='d'):
"""Return lower bound values.
dtype: string or None
If not None, return an array of this dtype. Otherwise just return
a list (useful if parameters may be of different types).
"""
result = []
for param in self._parameters.values():
result.extend(param.get_low())
if dtype:
result = array(result, dtype)
return result
def get_upper_bounds(self, dtype='d'):
"""Return upper bound values.
dtype: string or None
If not None, return an array of this dtype. Otherwise just return
a list (useful if parameters may be of different types).
"""
result = []
for param in self._parameters.values():
result.extend(param.get_high())
if dtype:
result = array(result, dtype)
return result
def get_fd_steps(self, dtype='d'):
"""Return fd_step values, they may include None.
dtype: string or None
If not None, return an array of this dtype. Otherwise just return
a list (useful if it's valid to have None for a step size).
"""
result = []
for param in self._parameters.values():
result.extend(param.get_fd_step())
if dtype:
result = array(result, dtype)
return result
def get_expr_depends(self):
"""Returns a list of tuples of the form (src_comp_name, dest_comp_name)
for each dependency introduced by a parameter.
"""
conn_list = []
pname = self.parent.name
for param in self._parameters.values():
for cname in param.get_referenced_compnames():
conn_list.append((pname, cname))
return conn_list
def get_referenced_compnames(self):
"""Return a set of Component names based on the
pathnames of Variables referenced in our target strings.
"""
result = set()
for param in self._parameters.values():
result.update(param.get_referenced_compnames())
return result
def get_referenced_varpaths(self):
"""Return a set of Variable names referenced in our target strings.
"""
result = set()
for param in self._parameters.values():
result.update(param.get_referenced_varpaths())
return result
def _get_scope(self, scope=None):
if scope is None:
try:
return self.parent.get_expr_scope()
except AttributeError:
pass
return scope
def mimic(self, target):
old = self._parameters
self.clear_parameters()
try:
for name, param in target.get_parameters().items():
self._parameters[name] = param.copy()
except Exception:
self._parameters = old
raise
class PyModule:
"""This is the module being called directly from the rest of ArchGenXML.
Through the __init__() you can feed it a file and it chops it up
in neat chunks of classes and methods. This way the other parts of
ArchGenXML can add/remove/augment those chunks.
"""
filebuf = None
splittedSource = None
ast = None
code = None
src = None
classes = OrderedDict()
functions = OrderedDict()
protectedSections = OrderedDict()
def __init__(self, file, mode='file'):
"""Start dividing 'file' in chunks.
'file' is the to chunk up. By default it is the name of a file
on the filesystem, but with 'mode' set to 'string', 'file' is
passed as a string.
"""
log.debug("Initialising module parser for file '%s'.",
file)
# Dictionary inits
self.classes = OrderedDict()
self.functions = OrderedDict()
self.protectedSections = OrderedDict()
self.protectionDeclarations = []
# Read and mangle the file
self.filebuf = self.readFile(file, mode)
self.splitSource()
# Note: ast = abstract syntax tree (python internal thingy),
# generated by the imported 'parser'.
self.filebuf = self.filebuf.encode(self.encoding)
self.ast = parser.suite(self.filebuf)
self.code = self.ast.compile()
# The next two filter out the classes and the top-level
# functions in the sourcefile and the protected
# sections. Beware that the rest is left out!
self.findClassesAndFunctions()
self.findProtectedSections()
self.findProtectionDeclarations()
def readFile(self, file, mode='file'):
""" Read the file into a string
File can be a filename, a file object or a big string.
"""
if type(file) in (type(''), type(u'')):
# filename or big string
if mode == 'string':
# Big string!
result = file
else:
# Filename!
result = open(file).read()
else:
# File object!
result = file.read()
# XXX: ugly hack: work only if file is utf-8
# TODO: read first 2 lines and find real encoding as in PEP 263 defined
self.encoding = 'utf-8'
if type(result) != types.UnicodeType:
result = result.decode(self.encoding)
return result
def findClassesAndFunctions(self):
""" Collect code elements in the source file
Code elements are seperate things like functions and classes.
The import-statements, local variables etcetera are not code
elements and are not extracted by this method.
The results are placed in self.classes and
self.functions. Functions are the top-level methods, methods
reside inside the classes.
"""
# First get all the code elements as seen by the python parser
codes = [c for c in self.code.co_consts if type(c) ==
types.CodeType]
# Get the classes
classes = [c for c in codes if self.isItAClass(c)]
for c in classes:
klass = PyClass(c, self)
self.classes[c.co_name] = klass
# Get the functions
functions = [c for c in codes if self.isItAFunction(c)]
for f in functions:
func = PyFunction(f, self)
self.functions[f.co_name] = func
def findProtectedSections(self):
""" Find the protected sections in the source file
The results are placed in self.protectedSections, which is a
dictionary. The keys are the names of the protected sections
(like 'module-header').
"""
for i in xrange(0, len(self.splittedSource)):
line = self.splittedSource[i]
sline = line.strip()
if sline.startswith(PROTECTED_BEGIN):
j = start = i
sectionname = sline.split()[1]
try:
while not self.splittedSource[j].strip().startswith(PROTECTED_END):
j = j+1
except IndexError:
return
end = j
protectedSection = '\n'.join(self.splittedSource[start+1:end])
self.protectedSections[sectionname] = protectedSection
log.debug("In total, we found %s protected sections.",
len(self.protectedSections))
def findProtectionDeclarations(self):
""" Find the protection declarations in the source file
The results are placed in self.protectionDeclarations, which
is a list. You can find the protected methods by looking for
their name in this list of strings. A bit brute-force, I
admit.
A restriction is that it has to be a one-line statement.
"""
for i in xrange(0, len(self.splittedSource)):
line = self.splittedSource[i]
strippedLine = line.strip()
if ('declarePublic' in strippedLine or
'declarePrivate' in strippedLine or
'declareProtected' in strippedLine):
self.protectionDeclarations.append(line)
# note: the line, so we get the good indentation
def isItAClass(self, c):
""" True if a code fragment is a class
Woooh - this is a very pillar of supreme machine intelligence
:-)
"""
fl = c.co_firstlineno
if self.splittedSource[fl-1].strip().startswith('class'):
return 1
res = len([o for o in c.co_consts if type(o) == types.CodeType])
#print 'Class:####',c.co_name, res, c.co_consts
return res
def isItAFunction(self, c):
""" True if a code fragment is a function
Woohoo, we're advanced! Heuristics!
"""
fl = c.co_firstlineno
if self.splittedSource[fl-1].startswith('def'):
return 1
def isItAComment(self, lineNumber):
""" True if a code fragment is a comment
Checks if the line starts with single or double quotes
"""
if self.splittedSource[lineNumber].strip().startswith('"""'):
# three double quotes
return 1
if self.splittedSource[lineNumber].strip().startswith("'''"):
# three single quotes
return 1
return 0
def getProtectedSection(self, section):
""" Return the named protected section
Simple wrapper function.
"""
return self.protectedSections.get(section)
def printit(self):
# Can probably be removed - early testing-by-printing
# Perhaps something for a --verbose option? Could be very
# handy.
print 'PyModule:'
print '========'
print 'CLASSES'
print '========'
for c in self.classes.values():
c.printit()
print '========'
print 'FUNCTIONS'
print '========'
for f in self.functions.values():
f.printit()
print '========'
print 'PROTECTED SECTIONS'
print '========'
for k, v in self.protectedSections.items():
print 'section:', k
print '-----------'
print v
print '-----------'
def splitSource(self):
self.filebuf = self.filebuf.replace(u'\r', u'\n')
self.splittedSource = self.filebuf.split(u'\n')
# cleanup trailing white spaces
self.splittedSource = [s.rstrip() for s in self.splittedSource]
class CnCGraph(object):
def __init__(self, name, g):
verifyCollectionDecls("item", g.itemColls)
steps = [x.step for x in g.stepRels]
verifyCollectionDecls("step", steps)
self.name = name
# items
self.itemDeclarations = OrderedDict(
(i.collName, makeItemDecl(i)) for i in g.itemColls)
self.concreteItems = [
i for i in self.itemDeclarations.values() if not i.isVirtual
]
# item virtual mappings
self.vms = [i for i in self.itemDeclarations.values() if i.isVirtual]
self.inlineVms = [i for i in self.vms if i.isInline]
self.externVms = [i for i in self.vms if not i.isInline]
# steps / pseudo-steps
self.stepFunctions = OrderedDict(
(x.step.collName, StepFunction(x)) for x in g.stepRels)
verifyEnv(self.stepFunctions)
self.initFunction = self.stepFunctions.pop(initNameRaw)
self.initFunction.collName = "cncInitialize"
self.finalizeFunction = self.stepFunctions.pop(finalizeNameRaw)
self.finalizeFunction.collName = "cncFinalize"
self.finalAndSteps = [self.finalizeFunction
] + self.stepFunctions.values()
# set up step attribute lookup dict
self.stepLikes = OrderedDict(self.stepFunctions)
self.stepLikes[self.initFunction.collName] = self.initFunction
self.stepLikes[self.finalizeFunction.collName] = self.finalizeFunction
# attribute tracking
self.allAttrNames = set()
# context
self.ctxParams = filter(bool, map(
strip, g.ctx.splitlines())) if g.ctx else []
def hasTuning(self, name):
return name in self.allAttrNames
def hasCustomDist(self):
return self.hasTuning('distfn') or self.hasTuning('placeWith')
def lookupType(self, item):
return self.itemDeclarations[item.collName].type
def itemDistFn(self, collName, ranksExpr):
return getDistFn(self.itemDeclarations, collName, ranksExpr)
def stepDistFn(self, collName, ranksExpr):
return getDistFn(self.stepLikes, collName, ranksExpr)
def itemTuningFn(self, collName, name, ranksExpr, default):
return getTuningFn(self.itemDeclarations, collName, name, ranksExpr,
default)
def stepTuningFn(self, collName, name, ranksExpr, default):
return getTuningFn(self.stepLikes, collName, name, ranksExpr, default)
def priorityFn(self, collName, ranksExpr):
return self.stepTuningFn(collName, 'priority', ranksExpr, "0")
def addTunings(self, tuningSpec):
for t in tuningSpec.itemTunings:
x = self.itemDeclarations.get(t.collName)
assert x, "Unknown item in tuning: {0}".format(t.collName)
x.attrs.update(t.attrs)
self.allAttrNames.update(t.attrs.keys())
for t in tuningSpec.stepTunings:
x = self.stepLikes.get(t.collName)
assert x, "Unknown step in tuning: {0}".format(t.collName)
if t.inputName:
i = x.inputsDict.get(t.inputName)
assert i, "Unknown input in tuning: {0} <- {1}".format(
t.collName, t.inputName)
i.attrs.update(t.attrs)
self.allAttrNames.update(t.attrs.keys())
else:
x.attrs.update(t.attrs)
self.allAttrNames.update(t.attrs.keys())
class _Section(_AbstractSection):
"""Simple abstract section object, container for Elements"""
def __init__(self, *args, **kwargs):
super(_Section, self).__init__(*args, **kwargs)
self.elements = OrderedDict()
def add_element(self, elt):
"""Helper to add a element to the current section. The Element name
will be used as an identifier."""
if not isinstance(elt, Element):
raise TypeError("argument should be a subclass of Element")
self.elements[elt.get_name()] = elt
return elt
def add_element_list(self, elt_list, **kwargs):
"""Helper to add a list of similar elements to the current section.
Element names will be used as an identifier."""
for e in elt_list:
self.add_element(Element(e, **kwargs))
def count(self):
"""This method will return the number of Element in the current
Section"""
return len(self.elements)
def reset(self):
for e in self.elements.values():
e.reset()
def load(self, file_parser):
section = self.get_section_name()
try:
for e in self.elements.values():
e.load(file_parser, section)
except ConfigParser.NoSectionError as e:
# pylint: disable-msg=W0621
log = logging.getLogger('argtoolbox')
if self._required:
log.error("Required section : " + section)
raise ValueError(e)
else:
log.debug("Missing section : " + section)
def __getattr__(self, name):
e = self.elements.get(name)
if e is not None:
return e
else:
raise AttributeError("'%(class)s' object has no attribute \
'%(name)s'" % {"name": name, "class": self.__class__.__name__})
def get_element(self, name):
return self.elements.get(name)
def write_config_file(self, f, comments):
"""This method write a sample file, with attributes, descriptions,
sample values, required flags, using the configuration object
properties.
"""
if len(self.elements) < 1:
return
super(_Section, self).write_config_file(f, comments)
for e in self.elements.values():
e.write_config_file(f, comments)
f.write("\n")
class HasParameters(object):
"""This class provides an implementation of the IHasParameters interface."""
_do_not_promote = [
'get_expr_depends', 'get_referenced_compnames',
'get_referenced_varpaths', 'get_metadata'
]
def __init__(self, parent):
self._parameters = OrderedDict()
self._allowed_types = ['continuous']
if obj_has_interface(parent, ISolver):
self._allowed_types.append('unbounded')
self._parent = None if parent is None else weakref.ref(parent)
def __getstate__(self):
state = self.__dict__.copy()
state['_parent'] = self.parent
return state
def __setstate__(self, state):
self.__dict__.update(state)
parent = state['_parent']
self._parent = None if parent is None else weakref.ref(parent)
@property
def parent(self):
""" The object we are a delegate of. """
return None if self._parent is None else self._parent()
def _item_count(self):
"""This is used by the replace function to determine if a delegate from
the target object is 'empty' or not. If it's empty, it's not an error
if the replacing object doesn't have this delegate.
"""
return len(self._parameters)
def add_parameter(self,
target,
low=None,
high=None,
scaler=None,
adder=None,
start=None,
fd_step=None,
name=None,
scope=None):
"""Adds a parameter or group of parameters to the driver.
target: string or iter of strings or Parameter
What the driver should vary during execution. A *target* is an
expression that can reside on the left-hand side of an assignment
statement, so typically it will be the name of a variable or
possibly a subscript expression indicating an entry within an array
variable, e.g., x[3]. If an iterator of targets is given, then the
driver will set all targets given to the same value whenever it
varies this parameter during execution. If a Parameter instance is
given, then that instance is copied into the driver with any other
arguments specified, overiding the values in the given parameter.
low: float (optional)
Minimum allowed value of the parameter. If scaler and/or adder
is supplied, use the transformed value here. If target is an array,
this may also be an array, but must have the same size.
high: float (optional)
Maximum allowed value of the parameter. If scaler and/or adder
is supplied, use the transformed value here. If target is an array,
this may also be an array, but must have the same size.
scaler: float (optional)
Value to multiply the possibly offset parameter value by. If target
is an array, this may also be an array, but must have the same size.
adder: float (optional)
Value to add to parameter prior to possible scaling. If target is
an array, this may also be an array, but must have the same size.
start: any (optional)
Value to set into the target or targets of a parameter before
starting any executions. If not given, analysis will start with
whatever values are in the target or targets at that time. If target
is an array, this may also be an array, but must have the same size.
fd_step: float (optional)
Step-size to use for finite difference calculation. If no value is
given, the differentiator will use its own default. If target is an
array, this may also be an array, but must have the same size.
name: str (optional)
Name used to refer to the parameter in place of the name of the
variable referred to in the parameter string.
This is sometimes useful if, for example, multiple entries in the
same array variable are declared as parameters.
scope: object (optional)
The object to be used as the scope when evaluating the expression.
If neither "low" nor "high" is specified, the min and max will
default to the values in the metadata of the variable being
referenced.
"""
if isinstance(target, (ParameterBase, ParameterGroup)):
self._parameters[target.name] = target
target.override(low, high, scaler, adder, start, fd_step, name)
else:
if isinstance(target, basestring):
names = [target]
key = target
else:
names = target
key = tuple(target)
if name is not None:
key = name
dups = set(self.list_param_targets()).intersection(names)
if len(dups) == 1:
self.parent.raise_exception(
"'%s' is already a Parameter"
" target" % dups.pop(), ValueError)
elif len(dups) > 1:
self.parent.raise_exception(
"%s are already Parameter targets" % sorted(list(dups)),
ValueError)
if key in self._parameters:
self.parent.raise_exception("%s is already a Parameter" % key,
ValueError)
try:
_scope = self._get_scope(scope)
if len(names) == 1:
target = self._create(names[0], low, high, scaler, adder,
start, fd_step, key, _scope)
else: # defining a ParameterGroup
parameters = [
self._create(n, low, high, scaler, adder, start,
fd_step, key, _scope) for n in names
]
types = set([p.valtypename for p in parameters])
if len(types) > 1:
raise ValueError("Can't add parameter %s because "
"%s are not all of the same type" %
(key, " and ".join(names)))
target = ParameterGroup(parameters)
self._parameters[key] = target
except Exception:
self.parent.reraise_exception(info=sys.exc_info())
self.parent.config_changed()
def _create(self, target, low, high, scaler, adder, start, fd_step, key,
scope):
""" Create one Parameter or ArrayParameter. """
try:
expreval = ExprEvaluator(target, scope)
except Exception as err:
raise err.__class__("Can't add parameter: %s" % err)
if not expreval.is_valid_assignee():
raise ValueError("Can't add parameter: '%s' is not a"
" valid parameter expression" % expreval.text)
try:
val = expreval.evaluate()
except Exception as err:
val = None # Let Parameter code sort out why.
name = key[0] if isinstance(key, tuple) else key
if isinstance(val, ndarray):
return ArrayParameter(target,
low=low,
high=high,
scaler=scaler,
adder=adder,
start=start,
fd_step=fd_step,
name=name,
scope=scope,
_expreval=expreval,
_val=val,
_allowed_types=self._allowed_types)
else:
return Parameter(target,
low=low,
high=high,
scaler=scaler,
adder=adder,
start=start,
fd_step=fd_step,
name=name,
scope=scope,
_expreval=expreval,
_val=val,
_allowed_types=self._allowed_types)
def remove_parameter(self, name):
"""Removes the parameter with the given name."""
param = self._parameters.get(name)
if param:
del self._parameters[name]
else:
self.parent.raise_exception(
"Trying to remove parameter '%s' "
"that is not in this driver." % (name, ), AttributeError)
self.parent.config_changed()
def config_parameters(self):
"""Reconfigure parameters from potentially changed targets."""
for param in self._parameters.values():
param.configure()
def get_references(self, name):
"""Return references to component `name` in preparation for subsequent
:meth:`restore_references` call.
name: string
Name of component being removed.
"""
refs = OrderedDict()
for pname, param in self._parameters.items():
if name in param.get_referenced_compnames():
refs[pname] = param
return refs
def remove_references(self, name):
"""Remove references to component `name`.
name: string
Name of component being removed.
"""
to_remove = []
for pname, param in self._parameters.items():
if name in param.get_referenced_compnames():
to_remove.append(pname)
for pname in to_remove:
self.remove_parameter(pname)
def restore_references(self, refs):
"""Restore references to component `name` from `refs`.
refs: object
Value returned by :meth:`get_references`.
"""
for pname, param in refs.items():
try:
self.add_parameter(param)
except Exception as err:
self.parent._logger.warning(
"Couldn't restore parameter '%s': %s" % (pname, str(err)))
def list_param_targets(self):
"""Returns a list of parameter targets. Note that this
list may contain more entries than the list of Parameter,
ParameterGroup, and ArrayParameter objects since ParameterGroup
instances have multiple targets.
"""
targets = []
for param in self._parameters.values():
targets.extend(param.targets)
return targets
def list_param_group_targets(self):
"""Returns a list of tuples that contain the targets for each
parameter group.
"""
targets = []
for param in self.get_parameters().values():
targets.append(tuple(param.targets))
return targets
def clear_parameters(self):
"""Removes all parameters."""
for name in self._parameters.keys():
self.remove_parameter(name)
self._parameters = OrderedDict()
def get_parameters(self):
"""Returns an ordered dict of parameter objects."""
return self._parameters
def total_parameters(self):
"""Returns the total number of values to be set."""
return sum([param.size for param in self._parameters.values()])
def init_parameters(self):
"""Sets all parameters to their start value if a
start value is given
"""
scope = self._get_scope()
for param in self._parameters.itervalues():
if param.start is not None:
param.set(param.start, scope)
def set_parameter_by_name(self, name, value, case=None, scope=None):
class Config(object):
# pylint: disable-msg=R0902
"""This is the entry point, this class will contains all Section and
Elements."""
def __init__(self, prog_name, config_file=None, desc=None,
mandatory=False, use_config_file=True):
self.prog_name = prog_name
self.config_file = config_file
self.use_config_file = use_config_file
self._desc = desc
self.mandatory = mandatory
self.sections = OrderedDict()
self._default_section = self.add_section(SimpleSection("DEFAULT"))
self.parser = None
self.file_parser = ConfigParser.SafeConfigParser()
def add_section(self, section):
"""Add a new Section object to the config. Should be a subclass of
_AbstractSection."""
if not issubclass(section.__class__, _AbstractSection):
raise TypeError("argument should be a subclass of Section")
self.sections[section.get_key_name()] = section
return section
def get_section(self, name):
if name.lower() == "default":
return self._default_section
return self.sections.get(name)
def get_default_section(self):
"""This method will return default section object"""
return self._default_section
def load(self, exit_on_failure=False):
"""One you have added all your configuration data (Section, Element,
...) you need to load data from the config file."""
if self.use_config_file:
self._load(exit_on_failure)
def _load(self, exit_on_failure):
"""One you have added all your configuration data (Section, Element,
...) you need to load data from the config file."""
# pylint: disable-msg=W0621
log = logging.getLogger('argtoolbox')
discoveredFileList = []
if self.config_file:
if isinstance(self.config_file, types.UnicodeType):
discoveredFileList = self.file_parser.read(self.config_file)
else:
discoveredFileList = self.file_parser.readfp(
self.config_file,
"file descriptor")
else:
defaultFileList = []
defaultFileList.append(self.prog_name + ".cfg")
defaultFileList.append(
os.path.expanduser('~/.' + self.prog_name + '.cfg'))
defaultFileList.append('/etc/' + self.prog_name + '.cfg')
log.debug("defaultFileList: " + str(defaultFileList))
discoveredFileList = self.file_parser.read(defaultFileList)
log.debug("discoveredFileList: " + str(discoveredFileList))
if self.mandatory and len(discoveredFileList) < 1:
msg = "The required config file was missing."
msg += " Default config files : " + str(defaultFileList)
log.error(msg)
raise EnvironmentError(msg)
log.debug("loading configuration ...")
if exit_on_failure:
for s in self.sections.values():
log.debug("loading section : " + s.get_section_name())
try:
s.load(self.file_parser)
except ValueError:
sys.exit(1)
else:
for s in self.sections.values():
log.debug("loading section : " + s.get_section_name())
s.load(self.file_parser)
log.debug("configuration loaded.")
def get_parser(self, **kwargs):
"""This method will create and return a new parser with prog_name,
description, and a config file argument.
"""
self.parser = argparse.ArgumentParser(prog=self.prog_name,
description=self._desc,
add_help=False, **kwargs)
# help is removed because parser.parse_known_args() show help,
# often partial help. help action will be added during
# reloading step for parser.parse_args()
if self.use_config_file:
self.parser.add_argument('--config-file',
action="store",
help="Other configuration file.")
return self.parser
def reload(self, hooks=None):
"""This method will reload the configuration using input argument
from the command line interface.
1. pasing arguments
2. applying hooks
3. addding help argument
4. reloading configuration using cli argument like a configuration
file name.
"""
#from argcomplete import debug
# Parsing the command line looking for the previous options like
# configuration file name or server section. Extra arguments
# will be store into argv.
args = None
if os.environ.get('_ARGCOMPLETE'):
# During argcomplete completion, parse_known_args will return an
# empty Namespace. In this case, we feed the previous function with
# data comming from the input completion data
compline = os.environ.get('COMP_LINE')
args = self.parser.parse_known_args(compline.split()[1:])[0]
else:
args = self.parser.parse_known_args()[0]
if hooks is not None:
if isinstance(hooks, list):
for h in hooks:
if isinstance(h, SectionHook):
h(args)
else:
if isinstance(hooks, SectionHook):
hooks(args)
# After the first argument parsing, for configuration reloading,
# we can add the help action.
self.parser.add_argument('-h', '--help', action='help',
default=argparse.SUPPRESS,
help='show this help message and exit')
# Reloading
if self.use_config_file:
# pylint: disable-msg=W0621
log = logging.getLogger('argtoolbox')
log.debug("reloading configuration ...")
if args.config_file:
self.file_parser = ConfigParser.SafeConfigParser()
discoveredFileList = self.file_parser.read(args.config_file)
log.debug("discoveredFileList: " + str(discoveredFileList))
for s in self.sections.values():
log.debug("loading section : " + s.get_section_name())
s.reset()
s.load(self.file_parser)
log.debug("configuration reloaded.")
def __getattr__(self, name):
if name.lower() == "default":
return self._default_section
s = self.sections.get(name)
if s is not None:
return s
else:
raise AttributeError("'%(class)s' object has no attribute \
'%(name)s'" % {"name": name, "class": self.__class__.__name__})
def __str__(self):
res = []
res.append("Configuration of %(prog_name)s : " % self.__dict__)
for s in self.sections.values():
res.append("".join(s.get_representation("\t")))
return "\n".join(res)
def write_default_config_file(self, output, comments=True):
"""This method write a sample file, with attributes, descriptions,
sample values, required flags, using the configuration object
properties.
"""
if self.use_config_file:
# pylint: disable-msg=W0621
log = logging.getLogger('argtoolbox')
with open(output, 'w') as f:
if comments:
f.write("#####################################\n")
f.write("# Description :\n")
f.write("# -------------\n# ")
for i in self._desc.split('\n'):
f.write("# ")
f.write(i)
f.write("\n")
f.write("\n\n")
for s in self.sections.values():
log.debug("loading section : " + s.get_section_name())
s.write_config_file(f, comments)
log.debug("config file generation completed : " + str(output))
def __call__(self):
request = self.request
params = self.get_base_params()
roles = OrderedDict([(str(ro.id), ro)
for ro in session.query(auth.Role
).order_by(auth.Role.name).all()])
permissions = OrderedDict([(str(p.id), p)
for p in session.query(
auth.Permission
).order_by(auth.Permission.name).all()])
data = OrderedDict()
params['data'] = data
data['permissions'] = permissions
data['roles'] = roles
form = construct_schema_acls(request, permissions=permissions, roles=roles)
if request.method == 'POST':
try:
controls = request.POST.items()
data = form.validate(controls)
perms = [permissions.get(pid, None)
for pid in data['roles']
if permissions.get(pid, None)]
modified, error = False, False
try:
for permission in perms:
mapping = data['roles'][str(permission.id)]
rles = [roles.get(rid, None)
for rid in mapping
if rid in roles]
for role in rles:
# maybe unactivate role
if not mapping[str(role.id)]:
if permission in role.global_permissions:
modified = True
role.global_permissions.pop(
role.global_permissions.index(
permission
)
)
session.add(role)
session.commit()
# maybe activate role
else:
if not permission in role.global_permissions:
modified = True
role.global_permissions.append(permission)
session.add(role)
session.commit()
except Exception, e:
request.session.flash(_('Something goes wrong while saving access parameters: %s')%e, 'error')
error = True
# sql failure !
try:
session.rollback()
except:
pass
if modified:
if not error:
request.session.flash(_('Access parameters have been saved'), 'info')
form = construct_schema_acls(request, permissions=permissions, roles=roles)
except ValidationFailure, e:
params['form'] = e.render()
def graph(host,interface,metric,timeperiod,viewOption,function):
graphiteMetricBase = "diamond.mjulian.interface.devices"
hosts = getDevices()
# Ordereddict to show time periods in this order
# Key is what's shown on the page, values get passed to Graphite API
timeperiods = OrderedDict()
timeperiods['15m'] = ['-15min', 'now']
timeperiods['1h'] = ['-1h', 'now']
timeperiods['24h'] = ['-24h', 'now']
timeperiods['7d'] = ['-7d', 'now']
timeperiods['30d'] = ['-30d', 'now']
timeperiods['6mo'] = ['-6mon', 'now']
timeperiods['1y'] = ['-1y', 'now']
# Ordereddict to show view options in this order
# Key is used in the URL, value gets shown on the page
viewOptions = OrderedDict()
viewOptions['bps'] = 'Bits/sec'
viewOptions['Bps'] = 'Bytes/sec'
viewOptions['pps'] = 'Packets/sec'
cleanedInterfaceName = getInterfaceName(hosts, host, interface)
# Template uses 'default' for timeperiod, so we we set it here
if timeperiod == "default":
timeperiod = "1h"
# Same as above. I used Bps as default because the counter from SNMP is already in octets
if viewOption == "default":
if metrics.get(metric)[2] == "packets":
viewOption = "pps"
else:
viewOption = "Bps"
# We use this variable to determine what view options are shown. See metrics dict above
metricUnit = metrics.get(metric)[2]
# From here on, we start building the URL to pass to Graphite
rxTargetOverlay = None
txTargetOverlay = None
rxTarget = "%s.%s.interface.%s.%s" % (graphiteMetricBase, host, cleanedInterfaceName, metrics.get(metric)[0])
txTarget = "%s.%s.interface.%s.%s" % (graphiteMetricBase, host, cleanedInterfaceName, metrics.get(metric)[1])
rxTarget = "scaleToSeconds(" + rxTarget + ",1)"
txTarget = "scaleToSeconds(" + txTarget + ",1)"
if viewOption == "bps":
rxTarget = "scale(" + rxTarget + ",0.125)"
txTarget = "scale(" + txTarget + ",0.125)"
if function == "average":
rxTarget = "movingAverage(" + rxTarget + ",30)"
txTarget = "movingAverage(" + txTarget + ",30)"
if function == "95th":
rxTargetOverlay = "nPercentile(" + rxTarget + ",95)"
txTargetOverlay = "nPercentile(" + txTarget + ",95)"
rxTargetOverlay = "alias(" + rxTargetOverlay + ",\"rx - 95th\")"
txTargetOverlay = "alias(" + txTargetOverlay + ",\"tx - 95th\")"
rxTarget = "alias(" + rxTarget + ",\"rx\")"
txTarget = "alias(" + txTarget + ",\"tx\")"
if rxTargetOverlay and txTargetOverlay:
graphLink = "http://" + graphiteServer + "/render?from=" + timeperiods.get(timeperiod)[0] + "&until=" + timeperiods.get(timeperiod)[1] + "&width=900&height=450" + "&target=" + rxTarget + "&target=" + txTarget + "&target=" + rxTargetOverlay + "&target=" + txTargetOverlay + "&hideGrid=true&fontSize=14&margin=25&vtitle=" + viewOptions.get(viewOption)
else:
graphLink = "http://" + graphiteServer + "/render?from=" + timeperiods.get(timeperiod)[0] + "&until=" + timeperiods.get(timeperiod)[1] + "&width=900&height=450" + "&target=" + rxTarget + "&target=" + txTarget + "&hideGrid=true&fontSize=14&margin=25&vtitle=" + viewOptions.get(viewOption)
return render_template('graph.html',
metricUnit=metricUnit,
viewOptions=viewOptions,
metrics=metrics,
timeperiods=timeperiods,
devices=hosts,
host=host,
interface=interface,
metric=metric,
timeperiod=timeperiod,
viewOption=viewOption,
function=function,
graph_link=graphLink
)
class SvxlinkTypeContainer(object):
def __init__(self, type_name, section_name, valid_options, data=None):
"""Type container for Svxlink Types.
It serves as an abstract class. We need to check for valid options
when setting sections. Instead of copying/pasting the code, this
abstract class checks for valid options when setting an item in class.
__dbase__ is an internal representation of key/values with an
OrderedDict. If data is provided, __dbase__ is filled with this
data. Otherwise, we know that the section is created from
scratch so we add TYPE accordingly to type_name argument by
default. "Data" is an array of tuple because it's the type that
ConfigParser returns. For the ease of use, it's directly used in
this way.
Note that svxlink.conf requires UPPERCASE for options in sections.
Section names can be arbitrary, however, options are presented
upper-case. Whenever you set an option name, it will be converted to
uppercase. It does not matter the way you set options. For
example:
f = SvxlinkTypeNet("foo")
f["tcp_PORT" = 5220
is converted to:
f["TCP_PORT"] 5220
so it's still valid to use as long as option is present in
VALID_OPTIONS
"""
# TODO: Implement a checker function. Now, we only check for
# valid options, not the values themselves. Later, we would need
# to check for values so we need to implement a function that
# checks it. Additionally, this function will be unique to the
# classes that extends SvxlinkTypeContainer. So it should be
# optional in __init__()
self._VALID_OPTIONS = valid_options
self._TYPE_NAME = type_name
self._SECTION_NAME = section_name
# internal ordered dictionary for storing key/values typical to
# section
self.__dbase__ = OrderedDict()
if data is None:
self.__dbase__.update({"TYPE": type_name})
else:
# start adding values that are in tuple to __dict__
for tuple_item in data:
self.__check_item_and_update(tuple_item[0],
tuple_item[1])
def __check_item_and_update(self, key, val):
"""Checks the item in VALID_OPTIONS and updates __dbase__ if the
option is valid.
"""
if not key.upper() in self._VALID_OPTIONS:
raise ValueError("Option '%s' is not valid for '%s'" %
(key, self._SECTION_NAME))
self.__dbase__.update({key.upper(): val})
def __str__(self):
return "<SvxlinkType%s: %s>" % (self._TYPE_NAME, self._SECTION_NAME)
def __getitem__(self, key):
return self.__dbase__.get(key.upper());
def __setitem__(self, key, val):
self.__check_item_and_update(key, val)
def __eq__(self, other):
# compare any object with our section name.
return other == self._SECTION_NAME
def get_section_name(self):
"""Returns a section name"""
return self._SECTION_NAME
def has_option(self, option):
"""Checks if there is an option in __dict__
"""
return self.__dbase__.has_key(option.upper())
def items(self):
"""Returns ConfigParser compatable output for items in this section.
The output is an array of tuples such as:
[(tcp_port, 5220), (type, "Net")]
"""
# iterate over __dict__, do not take variables that start with _
# into account.
output = []
for item in self.__dbase__:
if not item.startswith("_"):
output.append((item, self[item]))
return output
def is_online(self):
"""An abstract method for checking if the section is up.
This method should be implemented in SvxlinkType objects. By
default, it returns true. For example, for a
SvxlinkTypeRepeater, is_online() method can check if the
repeater is in LOGICS option in GLOBAL section. For a Local
device, this method can check if the card is listed by ALSA and
can be accessed without a problem.
"""
return True
class Application(object):
"""Poor WSGI application which is called by WSGI server.
Working of is describe in PEP 0333. This object store route dispatch table,
and have methods for it's using and of course __call__ method for use
as WSGI application.
"""
__instances = []
def __init__(self, name="__main__"):
"""Application class is per name singleton.
That means, there could be exist only one instance with same name.
"""
if Application.__instances.count(name):
raise RuntimeError('Application with name %s exist yet.' % name)
Application.__instances.append(name)
# Application name
self.__name = name
# list of pre and post process handlers
self.__pre = []
self.__post = []
# dhandlers table for default handers on methods {METHOD_GET: handler}
self.__dhandlers = {}
# handlers table of simple paths: {'/path': {METHOD_GET: handler}}
self.__handlers = {}
self.__filters = {
':int': (r'-?\d+', int),
':float': (r'-?\d+(\.\d+)?', float),
':word': (r'\w+', uni),
':hex': (r'[0-9a-fA-F]+', str),
':re:': (None, uni),
'none': (r'[^/]+', uni)
}
# handlers of regex paths: {r'/user/([a-z]?)': {METHOD_GET: handler}}
self.__rhandlers = OrderedDict()
# http state handlers: {HTTP_NOT_FOUND: {METHOD_GET: my_404_handler}}
self.__shandlers = {}
# -- Application variable
self.__config = {
'auto_args': True,
'auto_form': True,
'auto_json': True,
'keep_blank_values': 0,
'strict_parsing': 0,
'json_content_types': [
'application/json',
'application/javascript',
'application/merge-patch+json'],
'form_content_types': [
'application/x-www-form-urlencoded',
'multipart/form-data'
],
'auto_cookies': True,
'debug': 'Off',
'document_root': '',
'document_index': 'Off',
'secret_key': '%s%s%s%s' %
(__version__, version, getcwd(),
''.join(str(x) for x in uname()))
}
try:
self.__log_level = levels[environ.get('poor_LogLevel',
'warn').lower()]
except:
self.__log_level = LOG_WARNING
self.log_error('Bad poor_LogLevel, default is warn.', LOG_WARNING)
# endtry
# enddef
def __regex(self, match):
groups = match.groups()
_filter = str(groups[1]).lower()
if _filter in self.__filters:
regex = self.__filters[_filter][0]
elif _filter[:4] == ':re:': # :re: filter have user defined regex
regex = _filter[4:]
else:
try:
regex = self.__filters[_filter][0]
except KeyError:
raise RuntimeError("Undefined route group filter '%s'" %
_filter)
return "(?P<%s>%s)" % (groups[0], regex)
# enddef
def __convertor(self, _filter):
_filter = str(_filter).lower()
_filter = ':re:' if _filter[:4] == ':re:' else _filter
try:
return self.__filters[_filter][1]
except KeyError:
raise RuntimeError("Undefined route group filter '%s'" % _filter)
@property
def name(self):
"""Return application name."""
return self.__name
@property
def filters(self):
"""Copy of filter table.
Filter table contains regular expressions and convert functions,
see Application.set_filter and Application.route.
Default filters are:
:int - match number and convert it to int
:float - match number and convert it to float
:word - match one unicoee word
:hex - match hexadecimal value and convert it to str
:re: - match user defined regular expression
none - match any string withount '/' character
For more details see {/debug-info} page of your application, where
you see all filters with regular expression definition.
"""
return self.__filters.copy()
@property
def pre(self):
"""Tuple of table with pre-process handlers.
See Application.pre_process.
"""
return tuple(self.__pre)
@property
def post(self):
"""Tuple of table with post-process handlers.
See Application.post_process.
"""
return tuple(self.__post)
@property
def dhandlers(self):
"""Copy of table with default handlers.
See Application.set_default
"""
return self.__dhandlers.copy()
@property
def handlers(self):
"""Copy of table with static handlers.
See Application.route.
"""
return self.__handlers.copy()
@property
def rhandlers(self):
"""Copy of table with regular expression handlers.
See Application.route and Application.rroute.
"""
return self.__rhandlers.copy()
@property
def shandlers(self):
"""Copy of table with http state aka error handlers.
See Application.http_state
"""
return self.__shandlers.copy()
@property
def auto_args(self):
"""Automatic parsing request arguments from uri.
If it is True (default), Request object do automatic parsing request
uri to its args variable.
"""
return self.__config['auto_args']
@auto_args.setter
def auto_args(self, value):
self.__config['auto_args'] = bool(value)
@property
def auto_form(self):
"""Automatic parsing arguments from request body.
If it is True (default) and method is POST, PUT or PATCH, and
request content type is one of form_content_types, Request
object do automatic parsing request body to its form variable.
"""
return self.__config['auto_form']
@auto_form.setter
def auto_form(self, value):
self.__config['auto_form'] = bool(value)
@property
def auto_json(self):
"""Automatic parsing JSON from request body.
If it is True (default), method is POST, PUT or PATCH and request
content type is one of json_content_types, Request object do
automatic parsing request body to json variable.
"""
return self.__config['auto_json']
@auto_json.setter
def auto_json(self, value):
self.__config['auto_json'] = bool(value)
@property
def auto_cookies(self):
"""Automatic parsing cookies from request headers.
If it is True (default) and Cookie request header was set,
SimpleCookie object was paresed to Request property cookies.
"""
return self.__config['auto_cookies']
@auto_cookies.setter
def auto_cookies(self, value):
self.__config['auto_cookies'] = bool(value)
@property
def debug(self):
"""Application debug as another way how to set poor_Debug.
This setting will be rewrite by poor_Debug environment variable.
"""
return self.__config['debug'] == 'On'
@debug.setter
def debug(self, value):
self.__config['debug'] = 'On' if bool(value) else 'Off'
@property
def document_root(self):
"""Application document_root as another way how to set poor_DocumentRoot.
This setting will be rewrite by poor_DocumentRoot environ variable.
"""
return self.__config['document_root']
@document_root.setter
def document_root(self, value):
self.__config['document_root'] = value
@property
def document_index(self):
"""Application document_root as another way how to set poor_DocumentRoot.
This setting will be rewrite by poor_DocumentRoot environ variable.
"""
return self.__config['document_index'] == 'On'
@document_index.setter
def document_index(self, value):
self.__config['document_index'] = 'On' if bool(value) else 'Off'
@property
def secret_key(self):
"""Application secret_key could be replace by poor_SecretKey in request.
Secret key is used by PoorSession class. It is generate from
some server variables, and the best way is set to your own long
key."""
return self.__config['secret_key']
@secret_key.setter
def secret_key(self, value):
self.__config['secret_key'] = value
@property
def keep_blank_values(self):
"""Keep blank values in request arguments.
If it is 1 (0 is default), automatic parsing request uri or body
keep blank values as empty string.
"""
return self.__config['keep_blank_values']
@keep_blank_values.setter
def keep_blank_values(self, value):
self.__config['keep_blank_values'] = int(value)
@property
def strict_parsing(self):
"""Strict parse request arguments.
If it is 1 (0 is default), automatic parsing request uri or body
raise with exception on parsing error.
"""
return self.__config['strict_parsing']
@strict_parsing.setter
def strict_parsing(self, value):
self.__config['strict_parsing'] = int(value)
@property
def json_content_types(self):
"""Copy of json content type list.
Containt list of strings as json content types, which is use for
testing, when automatics Json object is create from request body.
"""
return self.__config['json_content_types']
@property
def form_content_types(self):
"""Copy of form content type list.
Containt list of strings as form content types, which is use for
testing, when automatics Form object is create from request body.
"""
return self.__config['form_content_types']
def set_filter(self, name, regex, convertor=uni):
"""Create new filter or overwrite builtins.
Arguments:
name - Name of filter which is used in route or set_route
method.
regex - regular expression which used for filter
convertor - convertor function or class, which gets unicode in
input. Default is uni function, which is wrapper
to unicode string.
app.set_filter('uint', r'\d+', int)
"""
name = ':'+name if name[0] != ':' else name
self.__filters[name] = (regex, convertor)
def pre_process(self):
"""Append pre process hendler.
This is decorator for function to call before each request.
@app.pre_process()
def before_each_request(req):
...
"""
def wrapper(fn):
self.__pre.append(fn)
return fn
return wrapper
# enddef
def add_pre_process(self, fn):
"""Append pre proccess handler.
Method adds function to list functions which is call before each
request.
app.add_pre_process(before_each_request)
"""
self.__pre.append(fn)
# enddef
def post_process(self):
"""Append post process handler.
This decorator append function to be called after each request,
if you want to use it redefined all outputs.
@app.pre_process()
def after_each_request(req):
...
"""
def wrapper(fn):
self.__post.append(fn)
return fn
return wrapper
# enddef
def add_post_process(self, fn):
"""Append post process handler.
Method for direct append function to list functions which are called
after each request.
app.add_post_process(after_each_request)
"""
self.__post.append(fn)
# enddef
def default(self, method=METHOD_HEAD | METHOD_GET):
"""Set default handler.
This is decorator for default handler for http method (called before
error_not_found).
@app.default(METHOD_GET_POST)
def default_get_post(req):
# this function will be called if no uri match in internal
# uri table with method. It's similar like not_found error,
# but without error
...
"""
def wrapper(fn):
self.set_default(fn, method)
return wrapper
# enddef
def set_default(self, fn, method=METHOD_HEAD | METHOD_GET):
"""Set default handler.
Set fn default handler for http method called befor error_not_found.
app.set_default(default_get_post, METHOD_GET_POST)
"""
for m in methods.values():
if method & m:
self.__dhandlers[m] = fn
# enddef
def pop_default(self, method):
"""Pop default handler for method."""
return self.__dhandlers(method)
def route(self, uri, method=METHOD_HEAD | METHOD_GET):
"""Wrap function to be handler for uri and specified method.
You can define uri as static path or as groups which are hand
to handler as next parameters.
# static uri
@app.route('/user/post', method=METHOD_POST)
def user_create(req):
...
# group regular expression
@app.route('/user/<name>')
def user_detail(req, name):
...
# group regular expression with filter
@app.route('/<surname:word>/<age:int>')
def surnames_by_age(req, surname, age):
...
# group with own regular expression filter
@app.route('/<car:re:\w+>/<color:re:#[\da-fA-F]+>')
def car(req, car, color):
...
If you can use some name of group which is python keyword, like class,
you can use **kwargs syntax:
@app.route('/<class>/<len:int>')
def classes(req, **kwargs):
return "'%s' class is %d lenght." % \
(kwargs['class'], kwargs['len'])
Be sure with ordering of call this decorator or set_route function with
groups regular expression. Regular expression routes are check with the
same ordering, as you create internal table of them. First match stops
any other searching. In fact, if groups are detect, they will be
transfer to normal regular expression, and will be add to second
internal table.
"""
def wrapper(fn):
self.set_route(uri, fn, method)
return fn
return wrapper
# enddef
def set_route(self, uri, fn, method=METHOD_HEAD | METHOD_GET):
"""Set handler for uri and method.
Another way to add fn as handler for uri. See Application.route
documentation for details.
app.set_route('/use/post', user_create, METHOD_POST)
"""
uri = uni(uri)
if re_filter.search(uri):
r_uri = re_filter.sub(self.__regex, uri) + '$'
convertors = tuple((g[0], self.__convertor(g[1]))
for g in (m.groups()
for m in re_filter.finditer(uri)))
self.set_rroute(r_uri, fn, method, convertors)
else:
if uri not in self.__handlers:
self.__handlers[uri] = {}
for m in methods.values():
if method & m:
self.__handlers[uri][m] = fn
# enddef
def pop_route(self, uri, method):
"""Pop handler for uri and method from handers table.
Method must be define unique, so METHOD_GET_POST could not be use.
If you want to remove handler for both methods, you must call pop route
for each method state.
"""
uri = uni(uri)
if re_filter.search(uri):
r_uri = re_filter.sub(self.__regex, uri) + '$'
return self.pop_rroute(r_uri, method)
else:
handlers = self.__handlers.get(uri, {})
rv = handlers.pop(method)
if not handlers: # is empty
self.__handlers.pop(uri, None)
return rv
def is_route(self, uri):
"""Check if uri have any registered record."""
uri = uni(uri)
if re_filter.search(uri):
r_uri = re_filter.sub(self.__regex, uri) + '$'
return self.is_rroute(r_uri)
return uri in self.__handlers
def rroute(self, ruri, method=METHOD_HEAD | METHOD_GET):
"""Wrap function to be handler for uri defined by regular expression.
Both of function, rroute and set_rroute store routes to special
internal table, which is another to table of static routes.
@app.rroute(r'/user/\w+') # simple regular expression
def any_user(req):
...
@app.rroute(r'/user/(?P<user>\w+)') # regular expression with
def user_detail(req, user): # groups
...
Be sure with ordering of call this decorator or set_rroute function.
Regular expression routes are check with the same ordering, as you
create internal table of them. First match stops any other searching.
"""
def wrapper(fn):
self.set_rroute(ruri, fn, method)
return fn
return wrapper
# enddef
def set_rroute(self, r_uri, fn, method=METHOD_HEAD | METHOD_GET,
convertors=()):
"""Set hanlder for uri defined by regular expression.
Another way to add fn as handler for uri defined by regular expression.
See Application.rroute documentation for details.
app.set_rroute('/use/\w+/post', user_create, METHOD_POST)
This method is internally use, when groups are found in static route,
adding by route or set_route method.
"""
r_uri = re.compile(r_uri, re.U)
if r_uri not in self.__rhandlers:
self.__rhandlers[r_uri] = {}
for m in methods.values():
if method & m:
self.__rhandlers[r_uri][m] = (fn, convertors)
# enddef
def pop_rroute(self, r_uri, method):
"""Pop handler and convertors for uri and method from handlers table.
For mor details see Application.pop_route.
"""
r_uri = re.compile(r_uri, re.U)
handlers = self.__rhandlers.get(r_uri, {})
rv = handlers.pop(method)
if not handlers: # is empty
self.__rhandlers.pop(r_uri, None)
return rv
def is_rroute(self, r_uri):
"""Check if regular expression uri have any registered record."""
r_uri = re.compile(r_uri, re.U)
return r_uri in self.__rhandlers
def http_state(self, code, method=METHOD_HEAD | METHOD_GET | METHOD_POST):
"""Wrap function to handle http status codes like http errors."""
def wrapper(fn):
self.set_http_state(code, fn, method)
return wrapper
# enddef
def set_http_state(self, code, fn,
method=METHOD_HEAD | METHOD_GET | METHOD_POST):
"""Set fn as handler for http state code and method."""
if code not in self.__shandlers:
self.__shandlers[code] = {}
for m in methods.values():
if method & m:
self.__shandlers[code][m] = fn
# enddef
def pop_http_state(self, code, method):
"""Pop handerl for http state and method.
As Application.pop_route, for pop multimethod handler, you must call
pop_http_state for each method.
"""
handlers = self.__shandlers(code, {})
return handlers.pop(method)
def error_from_table(self, req, code):
"""Internal method, which is called if error was accured.
If status code is in Application.shandlers (fill with http_state
function), call this handler.
"""
if code in self.__shandlers \
and req.method_number in self.__shandlers[code]:
try:
handler = self.__shandlers[code][req.method_number]
if 'uri_handler' not in req.__dict__:
req.uri_rule = '_%d_error_handler_' % code
req.uri_handler = handler
self.handler_from_pre(req) # call pre handlers now
handler(req)
except:
internal_server_error(req)
elif code in default_shandlers:
handler = default_shandlers[code][METHOD_GET]
handler(req)
else:
not_implemented(req, code)
# enddef
def handler_from_default(self, req):
"""Internal method, which is called if no handler is found."""
if req.method_number in self.__dhandlers:
req.uri_rule = '_default_handler_'
req.uri_handler = self.__dhandlers[req.method_number]
self.handler_from_pre(req) # call pre handlers now
retval = self.__dhandlers[req.method_number](req)
if retval != DECLINED:
raise SERVER_RETURN(retval)
# enddef
def handler_from_pre(self, req):
"""Internal method, which run all pre (pre_proccess) handlers.
This method was call before end-point route handler.
"""
for fn in self.__pre:
fn(req)
def handler_from_table(self, req):
"""Call right handler from handlers table (fill with route function).
If no handler is fined, try to find directory or file if Document Root,
resp. Document Index is set. Then try to call default handler for right
method or call handler for status code 404 - not found.
"""
# static routes
if req.uri in self.__handlers:
if req.method_number in self.__handlers[req.uri]:
handler = self.__handlers[req.uri][req.method_number]
req.uri_rule = req.uri # nice variable for pre handlers
req.uri_handler = handler
self.handler_from_pre(req) # call pre handlers now
retval = handler(req) # call right handler now
# return text is allowed
if isinstance(retval, str) \
or (_unicode_exist and isinstance(retval, unicode)):
req.write(retval, 1) # write data and flush
retval = DONE
if retval != DECLINED:
raise SERVER_RETURN(retval or DONE) # could be state.DONE
else:
raise SERVER_RETURN(HTTP_METHOD_NOT_ALLOWED)
# endif
# endif
# regular expression
for ruri in self.__rhandlers.keys():
match = ruri.match(req.uri)
if match and req.method_number in self.__rhandlers[ruri]:
handler, convertors = self.__rhandlers[ruri][req.method_number]
req.uri_rule = ruri.pattern # nice variable for pre handlers
req.uri_handler = handler
self.handler_from_pre(req) # call pre handlers now
if len(convertors):
# create OrderedDict from match insead of dict for
# convertors applying
req.groups = OrderedDict(
(g, c(v))for ((g, c), v) in zip(convertors,
match.groups()))
retval = handler(req, *req.groups.values())
else:
req.groups = match.groupdict()
retval = handler(req, *match.groups())
# return text is allowed
if isinstance(retval, str) \
or (_unicode_exist and isinstance(retval, unicode)):
req.write(retval, 1) # write data and flush
retval = DONE
if retval != DECLINED:
raise SERVER_RETURN(retval or DONE) # could be state.DONE
# endif - no METHOD_NOT_ALLOWED here
# endfor
# try file or index
if req.document_root():
rfile = "%s%s" % (uni(req.document_root()),
path.normpath("%s" % uni(req.uri)))
if not path.exists(rfile):
if req.debug and req.uri == '/debug-info': # work if debug
req.uri_rule = '_debug_info_'
req.uri_handler = debug_info
self.handler_from_pre(req) # call pre handlers now
raise SERVER_RETURN(debug_info(req, self))
self.handler_from_default(req) # try default
raise SERVER_RETURN(HTTP_NOT_FOUND) # not found
# return file
if path.isfile(rfile) and access(rfile, R_OK):
req.uri_rule = '_send_file_'
req.uri_handler = send_file
self.handler_from_pre(req) # call pre handlers now
req.log_error("Return file: %s" % req.uri, LOG_INFO)
raise SERVER_RETURN(send_file(req, rfile))
# return directory index
if req.document_index and path.isdir(rfile) \
and access(rfile, R_OK):
req.log_error("Return directory: %s" % req.uri, LOG_INFO)
req.uri_rule = '_directory_index_'
req.uri_handler = directory_index
self.handler_from_pre(req) # call pre handlers now
raise SERVER_RETURN(directory_index(req, rfile))
raise SERVER_RETURN(HTTP_FORBIDDEN)
# endif
if req.debug and req.uri == '/debug-info':
req.uri_rule = '_debug_info_'
req.uri_handler = debug_info
self.handler_from_pre(req) # call pre handlers now
raise SERVER_RETURN(debug_info(req, self))
self.handler_from_default(req)
req.log_error("404 Not Found: %s" % req.uri, LOG_ERR)
raise SERVER_RETURN(HTTP_NOT_FOUND)
# enddef
def __request__(self, environ, start_response):
"""Create Request instance and return wsgi response.
This method create Request object, call handlers from
Application.__pre (Application.handler_from_pre),
uri handler (handler_from_table), default handler
(Application.handler_from_default) or error handler
(Application.error_from_table), and handlers from
Application.__post.
"""
req = Request(environ, start_response, self.__config)
try:
self.handler_from_table(req)
except SERVER_RETURN as e:
code = e.args[0]
if code in (OK, HTTP_OK, DONE):
pass
# XXX: elif code in (HTTP_MOVED_PERMANENTLY,
# HTTP_MOVED_TEMPORARILY):
else:
req.status = code
self.error_from_table(req, code)
except (BrokenClientConnection, SystemExit) as e:
req.log_error(str(e), LOG_ERR)
req.log_error(' *** You shoud ignore next error ***',
LOG_ERR)
return ()
except:
self.error_from_table(req, 500)
# endtry
try: # call post_process handler
for fn in self.__post:
fn(req)
except:
self.error_from_table(req, 500)
# endtry
return req.__end_of_request__() # private call of request
# enddef
def __call__(self, environ, start_response):
"""Callable define for Application instance.
This method run __request__ method.
"""
if self.__name == '__poorwsgi__':
stderr.write("[W] Using deprecated instance of Application.\n")
stderr.write(" Please, create your own instance\n")
stderr.flush()
return self.__request__(environ, start_response)
def __profile_request__(self, environ, start_response):
"""Profiler version of __request__.
This method is used if set_profile is used."""
def wrapper(rv):
rv.append(self.__original_request__(environ, start_response))
rv = []
uri_dump = (self._dump + environ.get('PATH_INFO').replace('/', '_')
+ '.profile')
self.log_error('Generate %s' % uri_dump, LOG_INFO)
self._runctx('wrapper(rv)', globals(), locals(), filename=uri_dump)
return rv[0]
# enddef
def __repr__(self):
return '%s - callable Application class instance' % self.__name
def set_profile(self, runctx, dump):
"""Set profiler for __call__ function.
Arguments:
runctx - function from profiler module
dump - path and prefix for .profile files
Typical usage:
import cProfile
cProfile.runctx('from simple import *', globals(), locals(),
filename="log/init.profile")
app.set_profile(cProfile.runctx, 'log/req')
"""
self._runctx = runctx
self._dump = dump
self.__original_request__ = self.__request__
self.__request__ = self.__profile_request__
# enddef
def del_profile(self):
"""Remove profiler from application."""
self.__request__ = self.__original_request__
def get_options(self):
"""Returns dictionary with application variables from system environment.
Application variables start with {app_} prefix,
but in returned dictionary is set without this prefix.
#!ini
poor_LogLevel = warn # Poor WSGI variable
app_db_server = localhost # application variable db_server
app_templates = app/templ # application variable templates
This method works like Request.get_options, but work with
os.environ, so it works only with wsgi servers, which set not only
request environ, but os.environ too. Apaches mod_wsgi don't do that,
uWsgi and PoorHTTP do that.
"""
options = {}
for key, val in environ.items():
key = key.strip()
if key[:4].lower() == 'app_':
options[key[4:].lower()] = val.strip()
return options
# enddef
def log_error(self, message, level=LOG_ERR):
"""Logging method with the same functionality like in Request object.
But as get_options read configuration from os.environ which could
not work in same wsgi servers like Apaches mod_wsgi.
This method write to stderr so messages, could not be found in
servers error log!
"""
if self.__log_level[0] >= level[0]:
if _unicode_exist and isinstance(message, unicode):
message = message.encode('utf-8')
try:
stderr.write("<%s> [%s] %s\n" % (level[1], self.__name,
message))
except UnicodeEncodeError:
if _unicode_exist:
message = message.decode('utf-8').encode(
'ascii', 'backslashreplace')
else:
message = message.encode(
'ascii', 'backslashreplace').decode('ascii')
stderr.write("<%s> [%s] %s\n" % (level[1], self.__name,
message))
stderr.flush()
# enddef
def log_info(self, message):
"""Logging method, which create message as LOG_INFO level."""
self.log_error(message, LOG_INFO)
def log_debug(self, message):
"""Logging method, which create message as LOG_DEBUG level."""
self.log_error(message, LOG_DEBUG)
def log_warning(self, message):
"""Logging method, which create message as LOG_WARNING level."""
self.log_error(message, LOG_WARNING)
class FormData(object):
implements(IFormData)
def __init__(self, data=None):
if not data:
data = {}
super(FormData, self).__init__()
self._fields = OrderedDict()
self.from_dict(data)
def __repr__(self):
reprlist = ["FormData:", ""]
for field in self._fields.keys():
value = self._fields[field].value
# small hack for fields that have a dict as value (files)
if isinstance(value, dict):
if 'name' in value:
value = value['name']
if isinstance(field, unicode):
field = field.encode('utf-8')
if isinstance(value, unicode):
value = value.encode('utf-8')
reprlist.append("%s: %s\n" % (field, value))
return "\n".join(reprlist)
def __json__(self, request):
return self.as_dict()
def __getitem__(self, fieldId):
""" Always return something... even if the data isn't
there. This allows for a somewhat lax policy in evaluation of
requiredness, relevance, etc.
"""
try:
return self._fields[fieldId].value
except:
return None
def __setitem__(self, fieldId, val):
""" Item assignment on formdata. Setting the value of a non existing
field is NOT an error... """
if fieldId not in self._fields:
self._fields[fieldId] = Field(fieldId, val)
else:
self._fields[fieldId].value = val
def getField(self, fieldId):
return self._fields.get(fieldId, None)
def addField(self, field):
self._fields[field.id] = field
def getFields(self):
return self._fields.keys()
def update(self, data, ignore_missing=True):
""" Update self with fields from data arg """
for field_id in data.getFields():
field = data.getField(field_id)
if self.getField(field_id):
self.getField(field_id).value = field.value
else:
if not ignore_missing:
self.addField(Field(field.id, field.value))
def as_dict(self):
res = {}
for field_id in self._fields.keys():
res[field_id] = self._fields[field_id].value
return res
def from_dict(self, data=None, create_missing_fields=True):
""" Set the form fields and values from a dict """
self.clear()
if data:
for key, val in data.items():
if create_missing_fields:
self[key] = val
else:
fld = self.getField(key)
if fld:
fld.value = val
def clone(self):
""" clone the data """
return FormData(self.as_dict())
def clear(self):
""" clear all data """
for fieldId in self.getFields():
self._fields[fieldId].value = None
class ActionRunner(BaseActionRunner):
def __init__(self, optionMap, *args):
BaseActionRunner.__init__(self, optionMap,
logging.getLogger("Action Runner"))
self.currentTestRunner = None
self.previousTestRunner = None
self.appRunners = OrderedDict()
def addSuite(self, suite):
plugins.log.info("Using " +
suite.app.description(includeCheckout=True))
appRunner = ApplicationRunner(suite, self.diag)
self.appRunners[suite.app] = appRunner
def notifyAllReadAndNotified(self):
# kicks off processing. Don't use notifyAllRead as we end up running all the tests before
# everyone's been notified of the reading.
self.runAllTests()
def notifyRerun(self, test):
if self.currentTestRunner and self.currentTestRunner.test is test:
self.diag.info("Got rerun notification for " + repr(test) +
", resetting actions")
self.currentTestRunner.resetActionSequence()
def runTest(self, test):
# We have the lock coming in to here...
appRunner = self.appRunners.get(test.app)
if appRunner:
self.lock.acquire()
self.currentTestRunner = TestRunner(test, appRunner, self.diag,
self.exited, self.killSignal)
self.lock.release()
self.currentTestRunner.performActions(self.previousTestRunner)
self.previousTestRunner = self.currentTestRunner
self.lock.acquire()
self.currentTestRunner = None
self.notifyComplete(test)
self.lock.release()
def killTests(self):
if self.currentTestRunner:
self.currentTestRunner.kill(self.killSignal)
def killOrCancel(self, test):
if self.currentTestRunner and self.currentTestRunner.test is test:
self.currentTestRunner.kill()
else:
self.cancel(test)
def getAllActionClasses(self):
classes = set()
for appRunner in self.appRunners.values():
for action in appRunner.actionSequence:
classes.add(action.__class__)
return classes
def cleanup(self):
for actionClass in self.getAllActionClasses():
actionClass.finalise()
for appRunner in self.appRunners.values():
appRunner.cleanActions()
class Application(object):
"""Poor WSGI application which is called by WSGI server.
Working of is describe in PEP 0333. This object store route dispatch table,
and have methods for it's using and of course __call__ method for use
as WSGI application.
"""
__instances = []
def __init__(self, name="__main__"):
"""Application class is per name singleton.
That means, there could be exist only one instance with same name.
"""
if Application.__instances.count(name):
raise RuntimeError('Application with name %s exist yet.' % name)
Application.__instances.append(name)
# Application name
self.__name = name
# list of pre and post process handlers
self.__pre = []
self.__post = []
# dhandlers table for default handers on methods {METHOD_GET: handler}
self.__dhandlers = {}
# handlers table of simple paths: {'/path': {METHOD_GET: handler}}
self.__handlers = {}
self.__filters = {
':int': (r'-?\d+', int),
':float': (r'-?\d+(\.\d+)?', float),
':word': (r'\w+', uni),
':hex': (r'[0-9a-fA-F]+', str),
':re:': (None, uni),
'none': (r'[^/]+', uni)
}
# handlers of regex paths: {r'/user/([a-z]?)': {METHOD_GET: handler}}
self.__rhandlers = OrderedDict()
# http state handlers: {HTTP_NOT_FOUND: {METHOD_GET: my_404_handler}}
self.__shandlers = {}
# -- Application variable
self.__config = {
'auto_args': True,
'auto_form': True,
'auto_json': True,
'keep_blank_values': 0,
'strict_parsing': 0,
'json_content_types': [
'application/json',
'application/javascript',
'application/merge-patch+json'],
'auto_cookies': True,
'debug': 'Off',
'document_root': '',
'document_index': 'Off',
'secret_key': '%s%s%s%s' %
(__version__, version, getcwd(),
''.join(str(x) for x in uname()))
}
try:
self.__log_level = levels[environ.get('poor_LogLevel',
'warn').lower()]
except:
self.__log_level = LOG_WARNING
self.log_error('Bad poor_LogLevel, default is warn.', LOG_WARNING)
# endtry
# enddef
def __regex(self, match):
groups = match.groups()
_filter = str(groups[1]).lower()
if _filter in self.__filters:
regex = self.__filters[_filter][0]
elif _filter[:4] == ':re:': # :re: filter have user defined regex
regex = _filter[4:]
else:
try:
regex = self.__filters[_filter][0]
except KeyError:
raise RuntimeError("Undefined route group filter '%s'" %
_filter)
return "(?P<%s>%s)" % (groups[0], regex)
# enddef
def __convertor(self, _filter):
_filter = str(_filter).lower()
_filter = ':re:' if _filter[:4] == ':re:' else _filter
try:
return self.__filters[_filter][1]
except KeyError:
raise RuntimeError("Undefined route group filter '%s'" % _filter)
@property
def name(self):
"""Return application name."""
return self.__name
@property
def filters(self):
"""Copy of filter table.
Filter table contains regular expressions and convert functions,
see Application.set_filter and Application.route.
Default filters are:
:int - match number and convert it to int
:float - match number and convert it to float
:word - match one unicoee word
:hex - match hexadecimal value and convert it to str
:re: - match user defined regular expression
none - match any string withount '/' character
For more details see {/debug-info} page of your application, where
you see all filters with regular expression definition.
"""
return self.__filters.copy()
@property
def pre(self):
"""Tuple of table with pre-process handlers.
See Application.pre_process.
"""
return tuple(self.__pre)
@property
def post(self):
"""Tuple of table with post-process handlers.
See Application.post_process.
"""
return tuple(self.__post)
@property
def dhandlers(self):
"""Copy of table with default handlers.
See Application.set_default
"""
return self.__dhandlers.copy()
@property
def handlers(self):
"""Copy of table with static handlers.
See Application.route.
"""
return self.__handlers.copy()
@property
def rhandlers(self):
"""Copy of table with regular expression handlers.
See Application.route and Application.rroute.
"""
return self.__rhandlers.copy()
@property
def shandlers(self):
"""Copy of table with http state aka error handlers.
See Application.http_state
"""
return self.__shandlers.copy()
@property
def auto_args(self):
"""Automatic parsing request arguments from uri.
If it is True (default), Request object do automatic parsing request
uri to its args variable.
"""
return self.__config['auto_args']
@auto_args.setter
def auto_args(self, value):
self.__config['auto_args'] = bool(value)
@property
def auto_form(self):
"""Automatic parsing arguments from request body.
If it is True (default) and method is POST, PUT or PATCH, Request
object do automatic parsing request body to its form variable.
"""
return self.__config['auto_form']
@auto_form.setter
def auto_form(self, value):
self.__config['auto_form'] = bool(value)
@property
def auto_json(self):
"""Automatic parsing JSON from request body.
If it is True (default), method is POST, PUT or PATCH and request
content type is one of json_content_types, Request object do
automatic parsing request body to json variable.
"""
return self.__config['auto_json']
@auto_json.setter
def auto_json(self, value):
self.__config['auto_json'] = bool(value)
@property
def auto_cookies(self):
"""Automatic parsing cookies from request headers.
If it is True (default) and Cookie request header was set,
SimpleCookie object was paresed to Request property cookies.
"""
return self.__config['auto_cookies']
@auto_cookies.setter
def auto_cookies(self, value):
self.__config['auto_cookies'] = bool(value)
@property
def debug(self):
"""Application debug as another way how to set poor_Debug.
This setting will be rewrite by poor_Debug environment variable.
"""
return self.__config['debug'] == 'On'
@debug.setter
def debug(self, value):
self.__config['debug'] = 'On' if bool(value) else 'Off'
@property
def document_root(self):
"""Application document_root as another way how to set poor_DocumentRoot.
This setting will be rewrite by poor_DocumentRoot environ variable.
"""
return self.__config['document_root']
@document_root.setter
def document_root(self, value):
self.__config['document_root'] = value
@property
def document_index(self):
"""Application document_root as another way how to set poor_DocumentRoot.
This setting will be rewrite by poor_DocumentRoot environ variable.
"""
return self.__config['document_index'] == 'On'
@document_index.setter
def document_index(self, value):
self.__config['document_index'] = 'On' if bool(value) else 'Off'
@property
def secret_key(self):
"""Application secret_key could be replace by poor_SecretKey in request.
Secret key is used by PoorSession class. It is generate from
some server variables, and the best way is set to your own long
key."""
return self.__config['secret_key']
@secret_key.setter
def secret_key(self, value):
self.__config['secret_key'] = value
@property
def keep_blank_values(self):
"""Keep blank values in request arguments.
If it is 1 (0 is default), automatic parsing request uri or body
keep blank values as empty string.
"""
return self.__config['keep_blank_values']
@keep_blank_values.setter
def keep_blank_values(self, value):
self.__config['keep_blank_values'] = int(value)
@property
def strict_parsing(self):
"""Strict parse request arguments.
If it is 1 (0 is default), automatic parsing request uri or body
raise with exception on parsing error.
"""
return self.__config['strict_parsing']
@strict_parsing.setter
def strict_parsing(self, value):
self.__config['strict_parsing'] = int(value)
@property
def json_content_types(self):
"""Copy of json content type list.
Containt list of strings as json content types, which is use for
testing, when automatics Json object is create from request body.
"""
return self.__config['json_content_types']
def set_filter(self, name, regex, convertor=uni):
"""Create new filter or overwrite builtins.
Arguments:
name - Name of filter which is used in route or set_route
method.
regex - regular expression which used for filter
convertor - convertor function or class, which gets unicode in
input. Default is uni function, which is wrapper
to unicode string.
app.set_filter('uint', r'\d+', int)
"""
name = ':'+name if name[0] != ':' else name
self.__filters[name] = (regex, convertor)
def pre_process(self):
"""Append pre process hendler.
This is decorator for function to call before each request.
@app.pre_process()
def before_each_request(req):
...
"""
def wrapper(fn):
self.__pre.append(fn)
return fn
return wrapper
# enddef
def add_pre_process(self, fn):
"""Append pre proccess handler.
Method adds function to list functions which is call before each
request.
app.add_pre_process(before_each_request)
"""
self.__pre.append(fn)
# enddef
def post_process(self):
"""Append post process handler.
This decorator append function to be called after each request,
if you want to use it redefined all outputs.
@app.pre_process()
def after_each_request(req):
...
"""
def wrapper(fn):
self.__post.append(fn)
return fn
return wrapper
# enddef
def add_post_process(self, fn):
"""Append post process handler.
Method for direct append function to list functions which are called
after each request.
app.add_post_process(after_each_request)
"""
self.__post.append(fn)
# enddef
def default(self, method=METHOD_HEAD | METHOD_GET):
"""Set default handler.
This is decorator for default handler for http method (called before
error_not_found).
@app.default(METHOD_GET_POST)
def default_get_post(req):
# this function will be called if no uri match in internal
# uri table with method. It's similar like not_found error,
# but without error
...
"""
def wrapper(fn):
self.set_default(fn, method)
return wrapper
# enddef
def set_default(self, fn, method=METHOD_HEAD | METHOD_GET):
"""Set default handler.
Set fn default handler for http method called befor error_not_found.
app.set_default(default_get_post, METHOD_GET_POST)
"""
for m in methods.values():
if method & m:
self.__dhandlers[m] = fn
# enddef
def pop_default(self, method):
"""Pop default handler for method."""
return self.__dhandlers(method)
def route(self, uri, method=METHOD_HEAD | METHOD_GET):
"""Wrap function to be handler for uri and specified method.
You can define uri as static path or as groups which are hand
to handler as next parameters.
# static uri
@app.route('/user/post', method=METHOD_POST)
def user_create(req):
...
# group regular expression
@app.route('/user/<name>')
def user_detail(req, name):
...
# group regular expression with filter
@app.route('/<surname:word>/<age:int>')
def surnames_by_age(req, surname, age):
...
# group with own regular expression filter
@app.route('/<car:re:\w+>/<color:re:#[\da-fA-F]+>')
def car(req, car, color):
...
If you can use some name of group which is python keyword, like class,
you can use **kwargs syntax:
@app.route('/<class>/<len:int>')
def classes(req, **kwargs):
return "'%s' class is %d lenght." % \
(kwargs['class'], kwargs['len'])
Be sure with ordering of call this decorator or set_route function with
groups regular expression. Regular expression routes are check with the
same ordering, as you create internal table of them. First match stops
any other searching. In fact, if groups are detect, they will be
transfer to normal regular expression, and will be add to second
internal table.
"""
def wrapper(fn):
self.set_route(uri, fn, method)
return fn
return wrapper
# enddef
def set_route(self, uri, fn, method=METHOD_HEAD | METHOD_GET):
"""Set handler for uri and method.
Another way to add fn as handler for uri. See Application.route
documentation for details.
app.set_route('/use/post', user_create, METHOD_POST)
"""
uri = uni(uri)
if re_filter.search(uri):
r_uri = re_filter.sub(self.__regex, uri) + '$'
convertors = tuple((g[0], self.__convertor(g[1]))
for g in (m.groups()
for m in re_filter.finditer(uri)))
self.set_rroute(r_uri, fn, method, convertors)
else:
if uri not in self.__handlers:
self.__handlers[uri] = {}
for m in methods.values():
if method & m:
self.__handlers[uri][m] = fn
# enddef
def pop_route(self, uri, method):
"""Pop handler for uri and method from handers table.
Method must be define unique, so METHOD_GET_POST could not be use.
If you want to remove handler for both methods, you must call pop route
for each method state.
"""
uri = uni(uri)
if re_filter.search(uri):
r_uri = re_filter.sub(self.__regex, uri) + '$'
return self.pop_rroute(r_uri, method)
else:
handlers = self.__handlers.get(uri, {})
rv = handlers.pop(method)
if not handlers: # is empty
self.__handlers.pop(uri, None)
return rv
def is_route(self, uri):
"""Check if uri have any registered record."""
uri = uni(uri)
if re_filter.search(uri):
r_uri = re_filter.sub(self.__regex, uri) + '$'
return self.is_rroute(r_uri)
return uri in self.__handlers
def rroute(self, ruri, method=METHOD_HEAD | METHOD_GET):
"""Wrap function to be handler for uri defined by regular expression.
Both of function, rroute and set_rroute store routes to special
internal table, which is another to table of static routes.
@app.rroute(r'/user/\w+') # simple regular expression
def any_user(req):
...
@app.rroute(r'/user/(?P<user>\w+)') # regular expression with
def user_detail(req, user): # groups
...
Be sure with ordering of call this decorator or set_rroute function.
Regular expression routes are check with the same ordering, as you
create internal table of them. First match stops any other searching.
"""
def wrapper(fn):
self.set_rroute(ruri, fn, method)
return fn
return wrapper
# enddef
def set_rroute(self, r_uri, fn, method=METHOD_HEAD | METHOD_GET,
convertors=()):
"""Set hanlder for uri defined by regular expression.
Another way to add fn as handler for uri defined by regular expression.
See Application.rroute documentation for details.
app.set_rroute('/use/\w+/post', user_create, METHOD_POST)
This method is internally use, when groups are found in static route,
adding by route or set_route method.
"""
r_uri = re.compile(r_uri, re.U)
if r_uri not in self.__rhandlers:
self.__rhandlers[r_uri] = {}
for m in methods.values():
if method & m:
self.__rhandlers[r_uri][m] = (fn, convertors)
# enddef
def pop_rroute(self, r_uri, method):
"""Pop handler and convertors for uri and method from handlers table.
For mor details see Application.pop_route.
"""
r_uri = re.compile(r_uri, re.U)
handlers = self.__rhandlers.get(r_uri, {})
rv = handlers.pop(method)
if not handlers: # is empty
self.__rhandlers.pop(r_uri, None)
return rv
def is_rroute(self, r_uri):
"""Check if regular expression uri have any registered record."""
r_uri = re.compile(r_uri, re.U)
return r_uri in self.__rhandlers
def http_state(self, code, method=METHOD_HEAD | METHOD_GET | METHOD_POST):
"""Wrap function to handle http status codes like http errors."""
def wrapper(fn):
self.set_http_state(code, fn, method)
return wrapper
# enddef
def set_http_state(self, code, fn,
method=METHOD_HEAD | METHOD_GET | METHOD_POST):
"""Set fn as handler for http state code and method."""
if code not in self.__shandlers:
self.__shandlers[code] = {}
for m in methods.values():
if method & m:
self.__shandlers[code][m] = fn
# enddef
def pop_http_state(self, code, method):
"""Pop handerl for http state and method.
As Application.pop_route, for pop multimethod handler, you must call
pop_http_state for each method.
"""
handlers = self.__shandlers(code, {})
return handlers.pop(method)
def error_from_table(self, req, code):
"""Internal method, which is called if error was accured.
If status code is in Application.shandlers (fill with http_state
function), call this handler.
"""
if code in self.__shandlers \
and req.method_number in self.__shandlers[code]:
try:
handler = self.__shandlers[code][req.method_number]
if 'uri_handler' not in req.__dict__:
req.uri_rule = '_%d_error_handler_' % code
req.uri_handler = handler
self.handler_from_pre(req) # call pre handlers now
handler(req)
except:
internal_server_error(req)
elif code in default_shandlers:
handler = default_shandlers[code][METHOD_GET]
handler(req)
else:
not_implemented(req, code)
# enddef
def handler_from_default(self, req):
"""Internal method, which is called if no handler is found."""
if req.method_number in self.__dhandlers:
req.uri_rule = '_default_handler_'
req.uri_handler = self.__dhandlers[req.method_number]
self.handler_from_pre(req) # call pre handlers now
retval = self.__dhandlers[req.method_number](req)
if retval != DECLINED:
raise SERVER_RETURN(retval)
# enddef
def handler_from_pre(self, req):
"""Internal method, which run all pre (pre_proccess) handlers.
This method was call before end-point route handler.
"""
for fn in self.__pre:
fn(req)
def handler_from_table(self, req):
"""Call right handler from handlers table (fill with route function).
If no handler is fined, try to find directory or file if Document Root,
resp. Document Index is set. Then try to call default handler for right
method or call handler for status code 404 - not found.
"""
# static routes
if req.uri in self.__handlers:
if req.method_number in self.__handlers[req.uri]:
handler = self.__handlers[req.uri][req.method_number]
req.uri_rule = req.uri # nice variable for pre handlers
req.uri_handler = handler
self.handler_from_pre(req) # call pre handlers now
retval = handler(req) # call right handler now
# return text is allowed
if isinstance(retval, str) \
or (_unicode_exist and isinstance(retval, unicode)):
req.write(retval, 1) # write data and flush
retval = DONE
if retval != DECLINED:
raise SERVER_RETURN(retval or DONE) # could be state.DONE
else:
raise SERVER_RETURN(HTTP_METHOD_NOT_ALLOWED)
# endif
# endif
# regular expression
for ruri in self.__rhandlers.keys():
match = ruri.match(req.uri)
if match and req.method_number in self.__rhandlers[ruri]:
handler, convertors = self.__rhandlers[ruri][req.method_number]
req.uri_rule = ruri.pattern # nice variable for pre handlers
req.uri_handler = handler
self.handler_from_pre(req) # call pre handlers now
if len(convertors):
# create OrderedDict from match insead of dict for
# convertors applying
req.groups = OrderedDict(
(g, c(v))for ((g, c), v) in zip(convertors,
match.groups()))
retval = handler(req, *req.groups.values())
else:
req.groups = match.groupdict()
retval = handler(req, *match.groups())
# return text is allowed
if isinstance(retval, str) \
or (_unicode_exist and isinstance(retval, unicode)):
req.write(retval, 1) # write data and flush
retval = DONE
if retval != DECLINED:
raise SERVER_RETURN(retval or DONE) # could be state.DONE
# endif - no METHOD_NOT_ALLOWED here
# endfor
# try file or index
if req.document_root():
rfile = "%s%s" % (uni(req.document_root()),
path.normpath("%s" % uni(req.uri)))
if not path.exists(rfile):
if req.debug and req.uri == '/debug-info': # work if debug
req.uri_rule = '_debug_info_'
req.uri_handler = debug_info
self.handler_from_pre(req) # call pre handlers now
raise SERVER_RETURN(debug_info(req, self))
self.handler_from_default(req) # try default
raise SERVER_RETURN(HTTP_NOT_FOUND) # not found
# return file
if path.isfile(rfile) and access(rfile, R_OK):
req.uri_rule = '_send_file_'
req.uri_handler = send_file
self.handler_from_pre(req) # call pre handlers now
req.log_error("Return file: %s" % req.uri, LOG_INFO)
raise SERVER_RETURN(send_file(req, rfile))
# return directory index
if req.document_index and path.isdir(rfile) \
and access(rfile, R_OK):
req.log_error("Return directory: %s" % req.uri, LOG_INFO)
req.uri_rule = '_directory_index_'
req.uri_handler = directory_index
self.handler_from_pre(req) # call pre handlers now
raise SERVER_RETURN(directory_index(req, rfile))
raise SERVER_RETURN(HTTP_FORBIDDEN)
# endif
if req.debug and req.uri == '/debug-info':
req.uri_rule = '_debug_info_'
req.uri_handler = debug_info
self.handler_from_pre(req) # call pre handlers now
raise SERVER_RETURN(debug_info(req, self))
self.handler_from_default(req)
req.log_error("404 Not Found: %s" % req.uri, LOG_ERR)
raise SERVER_RETURN(HTTP_NOT_FOUND)
# enddef
def __request__(self, environ, start_response):
"""Create Request instance and return wsgi response.
This method create Request object, call handlers from
Application.__pre (Application.handler_from_pre),
uri handler (handler_from_table), default handler
(Application.handler_from_default) or error handler
(Application.error_from_table), and handlers from
Application.__post.
"""
req = Request(environ, start_response, self.__config)
try:
self.handler_from_table(req)
except SERVER_RETURN as e:
code = e.args[0]
if code in (OK, HTTP_OK, DONE):
pass
# XXX: elif code in (HTTP_MOVED_PERMANENTLY,
# HTTP_MOVED_TEMPORARILY):
else:
req.status = code
self.error_from_table(req, code)
except (BrokenClientConnection, SystemExit) as e:
req.log_error(str(e), LOG_ERR)
req.log_error(' *** You shoud ignore next error ***',
LOG_ERR)
return ()
except:
self.error_from_table(req, 500)
# endtry
try: # call post_process handler
for fn in self.__post:
fn(req)
except:
self.error_from_table(req, 500)
# endtry
return req.__end_of_request__() # private call of request
# enddef
def __call__(self, environ, start_response):
"""Callable define for Application instance.
This method run __request__ method.
"""
if self.__name == '__poorwsgi__':
stderr.write("[W] Using deprecated instance of Application.\n")
stderr.write(" Please, create your own instance\n")
stderr.flush()
return self.__request__(environ, start_response)
def __profile_request__(self, environ, start_response):
"""Profiler version of __request__.
This method is used if set_profile is used."""
def wrapper(rv):
rv.append(self.__original_request__(environ, start_response))
rv = []
uri_dump = (self._dump + environ.get('PATH_INFO').replace('/', '_')
+ '.profile')
self.log_error('Generate %s' % uri_dump, LOG_INFO)
self._runctx('wrapper(rv)', globals(), locals(), filename=uri_dump)
return rv[0]
# enddef
def __repr__(self):
return '%s - callable Application class instance' % self.__name
def set_profile(self, runctx, dump):
"""Set profiler for __call__ function.
Arguments:
runctx - function from profiler module
dump - path and prefix for .profile files
Typical usage:
import cProfile
cProfile.runctx('from simple import *', globals(), locals(),
filename="log/init.profile")
app.set_profile(cProfile.runctx, 'log/req')
"""
self._runctx = runctx
self._dump = dump
self.__original_request__ = self.__request__
self.__request__ = self.__profile_request__
# enddef
def del_profile(self):
"""Remove profiler from application."""
self.__request__ = self.__original_request__
def get_options(self):
"""Returns dictionary with application variables from system environment.
Application variables start with {app_} prefix,
but in returned dictionary is set without this prefix.
#!ini
poor_LogLevel = warn # Poor WSGI variable
app_db_server = localhost # application variable db_server
app_templates = app/templ # application variable templates
This method works like Request.get_options, but work with
os.environ, so it works only with wsgi servers, which set not only
request environ, but os.environ too. Apaches mod_wsgi don't do that,
uWsgi and PoorHTTP do that.
"""
options = {}
for key, val in environ.items():
key = key.strip()
if key[:4].lower() == 'app_':
options[key[4:].lower()] = val.strip()
return options
# enddef
def log_error(self, message, level=LOG_ERR):
"""Logging method with the same functionality like in Request object.
But as get_options read configuration from os.environ which could
not work in same wsgi servers like Apaches mod_wsgi.
This method write to stderr so messages, could not be found in
servers error log!
"""
if self.__log_level[0] >= level[0]:
if _unicode_exist and isinstance(message, unicode):
message = message.encode('utf-8')
try:
stderr.write("<%s> [%s] %s\n" % (level[1], self.__name,
message))
except UnicodeEncodeError:
if _unicode_exist:
message = message.decode('utf-8').encode(
'ascii', 'backslashreplace')
else:
message = message.encode(
'ascii', 'backslashreplace').decode('ascii')
stderr.write("<%s> [%s] %s\n" % (level[1], self.__name,
message))
stderr.flush()
# enddef
def log_info(self, message):
"""Logging method, which create message as LOG_INFO level."""
self.log_error(message, LOG_INFO)
def log_debug(self, message):
"""Logging method, which create message as LOG_DEBUG level."""
self.log_error(message, LOG_DEBUG)
def log_warning(self, message):
"""Logging method, which create message as LOG_WARNING level."""
self.log_error(message, LOG_WARNING)
class ResourceBuilder(object):
""" Helper to create a ressource """
def __init__(self, name=None, required=False):
self._name = name
self._fields = OrderedDict()
self._required = required
def add_field(self,
field,
arg=None,
value=None,
extended=False,
hidden=False,
e_type=str,
required=None):
"""Add a new field to the current ResourceBuilder.
Keyword arguments:
field -- field name
arg -- name of the attribute name in arg object (argparse)
value -- a default for this field, used for resource creation.
extended -- If set to true, the current field will be display in
extended list mode only.
hidden -- If set to true, the current field won't be exposed
as available keys.
e_type -- field data type (default str)
required -- True if the current field is required for create
and update methods
"""
if required is None:
required = self._required
if arg is None:
arg = re.sub('(?!^)([A-Z]+)', r'_\1', field).lower()
self._fields[field] = {
'field': field,
'arg': arg,
'value': value,
'extended': extended,
'required': required,
'e_type': e_type,
'hidden': hidden
}
def get_keys(self, extended=False):
res = []
for field in self._fields.values():
if field['hidden']:
continue
if not field['extended']:
res.append(field['field'])
if extended and field['extended']:
res.append(field['field'])
return res
def get_fields(self, extended=False, full=False):
res = []
if extended:
for field in self._fields.values():
if field['extended']:
res.append(field['field'])
elif full:
for field in self._fields.keys():
res.append(field)
else:
for field in self._fields.values():
if not field['extended']:
res.append(field['field'])
return res
def set_arg(self, key, arg):
field = self._fields.get(key, None)
if field is not None:
field['arg'] = arg
def get_value(self, key):
field = self._fields.get(key, None)
if field is not None:
return field['value']
else:
return None
def set_value(self, key, value):
field = self._fields.get(key, None)
if field is not None:
field['value'] = value
def to_resource(self):
ret = {}
for field in self._fields.values():
ret[field['field']] = field['value']
return ret
def load_from_args(self, namespace):
for field in self._fields.values():
value = getattr(namespace, field['arg'], None)
if value is not None:
field['value'] = value
def copy(self, data):
if isinstance(data, dict):
for field, val in self._fields.items():
val['value'] = data.get(field, "")
if isinstance(data, ResourceBuilder):
for field, val in self._fields.items():
val['value'] = data[field]['value']
def __str__(self):
return json.dumps(self.to_resource(), sort_keys=True, indent=2)
def check_required_fields(self):
for field in self._fields.values():
if field['required']:
value = field['value']
if value is None:
raise ValueError("missing value for required field : " +
field['field'])
e_type = field['e_type']
if e_type == int:
int(value)
if e_type == float:
float(value)
class FormData(object):
implements(IFormData)
def __init__(self, data=None):
if not data:
data = {}
super(FormData, self).__init__()
self._fields = OrderedDict()
self.from_dict(data)
def __repr__(self):
reprlist = ["FormData:", ""]
for field in self._fields.keys():
value = self._fields[field].value
# small hack for fields that have a dict as value (files)
if isinstance(value, dict):
if 'name' in value:
value = value['name']
reprlist.append("%s: %s\n" % (field, value))
return "\n".join(reprlist)
def __getitem__(self, fieldId):
""" Always return something... even if the data isn't
there. This allows for a somewhat lax policy in evaluation of
requiredness, relevance, etc.
"""
try:
return self._fields[fieldId].value
except:
return None
def __setitem__(self, fieldId, val):
""" Item assignment on formdata. Setting the value of a non existing
field is NOT an error... """
if not fieldId in self._fields:
self._fields[fieldId] = Field(fieldId, val)
else:
self._fields[fieldId].value = val
def getField(self, fieldId):
return self._fields.get(fieldId, None)
def addField(self, field):
self._fields[field.id] = field
def getFields(self):
return self._fields.keys()
def update(self, data, ignore_missing=True):
""" Update self with fields from data arg """
for field_id in data.getFields():
field = data.getField(field_id)
if self.getField(field_id):
self.getField(field_id).value = field.value
else:
if not ignore_missing:
self.addField(Field(field.id, field.value))
def as_dict(self):
res = {}
for field_id in self._fields.keys():
res[field_id] = self._fields[field_id].value
return res
def from_dict(self, data=None):
""" Set the form fields and values from a dict """
if data:
for key, val in data.items():
self[key] = val
class ResourceBuilder(object):
""" Helper to create a ressource """
"""Create a new ResourceBuilder:
Keyword arguments:
name -- resource name (display purpose)
required -- defined default required value for all fields.
"""
def __init__(self, name=None, required=False):
self._name = name
self._fields = OrderedDict()
self._required = required
l_name = "rbu"
if name:
l_name = "rbu:" + name
self.log = logging.getLogger(l_name)
def add_hook(self, key, hook):
field = self._fields.get(key, None)
if field is not None:
field['hook'] = hook
def add_field(self, field, arg=None, value=None, extended=False,
hidden=False, e_type=str, required=None):
"""Add a new field to the current ResourceBuilder.
Keyword arguments:
field -- field name
arg -- name of the attribute name in arg object (argparse)
value -- a default for this field, used for resource creation.
extended -- If set to true, the current field will be display in
extended list mode only.
hidden -- If set to true, the current field won't be exposed
as available keys.
e_type -- field data type (default str): int, float, str
required -- True if the current field is required for create
and update methods
"""
if required is None:
required = self._required
if arg is None:
arg = re.sub('(?!^)([A-Z]+)', r'_\1', field).lower()
self._fields[field] = {
'field': field,
'arg': arg,
'value': value,
'extended': extended,
'required': required,
'e_type': e_type,
'hidden': hidden
}
def get_keys(self, extended=False):
res = []
for field in self._fields.values():
if field['hidden']:
continue
if not field['extended']:
res.append(field['field'])
if extended and field['extended']:
res.append(field['field'])
return res
def get_fields(self, extended=False, full=False):
res = []
if extended:
for field in self._fields.values():
if field['extended']:
res.append(field['field'])
elif full:
for field in self._fields.keys():
res.append(field)
else:
for field in self._fields.values():
if not field['extended']:
res.append(field['field'])
return res
def set_arg(self, key, arg):
field = self._fields.get(key, None)
if field is not None:
field['arg'] = arg
def get_value(self, key):
field = self._fields.get(key, None)
if field is not None:
return field['value']
else:
return None
def set_value(self, key, value):
field = self._fields.get(key, None)
if field is not None:
field['value'] = value
def to_resource(self):
ret = {}
for field in self._fields.values():
ret[field['field']] = field['value']
return ret
def load_from_args(self, namespace):
for field in self._fields.values():
self.log.debug("config: %s", field)
value = getattr(namespace, field['arg'], None)
if value is not None:
if field.has_key('hook'):
value = field['hook'](value, self)
field['value'] = value
def copy(self, data):
if isinstance(data, dict):
self.log.debug("dict")
for field, val in self._fields.items():
self.log.debug("field: %s", field)
self.log.debug("config: %s", val)
value = data.get(field, None)
self.log.debug("value: %s", value)
if value is not None:
val['value'] = value
elif isinstance(data, ResourceBuilder):
self.log.debug("rbu")
for field, val in self._fields.items():
self.log.debug("field: %s", field)
self.log.debug("config: %s", val)
value = data[field]['value']
self.log.debug("value: %s", value)
val['value'] = value
else:
self.log.debug("type: %s", type(data))
raise ValueError("Invalid input data")
def __str__(self):
return json.dumps(self.to_resource(), sort_keys=True, indent=2)
def check_required_fields(self):
for field in self._fields.values():
if field['required']:
value = field['value']
if value is None:
raise ValueError("missing value for required field : "
+ field['field'])
e_type = field['e_type']
if e_type == int:
int(value)
if e_type == float:
float(value)
class UseCaseEditor:
enterTitle = "Enter Usecase names for auto-recorded actions"
def __init__(self, fileName, interface, mapFiles):
self.fileName = fileName
self.editTitle = self.fileName
if os.getenv("TEXTTEST_HOME"):
self.editTitle = self.fileName.replace(os.getenv("TEXTTEST_HOME") + "/", "")
self.interface = interface
self.uiMapFileHandler = UIMapFileHandler(mapFiles)
self.scriptEngine = EditorScriptEngine(uiMapFiles=[])
self.initShortcutManager()
self.allEntries = OrderedDict()
self.allDescriptionWidgets = []
self.popupSensitivities = {}
self.createdShortcuts = []
def initShortcutManager(self):
self.shortcutManager = ShortcutManager()
for shortcut in gtktoolkit.ScriptEngine.getShortcuts():
self.shortcutManager.add(shortcut)
def getAllCommands(self):
return [ line.strip() for line in encodingutils.openEncoded(self.fileName) ]
def getNewUsecaseNames(self):
return [ entry.get_text() for entry in self.allEntries.values() ]
def getNewWidgetDescriptions(self):
def get_text(widget):
return removeMarkup(unescape(widget.get_active_text())) if hasattr(widget, "get_active_text") else widget.get_text()
return map(get_text, self.allDescriptionWidgets)
def run(self):
commands = self.getAllCommands()
autoGenerated = self.getAutoGenerated(commands)
autoGeneratedInfo = self.parseAutoGenerated(autoGenerated)
self.isAutoGenerated = len(autoGenerated) > 0
dialog = self.createDialog(autoGeneratedInfo, commands)
self.runDialog(dialog, autoGenerated, autoGeneratedInfo)
def runDialog(self, dialog, autoGenerated, autoGeneratedInfo):
response = dialog.run()
if response == gtk.RESPONSE_ACCEPT:
newNames = self.getNewUsecaseNames()
if len(autoGenerated) > 0:
duplicateNames = self.getDuplicateNames(newNames)
if duplicateNames and not self.acceptDuplicateNames(dialog, duplicateNames):
self.runDialog(dialog, autoGenerated, autoGeneratedInfo)
dialog.destroy()
self.replaceInFile(self.fileName, self.makeReplacement, zip(autoGenerated, newNames))
toStore = zip(self.getNewWidgetDescriptions(), autoGeneratedInfo.values(), newNames)
for widgetDescription, signalInfo, eventName in toStore:
self.uiMapFileHandler.storeInfo(widgetDescription, signalInfo[-1], eventName)
self.uiMapFileHandler.write()
elif len(autoGenerated) == 0:
dialog.destroy()
else:
# Don't leave a half generated filename behind, if we didn't fill in the dialog properly
# we should remove it so nobody saves it...
for shortcutName in self.createdShortcuts:
os.remove(shortcutName)
os.remove(self.fileName)
dialog.destroy()
def getDuplicateNames(self, allNames):
duplicates = []
for name in allNames:
value, args = self.uiMapFileHandler.splitOptionValue(name)
if name and value and not args:
duplicates.append(name)
return duplicates
def acceptDuplicateNames(self, parent, duplicateNames):
message = "You have entered a name that already exists in the UI map.\n" + \
"You are allowed to do this but please be aware it may cause problems.\n" + \
"The following duplicate names were found:\n"
for duplicate in duplicateNames:
message += duplicate + "\n"
dialog = self.createMessageDialog(parent, message, gtk.MESSAGE_WARNING, "Warning", gtk.BUTTONS_OK_CANCEL)
dialog.show_all()
response = dialog.run()
dialog.hide()
return response == gtk.RESPONSE_OK
def getAutoGenerated(self, commands):
# Find the auto-generated commands and strip them of their arguments
autoGenerated = []
for command in commands:
if command.startswith("Auto."):
pos = command.rfind("'")
commandWithoutArg = command[:pos + 1]
if not commandWithoutArg in autoGenerated:
autoGenerated.append(commandWithoutArg)
return autoGenerated
def parseAutoGenerated(self, commands):
autoGenerated = OrderedDict()
for command in commands:
parts = command[5:].split("'")
initialPart = parts[0][:-1]
widgetType, signalName = initialPart.split(".", 1)
widgetDescription = self.uiMapFileHandler.unescape(parts[1])
autoGenerated[command] = widgetType, widgetDescription, signalName
return autoGenerated
def replaceInFile(self, fileName, replaceMethod, *args):
newFileName = fileName + ".tmp"
newFile = encodingutils.openEncoded(newFileName, "w")
for i, line in enumerate(encodingutils.openEncoded(fileName)):
newLine = replaceMethod(line, i, *args)
if newLine:
newFile.write(newLine)
newFile.close()
shutil.move(newFileName, fileName)
def makeReplacement(self, command, position, replacements):
for origName, newName in replacements:
if command.startswith(origName):
if newName:
return command.replace(origName, newName)
else:
return
return command
def createDialog(self, autoGenerated, commands):
title = self.enterTitle if len(autoGenerated) > 0 else self.editTitle
dialog = gtk.Dialog(title, flags=gtk.DIALOG_MODAL)
dialog.set_name("Name Entry Window")
height = int(gtk.gdk.screen_height() * 0.6)
if len(autoGenerated) > 0:
contents = self.createTable(autoGenerated, dialog)
dialog.vbox.pack_start(contents, expand=True, fill=True)
dialog.vbox.pack_start(gtk.HSeparator(), expand=False, fill=False)
dialog.set_default_size(-1, height)
else:
width = min(int(gtk.gdk.screen_width() * 0.2), 500)
dialog.set_default_size(width, height)
preview = self.createPreview(commands, autoGenerated)
dialog.vbox.pack_start(preview, expand=True, fill=True)
yesButton = dialog.add_button(gtk.STOCK_OK, gtk.RESPONSE_ACCEPT)
self.scriptEngine.monitorSignal("finish name entry editing", "clicked", yesButton)
self.scriptEngine.monitorSignal("close editor window", "delete-event", dialog)
if len(autoGenerated) > 0:
cancelButton = dialog.add_button(gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL)
self.scriptEngine.monitorSignal("cancel name entry editing", "clicked", cancelButton)
dialog.set_default_response(gtk.RESPONSE_ACCEPT)
dialog.show_all()
return dialog
def addScrollBar(self, widget, viewport=False):
window = gtk.ScrolledWindow()
window.set_policy(gtk.POLICY_NEVER, gtk.POLICY_AUTOMATIC)
if viewport:
window.add_with_viewport(widget)
else:
window.add(widget)
return window
def createMarkupLabel(self, markup):
label = gtk.Label()
label.set_markup(markup)
return label
def activateEntry(self, entry, dialog, *args):
entryPos = self.allEntries.values().index(entry)
if entryPos == len(self.allEntries) - 1:
dialog.response(gtk.RESPONSE_ACCEPT)
else:
nextEntry = self.allEntries.values()[entryPos + 1]
nextEntry.grab_focus()
def getActionDescription(self, signalName, widgetType):
try:
exec "from " + self.interface + "toolkit import ScriptEngine"
except ImportError:
# If we haven't even got any such interface, don't worry about this mechanism
return signalName
desc = ScriptEngine.getDisplayName(signalName) #@UndefinedVariable
if desc:
return desc
if signalName == "activate":
if "Entry" in widgetType:
return "pressed Enter"
else:
return "selected"
elif signalName == "changed":
if "Entry" in widgetType:
return "edited text"
else:
return "selected item"
parts = signalName.split(".")
if len(parts) == 1:
return signalName.replace("-", " ")
if parts[0] == "response":
text = parts[1]
if "--" in text:
return text.replace("--", "='") + "'"
else:
return text
columnName = parts[1]
remaining = parts[0]
if remaining == "toggled":
remaining = ".".join([ remaining, parts[-1] ])
return ScriptEngine.getColumnDisplayName(remaining) + " '" + columnName + "'" #@UndefinedVariable
def splitAutoCommand(self, command, autoGenerated):
for cmd in autoGenerated.keys():
if command.startswith(cmd):
arg = command.replace(cmd, "").strip()
return cmd, arg
return None, None
def addArgumentMarkup(self, arg):
return "<i>" + escape(arg) + "</i>" if arg else ""
def updatePreview(self, entry, data):
model, iter = data
text = entry.get_text() or "?"
args = model.get_value(iter, 2)
arg = " " + args[0] if args else ""
markupFullText = self.convertToMarkup(text + self.addArgumentMarkup(arg))
fullText = text + arg
model.set_value(iter, 0, markupFullText)
model.set_value(iter, 1, fullText)
def addText(self, model, rootIter, text, originalText, arguments, followIter=None):
return model.insert_before(rootIter, followIter, [self.convertToMarkup(text), originalText, arguments])
def addCommandToModel(self, command, model, rootIter=None):
shortcut, args = self.shortcutManager.findShortcut(command)
if shortcut:
self.addShortcutCommandToModel(shortcut, args, model, rootIter)
else:
self.addBasicCommandToModel(command, model, rootIter)
def addShortcutCommandToModel(self, shortcut, args, model, rootIter, followIter=None):
italicArgs = [ "<i>" + escape(arg) + "</i>" for arg in args ]
text = "<b>" + shortcut.getShortcutNameWithArgs(italicArgs) + "</b>"
iter = self.addText(model, rootIter, text, shortcut.getShortcutNameWithArgs(args), args, followIter)
if not followIter:
for step in shortcut.commands:
self.addCommandToModel(shortcut.replaceArgs(step, args), model, iter)
return iter
def extractArgsAddMarkup(self, text, cmd):
markup = text.replace(cmd, cmd + "<i>", 1) + "</i>"
arg = text.replace(cmd, "", 1).strip()
args = [ arg ] if arg else []
return markup, args
def getErrorColouredText(self, text):
return '<span foreground="red">' + text + "</span>"
def addBasicCommandToModel(self, command, model, rootIter):
args = []
if command.startswith(waitCommandName):
markup, _ = self.extractArgsAddMarkup(escape(command), waitCommandName)
# Ignore args for wait commands, they don't have anything in common
text = '<span foreground="#826200">' + markup + "</span>"
widgetDetails = []
else:
widgetDetails = self.uiMapFileHandler.findSectionsAndOptions(command)
text = escape(command)
if widgetDetails:
widgetDesc, signalName = widgetDetails[0]
cmd = self.uiMapFileHandler.get(widgetDesc, signalName)
if cmd != text:
text, args = self.extractArgsAddMarkup(text, cmd)
else:
text = self.getErrorColouredText(text)
iter = self.addText(model, rootIter, text, command, args)
for widgetDesc, signalName in widgetDetails:
msg = self.makeUIMapMessage(signalName, widgetDesc)
self.addText(model, iter, msg, signalName, [widgetDesc])
def makeUIMapMessage(self, signalName, widgetDesc):
return "Perform '" + signalName + "' on widget identified by '" + escape(widgetDesc) + "'"
def createPreview(self, commands, autoGenerated):
self.treeModel = gtk.TreeStore(str, str, object)
view = gtk.TreeView(self.treeModel)
view.set_headers_visible(False)
cell = gtk.CellRendererText()
column = gtk.TreeViewColumn("", cell, markup=0)
view.append_column(column)
view.get_selection().set_mode(gtk.SELECTION_MULTIPLE)
self.popup = self.createPopupMenu(view)
for command in commands:
autoCmdName, autoArg = self.splitAutoCommand(command, autoGenerated)
if autoCmdName:
args = [ autoArg ] if autoArg else []
autoArgMarkup = self.addArgumentMarkup(autoArg)
text = "? " + autoArgMarkup if autoArgMarkup else "?"
iter = self.addText(self.treeModel, None, text, None, args)
entry = self.allEntries.get(autoCmdName)
entry.connect("changed", self.updatePreview, (self.treeModel, iter))
widgetType, widgetDesc, signalName = autoGenerated.get(autoCmdName)
msg = "Perform '" + signalName + "' on widget of type '" + widgetType + "' identified by '" + widgetDesc + "'"
self.addText(self.treeModel, iter, msg, msg, [])
else:
self.addCommandToModel(command, self.treeModel)
view.connect("button-press-event", self.showPopupMenu)
self.scriptEngine.monitorSignal("expand preview node", "row-expanded", view)
self.scriptEngine.monitorSignal("select preview node", "changed", view)
self.scriptEngine.monitorSignal("show preview node options for", "button-press-event", view)
scrolled = self.addScrollBar(view)
if len(autoGenerated) > 0:
frame = gtk.Frame("Current Usecase Preview")
frame.add(scrolled)
return frame
else:
return scrolled
def convertToUtf8(self, text):
return self.convertEncoding(text, 'utf-8', 'replace')
def convertToMarkup(self, text):
return self.convertEncoding(text, 'ascii', 'xmlcharrefreplace')
def convertEncoding(self, text, targetEncoding, replaceMethod):
try:
return text.encode(targetEncoding, replaceMethod)
except ValueError:
return text
def findPossibleWidgetDescriptions(self, fullWidgetDesc):
parts = fullWidgetDesc.split(", ")
badNames = [ "Label=OK", "Label=Cancel", "Label=Yes", "Label=No" ]
sections, descs, badDescs = [], [], []
for i in range(1, len(parts) + 1):
for sectionNameParts in itertools.combinations(parts, i):
sectionName = ", ".join(sectionNameParts)
actualSection = self.uiMapFileHandler.getSection(sectionName)
if actualSection:
sections.append(self.convertToUtf8(escape(actualSection)))
else:
utf8Name = self.convertToUtf8(escape(sectionName))
if sectionName in badNames or (i == 1 and sectionName.startswith("Type=")):
badDescs.append(self.getErrorColouredText(utf8Name))
else:
descs.append(utf8Name)
return sections + descs + badDescs
def getWidgetDescriptionWidget(self, possibleWidgetDescs, signalName, widgetType):
if len(possibleWidgetDescs) == 1:
label = gtk.Label()
label.set_markup(possibleWidgetDescs[0])
return label
else:
liststore = gtk.ListStore(str)
for desc in possibleWidgetDescs:
liststore.append([ desc ])
combobox = gtk.ComboBox(liststore)
cell = gtk.CellRendererText()
combobox.pack_start(cell, True)
combobox.add_attribute(cell, 'markup', 0)
combobox.set_active(0)
scriptName = "choose widget description for signal '" + signalName + "' on " + widgetType + " '" + possibleWidgetDescs[0] + "' ="
self.scriptEngine.monitorSignal(scriptName, "changed", combobox)
return combobox
def createTable(self, autoGenerated, dialog):
table = gtk.Table(rows=len(autoGenerated) + 1, columns=4)
table.set_col_spacings(20)
headers = [ "Widget Type", "Identified By", "Action Performed", "Usecase Name" ]
for col, header in enumerate(headers):
table.attach(self.createMarkupLabel("<b><u>" + header + "</u></b>"),
col, col + 1, 0, 1, xoptions=gtk.FILL, yoptions=gtk.FILL)
for rowIndex, (command, (widgetType, fullWidgetDesc, signalName)) in enumerate(autoGenerated.items()):
table.attach(gtk.Label(widgetType), 0, 1, rowIndex + 1, rowIndex + 2, xoptions=gtk.FILL, yoptions=gtk.FILL)
actionDesc = self.getActionDescription(signalName, widgetType)
possibleWidgetDescs = self.findPossibleWidgetDescriptions(fullWidgetDesc)
widgetDescWidget = self.getWidgetDescriptionWidget(possibleWidgetDescs, signalName, widgetType)
table.attach(widgetDescWidget, 1, 2, rowIndex + 1, rowIndex + 2, xoptions=gtk.FILL, yoptions=gtk.FILL)
table.attach(gtk.Label(actionDesc), 2, 3, rowIndex + 1, rowIndex + 2, xoptions=gtk.FILL, yoptions=gtk.FILL)
entry = gtk.Entry()
fieldIdentifier = "for signal '" + signalName + "' on " + widgetType + " '" + removeMarkup(unescape(possibleWidgetDescs[0])) + "'"
scriptName = "enter usecase name " + fieldIdentifier + " ="
self.scriptEngine.monitorSignal(scriptName, "changed", entry)
entry.connect("activate", self.activateEntry, dialog)
self.scriptEngine.monitorSignal("press <enter> in field " + fieldIdentifier, "activate", entry)
self.allEntries[command] = entry
self.allDescriptionWidgets.append(widgetDescWidget)
table.attach(entry, 3, 4, rowIndex + 1, rowIndex + 2, yoptions=gtk.FILL)
table.show_all()
frame = gtk.Frame("Previously unseen actions: provide names for the interesting ones")
frame.add(self.addScrollBar(table, viewport=True))
return frame
def createPopupMenu(self, widget):
menu = gtk.Menu()
item = gtk.MenuItem("Create shortcut")
deleteItem = gtk.MenuItem("Delete shortcut")
updateUIMapItem = gtk.MenuItem("Update UI map file")
renameItem = gtk.MenuItem("Rename")
separator = gtk.SeparatorMenuItem()
menu.append(item)
menu.append(deleteItem)
menu.append(renameItem)
menu.append(separator)
menu.append(updateUIMapItem)
item.connect("activate", self.createShortcut, widget)
deleteItem.connect("activate", self.deleteShortcut, widget)
renameItem.connect("activate", self.rename, widget)
updateUIMapItem.connect("activate", self.updateUIMap, widget)
self.popupSensitivities[item] = self.setCreateShortcutSensitivity
self.popupSensitivities[deleteItem] = self.setDeleteShortcutSensitivity
self.popupSensitivities[renameItem] = self.setRenameSensitivity
self.popupSensitivities[updateUIMapItem] = self.setUpdateUIMapSensitivity
self.scriptEngine.monitorSignal("create a new shortcut", "activate", item)
self.scriptEngine.monitorSignal("delete shortcut", "activate", deleteItem)
self.scriptEngine.monitorSignal("rename a usecase name or shortcut", "activate", renameItem)
self.scriptEngine.monitorSignal("update ui map file", "activate", updateUIMapItem)
item.show()
deleteItem.show()
renameItem.show()
separator.show()
updateUIMapItem.show()
return menu
def applySensitivities(self, selection):
for item, method in self.popupSensitivities.items():
method(item, selection)
def setCreateShortcutSensitivity(self, item, selection):
# Check selection has at least 2 elements and is consecutive
item.set_sensitive(selection.count_selected_rows() > 1 and self.isConsecutive(selection))
def setDeleteShortcutSensitivity(self, item, selection):
item.set_sensitive(selection.count_selected_rows() == 1 and self.shortcutsSelected(selection))
def showPopupMenu(self, treeView, event):
if event.button == 3:
time = event.time
pathInfo = treeView.get_path_at_pos(int(event.x), int(event.y))
selection = treeView.get_selection()
selectedRows = selection.get_selected_rows()
# If they didnt right click on a currently selected
# row, change the selection
if pathInfo is not None:
if pathInfo[0] not in selectedRows[1]:
selection.unselect_all()
selection.select_path(pathInfo[0])
treeView.grab_focus()
self.popup.popup(None, None, None, event.button, time)
treeView.emit_stop_by_name("button-press-event")
self.applySensitivities(selection)
def createShortcut(self, widget, view):
selection = view.get_selection()
lines, arguments = self.selectionToModel(selection)
self.createShortcutFromLines(lines, arguments)
def selectionToModel(self, selection):
lines = []
allArguments = []
def addSelected(treemodel, path, iter, *args):
line = treemodel.get_value(iter, 1)
currArgs = treemodel.get_value(iter, 2)
lines.append(line)
for arg in currArgs:
allArguments.append(arg)
selection.selected_foreach(addSelected)
return lines, allArguments
def createShortcutFromLines(self, lines, arguments):
dialog = gtk.Dialog("New Shortcut", flags=gtk.DIALOG_MODAL)
dialog.set_name("New Shortcut Window")
label = gtk.Label("New name for shortcut:")
entry = gtk.Entry()
entry.set_name("New Name")
if arguments:
defaultText = "Do something with " + " and ".join(arguments)
entry.set_text(defaultText)
dialog.vbox.set_spacing(10)
dialog.vbox.pack_start(label, expand=False, fill=False)
dialog.vbox.pack_start(entry, expand=True, fill=True)
dialog.vbox.pack_start(gtk.HSeparator(), expand=False, fill=False)
self.scriptEngine.monitorSignal("enter new shortcut name", "changed", entry)
shortcutView = self.createShortcutPreview(lines, arguments, entry)
frame = gtk.Frame("")
frame.get_label_widget().set_use_markup(True)
self.updateShortcutName(entry, frame, arguments)
frame.add(shortcutView)
entry.connect("changed", self.updateShortcutName, frame, arguments)
dialog.vbox.pack_end(frame, expand=True, fill=True)
yesButton = dialog.add_button(gtk.STOCK_OK, gtk.RESPONSE_ACCEPT)
self.scriptEngine.monitorSignal("accept new shortcut name", "clicked", yesButton)
cancelButton = dialog.add_button(gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL)
self.scriptEngine.monitorSignal("cancel new shortcut name", "clicked", cancelButton)
dialog.connect("response", self.respond, entry, frame, shortcutView)
dialog.show_all()
def updateShortcutName(self, textEntry, frame, arguments):
newName = textEntry.get_text()
for arg in arguments:
newName = newName.replace(arg, "$")
markup = "<b><i>" + newName.lower().replace(" ", "_") + ".shortcut" + "</i></b>"
frame.get_label_widget().set_label(markup)
def getShortcutFileName(self, shortcutName):
return shortcutName.lower().replace(" ", "_") + ".shortcut"
def copyRow(self, iter, parentIter, followIter=None):
row = list(self.treeModel.get(iter, 0, 1, 2))
if followIter is None:
newIter = self.treeModel.append(parentIter, row)
else:
newIter = self.treeModel.insert_before(parentIter, followIter, row)
subIter = self.treeModel.iter_children(iter)
if subIter is not None:
self.copyRow(subIter, newIter)
def getTopLevelIters(self):
iters = []
def addSelected(model, path, iter, *args):
if len(path) == 1:
iters.append(iter)
self.treeModel.foreach(addSelected)
return iters
def getFirstDifferentIter(self, iters, commands):
for i, iter in enumerate(iters):
if commands[i] != self.treeModel.get_value(iter, 1):
return i, iter
def addShortcutToPreview(self):
allCommands = self.getAllCommands()
while True:
topLevelIters = self.getTopLevelIters()
topLevelNames = [ self.treeModel.get_value(iter, 1) for iter in topLevelIters ]
if topLevelNames == allCommands:
break
iterIx, iter = self.getFirstDifferentIter(topLevelIters, allCommands)
shortcut, args = self.shortcutManager.findShortcut(allCommands[iterIx])
if shortcut:
shortcutIter = self.addShortcutCommandToModel(shortcut, args, self.treeModel, None, iter)
shortcutLength = len(shortcut.commands)
currentIters = topLevelIters[iterIx:iterIx + shortcutLength]
for iter in currentIters:
self.copyRow(iter, shortcutIter)
self.treeModel.remove(iter)
else:
sys.stderr.write("ERROR: mismatch in files, expected shortcut for '" + allCommands[iterIx] + "', but found none.\n")
break
def findShortcutIters(self, shortcut):
iters = []
def addSelected(model, path, iter, *args):
value = model.get_value(iter, 1)
args = model.get_value(iter, 2)
currShortcut = self.getShortcut(value, args)
if currShortcut is shortcut:
iters.append(iter)
self.treeModel.foreach(addSelected)
return iters
def removeShortcutFromPreview(self, shortcut):
for iter in self.findShortcutIters(shortcut):
childIter = self.treeModel.iter_children(iter)
parentIter = self.treeModel.iter_parent(iter)
nextIter = self.treeModel.iter_next(iter)
while childIter is not None:
self.copyRow(childIter, parentIter, nextIter)
childIter = self.treeModel.iter_next(childIter)
self.treeModel.remove(iter)
def removeShortcutFromUsecase(self, shortcut):
recordScript = RecordScript(self.fileName, [])
for iter in self.getTopLevelIters():
value = self.treeModel.get_value(iter, 1)
args = self.treeModel.get_value(iter, 2)
if self.getShortcut(value, args) is shortcut:
childIter = self.treeModel.iter_children(iter)
while childIter is not None:
recordScript.record(self.treeModel.get_value(childIter, 1))
childIter = self.treeModel.iter_next(childIter)
else:
recordScript.record(value)
def respond(self, dialog, responseId, entry, frame, shortcutView):
if responseId == gtk.RESPONSE_ACCEPT:
if self.checkShortcutName(dialog, entry.get_text().lower()):
dialog.hide()
shortcut = self.saveShortcut(frame.get_label(), self.getShortcutLines(shortcutView))
self.shortcutManager.add(shortcut)
if self.isAutoGenerated:
self.createdShortcuts.append(shortcut.name)
self.recreateUsecaseFile()
self.addShortcutToPreview()
else:
dialog.hide()
def recreateUsecaseFile(self):
recordScript = RecordScript(self.fileName, [shortcut for _, shortcut in self.shortcutManager.shortcuts])
for iter in self.getTopLevelIters():
value =self.treeModel.get_value(iter, 1)
if "<b>" in self.treeModel.get_value(iter, 0):
args = self.treeModel.get_value(iter, 2)
shortcut = self.getShortcut(value, args)
self.runShortcutCommands(recordScript, shortcut, args)
else:
recordScript.record(value)
def runShortcutCommands(self, recordScript, shortcut, args):
shortcutCopy = ReplayScript(shortcut.name, True)
while not shortcutCopy.hasTerminated():
command = shortcutCopy.getCommand(args)
self.recordShortcutCommand(recordScript, command)
def recordShortcutCommand(self, recordScript, command):
shortcut, args = self.shortcutManager.findShortcut(command)
if shortcut:
self.runShortcutCommands(recordScript, shortcut, args)
else:
recordScript.record(command)
def getShortcut(self, shortcutNameWithArgs, args):
for _, shortcut in self.shortcutManager.shortcuts:
if shortcut.getShortcutNameWithArgs(args) == shortcutNameWithArgs:
return shortcut
def getShortcutLines(self, shortcutView):
model = shortcutView.get_model()
lines = []
def addSelected(model, path, iter, *args):
lines.append(model.get_value(iter, 0))
model.foreach(addSelected)
return lines
def checkShortcutName(self, parent, name):
if not name:
self.showErrorDialog(parent, "The shortcut name can't be empty.")
return False
elif self.isInUIMap(name):
self.showErrorDialog(parent, "The shortcut name already exists in the UI map file.")
return False
elif self.isInShortcuts(name):
self.showErrorDialog(parent, "The shortcut name is already being used for another shortcut.")
return False
return True
def isInUIMap(self, name):
return len(self.uiMapFileHandler.findSectionsAndOptions(name)) > 0
def isInShortcuts(self, name):
return self.shortcutManager.findShortcut(name)[0] is not None
def saveShortcut(self, name, lines):
storytextDir = os.environ["STORYTEXT_HOME"]
if not os.path.isdir(storytextDir):
os.makedirs(storytextDir)
fileName = os.path.join(storytextDir, name)
with open(fileName, "w") as f:
for line in lines:
f.write(line + "\n")
print "Shortcut", repr(name), "created."
return ReplayScript(fileName)
def shortcutsSelected(self, selection):
shortcuts = []
def addSelected(treemodel, path, iter, *args):
shortcuts.append("<b>" in treemodel.get_value(iter, 0))
selection.selected_foreach(addSelected)
return any(shortcuts)
def isConsecutive(self, selection):
paths = []
def addSelected(treemodel, path, *args):
paths.append(path)
selection.selected_foreach(addSelected)
prevIx = None
for path in paths:
if len(path) > 1:
return False # Can't make shortcuts out of lines further down the hierarchy
ix = path[0]
if prevIx is not None and ix - prevIx > 1:
return False
prevIx = ix
return True
def showConfirmationDialog(self, parent, message, *args):
self.showErrorWarningDialog(parent, message, gtk.MESSAGE_WARNING, "Confirmation", gtk.BUTTONS_OK_CANCEL, *args)
def showErrorDialog(self, parent, message):
self.showErrorWarningDialog(parent, message, gtk.MESSAGE_ERROR, "Error", gtk.BUTTONS_OK)
def showErrorWarningDialog(self, parent, message, stockIcon, alarmLevel, buttons, *args):
dialog = self.createMessageDialog(parent, message, stockIcon, alarmLevel, buttons)
dialog.connect("response", self.respondErrorWarning, *args)
dialog.show_all()
def respondErrorWarning(self, dialog, response, *args):
dialog.hide()
if response == gtk.RESPONSE_OK and args:
args[0](*args[1:])
def createMessageDialog(self, parent, message, stockIcon, alarmLevel, buttons=gtk.BUTTONS_OK):
dialogTitle = "StoryText " + alarmLevel
dialog = gtk.MessageDialog(parent, gtk.DIALOG_MODAL, stockIcon, buttons, None)
# Would like to use dialog.get_widget_for_response(gtk.RESPONSE_OK), introduced in gtk 2.22 instead
for button in dialog.action_area.get_children():
response = dialog.get_response_for_widget(button)
if response == gtk.RESPONSE_OK:
self.scriptEngine.monitorSignal("accept message", "clicked", button)
elif response == gtk.RESPONSE_CANCEL:
self.scriptEngine.monitorSignal("cancel message", "clicked", button)
dialog.set_title(dialogTitle)
dialog.set_markup(message)
dialog.set_default_response(gtk.RESPONSE_OK)
return dialog
def createShortcutPreview(self, commands, arguments, textEntry):
listModel = gtk.ListStore(str, str, object)
view = gtk.TreeView(listModel)
view.set_headers_visible(False)
cmdRenderer = gtk.CellRendererText()
cmdColumn = gtk.TreeViewColumn("", cmdRenderer, text=0)
view.append_column(cmdColumn)
view.get_selection().set_mode(gtk.SELECTION_MULTIPLE)
argumentIndex = 0
for command in commands:
shortcut, args =self.shortcutManager.findShortcut(command)
if shortcut:
text = shortcut.getShortcutName()
else:
arg = arguments[argumentIndex] if argumentIndex < len(arguments) else ""
text, argument = self.replaceArguments(command, arg)
args = [argument] if argument else []
if argument:
argumentIndex = argumentIndex + 1 % len(arguments)
iter1 = listModel.append([text, command, args])
if not shortcut and args:
textEntry.connect("changed", self.handleArguments, (listModel, iter1))
self.scriptEngine.monitorSignal("select preview node", "changed", view)
self.scriptEngine.monitorSignal("show preview node options for", "button-press-event", view)
return view
def replaceArguments(self, command, argument):
if argument and command.endswith(argument):
return command.replace(argument, "$"), argument
else:
return command, ""
def handleArguments(self, widget, data):
model, iter = data
newName = widget.get_text()
originalValue = model.get_value(iter, 1)
currentValue = model.get_value(iter, 0)
args = model.get_value(iter, 2)
if len(args) == 1 and re.search("\\b" + args[0] + "\\b", newName):
if originalValue == currentValue:
model.set_value(iter, 0, currentValue.replace(args[0], "$"))
else:
model.set_value(iter, 0, originalValue)
def setUpdateUIMapSensitivity(self, item, selection):
if selection.count_selected_rows() == 1 and self.uiMapSelected(selection):
item.set_sensitive(True)
else:
item.set_sensitive(False)
def uiMapSelected(self, selection):
uiMaps = []
def addSelected(treemodel, path, iter, *args):
uiMaps.append(" on widget identified by " in treemodel.get_value(iter, 0))
selection.selected_foreach(addSelected)
return any(uiMaps)
def updateUIMap(self, widget, view):
selection = view.get_selection()
signals, widgetDescriptions = self.selectionToModel(selection)
self.createUIMapDialog(signals[0], widgetDescriptions[0])
def createUIMapDialog(self, signal, widgetDescription):
cmd = self.uiMapFileHandler.get(widgetDescription, signal)
dialog = gtk.Dialog('Update UI map for ' + "'" + cmd + "'", flags=gtk.DIALOG_MODAL)
dialog.vbox.set_spacing(10)
label = gtk.Label("Widget description")
widgetDescEntry = gtk.Entry()
widgetDescEntry.set_text(widgetDescription)
hbox = gtk.HBox(False, 10)
hbox.pack_start(label, expand=False, fill=False)
hbox.pack_start(widgetDescEntry, expand=True, fill=True)
dialog.vbox.pack_start(hbox, expand=False, fill=False)
label2 = gtk.Label("Activity")
signalEntry = gtk.Entry()
signalEntry.set_text(signal)
hbox2 = gtk.HBox(False, 10)
hbox2.pack_start(label2, expand=False, fill=False)
hbox2.pack_start(signalEntry, expand=True, fill=True)
dialog.vbox.pack_start(hbox2, expand=False, fill=False)
updateButton = dialog.add_button('Update', gtk.RESPONSE_ACCEPT)
cancelButton = dialog.add_button('Cancel', gtk.RESPONSE_CANCEL)
self.scriptEngine.monitorSignal("enter new widget description", "changed", widgetDescEntry)
self.scriptEngine.monitorSignal("enter new activity name", "changed", signalEntry)
self.scriptEngine.monitorSignal("accept update ui map file", "clicked", updateButton)
self.scriptEngine.monitorSignal("cancel update ui map file", "clicked", cancelButton)
dialog.connect("response", self.respondUpdateUIMap, signalEntry, widgetDescEntry, signal, widgetDescription)
dialog.show_all()
def respondUpdateUIMap(self, dialog, responseId, signalEntry, widgetDescEntry, oldSignal, oldWidgetDescription):
if responseId == gtk.RESPONSE_ACCEPT:
newSignal = signalEntry.get_text()
newWidgetDesc = widgetDescEntry.get_text()
if self.checkUpdateUIMapEntryNames(dialog, newWidgetDesc, newSignal):
dialog.hide()
self.uiMapFileHandler.updateSectionAndOptionNames(oldWidgetDescription, newWidgetDesc, oldSignal, newSignal)
self.updateUIMapPreview(oldWidgetDescription, newWidgetDesc, oldSignal, newSignal)
else:
dialog.hide()
def updateUIMapPreview(self, oldWidgetDescription, newWidgetDesc, oldSignal, newSignal):
leafIters = self.getLeafIters()
for iter in leafIters:
descValue = self.treeModel.get_value(iter, 2)
if descValue:
if oldWidgetDescription == descValue[0]:
signalValue = self.treeModel.get_value(iter, 1)
if signalValue != oldSignal:
newSignal = oldSignal
msg = self.makeUIMapMessage(newSignal, newWidgetDesc)
self.treeModel.set(iter, 0, msg, 1, newSignal, 2, [newWidgetDesc])
def getLeafIters(self):
iters = []
def addLeafIters(model, path, iter, *args):
if not model.iter_has_child(iter):
iters.append(iter)
self.treeModel.foreach(addLeafIters)
return iters
def checkUpdateUIMapEntryNames(self, parent, widgetName, activityName):
if not widgetName:
self.showErrorDialog(parent, "The widget name can't be empty.")
return False
elif not activityName:
self.showErrorDialog(parent, "The activity name can't be empty.")
return False
return True
def setRenameSensitivity(self, item, selection):
if selection.count_selected_rows() == 1 and (self.shortcutsSelected(selection) or (self.usecaseSelected(selection) and self.isNewName(selection))):
item.set_sensitive(True)
else:
item.set_sensitive(False)
def isNewName(self, selection):
newNames = []
def addNames(treeModel, path, iter, *args):
command = treeModel.get_value(iter, 1)
cmd, args = self.uiMapFileHandler.splitOptionValue(command)
if cmd:
newNames.append(cmd)
selection.selected_foreach(addNames)
return any(newNames)
def usecaseSelected(self, selection):
return not self.shortcutsSelected(selection) and not self.uiMapSelected(selection) \
and not self.waitCommandSelected(selection)
def waitCommandSelected(self, selection):
commands, _ = self.selectionToModel(selection)
return any(cmd.startswith(waitCommandName) for cmd in commands)
def rename(self, widget, view):
selection = view.get_selection()
commands, _ = self.selectionToModel(selection)
self.createRenameDialog(commands[0], self.shortcutsSelected(selection))
def deleteShortcut(self, menuItem, view):
selection = view.get_selection()
command = self.selectionToModel(selection)[0][0]
shortcut, args = self.shortcutManager.findShortcut(command)
confirmationMessage = "You are about to delete the file '" + os.path.basename(shortcut.name) + "'\nand remove all references to it in the current usecase."
self.showConfirmationDialog(menuItem.get_toplevel(), confirmationMessage, self.performShortcutDeletion, shortcut)
def performShortcutDeletion(self, shortcut):
print "ShortcutRemove", repr(shortcut.getShortcutRegexp().pattern), "renamed to '" + "\\n".join(shortcut.commands) + "'"
self.removeShortcutFromUsecase(shortcut)
self.removeShortcutFromPreview(shortcut)
self.shortcutManager.remove(shortcut)
def createRenameDialog(self, command, isShortcut=False):
if isShortcut:
name = 'shortcut'
shortcut, args = self.shortcutManager.findShortcut(command)
cmd = shortcut.getShortcutName()
else:
cmd, args = self.uiMapFileHandler.splitOptionValue(command)
name = "usecase"
dialog = gtk.Dialog('Rename ' + name, flags=gtk.DIALOG_MODAL)
dialog.vbox.set_spacing(10)
label = gtk.Label(name.title())
nameEntry = gtk.Entry()
nameEntry.set_text(cmd)
hbox = gtk.HBox(False, 10)
hbox.pack_start(label, expand=False, fill=False)
hbox.pack_start(nameEntry, expand=True, fill=True)
dialog.vbox.pack_start(hbox, expand=False, fill=False)
renameButton = dialog.add_button('Rename', gtk.RESPONSE_ACCEPT)
cancelButton = dialog.add_button('Cancel', gtk.RESPONSE_CANCEL)
self.scriptEngine.monitorSignal("rename " + name, "changed", nameEntry)
self.scriptEngine.monitorSignal("accept rename", "clicked", renameButton)
self.scriptEngine.monitorSignal("cancel rename", "clicked", cancelButton)
dialog.connect("response", self.respondRenameShortcut if isShortcut else self.respondRenameUsecase, nameEntry, cmd)
dialog.show_all()
def respondRenameUsecase(self, dialog, responseId, nameEntry, oldValue):
methodName = "UsecaseRename"
newValue = nameEntry.get_text()
if responseId == gtk.RESPONSE_ACCEPT:
if self.checkUsecaseName(dialog, newValue):
self.updateUsecaseNameInUIMap(oldValue, newValue)
self.updateUsecaseNameInShorcuts(oldValue, newValue)
self.replaceInFile(self.fileName, self.makeReplacement, [(oldValue, newValue)])
print encodingutils.encodeToLocale(methodName + " '" + oldValue + "' renamed to '" + newValue + "'")
self.initShortcutManager()
self.updateNameInPreview(oldValue, newValue)
dialog.hide()
else:
dialog.hide()
def respondRenameShortcut(self, dialog, responseId, nameEntry, oldValue):
methodName = "ShortcutRename"
newValue = nameEntry.get_text()
if responseId == gtk.RESPONSE_ACCEPT:
if self.checkShortcutName(dialog, newValue) and self.checkShortcutArguments(dialog, oldValue, newValue):
self.shortcutManager.rename(oldValue, self.getShortcutFileName(newValue))
oldValueRegexp = ReplayScript.transformToRegexp(oldValue)
# Update shortcut name in shortcut files
for _, shortcut in self.shortcutManager.getShortcuts():
if shortcut.getShortcutName() != newValue:
self.replaceInFile(shortcut.name, self.replaceShortcutName, oldValueRegexp, newValue)
# Update shortcut name in current usecase file
self.replaceInFile(self.fileName, self.replaceShortcutName, oldValueRegexp, newValue)
print methodName, repr(oldValueRegexp), "renamed to", repr(newValue)
self.initShortcutManager()
self.updateShortcutNameInPreview(oldValueRegexp, newValue)
dialog.hide()
else:
nameEntry.set_text(oldValue)
else:
dialog.hide()
def checkUsecaseName(self, parent, name):
if not name:
self.showErrorDialog(parent, "The usecase name can't be empty.")
return False
elif self.isInUIMap(name):
self.showErrorDialog(parent, "The usecase name already exists in the UI map file.")
return False
return True
def updateUsecaseNameInUIMap(self, oldCommand, newCommand):
for section, option in self.uiMapFileHandler.findSectionsAndOptions(oldCommand):
self.uiMapFileHandler.updateOptionValue(section, option, newCommand)
def updateUsecaseNameInShorcuts(self, oldCommand, newCommand):
for _, shortcut in self.shortcutManager.getShortcuts():
self.replaceInFile(shortcut.name, self.makeReplacement, [(oldCommand, newCommand)])
def checkShortcutArguments(self, parentDialog, oldShortcut, newShortcut):
numArgs = oldShortcut.count("$")
if numArgs != newShortcut.count("$"):
self.showErrorDialog(parentDialog, "The number of shortcut arguments('$') must be "+ str(numArgs))
return False
return True
def updateNameInPreview(self, oldValue, newValue):
def updateNode(model, path, iter, *args):
markup = model.get_value(iter, 0)
text = model.get_value(iter, 1)
if text.startswith(oldValue):
model.set_value(iter, 0, markup.replace(oldValue, newValue))
model.set_value(iter, 1, text.replace(oldValue, newValue))
self.treeModel.foreach(updateNode)
def updateShortcutNameInPreview(self, oldNameRegexp, newName):
def updateNode(model, path, iter, *args):
markup = model.get_value(iter, 0)
if markup.startswith("<b>"):
text = model.get_value(iter, 1)
newText = self.replaceShortcutName(text, 0, oldNameRegexp , newName)
newMarkup = self.replaceShortcutName(markup, 0, oldNameRegexp, newName)
if text != newText:
if newMarkup == markup:
newMarkup = newMarkup.replace(text, newText)
model.set_value(iter, 0, newMarkup)
model.set_value(iter, 1, newText)
self.treeModel.foreach(updateNode)
def regexpReplace(self, regexp, line, newText):
return re.sub(regexp, newText, line)
def replaceShortcutName(self, line, position, oldNameRegexp, newName):
def replaceArgs(matchobj):
return ReplayScript.getTextWithArgs(newName, [arg for arg in matchobj.groups()])
return self.regexpReplace(oldNameRegexp, line, replaceArgs)
class RenderableContainer(object):
is_group = True
def __init__(self):
self._components = []
self._componentmap = OrderedDict()
self._bindmap = OrderedDict()
def __json__(self, request):
return {"components": self._componentmap}
def rmRenderable(self, renderable_id):
renderable = self._componentmap.pop(renderable_id)
self._components.remove(renderable)
def addRenderable(self, renderable, pos=None):
""" Add renderable. If pos is given, insert into that
position, otherwise just append"""
if pos is None:
self._components.append(renderable)
else:
self._components.insert(pos, renderable)
# todo: see if we can use the pos parameter for the insertion
# into the _componentmap ordereddict
self._componentmap[renderable.id] = renderable
if getattr(renderable, 'bind', None):
self._bindmap[renderable.bind] = renderable
def getRenderables(self, recursive=False):
""" retrieve all renderables. If recursive is true, then
also return all renderables from the children recursively """
if recursive:
result = self._componentmap.values()
for r in self._componentmap.values():
try:
result += r.getRenderables(recursive)
except:
pass
return result
else:
return self._components
def getRenderable(self, id):
""" find renderable by id in the complete (recursive) tree """
found = self._componentmap.get(id, None)
if not found:
# search children
for r in self.getRenderables(False):
try:
found = r.getRenderable(id)
if found:
break
except:
pass
return found
def getRenderableByBind(self, bind):
found = self._bindmap.get(bind, None)
if not found:
# search children
for r in self.getRenderables(False):
try:
found = r.getRenderableByBind(bind)
if found:
break
except:
pass
return found
class UseCaseReplayer(storytext.guishared.IdleHandlerUseCaseReplayer):
def __init__(self, *args, **kw):
storytext.guishared.IdleHandlerUseCaseReplayer.__init__(self, *args, **kw)
# Anyone calling events_pending doesn't mean to include our logging events
# so we intercept it and return the right answer for them...
self.orig_events_pending = gtk.events_pending
gtk.events_pending = self.events_pending
self.orig_idle_add = gobject.idle_add
gobject.idle_add = self.idle_add
self.orig_source_remove = gobject.source_remove
gobject.source_remove = self.source_remove
self.allIdleHandlers = OrderedDict()
def addUiMap(self, uiMap):
self.uiMap = uiMap
if not self.loggerActive:
self.tryAddDescribeHandler()
def makeDescribeHandler(self, method):
if "file chooser to read file system" not in self.waitingForEvents:
return gobject.idle_add(method, priority=describer.PRIORITY_STORYTEXT_IDLE)
def tryRemoveDescribeHandler(self):
if not self.isMonitoring() and not self.readingEnabled: # pragma: no cover - cannot test code with replayer disabled
self.logger.debug("Disabling all idle handlers")
self._disableIdleHandlers()
if self.uiMap:
self.uiMap.windows = [] # So we regenerate everything next time around
def idle_add(self, *args, **kw):
handler = self.orig_idle_add(*args, **kw)
self.allIdleHandlers[handler] = args, kw, handler
return handler
def source_remove(self, origHandler):
if origHandler in self.allIdleHandlers:
actualHandler = self.allIdleHandlers.get(origHandler)[-1]
result = self.orig_source_remove(actualHandler)
del self.allIdleHandlers[origHandler]
return result
else:
# Added by timeout_add etc
return self.orig_source_remove(origHandler)
def removeAllIdleHandlers(self):
idleArgs = []
self.logger.debug("Removing idle handlers")
for origHandler, data in self.allIdleHandlers.items():
handler = data[-1]
if handler != describer.idleScheduler.idleHandler:
self.removeHandler(handler)
idleArgs.append(origHandler)
return idleArgs
def readdAllIdleHandlers(self, idleArgs):
self.logger.debug("Readding idle handlers")
for origHandler in idleArgs:
args, kw, _ = self.allIdleHandlers.get(origHandler)
newHandler = self.orig_idle_add(*args, **kw)
self.allIdleHandlers[origHandler] = args, kw, newHandler
def events_pending(self): # pragma: no cover - cannot test code with replayer disabled
if not self.isActive():
self.logger.debug("Removing idle handler for descriptions")
self._disableIdleHandlers()
return_value = self.orig_events_pending()
if not self.isActive():
if self.readingEnabled:
self.enableReplayHandler()
else:
self.logger.debug("Re-adding idle handler for descriptions")
self.tryAddDescribeHandler()
return return_value
def events_pending_no_idle_handlers(self):
idleArgs = self.removeAllIdleHandlers()
return_value = self.orig_events_pending()
self.readdAllIdleHandlers(idleArgs)
return return_value
def removeHandler(self, handler):
return self.orig_source_remove(handler)
def makeTimeoutReplayHandler(self, method, milliseconds):
return gobject.timeout_add(milliseconds, method, priority=describer.PRIORITY_STORYTEXT_REPLAY_IDLE)
def makeIdleReplayHandler(self, method):
return gobject.idle_add(method, priority=describer.PRIORITY_STORYTEXT_REPLAY_IDLE)
def shouldMonitorWindow(self, window):
hint = window.get_type_hint()
if hint == gtk.gdk.WINDOW_TYPE_HINT_TOOLTIP or hint == gtk.gdk.WINDOW_TYPE_HINT_COMBO:
return False
elif isinstance(window.get_child(), gtk.Menu) and \
(window.get_child().get_name() == "gtk-combobox-popup-menu" or isinstance(window.get_child().get_attach_widget(), gtk.ComboBox)):
return False
else:
return True
def findWindowsForMonitoring(self):
return filter(self.shouldMonitorWindow, gtk.window_list_toplevels())
def describeNewWindow(self, window):
if window.get_property("visible"):
describer.describeNewWindow(window)
def callReplayHandlerAgain(self, *args):
return True # GTK's way of saying the handle should come again
def runMainLoopWithReplay(self):
while self.events_pending_no_idle_handlers():
gtk.main_iteration()
if self.delay:
time.sleep(self.delay)
if self.isActive():
self.describeAndRun()
class CnCGraph(object):
def __init__(self, name, g):
verifyCollectionDecls("item", g.itemColls)
steps = [ x.step for x in g.stepRels ]
verifyCollectionDecls("step", steps)
self.name = name
# items
self.itemDeclarations = OrderedDict((i.collName, makeItemDecl(i)) for i in g.itemColls)
self.concreteItems = [ i for i in self.itemDeclarations.values() if not i.isVirtual ]
# item virtual mappings
self.vms = [ i for i in self.itemDeclarations.values() if i.isVirtual ]
self.inlineVms = [ i for i in self.vms if i.isInline ]
self.externVms = [ i for i in self.vms if not i.isInline ]
# steps / pseudo-steps
self.stepFunctions = OrderedDict((x.step.collName, StepFunction(x)) for x in g.stepRels)
verifyEnv(self.stepFunctions)
self.initFunction = self.stepFunctions.pop(initNameRaw)
self.initFunction.collName = "cncInitialize"
self.finalizeFunction = self.stepFunctions.pop(finalizeNameRaw)
self.finalizeFunction.collName = "cncFinalize"
self.finalAndSteps = [self.finalizeFunction] + self.stepFunctions.values()
# set up step attribute lookup dict
self.stepLikes = OrderedDict(self.stepFunctions)
self.stepLikes[self.initFunction.collName] = self.initFunction
self.stepLikes[self.finalizeFunction.collName] = self.finalizeFunction
# attribute tracking
self.allAttrNames = set()
# context
self.ctxParams = filter(bool, map(strip, g.ctx.splitlines())) if g.ctx else []
def hasTuning(self, name):
return name in self.allAttrNames
def hasCustomDist(self):
return self.hasTuning('distfn') or self.hasTuning('placeWith')
def lookupType(self, item):
return self.itemDeclarations[item.collName].type
def itemDistFn(self, collName, ranksExpr):
return getDistFn(self.itemDeclarations, collName, ranksExpr)
def stepDistFn(self, collName, ranksExpr):
return getDistFn(self.stepLikes, collName, ranksExpr)
def itemTuningFn(self, collName, name, ranksExpr, default):
return getTuningFn(self.itemDeclarations, collName, name, ranksExpr, default)
def stepTuningFn(self, collName, name, ranksExpr, default):
return getTuningFn(self.stepLikes, collName, name, ranksExpr, default)
def priorityFn(self, collName, ranksExpr):
return self.stepTuningFn(collName, 'priority', ranksExpr, "0")
def addTunings(self, tuningSpec):
for t in tuningSpec.itemTunings:
x = self.itemDeclarations.get(t.collName)
assert x, "Unknown item in tuning: {0}".format(t.collName)
x.attrs.update(t.attrs)
self.allAttrNames.update(t.attrs.keys())
for t in tuningSpec.stepTunings:
x = self.stepLikes.get(t.collName)
assert x, "Unknown step in tuning: {0}".format(t.collName)
if t.inputName:
i = x.inputsDict.get(t.inputName)
assert i, "Unknown input in tuning: {0} <- {1}".format(t.collName, t.inputName)
i.attrs.update(t.attrs)
self.allAttrNames.update(t.attrs.keys())
else:
x.attrs.update(t.attrs)
self.allAttrNames.update(t.attrs.keys())
class UseCaseNameChooser:
title = "Enter Usecase names for auto-recorded actions"
def __init__(self, fileName, interface, mapFiles):
self.fileName = fileName
self.interface = interface
self.uiMapFileHandler = UIMapFileHandler(mapFiles)
self.scriptEngine = gtktoolkit.ScriptEngine(uiMapFiles=[])
self.allEntries = OrderedDict()
def collectNames(self):
commands = [ line.strip() for line in open(self.fileName) ]
autoGenerated = self.getAutoGenerated(commands)
if len(autoGenerated) == 0:
return
autoGeneratedInfo = self.parseAutoGenerated(autoGenerated)
dialog = self.createDialog(autoGeneratedInfo, commands)
response = dialog.run()
if response == gtk.RESPONSE_ACCEPT:
newNames = [ entry.get_text() for entry in self.allEntries.values() ]
dialog.destroy()
self.replaceInFile(self.fileName, zip(autoGenerated, newNames))
toStore = zip(autoGeneratedInfo, newNames)
for ((command, widgetType, widgetDescription, signalName), eventName) in toStore:
self.uiMapFileHandler.storeInfo(widgetDescription, signalName, eventName)
self.uiMapFileHandler.write()
else:
# Don't leave a half generated filename behind, if we didn't fill in the dialog properly
# we should remove it so nobody saves it...
os.remove(self.fileName)
dialog.destroy()
def getAutoGenerated(self, commands):
# Find the auto-generated commands and strip them of their arguments
autoGenerated = []
for command in commands:
if command.startswith("Auto."):
pos = command.rfind("'")
commandWithoutArg = command[:pos + 1]
if not commandWithoutArg in autoGenerated:
autoGenerated.append(commandWithoutArg)
return autoGenerated
def parseAutoGenerated(self, commands):
autoGenerated = []
for command in commands:
parts = command[5:].split("'")
initialPart = parts[0][:-1]
widgetType, signalName = initialPart.split(".", 1)
widgetDescription = parts[1].replace("<APOSTROPHE>", "'")
autoGenerated.append((command, widgetType, widgetDescription, signalName))
return autoGenerated
def replaceInFile(self, fileName, replacements):
newFileName = fileName + ".tmp"
newFile = open(newFileName, "w")
for line in open(fileName):
newLine = self.makeReplacement(line, replacements)
if newLine:
newFile.write(newLine)
newFile.close()
shutil.move(newFileName, fileName)
def makeReplacement(self, command, replacements):
for origName, newName in replacements:
if command.startswith(origName):
if newName:
return command.replace(origName, newName)
else:
return
return command
def createDialog(self, autoGenerated, commands):
dialog = gtk.Dialog(self.title, flags=gtk.DIALOG_MODAL)
dialog.set_name("Name Entry Window")
contents = self.createTable(autoGenerated, dialog)
dialog.vbox.pack_start(contents, expand=True, fill=True)
preview = self.createPreview(commands)
dialog.vbox.pack_start(gtk.HSeparator())
dialog.vbox.pack_start(preview, expand=True, fill=True)
yesButton = dialog.add_button(gtk.STOCK_OK, gtk.RESPONSE_ACCEPT)
cancelButton = dialog.add_button(gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL)
dialog.set_default_response(gtk.RESPONSE_ACCEPT)
self.scriptEngine.monitorSignal("finish name entry editing", "clicked", yesButton)
self.scriptEngine.monitorSignal("cancel name entry editing", "clicked", cancelButton)
dialog.show_all()
return dialog
def createMarkupLabel(self, markup):
label = gtk.Label()
label.set_markup(markup)
return label
def activateEntry(self, entry, dialog, *args):
dialog.response(gtk.RESPONSE_ACCEPT)
def getActionDescription(self, signalName, widgetType):
try:
exec "from " + self.interface + "toolkit import ScriptEngine"
except ImportError:
# If we haven't even got any such interface, don't worry about this mechanism
return signalName
desc = ScriptEngine.getDisplayName(signalName)
if desc:
return desc
if signalName == "activate":
if "Entry" in widgetType:
return "pressed Enter"
else:
return "selected"
elif signalName == "changed":
if "Entry" in widgetType:
return "edited text"
else:
return "selected item"
parts = signalName.split(".")
if len(parts) == 1:
return signalName.replace("-", " ")
if parts[0] == "response":
text = parts[1]
if "--" in text:
return text.replace("--", "='") + "'"
else:
return text
columnName = parts[1]
remaining = parts[0]
if remaining == "toggled":
remaining = ".".join([ remaining, parts[-1] ])
return ScriptEngine.getColumnDisplayName(remaining) + " '" + columnName + "'"
def splitAutoCommand(self, command):
for cmd in self.allEntries.keys():
if command.startswith(cmd):
arg = command.replace(cmd, "")
return cmd, arg
return None, None
def updatePreview(self, entry, data):
buffer, lineNo, arg = data
text = entry.get_text() or "?"
toUse = self.convertToUtf8(text + arg)
start = buffer.get_iter_at_line(lineNo)
end = buffer.get_iter_at_line(lineNo + 1)
buffer.delete(start, end)
buffer.insert(start, toUse + "\n")
def createPreview(self, commands):
frame = gtk.Frame("Current Usecase Preview")
view = gtk.TextView()
view.set_editable(False)
view.set_cursor_visible(False)
view.set_wrap_mode(gtk.WRAP_WORD)
buffer = view.get_buffer()
for ix, command in enumerate(commands):
autoCmdName, autoArg = self.splitAutoCommand(command)
if autoCmdName:
buffer.insert(buffer.get_end_iter(), self.convertToUtf8("?" + autoArg + "\n"))
entry = self.allEntries.get(autoCmdName)
entry.connect("changed", self.updatePreview, (buffer, ix, autoArg))
else:
buffer.insert(buffer.get_end_iter(), self.convertToUtf8(command) + "\n")
frame.add(view)
return frame
def convertToUtf8(self, text):
try:
localeEncoding = getdefaultlocale()[1]
if localeEncoding:
return unicode(text, localeEncoding, errors="replace").encode('utf-8', 'replace')
except ValueError:
pass
return text
def createTable(self, autoGenerated, dialog):
table = gtk.Table(rows=len(autoGenerated) + 1, columns=4)
table.set_col_spacings(20)
headers = [ "Widget Type", "Identified By", "Action Performed", "Usecase Name" ]
for col, header in enumerate(headers):
table.attach(self.createMarkupLabel("<b><u>" + header + "</u></b>"),
col, col + 1, 0, 1, xoptions=gtk.FILL)
for rowIndex, (command, widgetType, widgetDesc, signalName) in enumerate(autoGenerated):
table.attach(gtk.Label(widgetType), 0, 1, rowIndex + 1, rowIndex + 2, xoptions=gtk.FILL)
actionDesc = self.getActionDescription(signalName, widgetType)
widgetDescUtf8 = self.convertToUtf8(widgetDesc)
table.attach(gtk.Label(widgetDescUtf8), 1, 2, rowIndex + 1, rowIndex + 2, xoptions=gtk.FILL)
table.attach(gtk.Label(actionDesc), 2, 3, rowIndex + 1, rowIndex + 2, xoptions=gtk.FILL)
entry = gtk.Entry()
scriptName = "enter usecase name for signal '" + signalName + "' on " + widgetType + " '" + widgetDesc + "' ="
self.scriptEngine.monitorSignal(scriptName, "changed", entry)
entry.connect("activate", self.activateEntry, dialog)
self.scriptEngine.monitorSignal("finish name entry editing by pressing <enter>", "activate", entry)
self.allEntries[command] = entry
table.attach(entry, 3, 4, rowIndex + 1, rowIndex + 2)
frame = gtk.Frame("Previously unseen actions: provide names for the interesting ones")
frame.add(table)
return frame
class RenderableContainer(object):
is_group = True
def __init__(self):
self._components = []
self._componentmap = OrderedDict()
self._bindmap = OrderedDict()
def __json__(self, request):
return {"components": self._componentmap}
def rmRenderable(self, renderable_id):
renderable = self._componentmap.pop(renderable_id)
self._components.remove(renderable)
def addRenderable(self, renderable, pos=None):
""" Add renderable. If pos is given, insert into that
position, otherwise just append"""
if pos is None:
self._components.append(renderable)
else:
self._components.insert(pos, renderable)
# todo: see if we can use the pos parameter for the insertion
# into the _componentmap ordereddict
self._componentmap[renderable.id] = renderable
if getattr(renderable, "bind", None):
self._bindmap[renderable.bind] = renderable
def getRenderables(self, recursive=False):
""" retrieve all renderables. If recursive is true, then
also return all renderables from the children recursively """
if recursive:
result = self._componentmap.values()
for r in self._componentmap.values():
try:
result += r.getRenderables(recursive)
except:
pass
return result
else:
return self._components
def getRenderable(self, id):
""" find renderable by id in the complete (recursive) tree """
found = self._componentmap.get(id, None)
if not found:
# search children
for r in self.getRenderables(False):
try:
found = r.getRenderable(id)
if found:
break
except:
pass
return found
def getRenderableByBind(self, bind):
found = self._bindmap.get(bind, None)
if not found:
# search children
for r in self.getRenderables(False):
try:
found = r.getRenderableByBind(bind)
if found:
break
except:
pass
return found
