def generateDashboard(self):
""" Generates the HTML for the dashboard. """
currentConfig = config.currentConfiguration()
s = """
<html>
<head>
<title>Fantasm</title>
"""
s += STYLESHEET
s += """
</head>
<body>
<h1>Fantasm</h1>
<h4>Configured Machines</h4>
<table class='ae-table ae-table-striped' cellpadding='0' cellspacing='0'>
<thead>
<tr>
<th>Name</th>
<th>Queue</th>
<th>States</th>
<th>Transitions</th>
<th>Chart</th>
</tr>
</thead>
<tbody>
"""
even = True
for machineKey in sorted(currentConfig.machines.keys()):
machine = currentConfig.machines[machineKey]
even = False if even else True
s += """
<tr class='%(class)s'>
<td>%(machineName)s</td>
<td>%(queueName)s</td>
<td>%(numStates)d</td>
<td>%(numTransitions)d</td>
<td><a href='%(rootUrl)sgraphviz/%(machineName)s/'>view</a></td>
</tr>
""" % {
'class': 'ae-even' if even else '',
'machineName': machine.name,
'queueName': machine.queueName,
'numStates': len(machine.states),
'numTransitions': len(machine.transitions),
'rootUrl': currentConfig.rootUrl,
}
s += """
</tbody>
</table>
</body>
</html>
"""
return s
def generateDashboard(self):
""" Generates the HTML for the dashboard. """
currentConfig = config.currentConfiguration()
s = """
<html>
<head>
<title>Fantasm</title>
"""
s += STYLESHEET
s += """
</head>
<body>
<h1>Fantasm</h1>
<h4>Configured Machines</h4>
<table class='ae-table ae-table-striped' cellpadding='0' cellspacing='0'>
<thead>
<tr>
<th>Name</th>
<th>Queue</th>
<th>States</th>
<th>Transitions</th>
<th>Chart</th>
</tr>
</thead>
<tbody>
"""
even = True
for machineKey in sorted(currentConfig.machines.keys()):
machine = currentConfig.machines[machineKey]
even = False if even else True
s += """
<tr class='%(class)s'>
<td>%(machineName)s</td>
<td>%(queueName)s</td>
<td>%(numStates)d</td>
<td>%(numTransitions)d</td>
<td><a href='%(rootUrl)sgraphviz/%(machineName)s/'>view</a></td>
</tr>
""" % {
'class': 'ae-even' if even else '',
'machineName': machine.name,
'queueName': machine.queueName,
'numStates': len(machine.states),
'numTransitions': len(machine.transitions),
'rootUrl': currentConfig.rootUrl,
}
s += """
</tbody>
</table>
</body>
</html>
"""
return s
def getCurrentFSM():
""" Returns the current FSM singleton. """
# W0603: 32:currentConfiguration: Using the global statement
global _fsm # pylint: disable-msg=W0603
# always reload the FSM for dev_appserver to grab recent dev changes
if _fsm and not constants.DEV_APPSERVER:
return _fsm
currentConfig = config.currentConfiguration()
_fsm = FSM(currentConfig=currentConfig)
return _fsm
def getCurrentFSM():
""" Returns the current FSM singleton. """
# W0603: 32:currentConfiguration: Using the global statement
global _fsm # pylint: disable-msg=W0603
# always reload the FSM for dev_appserver to grab recent dev changes
if _fsm and not constants.DEV_APPSERVER:
return _fsm
currentConfig = config.currentConfiguration()
_fsm = FSM(currentConfig=currentConfig)
return _fsm
def _init(self, currentConfig=None):
""" Constructs a group of singleton States and Transitions from the machineConfig
@param currentConfig: a config._Configuration instance (dependency injection). if None,
then the factory uses config.currentConfiguration()
"""
import logging
logging.info("Initializing FSM factory.")
self.config = currentConfig or config.currentConfiguration()
self.machines = {}
self.pseudoInits, self.pseudoFinals = {}, {}
for machineConfig in self.config.machines.values():
self.machines[machineConfig.name] = {constants.MACHINE_STATES_ATTRIBUTE: {},
constants.MACHINE_TRANSITIONS_ATTRIBUTE: {}}
machine = self.machines[machineConfig.name]
# create a pseudo-init state for each machine that transitions to the initialState
pseudoInit = State(FSM.PSEUDO_INIT, None, None, None)
self.pseudoInits[machineConfig.name] = pseudoInit
self.machines[machineConfig.name][constants.MACHINE_STATES_ATTRIBUTE][FSM.PSEUDO_INIT] = pseudoInit
# create a pseudo-final state for each machine that transitions from the finalState(s)
pseudoFinal = State(FSM.PSEUDO_FINAL, None, None, None, isFinalState=True)
self.pseudoFinals[machineConfig.name] = pseudoFinal
self.machines[machineConfig.name][constants.MACHINE_STATES_ATTRIBUTE][FSM.PSEUDO_FINAL] = pseudoFinal
for stateConfig in machineConfig.states.values():
state = self._getState(machineConfig, stateConfig)
# add the transition from pseudo-init to initialState
if state.isInitialState:
transition = Transition(FSM.PSEUDO_INIT, state,
retryOptions = self._buildRetryOptions(machineConfig),
queueName=machineConfig.queueName)
self.pseudoInits[machineConfig.name].addTransition(transition, FSM.PSEUDO_INIT)
# add the transition from finalState to pseudo-final
if state.isFinalState:
transition = Transition(FSM.PSEUDO_FINAL, pseudoFinal,
retryOptions = self._buildRetryOptions(machineConfig),
queueName=machineConfig.queueName)
state.addTransition(transition, FSM.PSEUDO_FINAL)
machine[constants.MACHINE_STATES_ATTRIBUTE][stateConfig.name] = state
for transitionConfig in machineConfig.transitions.values():
source = machine[constants.MACHINE_STATES_ATTRIBUTE][transitionConfig.fromState.name]
transition = self._getTransition(machineConfig, transitionConfig)
machine[constants.MACHINE_TRANSITIONS_ATTRIBUTE][transitionConfig.name] = transition
event = transitionConfig.event
source.addTransition(transition, event)
def getMachineNameFromRequest(request):
""" Returns the machine name embedded in the request.
@param request: an HttpRequest
@return: the machineName (as a string)
"""
path = request.path
# strip off the mount-point
currentConfig = config.currentConfiguration()
mountPoint = currentConfig.rootUrl # e.g., '/fantasm/'
if not path.startswith(mountPoint):
raise FSMRuntimeError("rootUrl '%s' must match app.yaml mapping." % mountPoint)
path = path[len(mountPoint):]
# split on '/', the second item will be the machine name
parts = path.split('/')
return parts[1] # 0-based index
def getMachineNameFromRequest(request):
""" Returns the machine name embedded in the request.
@param request: an HttpRequest
@return: the machineName (as a string)
"""
path = request.path
# strip off the mount-point
currentConfig = config.currentConfiguration()
mountPoint = currentConfig.rootUrl # e.g., '/fantasm/'
if not path.startswith(mountPoint):
raise FSMRuntimeError("rootUrl '%s' must match app.yaml mapping." %
mountPoint)
path = path[len(mountPoint):]
# split on '/', the second item will be the machine name
parts = path.split('/')
return parts[1] # 0-based index
def getMachineConfig(request):
""" Returns the machine configuration specified by a URI in a HttpReuest
@param request: an HttpRequest
@return: a config._machineConfig instance
"""
# parse out the machine-name from the path {mount-point}/fsm/{machine-name}/startState/event/endState/
# NOTE: /startState/event/endState/ is optional
machineName = getMachineNameFromRequest(request)
# load the configuration, lookup the machine-specific configuration
# FIXME: sort out a module level cache for the configuration - it must be sensitive to YAML file changes
# for developer-time experience
currentConfig = config.currentConfiguration()
try:
machineConfig = currentConfig.machines[machineName]
return machineConfig
except KeyError:
raise UnknownMachineError(machineName)
def getMachineConfig(request):
""" Returns the machine configuration specified by a URI in a HttpReuest
@param request: an HttpRequest
@return: a config._machineConfig instance
"""
# parse out the machine-name from the path {mount-point}/fsm/{machine-name}/startState/event/endState/
# NOTE: /startState/event/endState/ is optional
machineName = getMachineNameFromRequest(request)
# load the configuration, lookup the machine-specific configuration
# FIXME: sort out a module level cache for the configuration - it must be sensitive to YAML file changes
# for developer-time experience
currentConfig = config.currentConfiguration()
try:
machineConfig = currentConfig.machines[machineName]
return machineConfig
except KeyError:
raise UnknownMachineError(machineName)
def __init__(self, currentConfig=None):
""" Constructor which either initializes the module/class-level cache, or simply uses it
@param currentConfig: a config._Configuration instance (dependency injection). if None,
then the factory uses config.currentConfiguration()
"""
currentConfig = currentConfig or config.currentConfiguration()
# if the FSM is not using the currentConfig (.yaml was edited etc.)
if not (FSM._CURRENT_CONFIG is currentConfig):
self._init(currentConfig=currentConfig)
FSM._CURRENT_CONFIG = self.config
FSM._MACHINES = self.machines
FSM._PSEUDO_INITS = self.pseudoInits
FSM._PSEUDO_FINALS = self.pseudoFinals
# otherwise simply use the cached currentConfig etc.
else:
self.config = FSM._CURRENT_CONFIG
self.machines = FSM._MACHINES
self.pseudoInits = FSM._PSEUDO_INITS
self.pseudoFinals = FSM._PSEUDO_FINALS
def __init__(self, currentConfig=None):
""" Constructor which either initializes the module/class-level cache, or simply uses it
@param currentConfig: a config._Configuration instance (dependency injection). if None,
then the factory uses config.currentConfiguration()
"""
currentConfig = currentConfig or config.currentConfiguration()
# if the FSM is not using the currentConfig (.yaml was edited etc.)
if not (FSM._CURRENT_CONFIG is currentConfig):
self._init(currentConfig=currentConfig)
FSM._CURRENT_CONFIG = self.config
FSM._MACHINES = self.machines
FSM._PSEUDO_INITS = self.pseudoInits
FSM._PSEUDO_FINALS = self.pseudoFinals
# otherwise simply use the cached currentConfig etc.
else:
self.config = FSM._CURRENT_CONFIG
self.machines = FSM._MACHINES
self.pseudoInits = FSM._PSEUDO_INITS
self.pseudoFinals = FSM._PSEUDO_FINALS
def _init(self, currentConfig=None):
""" Constructs a group of singleton States and Transitions from the machineConfig
@param currentConfig: a config._Configuration instance (dependency injection). if None,
then the factory uses config.currentConfiguration()
"""
import logging
logging.info("Initializing FSM factory.")
self.config = currentConfig or config.currentConfiguration()
self.machines = {}
self.pseudoInits, self.pseudoFinals = {}, {}
for machineConfig in self.config.machines.values():
self.machines[machineConfig.name] = {
constants.MACHINE_STATES_ATTRIBUTE: {},
constants.MACHINE_TRANSITIONS_ATTRIBUTE: {}
}
machine = self.machines[machineConfig.name]
# create a pseudo-init state for each machine that transitions to the initialState
pseudoInit = State(FSM.PSEUDO_INIT, None, None, None)
self.pseudoInits[machineConfig.name] = pseudoInit
self.machines[machineConfig.name][
constants.MACHINE_STATES_ATTRIBUTE][
FSM.PSEUDO_INIT] = pseudoInit
# create a pseudo-final state for each machine that transitions from the finalState(s)
pseudoFinal = State(FSM.PSEUDO_FINAL,
None,
None,
None,
isFinalState=True)
self.pseudoFinals[machineConfig.name] = pseudoFinal
self.machines[machineConfig.name][
constants.MACHINE_STATES_ATTRIBUTE][
FSM.PSEUDO_FINAL] = pseudoFinal
for stateConfig in machineConfig.states.values():
state = self._getState(machineConfig, stateConfig)
# add the transition from pseudo-init to initialState
if state.isInitialState:
transition = Transition(
FSM.PSEUDO_INIT,
state,
retryOptions=self._buildRetryOptions(machineConfig),
queueName=machineConfig.queueName)
self.pseudoInits[machineConfig.name].addTransition(
transition, FSM.PSEUDO_INIT)
# add the transition from finalState to pseudo-final
if state.isFinalState:
transition = Transition(
FSM.PSEUDO_FINAL,
pseudoFinal,
retryOptions=self._buildRetryOptions(machineConfig),
queueName=machineConfig.queueName)
state.addTransition(transition, FSM.PSEUDO_FINAL)
machine[constants.MACHINE_STATES_ATTRIBUTE][
stateConfig.name] = state
for transitionConfig in machineConfig.transitions.values():
source = machine[constants.MACHINE_STATES_ATTRIBUTE][
transitionConfig.fromState.name]
transition = self._getTransition(machineConfig,
transitionConfig)
machine[constants.MACHINE_TRANSITIONS_ATTRIBUTE][
transitionConfig.name] = transition
event = transitionConfig.event
source.addTransition(transition, event)
def get_or_post(self, method='POST'):
""" Handles the GET/POST request.
FIXME: this is getting a touch long
"""
# ensure that we have our services for the next 30s (length of a single request)
if config.currentConfiguration().enableCapabilitiesCheck:
unavailable = set()
for service in REQUIRED_SERVICES:
try:
if not CapabilitySet(service).is_enabled():
unavailable.add(service)
except Exception:
# Something failed while checking capabilities, just assume they are going to be available.
# These checks were from an era of lower-reliability which is no longer the case.
pass
if unavailable:
raise RequiredServicesUnavailableRuntimeError(unavailable)
# the case of headers is inconsistent on dev_appserver and appengine
# ie 'X-AppEngine-TaskRetryCount' vs. 'X-AppEngine-Taskretrycount'
lowerCaseHeaders = dict([
(key.lower(), value)
for key, value in self.request.headers.items()
])
taskName = lowerCaseHeaders.get('x-appengine-taskname')
retryCount = int(lowerCaseHeaders.get('x-appengine-taskretrycount', 0))
# pull out X-Fantasm-* headers
headers = None
for key, value in self.request.headers.items():
if key.startswith(HTTP_REQUEST_HEADER_PREFIX):
headers = headers or {}
if ',' in value:
headers[key] = [v.strip() for v in value.split(',')]
else:
headers[key] = value.strip()
requestData = {
'POST': self.request.POST,
'GET': self.request.GET
}[method]
method = requestData.get('method') or method
machineName = getMachineNameFromRequest(self.request)
# get the incoming instance name, if any
instanceName = requestData.get(INSTANCE_NAME_PARAM)
# get the incoming state, if any
fsmState = requestData.get(STATE_PARAM)
# get the incoming event, if any
fsmEvent = requestData.get(EVENT_PARAM)
assert (fsmState and instanceName
) or True # if we have a state, we should have an instanceName
assert (fsmState and fsmEvent
) or True # if we have a state, we should have an event
obj = TemporaryStateObject()
# make a copy, add the data
fsm = getCurrentFSM().createFSMInstance(machineName,
currentStateName=fsmState,
instanceName=instanceName,
method=method,
obj=obj,
headers=headers)
# Taskqueue can invoke multiple tasks of the same name occassionally. Here, we'll use
# a datastore transaction as a semaphore to determine if we should actually execute this or not.
if taskName and fsm.useRunOnceSemaphore:
semaphoreKey = '%s--%s' % (taskName, retryCount)
semaphore = RunOnceSemaphore(semaphoreKey, None)
if not semaphore.writeRunOnceSemaphore(payload='fantasm')[0]:
# we can simply return here, this is a duplicate fired task
logging.warn(
'A duplicate task "%s" has been queued by taskqueue infrastructure. Ignoring.',
taskName)
self.response.set_status(200)
return
# in "immediate mode" we try to execute as much as possible in the current request
# for the time being, this does not include things like fork/spawn/contuniuations/fan-in
immediateMode = IMMEDIATE_MODE_PARAM in requestData.keys()
if immediateMode:
obj[IMMEDIATE_MODE_PARAM] = immediateMode
obj[MESSAGES_PARAM] = []
fsm.Queue = NoOpQueue # don't queue anything else
# pylint: disable=W0201
# - initialized outside of ctor is ok in this case
self.fsm = fsm # used for logging in handle_exception
# pull all the data off the url and stuff into the context
for key, value in requestData.items():
if key in NON_CONTEXT_PARAMS:
continue # these are special, don't put them in the data
# deal with ...a=1&a=2&a=3...
value = requestData.get(key)
valueList = requestData.getall(key)
if len(valueList) > 1:
value = valueList
if key.endswith('[]'):
key = key[:-2]
value = [value]
if key in fsm.contextTypes.keys():
fsm.putTypedValue(key, value)
else:
fsm[key] = value
if not (fsmState or fsmEvent):
# just queue up a task to run the initial state transition using retries
fsm[STARTED_AT_PARAM] = time.time()
# initialize the fsm, which returns the 'pseudo-init' event
fsmEvent = fsm.initialize()
else:
# add the retry counter into the machine context from the header
obj[RETRY_COUNT_PARAM] = retryCount
# add the actual task name to the context
obj[TASK_NAME_PARAM] = taskName
# dispatch and return the next event
fsmEvent = fsm.dispatch(fsmEvent, obj)
# loop and execute until there are no more events - any exceptions
# will make it out to the user in the response - useful for debugging
if immediateMode:
while fsmEvent:
fsmEvent = fsm.dispatch(fsmEvent, obj)
self.response.headers['Content-Type'] = 'application/json'
data = {
'obj': obj,
'context': fsm,
}
self.response.out.write(json.dumps(data, cls=Encoder))
def get_or_post(self, method='POST'):
""" Handles the GET/POST request.
FIXME: this is getting a touch long
"""
# ensure that we have our services for the next 30s (length of a single request)
if config.currentConfiguration().enableCapabilitiesCheck:
unavailable = set()
for service in REQUIRED_SERVICES:
try:
if not CapabilitySet(service).is_enabled():
unavailable.add(service)
except Exception:
# Something failed while checking capabilities, just assume they are going to be available.
# These checks were from an era of lower-reliability which is no longer the case.
pass
if unavailable:
raise RequiredServicesUnavailableRuntimeError(unavailable)
# the case of headers is inconsistent on dev_appserver and appengine
# ie 'X-AppEngine-TaskRetryCount' vs. 'X-AppEngine-Taskretrycount'
lowerCaseHeaders = dict([(key.lower(), value) for key, value in self.request.headers.items()])
taskName = lowerCaseHeaders.get('x-appengine-taskname')
retryCount = int(lowerCaseHeaders.get('x-appengine-taskretrycount', 0))
# pull out X-Fantasm-* headers
headers = None
for key, value in self.request.headers.items():
if key.startswith(HTTP_REQUEST_HEADER_PREFIX):
headers = headers or {}
if ',' in value:
headers[key] = [v.strip() for v in value.split(',')]
else:
headers[key] = value.strip()
requestData = {'POST': self.request.POST, 'GET': self.request.GET}[method]
method = requestData.get('method') or method
machineName = getMachineNameFromRequest(self.request)
# get the incoming instance name, if any
instanceName = requestData.get(INSTANCE_NAME_PARAM)
# get the incoming state, if any
fsmState = requestData.get(STATE_PARAM)
# get the incoming event, if any
fsmEvent = requestData.get(EVENT_PARAM)
assert (fsmState and instanceName) or True # if we have a state, we should have an instanceName
assert (fsmState and fsmEvent) or True # if we have a state, we should have an event
obj = TemporaryStateObject()
# make a copy, add the data
fsm = getCurrentFSM().createFSMInstance(machineName,
currentStateName=fsmState,
instanceName=instanceName,
method=method,
obj=obj,
headers=headers)
# Taskqueue can invoke multiple tasks of the same name occassionally. Here, we'll use
# a datastore transaction as a semaphore to determine if we should actually execute this or not.
if taskName and fsm.useRunOnceSemaphore:
semaphoreKey = '%s--%s' % (taskName, retryCount)
semaphore = RunOnceSemaphore(semaphoreKey, None)
if not semaphore.writeRunOnceSemaphore(payload='fantasm')[0]:
# we can simply return here, this is a duplicate fired task
logging.warn('A duplicate task "%s" has been queued by taskqueue infrastructure. Ignoring.', taskName)
self.response.set_status(200)
return
# in "immediate mode" we try to execute as much as possible in the current request
# for the time being, this does not include things like fork/spawn/contuniuations/fan-in
immediateMode = IMMEDIATE_MODE_PARAM in requestData.keys()
if immediateMode:
obj[IMMEDIATE_MODE_PARAM] = immediateMode
obj[MESSAGES_PARAM] = []
fsm.Queue = NoOpQueue # don't queue anything else
# pylint: disable=W0201
# - initialized outside of ctor is ok in this case
self.fsm = fsm # used for logging in handle_exception
# pull all the data off the url and stuff into the context
for key, value in requestData.items():
if key in NON_CONTEXT_PARAMS:
continue # these are special, don't put them in the data
# deal with ...a=1&a=2&a=3...
value = requestData.get(key)
valueList = requestData.getall(key)
if len(valueList) > 1:
value = valueList
if key.endswith('[]'):
key = key[:-2]
value = [value]
if key in fsm.contextTypes.keys():
fsm.putTypedValue(key, value)
else:
fsm[key] = value
if not (fsmState or fsmEvent):
# just queue up a task to run the initial state transition using retries
fsm[STARTED_AT_PARAM] = time.time()
# initialize the fsm, which returns the 'pseudo-init' event
fsmEvent = fsm.initialize()
else:
# add the retry counter into the machine context from the header
obj[RETRY_COUNT_PARAM] = retryCount
# add the actual task name to the context
obj[TASK_NAME_PARAM] = taskName
# dispatch and return the next event
fsmEvent = fsm.dispatch(fsmEvent, obj)
# loop and execute until there are no more events - any exceptions
# will make it out to the user in the response - useful for debugging
if immediateMode:
while fsmEvent:
fsmEvent = fsm.dispatch(fsmEvent, obj)
self.response.headers['Content-Type'] = 'application/json'
data = {
'obj' : obj,
'context': fsm,
}
self.response.out.write(json.dumps(data, cls=Encoder))
