def _queueDispatchNormal(self,
nextEvent,
queue=True,
countdown=0,
retryOptions=None,
queueName=None):
""" Queues a call to .dispatch(nextEvent) in the appengine Task queue.
@param nextEvent: a string event
@param queue: a boolean indicating whether or not to queue a Task, or leave it to the caller
@param countdown: the number of seconds to countdown before the queued task fires
@param retryOptions: the RetryOptions for the task
@param queueName: the queue name to Queue into
@return: a taskqueue.Task instance which may or may not have been queued already
"""
assert nextEvent is not None
assert queueName
url = self.buildUrl(self.currentState, nextEvent)
params = self.buildParams(self.currentState, nextEvent)
taskName = self.getTaskName(nextEvent)
task = Task(name=taskName,
method=self.method,
url=url,
params=params,
countdown=countdown,
retry_options=retryOptions,
headers=self.headers)
if queue:
self.Queue(name=queueName).add(task)
return task
def runFixImageCaches(request, offset):
offset = int(offset)
limit = 50
query = ImageCache.all(keys_only=True).fetch(limit + 1, offset)
results = list(query)
if len(results) > limit:
results.pop()
continue_at = offset + limit
else:
continue_at = None
i = 0
for result in results:
i += 1
countdown = max(0, offset + (i / 2))
task = Task(url="/dreamer/admin/fix-image/%s" % result.id_or_name(), method="POST", countdown=countdown)
task.add('default')
if continue_at:
queueFixImageCaches(continue_at, countdown=max(10, limit/2))
return HttpResponse('OK')
def generateInitializationTask(self, countdown=0, taskName=None):
""" Generates a task for initializing the machine. """
assert self.currentState.name == FSM.PSEUDO_INIT
url = self.buildUrl(self.currentState, FSM.PSEUDO_INIT)
params = self.buildParams(self.currentState, FSM.PSEUDO_INIT)
taskName = taskName or self.getTaskName(FSM.PSEUDO_INIT)
transition = self.currentState.getTransition(FSM.PSEUDO_INIT)
task = Task(name=taskName,
method=self.method,
url=url,
params=params,
countdown=countdown,
headers=self.headers,
retry_options=transition.retryOptions)
return task
def _queueDispatchFanIn(self,
nextEvent,
fanInPeriod=0,
retryOptions=None,
queueName=None):
""" Queues a call to .dispatch(nextEvent) in the task queue, or saves the context to the
datastore for processing by the queued .dispatch(nextEvent)
@param nextEvent: a string event
@param fanInPeriod: the period of time between fan in Tasks
@param queueName: the queue name to Queue into
@return: a taskqueue.Task instance which may or may not have been queued already
"""
assert nextEvent is not None
assert not self.get(
constants.INDEX_PARAM) # fan-in after fan-in is not allowed
assert queueName
# we pop this off here because we do not want the fan-out/continuation param as part of the
# task name, otherwise we loose the fan-in - each fan-in gets one work unit.
self.pop(constants.GEN_PARAM, None)
fork = self.pop(constants.FORK_PARAM, None)
taskNameBase = self.getTaskName(nextEvent, fanIn=True)
rwlock = ReadWriteLock(taskNameBase, self)
index = rwlock.currentIndex()
# (***)
#
# grab the lock - memcache.incr()
#
# on Task retry, multiple incr() calls are possible. possible ways to handle:
#
# 1. release the lock in a 'finally' clause, but then risk missing a work
# package because acquiring the read lock will succeed even though the
# work package was not written yet.
#
# 2. allow the lock to get too high. the fan-in logic attempts to wait for
# work packages across multiple-retry attempts, so this seems like the
# best option. we basically trade a bit of latency in fan-in for reliability.
#
rwlock.acquireWriteLock(index, nextEvent=nextEvent)
# insert the work package, which is simply a serialized FSMContext
workIndex = '%s-%d' % (taskNameBase, knuthHash(index))
# on retry, we want to ensure we get the same work index for this task
actualTaskName = self.__obj[constants.TASK_NAME_PARAM]
indexKeyName = 'workIndex-' + '-'.join(
[str(i) for i in [actualTaskName, fork] if i]) or None
semaphore = RunOnceSemaphore(indexKeyName, self)
# check if the workIndex changed during retry
semaphoreWritten = False
if self.__obj[constants.RETRY_COUNT_PARAM] > 0:
# see comment (A) in self._queueDispatchFanIn(...)
time.sleep(constants.DATASTORE_ASYNCRONOUS_INDEX_WRITE_WAIT_TIME)
payload = semaphore.readRunOnceSemaphore(payload=workIndex,
transactional=False)
if payload:
semaphoreWritten = True
if payload != workIndex:
self.logger.info(
"Work index changed from '%s' to '%s' on retry.",
payload, workIndex)
workIndex = payload
# write down two models, one actual work package, one idempotency package
keyName = '-'.join([str(i)
for i in [actualTaskName, fork] if i]) or None
work = _FantasmFanIn(context=self,
workIndex=workIndex,
key_name=keyName)
# close enough to idempotent, but could still write only one of the entities
# FIXME: could be made faster using a bulk put, but this interface is cleaner
if not semaphoreWritten:
semaphore.writeRunOnceSemaphore(payload=workIndex,
transactional=False)
# put the work item
db.put(work)
# (A) now the datastore is asynchronously writing the indices, so the work package may
# not show up in a query for a period of time. there is a corresponding time.sleep()
# in the fan-in of self.mergeJoinDispatch(...)
# release the lock - memcache.decr()
rwlock.releaseWriteLock(index)
try:
# insert a task to run in the future and process a bunch of work packages
now = time.time()
self[constants.INDEX_PARAM] = index
url = self.buildUrl(self.currentState, nextEvent)
params = self.buildParams(self.currentState, nextEvent)
task = Task(name='%s-%d' % (taskNameBase, index),
method=self.method,
url=url,
params=params,
eta=datetime.datetime.utcfromtimestamp(now) +
datetime.timedelta(seconds=fanInPeriod),
headers=self.headers,
retry_options=retryOptions)
self.Queue(name=queueName).add(task)
return task
except (TaskAlreadyExistsError, TombstonedTaskError):
pass # Fan-in magic
def queueFixImageCaches(offset, countdown=0):
task = Task(url="/dreamer/admin/run-fix-image-caches/%d" % offset, method="POST", countdown=countdown)
task.add('default')
def dispatch(self, context, event, obj):
""" Fires the transition and executes the next States's entry, do and exit actions.
@param context: an FSMContext instance
@param event: a string event to dispatch to the State
@param obj: an object that the Transition can operate on
@return: the event returned from the next state's main action.
"""
transition = self.getTransition(event)
if context.currentState.exitAction:
try:
context.currentAction = context.currentState.exitAction
context.currentState.exitAction.execute(context, obj)
except Exception:
context.logger.error(
'Error processing entry action for state. (Machine %s, State %s, exitAction %s)',
context.machineName, context.currentState.name,
context.currentState.exitAction.__class__)
raise
# join the contexts of a fan-in
contextOrContexts = context
if transition.target.isFanIn:
taskNameBase = context.getTaskName(event, fanIn=True)
contextOrContexts = context.mergeJoinDispatch(event, obj)
if not contextOrContexts:
context.logger.info(
'Fan-in resulted in 0 contexts. Terminating machine. (Machine %s, State %s)',
context.machineName, context.currentState.name)
obj[constants.TERMINATED_PARAM] = True
transition.execute(context, obj)
if context.currentState.entryAction:
try:
context.currentAction = context.currentState.entryAction
context.currentState.entryAction.execute(
contextOrContexts, obj)
except Exception:
context.logger.error(
'Error processing entry action for state. (Machine %s, State %s, entryAction %s)',
context.machineName, context.currentState.name,
context.currentState.entryAction.__class__)
raise
if context.currentState.isContinuation:
try:
token = context.get(constants.CONTINUATION_PARAM, None)
nextToken = context.currentState.doAction.continuation(
contextOrContexts, obj, token=token)
if nextToken:
context.continuation(nextToken)
context.pop(constants.CONTINUATION_PARAM,
None) # pop this off because it is really long
except Exception:
context.logger.error(
'Error processing continuation for state. (Machine %s, State %s, continuation %s)',
context.machineName, context.currentState.name,
context.currentState.doAction.__class__)
raise
# either a fan-in resulted in no contexts, or a continuation was completed
if obj.get(constants.TERMINATED_PARAM):
return None
nextEvent = None
if context.currentState.doAction:
try:
context.currentAction = context.currentState.doAction
nextEvent = context.currentState.doAction.execute(
contextOrContexts, obj)
except Exception:
context.logger.error(
'Error processing action for state. (Machine %s, State %s, Action %s)',
context.machineName, context.currentState.name,
context.currentState.doAction.__class__)
raise
if transition.target.isFanIn:
# this prevents fan-in from re-counting the data if there is an Exception
# or DeadlineExceeded _after_ doAction.execute(...) succeeds
index = context.get(constants.INDEX_PARAM)
workIndex = '%s-%d' % (taskNameBase, knuthHash(index))
semaphore = RunOnceSemaphore(workIndex, context)
semaphore.writeRunOnceSemaphore(
payload=obj[constants.TASK_NAME_PARAM])
try:
# at this point we have processed the work items, delete them
task = Task(name=obj[constants.TASK_NAME_PARAM] + '-cleanup',
url=constants.DEFAULT_CLEANUP_URL,
params={constants.WORK_INDEX_PARAM: workIndex})
context.Queue(
name=constants.DEFAULT_CLEANUP_QUEUE_NAME).add(task)
except (TaskAlreadyExistsError, TombstonedTaskError):
context.logger.info("Fan-in cleanup Task already exists.")
if context.get('UNITTEST_RAISE_AFTER_FAN_IN'
): # only way to generate this failure
raise Exception()
if nextEvent:
if not isinstance(nextEvent,
str) or not constants.NAME_RE.match(nextEvent):
raise InvalidEventNameRuntimeError(nextEvent,
context.machineName,
context.currentState.name,
context.instanceName)
return nextEvent
def any(self):
if not self.visitorisnew:
pubfut=None
try:
visitorid = self.visitorkey.id()
logging.debug("Queuing refresh of "+'/publish/user/'+str(visitorid))
task=Task(url='/publish/user/'+str(visitorid))
pubfut=task.add_async('new-visitor-queue')
except Exception as e:
logging.exception(e)
logging.debug("add partials")
self.addpartials(DEFAULTPARTIALS)
logging.debug("added partials")
clients = getClients(async = True);
idtotopic = getAllTopics(async = True);
trendingstatements = getTrendingStatementsNow(async = True)
logging.debug("added clients, idtotopic, trending")
if self.visitorisnew:
visitor = generateVisitorData(None, isnew = True);
else:
visitor = generateVisitorData(self.getVisitor());
self.addjson("visitor",visitor)
logging.debug("added visitor")
idtoaxis = getAllAxis(async = True);
featuredtopic = getFeaturedTopic(async = True)
navtreejson = generateNavTree(async = True)
homestreamconfig = getHomeStreamConfig(async = True)
topicarticlehistory = getTopicArticleHistory(None, async = True)
randomstatements = randomStatements(async = True)
everyonestatements = everyoneElseStatements(async = True)
logging.debug("add clients")
self.addjson("clients",clients.get_result())
logging.debug("add trendingstatements")
self.addjson("trendingstatements",trendingstatements.get_result())
logging.debug("add idtotopic")
self.addjson("idtotopics",idtotopic.get_result())
logging.debug("add idtoaxis")
self.addjson("idtoaxis",idtoaxis.get_result())
logging.debug("add featuredtopicid")
self.addjson("featuredtopicid",featuredtopic.get_result())
logging.debug("add navtree")
self.addjson("navtree",navtreejson.get_result())
logging.debug("add topicarticlehistory")
self.addjson("latestarticles",topicarticlehistory.get_result())
logging.debug("add everyonestatements")
self.addjson("everyonetrendingstatements",everyonestatements.get_result())
logging.debug("add randomstatements")
self.addjson("randomstatements",randomstatements.get_result())
logging.debug("add homestreamconfig")
self.addjson("homestreamconfig",homestreamconfig.get_result())
logging.debug("add template")
self.settemplate('index.html')
if not self.visitorisnew:
if pubfut is not None:
try:
pubfut.get_result()
except Exception as e:
logging.exception(e)
super(MainHandler, self).any()
if transition.target.isFanIn:
# this prevents fan-in from re-counting the data if there is an Exception
# or DeadlineExceeded _after_ doAction.execute(...) succeeds
index = context.get(
constants.INDEX_PARAM) or contextOrContexts[0].get(
constants.INDEX_PARAM)
workIndex = '%s-%d' % (taskNameBase, knuthHash(index))
semaphore = RunOnceSemaphore(workIndex, context)
semaphore.writeRunOnceSemaphore(
payload=obj[constants.TASK_NAME_PARAM])
try:
# at this point we have processed the work items, delete them
task = Task(name=obj[constants.TASK_NAME_PARAM] + '-cleanup',
url=constants.DEFAULT_CLEANUP_URL,
params={constants.WORK_INDEX_PARAM: workIndex})
context.Queue(
name=constants.DEFAULT_CLEANUP_QUEUE_NAME).add(task)
except (TaskAlreadyExistsError, TombstonedTaskError):
context.logger.info("Fan-in cleanup Task already exists.")
if context.get('UNITTEST_RAISE_AFTER_FAN_IN'
): # only way to generate this failure
if not contextOrContexts.guarded:
raise Exception()
if nextEvent:
if not isinstance(nextEvent,
str) or not constants.NAME_RE.match(nextEvent):
