def _queueDispatchNormal(self, nextEvent, queue=True, countdown=0, retryOptions=None, queueName=None): """ Queues a call to .dispatch(nextEvent) in the appengine Task queue. @param nextEvent: a string event @param queue: a boolean indicating whether or not to queue a Task, or leave it to the caller @param countdown: the number of seconds to countdown before the queued task fires @param retryOptions: the RetryOptions for the task @param queueName: the queue name to Queue into @return: a taskqueue.Task instance which may or may not have been queued already """ assert nextEvent is not None assert queueName url = self.buildUrl(self.currentState, nextEvent) params = self.buildParams(self.currentState, nextEvent) taskName = self.getTaskName(nextEvent) task = Task(name=taskName, method=self.method, url=url, params=params, countdown=countdown, retry_options=retryOptions, headers=self.headers) if queue: self.Queue(name=queueName).add(task) return task
def generateInitializationTask(self, countdown=0, taskName=None): """ Generates a task for initializing the machine. """ assert self.currentState.name == FSM.PSEUDO_INIT url = self.buildUrl(self.currentState, FSM.PSEUDO_INIT) params = self.buildParams(self.currentState, FSM.PSEUDO_INIT) taskName = taskName or self.getTaskName(FSM.PSEUDO_INIT) transition = self.currentState.getTransition(FSM.PSEUDO_INIT) task = Task(name=taskName, method=self.method, url=url, params=params, countdown=countdown, headers=self.headers, retry_options=transition.retryOptions) return task
def runFixImageCaches(request, offset): offset = int(offset) limit = 50 query = ImageCache.all(keys_only=True).fetch(limit + 1, offset) results = list(query) if len(results) > limit: results.pop() continue_at = offset + limit else: continue_at = None i = 0 for result in results: i += 1 countdown = max(0, offset + (i / 2)) task = Task(url="/dreamer/admin/fix-image/%s" % result.id_or_name(), method="POST", countdown=countdown) task.add('default') if continue_at: queueFixImageCaches(continue_at, countdown=max(10, limit/2)) return HttpResponse('OK')
def _queueDispatchFanIn(self, nextEvent, fanInPeriod=0, retryOptions=None, queueName=None): """ Queues a call to .dispatch(nextEvent) in the task queue, or saves the context to the datastore for processing by the queued .dispatch(nextEvent) @param nextEvent: a string event @param fanInPeriod: the period of time between fan in Tasks @param queueName: the queue name to Queue into @return: a taskqueue.Task instance which may or may not have been queued already """ assert nextEvent is not None assert not self.get( constants.INDEX_PARAM) # fan-in after fan-in is not allowed assert queueName # we pop this off here because we do not want the fan-out/continuation param as part of the # task name, otherwise we loose the fan-in - each fan-in gets one work unit. self.pop(constants.GEN_PARAM, None) fork = self.pop(constants.FORK_PARAM, None) taskNameBase = self.getTaskName(nextEvent, fanIn=True) rwlock = ReadWriteLock(taskNameBase, self) index = rwlock.currentIndex() # (***) # # grab the lock - memcache.incr() # # on Task retry, multiple incr() calls are possible. possible ways to handle: # # 1. release the lock in a 'finally' clause, but then risk missing a work # package because acquiring the read lock will succeed even though the # work package was not written yet. # # 2. allow the lock to get too high. the fan-in logic attempts to wait for # work packages across multiple-retry attempts, so this seems like the # best option. we basically trade a bit of latency in fan-in for reliability. # rwlock.acquireWriteLock(index, nextEvent=nextEvent) # insert the work package, which is simply a serialized FSMContext workIndex = '%s-%d' % (taskNameBase, knuthHash(index)) # on retry, we want to ensure we get the same work index for this task actualTaskName = self.__obj[constants.TASK_NAME_PARAM] indexKeyName = 'workIndex-' + '-'.join( [str(i) for i in [actualTaskName, fork] if i]) or None semaphore = RunOnceSemaphore(indexKeyName, self) # check if the workIndex changed during retry semaphoreWritten = False if self.__obj[constants.RETRY_COUNT_PARAM] > 0: # see comment (A) in self._queueDispatchFanIn(...) time.sleep(constants.DATASTORE_ASYNCRONOUS_INDEX_WRITE_WAIT_TIME) payload = semaphore.readRunOnceSemaphore(payload=workIndex, transactional=False) if payload: semaphoreWritten = True if payload != workIndex: self.logger.info( "Work index changed from '%s' to '%s' on retry.", payload, workIndex) workIndex = payload # write down two models, one actual work package, one idempotency package keyName = '-'.join([str(i) for i in [actualTaskName, fork] if i]) or None work = _FantasmFanIn(context=self, workIndex=workIndex, key_name=keyName) # close enough to idempotent, but could still write only one of the entities # FIXME: could be made faster using a bulk put, but this interface is cleaner if not semaphoreWritten: semaphore.writeRunOnceSemaphore(payload=workIndex, transactional=False) # put the work item db.put(work) # (A) now the datastore is asynchronously writing the indices, so the work package may # not show up in a query for a period of time. there is a corresponding time.sleep() # in the fan-in of self.mergeJoinDispatch(...) # release the lock - memcache.decr() rwlock.releaseWriteLock(index) try: # insert a task to run in the future and process a bunch of work packages now = time.time() self[constants.INDEX_PARAM] = index url = self.buildUrl(self.currentState, nextEvent) params = self.buildParams(self.currentState, nextEvent) task = Task(name='%s-%d' % (taskNameBase, index), method=self.method, url=url, params=params, eta=datetime.datetime.utcfromtimestamp(now) + datetime.timedelta(seconds=fanInPeriod), headers=self.headers, retry_options=retryOptions) self.Queue(name=queueName).add(task) return task except (TaskAlreadyExistsError, TombstonedTaskError): pass # Fan-in magic
def queueFixImageCaches(offset, countdown=0): task = Task(url="/dreamer/admin/run-fix-image-caches/%d" % offset, method="POST", countdown=countdown) task.add('default')
def dispatch(self, context, event, obj): """ Fires the transition and executes the next States's entry, do and exit actions. @param context: an FSMContext instance @param event: a string event to dispatch to the State @param obj: an object that the Transition can operate on @return: the event returned from the next state's main action. """ transition = self.getTransition(event) if context.currentState.exitAction: try: context.currentAction = context.currentState.exitAction context.currentState.exitAction.execute(context, obj) except Exception: context.logger.error( 'Error processing entry action for state. (Machine %s, State %s, exitAction %s)', context.machineName, context.currentState.name, context.currentState.exitAction.__class__) raise # join the contexts of a fan-in contextOrContexts = context if transition.target.isFanIn: taskNameBase = context.getTaskName(event, fanIn=True) contextOrContexts = context.mergeJoinDispatch(event, obj) if not contextOrContexts: context.logger.info( 'Fan-in resulted in 0 contexts. Terminating machine. (Machine %s, State %s)', context.machineName, context.currentState.name) obj[constants.TERMINATED_PARAM] = True transition.execute(context, obj) if context.currentState.entryAction: try: context.currentAction = context.currentState.entryAction context.currentState.entryAction.execute( contextOrContexts, obj) except Exception: context.logger.error( 'Error processing entry action for state. (Machine %s, State %s, entryAction %s)', context.machineName, context.currentState.name, context.currentState.entryAction.__class__) raise if context.currentState.isContinuation: try: token = context.get(constants.CONTINUATION_PARAM, None) nextToken = context.currentState.doAction.continuation( contextOrContexts, obj, token=token) if nextToken: context.continuation(nextToken) context.pop(constants.CONTINUATION_PARAM, None) # pop this off because it is really long except Exception: context.logger.error( 'Error processing continuation for state. (Machine %s, State %s, continuation %s)', context.machineName, context.currentState.name, context.currentState.doAction.__class__) raise # either a fan-in resulted in no contexts, or a continuation was completed if obj.get(constants.TERMINATED_PARAM): return None nextEvent = None if context.currentState.doAction: try: context.currentAction = context.currentState.doAction nextEvent = context.currentState.doAction.execute( contextOrContexts, obj) except Exception: context.logger.error( 'Error processing action for state. (Machine %s, State %s, Action %s)', context.machineName, context.currentState.name, context.currentState.doAction.__class__) raise if transition.target.isFanIn: # this prevents fan-in from re-counting the data if there is an Exception # or DeadlineExceeded _after_ doAction.execute(...) succeeds index = context.get(constants.INDEX_PARAM) workIndex = '%s-%d' % (taskNameBase, knuthHash(index)) semaphore = RunOnceSemaphore(workIndex, context) semaphore.writeRunOnceSemaphore( payload=obj[constants.TASK_NAME_PARAM]) try: # at this point we have processed the work items, delete them task = Task(name=obj[constants.TASK_NAME_PARAM] + '-cleanup', url=constants.DEFAULT_CLEANUP_URL, params={constants.WORK_INDEX_PARAM: workIndex}) context.Queue( name=constants.DEFAULT_CLEANUP_QUEUE_NAME).add(task) except (TaskAlreadyExistsError, TombstonedTaskError): context.logger.info("Fan-in cleanup Task already exists.") if context.get('UNITTEST_RAISE_AFTER_FAN_IN' ): # only way to generate this failure raise Exception() if nextEvent: if not isinstance(nextEvent, str) or not constants.NAME_RE.match(nextEvent): raise InvalidEventNameRuntimeError(nextEvent, context.machineName, context.currentState.name, context.instanceName) return nextEvent
def any(self): if not self.visitorisnew: pubfut=None try: visitorid = self.visitorkey.id() logging.debug("Queuing refresh of "+'/publish/user/'+str(visitorid)) task=Task(url='/publish/user/'+str(visitorid)) pubfut=task.add_async('new-visitor-queue') except Exception as e: logging.exception(e) logging.debug("add partials") self.addpartials(DEFAULTPARTIALS) logging.debug("added partials") clients = getClients(async = True); idtotopic = getAllTopics(async = True); trendingstatements = getTrendingStatementsNow(async = True) logging.debug("added clients, idtotopic, trending") if self.visitorisnew: visitor = generateVisitorData(None, isnew = True); else: visitor = generateVisitorData(self.getVisitor()); self.addjson("visitor",visitor) logging.debug("added visitor") idtoaxis = getAllAxis(async = True); featuredtopic = getFeaturedTopic(async = True) navtreejson = generateNavTree(async = True) homestreamconfig = getHomeStreamConfig(async = True) topicarticlehistory = getTopicArticleHistory(None, async = True) randomstatements = randomStatements(async = True) everyonestatements = everyoneElseStatements(async = True) logging.debug("add clients") self.addjson("clients",clients.get_result()) logging.debug("add trendingstatements") self.addjson("trendingstatements",trendingstatements.get_result()) logging.debug("add idtotopic") self.addjson("idtotopics",idtotopic.get_result()) logging.debug("add idtoaxis") self.addjson("idtoaxis",idtoaxis.get_result()) logging.debug("add featuredtopicid") self.addjson("featuredtopicid",featuredtopic.get_result()) logging.debug("add navtree") self.addjson("navtree",navtreejson.get_result()) logging.debug("add topicarticlehistory") self.addjson("latestarticles",topicarticlehistory.get_result()) logging.debug("add everyonestatements") self.addjson("everyonetrendingstatements",everyonestatements.get_result()) logging.debug("add randomstatements") self.addjson("randomstatements",randomstatements.get_result()) logging.debug("add homestreamconfig") self.addjson("homestreamconfig",homestreamconfig.get_result()) logging.debug("add template") self.settemplate('index.html') if not self.visitorisnew: if pubfut is not None: try: pubfut.get_result() except Exception as e: logging.exception(e) super(MainHandler, self).any()
if transition.target.isFanIn: # this prevents fan-in from re-counting the data if there is an Exception # or DeadlineExceeded _after_ doAction.execute(...) succeeds index = context.get( constants.INDEX_PARAM) or contextOrContexts[0].get( constants.INDEX_PARAM) workIndex = '%s-%d' % (taskNameBase, knuthHash(index)) semaphore = RunOnceSemaphore(workIndex, context) semaphore.writeRunOnceSemaphore( payload=obj[constants.TASK_NAME_PARAM]) try: # at this point we have processed the work items, delete them task = Task(name=obj[constants.TASK_NAME_PARAM] + '-cleanup', url=constants.DEFAULT_CLEANUP_URL, params={constants.WORK_INDEX_PARAM: workIndex}) context.Queue( name=constants.DEFAULT_CLEANUP_QUEUE_NAME).add(task) except (TaskAlreadyExistsError, TombstonedTaskError): context.logger.info("Fan-in cleanup Task already exists.") if context.get('UNITTEST_RAISE_AFTER_FAN_IN' ): # only way to generate this failure if not contextOrContexts.guarded: raise Exception() if nextEvent: if not isinstance(nextEvent, str) or not constants.NAME_RE.match(nextEvent):