def assemble(self, build_func): """Build the canvas out of the task list.""" if self.model is None: raise RuntimeError('No database flow object found.') if self.model.tasks: raise RuntimeError( 'This flow instance was already assembled, use create' 'to create a new instance and restart the flow.') build_func(self) previous = [] for task in self._tasks: if isinstance(task, tuple): signature = self._new_task(*task, previous) self._canvas.append(signature) previous = [signature.id] elif isinstance(task, list): sub_canvas = [self._new_task(*t, previous) for t in task] previous = [t.id for t in sub_canvas] self._canvas.append(celery_group(sub_canvas, task_id=uuid())) else: raise RuntimeError('Error while parsing the task list %s', self._tasks) self._canvas = celery_chain(*self._canvas, task_id=self.id) return self
def _exec_callbacks(callback): """ Exec the callback or list of callbacks. Return asyncronous results as the TaskSetResult object. """ async_result = None if callback: if not isinstance(callback, (list, tuple)): # not iterable callback = [callback,] taskset = celery_group(*callback) async_result = taskset.apply_async() return async_result
def _exec_callbacks(callback): """ Exec the callback or list of callbacks. Return asyncronous results as the TaskSetResult object. """ async_result = None if callback: if not isinstance(callback, (list, tuple)): # not iterable callback = [ callback, ] taskset = celery_group(*callback) async_result = taskset.apply_async() return async_result
def apply_async(self): tasks = [] for node in self.children: func = node.func args = node.args kwargs = node.kwargs callback = kwargs.pop('callback', []) if not isinstance(callback, (list, tuple)): callback = [callback] subtasks = node._get_child_tasks() callback += subtasks kwargs = dict(callback=callback, **kwargs) _task = func.subtask(args=args, kwargs=kwargs) tasks.append(_task) taskset = celery_group(*tasks) result = taskset.apply_async() return result
def celery_worker( file_list, processing_func=None, limit=None, chunks=2500, most_common=10): # TODO: spawn it using process just for shortcut print( "Please start Celery server by running " "'celery -A words.tasks worker' and provide any args as you want") # NOTE: for redis use "config set stop-writes-on-bgsave-error" no if # fails to save on disk # prepare args file_list = chunked_generator(file_list, chunks, limit=limit) results = celery_group( processing_func.subtask((file_list, )) for file_list in file_list) results = results.apply_async().iterate() return process_results(results, most_common)
def load_data(flight_id): """ Cache the data, and then start to load the images. :param flight_id: :return: """ with connect_db() as db: path = flight_data.load_flight(db, flight_id) if path is None: return far_tiler = tile_geometry.Tiler(512) tiler = tile_geometry.Tiler(512, radius=1) night_tiler = tile_geometry.Tiler( zoom=4096, radius=1.5, params= 'LAYERS=ddl.simS3seriesNighttimeLightsGlob.brightness&STYLES=boxfill%2Fgreyscale' ) points = tiler.generate_points(path) far_points = far_tiler.generate_points(path) - points points = tiler.zoom_by(8, points) night_points = night_tiler.generate_points(path) grouped_points = tile_geometry.group_points(points) far_grouped_points = tile_geometry.group_points(far_points) night_grouped_points = tile_geometry.group_points(night_points) count = len(grouped_points) + len(night_grouped_points) + len( far_grouped_points) db.execute('UPDATE flightIDs SET tiles=? WHERE id=?', [count, flight_id]) db.commit() celery_group( load_group.s(flight_id, group, tiler.serialize(), 'day') for group in grouped_points)() celery_group( load_group.s(flight_id, group, far_tiler.serialize(), 'day') for group in far_grouped_points)() celery_group( load_group.s(flight_id, group, night_tiler.serialize(), 'night') for group in night_grouped_points)()