def assemble(self, build_func):
"""Build the canvas out of the task list."""
if self.model is None:
raise RuntimeError('No database flow object found.')
if self.model.tasks:
raise RuntimeError(
'This flow instance was already assembled, use create'
'to create a new instance and restart the flow.')
build_func(self)
previous = []
for task in self._tasks:
if isinstance(task, tuple):
signature = self._new_task(*task, previous)
self._canvas.append(signature)
previous = [signature.id]
elif isinstance(task, list):
sub_canvas = [self._new_task(*t, previous) for t in task]
previous = [t.id for t in sub_canvas]
self._canvas.append(celery_group(sub_canvas, task_id=uuid()))
else:
raise RuntimeError('Error while parsing the task list %s',
self._tasks)
self._canvas = celery_chain(*self._canvas, task_id=self.id)
return self
def _exec_callbacks(callback):
""" Exec the callback or list of callbacks. Return asyncronous results as
the TaskSetResult object.
"""
async_result = None
if callback:
if not isinstance(callback, (list, tuple)): # not iterable
callback = [callback,]
taskset = celery_group(*callback)
async_result = taskset.apply_async()
return async_result
def _exec_callbacks(callback):
""" Exec the callback or list of callbacks. Return asyncronous results as
the TaskSetResult object.
"""
async_result = None
if callback:
if not isinstance(callback, (list, tuple)): # not iterable
callback = [
callback,
]
taskset = celery_group(*callback)
async_result = taskset.apply_async()
return async_result
def apply_async(self):
tasks = []
for node in self.children:
func = node.func
args = node.args
kwargs = node.kwargs
callback = kwargs.pop('callback', [])
if not isinstance(callback, (list, tuple)):
callback = [callback]
subtasks = node._get_child_tasks()
callback += subtasks
kwargs = dict(callback=callback, **kwargs)
_task = func.subtask(args=args, kwargs=kwargs)
tasks.append(_task)
taskset = celery_group(*tasks)
result = taskset.apply_async()
return result
def apply_async(self):
tasks = []
for node in self.children:
func = node.func
args = node.args
kwargs = node.kwargs
callback = kwargs.pop('callback', [])
if not isinstance(callback, (list, tuple)):
callback = [callback]
subtasks = node._get_child_tasks()
callback += subtasks
kwargs = dict(callback=callback, **kwargs)
_task = func.subtask(args=args, kwargs=kwargs)
tasks.append(_task)
taskset = celery_group(*tasks)
result = taskset.apply_async()
return result
def celery_worker(
file_list, processing_func=None, limit=None, chunks=2500,
most_common=10):
# TODO: spawn it using process just for shortcut
print(
"Please start Celery server by running "
"'celery -A words.tasks worker' and provide any args as you want")
# NOTE: for redis use "config set stop-writes-on-bgsave-error" no if
# fails to save on disk
# prepare args
file_list = chunked_generator(file_list, chunks, limit=limit)
results = celery_group(
processing_func.subtask((file_list, )) for file_list in file_list)
results = results.apply_async().iterate()
return process_results(results, most_common)
def load_data(flight_id):
"""
Cache the data, and then start to load the images.
:param flight_id:
:return:
"""
with connect_db() as db:
path = flight_data.load_flight(db, flight_id)
if path is None:
return
far_tiler = tile_geometry.Tiler(512)
tiler = tile_geometry.Tiler(512, radius=1)
night_tiler = tile_geometry.Tiler(
zoom=4096,
radius=1.5,
params=
'LAYERS=ddl.simS3seriesNighttimeLightsGlob.brightness&STYLES=boxfill%2Fgreyscale'
)
points = tiler.generate_points(path)
far_points = far_tiler.generate_points(path) - points
points = tiler.zoom_by(8, points)
night_points = night_tiler.generate_points(path)
grouped_points = tile_geometry.group_points(points)
far_grouped_points = tile_geometry.group_points(far_points)
night_grouped_points = tile_geometry.group_points(night_points)
count = len(grouped_points) + len(night_grouped_points) + len(
far_grouped_points)
db.execute('UPDATE flightIDs SET tiles=? WHERE id=?',
[count, flight_id])
db.commit()
celery_group(
load_group.s(flight_id, group, tiler.serialize(), 'day')
for group in grouped_points)()
celery_group(
load_group.s(flight_id, group, far_tiler.serialize(), 'day')
for group in far_grouped_points)()
celery_group(
load_group.s(flight_id, group, night_tiler.serialize(), 'night')
for group in night_grouped_points)()
