def run_processors(self):
num_workers = cpu_count()
threads = []
print('Starting {} thumbnail processor workers\n'.format(num_workers))
for i in range(num_workers):
t = threading.Thread(target=worker)
t.start()
threads.append(t)
try:
while True:
requeue_stuck_tasks('generate_thumbnails')
num_remaining = Task.objects.filter(type='generate_thumbnails', status='P').count()
if num_remaining:
print('{} tasks remaining for thumbnail processing'.format(num_remaining))
# Load 'Pending' tasks onto worker threads
for task in Task.objects.filter(type='generate_thumbnails', status='P')[:64]:
q.put(task)
print('Finished thumbnail processing batch')
# Wait until all threads have finished
q.join()
sleep(1)
except KeyboardInterrupt:
# Shut down threads cleanly
for i in range(num_workers):
q.put(None)
for t in threads:
t.join()
def run(self, loop=True):
print('Starting {} {} workers\n'.format(self.num_workers, self.task_type))
if self.num_workers > 1:
for i in range(self.num_workers):
t = threading.Thread(target=self.__worker)
t.start()
self.threads.append(t)
try:
while True:
requeue_stuck_tasks(self.task_type)
for task in Task.objects.filter(type=self.task_type, status='P')[:64]:
if self.num_workers > 1:
print('putting task')
self.queue.put(task)
else:
self.__process_task(task)
if self.num_workers > 1:
self.queue.join()
if not loop:
self.__clean_up()
return
sleep(1)
except KeyboardInterrupt:
self.__clean_up()
def run(self, loop=True):
logger.info('Starting {} {} workers'.format(self.num_workers,
self.task_type))
if self.num_workers > 1:
for i in range(self.num_workers):
t = threading.Thread(target=self.__worker)
t.start()
self.threads.append(t)
try:
while True:
requeue_stuck_tasks(self.task_type)
if self.task_type == 'classify.color':
task_queryset = Task.objects.filter(
library__classification_color_enabled=True,
type=self.task_type,
status='P')
elif self.task_type == 'classify.location':
task_queryset = Task.objects.filter(
library__classification_location_enabled=True,
type=self.task_type,
status='P')
elif self.task_type == 'classify.face':
task_queryset = Task.objects.filter(
library__classification_face_enabled=True,
type=self.task_type,
status='P')
elif self.task_type == 'classify.style':
task_queryset = Task.objects.filter(
library__classification_style_enabled=True,
type=self.task_type,
status='P')
elif self.task_type == 'classify.object':
task_queryset = Task.objects.filter(
library__classification_object_enabled=True,
type=self.task_type,
status='P')
else:
task_queryset = Task.objects.filter(type=self.task_type,
status='P')
for task in task_queryset[:8]:
if self.num_workers > 1:
logger.debug('putting task')
self.queue.put(task)
else:
self.__process_task(task)
if self.num_workers > 1:
self.queue.join()
if not loop:
self.__clean_up()
return
sleep(1)
except KeyboardInterrupt:
self.__clean_up()
def run_processors(self):
num_workers = max(int(cpu_count() / 4), 1)
threads = []
logger.info(f'Starting {num_workers} raw processor workers')
for i in range(num_workers):
t = threading.Thread(target=worker)
t.start()
threads.append(t)
try:
while True:
requeue_stuck_tasks('process_raw')
num_remaining = Task.objects.filter(type='process_raw',
status='P').count()
if num_remaining:
logger.info(
f'{num_remaining} tasks remaining for raw processing')
# Load 'Pending' tasks onto worker threads
for task in Task.objects.filter(type='process_raw',
status='P')[:64]:
q.put(task)
logger.info('Finished raw processing batch')
# Wait until all threads have finished
q.join()
sleep(1)
except KeyboardInterrupt:
# Shut down threads cleanly
for i in range(num_workers):
q.put(None)
for t in threads:
t.join()