def revoke(self):
'''Revokes this task and returns its result'''
if self.task_id:
result = huey.result(self.task_id, preserve=True)
huey.revoke_by_id(self.task_id)
return result
def handle(self, *args, **options):
# clear any tasks in Huey DB, so they don't pile up over time,
# especially if you run the server without the timeoutworker to consume
# the tasks.
# this code is also in asgi.py. it should be in both places,
# to ensure the database is flushed in all circumstances.
from huey.contrib.djhuey import HUEY
HUEY.flush()
super(Command, self).handle(*args, **options)
def handle(self, *args, **options):
# clear any tasks in Huey DB, so they don't pile up over time,
# especially if you run the server without the timeoutworker to consume
# the tasks.
# this code is also in asgi.py. it should be in both places,
# to ensure the database is flushed in all circumstances.
from huey.contrib.djhuey import HUEY
HUEY.flush()
super(Command, self).handle(*args, **options)
def handle(self, *args, **options):
# clear any tasks in Huey DB, so they don't pile up over time,
# especially if you run the server without the timeoutworker to consume
# the tasks.
# this code is also in asgi.py. it should be in both places,
# to ensure the database is flushed in all circumstances.
from huey.contrib.djhuey import HUEY
HUEY.flush()
# need to set USE_REDIS = True, because it uses the test client
# to submit pages, and if the next page has a timeout as well,
# its timeout task should be queued.
import otree.common_internal
otree.common_internal.USE_REDIS = True
super().handle(*args, **options)
def add_monitor_entry(monitor_id, payload, sensor=None):
# CONSIDER: If we're skipping existing entries, we
# probably don't need this lock
key = '_'.join([
'purpleair.entry',
str(monitor_id),
sensor if sensor is not None else '',
str(payload[0]["created_at"]),
])
try:
with HUEY.lock_task(key):
monitor = PurpleAir.objects.get(pk=monitor_id)
entry = monitor.create_entry(payload, sensor=sensor)
# If the entry was pulled from the database and is not
# a new instance, we can end early since we already have it.
if not entry._state.adding:
return
print(f'[add_monitor_entry] {monitor.name} ({monitor.pk})')
entry = monitor.process_entry(entry)
entry.save()
monitor.check_latest(Entry.objects.get(pk=entry.pk))
except TaskLockedException:
print(
f'[LOCKED:add_monitor_entry] {monitor_id} ({payload[0]["created_at"]})'
)
def add_monitor_entry(monitor_id, payload, sensor=None):
key = '_'.join([
'purpleair.entry',
str(monitor_id),
sensor if sensor is not None else '',
str(payload[0]["created_at"]),
])
try:
with HUEY.lock_task(key):
monitor = PurpleAir.objects.get(pk=monitor_id)
print(f'[add_monitor_entry] {monitor.name} ({monitor.pk})')
entry = monitor.create_entry(payload, sensor=sensor)
entry = monitor.process_entry(entry)
entry.save()
is_latest = not monitor.latest or (
entry.timestamp > parse_datetime(monitor.latest['timestamp']))
sensor_match = monitor.DEFAULT_SENSOR is None or entry.sensor == monitor.DEFAULT_SENSOR
if sensor_match and is_latest:
monitor.set_latest(Entry.objects.get(pk=entry.pk))
monitor.save()
except TaskLockedException:
print(
f'[LOCKED:add_monitor_entry] {monitor_id} ({payload[0]["created_at"]})'
)
def load_tasks_async():
with HUEY.lock_task("import-lock"):
load_tasks(GitRepository(
settings.REPOSITORY_ROOT,
url=config.GITLOAD_URL,
branch=config.GITLOAD_BRANCH
))
def handle(self, *args, **options):
# clear any tasks in Huey DB, so they don't pile up over time,
# especially if you run the server without the timeoutworker to consume
# the tasks.
# this code is also in asgi.py. it should be in both places,
# to ensure the database is flushed in all circumstances.
from huey.contrib.djhuey import HUEY
if not os.environ.get('REDIS_URL'):
sys.exit('REDIS_URL env var must be defined')
try:
HUEY.flush()
except redis.exceptions.ConnectionError as exc:
sys.exit(f'Could not connect to Redis: {exc}')
super().handle(*args, **options)
def import_recent_data():
print('[import_recent_data]')
if HUEY.pending_count() > settings.MAX_QUEUE_SIZE:
return
for monitor in PurpleAir.objects.all():
print('\n' * 10, '[import_recent_data]', monitor.name, '\n' * 10)
import_monitor_data.schedule([monitor.pk], delay=1, priority=30)
def spawn_appserver(
instance_ref_id,
mark_active_on_success=False,
num_attempts=1,
success_tag=None,
failure_tag=None,
deployment_id=None,
old_server_ids=None,
target_count=None,
):
"""
Create a new AppServer for an existing instance.
instance_ref_id should be the ID of an InstanceReference (instance.ref.pk)
Optionally mark the new AppServer as active when the provisioning completes.
Optionally deactivate old AppServers when the provisioning completes.
Optionally retry up to 'num_attempts' times.
Optionally tag the instance with 'success_tag' when the deployment succeeds,
or failure_tag if it fails.
Optionally associate the AppServer with a deployment.
"""
logger.info('Retrieving instance: ID=%s', instance_ref_id)
instance = OpenEdXInstance.objects.get(ref_set__pk=instance_ref_id)
# NOTE: this is not async; blocks up to an hour.
# The actual appserver model is created fairly quickly though (after
# instance-wide provisioning things happen (mysql, dns records, mongo, s3,
# rabbitmq, etc.) but before appserver provision happens)
appserver = instance.spawn_appserver(
num_attempts=num_attempts,
success_tag=success_tag,
failure_tag=failure_tag,
deployment_id=deployment_id,
)
if appserver and mark_active_on_success:
pipeline = make_appserver_active.s(appserver, active=True).then(
check_deactivation,
instance_ref_id=instance_ref_id,
deployment_id=deployment_id,
old_server_ids=old_server_ids,
target_count=target_count,
)
HUEY.enqueue(pipeline)
def callback(self, target, producer, stop_event, path_qty, path_list,
task):
if stop_event.is_set():
raise NeuralNetworkStopExceptions(
'Neural network stopped by client')
start_qty = path_qty - len(path_list)
progress = int((start_qty + self._count) * 100 / path_qty)
result = {'path': path_list[self._count], 'target': target}
msg = {
'result': result,
'status': 'In progress {}%'.format(progress),
'extra': {}
}
producer.publish(body=json.dumps(msg))
self._store_result(result, task)
if progress % 5 == 0:
HUEY.emit_task(status='in progress', task=task, progress=progress)
self._count += 1
def update_layers(pkey):
with HUEY.lock_task('process-{}'.format(pkey)):
try:
d = Dataset.objects.get(pk=pkey)
d.update_layers()
return 'Processed {} ({!s})'.format(d.name, d.pk)
except Dataset.DoesNotExist:
return 'Dataset did not exist, can not complete task'
except AttributeError:
return 'No update_layers method on this dataset'
def update_dataset(pkey):
with HUEY.lock_task('updating-{}'.format(pkey)):
try:
Dataset.objects.filter(pk=pkey).update(update_task='UPDATING')
update_layers(pkey)
update_time_cache(pkey)
update_grid_cache(pkey)
return 'Done'
finally:
Dataset.objects.filter(pk=pkey).update(update_task='')
def update_grid_cache(pkey):
with HUEY.lock_task('grid-cache-{}'.format(pkey)):
try:
d = Dataset.objects.get(pk=pkey)
d.update_grid_cache()
return 'Updated {} ({!s})'.format(d.name, d.pk)
except Dataset.DoesNotExist:
return 'Dataset did not exist, can not complete task'
except AttributeError:
return 'No update_grid_cache method on this dataset'
def add(self, item):
try:
task = HUEY.serialize_task(item.task)
if task not in self.elements:
self.elements.add(task)
cache.set(self.task_group_id, self.elements)
return task
except PickleError:
raise
except AttributeError:
raise
def test_utc(self):
d1 = timezone.now()
d2 = timezone.now() + datetime.timedelta(minutes=1)
task1 = test_task.schedule(eta=timezone.localtime(d1).replace(
tzinfo=None))
task2 = test_task.schedule(eta=timezone.localtime(d2).replace(
tzinfo=None))
sleep(5)
result1 = huey.get_storage().peek_data(task1.task.task_id)
if result1 is not EmptyData:
result1 = pickle.loads(result1)
result2 = huey.get_storage().peek_data(task2.task.task_id)
if result2 is not EmptyData:
result2 = pickle.loads(result2)
self.assertNotEqual(result1, result2)
def update_time_cache(pkey):
with HUEY.lock_task('time-cache-{}'.format(pkey)):
try:
d = Dataset.objects.get(pk=pkey)
d.update_time_cache()
# Save without callbacks
Dataset.objects.filter(pk=pkey).update(
cache_last_updated=datetime.utcnow().replace(tzinfo=pytz.utc))
return 'Updated {} ({!s})'.format(d.name, d.pk)
except Dataset.DoesNotExist:
return 'Dataset did not exist, can not complete task'
except AttributeError:
return 'No update_time_cache method on this dataset'
def handle(self, *args, **options):
from huey.contrib.djhuey import HUEY
# Python 3.8+ on MacOS uses an incompatible multiprocess model. In this
# case we must explicitly configure mp to use fork().
if sys.version_info >= (3, 8) and sys.platform == 'darwin':
# Apparently this was causing a "context has already been set"
# error for some user. We'll just pass and hope for the best.
# They're apple users so presumably nothing important will be lost.
import multiprocessing
try:
multiprocessing.set_start_method('fork')
except RuntimeError:
pass
consumer_options = {}
try:
if isinstance(settings.HUEY, dict):
consumer_options.update(settings.HUEY.get('consumer', {}))
except AttributeError:
pass
for key, value in options.items():
if value is not None:
consumer_options[key] = value
consumer_options.setdefault('verbose',
consumer_options.pop('huey_verbose', None))
if not options.get('disable_autoload'):
autodiscover_modules("tasks")
logger = logging.getLogger('huey')
config = ConsumerConfig(**consumer_options)
config.validate()
# Only configure the "huey" logger if it has no handlers. For example,
# some users may configure the huey logger via the Django global
# logging config. This prevents duplicating log messages:
if not logger.handlers:
config.setup_logger(logger)
consumer = HUEY.create_consumer(**config.values)
consumer.run()
def add_monitor_entry(monitor_id, payload, sensor=None):
key = '_'.join([
'purpleair.entry',
str(monitor_id),
sensor if sensor is not None else '',
str(payload[0]["created_at"]),
])
try:
with HUEY.lock_task(key):
monitor = PurpleAir.objects.get(pk=monitor_id)
print(f'[add_monitor_entry] {monitor.name} ({monitor.pk})')
entry = monitor.create_entry(payload, sensor=sensor)
entry = monitor.process_entry(entry)
entry.save()
monitor.check_latest(Entry.objects.get(pk=entry.pk))
except TaskLockedException:
print(
f'[LOCKED:add_monitor_entry] {monitor_id} ({payload[0]["created_at"]})'
)
def handle_update(update_id, bot_id):
lock_id = 'handle_update_{}_{}'.format(update_id, bot_id)
with djhuey.lock_task(lock_id):
try:
update = caching.get_or_set(Update, update_id)
telegram_bot = caching.get_or_set(Bot, bot_id)
except Update.DoesNotExist:
logger.error("Update %s does not exists" % update_id)
except Bot.DoesNotExist:
logger.error("Bot %s does not exists or disabled" % bot_id)
except:
logger.error("Error handling update %s from bot %s" %
(update_id, bot_id))
else:
try:
telegram_bot.handle_message(update)
except:
logger.error("Error processing %s for bot %s" %
(update, telegram_bot))
else:
caching.delete(Update, update)
def add_unidentified_dataset(pkey):
with HUEY.lock_task('unidentified-{}'.format(pkey)):
try:
ud = UnidentifiedDataset.objects.get(pk=pkey)
klass = Dataset.identify(ud.uri)
if klass is not None:
try:
ud.delete()
ds = klass.objects.create(name=ud.name, uri=ud.uri)
return 'Added {}'.format(ds.name)
except IntegrityError:
msg = 'Could not add dataset, name "{}" already exists'.format(
ud.name)
ud.messages = msg
ud.save()
return msg
else:
msg = 'No dataset types found to process {}'.format(ud.uri)
ud.messages = msg
ud.save()
return msg
except UnidentifiedDataset.DoesNotExist:
return 'UnidentifiedDataset did not exist, can not complete task'
import os
import channels.asgi
from . import configure_settings
configure_settings()
channel_layer = channels.asgi.get_channel_layer()
from otree.common_internal import (
release_any_stale_locks, get_redis_conn # noqa
)
# clear any tasks in Huey DB, so they don't pile up over time,
# especially if you run the server without the timeoutworker to consume the
# tasks.
# ideally we would only schedule a task in Huey if timeoutworker is running,
# so that we don't pile up messages that never get consumed, but I don't know
# how and when to check if Huey is running, in a performant way.
# this code is also in timeoutworker.
from huey.contrib.djhuey import HUEY # noqa
HUEY.flush()
from otree.bots.browser import redis_flush_bots # noqa
redis_flush_bots(get_redis_conn())
# needs to happen after Django setup
release_any_stale_locks()
def task(self):
'''Returns the task result and preserves it'''
if self.task_id:
return huey.result(self.task_id, preserve=True)
def get_result(self, obj):
return HUEY.result(obj.id, preserve=True)
def load_tasks_async():
with HUEY.lock_task("import-lock"):
load_tasks(
GitRepository(settings.REPOSITORY_ROOT,
url=config.GITLOAD_URL,
branch=config.GITLOAD_BRANCH))
def handle(self, *args, **kwargs):
HUEY.pending_count()
HUEY.flush()
Entry.objects.all().delete()
def test_add_task(self):
task = self.group.add(add_one(4))
task = HUEY.deserialize_task(task)
self.assertTrue(task in self.group)
def get(self, request, project_id, task_id):
"""Returns the (possibly) new layer ID where the isochrone
data has been added. If the task has not yet completed a status message is returned
Note: `project_id` is only used for permissions checking!
Returns 500 in case of exceptions
Returns 404 in case of task not found
Returns 200 ok for all other cases
Response body:
{
"status": "complete", // or "pending" or "error", full list at
// https://huey.readthedocs.io/en/latest/signals.html#signals
"exception": "Normally empty, error message in case of errors",
"progress": [
100, // Progress %
],
"task_result": {
"qgis_layer_id": "4f2a88a1-ca93-4859-9de3-75d9728cde0e"
}
}
TODO: move this into core, make it generic for task status and result reporting
"""
try:
# Try to retrieve the task result, may throw an exception
try:
result = HUEY.result(task_id)
ret_status = status.HTTP_200_OK
except TaskException:
result = None
ret_status = status.HTTP_500_INTERNAL_SERVER_ERROR
task_model = TaskModel.objects.get(task_id=task_id)
progress_info = task_model.progress_info
try:
progress_percentage = int(100 * progress_info[0] /
task_model.total)
except:
progress_percentage = 0
try:
return Response(
{
'status': task_model.state.signal_name,
'exception': task_model.state.exception_line,
'progress': progress_percentage,
'task_result': result
},
status=ret_status)
except:
return Response(
{
'status': 'error',
'exception': 'Error retrieving task informations',
'progress': 0,
'task_result': result,
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
except TaskModel.DoesNotExist:
# Handle pending
pending_task_ids = [task.id for task in HUEY.pending()]
if task_id in pending_task_ids:
return Response({'result': True, 'status': 'pending'})
return Response({
'result': False,
'error': _('Task not found!')
},
status=status.HTTP_404_NOT_FOUND)
def post(self, request):
HUEY.flush()
messages.success(request, 'The task queue has been flushed.')
return redirect('admin:index')
def get(self, request):
return JSONResponse({
'timestamp': timezone.now(),
'queue_size': HUEY.pending_count(),
'entry_count': self.get_entry_count(),
})
def _check_redis_connection(self):
"""
Runs the redis ECHO command to check connection
"""
HUEY.get_storage().conn.echo('health_check')
import os
import django
from channels.routing import get_default_application
from . import configure_settings
from django.conf import settings
# needed if uvicorn is launched multi-process
if not settings.configured:
configure_settings()
django.setup()
application = get_default_application()
# clear any tasks in Huey DB, so they don't pile up over time,
# especially if you run the server without the timeoutworker to consume the
# tasks.
# ideally we would only schedule a task in Huey if timeoutworker is running,
# so that we don't pile up messages that never get consumed, but I don't know
# how and when to check if Huey is running, in a performant way.
# this code is also in timeoutworker.
from huey.contrib.djhuey import HUEY # noqa
import redis.exceptions
try:
HUEY.flush()
except redis.exceptions.ConnectionError:
# maybe Redis is not running
pass
def discard(self, item):
try:
task = HUEY.serialize_task(item)
self.elements.remove(task)
except ValueError:
pass
