def load_tasks_async():
with HUEY.lock_task("import-lock"):
load_tasks(GitRepository(
settings.REPOSITORY_ROOT,
url=config.GITLOAD_URL,
branch=config.GITLOAD_BRANCH
))
def add_monitor_entry(monitor_id, payload, sensor=None):
# CONSIDER: If we're skipping existing entries, we
# probably don't need this lock
key = '_'.join([
'purpleair.entry',
str(monitor_id),
sensor if sensor is not None else '',
str(payload[0]["created_at"]),
])
try:
with HUEY.lock_task(key):
monitor = PurpleAir.objects.get(pk=monitor_id)
entry = monitor.create_entry(payload, sensor=sensor)
# If the entry was pulled from the database and is not
# a new instance, we can end early since we already have it.
if not entry._state.adding:
return
print(f'[add_monitor_entry] {monitor.name} ({monitor.pk})')
entry = monitor.process_entry(entry)
entry.save()
monitor.check_latest(Entry.objects.get(pk=entry.pk))
except TaskLockedException:
print(
f'[LOCKED:add_monitor_entry] {monitor_id} ({payload[0]["created_at"]})'
)
def add_monitor_entry(monitor_id, payload, sensor=None):
key = '_'.join([
'purpleair.entry',
str(monitor_id),
sensor if sensor is not None else '',
str(payload[0]["created_at"]),
])
try:
with HUEY.lock_task(key):
monitor = PurpleAir.objects.get(pk=monitor_id)
print(f'[add_monitor_entry] {monitor.name} ({monitor.pk})')
entry = monitor.create_entry(payload, sensor=sensor)
entry = monitor.process_entry(entry)
entry.save()
is_latest = not monitor.latest or (
entry.timestamp > parse_datetime(monitor.latest['timestamp']))
sensor_match = monitor.DEFAULT_SENSOR is None or entry.sensor == monitor.DEFAULT_SENSOR
if sensor_match and is_latest:
monitor.set_latest(Entry.objects.get(pk=entry.pk))
monitor.save()
except TaskLockedException:
print(
f'[LOCKED:add_monitor_entry] {monitor_id} ({payload[0]["created_at"]})'
)
def update_dataset(pkey):
with HUEY.lock_task('updating-{}'.format(pkey)):
try:
Dataset.objects.filter(pk=pkey).update(update_task='UPDATING')
update_layers(pkey)
update_time_cache(pkey)
update_grid_cache(pkey)
return 'Done'
finally:
Dataset.objects.filter(pk=pkey).update(update_task='')
def update_grid_cache(pkey):
with HUEY.lock_task('grid-cache-{}'.format(pkey)):
try:
d = Dataset.objects.get(pk=pkey)
d.update_grid_cache()
return 'Updated {} ({!s})'.format(d.name, d.pk)
except Dataset.DoesNotExist:
return 'Dataset did not exist, can not complete task'
except AttributeError:
return 'No update_grid_cache method on this dataset'
def update_layers(pkey):
with HUEY.lock_task('process-{}'.format(pkey)):
try:
d = Dataset.objects.get(pk=pkey)
d.update_layers()
return 'Processed {} ({!s})'.format(d.name, d.pk)
except Dataset.DoesNotExist:
return 'Dataset did not exist, can not complete task'
except AttributeError:
return 'No update_layers method on this dataset'
def update_time_cache(pkey):
with HUEY.lock_task('time-cache-{}'.format(pkey)):
try:
d = Dataset.objects.get(pk=pkey)
d.update_time_cache()
# Save without callbacks
Dataset.objects.filter(pk=pkey).update(
cache_last_updated=datetime.utcnow().replace(tzinfo=pytz.utc))
return 'Updated {} ({!s})'.format(d.name, d.pk)
except Dataset.DoesNotExist:
return 'Dataset did not exist, can not complete task'
except AttributeError:
return 'No update_time_cache method on this dataset'
def add_monitor_entry(monitor_id, payload, sensor=None):
key = '_'.join([
'purpleair.entry',
str(monitor_id),
sensor if sensor is not None else '',
str(payload[0]["created_at"]),
])
try:
with HUEY.lock_task(key):
monitor = PurpleAir.objects.get(pk=monitor_id)
print(f'[add_monitor_entry] {monitor.name} ({monitor.pk})')
entry = monitor.create_entry(payload, sensor=sensor)
entry = monitor.process_entry(entry)
entry.save()
monitor.check_latest(Entry.objects.get(pk=entry.pk))
except TaskLockedException:
print(
f'[LOCKED:add_monitor_entry] {monitor_id} ({payload[0]["created_at"]})'
)
def handle_update(update_id, bot_id):
lock_id = 'handle_update_{}_{}'.format(update_id, bot_id)
with djhuey.lock_task(lock_id):
try:
update = caching.get_or_set(Update, update_id)
telegram_bot = caching.get_or_set(Bot, bot_id)
except Update.DoesNotExist:
logger.error("Update %s does not exists" % update_id)
except Bot.DoesNotExist:
logger.error("Bot %s does not exists or disabled" % bot_id)
except:
logger.error("Error handling update %s from bot %s" %
(update_id, bot_id))
else:
try:
telegram_bot.handle_message(update)
except:
logger.error("Error processing %s for bot %s" %
(update, telegram_bot))
else:
caching.delete(Update, update)
def add_unidentified_dataset(pkey):
with HUEY.lock_task('unidentified-{}'.format(pkey)):
try:
ud = UnidentifiedDataset.objects.get(pk=pkey)
klass = Dataset.identify(ud.uri)
if klass is not None:
try:
ud.delete()
ds = klass.objects.create(name=ud.name, uri=ud.uri)
return 'Added {}'.format(ds.name)
except IntegrityError:
msg = 'Could not add dataset, name "{}" already exists'.format(
ud.name)
ud.messages = msg
ud.save()
return msg
else:
msg = 'No dataset types found to process {}'.format(ud.uri)
ud.messages = msg
ud.save()
return msg
except UnidentifiedDataset.DoesNotExist:
return 'UnidentifiedDataset did not exist, can not complete task'
def load_tasks_async():
with HUEY.lock_task("import-lock"):
load_tasks(
GitRepository(settings.REPOSITORY_ROOT,
url=config.GITLOAD_URL,
branch=config.GITLOAD_BRANCH))
