def load_tasks_async(): with HUEY.lock_task("import-lock"): load_tasks(GitRepository( settings.REPOSITORY_ROOT, url=config.GITLOAD_URL, branch=config.GITLOAD_BRANCH ))
def add_monitor_entry(monitor_id, payload, sensor=None): # CONSIDER: If we're skipping existing entries, we # probably don't need this lock key = '_'.join([ 'purpleair.entry', str(monitor_id), sensor if sensor is not None else '', str(payload[0]["created_at"]), ]) try: with HUEY.lock_task(key): monitor = PurpleAir.objects.get(pk=monitor_id) entry = monitor.create_entry(payload, sensor=sensor) # If the entry was pulled from the database and is not # a new instance, we can end early since we already have it. if not entry._state.adding: return print(f'[add_monitor_entry] {monitor.name} ({monitor.pk})') entry = monitor.process_entry(entry) entry.save() monitor.check_latest(Entry.objects.get(pk=entry.pk)) except TaskLockedException: print( f'[LOCKED:add_monitor_entry] {monitor_id} ({payload[0]["created_at"]})' )
def add_monitor_entry(monitor_id, payload, sensor=None): key = '_'.join([ 'purpleair.entry', str(monitor_id), sensor if sensor is not None else '', str(payload[0]["created_at"]), ]) try: with HUEY.lock_task(key): monitor = PurpleAir.objects.get(pk=monitor_id) print(f'[add_monitor_entry] {monitor.name} ({monitor.pk})') entry = monitor.create_entry(payload, sensor=sensor) entry = monitor.process_entry(entry) entry.save() is_latest = not monitor.latest or ( entry.timestamp > parse_datetime(monitor.latest['timestamp'])) sensor_match = monitor.DEFAULT_SENSOR is None or entry.sensor == monitor.DEFAULT_SENSOR if sensor_match and is_latest: monitor.set_latest(Entry.objects.get(pk=entry.pk)) monitor.save() except TaskLockedException: print( f'[LOCKED:add_monitor_entry] {monitor_id} ({payload[0]["created_at"]})' )
def update_dataset(pkey): with HUEY.lock_task('updating-{}'.format(pkey)): try: Dataset.objects.filter(pk=pkey).update(update_task='UPDATING') update_layers(pkey) update_time_cache(pkey) update_grid_cache(pkey) return 'Done' finally: Dataset.objects.filter(pk=pkey).update(update_task='')
def update_grid_cache(pkey): with HUEY.lock_task('grid-cache-{}'.format(pkey)): try: d = Dataset.objects.get(pk=pkey) d.update_grid_cache() return 'Updated {} ({!s})'.format(d.name, d.pk) except Dataset.DoesNotExist: return 'Dataset did not exist, can not complete task' except AttributeError: return 'No update_grid_cache method on this dataset'
def update_layers(pkey): with HUEY.lock_task('process-{}'.format(pkey)): try: d = Dataset.objects.get(pk=pkey) d.update_layers() return 'Processed {} ({!s})'.format(d.name, d.pk) except Dataset.DoesNotExist: return 'Dataset did not exist, can not complete task' except AttributeError: return 'No update_layers method on this dataset'
def update_time_cache(pkey): with HUEY.lock_task('time-cache-{}'.format(pkey)): try: d = Dataset.objects.get(pk=pkey) d.update_time_cache() # Save without callbacks Dataset.objects.filter(pk=pkey).update( cache_last_updated=datetime.utcnow().replace(tzinfo=pytz.utc)) return 'Updated {} ({!s})'.format(d.name, d.pk) except Dataset.DoesNotExist: return 'Dataset did not exist, can not complete task' except AttributeError: return 'No update_time_cache method on this dataset'
def add_monitor_entry(monitor_id, payload, sensor=None): key = '_'.join([ 'purpleair.entry', str(monitor_id), sensor if sensor is not None else '', str(payload[0]["created_at"]), ]) try: with HUEY.lock_task(key): monitor = PurpleAir.objects.get(pk=monitor_id) print(f'[add_monitor_entry] {monitor.name} ({monitor.pk})') entry = monitor.create_entry(payload, sensor=sensor) entry = monitor.process_entry(entry) entry.save() monitor.check_latest(Entry.objects.get(pk=entry.pk)) except TaskLockedException: print( f'[LOCKED:add_monitor_entry] {monitor_id} ({payload[0]["created_at"]})' )
def handle_update(update_id, bot_id): lock_id = 'handle_update_{}_{}'.format(update_id, bot_id) with djhuey.lock_task(lock_id): try: update = caching.get_or_set(Update, update_id) telegram_bot = caching.get_or_set(Bot, bot_id) except Update.DoesNotExist: logger.error("Update %s does not exists" % update_id) except Bot.DoesNotExist: logger.error("Bot %s does not exists or disabled" % bot_id) except: logger.error("Error handling update %s from bot %s" % (update_id, bot_id)) else: try: telegram_bot.handle_message(update) except: logger.error("Error processing %s for bot %s" % (update, telegram_bot)) else: caching.delete(Update, update)
def add_unidentified_dataset(pkey): with HUEY.lock_task('unidentified-{}'.format(pkey)): try: ud = UnidentifiedDataset.objects.get(pk=pkey) klass = Dataset.identify(ud.uri) if klass is not None: try: ud.delete() ds = klass.objects.create(name=ud.name, uri=ud.uri) return 'Added {}'.format(ds.name) except IntegrityError: msg = 'Could not add dataset, name "{}" already exists'.format( ud.name) ud.messages = msg ud.save() return msg else: msg = 'No dataset types found to process {}'.format(ud.uri) ud.messages = msg ud.save() return msg except UnidentifiedDataset.DoesNotExist: return 'UnidentifiedDataset did not exist, can not complete task'
def load_tasks_async(): with HUEY.lock_task("import-lock"): load_tasks( GitRepository(settings.REPOSITORY_ROOT, url=config.GITLOAD_URL, branch=config.GITLOAD_BRANCH))