def test_custom_lock_timeout(self): """ Tests that the custom lock timeout works when an hour is configured as the timeout. """ with freeze_time('2014-02-01'): # Create a lock orig_lock = DBMutex.objects.create(lock_id='lock_id') # Try to acquire the lock one minute in the future. It should fail with freeze_time('2014-02-01 00:01:00'): with self.assertRaises(DBMutexError): with db_mutex('lock_id'): raise NotImplementedError # Try to acquire the lock 31 minutes in the future. It should fail with freeze_time('2014-02-01 00:31:00'): with self.assertRaises(DBMutexError): with db_mutex('lock_id'): raise NotImplementedError # Try to acquire the lock 60 minutes in the future. It should pass with freeze_time('2014-02-01 01:00:00'): with db_mutex('lock_id'): self.assertFalse(DBMutex.objects.filter(id=orig_lock.id).exists()) self.assertEqual(DBMutex.objects.count(), 1) m = DBMutex.objects.get(lock_id='lock_id') self.assertEqual(m.creation_time, datetime(2014, 2, 1, 1))
def test_lock_timeout_default(self): """ Tests that the lock timeout works with the default value of 30 minutes. """ with freeze_time('2014-02-01'): # Create a lock orig_lock = DBMutex.objects.create(lock_id='lock_id') # Try to acquire the lock one minute in the future. It should fail with freeze_time('2014-02-01 00:01:00'): with self.assertRaises(DBMutexError): with db_mutex('lock_id'): raise NotImplementedError # Try to acquire the lock 9 minutes in the future. It should fail with freeze_time('2014-02-01 00:09:00'): with self.assertRaises(DBMutexError): with db_mutex('lock_id'): raise NotImplementedError # Try to acquire the lock 30 minutes in the future. It should pass since the lock timed out with freeze_time('2014-02-01 00:30:00'): with db_mutex('lock_id'): self.assertFalse(DBMutex.objects.filter(id=orig_lock.id).exists()) self.assertEqual(DBMutex.objects.count(), 1) m = DBMutex.objects.get(lock_id='lock_id') self.assertEqual(m.creation_time, datetime(2014, 2, 1, 0, 30))
def test_no_lock_timeout(self): """ Tests that the lock timeout works when None is configured as the timeout. """ with freeze_time('2014-02-01'): # Create a lock DBMutex.objects.create(lock_id='lock_id') # Try to acquire the lock one minute in the future. It should fail with freeze_time('2014-02-01 00:01:00'): with self.assertRaises(DBMutexError): with db_mutex('lock_id'): raise NotImplementedError # Try to acquire the lock 9 minutes in the future. It should fail with freeze_time('2014-02-01 00:09:00'): with self.assertRaises(DBMutexError): with db_mutex('lock_id'): raise NotImplementedError # Try to acquire the lock 30 minutes in the future. It should fail with freeze_time('2014-02-01 00:30:00'): with self.assertRaises(DBMutexError): with db_mutex('lock_id'): raise NotImplementedError # Try to acquire the lock years in the future. It should fail with freeze_time('2016-02-01 00:30:00'): with self.assertRaises(DBMutexError): with db_mutex('lock_id'): raise NotImplementedError
def run_exclusive(self, lock_id, callback, error_on_locked = False, wait = True, timeout = 600, interval = 2): if not lock_id: callback() else: start_time = time.time() current_time = start_time while (current_time - start_time) <= timeout: try: with db_mutex(lock_id): callback() break except DBMutexError: if error_on_locked: self.error("Could not obtain lock for {}".format(lock_id)) if not wait: break except DBMutexTimeoutError: logger.warning("Task {} completed but the lock timed out".format(lock_id)) break except Exception as e: DBMutex.objects.filter(lock_id = lock_id).delete() raise e time.sleep(interval) current_time = time.time()
def update_categories(): success = True lock_id = 'categories.update_categories' old_stdout = sys.stdout sys.stdout = mystdout = io.StringIO() try: with db_mutex(lock_id): # Commands don't return anything call_command('parse_categories') except DBMutexError: success = False print('update_categories: Could not obtain lock') except DBMutexTimeoutError: print('update_categories: Task completed but the lock timed out') except Exception as error: print(error) DBMutex.objects.filter(lock_id=lock_id).delete() success = False sys.stdout = old_stdout if not success: raise TaskError(mystdout.getvalue()) return { "task": "update_categories", "params": {}, "message": mystdout.getvalue() }
def run(self, component, guid, value, *args, **kwargs): try: with db_mutex('{0}:{1}'.format(AGENT_MUTEX_ID_NAMESPACE, component.guid)): # add value to timeslice component.push(guid, value) except DBMutexError as exc: raise self.retry(exc=exc, countdown=settings.TIMESLICE_LOCK_RETRY_DELAY_MS)
def test_no_lock_before(self): """ Tests that a lock is succesfully acquired. """ # There should be no locks before and after the context manager self.assertEqual(DBMutex.objects.count(), 0) with db_mutex('lock_id'): self.assertEqual(DBMutex.objects.count(), 1) m = DBMutex.objects.get(lock_id='lock_id') self.assertEqual(m.creation_time, datetime(2014, 2, 1)) self.assertEqual(DBMutex.objects.count(), 0)
def test_lock_before(self): """ Tests when a lock already exists. """ # Create a lock m = DBMutex.objects.create(lock_id='lock_id') # Try to acquire the lock. It should raise an exception with self.assertRaises(DBMutexError): with db_mutex('lock_id'): raise NotImplementedError # The lock should still exist self.assertTrue(DBMutex.objects.filter(id=m.id).exists())
def test_lock_before_suppress_acquisition_errors(self): """ Tests when a lock already exists. Verifies that an exception is thrown when suppress_acquisition_errors is True. The exception is still thrown because we are using it as a context manager """ # Create a lock m = DBMutex.objects.create(lock_id='lock_id') # Try to acquire the lock. It should neither acquire nor release it with self.assertRaises(DBMutexError): with db_mutex('lock_id', suppress_acquisition_exceptions=True): raise NotImplementedError # The lock should still exist self.assertTrue(DBMutex.objects.filter(id=m.id).exists())
def test_lock_different_id(self): """ Tests that the lock still works even when another lock with a different id exists. """ # Create a lock m = DBMutex.objects.create(lock_id='lock_id') # Try to acquire the lock with a different ID with db_mutex('lock_id2'): self.assertEqual(DBMutex.objects.count(), 2) m2 = DBMutex.objects.get(lock_id='lock_id2') self.assertEqual(m2.creation_time, datetime(2014, 2, 1)) # The original lock should still exist but the other one should be gone self.assertTrue(DBMutex.objects.filter(id=m.id).exists()) self.assertEqual(DBMutex.objects.count(), 1)
def test_lock_timeout_error(self): """ Tests the case when a lock expires while the context manager is executing. """ with freeze_time('2014-02-01'): # Acquire a lock at the given time and release it before it is finished. It # should result in an error with self.assertRaises(DBMutexTimeoutError): with db_mutex('lock_id'): self.assertEqual(DBMutex.objects.count(), 1) m = DBMutex.objects.get(lock_id='lock_id') self.assertEqual(m.creation_time, datetime(2014, 2, 1)) # Release the lock before the context manager finishes m.delete()
def sync(self): try: time.sleep(random.randrange(10)) with db_mutex(self.lock_id): super().sync() except DBMutexError: logger.warning("Scheduler could not obtain lock for {}".format( self.lock_id)) except DBMutexTimeoutError: logger.warning("Scheduler sync completed but the lock timed out") except Exception as e: DBMutex.objects.filter(lock_id=self.lock_id).delete() raise e
def update_contracts(period=260, load=260, count=500, pause=1): success = True lock_id = 'contracts.update_contracts' old_stdout = sys.stdout sys.stdout = mystdout = io.StringIO() try: with db_mutex(lock_id): # Commands don't return anything call_command('load_fpds', period=period, load=load, count=count, pause=pause) except DBMutexError: success = False print('update_contracts: Could not obtain lock') except DBMutexTimeoutError: print('update_contracts: Task completed but the lock timed out') except Exception as error: print(error) DBMutex.objects.filter(lock_id=lock_id).delete() success = False sys.stdout = old_stdout if not success: raise TaskError(mystdout.getvalue()) return { "task": "update_fpds", "params": { "period": period, "load": load, "count": count, "pause": pause }, "message": mystdout.getvalue() }
def update_vendors_sam(tries=3, pause=1): success = True lock_id = 'vendors.update_vendors_sam' old_stdout = sys.stdout sys.stdout = mystdout = io.StringIO() try: with db_mutex(lock_id): # Commands don't return anything call_command('load_sam', tries=tries, pause=pause) except DBMutexError: success = False print('update_vendors_sam: Could not obtain lock') except DBMutexTimeoutError: print('update_vendors_sam: Task completed but the lock timed out') except Exception as error: print(error) DBMutex.objects.filter(lock_id=lock_id).delete() success = False sys.stdout = old_stdout if not success: raise TaskError(mystdout.getvalue()) return { "task": "update_vendors_sam", "params": { "tries": tries, "pause": pause }, "message": mystdout.getvalue() }
def run(self, *args, **kwargs): for component in NewRelicComponent.objects.all(): try: with db_mutex('{0}:{1}'.format(AGENT_MUTEX_ID_NAMESPACE, component.guid)): last_pushed_time = component.last_pushed_time metrics = self._get_component_metrics(component) component.last_pushed_time = datetime.now() component.save() except DBMutexError as exc: raise self.retry( exc=exc, countdown=settings.TIMESLICE_LOCK_RETRY_DELAY_MS) # note regarding failure to deliver (http) metrics: # I considered merging the metrics back and resetting the last # push time of the component, but couldn't easily solve the race # condition observed when this component is repopulated and # pushed in the time between releasing the lock and getting a # bad http response from NR. The other obvious alternative is # to lock until we receive a successful response, but I don't # consider that an acceptable solution. Considered this lost # request an acceptable risk for the time being. No data is # better than bad data imo self._push_component_metrics(component, metrics, last_pushed_time)
def prune_contracts(period=260): success = True lock_id = 'contracts.prune_contracts' old_stdout = sys.stdout sys.stdout = mystdout = io.StringIO() try: with db_mutex(lock_id): # Commands don't return anything call_command('prune_contracts', period=period) except DBMutexError: success = False print('prune_contracts: Could not obtain lock') except DBMutexTimeoutError: print('prune_contracts: Task completed but the lock timed out') except Exception as error: print(error) DBMutex.objects.filter(lock_id=lock_id).delete() success = False sys.stdout = old_stdout if not success: raise TaskError(mystdout.getvalue()) return { "task": "prune_contracts", "params": { "period": period }, "message": mystdout.getvalue() }
def run(self, *args, **kwargs): with db_mutex('push-slack-events'): self.run_worker(*args, **kwargs)