def test_custom_lock_timeout(self):
"""
Tests that the custom lock timeout works when an hour is configured as the timeout.
"""
with freeze_time('2014-02-01'):
# Create a lock
orig_lock = DBMutex.objects.create(lock_id='lock_id')
# Try to acquire the lock one minute in the future. It should fail
with freeze_time('2014-02-01 00:01:00'):
with self.assertRaises(DBMutexError):
with db_mutex('lock_id'):
raise NotImplementedError
# Try to acquire the lock 31 minutes in the future. It should fail
with freeze_time('2014-02-01 00:31:00'):
with self.assertRaises(DBMutexError):
with db_mutex('lock_id'):
raise NotImplementedError
# Try to acquire the lock 60 minutes in the future. It should pass
with freeze_time('2014-02-01 01:00:00'):
with db_mutex('lock_id'):
self.assertFalse(DBMutex.objects.filter(id=orig_lock.id).exists())
self.assertEqual(DBMutex.objects.count(), 1)
m = DBMutex.objects.get(lock_id='lock_id')
self.assertEqual(m.creation_time, datetime(2014, 2, 1, 1))
def test_lock_timeout_default(self):
"""
Tests that the lock timeout works with the default value of 30 minutes.
"""
with freeze_time('2014-02-01'):
# Create a lock
orig_lock = DBMutex.objects.create(lock_id='lock_id')
# Try to acquire the lock one minute in the future. It should fail
with freeze_time('2014-02-01 00:01:00'):
with self.assertRaises(DBMutexError):
with db_mutex('lock_id'):
raise NotImplementedError
# Try to acquire the lock 9 minutes in the future. It should fail
with freeze_time('2014-02-01 00:09:00'):
with self.assertRaises(DBMutexError):
with db_mutex('lock_id'):
raise NotImplementedError
# Try to acquire the lock 30 minutes in the future. It should pass since the lock timed out
with freeze_time('2014-02-01 00:30:00'):
with db_mutex('lock_id'):
self.assertFalse(DBMutex.objects.filter(id=orig_lock.id).exists())
self.assertEqual(DBMutex.objects.count(), 1)
m = DBMutex.objects.get(lock_id='lock_id')
self.assertEqual(m.creation_time, datetime(2014, 2, 1, 0, 30))
def test_no_lock_timeout(self):
"""
Tests that the lock timeout works when None is configured as the timeout.
"""
with freeze_time('2014-02-01'):
# Create a lock
DBMutex.objects.create(lock_id='lock_id')
# Try to acquire the lock one minute in the future. It should fail
with freeze_time('2014-02-01 00:01:00'):
with self.assertRaises(DBMutexError):
with db_mutex('lock_id'):
raise NotImplementedError
# Try to acquire the lock 9 minutes in the future. It should fail
with freeze_time('2014-02-01 00:09:00'):
with self.assertRaises(DBMutexError):
with db_mutex('lock_id'):
raise NotImplementedError
# Try to acquire the lock 30 minutes in the future. It should fail
with freeze_time('2014-02-01 00:30:00'):
with self.assertRaises(DBMutexError):
with db_mutex('lock_id'):
raise NotImplementedError
# Try to acquire the lock years in the future. It should fail
with freeze_time('2016-02-01 00:30:00'):
with self.assertRaises(DBMutexError):
with db_mutex('lock_id'):
raise NotImplementedError
def run_exclusive(self, lock_id, callback, error_on_locked = False, wait = True, timeout = 600, interval = 2):
if not lock_id:
callback()
else:
start_time = time.time()
current_time = start_time
while (current_time - start_time) <= timeout:
try:
with db_mutex(lock_id):
callback()
break
except DBMutexError:
if error_on_locked:
self.error("Could not obtain lock for {}".format(lock_id))
if not wait:
break
except DBMutexTimeoutError:
logger.warning("Task {} completed but the lock timed out".format(lock_id))
break
except Exception as e:
DBMutex.objects.filter(lock_id = lock_id).delete()
raise e
time.sleep(interval)
current_time = time.time()
def update_categories():
success = True
lock_id = 'categories.update_categories'
old_stdout = sys.stdout
sys.stdout = mystdout = io.StringIO()
try:
with db_mutex(lock_id):
# Commands don't return anything
call_command('parse_categories')
except DBMutexError:
success = False
print('update_categories: Could not obtain lock')
except DBMutexTimeoutError:
print('update_categories: Task completed but the lock timed out')
except Exception as error:
print(error)
DBMutex.objects.filter(lock_id=lock_id).delete()
success = False
sys.stdout = old_stdout
if not success:
raise TaskError(mystdout.getvalue())
return {
"task": "update_categories",
"params": {},
"message": mystdout.getvalue()
}
def run(self, component, guid, value, *args, **kwargs):
try:
with db_mutex('{0}:{1}'.format(AGENT_MUTEX_ID_NAMESPACE,
component.guid)):
# add value to timeslice
component.push(guid, value)
except DBMutexError as exc:
raise self.retry(exc=exc,
countdown=settings.TIMESLICE_LOCK_RETRY_DELAY_MS)
def test_no_lock_before(self):
"""
Tests that a lock is succesfully acquired.
"""
# There should be no locks before and after the context manager
self.assertEqual(DBMutex.objects.count(), 0)
with db_mutex('lock_id'):
self.assertEqual(DBMutex.objects.count(), 1)
m = DBMutex.objects.get(lock_id='lock_id')
self.assertEqual(m.creation_time, datetime(2014, 2, 1))
self.assertEqual(DBMutex.objects.count(), 0)
def test_lock_before(self):
"""
Tests when a lock already exists.
"""
# Create a lock
m = DBMutex.objects.create(lock_id='lock_id')
# Try to acquire the lock. It should raise an exception
with self.assertRaises(DBMutexError):
with db_mutex('lock_id'):
raise NotImplementedError
# The lock should still exist
self.assertTrue(DBMutex.objects.filter(id=m.id).exists())
def test_lock_before_suppress_acquisition_errors(self):
"""
Tests when a lock already exists. Verifies that an exception is thrown when
suppress_acquisition_errors is True. The exception is still thrown because
we are using it as a context manager
"""
# Create a lock
m = DBMutex.objects.create(lock_id='lock_id')
# Try to acquire the lock. It should neither acquire nor release it
with self.assertRaises(DBMutexError):
with db_mutex('lock_id', suppress_acquisition_exceptions=True):
raise NotImplementedError
# The lock should still exist
self.assertTrue(DBMutex.objects.filter(id=m.id).exists())
def test_lock_different_id(self):
"""
Tests that the lock still works even when another lock with a different id exists.
"""
# Create a lock
m = DBMutex.objects.create(lock_id='lock_id')
# Try to acquire the lock with a different ID
with db_mutex('lock_id2'):
self.assertEqual(DBMutex.objects.count(), 2)
m2 = DBMutex.objects.get(lock_id='lock_id2')
self.assertEqual(m2.creation_time, datetime(2014, 2, 1))
# The original lock should still exist but the other one should be gone
self.assertTrue(DBMutex.objects.filter(id=m.id).exists())
self.assertEqual(DBMutex.objects.count(), 1)
def test_lock_timeout_error(self):
"""
Tests the case when a lock expires while the context manager is executing.
"""
with freeze_time('2014-02-01'):
# Acquire a lock at the given time and release it before it is finished. It
# should result in an error
with self.assertRaises(DBMutexTimeoutError):
with db_mutex('lock_id'):
self.assertEqual(DBMutex.objects.count(), 1)
m = DBMutex.objects.get(lock_id='lock_id')
self.assertEqual(m.creation_time, datetime(2014, 2, 1))
# Release the lock before the context manager finishes
m.delete()
def sync(self):
try:
time.sleep(random.randrange(10))
with db_mutex(self.lock_id):
super().sync()
except DBMutexError:
logger.warning("Scheduler could not obtain lock for {}".format(
self.lock_id))
except DBMutexTimeoutError:
logger.warning("Scheduler sync completed but the lock timed out")
except Exception as e:
DBMutex.objects.filter(lock_id=self.lock_id).delete()
raise e
def update_contracts(period=260, load=260, count=500, pause=1):
success = True
lock_id = 'contracts.update_contracts'
old_stdout = sys.stdout
sys.stdout = mystdout = io.StringIO()
try:
with db_mutex(lock_id):
# Commands don't return anything
call_command('load_fpds',
period=period,
load=load,
count=count,
pause=pause)
except DBMutexError:
success = False
print('update_contracts: Could not obtain lock')
except DBMutexTimeoutError:
print('update_contracts: Task completed but the lock timed out')
except Exception as error:
print(error)
DBMutex.objects.filter(lock_id=lock_id).delete()
success = False
sys.stdout = old_stdout
if not success:
raise TaskError(mystdout.getvalue())
return {
"task": "update_fpds",
"params": {
"period": period,
"load": load,
"count": count,
"pause": pause
},
"message": mystdout.getvalue()
}
def update_vendors_sam(tries=3, pause=1):
success = True
lock_id = 'vendors.update_vendors_sam'
old_stdout = sys.stdout
sys.stdout = mystdout = io.StringIO()
try:
with db_mutex(lock_id):
# Commands don't return anything
call_command('load_sam', tries=tries, pause=pause)
except DBMutexError:
success = False
print('update_vendors_sam: Could not obtain lock')
except DBMutexTimeoutError:
print('update_vendors_sam: Task completed but the lock timed out')
except Exception as error:
print(error)
DBMutex.objects.filter(lock_id=lock_id).delete()
success = False
sys.stdout = old_stdout
if not success:
raise TaskError(mystdout.getvalue())
return {
"task": "update_vendors_sam",
"params": {
"tries": tries,
"pause": pause
},
"message": mystdout.getvalue()
}
def run(self, *args, **kwargs):
for component in NewRelicComponent.objects.all():
try:
with db_mutex('{0}:{1}'.format(AGENT_MUTEX_ID_NAMESPACE,
component.guid)):
last_pushed_time = component.last_pushed_time
metrics = self._get_component_metrics(component)
component.last_pushed_time = datetime.now()
component.save()
except DBMutexError as exc:
raise self.retry(
exc=exc, countdown=settings.TIMESLICE_LOCK_RETRY_DELAY_MS)
# note regarding failure to deliver (http) metrics:
# I considered merging the metrics back and resetting the last
# push time of the component, but couldn't easily solve the race
# condition observed when this component is repopulated and
# pushed in the time between releasing the lock and getting a
# bad http response from NR. The other obvious alternative is
# to lock until we receive a successful response, but I don't
# consider that an acceptable solution. Considered this lost
# request an acceptable risk for the time being. No data is
# better than bad data imo
self._push_component_metrics(component, metrics, last_pushed_time)
def prune_contracts(period=260):
success = True
lock_id = 'contracts.prune_contracts'
old_stdout = sys.stdout
sys.stdout = mystdout = io.StringIO()
try:
with db_mutex(lock_id):
# Commands don't return anything
call_command('prune_contracts', period=period)
except DBMutexError:
success = False
print('prune_contracts: Could not obtain lock')
except DBMutexTimeoutError:
print('prune_contracts: Task completed but the lock timed out')
except Exception as error:
print(error)
DBMutex.objects.filter(lock_id=lock_id).delete()
success = False
sys.stdout = old_stdout
if not success:
raise TaskError(mystdout.getvalue())
return {
"task": "prune_contracts",
"params": {
"period": period
},
"message": mystdout.getvalue()
}
def run(self, *args, **kwargs):
with db_mutex('push-slack-events'):
self.run_worker(*args, **kwargs)
def run(self, *args, **kwargs):
with db_mutex('push-slack-events'):
self.run_worker(*args, **kwargs)
