def test_process_labrequests_lifecycle_bug(self):
expected_lab = ("testlab", "sprint", "netact-3", False)
testlabs = [("testlab-1", "sprint", "netact-5", False),
("testlab2", "sprint", "netact-2", False),
("testlab-2", "sprint", "netact-2", False), expected_lab,
("2", "sprint", "netact-2", False),
("lab-2", "sprint", "netact-2", False),
("testlab-3", "cloud", "netact-1", False)]
test_data.test_setup(testlabs, lab_status="ready")
labs = test_data.get_resources('labs')['_items']
expected_lab_ids = [
lab['_id'] for lab in labs if lab['lab_name'] == expected_lab[0]
]
self.assertEqual(len(expected_lab_ids), 1)
expected_lab_id = expected_lab_ids.pop()
labrequest_response = test_data.post_labrequest(
"mikko", "{{\"build.id\": \"{0}\"}}".format(expected_lab[2]))
labrequest_id = labrequest_response['_id']
labrequest_data = test_data.get_resources('labrequests')['_items'][0]
self.assertIsNone(labrequest_data.get('lab'))
self.assertEqual(labrequest_data['username'], "mikko")
self.assertEqual(labrequest_data['status'], "pending")
labrequest_tasks.process_labrequests()
labrequest_processed_data = test_data.get_item('labrequests',
labrequest_id)
self.assertEqual(labrequest_processed_data['lab'], expected_lab_id)
self.assertEqual(labrequest_processed_data['username'], "mikko")
my_lab = test_data.get_item('labs', expected_lab_id)
my_reservation = test_data.get_item('reservations',
my_lab['reservation'])
self.assertEqual(my_reservation['username'], "mikko")
self.assertEqual(my_reservation['username'], "mikko")
self.assertEqual(labrequest_processed_data['status'], "ready")
self.assertEqual(my_lab.get('status'), "queued_for_revert")
_mock_snapshot_revert_task(my_lab['_id'])
response = test_data.delete_item('reservations',
my_reservation['_id'],
force=True)
self.assertEqual(response.status_code, 204)
labrequest_response = test_data.post_labrequest(
"mikko", "{{\"build.id\": \"{0}\"}}".format(expected_lab[2]))
labrequest_id = labrequest_response['_id']
labrequest_data = test_data.get_item('labrequests', labrequest_id)
self.assertIsNone(labrequest_data.get('lab'))
self.assertEqual(labrequest_data['username'], "mikko")
self.assertEqual(labrequest_data['status'], "pending")
labrequest_tasks.process_labrequests()
labrequest_processed_data = test_data.get_item('labrequests',
labrequest_id)
self.assertEqual(labrequest_processed_data['lab'], expected_lab_id)
self.assertEqual(labrequest_processed_data['username'], "mikko")
my_lab = test_data.get_item('labs', expected_lab_id)
my_reservation = test_data.get_item('reservations',
my_lab['reservation'])
self.assertEqual(my_reservation['username'], "mikko")
self.assertEqual(my_reservation['username'], "mikko")
self.assertEqual(labrequest_processed_data['status'], "ready")
def test_process_labrequest_match_only_available_snapshots(self):
expected_lab = ("testlab", "sprint", "netact-3", False)
testlabs = [("testlab-1", "sprint", "netact-5", False),
("testlab2", "sprint", "netact-2", False),
("testlab-2", "sprint", "netact-2", False), expected_lab,
("2", "sprint", "netact-2", False),
("lab-2", "sprint", "netact-2", False),
("testlab-3", "cloud", "netact-1", False)]
test_data.test_setup(testlabs, snapshots_status="failed")
labs = test_data.get_resources('labs')['_items']
expected_lab_ids = [
lab['_id'] for lab in labs if lab['lab_name'] == expected_lab[0]
]
self.assertEqual(len(expected_lab_ids), 1)
labrequest_response = test_data.post_labrequest(
"mikko", "{{\"build.id\": \"{0}\"}}".format(expected_lab[2]))
labrequest_id = labrequest_response['_id']
labrequest_data = test_data.get_item('labrequests', labrequest_id)
self.assertIsNone(labrequest_data.get('lab'))
self.assertEqual(labrequest_data['username'], "mikko")
self.assertEqual(labrequest_data['status'], "pending")
labrequest_tasks.LabrequestProcessor(
labrequest_data).process_labrequest()
labrequest_processed_data = test_data.get_item('labrequests',
labrequest_id)
self.assertEqual(labrequest_processed_data.get('status'),
"no_lab_available")
self.assertIsNone(labrequest_processed_data.get('lab'))
def _do_lab_request(self,
user,
query,
message,
expected_state='ready',
expected_lab_is_None=False,
expected_lab=None,
**kwargs):
''' create & execute labrequest and checks results'''
response = test_data.post_labrequest(user, query, **kwargs)
labrequest = test_data.get_item('labrequests', response['_id'])
# state of fresh request is 'pending'
self.assertEqual(labrequest.get('username'), user)
self.assertEqual(labrequest.get('status'), 'pending')
self.assertIsNone(labrequest.get('lab'))
labrequest_tasks.LabrequestProcessor(labrequest).process_labrequest()
labrequest = test_data.get_item('labrequests', response['_id'])
logging.info('labrequest: %s', labrequest)
self.assertEqual(labrequest.get('username'), user)
self.assertEqual(labrequest.get('status'), expected_state)
if expected_lab_is_None:
self.assertIsNone(labrequest.get('lab'))
else:
self.assertIsNotNone(labrequest.get('lab'))
return labrequest.get('lab')
def test_process_labrequests_concurrency_problem(self):
# It's possible that labrequest processing and keepalive are both updating labrequest at the same time.
# Both are now supporting retry in case of etag mismatch that seems to help and should tackle all meaningful scenarios.
# i.e. where keepalive is called by one client once per second etc.
#
# There's still some problem that causes labrequest processing to add a lab even when the request itself seems to be successful.
# To reproduce, increase the number of keepalive_threads.
expected_lab = ("testlab", "sprint", "netact-3", False)
testlabs = [("testlab-1", "sprint", "netact-5", False),
("testlab2", "sprint", "netact-2", False),
("testlab-2", "sprint", "netact-2", False), expected_lab,
("2", "sprint", "netact-2", False),
("lab-2", "sprint", "netact-2", False),
("testlab-3", "cloud", "netact-1", False)]
test_data.test_setup(testlabs, lab_status="ready")
labs = test_data.get_resources('labs')['_items']
expected_lab_ids = [
lab['_id'] for lab in labs if lab['lab_name'] == expected_lab[0]
]
self.assertEqual(len(expected_lab_ids), 1)
expected_lab_id = expected_lab_ids.pop()
labrequest_response = test_data.post_labrequest(
"mikko", "{{\"build.id\": \"{0}\"}}".format(expected_lab[2]))
labrequest_id = labrequest_response['_id']
labrequest_data = test_data.get_resources('labrequests')['_items'][0]
self.assertIsNone(labrequest_data.get('lab'))
self.assertEqual(labrequest_data['username'], "mikko")
self.assertEqual(labrequest_data['status'], "pending")
def _process():
labrequest_tasks.process_labrequests()
def _keepalive():
requests.patch("{}/{}/{}/keepalive".format(STATE_REST_API_URL,
'labrequests',
labrequest_id))
process = threading.Thread(target=_process)
keepalive_threads = []
for i in xrange(1, 10):
keepalive_threads.append(threading.Thread(target=_keepalive))
process.start()
for thread in keepalive_threads:
thread.start()
process.join()
for thread in keepalive_threads:
thread.join()
labrequest_processed_data = test_data.get_item('labrequests',
labrequest_id)
self.assertEqual(labrequest_processed_data['lab'], expected_lab_id)
self.assertEqual(labrequest_processed_data['username'], "mikko")
my_lab = test_data.get_item('labs', expected_lab_id)
my_reservation = test_data.get_item('reservations',
my_lab['reservation'])
self.assertEqual(my_reservation['username'], "mikko")
self.assertEqual(my_reservation['username'], "mikko")
self.assertEqual(labrequest_processed_data['status'], "ready")
def test_process_labrequests_process_all_even_if_failures(
self, mockLabrequestProcessor):
mockLabrequestProcessor.side_effect = [
Exception, Mock(), Exception, RuntimeError
]
test_data.post_labrequest("mikko",
"{{\"build.id\": \"{0}\"}}".format("dsa"))
test_data.post_labrequest("mikko",
"{{\"build.id\": \"{0}\"}}".format("dsa"))
test_data.post_labrequest("mikko",
"{{\"build.id\": \"{0}\"}}".format("dsa"))
test_data.post_labrequest("mikko",
"{{\"build.id\": \"{0}\"}}".format("dsa"))
labrequest_tasks.process_labrequests()
self.assertEqual(mockLabrequestProcessor.call_count, 4)
def test_process_labrequest_no_lab_available_no_final_state(self):
testlabs = [("testlab-1", "sprint", "netact-5", False),
("lab-2", "sprint", "netact-2", False),
("testlab-3", "cloud", "netact-1", False)]
test_data.test_setup(testlabs, lab_status="reverting_state")
labrequest_response = test_data.post_labrequest(
"mikko", "{{\"build.id\": \"{0}\"}}".format("netact-1"))
labrequest_id = labrequest_response['_id']
labrequest_data = test_data.get_item('labrequests', labrequest_id)
self.assertIsNone(labrequest_data.get('lab'))
self.assertEqual(labrequest_data['username'], "mikko")
self.assertEqual(labrequest_data['status'], "pending")
labrequest_tasks.LabrequestProcessor(
labrequest_data).process_labrequest()
labrequest_processed_data = test_data.get_item('labrequests',
labrequest_id)
self.assertIsNone(labrequest_data.get('lab'))
self.assertEqual(labrequest_processed_data['username'], "mikko")
self.assertEqual(labrequest_processed_data['status'],
"no_lab_available")
def test_process_labrequest_no_matching_lab(self):
testlabs = [("testlab-1", "sprint", "netact-5", True),
("lab-2", "sprint", "netact-2", True),
("testlab-3", "cloud", "netact-1", True)]
test_data.test_setup(testlabs)
labrequest_response = test_data.post_labrequest(
"mikko",
"{{\"build.id\": \"{0}\"}}".format("netact-1"),
lab_search_query="lab_type==sprint")
labrequest_id = labrequest_response['_id']
labrequest_data = test_data.get_item('labrequests', labrequest_id)
self.assertIsNone(labrequest_data.get('lab'))
self.assertEqual(labrequest_data['username'], "mikko")
self.assertEqual(labrequest_data['status'], "pending")
labrequest_tasks.LabrequestProcessor(
labrequest_data).process_labrequest()
labrequest_processed_data = test_data.get_item('labrequests',
labrequest_id)
self.assertIsNone(labrequest_data.get('lab'))
self.assertEqual(labrequest_processed_data['username'], "mikko")
self.assertEqual(labrequest_processed_data['status'],
"no_matching_state")
