def match_found(flock_manager, sheep_identity, request):
logger.info(
"Sending test request for submission [%s] to sheep [%s].",
request.submission_id,
repr(sheep_identity)
)
# Get submission and test harness to send to sheep
submission = Submission.objects(id = request.submission_id).exclude(
"most_recent",
"uploaded_filenames"
).first()
assignment = Assignment.objects.get(id = submission.assignment)
test_harness = TestHarness.objects.get(id = assignment.test_harness)
# Apply any personal deadlines to the assignment object.
user = User.objects.get(email = submission.user)
assignment.apply_personal_deadlines(user)
data = {
"assignment": assignment.to_dict(),
"submission": submission.to_dict(),
"test_harness": test_harness.to_dict()
}
router_send_json(
sheep,
sheep_identity,
FlockMessage("request", data).to_dict()
)
return True
def main():
flock = FlockManager(
match_found,
config["BLEET_TIMEOUT"],
config["SERVICE_TIMEOUT"]
)
logger.info("Shepherd starting.")
while True:
# Wait until either the public or sheep socket has messages waiting
zmq.select([public, sheep], [], [], timeout = 5)
# Will grab all of the outstanding messages from the outside and place them
# in the request queue
while public.getsockopt(zmq.EVENTS) & zmq.POLLIN != 0:
request = public.recv_json()
logger.debug("Raw test request: %s", str(request))
request = TestRequest.from_dict(request)
try:
submission = \
Submission.objects.get(id = ObjectId(request.submission_id))
except Submission.DoesNotExist as e:
logger.warning(
"Received test request for non-existant submission [%s].",
str(request.submission_id)
)
continue
except bson.errors.InvalidId as e:
logger.warning("Received malformed test request. %s", str(e))
continue
try:
assignment = Assignment.objects.get(id = submission.assignment)
except Assignment.DoesNotExist as e:
logger.error(
"Received test request for a submission [%s] referencing "
"an invalid assignment [%s].",
str(submission.id),
str(submission.assignment)
)
continue
if not assignment.test_harness:
logger.warning(
"Received test request for a submission [%s] referencing "
"an assignment [%s] that does not have a test harness "
"associated with it.",
str(submission.id),
str(submission.assignment)
)
continue
try:
test_harness = \
TestHarness.objects.get(id = assignment.test_harness)
except TestHarness.DoesNotExit as e:
logger.error(
"Received test request for a submission [%s] referencing "
"an assignment [%s] that references a non-existant test "
"harness [%s].",
str(submission.id),
str(submission.assignment),
str(assignment.test_harness)
)
continue
# Gather all the necessary information from the test request
# received from the outside.
processed_request = InternalTestRequest(
submission.id,
test_harness.config.get("galah/timeout",
config["BLEET_TIMEOUT"].seconds),
test_harness.config.get("galah/environment", {})
)
logger.info("Received test request.")
flock.received_request(processed_request)
# Will grab all of the outstanding messages from the sheep and process them
while sheep.getsockopt(zmq.EVENTS) & zmq.POLLIN != 0:
try:
sheep_identity, sheep_message = router_recv_json(sheep)
sheep_message = FlockMessage.from_dict(sheep_message)
logger.debug(
"Received message from sheep: %s",
str(sheep_message)
)
except ValueError as e:
logger.error("Could not decode sheep's message: %s", str(e))
logger.debug(
"Exception thrown while decoding sheep's message...",
exc_info = sys.exc_info()
)
continue
if sheep_message.type == "distress":
logger.warn("Received distress message. Sending bloot.")
router_send_json(
sheep, sheep_identity, FlockMessage("bloot", "").to_dict()
)
elif sheep_message.type == "bleet":
logger.debug(
"Sheep [%s] bleeted. Sending bloot.",
repr(sheep_identity)
)
result = flock.sheep_bleeted(sheep_identity)
# Under certain circumstances we want to completely ignore a
# bleet (see FlockManager.sheep_bleeted() for more details)
if result is FlockManager.IGNORE:
logger.debug("Ignoring bleet.")
continue
if not result:
router_send_json(
sheep,
sheep_identity,
FlockMessage("identify", "").to_dict()
)
logger.info(
"Unrecognized sheep [%s] connected, identify sent.",
repr(sheep_identity)
)
continue
router_send_json(
sheep,
sheep_identity,
FlockMessage("bloot", "").to_dict()
)
elif sheep_message.type == "environment":
if not flock.manage_sheep(sheep_identity, sheep_message.body):
logger.warn(
"Received environment from an already-recognized sheep."
)
elif sheep_message.type == "result":
logger.info("Received test result from sheep.")
logger.debug(
"Received test result from sheep: %s",
str(sheep_message.body)
)
try:
submission_id = ObjectId(sheep_message.body["id"])
submission = Submission.objects.get(id = submission_id)
test_result = TestResult.from_dict(sheep_message.body)
try:
test_result.save()
except InvalidDocument:
logger.warn(
"Test result is too large for the database.",
exc_info = True
)
test_result = TestResult(failed = True)
test_result.save()
submission.test_results = test_result.id
submission.save()
except (InvalidId, Submission.DoesNotExist) as e:
logger.warn(
"Could not retrieve submission [%s] for test result "
"received from sheep [%s].",
str(submission_id),
repr(sheep_identity)
)
continue
router_send_json(
sheep,
sheep_identity,
FlockMessage(
"bloot", sheep_message.body["id"]
).to_dict()
)
if not flock.sheep_finished(sheep_identity):
logger.info(
"Got result from sheep [%s] who was not processing "
"a test request.",
repr(sheep_identity)
)
# Let the flock manager get rid of any dead or killed sheep.
lost_sheep, killed_sheep = flock.cleanup()
if lost_sheep:
logger.warn(
"%d sheep lost due to bleet timeout: %s",
len(lost_sheep),
str([repr(i) for i in lost_sheep])
)
if killed_sheep:
logger.warn(
"%d sheep lost due to request timeout: %s",
len(killed_sheep),
str([repr(i) for i in killed_sheep])
)