def __init__(self, evaluation_service):
"""Create the single executor for ES.
The executor just delegates work to the worker pool.
"""
super(EvaluationExecutor, self).__init__(True)
self.evaluation_service = evaluation_service
self.pool = WorkerPool(self.evaluation_service)
# List of QueueItem (ESOperation) we have extracted from the
# queue, but not yet finished to execute.
self._currently_executing = []
# Lock used to guard the currently executing operations
self._current_execution_lock = gevent.coros.RLock()
# Whether execute need to drop the currently executing
# operation.
self._drop_current = False
for i in xrange(get_service_shards("Worker")):
worker = ServiceCoord("Worker", i)
self.pool.add_worker(worker)
def __init__(self, evaluation_service):
"""Create the single executor for ES.
The executor just delegates work to the worker pool.
"""
super(EvaluationExecutor, self).__init__(True)
self.evaluation_service = evaluation_service
self.pool = WorkerPool(self.evaluation_service)
# List of QueueItem (ESOperation) we have extracted from the
# queue, but not yet finished to execute.
self._currently_executing = []
# Lock used to guard the currently executing operations
self._current_execution_lock = gevent.lock.RLock()
# Whether execute need to drop the currently executing
# operation.
self._drop_current = False
for i in range(get_service_shards("Worker")):
worker = ServiceCoord("Worker", i)
self.pool.add_worker(worker)
def __init__(self, shard):
logger.initialize(ServiceCoord("AdminWebServer", shard))
# A list of pending notifications.
self.notifications = []
parameters = {
"login_url": "/",
"template_path": os.path.join(os.path.dirname(__file__),
"templates", "admin"),
"static_path": os.path.join(os.path.dirname(__file__),
"static"),
"cookie_secret": base64.b64encode(config.secret_key),
"debug": config.tornado_debug,
}
WebService.__init__(self,
config.admin_listen_port,
_aws_handlers,
parameters,
shard=shard,
custom_logger=logger,
listen_address=config.admin_listen_address)
self.file_cacher = FileCacher(self)
self.evaluation_service = self.connect_to(
ServiceCoord("EvaluationService", 0))
self.scoring_service = self.connect_to(
ServiceCoord("ScoringService", 0))
self.resource_services = []
for i in xrange(get_service_shards("ResourceService")):
self.resource_services.append(self.connect_to(
ServiceCoord("ResourceService", i)))
self.logservice = self.connect_to(ServiceCoord("LogService", 0))
def get(self, shard=None, contest_id=None):
if contest_id is not None:
self.contest = self.safe_get_item(Contest, contest_id)
contest_address = "/%s" % contest_id
else:
contest_address = ""
if shard is None:
shard = "all"
self.r_params = self.render_params()
self.r_params["resource_shards"] = \
get_service_shards("ResourceService")
self.r_params["resource_addresses"] = {}
if shard == "all":
for i in range(self.r_params["resource_shards"]):
self.r_params["resource_addresses"][i] = get_service_address(
ServiceCoord("ResourceService", i)).ip
else:
shard = int(shard)
try:
address = get_service_address(
ServiceCoord("ResourceService", shard))
except KeyError:
self.redirect("/resourceslist%s" % contest_address)
return
self.r_params["resource_addresses"][shard] = address.ip
self.render("resources.html", **self.r_params)
def __init__(self, shard, contest_id):
logger.initialize(ServiceCoord("EvaluationService", shard))
Service.__init__(self, shard, custom_logger=logger)
self.contest_id = contest_id
self.queue = JobQueue()
self.pool = WorkerPool(self)
self.scoring_service = self.connect_to(
ServiceCoord("ScoringService", 0))
for i in xrange(get_service_shards("Worker")):
worker = ServiceCoord("Worker", i)
self.pool.add_worker(worker)
self.add_timeout(self.dispatch_jobs, None,
EvaluationService.CHECK_DISPATCH_TIME,
immediately=True)
self.add_timeout(self.check_workers_timeout, None,
EvaluationService.WORKER_TIMEOUT_CHECK_TIME,
immediately=False)
self.add_timeout(self.check_workers_connection, None,
EvaluationService.WORKER_CONNECTION_CHECK_TIME,
immediately=False)
self.add_timeout(self.search_jobs_not_done, None,
EvaluationService.JOBS_NOT_DONE_CHECK_TIME,
immediately=True)
def __init__(self, shard):
parameters = {
"ui_modules": views,
"login_url": "/login",
"template_path": pkg_resources.resource_filename("cms.server.admin", "templates"),
"static_files": [("cms.server", "static"), ("cms.server.admin", "static")],
"cookie_secret": base64.b64encode(config.secret_key),
"debug": config.tornado_debug,
"auth_middleware": AWSAuthMiddleware,
"rpc_enabled": True,
"rpc_auth": rpc_authorization_checker,
}
super(AdminWebServer, self).__init__(
config.admin_listen_port, HANDLERS, parameters, shard=shard, listen_address=config.admin_listen_address
)
# A list of pending notifications.
self.notifications = []
self.file_cacher = FileCacher(self)
self.evaluation_service = self.connect_to(ServiceCoord("EvaluationService", 0))
self.scoring_service = self.connect_to(ServiceCoord("ScoringService", 0))
ranking_enabled = len(config.rankings) > 0
self.proxy_service = self.connect_to(ServiceCoord("ProxyService", 0), must_be_present=ranking_enabled)
self.resource_services = []
for i in range(get_service_shards("ResourceService")):
self.resource_services.append(self.connect_to(ServiceCoord("ResourceService", i)))
self.logservice = self.connect_to(ServiceCoord("LogService", 0))
def get(self, contest_id=None):
if contest_id is not None:
self.contest = self.safe_get_item(Contest, contest_id)
self.r_params = self.render_params()
self.r_params["resource_addresses"] = {}
services = get_service_shards("ResourceService")
for i in range(services):
self.r_params["resource_addresses"][i] = get_service_address(ServiceCoord("ResourceService", i)).ip
self.render("resourceslist.html", **self.r_params)
def __init__(self, shard, contest_id=None):
super(QueueService, self).__init__(shard)
self.contest_id = contest_id
# This lock is used to avoid inserting in the queue (which
# itself is already thread-safe) an operation which is already
# being processed. Such operation might be in one of the
# following state:
# 1. in the queue;
# 2. extracted from the queue by the executor, but not yet
# dispatched to a worker;
# 3. being processed by a worker ("in the worker pool");
# 4. being processed by action_finished, but with the results
# not yet written to the database.
# 5. with results written in the database.
#
# The methods enqueuing operations already check that the
# operation is not in state 5, and enqueue() checks that it is
# not in the first three states.
#
# Therefore, the lock guarantees that the methods adding
# operations to the queue (_missing_operations,
# invalidate_submission, enqueue) are not executed
# concurrently with action_finished to avoid picking
# operations in state 4.
self.post_finish_lock = gevent.lock.RLock()
# Data structure holding pending results.
self.pending = PendingResults()
# Neverending greenlet consuming results, by sending them to ES.
gevent.spawn(self.process_results)
self.evaluation_services = [
self.connect_to(ServiceCoord("EvaluationService", i))
for i in range(get_service_shards("EvaluationService"))
]
# How many calls to the sweeper will be ignored. Used to
# temporarily disable it in case of invalidate.
self.avoid_next_sweepers = 0
self.add_executor(EvaluationExecutor(self))
self.start_sweeper(1200.0)
self.add_timeout(
self.check_workers_timeout,
None,
QueueService.WORKER_TIMEOUT_CHECK_TIME.total_seconds(),
immediately=False)
self.add_timeout(
self.check_workers_connection,
None,
QueueService.WORKER_CONNECTION_CHECK_TIME.total_seconds(),
immediately=False)
def get(self, contest_id=None):
if contest_id is not None:
self.contest = self.safe_get_item(Contest, contest_id)
self.r_params = self.render_params()
self.r_params["resource_addresses"] = {}
services = get_service_shards("ResourceService")
for i in range(services):
self.r_params["resource_addresses"][i] = get_service_address(
ServiceCoord("ResourceService", i)).ip
self.render("resourceslist.html", **self.r_params)
def get(self, contest_id=None):
if contest_id is not None:
self.contest = self.safe_get_item(Contest, contest_id)
r_params = self.render_params()
r_params["resource_shards"] = get_service_shards("ResourceService")
r_params["resource_addresses"] = {}
for i in xrange(r_params["resource_shards"]):
r_params["resource_addresses"][i] = get_service_address(
ServiceCoord("ResourceService", i)).ip
self.render("resources.html", **r_params)
def __init__(self, shard):
parameters = {
"static_files": [("cms.server", "static"),
("cms.server.admin", "static")],
"cookie_secret":
hex_to_bin(config.secret_key),
"debug":
config.tornado_debug,
"auth_middleware":
AWSAuthMiddleware,
"rpc_enabled":
True,
"rpc_auth":
self.is_rpc_authorized,
"xsrf_cookies":
True,
}
super().__init__(config.admin_listen_port,
HANDLERS,
parameters,
shard=shard,
listen_address=config.admin_listen_address)
# For printing of detailed results
try:
locale.setlocale(locale.LC_COLLATE, 'lv_LV.UTF-8')
locale.setlocale(locale.LC_NUMERIC, 'lv_LV.UTF-8')
except locale.Error:
logger.warning("Failed to set lv_LV.UTF-8 locale")
self.jinja2_environment = AWS_ENVIRONMENT
# A list of pending notifications.
self.notifications = []
self.admin_web_server = self.connect_to(
ServiceCoord("AdminWebServer", 0))
self.evaluation_service = self.connect_to(
ServiceCoord("EvaluationService", 0))
self.scoring_service = self.connect_to(
ServiceCoord("ScoringService", 0))
ranking_enabled = len(config.rankings) > 0
self.proxy_service = self.connect_to(ServiceCoord("ProxyService", 0),
must_be_present=ranking_enabled)
self.resource_services = []
for i in range(get_service_shards("ResourceService")):
self.resource_services.append(
self.connect_to(ServiceCoord("ResourceService", i)))
self.logservice = self.connect_to(ServiceCoord("LogService", 0))
def __init__(self, shard):
parameters = {
"ui_modules":
views,
"template_path":
pkg_resources.resource_filename("cms.server.admin", "templates"),
"static_files": [("cms.server", "static"),
("cms.server.admin", "static")],
"cookie_secret":
base64.b64encode(config.secret_key),
"debug":
config.tornado_debug,
"auth_middleware":
AWSAuthMiddleware,
"rpc_enabled":
True,
"rpc_auth":
self.is_rpc_authorized,
"xsrf_cookies":
True,
}
super(AdminWebServer,
self).__init__(config.admin_listen_port,
HANDLERS,
parameters,
shard=shard,
listen_address=config.admin_listen_address)
# A list of pending notifications.
self.notifications = []
self.file_cacher = FileCacher(self)
self.admin_web_server = self.connect_to(
ServiceCoord("AdminWebServer", 0))
self.queue_service = self.connect_to(ServiceCoord("QueueService", 0))
# TODO: does it make sense to use a random one?
self.evaluation_services = self.connect_to(
ServiceCoord("EvaluationService", 0))
self.scoring_service = self.connect_to(
ServiceCoord("ScoringService", 0))
ranking_enabled = len(config.rankings) > 0
self.proxy_service = self.connect_to(ServiceCoord("ProxyService", 0),
must_be_present=ranking_enabled)
self.resource_services = []
for i in range(get_service_shards("ResourceService")):
self.resource_services.append(
self.connect_to(ServiceCoord("ResourceService", i)))
self.logservice = self.connect_to(ServiceCoord("LogService", 0))
def __init__(self, shard):
parameters = {
"static_files": [("cms.server", "static"),
("cms.server.admin", "static")],
"cookie_secret":
hex_to_bin(config.secret_key),
"debug":
config.tornado_debug,
"num_proxies_used":
config.admin_num_proxies_used,
"auth_middleware":
AWSAuthMiddleware,
"rpc_enabled":
True,
"rpc_auth":
self.is_rpc_authorized,
"xsrf_cookies":
True,
"xsrf_cookie_kwargs": {
"samesite": "Strict"
},
}
super().__init__(config.admin_listen_port,
HANDLERS,
parameters,
shard=shard,
listen_address=config.admin_listen_address)
self.jinja2_environment = AWS_ENVIRONMENT
# A list of pending notifications.
self.notifications = []
self.admin_web_server = self.connect_to(
ServiceCoord("AdminWebServer", 0))
self.evaluation_service = self.connect_to(
ServiceCoord("EvaluationService", 0))
self.scoring_service = self.connect_to(
ServiceCoord("ScoringService", 0))
ranking_enabled = len(config.rankings) > 0
self.proxy_service = self.connect_to(ServiceCoord("ProxyService", 0),
must_be_present=ranking_enabled)
self.resource_services = []
for i in range(get_service_shards("ResourceService")):
self.resource_services.append(
self.connect_to(ServiceCoord("ResourceService", i)))
self.logservice = self.connect_to(ServiceCoord("LogService", 0))
def __init__(self, shard):
parameters = {
"static_files": [("cms.server", "static"),
("cms.server.admin", "static")],
"cookie_secret": hex_to_bin(config.secret_key),
"debug": config.tornado_debug,
"auth_middleware": AWSAuthMiddleware,
"rpc_enabled": True,
"rpc_auth": self.is_rpc_authorized,
"xsrf_cookies": True,
}
super(AdminWebServer, self).__init__(
config.admin_listen_port,
HANDLERS,
parameters,
shard=shard,
listen_address=config.admin_listen_address)
self.jinja2_environment = AWS_ENVIRONMENT
# A list of pending notifications.
self.notifications = []
self.admin_web_server = self.connect_to(
ServiceCoord("AdminWebServer", 0))
self.evaluation_service = self.connect_to(
ServiceCoord("EvaluationService", 0))
self.scoring_service = self.connect_to(
ServiceCoord("ScoringService", 0))
ranking_enabled = len(config.rankings) > 0
self.proxy_service = self.connect_to(
ServiceCoord("ProxyService", 0),
must_be_present=ranking_enabled)
self.resource_services = []
for i in range(get_service_shards("ResourceService")):
self.resource_services.append(self.connect_to(
ServiceCoord("ResourceService", i)))
self.logservice = self.connect_to(ServiceCoord("LogService", 0))
def __init__(self, evaluation_service):
"""Create the single executor for ES.
The executor just delegates work to the worker pool.
"""
super(EvaluationExecutor, self).__init__()
self.evaluation_service = evaluation_service
self.pool = WorkerPool(self.evaluation_service)
# QueueItem (ESOperation) we have extracted from the queue,
# but not yet finished to execute.
self._currently_executing = None
# Whether execute need to drop the currently executing
# operation.
self._drop_current = False
for i in xrange(get_service_shards("Worker")):
worker = ServiceCoord("Worker", i)
self.pool.add_worker(worker)
def __init__(self, evaluation_service):
"""Create the single executor for ES.
The executor just delegates work to the worker pool.
"""
super(EvaluationExecutor, self).__init__()
self.evaluation_service = evaluation_service
self.pool = WorkerPool(self.evaluation_service)
# QueueItem (ESOperation) we have extracted from the queue,
# but not yet finished to execute.
self._currently_executing = None
# Whether execute need to drop the currently executing
# operation.
self._drop_current = False
for i in xrange(get_service_shards("Worker")):
worker = ServiceCoord("Worker", i)
self.pool.add_worker(worker)
def test_success(self):
"""Test success cases."""
self.assertEqual(get_service_shards("Service"), 2)
self.assertEqual(get_service_shards("ServiceNotPresent"), 0)
def test_success(self):
"""Test success cases."""
self.assertEqual(get_service_shards("Service"), 2)
self.assertEqual(get_service_shards("ServiceNotPresent"), 0)
def __init__(self, shard, contest_id=None):
parameters = {
"template_path":
pkg_resources.resource_filename("cms.server.contest", "templates"),
"static_files": [("cms.server", "static"),
("cms.server.contest", "static")],
"cookie_secret":
base64.b64encode(config.secret_key),
"debug":
config.tornado_debug,
"is_proxy_used":
config.is_proxy_used,
"num_proxies_used":
config.num_proxies_used,
"xsrf_cookies":
True,
}
try:
listen_address = config.contest_listen_address[shard]
listen_port = config.contest_listen_port[shard]
except IndexError:
raise ConfigError("Wrong shard number for %s, or missing "
"address/port configuration. Please check "
"contest_listen_address and contest_listen_port "
"in cms.conf." % __name__)
self.contest_id = contest_id
if self.contest_id is None:
HANDLERS.append((r"", MainHandler))
handlers = [(r'/', BaseHandler)]
for h in HANDLERS:
handlers.append((r'/([^/]+)' + h[0], ) + h[1:])
else:
HANDLERS.append((r"/", MainHandler))
handlers = HANDLERS
super(ContestWebServer, self).__init__(listen_port,
handlers,
parameters,
shard=shard,
listen_address=listen_address)
# This is a dictionary (indexed by username) of pending
# notification. Things like "Yay, your submission went
# through.", not things like "Your question has been replied",
# that are handled by the db. Each username points to a list
# of tuples (timestamp, subject, text).
self.notifications = {}
# Retrieve the available translations.
self.langs = {
lang_code: wrap_translations_for_tornado(trans)
for lang_code, trans in get_translations().iteritems()
}
self.file_cacher = FileCacher(self)
self.evaluation_services = [
self.connect_to(ServiceCoord("EvaluationService", i))
for i in range(get_service_shards("EvaluationService"))
]
self.scoring_service = self.connect_to(
ServiceCoord("ScoringService", 0))
ranking_enabled = len(config.rankings) > 0
self.proxy_service = self.connect_to(ServiceCoord("ProxyService", 0),
must_be_present=ranking_enabled)
printing_enabled = config.printer is not None
self.printing_service = self.connect_to(
ServiceCoord("PrintingService", 0),
must_be_present=printing_enabled)
