def __init__(self, evaluation_service): """Create the single executor for ES. The executor just delegates work to the worker pool. """ super(EvaluationExecutor, self).__init__(True) self.evaluation_service = evaluation_service self.pool = WorkerPool(self.evaluation_service) # List of QueueItem (ESOperation) we have extracted from the # queue, but not yet finished to execute. self._currently_executing = [] # Lock used to guard the currently executing operations self._current_execution_lock = gevent.coros.RLock() # Whether execute need to drop the currently executing # operation. self._drop_current = False for i in xrange(get_service_shards("Worker")): worker = ServiceCoord("Worker", i) self.pool.add_worker(worker)
def __init__(self, evaluation_service): """Create the single executor for ES. The executor just delegates work to the worker pool. """ super(EvaluationExecutor, self).__init__(True) self.evaluation_service = evaluation_service self.pool = WorkerPool(self.evaluation_service) # List of QueueItem (ESOperation) we have extracted from the # queue, but not yet finished to execute. self._currently_executing = [] # Lock used to guard the currently executing operations self._current_execution_lock = gevent.lock.RLock() # Whether execute need to drop the currently executing # operation. self._drop_current = False for i in range(get_service_shards("Worker")): worker = ServiceCoord("Worker", i) self.pool.add_worker(worker)
def __init__(self, shard): logger.initialize(ServiceCoord("AdminWebServer", shard)) # A list of pending notifications. self.notifications = [] parameters = { "login_url": "/", "template_path": os.path.join(os.path.dirname(__file__), "templates", "admin"), "static_path": os.path.join(os.path.dirname(__file__), "static"), "cookie_secret": base64.b64encode(config.secret_key), "debug": config.tornado_debug, } WebService.__init__(self, config.admin_listen_port, _aws_handlers, parameters, shard=shard, custom_logger=logger, listen_address=config.admin_listen_address) self.file_cacher = FileCacher(self) self.evaluation_service = self.connect_to( ServiceCoord("EvaluationService", 0)) self.scoring_service = self.connect_to( ServiceCoord("ScoringService", 0)) self.resource_services = [] for i in xrange(get_service_shards("ResourceService")): self.resource_services.append(self.connect_to( ServiceCoord("ResourceService", i))) self.logservice = self.connect_to(ServiceCoord("LogService", 0))
def get(self, shard=None, contest_id=None): if contest_id is not None: self.contest = self.safe_get_item(Contest, contest_id) contest_address = "/%s" % contest_id else: contest_address = "" if shard is None: shard = "all" self.r_params = self.render_params() self.r_params["resource_shards"] = \ get_service_shards("ResourceService") self.r_params["resource_addresses"] = {} if shard == "all": for i in range(self.r_params["resource_shards"]): self.r_params["resource_addresses"][i] = get_service_address( ServiceCoord("ResourceService", i)).ip else: shard = int(shard) try: address = get_service_address( ServiceCoord("ResourceService", shard)) except KeyError: self.redirect("/resourceslist%s" % contest_address) return self.r_params["resource_addresses"][shard] = address.ip self.render("resources.html", **self.r_params)
def __init__(self, shard, contest_id): logger.initialize(ServiceCoord("EvaluationService", shard)) Service.__init__(self, shard, custom_logger=logger) self.contest_id = contest_id self.queue = JobQueue() self.pool = WorkerPool(self) self.scoring_service = self.connect_to( ServiceCoord("ScoringService", 0)) for i in xrange(get_service_shards("Worker")): worker = ServiceCoord("Worker", i) self.pool.add_worker(worker) self.add_timeout(self.dispatch_jobs, None, EvaluationService.CHECK_DISPATCH_TIME, immediately=True) self.add_timeout(self.check_workers_timeout, None, EvaluationService.WORKER_TIMEOUT_CHECK_TIME, immediately=False) self.add_timeout(self.check_workers_connection, None, EvaluationService.WORKER_CONNECTION_CHECK_TIME, immediately=False) self.add_timeout(self.search_jobs_not_done, None, EvaluationService.JOBS_NOT_DONE_CHECK_TIME, immediately=True)
def __init__(self, shard): parameters = { "ui_modules": views, "login_url": "/login", "template_path": pkg_resources.resource_filename("cms.server.admin", "templates"), "static_files": [("cms.server", "static"), ("cms.server.admin", "static")], "cookie_secret": base64.b64encode(config.secret_key), "debug": config.tornado_debug, "auth_middleware": AWSAuthMiddleware, "rpc_enabled": True, "rpc_auth": rpc_authorization_checker, } super(AdminWebServer, self).__init__( config.admin_listen_port, HANDLERS, parameters, shard=shard, listen_address=config.admin_listen_address ) # A list of pending notifications. self.notifications = [] self.file_cacher = FileCacher(self) self.evaluation_service = self.connect_to(ServiceCoord("EvaluationService", 0)) self.scoring_service = self.connect_to(ServiceCoord("ScoringService", 0)) ranking_enabled = len(config.rankings) > 0 self.proxy_service = self.connect_to(ServiceCoord("ProxyService", 0), must_be_present=ranking_enabled) self.resource_services = [] for i in range(get_service_shards("ResourceService")): self.resource_services.append(self.connect_to(ServiceCoord("ResourceService", i))) self.logservice = self.connect_to(ServiceCoord("LogService", 0))
def get(self, contest_id=None): if contest_id is not None: self.contest = self.safe_get_item(Contest, contest_id) self.r_params = self.render_params() self.r_params["resource_addresses"] = {} services = get_service_shards("ResourceService") for i in range(services): self.r_params["resource_addresses"][i] = get_service_address(ServiceCoord("ResourceService", i)).ip self.render("resourceslist.html", **self.r_params)
def __init__(self, shard, contest_id=None): super(QueueService, self).__init__(shard) self.contest_id = contest_id # This lock is used to avoid inserting in the queue (which # itself is already thread-safe) an operation which is already # being processed. Such operation might be in one of the # following state: # 1. in the queue; # 2. extracted from the queue by the executor, but not yet # dispatched to a worker; # 3. being processed by a worker ("in the worker pool"); # 4. being processed by action_finished, but with the results # not yet written to the database. # 5. with results written in the database. # # The methods enqueuing operations already check that the # operation is not in state 5, and enqueue() checks that it is # not in the first three states. # # Therefore, the lock guarantees that the methods adding # operations to the queue (_missing_operations, # invalidate_submission, enqueue) are not executed # concurrently with action_finished to avoid picking # operations in state 4. self.post_finish_lock = gevent.lock.RLock() # Data structure holding pending results. self.pending = PendingResults() # Neverending greenlet consuming results, by sending them to ES. gevent.spawn(self.process_results) self.evaluation_services = [ self.connect_to(ServiceCoord("EvaluationService", i)) for i in range(get_service_shards("EvaluationService")) ] # How many calls to the sweeper will be ignored. Used to # temporarily disable it in case of invalidate. self.avoid_next_sweepers = 0 self.add_executor(EvaluationExecutor(self)) self.start_sweeper(1200.0) self.add_timeout( self.check_workers_timeout, None, QueueService.WORKER_TIMEOUT_CHECK_TIME.total_seconds(), immediately=False) self.add_timeout( self.check_workers_connection, None, QueueService.WORKER_CONNECTION_CHECK_TIME.total_seconds(), immediately=False)
def get(self, contest_id=None): if contest_id is not None: self.contest = self.safe_get_item(Contest, contest_id) self.r_params = self.render_params() self.r_params["resource_addresses"] = {} services = get_service_shards("ResourceService") for i in range(services): self.r_params["resource_addresses"][i] = get_service_address( ServiceCoord("ResourceService", i)).ip self.render("resourceslist.html", **self.r_params)
def get(self, contest_id=None): if contest_id is not None: self.contest = self.safe_get_item(Contest, contest_id) r_params = self.render_params() r_params["resource_shards"] = get_service_shards("ResourceService") r_params["resource_addresses"] = {} for i in xrange(r_params["resource_shards"]): r_params["resource_addresses"][i] = get_service_address( ServiceCoord("ResourceService", i)).ip self.render("resources.html", **r_params)
def __init__(self, shard): parameters = { "static_files": [("cms.server", "static"), ("cms.server.admin", "static")], "cookie_secret": hex_to_bin(config.secret_key), "debug": config.tornado_debug, "auth_middleware": AWSAuthMiddleware, "rpc_enabled": True, "rpc_auth": self.is_rpc_authorized, "xsrf_cookies": True, } super().__init__(config.admin_listen_port, HANDLERS, parameters, shard=shard, listen_address=config.admin_listen_address) # For printing of detailed results try: locale.setlocale(locale.LC_COLLATE, 'lv_LV.UTF-8') locale.setlocale(locale.LC_NUMERIC, 'lv_LV.UTF-8') except locale.Error: logger.warning("Failed to set lv_LV.UTF-8 locale") self.jinja2_environment = AWS_ENVIRONMENT # A list of pending notifications. self.notifications = [] self.admin_web_server = self.connect_to( ServiceCoord("AdminWebServer", 0)) self.evaluation_service = self.connect_to( ServiceCoord("EvaluationService", 0)) self.scoring_service = self.connect_to( ServiceCoord("ScoringService", 0)) ranking_enabled = len(config.rankings) > 0 self.proxy_service = self.connect_to(ServiceCoord("ProxyService", 0), must_be_present=ranking_enabled) self.resource_services = [] for i in range(get_service_shards("ResourceService")): self.resource_services.append( self.connect_to(ServiceCoord("ResourceService", i))) self.logservice = self.connect_to(ServiceCoord("LogService", 0))
def __init__(self, shard): parameters = { "ui_modules": views, "template_path": pkg_resources.resource_filename("cms.server.admin", "templates"), "static_files": [("cms.server", "static"), ("cms.server.admin", "static")], "cookie_secret": base64.b64encode(config.secret_key), "debug": config.tornado_debug, "auth_middleware": AWSAuthMiddleware, "rpc_enabled": True, "rpc_auth": self.is_rpc_authorized, "xsrf_cookies": True, } super(AdminWebServer, self).__init__(config.admin_listen_port, HANDLERS, parameters, shard=shard, listen_address=config.admin_listen_address) # A list of pending notifications. self.notifications = [] self.file_cacher = FileCacher(self) self.admin_web_server = self.connect_to( ServiceCoord("AdminWebServer", 0)) self.queue_service = self.connect_to(ServiceCoord("QueueService", 0)) # TODO: does it make sense to use a random one? self.evaluation_services = self.connect_to( ServiceCoord("EvaluationService", 0)) self.scoring_service = self.connect_to( ServiceCoord("ScoringService", 0)) ranking_enabled = len(config.rankings) > 0 self.proxy_service = self.connect_to(ServiceCoord("ProxyService", 0), must_be_present=ranking_enabled) self.resource_services = [] for i in range(get_service_shards("ResourceService")): self.resource_services.append( self.connect_to(ServiceCoord("ResourceService", i))) self.logservice = self.connect_to(ServiceCoord("LogService", 0))
def __init__(self, shard): parameters = { "static_files": [("cms.server", "static"), ("cms.server.admin", "static")], "cookie_secret": hex_to_bin(config.secret_key), "debug": config.tornado_debug, "num_proxies_used": config.admin_num_proxies_used, "auth_middleware": AWSAuthMiddleware, "rpc_enabled": True, "rpc_auth": self.is_rpc_authorized, "xsrf_cookies": True, "xsrf_cookie_kwargs": { "samesite": "Strict" }, } super().__init__(config.admin_listen_port, HANDLERS, parameters, shard=shard, listen_address=config.admin_listen_address) self.jinja2_environment = AWS_ENVIRONMENT # A list of pending notifications. self.notifications = [] self.admin_web_server = self.connect_to( ServiceCoord("AdminWebServer", 0)) self.evaluation_service = self.connect_to( ServiceCoord("EvaluationService", 0)) self.scoring_service = self.connect_to( ServiceCoord("ScoringService", 0)) ranking_enabled = len(config.rankings) > 0 self.proxy_service = self.connect_to(ServiceCoord("ProxyService", 0), must_be_present=ranking_enabled) self.resource_services = [] for i in range(get_service_shards("ResourceService")): self.resource_services.append( self.connect_to(ServiceCoord("ResourceService", i))) self.logservice = self.connect_to(ServiceCoord("LogService", 0))
def __init__(self, shard): parameters = { "static_files": [("cms.server", "static"), ("cms.server.admin", "static")], "cookie_secret": hex_to_bin(config.secret_key), "debug": config.tornado_debug, "auth_middleware": AWSAuthMiddleware, "rpc_enabled": True, "rpc_auth": self.is_rpc_authorized, "xsrf_cookies": True, } super(AdminWebServer, self).__init__( config.admin_listen_port, HANDLERS, parameters, shard=shard, listen_address=config.admin_listen_address) self.jinja2_environment = AWS_ENVIRONMENT # A list of pending notifications. self.notifications = [] self.admin_web_server = self.connect_to( ServiceCoord("AdminWebServer", 0)) self.evaluation_service = self.connect_to( ServiceCoord("EvaluationService", 0)) self.scoring_service = self.connect_to( ServiceCoord("ScoringService", 0)) ranking_enabled = len(config.rankings) > 0 self.proxy_service = self.connect_to( ServiceCoord("ProxyService", 0), must_be_present=ranking_enabled) self.resource_services = [] for i in range(get_service_shards("ResourceService")): self.resource_services.append(self.connect_to( ServiceCoord("ResourceService", i))) self.logservice = self.connect_to(ServiceCoord("LogService", 0))
def __init__(self, evaluation_service): """Create the single executor for ES. The executor just delegates work to the worker pool. """ super(EvaluationExecutor, self).__init__() self.evaluation_service = evaluation_service self.pool = WorkerPool(self.evaluation_service) # QueueItem (ESOperation) we have extracted from the queue, # but not yet finished to execute. self._currently_executing = None # Whether execute need to drop the currently executing # operation. self._drop_current = False for i in xrange(get_service_shards("Worker")): worker = ServiceCoord("Worker", i) self.pool.add_worker(worker)
def test_success(self): """Test success cases.""" self.assertEqual(get_service_shards("Service"), 2) self.assertEqual(get_service_shards("ServiceNotPresent"), 0)
def __init__(self, shard, contest_id=None): parameters = { "template_path": pkg_resources.resource_filename("cms.server.contest", "templates"), "static_files": [("cms.server", "static"), ("cms.server.contest", "static")], "cookie_secret": base64.b64encode(config.secret_key), "debug": config.tornado_debug, "is_proxy_used": config.is_proxy_used, "num_proxies_used": config.num_proxies_used, "xsrf_cookies": True, } try: listen_address = config.contest_listen_address[shard] listen_port = config.contest_listen_port[shard] except IndexError: raise ConfigError("Wrong shard number for %s, or missing " "address/port configuration. Please check " "contest_listen_address and contest_listen_port " "in cms.conf." % __name__) self.contest_id = contest_id if self.contest_id is None: HANDLERS.append((r"", MainHandler)) handlers = [(r'/', BaseHandler)] for h in HANDLERS: handlers.append((r'/([^/]+)' + h[0], ) + h[1:]) else: HANDLERS.append((r"/", MainHandler)) handlers = HANDLERS super(ContestWebServer, self).__init__(listen_port, handlers, parameters, shard=shard, listen_address=listen_address) # This is a dictionary (indexed by username) of pending # notification. Things like "Yay, your submission went # through.", not things like "Your question has been replied", # that are handled by the db. Each username points to a list # of tuples (timestamp, subject, text). self.notifications = {} # Retrieve the available translations. self.langs = { lang_code: wrap_translations_for_tornado(trans) for lang_code, trans in get_translations().iteritems() } self.file_cacher = FileCacher(self) self.evaluation_services = [ self.connect_to(ServiceCoord("EvaluationService", i)) for i in range(get_service_shards("EvaluationService")) ] self.scoring_service = self.connect_to( ServiceCoord("ScoringService", 0)) ranking_enabled = len(config.rankings) > 0 self.proxy_service = self.connect_to(ServiceCoord("ProxyService", 0), must_be_present=ranking_enabled) printing_enabled = config.printer is not None self.printing_service = self.connect_to( ServiceCoord("PrintingService", 0), must_be_present=printing_enabled)